codebase_index 0.1.0

data/lib/codebase_index/notion/mappers/model_mapper.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Notion
+    module Mappers
+      # Maps a model ExtractedUnit to Notion page properties for the Data Models database.
+      #
+      # Transforms model metadata (associations, validations, callbacks, scopes, git data)
+      # into Notion API property format for the Data Models database.
+      #
+      # @example
+      #   mapper = ModelMapper.new
+      #   properties = mapper.map(unit_data)
+      #   client.create_page(database_id: db_id, properties: properties)
+      #
+      class ModelMapper
+        MAX_RICH_TEXT_LENGTH = 2000
+
+        # Map a model unit to Notion Data Models page properties.
+        #
+        # @param unit_data [Hash] Parsed model ExtractedUnit JSON
+        # @return [Hash] Notion page properties hash
+        def map(unit_data)
+          metadata = unit_data['metadata'] || {}
+          properties = build_text_properties(unit_data, metadata)
+          properties['Column Count'] = { number: column_count(metadata) }
+          add_git_properties(properties, metadata['git'] || {})
+          properties
+        end
+
+        private
+
+        # @return [Hash] Text-based Notion properties
+        def build_text_properties(unit_data, metadata)
+          {
+            'Table Name' => title_property(table_name(unit_data, metadata)),
+            'Model Name' => rich_text_property(unit_data['identifier']),
+            'Description' => rich_text_property(extract_description(unit_data['source_code'])),
+            'Associations' => rich_text_property(format_associations(metadata['associations'])),
+            'Validations' => rich_text_property(format_validations(metadata['validations'])),
+            'Callbacks' => rich_text_property(format_callbacks(metadata['callbacks'])),
+            'Scopes' => rich_text_property(format_scopes(metadata['scopes'])),
+            'File Path' => rich_text_property(unit_data['file_path'] || ''),
+            'Dependencies' => rich_text_property(format_dependencies(unit_data['dependencies']))
+          }
+        end
+
+        # @return [void]
+        def add_git_properties(properties, git)
+          properties['Last Modified'] = { date: { start: git['last_modified'] } } if git['last_modified']
+          properties['Change Frequency'] = { select: { name: git['change_frequency'] } } if git['change_frequency']
+        end
+
+        # @return [String]
+        def table_name(unit_data, metadata)
+          return metadata['table_name'] if metadata['table_name']
+
+          identifier = unit_data['identifier'] || ''
+          "#{identifier.split('::').last.to_s.gsub(/([a-z])([A-Z])/, '\1_\2').downcase}s"
+        end
+
+        # @return [Integer]
+        def column_count(metadata)
+          metadata['column_count'] || (metadata['columns'] || []).size
+        end
+
+        # @return [String]
+        def extract_description(source_code)
+          return '' unless source_code
+
+          comment_lines = []
+          source_code.lines.each do |line|
+            stripped = line.strip
+            if stripped.start_with?('#')
+              comment_lines << stripped.sub(/^#\s?/, '')
+            elsif comment_lines.any?
+              break
+            end
+          end
+
+          comment_lines.any? ? comment_lines.join(' ').strip : ''
+        end
+
+        # @return [String]
+        def format_associations(associations)
+          return 'None' if associations.nil? || associations.empty?
+
+          associations.map { |a| format_single_association(a) }.join("\n")
+        end
+
+        # @return [String]
+        def format_single_association(assoc)
+          parts = ["#{assoc['type']} :#{assoc['name']}"]
+          parts << "through: :#{assoc['through']}" if assoc['through']
+          parts << "class_name: '#{assoc['class_name']}'" if assoc['class_name']
+          parts << "foreign_key: :#{assoc['foreign_key']}" if assoc['foreign_key']
+          parts.join(', ')
+        end
+
+        # @return [String]
+        def format_validations(validations)
+          return 'None' if validations.nil? || validations.empty?
+
+          validations.group_by { |v| v['attribute'] }.map do |attr, vals|
+            "#{attr}: #{vals.map { |v| v['type'] }.join(', ')}"
+          end.join("\n")
+        end
+
+        # @return [String]
+        def format_callbacks(callbacks)
+          return 'None' if callbacks.nil? || callbacks.empty?
+
+          callbacks.map { |callback| format_single_callback(callback) }.join("\n")
+        end
+
+        # @return [String]
+        def format_single_callback(callback)
+          parts = ["#{callback['type']}: #{callback['filter']}"]
+          effects = callback_side_effects(callback['side_effects'])
+          parts << "(#{effects.join('; ')})" if effects.any?
+          parts.join(' ')
+        end
+
+        # @return [Array<String>]
+        def callback_side_effects(side_effects)
+          return [] unless side_effects
+
+          effects = []
+          jobs = side_effects['jobs_enqueued']
+          effects << "enqueues #{jobs.join(', ')}" if jobs&.any?
+          services = side_effects['services_called']
+          effects << "calls #{services.join(', ')}" if services&.any?
+          effects
+        end
+
+        # @return [String]
+        def format_scopes(scopes)
+          return 'None' if scopes.nil? || scopes.empty?
+
+          scopes.map { |s| s['name'] }.join(', ')
+        end
+
+        # @return [String]
+        def format_dependencies(dependencies)
+          return 'None' if dependencies.nil? || dependencies.empty?
+
+          dependencies.map { |dep| "#{dep['target']} (via #{dep['via']})" }.join(', ')
+        end
+
+        # @return [Hash]
+        def title_property(text)
+          { title: [{ text: { content: text } }] }
+        end
+
+        # @return [Hash]
+        def rich_text_property(text)
+          content = text.to_s
+          content = "#{content[0...1997]}..." if content.length > MAX_RICH_TEXT_LENGTH
+          { rich_text: [{ text: { content: content } }] }
+        end
+      end
+    end
+  end
+end

data/lib/codebase_index/notion/rate_limiter.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Notion
+    # Thread-safe rate limiter for Notion API (3 requests/second default).
+    #
+    # Enforces a minimum interval between API calls by sleeping when necessary.
+    # Uses a Mutex to ensure thread safety when called from concurrent contexts.
+    #
+    # @example
+    #   limiter = RateLimiter.new(requests_per_second: 3)
+    #   limiter.throttle { client.create_page(...) }
+    #   limiter.throttle { client.update_page(...) }
+    #
+    class RateLimiter
+      # @param requests_per_second [Numeric] Maximum requests per second (default: 3)
+      # @raise [ArgumentError] if requests_per_second is not positive
+      def initialize(requests_per_second: 3)
+        unless requests_per_second.is_a?(Numeric) && requests_per_second.positive?
+          raise ArgumentError, "requests_per_second must be positive, got #{requests_per_second.inspect}"
+        end
+
+        @min_interval = 1.0 / requests_per_second
+        @last_request_at = nil
+        @mutex = Mutex.new
+      end
+
+      # Execute a block after enforcing the rate limit.
+      #
+      # Sleeps if the minimum interval since the last request hasn't elapsed.
+      # Thread-safe — only one request proceeds at a time.
+      #
+      # @yield The block to execute after rate limiting
+      # @return [Object] The block's return value
+      # @raise [ArgumentError] if no block is given
+      def throttle
+        raise ArgumentError, 'block required' unless block_given?
+
+        @mutex.synchronize do
+          wait_for_interval
+          @last_request_at = monotonic_now
+        end
+
+        yield
+      end
+
+      private
+
+      # Sleep if minimum interval hasn't elapsed since last request.
+      #
+      # @return [void]
+      def wait_for_interval
+        return unless @last_request_at
+
+        elapsed = monotonic_now - @last_request_at
+        remaining = @min_interval - elapsed
+        sleep(remaining) if remaining.positive?
+      end
+
+      # Monotonic clock for accurate interval measurement.
+      #
+      # @return [Float] Current monotonic time in seconds
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/codebase_index/observability/health_check.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Observability
+    # Probes configured components and reports overall system health.
+    #
+    # Checks vector store, metadata store, and embedding provider by calling
+    # lightweight operations on each. Components that are nil are reported
+    # as :not_configured and do not affect the overall healthy? status.
+    #
+    # @example
+    #   check = HealthCheck.new(
+    #     vector_store: vector_store,
+    #     metadata_store: metadata_store,
+    #     embedding_provider: provider
+    #   )
+    #   status = check.run
+    #   status.healthy?    # => true
+    #   status.components  # => { vector_store: :ok, metadata_store: :ok, embedding_provider: :ok }
+    #
+    class HealthCheck
+      # Value object representing the result of a health check.
+      HealthStatus = Struct.new(:healthy?, :components, keyword_init: true)
+
+      # @param vector_store [Object, nil] Vector store adapter (must respond to #count)
+      # @param metadata_store [Object, nil] Metadata store adapter (must respond to #count)
+      # @param embedding_provider [Object, nil] Embedding provider (must respond to #embed)
+      def initialize(vector_store: nil, metadata_store: nil, embedding_provider: nil)
+        @vector_store = vector_store
+        @metadata_store = metadata_store
+        @embedding_provider = embedding_provider
+      end
+
+      # Run health probes on all configured components.
+      #
+      # @return [HealthStatus] Result with healthy? flag and per-component status
+      def run
+        components = {
+          vector_store: probe_store(@vector_store),
+          metadata_store: probe_store(@metadata_store),
+          embedding_provider: probe_provider(@embedding_provider)
+        }
+
+        all_healthy = components.values.all? { |status| %i[ok not_configured].include?(status) }
+
+        HealthStatus.new(healthy?: all_healthy, components: components)
+      end
+
+      private
+
+      # Probe a store component by calling #count.
+      #
+      # @param store [Object, nil] Store adapter
+      # @return [Symbol] :ok, :error, or :not_configured
+      def probe_store(store)
+        return :not_configured if store.nil?
+
+        store.count
+        :ok
+      rescue StandardError
+        :error
+      end
+
+      # Probe an embedding provider by checking its capabilities without making network calls.
+      #
+      # @param provider [Object, nil] Embedding provider
+      # @return [Symbol] :ok, :error, or :not_configured
+      def probe_provider(provider)
+        return :not_configured if provider.nil?
+
+        if provider.respond_to?(:embed) && provider.respond_to?(:dimensions)
+          :ok
+        else
+          :error
+        end
+      rescue StandardError
+        :error
+      end
+    end
+  end
+end

data/lib/codebase_index/observability/instrumentation.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Observability
+    # Lightweight instrumentation wrapper that delegates to ActiveSupport::Notifications
+    # when available, and falls back to a simple yield otherwise.
+    #
+    # @example
+    #   Instrumentation.instrument('codebase_index.extraction', unit: 'User') do
+    #     extract_unit(user_model)
+    #   end
+    #
+    module Instrumentation
+      module_function
+
+      # Instrument a block of code with an event name and payload.
+      #
+      # Delegates to ActiveSupport::Notifications.instrument when available.
+      # Otherwise, yields the block directly.
+      #
+      # @param event [String] Event name (e.g., 'codebase_index.extraction')
+      # @param payload [Hash] Additional data to include with the event
+      # @yield [payload] The block to instrument
+      # @return [Object] The return value of the block
+      def instrument(event, payload = {}, &block)
+        if defined?(ActiveSupport::Notifications)
+          ActiveSupport::Notifications.instrument(event, payload, &block)
+        elsif block
+          yield payload
+        end
+      end
+    end
+  end
+end

data/lib/codebase_index/observability/structured_logger.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+
+module CodebaseIndex
+  module Observability
+    # Structured JSON logger that writes one JSON object per line.
+    #
+    # Each log entry includes a timestamp, level, event name, and any
+    # additional data passed as keyword arguments.
+    #
+    # @example
+    #   logger = StructuredLogger.new(output: $stderr)
+    #   logger.info('extraction.complete', units: 42, duration_ms: 1200)
+    #   # => {"timestamp":"2026-02-15T12:00:00Z","level":"info",
+    #   #     "event":"extraction.complete","units":42,"duration_ms":1200}
+    #
+    class StructuredLogger
+      # @param output [IO] Output stream (default: $stderr)
+      def initialize(output: $stderr)
+        @output = output
+      end
+
+      # Log at info level.
+      #
+      # @param event [String] Event name
+      # @param data [Hash] Additional structured data
+      def info(event, **data)
+        write_entry('info', event, data)
+      end
+
+      # Log at warn level.
+      #
+      # @param event [String] Event name
+      # @param data [Hash] Additional structured data
+      def warn(event, **data)
+        write_entry('warn', event, data)
+      end
+
+      # Log at error level.
+      #
+      # @param event [String] Event name
+      # @param data [Hash] Additional structured data
+      def error(event, **data)
+        write_entry('error', event, data)
+      end
+
+      # Log at debug level.
+      #
+      # @param event [String] Event name
+      # @param data [Hash] Additional structured data
+      def debug(event, **data)
+        write_entry('debug', event, data)
+      end
+
+      private
+
+      # Write a single JSON log line.
+      #
+      # @param level [String] Log level
+      # @param event [String] Event name
+      # @param data [Hash] Additional data
+      def write_entry(level, event, data)
+        entry = {
+          timestamp: Time.now.utc.iso8601,
+          level: level,
+          event: event
+        }.merge(data)
+
+        @output.puts(JSON.generate(entry))
+      end
+    end
+  end
+end

data/lib/codebase_index/operator/error_escalator.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Operator
+    # Classifies pipeline errors by severity and suggests remediation.
+    #
+    # @example
+    #   escalator = ErrorEscalator.new
+    #   result = escalator.classify(Timeout::Error.new("connection timed out"))
+    #   result[:severity]     # => :transient
+    #   result[:remediation]  # => "Retry after a short delay"
+    #
+    class ErrorEscalator
+      TRANSIENT_PATTERNS = [
+        { class_pattern: /Timeout|ETIMEDOUT/, category: 'timeout', remediation: 'Retry after a short delay' },
+        { class_pattern: /Net::/, category: 'network', remediation: 'Check network connectivity and retry' },
+        { class_pattern: /RateLimited|429/, category: 'rate_limit',
+          remediation: 'Back off and retry with exponential delay' },
+        { class_pattern: /CircuitOpenError/, category: 'circuit_open',
+          remediation: 'Wait for circuit breaker reset timeout' },
+        { class_pattern: /ConnectionPool|Busy/, category: 'resource_contention',
+          remediation: 'Wait for resources to free up' }
+      ].freeze
+
+      PERMANENT_PATTERNS = [
+        { class_pattern: /NameError|NoMethodError/, category: 'code_error',
+          remediation: 'Fix the code error and re-extract' },
+        { class_pattern: /Errno::ENOENT|FileNotFoundError/, category: 'missing_file',
+          remediation: 'Verify file paths and re-run extraction' },
+        { class_pattern: /JSON::ParserError/, category: 'corrupt_data',
+          remediation: 'Clean index and re-extract' },
+        { class_pattern: /ConfigurationError/, category: 'configuration',
+          remediation: 'Review CodebaseIndex configuration' },
+        { class_pattern: /ExtractionError/, category: 'extraction_failure',
+          remediation: 'Check extraction logs for specific failure details' }
+      ].freeze
+
+      # Classify an error by severity and suggest remediation.
+      #
+      # @param error [StandardError] The error to classify
+      # @return [Hash] :severity (:transient or :permanent), :category, :remediation, :error_class, :message
+      def classify(error)
+        error_string = "#{error.class} #{error.message}"
+
+        match = find_match(error_string, TRANSIENT_PATTERNS, :transient) ||
+                find_match(error_string, PERMANENT_PATTERNS, :permanent)
+
+        if match
+          match.merge(error_class: error.class.name, message: error.message)
+        else
+          {
+            severity: :unknown,
+            category: 'unclassified',
+            remediation: 'Investigate error details and check logs',
+            error_class: error.class.name,
+            message: error.message
+          }
+        end
+      end
+
+      private
+
+      # @param error_string [String]
+      # @param patterns [Array<Hash>]
+      # @param severity [Symbol]
+      # @return [Hash, nil]
+      def find_match(error_string, patterns, severity)
+        patterns.each do |pattern|
+          next unless error_string.match?(pattern[:class_pattern])
+
+          return {
+            severity: severity,
+            category: pattern[:category],
+            remediation: pattern[:remediation]
+          }
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/codebase_index/operator/pipeline_guard.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+require 'time'
+
+module CodebaseIndex
+  module Operator
+    # Rate limiter for pipeline operations using file-based state.
+    #
+    # Enforces a cooldown between consecutive runs of the same operation
+    # to prevent accidental repeated extraction or embedding.
+    #
+    # @example
+    #   guard = PipelineGuard.new(state_dir: '/tmp', cooldown: 300)
+    #   if guard.allow?(:extraction)
+    #     run_extraction
+    #     guard.record!(:extraction)
+    #   end
+    #
+    class PipelineGuard
+      # @param state_dir [String] Directory for persisting state
+      # @param cooldown [Integer] Minimum seconds between runs
+      def initialize(state_dir:, cooldown: 300)
+        @state_dir = state_dir
+        @cooldown = cooldown
+        @state_path = File.join(state_dir, 'pipeline_guard.json')
+      end
+
+      # Check if an operation is allowed (cooldown elapsed).
+      #
+      # @param operation [Symbol, String] Operation name
+      # @return [Boolean]
+      def allow?(operation)
+        last = last_run(operation)
+        return true if last.nil?
+
+        (Time.now - last) >= @cooldown
+      end
+
+      # Record that an operation has just run.
+      #
+      # @param operation [Symbol, String] Operation name
+      # @return [void]
+      def record!(operation)
+        FileUtils.mkdir_p(@state_dir)
+        File.open(@state_path, File::RDWR | File::CREAT) do |f|
+          f.flock(File::LOCK_EX)
+          content = f.read
+          state = if content.empty?
+                    {}
+                  else
+                    begin
+                      JSON.parse(content)
+                    rescue StandardError
+                      {}
+                    end
+                  end
+          state[operation.to_s] = Time.now.iso8601
+          f.rewind
+          f.write(JSON.generate(state))
+          f.truncate(f.pos)
+        end
+      end
+
+      # Get the last run time for an operation.
+      #
+      # @param operation [Symbol, String] Operation name
+      # @return [Time, nil]
+      def last_run(operation)
+        state = read_state
+        timestamp = state[operation.to_s]
+        return nil if timestamp.nil?
+
+        Time.parse(timestamp)
+      rescue ArgumentError
+        nil
+      end
+
+      private
+
+      # @return [Hash]
+      def read_state
+        return {} unless File.exist?(@state_path)
+
+        JSON.parse(File.read(@state_path))
+      rescue JSON::ParserError
+        {}
+      end
+
+      # @param state [Hash]
+      # @return [void]
+      def write_state(state)
+        FileUtils.mkdir_p(@state_dir)
+        File.write(@state_path, JSON.generate(state))
+      end
+    end
+  end
+end

data/lib/codebase_index/operator/status_reporter.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+
+module CodebaseIndex
+  module Operator
+    # Reports pipeline status by reading extraction output metadata.
+    #
+    # @example
+    #   reporter = StatusReporter.new(output_dir: 'tmp/codebase_index')
+    #   status = reporter.report
+    #   status[:status]           # => :ok
+    #   status[:staleness_seconds] # => 3600
+    #
+    class StatusReporter
+      STALE_THRESHOLD = 86_400 # 24 hours
+
+      # @param output_dir [String] Path to extraction output directory
+      def initialize(output_dir:)
+        @output_dir = output_dir
+      end
+
+      # Generate a pipeline status report.
+      #
+      # @return [Hash] Status report with :status, :extracted_at, :total_units, :counts, :staleness_seconds
+      def report
+        manifest = read_manifest
+        return not_extracted_report if manifest.nil?
+
+        staleness = compute_staleness(manifest['extracted_at'])
+
+        {
+          status: staleness < STALE_THRESHOLD ? :ok : :stale,
+          extracted_at: manifest['extracted_at'],
+          total_units: manifest['total_units'] || 0,
+          counts: manifest['counts'] || {},
+          git_sha: manifest['git_sha'],
+          git_branch: manifest['git_branch'],
+          staleness_seconds: staleness
+        }
+      end
+
+      private
+
+      # @return [Hash, nil]
+      def read_manifest
+        path = File.join(@output_dir, 'manifest.json')
+        return nil unless File.exist?(path)
+
+        JSON.parse(File.read(path))
+      rescue JSON::ParserError
+        nil
+      end
+
+      # @return [Hash]
+      def not_extracted_report
+        {
+          status: :not_extracted,
+          extracted_at: nil,
+          total_units: 0,
+          counts: {},
+          git_sha: nil,
+          git_branch: nil,
+          staleness_seconds: nil
+        }
+      end
+
+      # @param extracted_at [String, nil] ISO8601 timestamp
+      # @return [Numeric]
+      def compute_staleness(extracted_at)
+        return Float::INFINITY if extracted_at.nil?
+
+        Time.now - Time.parse(extracted_at)
+      rescue ArgumentError
+        Float::INFINITY
+      end
+    end
+  end
+end