smart_brain 0.1.0

data/lib/smart_brain/adapters/smart_rag/http_client.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module SmartBrain
+  module Adapters
+    module SmartRag
+      class HttpClient
+        def initialize(transport:, timeout_seconds: 2)
+          @transport = transport
+          @timeout_seconds = timeout_seconds
+        end
+
+        def retrieve(plan)
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          raw = transport.call(plan, timeout_seconds: timeout_seconds)
+          build_pack(raw: raw, request_id: plan[:request_id], took_ms: elapsed_ms(started_at))
+        rescue Timeout::Error
+          {
+            version: '0.1',
+            request_id: plan[:request_id],
+            plan_id: "timeout-#{plan[:request_id]}",
+            generated_at: Time.now.utc.iso8601,
+            evidences: [],
+            stats: { candidates: 0, returned: 0, took_ms: elapsed_ms(started_at) },
+            explain: { ignored_fields: [] },
+            warnings: ['smart_rag timeout; fallback to memory-only evidence']
+          }
+        end
+
+        private
+
+        attr_reader :transport, :timeout_seconds
+
+        def build_pack(raw:, request_id:, took_ms:)
+          ignored = []
+          ignored << 'global_filters.language not supported' unless raw.key?(:supports_language_filter) && raw[:supports_language_filter]
+
+          {
+            version: '0.1',
+            request_id: request_id,
+            plan_id: raw[:plan_id] || "remote-#{request_id}",
+            generated_at: Time.now.utc.iso8601,
+            evidences: Array(raw[:evidences]),
+            stats: {
+              candidates: raw.dig(:stats, :candidates) || Array(raw[:evidences]).size,
+              returned: Array(raw[:evidences]).size,
+              took_ms: took_ms
+            },
+            explain: {
+              ignored_fields: ignored + Array(raw.dig(:explain, :ignored_fields))
+            },
+            warnings: Array(raw[:warnings])
+          }
+        end
+
+        def elapsed_ms(start)
+          ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000).round(2)
+        end
+      end
+    end
+  end
+end

data/lib/smart_brain/adapters/smart_rag/null_client.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SmartBrain
+  module Adapters
+    module SmartRag
+      class NullClient
+        def retrieve(plan)
+          {
+            version: '0.1',
+            plan_id: "local-#{plan[:request_id]}",
+            request_id: plan[:request_id],
+            generated_at: Time.now.utc.iso8601,
+            evidences: [],
+            stats: { candidates: 0, returned: 0, took_ms: 0 },
+            explain: { ignored_fields: [] },
+            warnings: ['smart_rag client not configured; returned empty evidences']
+          }
+        end
+      end
+    end
+  end
+end

data/lib/smart_brain/configuration.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module SmartBrain
+  class Configuration
+    DEFAULT_PATH = File.expand_path('../../config/brain.yml', __dir__)
+
+    def self.load(path = nil)
+      file_path = path || DEFAULT_PATH
+      data = File.exist?(file_path) ? YAML.safe_load(File.read(file_path), symbolize_names: true) : {}
+      new(data || {})
+    end
+
+    attr_reader :raw
+
+    def initialize(raw)
+      @raw = raw
+    end
+
+    def policies
+      raw.fetch(:policies, {})
+    end
+
+    def retention
+      policies.fetch(:retention, {})
+    end
+
+    def retrieval
+      policies.fetch(:retrieval, {})
+    end
+
+    def composition
+      policies.fetch(:composition, {})
+    end
+
+    def observability
+      policies.fetch(:observability, {})
+    end
+  end
+end

data/lib/smart_brain/consolidator/working_summary.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module SmartBrain
+  module Consolidator
+    class WorkingSummary
+      def initialize(config:, clock:)
+        @config = config
+        @clock = clock
+        @summaries = {}
+        @last_summary_turn = Hash.new(0)
+      end
+
+      def update(session_id:, turn_count:, recent_turns:, memory_items:, stage_event: false)
+        reason = trigger_reason(session_id: session_id, turn_count: turn_count, recent_turns: recent_turns, stage_event: stage_event)
+        return latest_summary(session_id).merge(triggered: false, trigger_reason: 'not_triggered') unless reason
+
+        summary = {
+          summary_version: next_version(session_id),
+          summary_source_turn_range: source_turn_range(turn_count),
+          summary_generated_at: clock.call.iso8601,
+          text: build_text(memory_items),
+          triggered: true,
+          trigger_reason: reason
+        }
+        summaries[session_id] = summary
+        last_summary_turn[session_id] = turn_count
+        summary
+      end
+
+      def latest_summary(session_id)
+        summaries[session_id] || default_summary
+      end
+
+      private
+
+      attr_reader :config, :clock, :summaries, :last_summary_turn
+
+      def trigger_reason(session_id:, turn_count:, recent_turns:, stage_event:)
+        turns_since_last = turn_count - last_summary_turn[session_id]
+        threshold = config.retention.fetch(:summarize_after_turns, 12)
+        return 'turn_threshold' if turns_since_last >= threshold
+
+        token_limit = config.composition.fetch(:token_limit, 8192)
+        token_used = estimate_tokens(recent_turns)
+        return 'token_pressure' if token_used > (token_limit * 0.7)
+        return 'stage_event' if stage_event
+
+        nil
+      end
+
+      def estimate_tokens(recent_turns)
+        recent_turns.sum { |t| t[:content].to_s.length / 4 }
+      end
+
+      def next_version(session_id)
+        previous = summaries[session_id]
+        previous ? previous[:summary_version] + 1 : 1
+      end
+
+      def source_turn_range(turn_count)
+        { from: [turn_count - (config.retention.fetch(:summarize_after_turns, 12) - 1), 1].max, to: turn_count }
+      end
+
+      def build_text(memory_items)
+        goals = memory_items.select { |i| i[:type] == 'goals' }
+        tasks = memory_items.select { |i| i[:type] == 'tasks' }
+        decisions = memory_items.select { |i| i[:type] == 'decisions' }
+        refs = memory_items.select { |i| i[:type] == 'entities' }
+
+        [
+          'Goals:',
+          *to_lines(goals, fallback: '- None'),
+          'Decisions:',
+          *to_lines(decisions, fallback: '- None'),
+          'Tasks:',
+          *to_lines(tasks, fallback: '- None'),
+          'Key References:',
+          *to_lines(refs, fallback: '- None'),
+          'Open Questions:',
+          '- None'
+        ].join("\n")
+      end
+
+      def to_lines(items, fallback:)
+        return [fallback] if items.empty?
+
+        items.first(5).map { |i| "- #{i[:key]}" }
+      end
+
+      def default_summary
+        {
+          summary_version: 0,
+          summary_source_turn_range: { from: 0, to: 0 },
+          summary_generated_at: clock.call.iso8601,
+          text: "Goals:\n- None\nDecisions:\n- None\nTasks:\n- None\nKey References:\n- None\nOpen Questions:\n- None",
+          triggered: false,
+          trigger_reason: 'empty'
+        }
+      end
+    end
+  end
+end

data/lib/smart_brain/context_composer/composer.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module SmartBrain
+  module ContextComposer
+    class Composer
+      def initialize(config:, clock:)
+        @config = config
+        @clock = clock
+      end
+
+      def compose(session_id:, user_message:, plan:, plan_id:, summary:, recent_turns:, evidence_bundle:)
+        context_id = SecureRandom.uuid
+        evidence = evidence_bundle.fetch(:selected, [])
+        used_estimate = estimate_tokens(summary: summary[:text], recent_turns: recent_turns, evidence: evidence, user_message: user_message)
+        token_limit = config.composition.fetch(:token_limit, 8192)
+
+        {
+          version: '0.1',
+          context_id: context_id,
+          session_id: session_id,
+          created_at: clock.call.iso8601,
+          system_blocks: [],
+          developer_blocks: [],
+          working_summary: summary[:text],
+          recent_turns: recent_turns.first(config.composition.fetch(:recent_turns_max, 8)),
+          evidence: evidence,
+          user_message: { role: 'user', content: user_message },
+          constraints: {
+            token_budget: {
+              limit: token_limit,
+              used_estimate: used_estimate
+            },
+            diversity: {
+              by_document: config.composition.dig(:diversity, :by_document) || 3,
+              by_source: config.composition.dig(:diversity, :by_source_uri) || 2
+            },
+            truncation: {
+              snippets_max_chars: config.composition.fetch(:max_snippet_chars, 800),
+              recent_turns_max: config.composition.fetch(:recent_turns_max, 8)
+            }
+          },
+          debug: {
+            trace: {
+              context_id: context_id,
+              request_id: plan[:request_id],
+              plan_id: plan_id
+            },
+            planner: {
+              request_id: plan[:request_id],
+              purpose: plan[:purpose],
+              queries: plan[:queries].map { |q| q[:text] }
+            },
+            why_selected: evidence.map { |e| "#{e[:id]} score=#{e[:score]} source=#{e[:source]}" },
+            ignored: evidence_bundle[:ignored_fields] || [],
+            dropped: (evidence_bundle[:dropped] || []).map { |e| { id: e[:id], reason: e[:drop_reason] } }
+          }
+        }
+      end
+
+      private
+
+      attr_reader :config, :clock
+
+      def estimate_tokens(summary:, recent_turns:, evidence:, user_message:)
+        text_size = summary.to_s.length
+        text_size += recent_turns.sum { |t| t[:content].to_s.length }
+        text_size += evidence.sum { |e| e[:snippet].to_s.length }
+        text_size += user_message.to_s.length
+        (text_size / 4.0).ceil
+      end
+    end
+  end
+end

data/lib/smart_brain/contracts/context_package.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module SmartBrain
+  module Contracts
+    class ContextPackage
+      REQUIRED_KEYS = %i[version context_id session_id created_at user_message evidence].freeze
+
+      def self.validate!(pkg)
+        missing = REQUIRED_KEYS.reject { |key| pkg.key?(key) }
+        raise ArgumentError, "invalid context package: missing #{missing.join(', ')}" unless missing.empty?
+
+        true
+      end
+    end
+  end
+end

data/lib/smart_brain/contracts/evidence_pack.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module SmartBrain
+  module Contracts
+    class EvidencePack
+      REQUIRED_KEYS = %i[version request_id plan_id generated_at evidences].freeze
+
+      def self.validate!(pack)
+        missing = REQUIRED_KEYS.reject { |key| pack.key?(key) }
+        raise ArgumentError, "invalid evidence pack: missing #{missing.join(', ')}" unless missing.empty?
+
+        true
+      end
+    end
+  end
+end

data/lib/smart_brain/contracts/retrieval_plan.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SmartBrain
+  module Contracts
+    class RetrievalPlan
+      REQUIRED_KEYS = %i[version request_id purpose queries budget].freeze
+
+      def self.validate!(plan)
+        missing = REQUIRED_KEYS.reject { |key| plan.key?(key) }
+        raise ArgumentError, "invalid retrieval plan: missing #{missing.join(', ')}" unless missing.empty?
+        raise ArgumentError, 'invalid retrieval plan: queries must not be empty' if Array(plan[:queries]).empty?
+
+        true
+      end
+    end
+  end
+end

data/lib/smart_brain/event_store/in_memory.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module SmartBrain
+  module EventStore
+    class InMemory
+      def initialize
+        @sessions = {}
+      end
+
+      def append_turn(session_id:, turn_events:, created_at:)
+        session = sessions[session_id] ||= { seq: 0, turns: [] }
+        session[:seq] += 1
+        turn_id = SecureRandom.uuid
+        messages = normalize_messages(turn_id: turn_id, messages: turn_events[:messages] || [], created_at: created_at)
+        refs = normalize_refs(turn_id: turn_id, refs: turn_events[:refs] || [], created_at: created_at)
+        turn = {
+          id: turn_id,
+          session_id: session_id,
+          seq: session[:seq],
+          created_at: created_at.iso8601,
+          turn_events: turn_events.merge(messages: messages, refs: refs)
+        }
+        session[:turns] << turn
+        turn
+      end
+
+      def turns_count(session_id:)
+        (sessions[session_id] || { turns: [] })[:turns].size
+      end
+
+      def recent_turns(session_id:, limit:)
+        session = sessions[session_id]
+        return [] unless session
+
+        session[:turns].last(limit).flat_map do |turn|
+          (turn.dig(:turn_events, :messages) || []).map do |m|
+            {
+              turn_id: turn[:id],
+              message_id: m[:id],
+              role: m[:role],
+              content: m[:content],
+              created_at: m[:created_at]
+            }
+          end
+        end
+      end
+
+      def recent_refs(session_id:, limit:)
+        session = sessions[session_id]
+        return [] unless session
+
+        session[:turns].last(limit).flat_map { |t| t.dig(:turn_events, :refs) || [] }
+      end
+
+      def entity_frequencies(session_id:, window_turns:)
+        session = sessions[session_id]
+        return Hash.new(0) unless session
+
+        freq = Hash.new(0)
+        session[:turns].last(window_turns).each do |turn|
+          Array(turn.dig(:turn_events, :entities)).each do |entity|
+            canonical = entity[:canonical] || entity[:name]
+            freq[canonical.to_s.downcase] += 1
+          end
+        end
+        freq
+      end
+
+      def all_turns(session_id:)
+        return sessions.values.flat_map { |s| s[:turns] } if session_id.nil?
+
+        (sessions[session_id] || { turns: [] })[:turns]
+      end
+
+      private
+
+      attr_reader :sessions
+
+      def normalize_messages(turn_id:, messages:, created_at:)
+        messages.map do |m|
+          m.merge(
+            id: m[:id] || SecureRandom.uuid,
+            turn_id: turn_id,
+            created_at: m[:created_at] || created_at.iso8601
+          )
+        end
+      end
+
+      def normalize_refs(turn_id:, refs:, created_at:)
+        refs.map do |ref|
+          ref.merge(
+            id: ref[:id] || SecureRandom.uuid,
+            turn_id: turn_id,
+            created_at: ref[:created_at] || created_at.iso8601,
+            ref_meta_json: ref[:ref_meta_json] || {}
+          )
+        end
+      end
+    end
+  end
+end

data/lib/smart_brain/fusion/merger.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module SmartBrain
+  module Fusion
+    class Merger
+      def initialize(config:)
+        @config = config
+      end
+
+      def merge(query:, memory_evidence:, resource_evidence:)
+        combined = (memory_evidence + normalize_resource(resource_evidence))
+        deduped = dedupe(combined)
+        ranked = deduped.sort_by { |item| -rerank_score(item, query) }
+        diversified, dropped = apply_diversity(ranked)
+        selected = apply_budget(diversified)
+
+        {
+          selected: selected,
+          dropped: dropped,
+          ignored_fields: []
+        }
+      end
+
+      private
+
+      attr_reader :config
+
+      def normalize_resource(items)
+        items.map do |item|
+          item.merge(
+            source: 'resource',
+            score: item.fetch(:score, item.dig(:signals, :rerank_score) || item.dig(:signals, :rrf_score) || 0.4),
+            ref: item[:ref] || {
+              document_id: item[:document_id],
+              section_id: item[:section_id],
+              chunk_index: item.dig(:metadata, :chunk_index)
+            }
+          )
+        end
+      end
+
+      def dedupe(items)
+        deduped = {}
+        items.each do |item|
+          key = dedupe_key(item)
+          existing = deduped[key]
+          deduped[key] = item if existing.nil? || item.fetch(:score, 0.0) > existing.fetch(:score, 0.0)
+        end
+        deduped.values
+      end
+
+      def apply_diversity(items)
+        by_document = config.composition.dig(:diversity, :by_document) || 3
+        by_source = config.composition.dig(:diversity, :by_source_uri) || 2
+
+        doc_counter = Hash.new(0)
+        source_counter = Hash.new(0)
+        kept = []
+        dropped = []
+
+        items.each do |item|
+          ref = item[:ref] || {}
+          document_key = ref[:document_id] || item[:title]
+          source_prefix = item[:source_uri].to_s.split('/')[0, 3].join('/')
+          source_key = source_prefix.empty? ? item[:source_uri] : source_prefix
+
+          if doc_counter[document_key] >= by_document || source_counter[source_key] >= by_source
+            dropped << item.merge(drop_reason: 'diversity')
+            next
+          end
+
+          doc_counter[document_key] += 1
+          source_counter[source_key] += 1
+          kept << item
+        end
+
+        [kept, dropped]
+      end
+
+      def apply_budget(items)
+        limit = config.composition.fetch(:evidence_max_items, 12)
+        max_chars = config.composition.fetch(:max_snippet_chars, 800)
+        ratio = parse_ratio(config.composition.dig(:diversity, :memory_resource_ratio) || '40/60')
+
+        memory_limit = (limit * ratio[:memory]).floor
+        resource_limit = limit - memory_limit
+        selected = []
+
+        memory_items = items.select { |i| i[:source] == 'memory' }.first(memory_limit)
+        resource_items = items.select { |i| i[:source] == 'resource' }.first(resource_limit)
+        selected.concat(memory_items).concat(resource_items)
+
+        if selected.length < limit
+          leftovers = (items - selected).first(limit - selected.length)
+          selected.concat(leftovers)
+        end
+
+        selected.first(limit).map do |item|
+          snippet = item[:snippet].to_s
+          item.merge(snippet: snippet.length > max_chars ? "#{snippet[0...max_chars]}..." : snippet)
+        end
+      end
+
+      def parse_ratio(text)
+        memory, resource = text.to_s.split('/').map(&:to_i)
+        total = memory + resource
+        return { memory: 0.4, resource: 0.6 } if total <= 0
+
+        { memory: memory.to_f / total, resource: resource.to_f / total }
+      end
+
+      def rerank_score(item, query)
+        score = item.fetch(:score, 0.0)
+        score + lexical_boost(item: item, query: query)
+      end
+
+      def lexical_boost(item:, query:)
+        terms = query.to_s.downcase.scan(/[[:alnum:]_\-\p{Han}]+/)
+        text = "#{item[:title]} #{item[:snippet]}".downcase
+        terms.count { |term| text.include?(term) } * 0.05
+      end
+
+      def dedupe_key(item)
+        ref = item[:ref] || {}
+        if ref[:document_id] && ref[:section_id]
+          "resource:#{ref[:document_id]}:#{ref[:section_id]}:#{ref[:chunk_index]}"
+        elsif ref[:memory_item_id]
+          "memory:#{ref[:memory_item_id]}"
+        elsif ref[:turn_id] && ref[:message_id]
+          "memory-turn:#{ref[:turn_id]}:#{ref[:message_id]}"
+        else
+          "fallback:#{item[:id]}"
+        end
+      end
+    end
+  end
+end

data/lib/smart_brain/memory_extractor/extractor.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module SmartBrain
+  module MemoryExtractor
+    class Extractor
+      def initialize(config:)
+        @config = config
+      end
+
+      def extract(session_id:, turn:, entity_frequencies: Hash.new(0))
+        events = turn.fetch(:turn_events)
+        explain = []
+        items = []
+
+        collect(items, explain, events[:tasks], type: 'tasks', confidence: confidence(:tool_derived), source_turn_id: turn[:id])
+        collect(items, explain, events[:decisions], type: 'decisions', confidence: confidence(:user_asserted), source_turn_id: turn[:id])
+        collect(items, explain, events[:goals], type: 'goals', confidence: confidence(:user_asserted), source_turn_id: turn[:id])
+        collect(items, explain, events[:events], type: 'events', confidence: confidence(:tool_derived), source_turn_id: turn[:id])
+
+        Array(events[:preferences]).each do |preference|
+          key = preference.fetch(:key)
+          if preference[:confirmed]
+            items << build_item(type: 'preferences', key: key, value_json: preference, source_turn_id: turn[:id], confidence: confidence(:user_asserted))
+            explain << "write preferences:#{key}"
+          else
+            explain << "skip preferences:#{key} not confirmed"
+          end
+        end
+
+        Array(events[:entities]).each do |entity|
+          key = entity.fetch(:key)
+          canonical = (entity[:canonical] || entity[:name]).to_s.downcase
+          should_write = entity[:remember] || entity_structure_signal?(entity) || entity_frequencies[canonical] >= freq_threshold
+          if should_write
+            items << build_item(type: 'entities', key: key, value_json: entity, source_turn_id: turn[:id], confidence: confidence(:inferred))
+            explain << "write entities:#{key}"
+          else
+            explain << "skip entities:#{key} below threshold"
+          end
+        end
+
+        Array(events[:retractions]).each do |retraction|
+          items << build_item(type: retraction.fetch(:type), key: retraction.fetch(:key), value_json: retraction, source_turn_id: turn[:id], confidence: confidence(:user_asserted), status: 'retracted')
+          explain << "retract #{retraction.fetch(:type)}:#{retraction.fetch(:key)}"
+        end
+
+        {
+          session_id: session_id,
+          items: items,
+          explain: explain
+        }
+      end
+
+      private
+
+      attr_reader :config
+
+      def collect(items, explain, raw_items, type:, confidence:, source_turn_id:)
+        Array(raw_items).each do |entry|
+          key = entry.fetch(:key)
+          items << build_item(type: type, key: key, value_json: entry, source_turn_id: source_turn_id, confidence: confidence)
+          explain << "write #{type}:#{key}"
+        end
+      end
+
+      def build_item(type:, key:, value_json:, source_turn_id:, confidence:, status: 'active')
+        {
+          type: type,
+          key: key,
+          value_json: value_json,
+          source_turn_id: source_turn_id,
+          confidence: confidence,
+          status: status,
+          updated_at: Time.now.utc.iso8601
+        }
+      end
+
+      def confidence(name)
+        config.retention.fetch(:confidence, {}).fetch(name, 0.6)
+      end
+
+      def freq_threshold
+        config.retention.dig(:entity_gate, :freq_threshold) || 2
+      end
+
+      def entity_structure_signal?(entity)
+        canonical = entity[:canonical].to_s
+        canonical.include?('/') || canonical.include?('http') || canonical.include?('.')
+      end
+    end
+  end
+end