smart_brain 0.1.2 → 0.2.0

data/lib/smart_brain/governance/fact_check.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require 'time'
+require_relative '../support/levenshtein'
+
+module SmartBrain
+  module Governance
+    # Offline, zero-LLM, zero-network contradiction detector (mempal parity).
+    # Scans a piece of text against a session's entities + KG edges and reports
+    # three finding types:
+    #   - SimilarNameConflict: a token within Levenshtein <= N of a known entity
+    #   - RelationContradiction: text mentions both endpoints of an active edge
+    #     alongside a negation/contrary cue (may contradict the relation)
+    #   - StaleFact: text references an edge whose valid_to < now (invalidated)
+    #
+    # Heuristic by design — surfaces candidates for human/agent review.
+    class FactCheck
+      TOKEN_RE = /[[:alnum:]_\-\p{Han}]+/.freeze
+
+      def initialize(memory_store:, config:)
+        @memory_store = memory_store
+        @config = config
+      end
+
+      def check(session_id:, text:, now: Time.now.utc, scope_ids: nil)
+        now = Time.parse(now).utc if now.is_a?(String)
+        text_l = text.to_s.downcase
+        tokens = text.to_s.downcase.scan(TOKEN_RE)
+
+        findings = []
+        findings.concat(similar_name_conflicts(session_id:, scope_ids:, tokens: tokens))
+        findings.concat(relation_contradictions(session_id:, scope_ids:, text_l: text_l))
+        findings.concat(stale_facts(session_id:, scope_ids:, text_l: text_l, now: now))
+
+        { findings: findings, checked_at: Time.now.utc.iso8601,
+          counts: { total: findings.size,
+                    similar_name: findings.count { |f| f[:type] == 'SimilarNameConflict' },
+                    relation_contradiction: findings.count { |f| f[:type] == 'RelationContradiction' },
+                    stale_fact: findings.count { |f| f[:type] == 'StaleFact' } } }
+      end
+
+      private
+
+      attr_reader :memory_store, :config
+
+      def similar_name_conflicts(session_id:, scope_ids:, tokens:)
+        distance = config.fact_check.fetch(:similar_name_distance, 2) rescue 2
+        names = memory_store.entities(session_id: session_id, scope_ids: scope_ids).each_with_object([]) do |e, acc|
+          acc << { name: e[:canonical].to_s, entity: e } unless e[:canonical].to_s.empty?
+          acc << { name: e[:name].to_s, entity: e } unless e[:name].to_s.empty?
+        end
+
+        tokens.product(names).each_with_object([]) do |(token, named), acc|
+          next if token.length < 3 || named[:name].length < 3
+          next if token == named[:name]
+
+          d = Support::Levenshtein.distance(token, named[:name])
+          next unless d.between?(1, distance)
+
+          acc << { type: 'SimilarNameConflict', severity: 'warning',
+                   detail: "'#{token}' is within Levenshtein #{d} of entity '#{named[:name]}'",
+                   refs: { token: token, entity: named[:name], entity_id: named[:entity][:id], distance: d } }
+        end
+      end
+
+      def relation_contradictions(session_id:, scope_ids:, text_l:)
+        cues = negation_cues
+        edges = safe_edges(session_id, scope_ids).select { |e| e[:status] == 'active' }
+        return [] if edges.empty? || cues.empty?
+
+        edges.each_with_object([]) do |edge, acc|
+          s, o = edge[:subject].downcase, edge[:object].downcase
+          next unless text_l.include?(s) && text_l.include?(o)
+          next unless cues.any? { |cue| text_l.include?(cue.to_s.downcase) }
+
+          acc << { type: 'RelationContradiction', severity: 'warning',
+                   detail: "text may contradict relation '#{edge[:subject]} #{edge[:predicate]} #{edge[:object]}'",
+                   refs: { edge_id: edge[:id], subject: edge[:subject], predicate: edge[:predicate], object: edge[:object] } }
+        end
+      end
+
+      def stale_facts(session_id:, scope_ids:, text_l:, now:)
+        safe_edges(session_id, scope_ids).select { |e| e[:status] == 'invalidated' && e[:valid_to] }
+          .each_with_object([]) do |edge, acc|
+            valid_to = Time.parse(edge[:valid_to].to_s).utc rescue next
+            next if valid_to >= now
+
+            s, o = edge[:subject].downcase, edge[:object].downcase
+            next unless text_l.include?(s) || text_l.include?(o)
+
+            acc << { type: 'StaleFact', severity: 'info',
+                     detail: "text references invalidated fact '#{edge[:subject]} #{edge[:predicate]} #{edge[:object]}' (valid_to #{edge[:valid_to]})",
+                     refs: { edge_id: edge[:id], subject: edge[:subject], object: edge[:object], valid_to: edge[:valid_to] } }
+          end
+      end
+
+      def safe_edges(session_id, scope_ids)
+        return [] unless memory_store.respond_to?(:query_edges)
+
+        memory_store.query_edges(session_id: session_id, scope_ids: scope_ids, include_invalid: true)
+      rescue StandardError
+        []
+      end
+
+      def negation_cues
+        config.fact_check.fetch(:negation_cues, []) rescue []
+      end
+    end
+  end
+end

data/lib/smart_brain/governance/knowledge_graph.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module SmartBrain
+  module Governance
+    # Knowledge graph: typed triples (subject-predicate-object) with temporal
+    # validity (valid_from/valid_to) over a MemoryStore. Orthogonal to the
+    # Stage-1 lifecycle and to memory_items — edges live in kg_edges and are
+    # added explicitly (via turn_events[:edges] extraction) or by the agent.
+    class KnowledgeGraph
+      def initialize(memory_store:, config:)
+        @memory_store = memory_store
+        @config = config
+      end
+
+      def add(session_id:, subject:, predicate:, object:, source_turn_id: nil,
+              source_memory_item_id: nil, confidence: nil, valid_from: nil,
+              scope_id: nil, scope: nil, source_session_id: nil)
+        memory_store.add_edge(
+          session_id: session_id,
+          scope_id: scope_id,
+          scope: scope,
+          source_session_id: source_session_id,
+          edge: {
+            subject: subject, predicate: predicate, object: object,
+            source_turn_id: source_turn_id, source_memory_item_id: source_memory_item_id,
+            confidence: confidence || default_confidence, valid_from: valid_from
+          }
+        )
+      end
+
+      def query(session_id: nil, scope_ids: nil, subject: nil, predicate: nil, object: nil, include_invalid: false)
+        memory_store.query_edges(session_id: session_id, scope_ids: scope_ids, subject: subject,
+                                 predicate: predicate, object: object,
+                                 include_invalid: include_invalid)
+      end
+
+      def timeline(session_id: nil, scope_ids: nil, subject:)
+        memory_store.edges_for_subject(session_id: session_id, scope_ids: scope_ids, subject: subject)
+      end
+
+      def invalidate(edge_id:, reason: nil)
+        memory_store.invalidate_edge(id: edge_id, reason: reason)
+      end
+
+      def stats(session_id: nil, scope_ids: nil)
+        memory_store.edge_stats(session_id: session_id, scope_ids: scope_ids)
+      end
+
+      private
+
+      attr_reader :memory_store, :config
+
+      def default_confidence
+        config.kg.fetch(:default_confidence, 0.6)
+      rescue StandardError
+        0.6
+      end
+    end
+  end
+end

data/lib/smart_brain/governance/lifecycle.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require_relative 'tiers'
+
+module SmartBrain
+  module Governance
+    # Stage-1 knowledge lifecycle (mempal parity):
+    #
+    #   evidence → distill(candidate) → gate → promote(promoted) → demote/retire
+    #
+    # Operates over a MemoryStore (independent of backend). distill creates a
+    # candidate knowledge item at tier dao_ren|qi; gate is a read-only readiness
+    # check; promote/demote are evidence-backed transitions that append to the
+    # knowledge_events audit trail. Orthogonal to the conflict `status`.
+    class Lifecycle
+      DEMOTE_REASONS = %w[contradicted obsolete superseded].freeze
+
+      def initialize(memory_store:, config:, model_provider: nil)
+        @memory_store = memory_store
+        @config = config
+        @model_provider = model_provider
+      end
+
+      def distill(session_id:, statement:, content: nil, tier:, supporting_refs: [], domain: nil, field: nil,
+                  reviewer: nil, key: nil, scope_id: nil, scope: nil)
+        tier = tier.to_s
+        raise ArgumentError, "tier must be one of #{Tiers::DISTILLABLE.join('/')}" unless Tiers.distillable?(tier)
+
+        statement = draft_statement(statement, supporting_refs: supporting_refs, content: content) if statement.nil? || statement.strip.empty?
+        raise ArgumentError, 'statement is required (provide one or wire an LLM provider)' if statement.nil? || statement.strip.empty?
+
+        item = {
+          type: 'knowledge',
+          key: key || "knowledge:#{tier}:#{slugify(statement)}",
+          value_json: {
+            statement: statement,
+            content: content,
+            tier: tier,
+            domain: domain,
+            field: field,
+            supporting_refs: Array(supporting_refs)
+          },
+          confidence: 0.5,
+          status: 'active',
+          tier: tier,
+          lifecycle_status: 'candidate',
+          source_turn_id: nil,
+          updated_at: Time.now.utc.iso8601
+        }
+        record = memory_store.create_item(session_id: session_id, item: item, scope_id: scope_id, scope: scope)
+        memory_store.record_event(
+          memory_item_id: record[:id], event_type: 'distill',
+          from_lifecycle: nil, to_lifecycle: 'candidate',
+          reason: 'distilled from evidence', reviewer: reviewer,
+          evidence_refs: Array(supporting_refs)
+        )
+        { memory_item: record, event: last_event(record[:id]) }
+      end
+
+      # Read-only promotion readiness check.
+      def gate(memory_item_id:)
+        item = memory_store.find_item(id: memory_item_id)
+        raise ArgumentError, "memory item not found: #{memory_item_id}" unless item
+
+        refs = Array(item.dig(:value_json, :supporting_refs) || item.dig(:value_json, 'supporting_refs'))
+        min = min_supporting_refs
+        reviewer = item.dig(:value_json, :reviewer) || item.dig(:value_json, 'reviewer')
+        checks = {
+          lifecycle: item[:lifecycle_status],
+          supporting_refs: refs.size,
+          min_required: min,
+          reviewer: reviewer
+        }
+        ready = item[:lifecycle_status] == 'candidate' && (refs.size >= min || reviewer == 'human')
+        { ready: ready, target_status: 'promoted', checks: checks }
+      end
+
+      def promote(memory_item_id:, verification_refs:, reason:, reviewer: nil, force: false)
+        item = memory_store.find_item(id: memory_item_id)
+        raise ArgumentError, "memory item not found: #{memory_item_id}" unless item
+
+        decision = gate(memory_item_id: memory_item_id)
+        if !decision[:ready] && !force
+          raise LifecycleGateError, "promotion gate failed: #{decision[:checks].inspect}"
+        end
+
+        from = item[:lifecycle_status]
+        updated = memory_store.set_lifecycle(
+          id: memory_item_id, lifecycle_status: 'promoted',
+          merge_value: { verification_refs: Array(verification_refs), reviewer: reviewer }
+        )
+        memory_store.record_event(
+          memory_item_id: memory_item_id, event_type: 'promote',
+          from_lifecycle: from, to_lifecycle: 'promoted',
+          reason: reason, reviewer: reviewer, evidence_refs: Array(verification_refs)
+        )
+        { memory_item: updated, event: last_event(memory_item_id) }
+      end
+
+      def demote(memory_item_id:, evidence_refs:, reason:, reason_type:, reviewer: nil)
+        raise ArgumentError, "reason_type must be one of #{DEMOTE_REASONS.join('/')}" unless DEMOTE_REASONS.include?(reason_type.to_s)
+
+        item = memory_store.find_item(id: memory_item_id)
+        raise ArgumentError, "memory item not found: #{memory_item_id}" unless item
+
+        from = item[:lifecycle_status]
+        updated = memory_store.set_lifecycle(
+          id: memory_item_id, lifecycle_status: 'demoted',
+          merge_value: { demotion_reason: reason, reason_type: reason_type }
+        )
+        memory_store.record_event(
+          memory_item_id: memory_item_id, event_type: 'demote',
+          from_lifecycle: from, to_lifecycle: 'demoted',
+          reason: reason, reason_type: reason_type, reviewer: reviewer, evidence_refs: Array(evidence_refs)
+        )
+        { memory_item: updated, event: last_event(memory_item_id) }
+      end
+
+      def promote_to_scope(source_item:, target_scope:, session_id:, verification_refs:, reason:, reviewer: nil)
+        refs = Array(verification_refs)
+        unless reviewer == 'human' || refs.size >= min_supporting_refs
+          raise LifecycleGateError, "scope promotion requires human review or #{min_supporting_refs} verification refs"
+        end
+
+        provenance = {
+          source_memory_item_id: source_item[:id],
+          source_scope: source_item[:scope],
+          target_scope: { type: target_scope[:type], id: target_scope[:id] },
+          source_session_id: source_item[:source_session_id] || source_item[:session_id],
+          verification_refs: refs,
+          reviewer: reviewer,
+          reason: reason
+        }
+        item = source_item.slice(:type, :key, :confidence, :tier).merge(
+          value_json: (source_item[:value_json] || {}).merge(provenance: provenance),
+          status: 'active', lifecycle_status: 'promoted', source_turn_id: source_item[:source_turn_id],
+          source_message_id: source_item[:source_message_id],
+          source_session_id: source_item[:source_session_id] || source_item[:session_id],
+          updated_at: Time.now.utc.iso8601
+        )
+        target = memory_store.create_item(
+          session_id: session_id, item: item, scope_id: target_scope[:scope_id],
+          scope: { type: target_scope[:type], id: target_scope[:id] }
+        )
+        event = memory_store.record_event(
+          memory_item_id: target[:id], event_type: 'promote_to_scope',
+          from_lifecycle: source_item[:lifecycle_status], to_lifecycle: 'promoted',
+          reason: reason, reviewer: reviewer, evidence_refs: refs
+        )
+        target = memory_store.set_lifecycle(
+          id: target[:id], lifecycle_status: 'promoted',
+          merge_value: { provenance: provenance.merge(promotion_event_id: event[:id]) }
+        )
+        { source_memory_item: source_item, memory_item: target, event: event }
+      end
+
+      def retract(memory_item_id:, evidence_refs:, reason:, reviewer: nil)
+        item = memory_store.find_item(id: memory_item_id)
+        raise ArgumentError, "memory item not found: #{memory_item_id}" unless item
+
+        updated = memory_store.set_status(id: memory_item_id, status: 'retracted', merge_value: { retraction_reason: reason })
+        event = memory_store.record_event(
+          memory_item_id: memory_item_id, event_type: 'retract',
+          from_lifecycle: item[:lifecycle_status], to_lifecycle: item[:lifecycle_status],
+          reason: reason, reviewer: reviewer, evidence_refs: Array(evidence_refs)
+        )
+        { memory_item: updated, event: event }
+      end
+
+      def lineage(memory_item_id:)
+        lineage = []
+        current = memory_store.find_item(id: memory_item_id)
+        while current
+          lineage << current
+          source_id = current.dig(:value_json, :provenance, :source_memory_item_id)
+          break unless source_id
+
+          current = memory_store.find_item(id: source_id)
+        end
+        lineage
+      end
+
+      def events(memory_item_id:)
+        memory_store.events_for(memory_item_id: memory_item_id)
+      end
+
+      private
+
+      attr_reader :memory_store, :config, :model_provider
+
+      def min_supporting_refs
+        config.lifecycle.fetch(:min_supporting_refs, 2) rescue 2
+      end
+
+      def last_event(memory_item_id)
+        memory_store.events_for(memory_item_id: memory_item_id).last
+      end
+
+      def draft_statement(statement, supporting_refs:, content:)
+        return statement unless model_provider&.llm?
+        return nil if Array(supporting_refs).empty?
+
+        refs_text = supporting_refs_for(supporting_refs).map { |i| "- #{i[:key]}: #{i.dig(:value_json, :statement) || i[:value_json]}" }.join("\n")
+        result = model_provider.complete(
+          prompt: "Distill ONE concise knowledge statement (<= 1 line) from these observations; reply with only the statement.\n#{refs_text}",
+          temperature: 0.2, max_tokens: 120
+        )
+        result[:error] ? nil : result[:text]
+      rescue StandardError
+        nil
+      end
+
+      def supporting_refs_for(refs)
+        Array(refs).filter_map { |id| memory_store.find_item(id: id) }
+      end
+
+      def slugify(text)
+        text.to_s.downcase.scan(/[[:alnum:]]+/).first(6).join('-')[0, 60]
+      end
+    end
+
+    class LifecycleGateError < StandardError; end
+  end
+end

data/lib/smart_brain/governance/tiers.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module SmartBrain
+  module Governance
+    # Mind-model tiers (mempal parity): dao_tian 天道 / dao_ren 人道 / shu 术 /
+    # qi 器 / evidence 证据. Lower ORDER index = higher assembly priority.
+    module Tiers
+      ORDER = %w[dao_tian dao_ren shu qi evidence].freeze
+      PRIORITY = ORDER.each_with_index.to_h.freeze
+
+      # Default tier per memory type. dao_tian is never auto-assigned — it is
+      # reserved for user-pinned immutable principles.
+      DEFAULT_TIER_BY_TYPE = {
+        'profile' => 'dao_ren',
+        'decisions' => 'dao_ren',
+        'patterns' => 'dao_ren',
+        'goals' => 'shu',
+        'tasks' => 'shu',
+        'preferences' => 'qi',
+        'cases' => 'qi',
+        'entities' => 'evidence',
+        'events' => 'evidence'
+      }.freeze
+
+      # Only dao_ren / qi knowledge can enter the candidate stage via distill
+      # (mempal rule: dao_tian and shu are not open to candidate state).
+      DISTILLABLE = %w[dao_ren qi].freeze
+
+      # lifecycle statuses that participate in automatic context assembly.
+      CONTEXT_LIFECYCLE = %w[raw promoted].freeze
+
+      module_function
+
+      def tier_for(type)
+        DEFAULT_TIER_BY_TYPE.fetch(type.to_s, 'evidence')
+      end
+
+      def priority(tier)
+        PRIORITY.fetch(tier.to_s, PRIORITY.fetch('evidence'))
+      end
+
+      def distillable?(tier)
+        DISTILLABLE.include?(tier.to_s)
+      end
+
+      def valid?(tier)
+        PRIORITY.key?(tier.to_s)
+      end
+
+      def dao_tian_limit(config)
+        (config.tiers.fetch(:dao_tian_limit, 1) rescue 1)
+      end
+
+      # Sort by tier priority asc (dao_tian first); stable on equal priority.
+      def sort_by_tier(items)
+        items.sort_by.with_index { |i, idx| [priority(i[:tier] || i['tier'] || 'evidence'), idx] }
+      end
+    end
+  end
+end

data/lib/smart_brain/memory_extractor/extractor.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../governance/tiers'
+
 module SmartBrain
   module MemoryExtractor
     class Extractor
@@ -16,11 +18,14 @@ module SmartBrain
         collect(items, explain, events[:decisions], type: 'decisions', confidence: confidence(:user_asserted), source_turn_id: turn[:id])
         collect(items, explain, events[:goals], type: 'goals', confidence: confidence(:user_asserted), source_turn_id: turn[:id])
         collect(items, explain, events[:events], type: 'events', confidence: confidence(:tool_derived), source_turn_id: turn[:id])
+        collect(items, explain, events[:profile], type: 'profile', confidence: confidence(:user_asserted), source_turn_id: turn[:id])
+        collect(items, explain, events[:cases], type: 'cases', confidence: confidence(:tool_derived), source_turn_id: turn[:id])
+        collect(items, explain, events[:patterns], type: 'patterns', confidence: confidence(:inferred), source_turn_id: turn[:id])
 
         Array(events[:preferences]).each do |preference|
           key = preference.fetch(:key)
           if preference[:confirmed]
-            items << build_item(type: 'preferences', key: key, value_json: preference, source_turn_id: turn[:id], confidence: confidence(:user_asserted))
+            items << build_item(type: 'preferences', key: key, value_json: preference, source_turn_id: turn[:id], confidence: confidence(:user_asserted), scope_ref: preference[:scope_ref])
             explain << "write preferences:#{key}"
           else
             explain << "skip preferences:#{key} not confirmed"
@@ -32,7 +37,7 @@ module SmartBrain
           canonical = (entity[:canonical] || entity[:name]).to_s.downcase
           should_write = entity[:remember] || entity_structure_signal?(entity) || entity_frequencies[canonical] >= freq_threshold
           if should_write
-            items << build_item(type: 'entities', key: key, value_json: entity, source_turn_id: turn[:id], confidence: confidence(:inferred))
+            items << build_item(type: 'entities', key: key, value_json: entity, source_turn_id: turn[:id], confidence: confidence(:inferred), scope_ref: entity[:scope_ref])
             explain << "write entities:#{key}"
           else
             explain << "skip entities:#{key} below threshold"
@@ -40,14 +45,24 @@ module SmartBrain
         end
 
         Array(events[:retractions]).each do |retraction|
-          items << build_item(type: retraction.fetch(:type), key: retraction.fetch(:key), value_json: retraction, source_turn_id: turn[:id], confidence: confidence(:user_asserted), status: 'retracted')
+          items << build_item(type: retraction.fetch(:type), key: retraction.fetch(:key), value_json: retraction, source_turn_id: turn[:id], confidence: confidence(:user_asserted), status: 'retracted', scope_ref: retraction[:scope_ref])
           explain << "retract #{retraction.fetch(:type)}:#{retraction.fetch(:key)}"
         end
 
+        # KG edges: pass-through (subject/predicate/object). Orthogonal to
+        # memory_items — runtime persists them to kg_edges via KnowledgeGraph.
+        edges = Array(events[:edges]).map do |edge|
+          next unless edge[:subject] && edge[:predicate] && edge[:object]
+
+          { subject: edge[:subject], predicate: edge[:predicate],
+            object: edge[:object], confidence: edge[:confidence], scope_ref: edge[:scope_ref] }
+        end.compact
+
         {
           session_id: session_id,
           items: items,
-          explain: explain
+          explain: explain,
+          edges: edges
         }
       end
 
@@ -58,12 +73,12 @@ module SmartBrain
       def collect(items, explain, raw_items, type:, confidence:, source_turn_id:)
         Array(raw_items).each do |entry|
           key = entry.fetch(:key)
-          items << build_item(type: type, key: key, value_json: entry, source_turn_id: source_turn_id, confidence: confidence)
+          items << build_item(type: type, key: key, value_json: entry, source_turn_id: source_turn_id, confidence: confidence, scope_ref: entry[:scope_ref])
           explain << "write #{type}:#{key}"
         end
       end
 
-      def build_item(type:, key:, value_json:, source_turn_id:, confidence:, status: 'active')
+      def build_item(type:, key:, value_json:, source_turn_id:, confidence:, status: 'active', scope_ref: nil)
         {
           type: type,
           key: key,
@@ -71,7 +86,10 @@ module SmartBrain
           source_turn_id: source_turn_id,
           confidence: confidence,
           status: status,
-          updated_at: Time.now.utc.iso8601
+          tier: Governance::Tiers.tier_for(type),
+          lifecycle_status: 'raw',
+          updated_at: Time.now.utc.iso8601,
+          scope_ref: scope_ref
         }
       end