fact_db 0.0.1

data/lib/fact_db/models/fact.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+module FactDb
+  module Models
+    class Fact < ActiveRecord::Base
+      self.table_name = "fact_db_facts"
+
+      has_many :entity_mentions, class_name: "FactDb::Models::EntityMention",
+               foreign_key: :fact_id, dependent: :destroy
+      has_many :entities, through: :entity_mentions
+
+      has_many :fact_sources, class_name: "FactDb::Models::FactSource",
+               foreign_key: :fact_id, dependent: :destroy
+      has_many :source_contents, through: :fact_sources, source: :content
+
+      belongs_to :superseded_by, class_name: "FactDb::Models::Fact",
+                 foreign_key: :superseded_by_id, optional: true
+      has_many :supersedes, class_name: "FactDb::Models::Fact",
+               foreign_key: :superseded_by_id
+
+      validates :fact_text, presence: true
+      validates :fact_hash, presence: true
+      validates :valid_at, presence: true
+      validates :status, presence: true
+
+      before_validation :generate_fact_hash, on: :create
+
+      # Fact statuses
+      STATUSES = %w[canonical superseded corroborated synthesized].freeze
+      EXTRACTION_METHODS = %w[manual llm rule_based].freeze
+
+      validates :status, inclusion: { in: STATUSES }
+      validates :extraction_method, inclusion: { in: EXTRACTION_METHODS }, allow_nil: true
+
+      # Core scopes
+      scope :canonical, -> { where(status: "canonical") }
+      scope :superseded, -> { where(status: "superseded") }
+      scope :synthesized, -> { where(status: "synthesized") }
+
+      # Temporal scopes - the heart of the Event Clock
+      scope :currently_valid, -> { where(invalid_at: nil) }
+      scope :historical, -> { where.not(invalid_at: nil) }
+
+      scope :valid_at, lambda { |date|
+        where("valid_at <= ?", date)
+          .where("invalid_at > ? OR invalid_at IS NULL", date)
+      }
+
+      scope :valid_between, lambda { |from, to|
+        where("valid_at <= ? AND (invalid_at > ? OR invalid_at IS NULL)", to, from)
+      }
+
+      scope :became_valid_between, lambda { |from, to|
+        where(valid_at: from..to)
+      }
+
+      scope :became_invalid_between, lambda { |from, to|
+        where(invalid_at: from..to)
+      }
+
+      # Entity filtering
+      scope :mentioning_entity, lambda { |entity_id|
+        joins(:entity_mentions).where(fact_db_entity_mentions: { entity_id: entity_id })
+      }
+
+      scope :with_role, lambda { |entity_id, role|
+        joins(:entity_mentions).where(
+          fact_db_entity_mentions: { entity_id: entity_id, mention_role: role }
+        )
+      }
+
+      # Full-text search
+      scope :search_text, lambda { |query|
+        where("to_tsvector('english', fact_text) @@ plainto_tsquery('english', ?)", query)
+      }
+
+      # Extraction method
+      scope :extracted_by, ->(method) { where(extraction_method: method) }
+      scope :by_extraction_method, ->(method) { where(extraction_method: method) }
+
+      # Confidence filtering
+      scope :high_confidence, -> { where("confidence >= ?", 0.9) }
+      scope :low_confidence, -> { where("confidence < ?", 0.5) }
+
+      def currently_valid?
+        invalid_at.nil?
+      end
+
+      def valid_at?(date)
+        valid_at <= date && (invalid_at.nil? || invalid_at > date)
+      end
+
+      def duration
+        return nil if invalid_at.nil?
+
+        invalid_at - valid_at
+      end
+
+      def duration_days
+        return nil if invalid_at.nil?
+
+        (invalid_at.to_date - valid_at.to_date).to_i
+      end
+
+      def superseded?
+        status == "superseded"
+      end
+
+      def synthesized?
+        status == "synthesized"
+      end
+
+      def invalidate!(at: Time.current)
+        update!(invalid_at: at)
+      end
+
+      def supersede_with!(new_fact_text, valid_at:)
+        transaction do
+          new_fact = self.class.create!(
+            fact_text: new_fact_text,
+            valid_at: valid_at,
+            status: "canonical",
+            extraction_method: extraction_method
+          )
+
+          update!(
+            status: "superseded",
+            superseded_by_id: new_fact.id,
+            invalid_at: valid_at
+          )
+
+          new_fact
+        end
+      end
+
+      def add_mention(entity:, text:, role: nil, confidence: 1.0)
+        entity_mentions.find_or_create_by!(entity: entity, mention_text: text) do |m|
+          m.mention_role = role
+          m.confidence = confidence
+        end
+      end
+
+      def add_source(content:, type: "primary", excerpt: nil, confidence: 1.0)
+        fact_sources.find_or_create_by!(content: content) do |s|
+          s.source_type = type
+          s.excerpt = excerpt
+          s.confidence = confidence
+        end
+      end
+
+      # Get source facts for synthesized facts
+      def source_facts
+        return Fact.none unless derived_from_ids.any?
+
+        Fact.where(id: derived_from_ids)
+      end
+
+      # Get facts that corroborate this one
+      def corroborating_facts
+        return Fact.none unless corroborated_by_ids.any?
+
+        Fact.where(id: corroborated_by_ids)
+      end
+
+      # Evidence chain - trace back to original content
+      def evidence_chain
+        sources = source_contents.to_a
+
+        if synthesized? && derived_from_ids.any?
+          source_facts.each do |source_fact|
+            sources.concat(source_fact.evidence_chain)
+          end
+        end
+
+        sources.uniq
+      end
+
+      # Vector similarity search
+      def self.nearest_neighbors(embedding, limit: 10)
+        return none unless embedding
+
+        order(Arel.sql("embedding <=> '#{embedding}'")).limit(limit)
+      end
+
+      private
+
+      def generate_fact_hash
+        self.fact_hash = Digest::SHA256.hexdigest(fact_text) if fact_text.present?
+      end
+    end
+  end
+end

data/lib/fact_db/models/fact_source.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module FactDb
+  module Models
+    class FactSource < ActiveRecord::Base
+      self.table_name = "fact_db_fact_sources"
+
+      belongs_to :fact, class_name: "FactDb::Models::Fact"
+      belongs_to :content, class_name: "FactDb::Models::Content"
+
+      validates :fact_id, uniqueness: { scope: :content_id }
+
+      # Source types
+      TYPES = %w[primary supporting corroborating].freeze
+
+      validates :source_type, inclusion: { in: TYPES }
+
+      scope :primary, -> { where(source_type: "primary") }
+      scope :supporting, -> { where(source_type: "supporting") }
+      scope :corroborating, -> { where(source_type: "corroborating") }
+      scope :high_confidence, -> { where("confidence >= ?", 0.9) }
+
+      def primary?
+        source_type == "primary"
+      end
+
+      def excerpt_preview(length: 100)
+        return nil if excerpt.nil?
+        return excerpt if excerpt.length <= length
+
+        "#{excerpt[0, length]}..."
+      end
+    end
+  end
+end

data/lib/fact_db/pipeline/extraction_pipeline.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "simple_flow"
+
+module FactDb
+  module Pipeline
+    # Pipeline for extracting facts from content using SimpleFlow
+    # Supports parallel processing of multiple content items
+    #
+    # @example Sequential extraction
+    #   pipeline = ExtractionPipeline.new(config)
+    #   results = pipeline.process([content1, content2], extractor: :llm)
+    #
+    # @example Parallel extraction
+    #   pipeline = ExtractionPipeline.new(config)
+    #   results = pipeline.process_parallel([content1, content2, content3], extractor: :llm)
+    #
+    class ExtractionPipeline
+      attr_reader :config
+
+      def initialize(config = FactDb.config)
+        @config = config
+      end
+
+      # Process multiple content items sequentially
+      #
+      # @param contents [Array<Models::Content>] Content records to process
+      # @param extractor [Symbol] Extractor type (:manual, :llm, :rule_based)
+      # @return [Array<Hash>] Results with extracted facts per content
+      def process(contents, extractor: config.default_extractor)
+        pipeline = build_extraction_pipeline(extractor)
+
+        contents.map do |content|
+          result = pipeline.call(SimpleFlow::Result.new(content))
+          {
+            content_id: content.id,
+            facts: result.success? ? result.value : [],
+            error: result.halted? ? result.error : nil
+          }
+        end
+      end
+
+      # Process multiple content items in parallel
+      # Uses SimpleFlow's parallel execution capabilities
+      #
+      # @param contents [Array<Models::Content>] Content records to process
+      # @param extractor [Symbol] Extractor type (:manual, :llm, :rule_based)
+      # @return [Array<Hash>] Results with extracted facts per content
+      def process_parallel(contents, extractor: config.default_extractor)
+        pipeline = build_parallel_pipeline(contents, extractor)
+        initial_result = SimpleFlow::Result.new(contents: contents, results: {})
+
+        final_result = pipeline.call(initial_result)
+
+        contents.map do |content|
+          result = final_result.value[:results][content.id]
+          {
+            content_id: content.id,
+            facts: result&.dig(:facts) || [],
+            error: result&.dig(:error)
+          }
+        end
+      end
+
+      private
+
+      def build_extraction_pipeline(extractor)
+        extractor_instance = get_extractor(extractor)
+
+        SimpleFlow::Pipeline.new do
+          # Step 1: Validate content
+          step ->(result) {
+            content = result.value
+            if content.nil? || content.raw_text.blank?
+              result.halt("Content is empty or missing")
+            else
+              result.continue(content)
+            end
+          }
+
+          # Step 2: Extract facts
+          step ->(result) {
+            content = result.value
+            begin
+              facts = extractor_instance.extract(content)
+              result.continue(facts)
+            rescue StandardError => e
+              result.halt("Extraction failed: #{e.message}")
+            end
+          }
+
+          # Step 3: Validate extracted facts
+          step ->(result) {
+            facts = result.value
+            valid_facts = facts.select { |f| f.valid? }
+            result.continue(valid_facts)
+          }
+        end
+      end
+
+      def build_parallel_pipeline(contents, extractor)
+        extractor_instance = get_extractor(extractor)
+
+        SimpleFlow::Pipeline.new do
+          # Create a step for each content item
+          contents.each do |content|
+            step "extract_#{content.id}", depends_on: [] do |result|
+              begin
+                facts = extractor_instance.extract(content)
+                valid_facts = facts.select { |f| f.valid? }
+
+                new_results = result.value[:results].merge(
+                  content.id => { facts: valid_facts, error: nil }
+                )
+                result.continue(result.value.merge(results: new_results))
+              rescue StandardError => e
+                new_results = result.value[:results].merge(
+                  content.id => { facts: [], error: e.message }
+                )
+                result.continue(result.value.merge(results: new_results))
+              end
+            end
+          end
+
+          # Aggregate results
+          step "aggregate", depends_on: contents.map { |c| "extract_#{c.id}" } do |result|
+            result.continue(result.value)
+          end
+        end
+      end
+
+      def get_extractor(extractor)
+        case extractor.to_sym
+        when :manual
+          Extractors::ManualExtractor.new(config)
+        when :llm
+          Extractors::LLMExtractor.new(config)
+        when :rule_based
+          Extractors::RuleBasedExtractor.new(config)
+        else
+          raise ConfigurationError, "Unknown extractor: #{extractor}"
+        end
+      end
+    end
+  end
+end

data/lib/fact_db/pipeline/resolution_pipeline.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "simple_flow"
+
+module FactDb
+  module Pipeline
+    # Pipeline for resolving entities and facts using SimpleFlow
+    # Supports parallel resolution of multiple items
+    #
+    # @example Resolve entities in parallel
+    #   pipeline = ResolutionPipeline.new(config)
+    #   results = pipeline.resolve_entities(["John Smith", "Jane Doe", "Acme Corp"])
+    #
+    class ResolutionPipeline
+      attr_reader :config, :entity_resolver, :fact_resolver
+
+      def initialize(config = FactDb.config)
+        @config = config
+        @entity_resolver = Resolution::EntityResolver.new(config)
+        @fact_resolver = Resolution::FactResolver.new(config)
+      end
+
+      # Resolve multiple entity names in parallel
+      #
+      # @param names [Array<String>] Entity names to resolve
+      # @param type [Symbol, nil] Entity type filter
+      # @return [Array<Hash>] Resolution results
+      def resolve_entities(names, type: nil)
+        pipeline = build_entity_resolution_pipeline(names, type)
+        initial_result = SimpleFlow::Result.new(names: names, resolved: {})
+
+        final_result = pipeline.call(initial_result)
+
+        names.map do |name|
+          resolution = final_result.value[:resolved][name]
+          {
+            name: name,
+            entity: resolution&.dig(:entity),
+            status: resolution&.dig(:status) || :failed,
+            error: resolution&.dig(:error)
+          }
+        end
+      end
+
+      # Find and resolve conflicts for multiple entities in parallel
+      #
+      # @param entity_ids [Array<Integer>] Entity IDs to check for conflicts
+      # @return [Array<Hash>] Conflict detection results
+      def detect_conflicts(entity_ids)
+        pipeline = build_conflict_detection_pipeline(entity_ids)
+        initial_result = SimpleFlow::Result.new(entity_ids: entity_ids, conflicts: {})
+
+        final_result = pipeline.call(initial_result)
+
+        entity_ids.map do |entity_id|
+          conflicts = final_result.value[:conflicts][entity_id]
+          {
+            entity_id: entity_id,
+            conflicts: conflicts || [],
+            conflict_count: conflicts&.size || 0
+          }
+        end
+      end
+
+      private
+
+      def build_entity_resolution_pipeline(names, type)
+        resolver = @entity_resolver
+
+        SimpleFlow::Pipeline.new do
+          # Create parallel resolution steps
+          names.each do |name|
+            step "resolve_#{name.hash.abs}", depends_on: [] do |result|
+              begin
+                entity = resolver.resolve(name, type: type)
+                status = entity ? :resolved : :not_found
+
+                new_resolved = result.value[:resolved].merge(
+                  name => { entity: entity, status: status, error: nil }
+                )
+                result.continue(result.value.merge(resolved: new_resolved))
+              rescue StandardError => e
+                new_resolved = result.value[:resolved].merge(
+                  name => { entity: nil, status: :error, error: e.message }
+                )
+                result.continue(result.value.merge(resolved: new_resolved))
+              end
+            end
+          end
+
+          # Aggregate
+          step "aggregate", depends_on: names.map { |n| "resolve_#{n.hash.abs}" } do |result|
+            result.continue(result.value)
+          end
+        end
+      end
+
+      def build_conflict_detection_pipeline(entity_ids)
+        resolver = @fact_resolver
+
+        SimpleFlow::Pipeline.new do
+          # Create parallel conflict detection steps
+          entity_ids.each do |entity_id|
+            step "conflicts_#{entity_id}", depends_on: [] do |result|
+              begin
+                conflicts = resolver.find_conflicts(entity_id: entity_id)
+
+                new_conflicts = result.value[:conflicts].merge(
+                  entity_id => conflicts
+                )
+                result.continue(result.value.merge(conflicts: new_conflicts))
+              rescue StandardError
+                new_conflicts = result.value[:conflicts].merge(
+                  entity_id => []
+                )
+                result.continue(result.value.merge(conflicts: new_conflicts))
+              end
+            end
+          end
+
+          # Aggregate
+          step "aggregate", depends_on: entity_ids.map { |id| "conflicts_#{id}" } do |result|
+            result.continue(result.value)
+          end
+        end
+      end
+    end
+  end
+end