fact_db 0.0.1

data/lib/fact_db/resolution/entity_resolver.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+module FactDb
+  module Resolution
+    class EntityResolver
+      attr_reader :config
+
+      def initialize(config = FactDb.config)
+        @config = config
+        @threshold = config.fuzzy_match_threshold
+        @auto_merge_threshold = config.auto_merge_threshold
+      end
+
+      # Resolve a name to an entity
+      def resolve(name, type: nil)
+        return nil if name.nil? || name.empty?
+
+        # 1. Exact alias match
+        exact = find_by_exact_alias(name, type: type)
+        return ResolvedEntity.new(exact, confidence: 1.0, match_type: :exact_alias) if exact
+
+        # 2. Canonical name match
+        canonical = find_by_canonical_name(name, type: type)
+        return ResolvedEntity.new(canonical, confidence: 1.0, match_type: :canonical_name) if canonical
+
+        # 3. Fuzzy matching
+        fuzzy = find_by_fuzzy_match(name, type: type)
+        return fuzzy if fuzzy && fuzzy.confidence >= @threshold
+
+        # 4. No match found
+        nil
+      end
+
+      # Resolve or create an entity
+      def resolve_or_create(name, type:, aliases: [], attributes: {})
+        resolved = resolve(name, type: type)
+        return resolved.entity if resolved
+
+        create_entity(name, type: type, aliases: aliases, attributes: attributes)
+      end
+
+      # Merge two entities, keeping one as canonical
+      def merge(keep_id, merge_id)
+        keep = Models::Entity.find(keep_id)
+        merge_entity = Models::Entity.find(merge_id)
+
+        raise ResolutionError, "Cannot merge entity into itself" if keep_id == merge_id
+        raise ResolutionError, "Cannot merge already merged entity" if merge_entity.merged?
+
+        Models::Entity.transaction do
+          # Move all aliases to kept entity
+          merge_entity.aliases.each do |alias_record|
+            keep.aliases.find_or_create_by!(alias_text: alias_record.alias_text) do |a|
+              a.alias_type = alias_record.alias_type
+              a.confidence = alias_record.confidence
+            end
+          end
+
+          # Add the merged entity's canonical name as an alias
+          keep.aliases.find_or_create_by!(alias_text: merge_entity.canonical_name) do |a|
+            a.alias_type = "name"
+            a.confidence = 1.0
+          end
+
+          # Update all entity mentions to point to kept entity
+          Models::EntityMention.where(entity_id: merge_id).update_all(entity_id: keep_id)
+
+          # Mark merged entity
+          merge_entity.update!(
+            resolution_status: "merged",
+            merged_into_id: keep_id
+          )
+        end
+
+        keep.reload
+      end
+
+      # Split an entity into multiple entities
+      def split(entity_id, split_configs)
+        original = Models::Entity.find(entity_id)
+
+        Models::Entity.transaction do
+          new_entities = split_configs.map do |config|
+            create_entity(
+              config[:name],
+              type: config[:type] || original.entity_type,
+              aliases: config[:aliases] || [],
+              attributes: config[:attributes] || {}
+            )
+          end
+
+          original.update!(resolution_status: "split")
+
+          new_entities
+        end
+      end
+
+      # Find potential duplicate entities
+      def find_duplicates(threshold: nil)
+        threshold ||= @threshold
+        duplicates = []
+
+        entities = Models::Entity.resolved.to_a
+
+        entities.each_with_index do |entity, i|
+          entities[(i + 1)..].each do |other|
+            similarity = calculate_similarity(entity.canonical_name, other.canonical_name)
+            if similarity >= threshold
+              duplicates << {
+                entity1: entity,
+                entity2: other,
+                similarity: similarity
+              }
+            end
+          end
+        end
+
+        duplicates.sort_by { |d| -d[:similarity] }
+      end
+
+      # Auto-merge high-confidence duplicates
+      def auto_merge_duplicates!
+        duplicates = find_duplicates(threshold: @auto_merge_threshold)
+
+        duplicates.each do |dup|
+          next if dup[:entity1].merged? || dup[:entity2].merged?
+
+          # Keep the entity with more mentions
+          keep, merge_entity = if dup[:entity1].entity_mentions.count >= dup[:entity2].entity_mentions.count
+                                 [dup[:entity1], dup[:entity2]]
+                               else
+                                 [dup[:entity2], dup[:entity1]]
+                               end
+
+          merge(keep.id, merge_entity.id)
+        end
+      end
+
+      private
+
+      def find_by_exact_alias(name, type:)
+        scope = Models::EntityAlias.where(["LOWER(alias_text) = ?", name.downcase])
+        scope = scope.joins(:entity).where(fact_db_entities: { entity_type: type }) if type
+        scope = scope.joins(:entity).where.not(fact_db_entities: { resolution_status: "merged" })
+        scope.first&.entity
+      end
+
+      def find_by_canonical_name(name, type:)
+        scope = Models::Entity.where(["LOWER(canonical_name) = ?", name.downcase])
+        scope = scope.where(entity_type: type) if type
+        scope.not_merged.first
+      end
+
+      def find_by_fuzzy_match(name, type:)
+        candidates = Models::Entity.not_merged
+        candidates = candidates.where(entity_type: type) if type
+
+        best_match = nil
+        best_similarity = 0
+
+        candidates.find_each do |entity|
+          # Check canonical name
+          similarity = calculate_similarity(name, entity.canonical_name)
+          if similarity > best_similarity
+            best_similarity = similarity
+            best_match = entity
+          end
+
+          # Check aliases
+          entity.aliases.each do |alias_record|
+            alias_similarity = calculate_similarity(name, alias_record.alias_text)
+            if alias_similarity > best_similarity
+              best_similarity = alias_similarity
+              best_match = entity
+            end
+          end
+        end
+
+        return nil if best_match.nil? || best_similarity < @threshold
+
+        ResolvedEntity.new(best_match, confidence: best_similarity, match_type: :fuzzy)
+      end
+
+      def create_entity(name, type:, aliases: [], attributes: {})
+        entity = Models::Entity.create!(
+          canonical_name: name,
+          entity_type: type,
+          attributes: attributes,
+          resolution_status: "resolved"
+        )
+
+        aliases.each do |alias_text|
+          entity.add_alias(alias_text)
+        end
+
+        entity
+      end
+
+      def calculate_similarity(a, b)
+        return 1.0 if a.downcase == b.downcase
+
+        max_len = [a.length, b.length].max
+        return 1.0 if max_len.zero?
+
+        1.0 - (levenshtein_distance(a.downcase, b.downcase).to_f / max_len)
+      end
+
+      def levenshtein_distance(a, b)
+        m = a.length
+        n = b.length
+        d = Array.new(m + 1) { |i| i }
+
+        (1..n).each do |j|
+          prev = d[0]
+          d[0] = j
+          (1..m).each do |i|
+            temp = d[i]
+            d[i] = if a[i - 1] == b[j - 1]
+                     prev
+                   else
+                     [prev + 1, d[i] + 1, d[i - 1] + 1].min
+                   end
+            prev = temp
+          end
+        end
+
+        d[m]
+      end
+    end
+
+    class ResolvedEntity
+      attr_reader :entity, :confidence, :match_type
+
+      def initialize(entity, confidence:, match_type:)
+        @entity = entity
+        @confidence = confidence
+        @match_type = match_type
+      end
+
+      def exact_match?
+        confidence == 1.0
+      end
+
+      def fuzzy_match?
+        match_type == :fuzzy
+      end
+
+      def id
+        entity.id
+      end
+
+      def canonical_name
+        entity.canonical_name
+      end
+
+      def entity_type
+        entity.entity_type
+      end
+    end
+  end
+end

data/lib/fact_db/resolution/fact_resolver.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+module FactDb
+  module Resolution
+    class FactResolver
+      attr_reader :config
+
+      def initialize(config = FactDb.config)
+        @config = config
+      end
+
+      # Supersede an existing fact with a new one
+      def supersede(old_fact_id, new_fact_text, valid_at:, mentions: [])
+        old_fact = Models::Fact.find(old_fact_id)
+
+        raise ResolutionError, "Cannot supersede already superseded fact" if old_fact.superseded?
+
+        Models::Fact.transaction do
+          new_fact = Models::Fact.create!(
+            fact_text: new_fact_text,
+            valid_at: valid_at,
+            status: "canonical",
+            extraction_method: old_fact.extraction_method,
+            confidence: old_fact.confidence
+          )
+
+          # Copy mentions from old fact if not provided
+          if mentions.empty?
+            old_fact.entity_mentions.each do |mention|
+              new_fact.add_mention(
+                entity: mention.entity,
+                text: mention.mention_text,
+                role: mention.mention_role,
+                confidence: mention.confidence
+              )
+            end
+          else
+            mentions.each do |mention|
+              entity = mention[:entity] || Models::Entity.find(mention[:entity_id])
+              new_fact.add_mention(
+                entity: entity,
+                text: mention[:text],
+                role: mention[:role],
+                confidence: mention[:confidence] || 1.0
+              )
+            end
+          end
+
+          # Copy sources from old fact
+          old_fact.fact_sources.each do |source|
+            new_fact.add_source(
+              content: source.content,
+              type: source.source_type,
+              excerpt: source.excerpt,
+              confidence: source.confidence
+            )
+          end
+
+          # Mark old fact as superseded
+          old_fact.update!(
+            status: "superseded",
+            superseded_by_id: new_fact.id,
+            invalid_at: valid_at
+          )
+
+          new_fact
+        end
+      end
+
+      # Synthesize a new fact from multiple source facts
+      def synthesize(source_fact_ids, synthesized_text, valid_at:, invalid_at: nil, mentions: [])
+        source_facts = Models::Fact.where(id: source_fact_ids)
+
+        raise ResolutionError, "No source facts found" if source_facts.empty?
+
+        Models::Fact.transaction do
+          synthesized = Models::Fact.create!(
+            fact_text: synthesized_text,
+            valid_at: valid_at,
+            invalid_at: invalid_at,
+            status: "synthesized",
+            derived_from_ids: source_fact_ids,
+            extraction_method: "synthesized",
+            confidence: calculate_synthesized_confidence(source_facts)
+          )
+
+          # Aggregate entity mentions from source facts if not provided
+          if mentions.empty?
+            aggregate_mentions(source_facts).each do |mention|
+              synthesized.add_mention(**mention)
+            end
+          else
+            mentions.each do |mention|
+              entity = mention[:entity] || Models::Entity.find(mention[:entity_id])
+              synthesized.add_mention(
+                entity: entity,
+                text: mention[:text],
+                role: mention[:role],
+                confidence: mention[:confidence] || 1.0
+              )
+            end
+          end
+
+          # Link all source content
+          source_facts.each do |source_fact|
+            source_fact.fact_sources.each do |source|
+              synthesized.add_source(
+                content: source.content,
+                type: "supporting",
+                excerpt: source.excerpt,
+                confidence: source.confidence
+              )
+            end
+          end
+
+          synthesized
+        end
+      end
+
+      # Mark a fact as corroborated by another fact
+      def corroborate(fact_id, corroborating_fact_id)
+        fact = Models::Fact.find(fact_id)
+        _corroborating = Models::Fact.find(corroborating_fact_id)
+
+        raise ResolutionError, "Cannot corroborate with same fact" if fact_id == corroborating_fact_id
+
+        fact.update!(
+          corroborated_by_ids: (fact.corroborated_by_ids + [corroborating_fact_id]).uniq
+        )
+
+        # Optionally update status to corroborated if it was just canonical
+        fact.update!(status: "corroborated") if fact.status == "canonical" && fact.corroborated_by_ids.size >= 2
+
+        fact
+      end
+
+      # Invalidate a fact without replacement
+      def invalidate(fact_id, at: Time.current)
+        fact = Models::Fact.find(fact_id)
+        fact.update!(invalid_at: at)
+        fact
+      end
+
+      # Find potentially conflicting facts
+      def find_conflicts(entity_id: nil, topic: nil)
+        scope = Models::Fact.canonical.currently_valid
+
+        if entity_id
+          scope = scope.mentioning_entity(entity_id)
+        end
+
+        if topic
+          scope = scope.search_text(topic)
+        end
+
+        # Group facts that might be about the same thing
+        facts = scope.to_a
+        conflicts = []
+
+        facts.each_with_index do |fact, i|
+          facts[(i + 1)..].each do |other|
+            similarity = text_similarity(fact.fact_text, other.fact_text)
+            if similarity > 0.5 && similarity < 0.95
+              conflicts << {
+                fact1: fact,
+                fact2: other,
+                similarity: similarity
+              }
+            end
+          end
+        end
+
+        conflicts.sort_by { |c| -c[:similarity] }
+      end
+
+      # Resolve conflicts by keeping one fact and superseding others
+      def resolve_conflict(keep_fact_id, supersede_fact_ids, reason: nil)
+        Models::Fact.transaction do
+          supersede_fact_ids.each do |fact_id|
+            fact = Models::Fact.find(fact_id)
+            fact.update!(
+              status: "superseded",
+              superseded_by_id: keep_fact_id,
+              invalid_at: Time.current,
+              metadata: fact.metadata.merge(supersede_reason: reason)
+            )
+          end
+        end
+
+        Models::Fact.find(keep_fact_id)
+      end
+
+      # Build a timeline fact from point-in-time facts
+      def build_timeline_fact(entity_id:, topic: nil)
+        facts = Models::Fact.mentioning_entity(entity_id)
+        facts = facts.search_text(topic) if topic
+        facts = facts.order(valid_at: :asc).to_a
+
+        return nil if facts.empty?
+
+        # Find start and end dates
+        start_date = facts.first.valid_at
+        end_date = facts.select { |f| f.invalid_at }.map(&:invalid_at).max
+
+        entity = Models::Entity.find(entity_id)
+        synthesized_text = "#{entity.canonical_name}: #{topic || 'timeline'} from #{start_date.to_date}"
+        synthesized_text += " to #{end_date.to_date}" if end_date
+
+        synthesize(
+          facts.map(&:id),
+          synthesized_text,
+          valid_at: start_date,
+          invalid_at: end_date
+        )
+      end
+
+      private
+
+      def calculate_synthesized_confidence(source_facts)
+        confidences = source_facts.map(&:confidence)
+        confidences.sum / confidences.size
+      end
+
+      def aggregate_mentions(source_facts)
+        mentions = {}
+
+        source_facts.each do |fact|
+          fact.entity_mentions.each do |mention|
+            key = [mention.entity_id, mention.mention_role]
+            existing = mentions[key]
+
+            if existing.nil? || mention.confidence > existing[:confidence]
+              mentions[key] = {
+                entity: mention.entity,
+                text: mention.mention_text,
+                role: mention.mention_role,
+                confidence: mention.confidence
+              }
+            end
+          end
+        end
+
+        mentions.values
+      end
+
+      def text_similarity(text1, text2)
+        words1 = text1.downcase.split
+        words2 = text2.downcase.split
+
+        return 0.0 if words1.empty? || words2.empty?
+
+        intersection = words1 & words2
+        union = words1 | words2
+
+        intersection.size.to_f / union.size
+      end
+    end
+  end
+end

data/lib/fact_db/services/content_service.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module FactDb
+  module Services
+    class ContentService
+      attr_reader :config
+
+      def initialize(config = FactDb.config)
+        @config = config
+      end
+
+      def create(raw_text, type:, captured_at: Time.current, metadata: {}, title: nil, source_uri: nil)
+        content_hash = Digest::SHA256.hexdigest(raw_text)
+
+        # Check for duplicate content
+        existing = Models::Content.find_by(content_hash: content_hash)
+        return existing if existing
+
+        embedding = generate_embedding(raw_text)
+
+        Models::Content.create!(
+          raw_text: raw_text,
+          content_hash: content_hash,
+          content_type: type.to_s,
+          title: title,
+          source_uri: source_uri,
+          source_metadata: metadata,
+          captured_at: captured_at,
+          embedding: embedding
+        )
+      end
+
+      def find(id)
+        Models::Content.find(id)
+      end
+
+      def find_by_hash(hash)
+        Models::Content.find_by(content_hash: hash)
+      end
+
+      def search(query, type: nil, from: nil, to: nil, limit: 20)
+        scope = Models::Content.search_text(query)
+        scope = scope.by_type(type) if type
+        scope = scope.captured_after(from) if from
+        scope = scope.captured_before(to) if to
+        scope.order(captured_at: :desc).limit(limit)
+      end
+
+      def semantic_search(query, limit: 20)
+        embedding = generate_embedding(query)
+        return Models::Content.none unless embedding
+
+        Models::Content.nearest_neighbors(embedding, limit: limit)
+      end
+
+      def by_type(type, limit: nil)
+        scope = Models::Content.by_type(type).order(captured_at: :desc)
+        scope = scope.limit(limit) if limit
+        scope
+      end
+
+      def between(from, to)
+        Models::Content.captured_between(from, to).order(captured_at: :asc)
+      end
+
+      def recent(limit: 10)
+        Models::Content.order(captured_at: :desc).limit(limit)
+      end
+
+      def stats
+        {
+          total: Models::Content.count,
+          total_count: Models::Content.count,
+          by_type: Models::Content.group(:content_type).count,
+          earliest: Models::Content.minimum(:captured_at),
+          latest: Models::Content.maximum(:captured_at),
+          total_words: Models::Content.sum("array_length(regexp_split_to_array(raw_text, '\\s+'), 1)")
+        }
+      end
+
+      private
+
+      def generate_embedding(text)
+        return nil unless config.embedding_generator
+
+        config.embedding_generator.call(text)
+      rescue StandardError => e
+        config.logger&.warn("Failed to generate embedding: #{e.message}")
+        nil
+      end
+    end
+  end
+end