fact_db 0.0.2 → 0.0.3

data/lib/fact_db/transformers/raw_transformer.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module FactDb
+  module Transformers
+    # Returns raw ActiveRecord objects without transformation.
+    #
+    # Use this format when you need direct access to the database objects,
+    # such as when you want to:
+    # - Access ActiveRecord associations (entity_mentions, fact_sources)
+    # - Perform additional database queries on the results
+    # - Use ActiveRecord methods like update, destroy, or reload
+    # - Chain additional scopes or queries
+    #
+    # @example Basic usage
+    #   results = facts.query_facts(topic: "Paula Chen", format: :raw)
+    #   results.each do |fact|
+    #     puts fact.text
+    #     fact.entity_mentions.each { |m| puts m.entity.name }
+    #   end
+    #
+    # @example Chaining queries
+    #   results = facts.query_facts(topic: "Microsoft", format: :raw)
+    #   recent = results.select { |f| f.valid_at > 1.month.ago }
+    #
+    class RawTransformer < Base
+      # Return raw results without transformation.
+      #
+      # @param results [QueryResult] The query results
+      # @return [Array<FactDb::Models::Fact>] Original ActiveRecord Fact objects
+      def transform(results)
+        results.raw_facts
+      end
+    end
+  end
+end

data/lib/fact_db/transformers/text_transformer.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module FactDb
+  module Transformers
+    # Transforms results into human-readable text format.
+    # Useful for direct LLM consumption or debugging.
+    class TextTransformer < Base
+      # Transform results to text format.
+      #
+      # @param results [QueryResult] The query results
+      # @return [String] Human-readable text
+      def transform(results)
+        sections = []
+
+        # Entities section
+        sections << format_entities_section(results) unless results.entities.empty?
+
+        # Facts section
+        sections << format_facts_section(results) unless results.facts.empty?
+
+        if sections.empty?
+          "No results found for query: #{results.query}"
+        else
+          sections.join("\n\n")
+        end
+      end
+
+      private
+
+      def format_entities_section(results)
+        lines = ["## Entities"]
+
+        results.each_entity do |entity|
+          name = get_value(entity, :name)
+          entity_kind = get_value(entity, :kind)
+
+          line = "- **#{name}**"
+          line += " (#{entity_kind})" if entity_kind
+
+          aliases = get_value(entity, :aliases)
+          if aliases && !aliases.empty?
+            alias_texts = aliases.map { |a| a.is_a?(Hash) ? a[:name] : a.to_s }
+            line += " - also known as: #{alias_texts.join(', ')}"
+          end
+
+          lines << line
+        end
+
+        lines.join("\n")
+      end
+
+      def format_facts_section(results)
+        lines = ["## Facts"]
+
+        # Group by status
+        facts_by_status = results.facts.group_by { |f| get_value(f, :status) || "unknown" }
+
+        # Show canonical facts first
+        if facts_by_status["canonical"]
+          lines << "\n### Current Facts"
+          facts_by_status["canonical"].each do |fact|
+            lines << format_fact(fact, results.entities)
+          end
+        end
+
+        # Show corroborated facts
+        if facts_by_status["corroborated"]
+          lines << "\n### Corroborated Facts"
+          facts_by_status["corroborated"].each do |fact|
+            lines << format_fact(fact, results.entities)
+          end
+        end
+
+        # Show superseded facts (historical)
+        if facts_by_status["superseded"]
+          lines << "\n### Historical Facts (Superseded)"
+          facts_by_status["superseded"].each do |fact|
+            lines << format_fact(fact, results.entities)
+          end
+        end
+
+        # Show synthesized facts
+        if facts_by_status["synthesized"]
+          lines << "\n### Synthesized Facts"
+          facts_by_status["synthesized"].each do |fact|
+            lines << format_fact(fact, results.entities)
+          end
+        end
+
+        lines.join("\n")
+      end
+
+      def format_fact(fact, _entities)
+        text = get_value(fact, :text)
+        valid_at = get_value(fact, :valid_at)
+        invalid_at = get_value(fact, :invalid_at)
+        confidence = get_value(fact, :confidence)
+
+        line = "- #{text}"
+
+        # Add temporal info
+        temporal = []
+        temporal << "from #{format_date(valid_at)}" if valid_at
+        temporal << "until #{format_date(invalid_at)}" if invalid_at
+        line += " (#{temporal.join(' ')})" unless temporal.empty?
+
+        # Add confidence
+        line += " [confidence: #{(confidence * 100).round}%]" if confidence
+
+        line
+      end
+    end
+  end
+end

data/lib/fact_db/transformers/triple_transformer.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module FactDb
+  module Transformers
+    # Transforms results into Subject-Predicate-Object triples.
+    # This format encodes semantic structure that LLMs can leverage.
+    #
+    # @example Output format
+    #   [
+    #     ["Paula Chen", "type", "Person"],
+    #     ["Paula Chen", "works_at", "Microsoft"],
+    #     ["Paula Chen", "works_at.valid_from", "2024-01-10"]
+    #   ]
+    class TripleTransformer < Base
+      # Transform results to triples format.
+      #
+      # @param results [QueryResult] The query results
+      # @return [Array<Array>] Array of [subject, predicate, object] triples
+      def transform(results)
+        triples = []
+
+        # Transform entities
+        results.each_entity do |entity|
+          triples += entity_to_triples(entity)
+        end
+
+        # Transform facts
+        results.each_fact do |fact|
+          triples += fact_to_triples(fact, results.entities)
+        end
+
+        triples
+      end
+
+      private
+
+      def entity_to_triples(entity)
+        triples = []
+        name = get_value(entity, :name)
+        return triples unless name
+
+        # Kind triple
+        entity_kind = get_value(entity, :kind)
+        triples << [name, "kind", entity_kind.to_s.capitalize] if entity_kind
+
+        # Aliases
+        aliases = get_value(entity, :aliases) || []
+        aliases.each do |aka|
+          alias_name = aka.is_a?(Hash) ? aka[:name] : aka.to_s
+          triples << [name, "also_known_as", alias_name]
+        end
+
+        # Resolution status
+        status = get_value(entity, :resolution_status)
+        triples << [name, "resolution_status", status] if status
+
+        triples
+      end
+
+      def fact_to_triples(fact, entities)
+        triples = []
+
+        text = get_value(fact, :text)
+        return triples unless text
+
+        # Try to extract subject from entity mentions
+        mentions = get_value(fact, :entity_mentions) || []
+        subject_mention = mentions.find { |m| get_value(m, :mention_role) == "subject" }
+
+        if subject_mention
+          entity_id = get_value(subject_mention, :entity_id)
+          entity = entities[entity_id]
+          subject = entity ? get_value(entity, :name) : "Entity_#{entity_id}"
+        else
+          subject = extract_subject(text)
+        end
+
+        # Main fact assertion
+        predicate, object = extract_predicate_object(text, subject)
+        triples << [subject, predicate, object]
+
+        # Temporal metadata
+        valid_at = get_value(fact, :valid_at)
+        triples << [subject, "#{predicate}.valid_from", format_date(valid_at)] if valid_at
+
+        invalid_at = get_value(fact, :invalid_at)
+        triples << [subject, "#{predicate}.valid_until", format_date(invalid_at)] if invalid_at
+
+        # Status
+        status = get_value(fact, :status)
+        triples << [subject, "#{predicate}.status", status] if status
+
+        # Confidence
+        confidence = get_value(fact, :confidence)
+        triples << [subject, "#{predicate}.confidence", confidence.to_s] if confidence
+
+        # Add other entity mentions as relationships
+        mentions.each do |mention|
+          role = get_value(mention, :mention_role)
+          next if role == "subject"
+
+          entity_id = get_value(mention, :entity_id)
+          entity = entities[entity_id]
+          entity_name = entity ? get_value(entity, :name) : "Entity_#{entity_id}"
+
+          triples << [subject, role, entity_name]
+        end
+
+        triples
+      end
+
+      def extract_subject(text)
+        words = text.split(/\s+/)
+        words.take_while { |w| !w.match?(/^(is|are|was|were|has|have|works|worked)$/i) }.join(" ")
+      end
+
+      def extract_predicate_object(text, subject)
+        remainder = text.sub(/^#{Regexp.escape(subject)}\s*/i, "")
+
+        if (match = remainder.match(/^(is|are|was|were|has|have|works?|worked)\s+(.+)/i))
+          verb = match[1].downcase
+          object = match[2]
+
+          predicate = case verb
+                      when "is", "are", "was", "were" then "is"
+                      when "has", "have" then "has"
+                      when "works", "worked" then "works_at"
+                      else verb
+                      end
+
+          [predicate, object]
+        else
+          ["asserts", remainder]
+        end
+      end
+    end
+  end
+end

data/lib/fact_db/validation/alias_filter.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module FactDb
+  module Validation
+    # Filters out invalid aliases such as pronouns, common terms, and generic references.
+    # Used by extractors, services, and models to ensure alias quality.
+    class AliasFilter
+      # English pronouns (subject, object, possessive, reflexive)
+      PRONOUNS = %w[
+        i me my mine myself
+        you your yours yourself yourselves
+        he him his himself
+        she her hers herself
+        it its itself
+        we us our ours ourselves
+        they them their theirs themselves
+        who whom whose
+        this that these those
+        what which
+        one ones
+        all any both each either neither none some
+        another other others
+      ].freeze
+
+      # Common generic terms that shouldn't be aliases
+      GENERIC_TERMS = %w[
+        a an the
+        man woman person people men women
+        boy girl child children
+        husband wife brother sister father mother son daughter
+        king queen prince princess lord lady
+        sir madam mr mrs ms miss dr
+        someone something somewhere anyone anything anywhere
+        everyone everything everywhere nobody nothing nowhere
+        here there
+        today yesterday tomorrow
+        now then
+      ].freeze
+
+      # Common role/title references that are too generic
+      GENERIC_ROLES = %w[
+        the\ man the\ woman the\ person the\ people
+        a\ man a\ woman a\ person
+        this\ man this\ woman this\ person
+        that\ man that\ woman that\ person
+        the\ king the\ queen the\ lord the\ lady
+        the\ brother the\ sister the\ father the\ mother
+        the\ husband the\ wife
+        the\ boy the\ girl the\ child
+        believers disciples apostles
+        men greek\ men
+      ].freeze
+
+      # Common first names that are too ambiguous to use as standalone aliases
+      # These should only be valid when part of a fuller name
+      AMBIGUOUS_FIRST_NAMES = %w[
+        simon peter john james paul mark matthew luke andrew philip
+        thomas james joseph mary martha elizabeth sarah anna david
+        michael robert william richard henry george charles edward
+        mary ann jane elizabeth margaret catherine alice
+      ].freeze
+
+      class << self
+        # Check if a potential alias is valid
+        # @param text [String] The alias text to validate
+        # @param name [String, nil] The entity's name (for comparison)
+        # @return [Boolean] true if the alias is valid
+        def valid?(text, name: nil)
+          return false if text.nil?
+
+          normalized = text.to_s.strip.downcase
+
+          return false if normalized.empty?
+          return false if too_short?(normalized)
+          return false if pronoun?(normalized)
+          return false if generic_term?(normalized)
+          return false if generic_role?(normalized)
+          return false if matches_canonical?(normalized, name)
+          return false if only_articles_and_generic?(normalized)
+          return false if ambiguous_standalone_name?(normalized, name)
+
+          true
+        end
+
+        # Filter an array of aliases, returning only valid ones
+        # @param aliases [Array<String>] Array of potential aliases
+        # @param name [String, nil] The entity's name
+        # @return [Array<String>] Array of valid aliases
+        def filter(aliases, name: nil)
+          return [] unless aliases.is_a?(Array)
+
+          aliases
+            .map { |a| a.to_s.strip }
+            .reject { |a| a.empty? }
+            .select { |a| valid?(a, name: name) }
+            .uniq { |a| a.downcase }
+        end
+
+        # Get a human-readable reason why an alias was rejected
+        # @param text [String] The alias text
+        # @param name [String, nil] The entity's name
+        # @return [String, nil] Rejection reason or nil if valid
+        def rejection_reason(text, name: nil)
+          return "empty or nil" if text.nil? || text.to_s.strip.empty?
+
+          normalized = text.to_s.strip.downcase
+
+          return "too short (less than 2 characters)" if too_short?(normalized)
+          return "is a pronoun" if pronoun?(normalized)
+          return "is a generic term" if generic_term?(normalized)
+          return "is a generic role reference" if generic_role?(normalized)
+          return "contains only articles and generic words" if only_articles_and_generic?(normalized)
+          return "is an ambiguous standalone first name" if ambiguous_standalone_name?(normalized, name)
+
+          nil
+        end
+
+        private
+
+        def too_short?(text)
+          # Single characters are almost never valid aliases
+          # Exception: single uppercase letters could be initials
+          text.length < 2
+        end
+
+        def pronoun?(text)
+          PRONOUNS.include?(text)
+        end
+
+        def generic_term?(text)
+          GENERIC_TERMS.include?(text)
+        end
+
+        def generic_role?(text)
+          GENERIC_ROLES.include?(text)
+        end
+
+        def matches_canonical?(text, canonical_name)
+          return false if canonical_name.nil?
+
+          text == canonical_name.to_s.strip.downcase
+        end
+
+        def only_articles_and_generic?(text)
+          words = text.split(/\s+/)
+          return false if words.empty?
+
+          # Check if all words are articles or generic terms
+          filler_words = %w[a an the this that these those of and or]
+          non_filler = words.reject { |w| filler_words.include?(w) || GENERIC_TERMS.include?(w) }
+
+          non_filler.empty? || non_filler.all? { |w| PRONOUNS.include?(w) }
+        end
+
+        # Check if text is a standalone ambiguous first name
+        # Single common first names are too likely to cause entity confusion
+        # But "Simon Peter" or "John Mark" would be acceptable
+        def ambiguous_standalone_name?(text, canonical_name)
+          return false if text.nil?
+
+          words = text.split(/\s+/)
+
+          # Only reject if it's a single word that's a common first name
+          return false unless words.length == 1
+
+          # Check if it's in our list of ambiguous first names
+          return false unless AMBIGUOUS_FIRST_NAMES.include?(text)
+
+          # Allow if the canonical name is essentially the same
+          # (e.g., "Peter" as alias for "Peter" entity)
+          return false if canonical_name && canonical_name.to_s.strip.downcase == text
+
+          # Allow if the first name matches the first word of canonical name
+          # (e.g., "Simon" for "Simon Peter" is ok, but "Simon" for "Jesus" is not)
+          if canonical_name
+            canonical_first = canonical_name.to_s.strip.downcase.split(/\s+/).first
+            return false if canonical_first == text
+          end
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fact_db/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FactDb
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end