fact_db 0.0.2 → 0.0.3

data/lib/fact_db.rb

@@ -9,7 +9,7 @@ require_relative "fact_db/config"
 require_relative "fact_db/database"
 
 # Models
-require_relative "fact_db/models/content"
+require_relative "fact_db/models/source"
 require_relative "fact_db/models/entity"
 require_relative "fact_db/models/entity_alias"
 require_relative "fact_db/models/fact"
@@ -19,11 +19,15 @@ require_relative "fact_db/models/fact_source"
 # Temporal queries
 require_relative "fact_db/temporal/query"
 require_relative "fact_db/temporal/timeline"
+require_relative "fact_db/temporal/query_builder"
 
 # Resolution
 require_relative "fact_db/resolution/entity_resolver"
 require_relative "fact_db/resolution/fact_resolver"
 
+# Validation
+require_relative "fact_db/validation/alias_filter"
+
 # Extractors
 require_relative "fact_db/extractors/base"
 require_relative "fact_db/extractors/manual_extractor"
@@ -38,31 +42,49 @@ require_relative "fact_db/pipeline/extraction_pipeline"
 require_relative "fact_db/pipeline/resolution_pipeline"
 
 # Services
-require_relative "fact_db/services/content_service"
+require_relative "fact_db/services/source_service"
 require_relative "fact_db/services/entity_service"
 require_relative "fact_db/services/fact_service"
 
+# Transformers (output formatting)
+require_relative "fact_db/transformers/base"
+require_relative "fact_db/transformers/raw_transformer"
+require_relative "fact_db/transformers/json_transformer"
+require_relative "fact_db/transformers/triple_transformer"
+require_relative "fact_db/transformers/cypher_transformer"
+require_relative "fact_db/transformers/text_transformer"
+
+# Query Result
+require_relative "fact_db/query_result"
+
 module FactDb
   class Facts
-    attr_reader :config, :content_service, :entity_service, :fact_service,
+    # Available output formats for LLM consumption
+    FORMATS = %i[raw json triples cypher text].freeze
+
+    # Available retrieval strategies
+    STRATEGIES = %i[auto semantic fulltext graph temporal hybrid].freeze
+
+    attr_reader :config, :source_service, :entity_service, :fact_service,
                 :extraction_pipeline, :resolution_pipeline
 
     def initialize(config: nil)
       @config = config || FactDb.config
       Database.establish_connection!(@config)
 
-      @content_service = Services::ContentService.new(@config)
+      @source_service = Services::SourceService.new(@config)
       @entity_service = Services::EntityService.new(@config)
       @fact_service = Services::FactService.new(@config)
       @extraction_pipeline = Pipeline::ExtractionPipeline.new(@config)
       @resolution_pipeline = Pipeline::ResolutionPipeline.new(@config)
+      @transformers = build_transformers
     end
 
-    # Ingest raw content into the event clock
-    def ingest(raw_text, type:, captured_at: Time.current, metadata: {}, title: nil, source_uri: nil)
-      @content_service.create(
-        raw_text,
-        type: type,
+    # Ingest raw content
+    def ingest(content, kind:, captured_at: Time.current, metadata: {}, title: nil, source_uri: nil)
+      @source_service.create(
+        content,
+        kind: kind,
         captured_at: captured_at,
         metadata: metadata,
         title: title,
@@ -70,19 +92,27 @@ module FactDb
       )
     end
 
-    # Extract facts from content
-    def extract_facts(content_id, extractor: @config.default_extractor)
-      @fact_service.extract_from_content(content_id, extractor: extractor)
+    # Extract facts from source
+    def extract_facts(source_id, extractor: @config.default_extractor)
+      @fact_service.extract_from_source(source_id, extractor: extractor)
     end
 
     # Query facts with temporal and entity filtering
-    def query_facts(topic: nil, at: nil, entity: nil, status: :canonical)
-      @fact_service.query(topic: topic, at: at, entity: entity, status: status)
+    #
+    # @param topic [String, nil] Topic to search for
+    # @param at [Date, Time, String, nil] Point in time for temporal query
+    # @param entity [Integer, nil] Entity ID to filter by
+    # @param status [Symbol] Fact status (:canonical, :superseded, :synthesized, :all)
+    # @param format [Symbol] Output format (:json, :triples, :cypher, :text, :prolog)
+    # @return [Array, String, Hash] Results in requested format
+    def query_facts(topic: nil, at: nil, entity: nil, status: :canonical, format: :json)
+      results = @fact_service.query(topic: topic, at: at, entity: entity, status: status)
+      transform_results(results, topic: topic, format: format)
     end
 
     # Resolve a name to an entity
-    def resolve_entity(name, type: nil)
-      @entity_service.resolve(name, type: type)
+    def resolve_entity(name, kind: nil)
+      @entity_service.resolve(name, kind: kind)
     end
 
     # Build a timeline for an entity
@@ -91,37 +121,152 @@ module FactDb
     end
 
     # Get currently valid facts about an entity
-    def current_facts_for(entity_id)
-      query_facts(entity: entity_id, at: nil, status: :canonical)
+    #
+    # @param entity_id [Integer] Entity ID
+    # @param format [Symbol] Output format
+    # @return [Array, String, Hash] Results in requested format
+    def current_facts_for(entity_id, format: :json)
+      results = @fact_service.current_facts(entity: entity_id)
+      transform_results(results, topic: "entity_#{entity_id}", format: format)
     end
 
     # Get facts valid at a specific point in time
-    def facts_at(at, entity: nil, topic: nil)
-      query_facts(at: at, entity: entity, topic: topic, status: :canonical)
+    #
+    # @param at [Date, Time, String] Point in time
+    # @param entity [Integer, nil] Entity ID to filter by
+    # @param topic [String, nil] Topic to search for
+    # @param format [Symbol] Output format
+    # @return [Array, String, Hash] Results in requested format
+    def facts_at(at, entity: nil, topic: nil, format: :json)
+      results = @fact_service.facts_at(at, entity: entity, topic: topic)
+      transform_results(results, topic: topic || "facts_at_#{at}", format: format)
+    end
+
+    # Temporal query builder - query at a specific point in time
+    #
+    # @param date [Date, Time, String] Point in time
+    # @return [Temporal::QueryBuilder] Scoped query builder
+    #
+    # @example
+    #   facts.at("2024-01-15").query("Paula's role", format: :cypher)
+    #   facts.at("2024-01-15").facts_for(entity_id)
+    #   facts.at("2024-01-15").compare_to("2024-06-15")
+    def at(date)
+      Temporal::QueryBuilder.new(self, parse_date(date))
+    end
+
+    # Compare what changed between two dates
+    #
+    # @param topic [String, nil] Topic to compare (nil for all facts)
+    # @param from [Date, Time, String] Start date
+    # @param to [Date, Time, String] End date
+    # @return [Hash] Differences with :added, :removed, :unchanged keys
+    def diff(topic = nil, from:, to:)
+      from_date = parse_date(from)
+      to_date = parse_date(to)
+
+      from_results = @fact_service.query(topic: topic, at: from_date, status: :canonical)
+      to_results = @fact_service.query(topic: topic, at: to_date, status: :canonical)
+
+      from_set = facts_to_comparable(from_results)
+      to_set = facts_to_comparable(to_results)
+
+      {
+        topic: topic,
+        from: from_date,
+        to: to_date,
+        added: to_results.select { |f| !from_set.include?(comparable_key(f)) },
+        removed: from_results.select { |f| !to_set.include?(comparable_key(f)) },
+        unchanged: from_results.select { |f| to_set.include?(comparable_key(f)) }
+      }
+    end
+
+    # Introspect the schema - what does the layer know about?
+    #
+    # @param topic [String, nil] Optional topic to introspect specifically
+    # @return [Hash] Schema information or topic-specific coverage
+    def introspect(topic = nil)
+      topic ? introspect_topic(topic) : introspect_schema
+    end
+
+    # Suggest queries based on what's stored for a topic
+    #
+    # @param topic [String] Topic to get suggestions for
+    # @return [Array<String>] Suggested queries
+    def suggest_queries(topic)
+      resolved = resolve_entity(topic)
+      return [] unless resolved
+
+      entity = resolved.respond_to?(:entity) ? resolved.entity : resolved
+      suggestions = []
+
+      entity_kind = entity.respond_to?(:kind) ? entity.kind : nil
+      suggestions << "current status" if entity_kind == "person"
+
+      # Check relationships
+      relationships = @entity_service.relationship_types_for(entity.id)
+      suggestions << "employment history" if relationships.include?(:works_at) || relationships.include?(:object)
+      suggestions << "team members" if relationships.include?(:works_with)
+      suggestions << "reporting chain" if relationships.include?(:reports_to)
+
+      # Check fact coverage
+      fact_stats = @fact_service.fact_stats(entity.id)
+      suggestions << "timeline" if fact_stats[:canonical]&.positive?
+      suggestions << "historical changes" if fact_stats[:superseded]&.positive?
+
+      suggestions
+    end
+
+    # Suggest retrieval strategies for a query
+    #
+    # @param query_text [String] The query
+    # @return [Array<Hash>] Strategy options with descriptions
+    def suggest_strategies(query_text)
+      strategies = []
+
+      # Check for temporal keywords
+      if query_text.match?(/\b(yesterday|last\s+week|last\s+month|ago|since|before|after|between)\b/i)
+        strategies << { strategy: :temporal, description: "Filter by date range" }
+      end
+
+      # Check for semantic intent
+      if query_text.match?(/\b(about|related|similar|like)\b/i)
+        strategies << { strategy: :semantic, description: "Search by semantic similarity" }
+      end
+
+      # Check for entity focus
+      if query_text.match?(/\b(who|what|where)\b/i)
+        strategies << { strategy: :graph, description: "Traverse from entity node" }
+      end
+
+      # Default: hybrid
+      strategies << { strategy: :hybrid, description: "Combine multiple strategies" }
+
+      strategies
     end
 
-    # Batch extract facts from multiple content items
+    # Batch extract facts from multiple sources
     #
-    # @param content_ids [Array<Integer>] Content IDs to process
+    # @param source_ids [Array<Integer>] Source IDs to process
     # @param extractor [Symbol] Extractor type (:manual, :llm, :rule_based)
     # @param parallel [Boolean] Whether to use parallel processing
-    # @return [Array<Hash>] Results with extracted facts per content
-    def batch_extract(content_ids, extractor: @config.default_extractor, parallel: true)
-      contents = Models::Content.where(id: content_ids).to_a
+    # @return [Array<Hash>] Results with extracted facts per source
+    def batch_extract(source_ids, extractor: @config.default_extractor, parallel: true)
+      sources = Models::Source.where(id: source_ids).to_a
       if parallel
-        @extraction_pipeline.process_parallel(contents, extractor: extractor)
+        @extraction_pipeline.process_parallel(sources, extractor: extractor)
       else
-        @extraction_pipeline.process(contents, extractor: extractor)
+        @extraction_pipeline.process(sources, extractor: extractor)
       end
     end
 
     # Batch resolve entity names
     #
     # @param names [Array<String>] Entity names to resolve
-    # @param type [Symbol, nil] Entity type filter
+    # @param kind [Symbol, nil] Entity kind filter
     # @return [Array<Hash>] Resolution results
-    def batch_resolve_entities(names, type: nil)
-      @resolution_pipeline.resolve_entities(names, type: type)
+    def batch_resolve_entities(names, kind: nil)
+      @resolution_pipeline.resolve_entities(names, kind: kind)
     end
 
     # Detect fact conflicts for multiple entities
@@ -131,6 +276,112 @@ module FactDb
     def detect_fact_conflicts(entity_ids)
       @resolution_pipeline.detect_conflicts(entity_ids)
     end
+
+    private
+
+    def build_transformers
+      {
+        raw: Transformers::RawTransformer.new,
+        json: Transformers::JsonTransformer.new,
+        triples: Transformers::TripleTransformer.new,
+        cypher: Transformers::CypherTransformer.new,
+        text: Transformers::TextTransformer.new
+      }
+    end
+
+    def transform_results(results, topic:, format:)
+      validate_format!(format)
+
+      query_result = QueryResult.new(query: topic || "query")
+      query_result.add_facts(results)
+      query_result.resolve_entities(@entity_service)
+
+      # Return QueryResult directly for :json format to support fluent API methods
+      # like each_fact, fact_count, etc. Use query_result.to_h for Hash output.
+      return query_result if format == :json
+
+      @transformers[format].transform(query_result)
+    end
+
+    def validate_format!(format)
+      return if FORMATS.include?(format)
+
+      raise ArgumentError, "Unknown format: #{format}. Available: #{FORMATS.join(', ')}"
+    end
+
+    def parse_date(date)
+      return nil if date.nil?
+      return date if date.is_a?(Date) || date.is_a?(Time)
+
+      Date.parse(date.to_s)
+    rescue ArgumentError
+      nil
+    end
+
+    def introspect_schema
+      {
+        capabilities: collect_capabilities,
+        entity_kinds: Models::Entity.distinct.pluck(:kind).compact,
+        fact_statuses: %w[canonical superseded corroborated synthesized],
+        extraction_methods: %w[manual llm rule_based],
+        output_formats: FORMATS,
+        retrieval_strategies: STRATEGIES,
+        statistics: collect_statistics
+      }
+    end
+
+    def introspect_topic(topic)
+      resolved = resolve_entity(topic)
+      return nil unless resolved
+
+      entity = resolved.respond_to?(:entity) ? resolved.entity : resolved
+
+      {
+        entity: entity_info(entity),
+        coverage: {
+          facts: @fact_service.fact_stats(entity.id),
+          timespan: @entity_service.timespan_for(entity.id)
+        },
+        relationships: @entity_service.relationship_types_for(entity.id),
+        suggested_queries: suggest_queries(topic)
+      }
+    end
+
+    def collect_capabilities
+      capabilities = [:temporal_query, :entity_resolution, :introspection]
+
+      capabilities << :semantic_search if @config.embedding_generator
+      capabilities << :llm_extraction if @config.llm_client || @config.llm&.provider
+
+      capabilities
+    end
+
+    def collect_statistics
+      {
+        facts: @fact_service.stats,
+        entities: @entity_service.stats,
+        sources: @source_service.stats
+      }
+    end
+
+    def entity_info(entity)
+      {
+        id: entity.id,
+        name: entity.name,
+        kind: entity.kind,
+        resolution_status: entity.resolution_status,
+        aliases: entity.aliases.map { |a| { name: a.name, kind: a.kind } }
+      }
+    end
+
+    def facts_to_comparable(facts)
+      facts.map { |f| comparable_key(f) }.to_set
+    end
+
+    def comparable_key(fact)
+      text = fact.respond_to?(:text) ? fact.text : fact[:text]
+      "#{text}".downcase.strip
+    end
   end
 
   class << self

data/mkdocs.yml

@@ -174,12 +174,12 @@ nav:
       - Facts: api/facts.md
       - Models:
           - api/models/index.md
-          - Content: api/models/content.md
+          - Source: api/models/source.md
           - Entity: api/models/entity.md
           - Fact: api/models/fact.md
       - Services:
           - api/services/index.md
-          - ContentService: api/services/content-service.md
+          - SourceService: api/services/source-service.md
           - EntityService: api/services/entity-service.md
           - FactService: api/services/fact-service.md
       - Extractors:

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fact_db
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Dewayne VanHoozer
@@ -52,19 +52,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: anyway_config
+  name: myway_config
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 0.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: chronic
   requirement: !ruby/object:Gem::Requirement
@@ -149,6 +149,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -175,12 +189,13 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".envrc"
+- ".yardopts"
 - CHANGELOG.md
 - COMMITS.md
 - README.md
 - Rakefile
 - db/migrate/001_enable_extensions.rb
-- db/migrate/002_create_contents.rb
+- db/migrate/002_create_sources.rb
 - db/migrate/003_create_entities.rb
 - db/migrate/004_create_entity_aliases.rb
 - db/migrate/005_create_facts.rb
@@ -192,17 +207,17 @@ files:
 - docs/api/extractors/rule-based.md
 - docs/api/facts.md
 - docs/api/index.md
-- docs/api/models/content.md
 - docs/api/models/entity.md
 - docs/api/models/fact.md
 - docs/api/models/index.md
+- docs/api/models/source.md
 - docs/api/pipeline/extraction.md
 - docs/api/pipeline/index.md
 - docs/api/pipeline/resolution.md
-- docs/api/services/content-service.md
 - docs/api/services/entity-service.md
 - docs/api/services/fact-service.md
 - docs/api/services/index.md
+- docs/api/services/source-service.md
 - docs/architecture/database-schema.md
 - docs/architecture/entity-resolution.md
 - docs/architecture/index.md
@@ -229,14 +244,34 @@ files:
 - docs/guides/llm-integration.md
 - docs/guides/temporal-queries.md
 - docs/index.md
+- examples/.envrc
+- examples/.gitignore
+- examples/001_configuration.rb
+- examples/010_basic_usage.rb
+- examples/020_entity_management.rb
+- examples/030_temporal_queries.rb
+- examples/040_output_formats.rb
+- examples/050_rule_based_extraction.rb
+- examples/060_fluent_temporal_api.rb
+- examples/070_introspection.rb
+- examples/080_hr_system.rb
+- examples/090_ingest_demo.rb
+- examples/100_query_context.rb
+- examples/110_prove_it.rb
+- examples/120_dump_database.rb
+- examples/130_rag_feedback_loop.rb
 - examples/README.md
-- examples/basic_usage.rb
-- examples/entity_management.rb
-- examples/hr_system.rb
-- examples/rule_based_extraction.rb
-- examples/temporal_queries.rb
+- examples/data/lincoln_associates.md
+- examples/data/lincoln_biography.md
+- examples/data/lincoln_cabinet.md
+- examples/data/lincoln_family.md
+- examples/data/lincoln_military.md
+- examples/data/lincoln_todd_family.md
+- examples/ingest_reporter.rb
+- examples/utilities.rb
 - lib/fact_db.rb
 - lib/fact_db/config.rb
+- lib/fact_db/config/defaults.yml
 - lib/fact_db/database.rb
 - lib/fact_db/errors.rb
 - lib/fact_db/extractors/base.rb
@@ -244,21 +279,30 @@ files:
 - lib/fact_db/extractors/manual_extractor.rb
 - lib/fact_db/extractors/rule_based_extractor.rb
 - lib/fact_db/llm/adapter.rb
-- lib/fact_db/models/content.rb
 - lib/fact_db/models/entity.rb
 - lib/fact_db/models/entity_alias.rb
 - lib/fact_db/models/entity_mention.rb
 - lib/fact_db/models/fact.rb
 - lib/fact_db/models/fact_source.rb
+- lib/fact_db/models/source.rb
 - lib/fact_db/pipeline/extraction_pipeline.rb
 - lib/fact_db/pipeline/resolution_pipeline.rb
+- lib/fact_db/query_result.rb
 - lib/fact_db/resolution/entity_resolver.rb
 - lib/fact_db/resolution/fact_resolver.rb
-- lib/fact_db/services/content_service.rb
 - lib/fact_db/services/entity_service.rb
 - lib/fact_db/services/fact_service.rb
+- lib/fact_db/services/source_service.rb
 - lib/fact_db/temporal/query.rb
+- lib/fact_db/temporal/query_builder.rb
 - lib/fact_db/temporal/timeline.rb
+- lib/fact_db/transformers/base.rb
+- lib/fact_db/transformers/cypher_transformer.rb
+- lib/fact_db/transformers/json_transformer.rb
+- lib/fact_db/transformers/raw_transformer.rb
+- lib/fact_db/transformers/text_transformer.rb
+- lib/fact_db/transformers/triple_transformer.rb
+- lib/fact_db/validation/alias_filter.rb
 - lib/fact_db/version.rb
 - mkdocs.yml
 homepage: https://github.com/MadBomber/fact_db

data/db/migrate/002_create_contents.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-class CreateContents < ActiveRecord::Migration[7.0]
-  def change
-    create_table :fact_db_contents do |t|
-      # Content identification
-      t.string :content_hash, null: false, limit: 64
-      t.string :content_type, null: false, limit: 50
-
-      # The raw content (immutable)
-      t.text :raw_text, null: false
-      t.string :title, limit: 500
-
-      # Source metadata
-      t.text :source_uri
-      t.jsonb :source_metadata, null: false, default: {}
-
-      # Vector embedding for semantic search
-      t.vector :embedding, limit: 1536
-
-      # Timestamps
-      t.timestamptz :captured_at, null: false
-      t.timestamps
-    end
-
-    add_index :fact_db_contents, :content_hash, unique: true
-    add_index :fact_db_contents, :captured_at
-    add_index :fact_db_contents, :content_type
-    add_index :fact_db_contents, :source_metadata, using: :gin
-
-    # Full-text search index
-    execute <<-SQL
-      CREATE INDEX idx_contents_fulltext ON fact_db_contents
-      USING gin(to_tsvector('english', raw_text));
-    SQL
-
-    # HNSW index for vector similarity search (if pgvector supports it)
-    # This creates a cosine similarity index for fast nearest neighbor queries
-    execute <<-SQL
-      CREATE INDEX idx_contents_embedding ON fact_db_contents
-      USING hnsw (embedding vector_cosine_ops);
-    SQL
-  end
-end

data/lib/fact_db/models/content.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-module FactDb
-  module Models
-    class Content < ActiveRecord::Base
-      self.table_name = "fact_db_contents"
-
-      has_many :fact_sources, class_name: "FactDb::Models::FactSource",
-               foreign_key: :content_id, dependent: :destroy
-      has_many :facts, through: :fact_sources
-
-      validates :content_hash, presence: true, uniqueness: true
-      validates :content_type, presence: true
-      validates :raw_text, presence: true
-      validates :captured_at, presence: true
-
-      before_validation :generate_content_hash, on: :create
-
-      # Content types
-      TYPES = %w[email transcript document slack meeting_notes contract report].freeze
-
-      validates :content_type, inclusion: { in: TYPES }, allow_nil: false
-
-      scope :by_type, ->(type) { where(content_type: type) }
-      scope :captured_between, ->(from, to) { where(captured_at: from..to) }
-      scope :captured_after, ->(date) { where("captured_at >= ?", date) }
-      scope :captured_before, ->(date) { where("captured_at <= ?", date) }
-
-      # Full-text search
-      scope :search_text, lambda { |query|
-        where("to_tsvector('english', raw_text) @@ plainto_tsquery('english', ?)", query)
-      }
-
-      # Vector similarity search (requires neighbor gem configured)
-      def self.nearest_neighbors(embedding, limit: 10)
-        return none unless embedding
-
-        order(Arel.sql("embedding <=> '#{embedding}'")).limit(limit)
-      end
-
-      def immutable?
-        true
-      end
-
-      def word_count
-        raw_text.split.size
-      end
-
-      def preview(length: 200)
-        return raw_text if raw_text.length <= length
-
-        "#{raw_text[0, length]}..."
-      end
-
-      private
-
-      def generate_content_hash
-        self.content_hash = Digest::SHA256.hexdigest(raw_text) if raw_text.present?
-      end
-    end
-  end
-end