fact_db 0.0.2 → 0.0.3

data/db/migrate/004_create_entity_aliases.rb

@@ -2,17 +2,30 @@
 
 class CreateEntityAliases < ActiveRecord::Migration[7.0]
   def change
-    create_table :fact_db_entity_aliases do |t|
-      t.references :entity, null: false, foreign_key: { to_table: :fact_db_entities, on_delete: :cascade }
-      t.string :alias_text, null: false, limit: 500
-      t.string :alias_type, limit: 50  # name, nickname, email, handle, abbreviation
-      t.float :confidence, default: 1.0
+    create_table :fact_db_entity_aliases, comment: "Alternative names and identifiers for entities enabling flexible matching" do |t|
+      t.references :entity, null: false, foreign_key: { to_table: :fact_db_entities, on_delete: :cascade },
+                   comment: "The canonical entity this alias refers to"
+      t.string :name, null: false, limit: 500,
+               comment: "The alternative name, identifier, or reference text"
+      t.string :kind, limit: 50,
+               comment: "Classification of alias: name, nickname, email, handle, abbreviation, former_name"
+      t.float :confidence, default: 1.0,
+              comment: "Confidence score (0.0-1.0) that this alias correctly refers to the entity"
 
       t.timestamps
     end
 
-    add_index :fact_db_entity_aliases, :alias_text
-    add_index :fact_db_entity_aliases, [:entity_id, :alias_text], unique: true,
+    add_index :fact_db_entity_aliases, :name
+    add_index :fact_db_entity_aliases, [:entity_id, :name], unique: true,
               name: "idx_unique_entity_alias"
+
+    # GIN trigram index on name for fuzzy alias matching
+    execute <<-SQL
+      CREATE INDEX idx_entity_aliases_name_trgm ON fact_db_entity_aliases
+      USING gin (name gin_trgm_ops);
+    SQL
+
+    execute "COMMENT ON COLUMN fact_db_entity_aliases.created_at IS 'When this alias association was created';"
+    execute "COMMENT ON COLUMN fact_db_entity_aliases.updated_at IS 'When this alias record was last modified';"
   end
 end

data/db/migrate/005_create_facts.rb

@@ -2,35 +2,42 @@
 
 class CreateFacts < ActiveRecord::Migration[7.0]
   def change
-    create_table :fact_db_facts do |t|
-      # The assertion
-      t.text :fact_text, null: false
-      t.string :fact_hash, null: false, limit: 64
+    create_table :fact_db_facts, comment: "Extracted factual assertions with temporal validity tracking (Event Clock pattern)" do |t|
+      t.text :text, null: false,
+             comment: "The factual assertion in natural language form"
+      t.string :digest, null: false, limit: 64,
+               comment: "SHA-256 hash of normalized text for deduplication"
 
-      # Temporal validity (the Event Clock core concept)
-      t.timestamptz :valid_at, null: false
-      t.timestamptz :invalid_at  # NULL = still valid
+      t.timestamptz :valid_at, null: false,
+                    comment: "When this fact became true (Event Clock valid_from)"
+      t.timestamptz :invalid_at,
+                    comment: "When this fact ceased to be true; NULL means still valid (Event Clock valid_to)"
 
-      # Fact status
-      t.string :status, null: false, default: "canonical", limit: 20
+      t.string :status, null: false, default: "canonical", limit: 20,
+               comment: "Fact lifecycle state: canonical, superseded, retracted, or disputed"
 
-      # Resolution relationships
-      t.bigint :superseded_by_id
-      t.bigint :derived_from_ids, array: true, default: []
-      t.bigint :corroborated_by_ids, array: true, default: []
+      t.bigint :superseded_by_id,
+               comment: "Reference to newer fact that replaces this one"
+      t.bigint :derived_from_ids, array: true, default: [],
+               comment: "Array of fact IDs from which this fact was inferred or derived"
+      t.bigint :corroborated_by_ids, array: true, default: [],
+               comment: "Array of fact IDs that independently confirm this fact"
 
-      # Confidence and metadata
-      t.float :confidence, default: 1.0
-      t.string :extraction_method, limit: 50
-      t.jsonb :metadata, null: false, default: {}
+      t.float :confidence, default: 1.0,
+              comment: "Confidence score (0.0-1.0) in the accuracy of this fact"
+      t.string :extraction_method, limit: 50,
+               comment: "How fact was extracted: manual, llm_extraction, rule_based, etc."
+      t.jsonb :metadata, null: false, default: {},
+              comment: "Additional structured data: extraction context, source details, tags"
 
-      # Vector embedding for semantic search
-      t.vector :embedding, limit: 1536
+      t.vector :embedding, limit: 1536,
+               comment: "Vector embedding for semantic fact search and similarity matching"
 
       t.timestamps
     end
 
-    add_index :fact_db_facts, :fact_hash
+    # Unique constraint on digest + valid_at allows same fact text at different times
+    add_index :fact_db_facts, [:digest, :valid_at], unique: true, name: "index_fact_db_facts_on_digest_valid_at"
     add_index :fact_db_facts, :valid_at
     add_index :fact_db_facts, :invalid_at
     add_index :fact_db_facts, :status
@@ -53,7 +60,7 @@ class CreateFacts < ActiveRecord::Migration[7.0]
     # Full-text search index
     execute <<-SQL
       CREATE INDEX idx_facts_fulltext ON fact_db_facts
-      USING gin(to_tsvector('english', fact_text));
+      USING gin(to_tsvector('english', text));
     SQL
 
     # HNSW index for vector similarity search
@@ -61,5 +68,14 @@ class CreateFacts < ActiveRecord::Migration[7.0]
       CREATE INDEX idx_facts_embedding ON fact_db_facts
       USING hnsw (embedding vector_cosine_ops);
     SQL
+
+    # GIN trigram index on text for fuzzy fact search
+    execute <<-SQL
+      CREATE INDEX idx_facts_text_trgm ON fact_db_facts
+      USING gin (text gin_trgm_ops);
+    SQL
+
+    execute "COMMENT ON COLUMN fact_db_facts.created_at IS 'When this fact was recorded in the database';"
+    execute "COMMENT ON COLUMN fact_db_facts.updated_at IS 'When this fact record was last modified';"
   end
 end

data/db/migrate/006_create_entity_mentions.rb

@@ -2,17 +2,25 @@
 
 class CreateEntityMentions < ActiveRecord::Migration[7.0]
   def change
-    create_table :fact_db_entity_mentions do |t|
-      t.references :fact, null: false, foreign_key: { to_table: :fact_db_facts, on_delete: :cascade }
-      t.references :entity, null: false, foreign_key: { to_table: :fact_db_entities, on_delete: :cascade }
-      t.string :mention_text, null: false, limit: 500
-      t.string :mention_role, limit: 50  # subject, object, location, etc.
-      t.float :confidence, default: 1.0
+    create_table :fact_db_entity_mentions, comment: "Links entities to facts where they are mentioned, with role context" do |t|
+      t.references :fact, null: false, foreign_key: { to_table: :fact_db_facts, on_delete: :cascade },
+                   comment: "The fact containing this entity mention"
+      t.references :entity, null: false, foreign_key: { to_table: :fact_db_entities, on_delete: :cascade },
+                   comment: "The resolved entity being mentioned"
+      t.string :mention_text, null: false, limit: 500,
+               comment: "The exact text used to reference the entity in the fact"
+      t.string :mention_role, limit: 50,
+               comment: "Semantic role of entity in fact: subject, object, location, time, instrument, etc."
+      t.float :confidence, default: 1.0,
+              comment: "Confidence score (0.0-1.0) that mention correctly resolves to entity"
 
       t.timestamps
     end
 
     add_index :fact_db_entity_mentions, [:fact_id, :entity_id, :mention_text],
               unique: true, name: "idx_unique_fact_entity_mention"
+
+    execute "COMMENT ON COLUMN fact_db_entity_mentions.created_at IS 'When this mention link was created';"
+    execute "COMMENT ON COLUMN fact_db_entity_mentions.updated_at IS 'When this mention record was last modified';"
   end
 end

data/db/migrate/007_create_fact_sources.rb

@@ -2,17 +2,25 @@
 
 class CreateFactSources < ActiveRecord::Migration[7.0]
   def change
-    create_table :fact_db_fact_sources do |t|
-      t.references :fact, null: false, foreign_key: { to_table: :fact_db_facts, on_delete: :cascade }
-      t.references :content, null: false, foreign_key: { to_table: :fact_db_contents, on_delete: :cascade }
-      t.string :source_type, default: "primary", limit: 50  # primary, supporting, corroborating
-      t.text :excerpt  # The specific portion that supports the fact
-      t.float :confidence, default: 1.0
+    create_table :fact_db_fact_sources, comment: "Links facts to their source content for provenance tracking" do |t|
+      t.references :fact, null: false, foreign_key: { to_table: :fact_db_facts, on_delete: :cascade },
+                   comment: "The fact derived from this source"
+      t.references :source, null: false, foreign_key: { to_table: :fact_db_sources, on_delete: :cascade },
+                   comment: "The source content from which the fact was extracted"
+      t.string :kind, default: "primary", limit: 50,
+               comment: "Relationship type: primary (direct extraction), supporting, or corroborating"
+      t.text :excerpt,
+             comment: "The specific text passage within the content that supports this fact"
+      t.float :confidence, default: 1.0,
+              comment: "Confidence score (0.0-1.0) that this source supports the fact"
 
       t.timestamps
     end
 
-    add_index :fact_db_fact_sources, [:fact_id, :content_id], unique: true,
-              name: "idx_unique_fact_content"
+    add_index :fact_db_fact_sources, [:fact_id, :source_id], unique: true,
+              name: "idx_unique_fact_source"
+
+    execute "COMMENT ON COLUMN fact_db_fact_sources.created_at IS 'When this source link was established';"
+    execute "COMMENT ON COLUMN fact_db_fact_sources.updated_at IS 'When this source record was last modified';"
   end
 end

data/docs/api/extractors/index.md

@@ -34,11 +34,11 @@ end
 
 ```ruby
 class MyExtractor < FactDb::Extractors::Base
-  def extract(content)
+  def extract(source)
     facts = []
 
     # Your extraction logic
-    # Parse content.raw_text
+    # Parse source.content
     # Create fact records
 
     facts
@@ -48,18 +48,18 @@ end
 
 ## Using Extractors
 
-### Via Facts
+### Via FactDb
 
 ```ruby
 facts = FactDb.new
-extracted = facts.extract_facts(content.id, extractor: :llm)
+extracted = facts.extract_facts(source.id, extractor: :llm)
 ```
 
 ### Directly
 
 ```ruby
 extractor = FactDb::Extractors::LLMExtractor.new(config)
-facts = extractor.extract(content)
+facts = extractor.extract(source)
 ```
 
 ## Extractor Selection

data/docs/api/extractors/llm.md

@@ -17,9 +17,9 @@ extractor = FactDb::Extractors::LLMExtractor.new(config)
 
 ```ruby
 FactDb.configure do |config|
-  config.llm_provider = :openai
-  config.llm_model = "gpt-4o-mini"
-  config.llm_api_key = ENV['OPENAI_API_KEY']
+  config.llm.provider = :openai
+  config.llm.model = "gpt-4o-mini"
+  config.llm.api_key = ENV['OPENAI_API_KEY']
 end
 ```
 
@@ -35,7 +35,7 @@ Extract facts from content using LLM.
 
 **Parameters:**
 
-- `content` (Models::Content) - Content to process
+- `source` (Models::Source) - Source to process
 
 **Returns:** `Array<Models::Fact>`
 
@@ -43,10 +43,10 @@ Extract facts from content using LLM.
 
 ```ruby
 extractor = LLMExtractor.new(config)
-facts = extractor.extract(content)
+facts = extractor.extract(source)
 
 facts.each do |fact|
-  puts fact.fact_text
+  puts fact.text
   puts "  Valid: #{fact.valid_at}"
   puts "  Confidence: #{fact.confidence}"
 end
@@ -73,7 +73,7 @@ Extract temporal facts from this content. For each fact:
 4. Assess confidence level
 
 Content:
-{content.raw_text}
+{source.content}
 
 Return JSON:
 {
@@ -94,12 +94,12 @@ Return JSON:
 
 | Provider | Models | Config |
 |----------|--------|--------|
-| OpenAI | gpt-4o, gpt-4o-mini | `llm_provider: :openai` |
-| Anthropic | claude-sonnet-4, claude-3-haiku | `llm_provider: :anthropic` |
-| Google | gemini-2.0-flash | `llm_provider: :gemini` |
-| Ollama | llama3.2, mistral | `llm_provider: :ollama` |
-| AWS Bedrock | claude-sonnet-4 | `llm_provider: :bedrock` |
-| OpenRouter | Various | `llm_provider: :openrouter` |
+| OpenAI | gpt-4o, gpt-4o-mini | `llm.provider = :openai` |
+| Anthropic | claude-sonnet-4, claude-3-haiku | `llm.provider = :anthropic` |
+| Google | gemini-2.0-flash | `llm.provider = :gemini` |
+| Ollama | llama3.2, mistral | `llm.provider = :ollama` |
+| AWS Bedrock | claude-sonnet-4 | `llm.provider = :bedrock` |
+| OpenRouter | Various | `llm.provider = :openrouter` |
 
 ## Error Handling
 
@@ -134,7 +134,7 @@ end
 ### 1. Validate Results
 
 ```ruby
-facts = extractor.extract(content)
+facts = extractor.extract(source)
 facts.each do |fact|
   if fact.confidence < 0.7
     fact.update!(metadata: { needs_review: true })
@@ -145,9 +145,9 @@ end
 ### 2. Cache Responses
 
 ```ruby
-cache_key = "llm:#{content.content_hash}"
+cache_key = "llm:#{source.content_hash}"
 facts = Rails.cache.fetch(cache_key) do
-  extractor.extract(content)
+  extractor.extract(source)
 end
 ```
 
@@ -157,6 +157,6 @@ end
 require 'retryable'
 
 Retryable.retryable(tries: 3, sleep: lambda { |n| 2**n }) do
-  extractor.extract(content)
+  extractor.extract(source)
 end
 ```

data/docs/api/extractors/rule-based.md

@@ -65,13 +65,13 @@ The extractor includes patterns for common fact types:
 ```ruby
 extractor = RuleBasedExtractor.new(config)
 
-content = Models::Content.create!(
-  raw_text: "Paula Chen joined Microsoft on January 10, 2024 as Principal Engineer.",
-  content_type: "announcement",
+source = Models::Source.create!(
+  content: "Paula Chen joined Microsoft on January 10, 2024 as Principal Engineer.",
+  type: "announcement",
   captured_at: Time.current
 )
 
-facts = extractor.extract(content)
+facts = extractor.extract(source)
 # Returns facts about:
 # - Paula joining Microsoft
 # - Paula's title as Principal Engineer
@@ -99,20 +99,20 @@ class CustomRuleExtractor < FactDb::Extractors::RuleBasedExtractor
 
   private
 
-  def extract_custom_patterns(content)
+  def extract_custom_patterns(source)
     facts = []
     CUSTOM_PATTERNS.each do |rule|
-      content.raw_text.scan(rule[:pattern]) do |match|
-        facts << send(rule[:handler], match, content)
+      source.content.scan(rule[:pattern]) do |match|
+        facts << send(rule[:handler], match, source)
       end
     end
     facts
   end
 
-  def extract_revenue(match, content)
+  def extract_revenue(match, source)
     Models::Fact.create!(
-      fact_text: "Revenue of $#{match[:amount]}",
-      valid_at: content.captured_at,
+      text: "Revenue of $#{match[:amount]}",
+      valid_at: source.captured_at,
       extraction_method: "rule_based",
       # ...
     )
@@ -141,7 +141,7 @@ end
 
 ```ruby
 # Use rule-based for structured content
-if content.content_type == "form"
+if content.type == "form"
   facts = rule_extractor.extract(content)
 else
   facts = llm_extractor.extract(content)
@@ -155,11 +155,11 @@ facts = extractor.extract(content)
 facts.select { |f| f.confidence > 0.8 }
 ```
 
-### 3. Log Unmatched Content
+### 3. Log Unmatched Sources
 
 ```ruby
-facts = extractor.extract(content)
+facts = extractor.extract(source)
 if facts.empty?
-  logger.info "No patterns matched for content #{content.id}"
+  logger.info "No patterns matched for source #{source.id}"
 end
 ```

data/docs/api/facts.md

@@ -15,7 +15,7 @@ facts = FactDb::Facts.new(config: custom_config)
 | Attribute | Type | Description |
 |-----------|------|-------------|
 | `config` | Config | Configuration instance |
-| `content_service` | ContentService | Service for content operations |
+| `source_service` | SourceService | Service for source operations |
 | `entity_service` | EntityService | Service for entity operations |
 | `fact_service` | FactService | Service for fact operations |
 | `extraction_pipeline` | ExtractionPipeline | Pipeline for batch extraction |
@@ -43,7 +43,7 @@ facts = FactDb.new
 
 # Use custom configuration
 config = FactDb::Config.new
-config.database_url = "postgresql://localhost/my_db"
+config.database.url = "postgresql://localhost/my_db"
 facts = FactDb.new(config: config)
 ```
 
@@ -52,28 +52,28 @@ facts = FactDb.new(config: config)
 ### ingest
 
 ```ruby
-def ingest(raw_text, type:, captured_at: Time.current, metadata: {}, title: nil, source_uri: nil)
+def ingest(content, kind:, captured_at: Time.current, metadata: {}, title: nil, source_uri: nil)
 ```
 
 Ingest raw content into the fact database.
 
 **Parameters:**
 
-- `raw_text` (String) - The content text
-- `type` (Symbol) - Content type (:email, :document, :article, etc.)
+- `content` (String) - The source text content
+- `kind` (Symbol) - Content kind (:email, :document, :article, etc.)
 - `captured_at` (Time, optional) - When content was captured
 - `metadata` (Hash, optional) - Additional metadata
 - `title` (String, optional) - Content title
 - `source_uri` (String, optional) - Original location
 
-**Returns:** `Models::Content`
+**Returns:** `Models::Source`
 
 **Example:**
 
 ```ruby
-content = facts.ingest(
+source = facts.ingest(
   "Paula joined Microsoft on Jan 10, 2024",
-  type: :announcement,
+  kind: :announcement,
   title: "New Hire",
   captured_at: Time.current
 )
@@ -84,14 +84,14 @@ content = facts.ingest(
 ### extract_facts
 
 ```ruby
-def extract_facts(content_id, extractor: @config.default_extractor)
+def extract_facts(source_id, extractor: @config.default_extractor)
 ```
 
 Extract facts from content.
 
 **Parameters:**
 
-- `content_id` (Integer) - Content ID
+- `source_id` (Integer) - Source ID
 - `extractor` (Symbol, optional) - Extraction method (:manual, :llm, :rule_based)
 
 **Returns:** `Array<Models::Fact>`
@@ -99,7 +99,7 @@ Extract facts from content.
 **Example:**
 
 ```ruby
-extracted = facts.extract_facts(content.id, extractor: :llm)
+extracted = facts.extract_facts(source.id, extractor: :llm)
 ```
 
 ---
@@ -139,7 +139,7 @@ results = facts.query_facts(at: Date.parse("2023-06-15"))
 ### resolve_entity
 
 ```ruby
-def resolve_entity(name, type: nil)
+def resolve_entity(name, kind: nil)
 ```
 
 Resolve a name to an entity.
@@ -147,14 +147,14 @@ Resolve a name to an entity.
 **Parameters:**
 
 - `name` (String) - Name to resolve
-- `type` (Symbol, optional) - Entity type filter
+- `kind` (Symbol, optional) - Entity kind filter
 
 **Returns:** `Models::Entity` or `nil`
 
 **Example:**
 
 ```ruby
-entity = facts.resolve_entity("Paula Chen", type: :person)
+entity = facts.resolve_entity("Paula Chen", kind: :person)
 ```
 
 ---
@@ -232,14 +232,14 @@ historical = facts.facts_at(Date.parse("2023-06-15"), entity: paula.id)
 ### batch_extract
 
 ```ruby
-def batch_extract(content_ids, extractor: @config.default_extractor, parallel: true)
+def batch_extract(source_ids, extractor: @config.default_extractor, parallel: true)
 ```
 
 Batch extract facts from multiple content items.
 
 **Parameters:**
 
-- `content_ids` (Array<Integer>) - Content IDs to process
+- `source_ids` (Array<Integer>) - Source IDs to process
 - `extractor` (Symbol, optional) - Extraction method
 - `parallel` (Boolean, optional) - Use parallel processing (default: true)
 
@@ -248,9 +248,9 @@ Batch extract facts from multiple content items.
 **Example:**
 
 ```ruby
-results = facts.batch_extract([c1.id, c2.id, c3.id], parallel: true)
+results = facts.batch_extract([s1.id, s2.id, s3.id], parallel: true)
 results.each do |r|
-  puts "#{r[:content_id]}: #{r[:facts].count} facts"
+  puts "#{r[:source_id]}: #{r[:facts].count} facts"
 end
 ```
 
@@ -259,7 +259,7 @@ end
 ### batch_resolve_entities
 
 ```ruby
-def batch_resolve_entities(names, type: nil)
+def batch_resolve_entities(names, kind: nil)
 ```
 
 Batch resolve entity names.
@@ -267,7 +267,7 @@ Batch resolve entity names.
 **Parameters:**
 
 - `names` (Array<String>) - Names to resolve
-- `type` (Symbol, optional) - Entity type filter
+- `kind` (Symbol, optional) - Entity kind filter
 
 **Returns:** `Array<Hash>` - Resolution results
 

data/docs/api/index.md

@@ -8,13 +8,13 @@ Complete API documentation for FactDb.
 
 ## Models
 
-- [Content](models/content.md) - Immutable source documents
+- [Source](models/source.md) - Immutable source content
 - [Entity](models/entity.md) - Resolved identities
 - [Fact](models/fact.md) - Temporal assertions
 
 ## Services
 
-- [ContentService](services/content-service.md) - Ingest and manage content
+- [SourceService](services/source-service.md) - Ingest and manage sources
 - [EntityService](services/entity-service.md) - Create and resolve entities
 - [FactService](services/fact-service.md) - Extract and query facts
 
@@ -37,14 +37,14 @@ FactDb
 ├── Config                   # Configuration
 ├── Database                 # Database connection
 ├── Models
-│   ├── Content
+│   ├── Source
 │   ├── Entity
 │   ├── EntityAlias
 │   ├── Fact
 │   ├── EntityMention
 │   └── FactSource
 ├── Services
-│   ├── ContentService
+│   ├── SourceService
 │   ├── EntityService
 │   └── FactService
 ├── Extractors