fact_db 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a87c848e8b274c26d7960e6ac91c5efaf3238b568fb264e2c92e3d37548aa398
-  data.tar.gz: 72d964331f79436f8efefc0627896709fe2a32487195da21a180504c15c67082
+  metadata.gz: ac11ce032c5e56849e6910c26154add65d81d3aaaa349969e584c493633e8db9
+  data.tar.gz: c0ace2cd5605d4530fc3f2a16e91a2edb85dd5532ccc80bdf3d0a2248a0e2a27
 SHA512:
-  metadata.gz: 74c901b77d7081e53ff87dc81800cc4cb862b83de96ce1e8b15de2f46e0b4d7b8e7b91daaaaedaca9fa723c36da9b0142fa68c2a9583a1f52731b7642ed29245
-  data.tar.gz: 6f7359db8aaaa3c60c8fba762d4c96157834f2427e4295e760b74671ca30dfa4340c2070078d287b9263d77be7e8f89cb6a6731792a4273cc3233d4bb11aba81
+  metadata.gz: e0ecf9c10caaa2e5836282bfac4b8de0b0f287948ca7f9de5413a8a00bb6d188e272b9a0f10d1832ca68806c9743ecdeac3939c304c6636c06d0b96847e33eb9
+  data.tar.gz: efd5e98fb240194a467bf6a8ddec659c754635290f54f99dab4c11779f627589fa659b8827feb9b0bd7e1c2d761e592e3c03fc30965d7a54095c32e05c48f310

data/.envrc

@@ -1 +1,3 @@
 export RR=`pwd`
+
+export FDB_ENV=development

data/.yardopts

@@ -0,0 +1,5 @@
+--output-dir doc
+--readme README.md
+--markup markdown
+--asset docs/assets:assets
+lib/**/*.rb

data/CHANGELOG.md

@@ -8,6 +8,109 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.0] - Unreleased
+
+### Added
+
+- **Persistent TSVectors** - Precomputed `tsvector` columns for full-text search
+  - `text_vector` column on `fact_db_facts` replaces on-the-fly `to_tsvector()` computation
+  - `content_vector` column on `fact_db_sources` replaces on-the-fly `to_tsvector()` computation
+  - Database triggers automatically keep vectors in sync on INSERT/UPDATE
+  - GIN indexes created `CONCURRENTLY` for non-blocking deployment
+  - Migration backfills existing rows and drops redundant expression-based indexes
+- **Configurable LLM Prompts** - Extraction prompts moved to configuration
+  - `config.prompts.fact_extraction` - Customizable fact extraction prompt template
+  - `config.prompts.entity_extraction` - Customizable entity extraction prompt template
+  - `config.prompts.rag_system` - Customizable RAG system prompt
+  - Override via config files, environment variables (`FDB_PROMPTS__*`), or programmatic configuration
+- **Configuration Defaults File** - `lib/fact_db/config/defaults.yml` as single source of truth for all config schema and defaults
+- **Configuration Example** - New `examples/001_configuration.rb` demonstrating all configuration methods
+- **Ingest Reporter** - New `examples/ingest_reporter.rb` with structured reporting for markdown ingestion
+
+### Changed
+
+- **Full-text search scopes** now query persisted `tsvector` columns instead of computing them at query time
+  - `Fact.search_text` queries `text_vector` column directly
+  - `Source.search_text` queries `content_vector` column directly
+  - `ts_rank_cd()` calls in examples use persisted columns instead of recomputing
+- **LLM Extractor** - Removed hardcoded prompt constants; prompts now loaded from configuration
+- **Markdown ingestion** - Refactored with new reporter and reduced progress verbosity
+- Version bump from 0.0.4 to 0.1.0
+
+## [0.0.4] - 2026-01-12
+
+### Added
+
+- **LLM prompt configuration** - Prompts for fact/entity extraction are now configurable
+
+### Changed
+
+- Version bump from 0.0.3 to 0.0.4
+
+## [0.0.3] - 2026-01-12
+
+### Added
+
+- **Rake Tasks** - New database and documentation tasks
+  - `db:dump` - Dump database to file with timestamped naming convention
+  - `db:restore` - Restore database from dump file with interactive selection
+  - `db:schema:dump` - Dump database schema to `db/schema.sql`
+  - `db:schema:load` - Load database schema from `db/schema.sql`
+  - `docs:mkdocs` - Build MkDocs documentation site
+  - `docs:yard` - Build YARD API documentation
+  - `docs:all` - Build all documentation
+- **YARD Documentation** - API documentation with GitHub Pages deployment
+  - Added `.yardopts` configuration
+  - Added GitHub Actions workflow for YARD deployment
+  - YARD docs available at `/yard` subdirectory on GitHub Pages
+- **Trigram Search** - Added pg_trgm extension for fuzzy text matching
+- **RAG Feedback Loop Example** - New example demonstrating retrieval-augmented generation patterns
+- **Output Transformers** - Transform query results into multiple formats optimized for LLM consumption
+  - `RawTransformer` - Returns original ActiveRecord objects unchanged for direct database access
+  - `JsonTransformer` - JSON-serializable hash format (default)
+  - `TripleTransformer` - Subject-Predicate-Object triples for semantic encoding
+  - `CypherTransformer` - Cypher-like graph notation with nodes and relationships
+  - `TextTransformer` - Human-readable markdown format grouped by fact status
+- **QueryResult** - Unified container for query results that works with all transformers
+  - Normalizes facts from ActiveRecord objects or hashes
+  - Resolves and caches entities referenced in facts
+  - Provides iteration methods (`each_fact`, `each_entity`)
+- **Temporal Query Builder** - Fluent API for point-in-time queries via `facts.at(date)`
+  - Chain queries: `facts.at("2024-01-15").query("Paula's role", format: :cypher)`
+  - Get facts for entity: `facts.at("2024-01-15").facts_for(entity_id)`
+  - Compare dates: `facts.at("2024-01-15").compare_to("2024-06-15")`
+- **Temporal Diff** - Compare what changed between two dates with `facts.diff(topic, from:, to:)`
+  - Returns `:added`, `:removed`, and `:unchanged` fact arrays
+- **Introspection API** - Discover what the fact database knows about
+  - `facts.introspect` - Get schema, capabilities, entity types, and statistics
+  - `facts.introspect("Paula Chen")` - Get coverage and relationships for a topic
+  - `facts.suggest_queries(topic)` - Get suggested queries based on stored data
+  - `facts.suggest_strategies(query)` - Get recommended retrieval strategies
+- **Format Parameter** - All query methods now accept `format:` parameter
+  - Available formats: `:raw`, `:json`, `:triples`, `:cypher`, `:text`
+  - Example: `facts.query_facts(topic: "Paula", format: :cypher)`
+
+### Changed
+
+- **Configuration** - Replaced `anyway_config` with `myway_config` for configuration management
+  - Added environment-specific configuration support
+- `EntityService` now includes `relationship_types_for(entity_id)` and `timespan_for(entity_id)` methods
+- `FactService` now includes `fact_stats(entity_id)` for per-entity statistics
+
+### Breaking Changes
+
+- **Database Schema Renames** - Multiple columns and tables renamed for consistency
+  - Table: `contents` → `sources`
+  - Column: `content_type` → `type` (in sources)
+  - Column: `source_metadata` → `metadata` (in sources)
+  - Column: `entity_type` → `type` (in entities)
+  - Column: `canonical_name` → `name` (in entities)
+  - Column: `merged_into_id` → `canonical_id` (in entities)
+  - Column: `alias_text` → `name` (in entity_aliases)
+  - Column: `alias_type` → `type` (in entity_aliases)
+  - Column: `fact_text` → `text` (in facts)
+- **Terminology** - Replaced `type` with `kind` throughout the codebase for entity and content classification to avoid conflicts with Ruby's reserved `type` method
+
 ## [0.0.2] - 2025-01-08
 
 ### Fixed

data/README.md

@@ -19,7 +19,9 @@ FactDb implements the Event Clock concept - capturing organizational knowledge t
 - <strong>Audit Trails</strong> - Every fact links back to source content<br>
 - <strong>Multiple Extractors</strong> - Extract facts manually, via LLM, or rule-based<br>
 - <strong>Semantic Search</strong> - PostgreSQL with pgvector<br>
-- <strong>Concurrent Processing</strong> - Batch process with parallel pipelines
+- <strong>Concurrent Processing</strong> - Batch process with parallel pipelines<br>
+- <strong>Output Formats</strong> - JSON, triples, Cypher, or text for LLM consumption<br>
+- <strong>Temporal Queries</strong> - Fluent API for point-in-time queries and diffs
 </td>
 </tr>
 </table>
@@ -50,8 +52,10 @@ bundle install
 require 'fact_db'
 
 # Configure with a PostgreSQL database URL
+# If you want to use an envar name different from the standard
+# FDB_DATABASE__URL then you must set the config.database.url in code ...
 FactDb.configure do |config|
-  config.database_url = ENV.fetch("DATABASE_URL", "postgres://#{ENV['USER']}@localhost/fact_db_demo")
+  config.database.url = ENV["YOUR_DATABASE_URL_ENVAR_NAME"]
 end
 
 # Run migrations to create the schema (only needed once)
@@ -61,19 +65,27 @@ FactDb::Database.migrate!
 facts = FactDb.new
 ```
 
+Configuration uses nested sections. You can also use environment variables:
+
+```bash
+export FDB_DATABASE__URL="postgresql://localhost/fact_db"
+export FDB_LLM__PROVIDER="openai"
+export FDB_LLM__API_KEY="sk-..."
+```
+
 Once configured, you can ingest content and create facts:
 
 ```ruby
 # Ingest content
 content = facts.ingest(
   "Paula Chen joined Microsoft as Principal Engineer on January 10, 2024.",
-  type: :email,
+  kind: :email,
   captured_at: Time.now
 )
 
 # Create entities
-paula = facts.entity_service.create("Paula Chen", type: :person)
-microsoft = facts.entity_service.create("Microsoft", type: :organization)
+paula = facts.entity_service.create("Paula Chen", kind: :person)
+microsoft = facts.entity_service.create("Microsoft", kind: :organization)
 
 # Create a fact with entity mentions
 facts.fact_service.create(
@@ -91,17 +103,106 @@ Query facts temporally:
 ```ruby
 # Query current facts about Paula
 facts.current_facts_for(paula.id).each do |fact|
-  puts fact.fact_text
+  puts fact.text
 end
 
 # Query facts at a point in time (before she joined)
 facts.facts_at(Date.new(2023, 6, 15), entity: paula.id)
 ```
 
+## Output Formats
+
+Query results can be transformed into multiple formats for different use cases:
+
+```ruby
+# Raw - original ActiveRecord objects for direct database access
+results = facts.query_facts(topic: "Paula Chen", format: :raw)
+results.each do |fact|
+  puts fact.text
+  puts fact.entity_mentions.map(&:entity).map(&:name)
+end
+
+# JSON (default) - structured hash
+facts.query_facts(topic: "Paula Chen", format: :json)
+
+# Triples - Subject-Predicate-Object for semantic encoding
+facts.query_facts(topic: "Paula Chen", format: :triples)
+# => [["Paula Chen", "kind", "Person"],
+#     ["Paula Chen", "works_at", "Microsoft"],
+#     ["Paula Chen", "works_at.valid_from", "2024-01-10"]]
+
+# Cypher - graph notation with nodes and relationships
+facts.query_facts(topic: "Paula Chen", format: :cypher)
+# => (paula_chen:Person {name: "Paula Chen"})
+#    (microsoft:Organization {name: "Microsoft"})
+#    (paula_chen)-[:WORKS_AT {since: "2024-01-10"}]->(microsoft)
+
+# Text - human-readable markdown
+facts.query_facts(topic: "Paula Chen", format: :text)
+```
+
+## Temporal Query Builder
+
+Use the fluent API for point-in-time queries:
+
+```ruby
+# Query at a specific date
+facts.at("2024-01-15").query("Paula's role", format: :cypher)
+
+# Get all facts valid at a date
+facts.at("2024-01-15").facts
+
+# Get facts for a specific entity at that date
+facts.at("2024-01-15").facts_for(paula.id)
+
+# Compare what changed between two dates
+facts.at("2024-01-15").compare_to("2024-06-15")
+```
+
+## Comparing Changes Over Time
+
+Track what changed between two points in time:
+
+```ruby
+diff = facts.diff("Paula Chen", from: "2024-01-01", to: "2024-06-01")
+
+diff[:added]     # Facts that became valid
+diff[:removed]   # Facts that were superseded
+diff[:unchanged] # Facts that remained valid
+```
+
+## Introspection
+
+Discover what the fact database knows about:
+
+```ruby
+# Get schema and capabilities
+facts.introspect
+# => { capabilities: [:temporal_query, :entity_resolution, ...],
+#      entity_kinds: ["person", "organization", ...],
+#      output_formats: [:raw, :json, :triples, :cypher, :text],
+#      statistics: { facts: {...}, entities: {...} } }
+
+# Get coverage for a specific topic
+facts.introspect("Paula Chen")
+# => { entity: {...}, coverage: {...}, relationships: [...],
+#      suggested_queries: ["current status", "employment history"] }
+
+# Get query suggestions
+facts.suggest_queries("Paula Chen")
+# => ["current status", "employment history", "timeline"]
+
+# Get retrieval strategy recommendations
+facts.suggest_strategies("What happened last week?")
+# => [{ strategy: :temporal, description: "Filter by date range" }]
+```
+
 ## Documentation
 
 Full documentation is available at **[https://madbomber.github.io/fact_db](https://madbomber.github.io/fact_db)**
 
+API documentation (YARD) is available at **[https://madbomber.github.io/fact_db/yard](https://madbomber.github.io/fact_db/yard)**
+
 ## Examples
 
 See the [examples directory](examples/README.md) for runnable demo programs covering:

data/Rakefile

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "bundler/gem_tasks"
+require "fileutils"
 require "rake/testtask"
 
 Rake::TestTask.new(:test) do |t|
@@ -9,33 +10,275 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/**/*_test.rb"]
 end
 
+# Ensure test environment is set before running tests
+task :set_test_env do
+  ENV["FDB_ENV"] = "test"
+end
+
+task test: [:set_test_env, "db:reset:test"]
+
 namespace :db do
+  desc "Drop the database"
+  task :drop do
+    require_relative "lib/fact_db"
+    puts "Environment: #{FactDb.config.environment}"
+    puts "Database: #{FactDb.config.database.name}"
+    FactDb::Database.drop!
+  end
+
+  desc "Create the database"
+  task :create do
+    require_relative "lib/fact_db"
+    puts "Environment: #{FactDb.config.environment}"
+    puts "Database: #{FactDb.config.database.name}"
+    FactDb::Database.create!
+  end
+
   desc "Run database migrations"
   task :migrate do
     require_relative "lib/fact_db"
-    FactDb.configure do |config|
-      config.database_url = ENV.fetch("DATABASE_URL")
-    end
+    puts "Environment: #{FactDb.config.environment}"
+    puts "Database: #{FactDb.config.database.name}"
     FactDb::Database.migrate!
   end
 
   desc "Rollback the last migration"
   task :rollback do
     require_relative "lib/fact_db"
-    FactDb.configure do |config|
-      config.database_url = ENV.fetch("DATABASE_URL")
-    end
+    puts "Environment: #{FactDb.config.environment}"
+    puts "Database: #{FactDb.config.database.name}"
     FactDb::Database.rollback!
   end
 
-  desc "Reset the database (drop, create, migrate)"
+  desc "Reset the database (drop, create, migrate) - honors FDB_ENV"
   task :reset do
     require_relative "lib/fact_db"
-    FactDb.configure do |config|
-      config.database_url = ENV.fetch("DATABASE_URL")
-    end
+    puts "Environment: #{FactDb.config.environment}"
+    puts "Database: #{FactDb.config.database.name}"
     FactDb::Database.reset!
   end
+
+  namespace :reset do
+    def reset_for_environment(env_name)
+      original_env = ENV["FDB_ENV"]
+      ENV["FDB_ENV"] = env_name
+
+      require_relative "lib/fact_db"
+      Anyway::Settings.current_environment = env_name
+      FactDb.reset_configuration!
+
+      puts "Environment: #{FactDb.config.environment}"
+      puts "Database: #{FactDb.config.database.name}"
+      FactDb::Database.reset!
+    ensure
+      ENV["FDB_ENV"] = original_env
+      Anyway::Settings.current_environment = original_env || "development"
+      FactDb.reset_configuration!
+    end
+
+    desc "Reset development database"
+    task :development do
+      reset_for_environment("development")
+    end
+
+    desc "Reset test database"
+    task :test do
+      reset_for_environment("test")
+    end
+
+    desc "Reset demo database"
+    task :demo do
+      reset_for_environment("demo")
+    end
+
+    desc "Reset all databases (development, test, demo)"
+    task :all do
+      %w[development test demo].each do |env_name|
+        puts "\n#{"=" * 50}"
+        reset_for_environment(env_name)
+      end
+      puts "\n#{"=" * 50}"
+      puts "All databases reset."
+    end
+  end
+
+  namespace :schema do
+    desc "Dump database schema to db/schema.sql"
+    task :dump do
+      require_relative "lib/fact_db"
+      db = FactDb.config.database
+
+      puts "Environment: #{FactDb.config.environment}"
+      puts "Database: #{db.name}"
+
+      schema_file = File.expand_path("db/schema.sql", __dir__)
+      host = db.host || "localhost"
+      port = db.port || 5432
+      user = db.username || ENV["USER"]
+
+      cmd = ["pg_dump", "--schema-only", "--no-owner", "--no-acl"]
+      cmd += ["-h", host, "-p", port.to_s]
+      cmd += ["-U", user] if user
+      cmd << db.name
+
+      File.open(schema_file, "w") do |f|
+        f.puts "-- Schema dump for #{db.name}"
+        f.puts "-- Generated: #{Time.now.utc.iso8601}"
+        f.puts "-- Environment: #{FactDb.config.environment}"
+        f.puts
+      end
+
+      system(*cmd, out: [schema_file, "a"]) || abort("pg_dump failed")
+      puts "Schema dumped to #{schema_file}"
+    end
+
+    desc "Load database schema from db/schema.sql"
+    task :load do
+      require_relative "lib/fact_db"
+      db = FactDb.config.database
+
+      puts "Environment: #{FactDb.config.environment}"
+      puts "Database: #{db.name}"
+
+      schema_file = File.expand_path("db/schema.sql", __dir__)
+      abort("Schema file not found: #{schema_file}") unless File.exist?(schema_file)
+
+      host = db.host || "localhost"
+      port = db.port || 5432
+      user = db.username || ENV["USER"]
+
+      cmd = ["psql", "-q"]
+      cmd += ["-h", host, "-p", port.to_s]
+      cmd += ["-U", user] if user
+      cmd += ["-d", db.name, "-f", schema_file]
+
+      system(*cmd) || abort("psql failed")
+      puts "Schema loaded from #{schema_file}"
+    end
+  end
+
+  desc "Dump database to file. DIR=path (default: .)"
+  task :dump do
+    require_relative "lib/fact_db"
+    db = FactDb.config.database
+
+    puts "Environment: #{FactDb.config.environment}"
+    puts "Database: #{db.name}"
+
+    dumps_dir = ENV["DIR"] || Dir.pwd
+    timestamp = Time.now.strftime("%Y%m%d_%H%M%S")
+    dump_file = File.join(dumps_dir, "fact_db_#{FactDb.config.environment}_#{timestamp}.dump")
+
+    host = db.host || "localhost"
+    port = db.port || 5432
+    user = db.username || ENV["USER"]
+
+    cmd = ["pg_dump", "-Fc", "--no-owner", "--no-acl"]
+    cmd += ["-h", host, "-p", port.to_s]
+    cmd += ["-U", user] if user
+    cmd += ["-f", dump_file]
+    cmd << db.name
+
+    system(*cmd) || abort("pg_dump failed")
+    puts "Database dumped to #{dump_file}"
+  end
+
+  desc "Restore database from dump file. DIR=path (default: .)"
+  task :restore do
+    require_relative "lib/fact_db"
+    db = FactDb.config.database
+
+    dumps_dir = ENV["DIR"] || Dir.pwd
+    pattern = File.join(dumps_dir, "fact_db_#{FactDb.config.environment}_*.dump")
+    dump_files = Dir.glob(pattern).sort
+    abort("No dump files found matching: #{pattern}") if dump_files.empty?
+
+    dump_file = if dump_files.size == 1
+      dump_files.first
+    else
+      puts "Available dump files:"
+      dump_files.each_with_index do |file, index|
+        puts "  #{index + 1}. #{File.basename(file)}"
+      end
+      print "\nSelect file (1-#{dump_files.size}): "
+      choice = $stdin.gets.to_i
+      abort("Invalid selection") unless choice.between?(1, dump_files.size)
+      dump_files[choice - 1]
+    end
+
+    puts "Environment: #{FactDb.config.environment}"
+    puts "Database: #{db.name}"
+    puts "Restoring from: #{dump_file}"
+
+    host = db.host || "localhost"
+    port = db.port || 5432
+    user = db.username || ENV["USER"]
+
+    cmd = ["pg_restore", "--no-owner", "--no-acl", "-d", db.name]
+    cmd += ["-h", host, "-p", port.to_s]
+    cmd += ["-U", user] if user
+    cmd << dump_file
+
+    system(*cmd) || abort("pg_restore failed")
+    puts "Database restored from #{dump_file}"
+  end
+
+  desc "Clean up invalid aliases (pronouns, generic terms). Use EXECUTE=1 to apply changes."
+  task :cleanup_aliases do
+    require_relative "lib/fact_db"
+    puts "Environment: #{FactDb.config.environment}"
+    puts "Database: #{FactDb.config.database.name}"
+    FactDb::Database.establish_connection!
+
+    dry_run = ENV["EXECUTE"] != "1"
+    stats = { checked: 0, removed: 0 }
+
+    puts dry_run ? "\n=== DRY RUN ===" : "\n=== EXECUTING ==="
+    puts
+
+    FactDb::Models::Entity.not_merged.find_each do |entity|
+      entity.aliases.each do |alias_record|
+        stats[:checked] += 1
+        next if FactDb::Validation::AliasFilter.valid?(alias_record.name, name: entity.name)
+
+        reason = FactDb::Validation::AliasFilter.rejection_reason(alias_record.name, name: entity.name)
+        puts "#{entity.name}: removing \"#{alias_record.name}\" (#{reason})"
+        alias_record.destroy unless dry_run
+        stats[:removed] += 1
+      end
+    end
+
+    puts "\nChecked: #{stats[:checked]}, Removed: #{stats[:removed]}"
+    puts "\nRun with EXECUTE=1 to apply changes." if dry_run && stats[:removed] > 0
+  end
+end
+
+namespace :docs do
+  desc "Build mkdocs documentation site"
+  task :mkdocs do
+    output_dir = File.expand_path("site", __dir__)
+    system("mkdocs", "build", "--clean") || abort("mkdocs build failed")
+    puts "MkDocs site built to #{output_dir}"
+  end
+
+  desc "Build YARD API documentation"
+  task :yard do
+    output_dir = File.expand_path("doc", __dir__)
+    system("yard", "doc") || abort("yard doc failed")
+
+    # Create symlink for README.md image path (docs/assets -> assets)
+    # README.md references docs/assets/fact_db.jpg which needs to resolve in YARD output
+    docs_dir = File.join(output_dir, "docs")
+    FileUtils.mkdir_p(docs_dir)
+    symlink_path = File.join(docs_dir, "assets")
+    FileUtils.rm_f(symlink_path)
+    FileUtils.ln_sf("../assets", symlink_path)
+
+    puts "YARD documentation built to #{output_dir}"
+  end
+
+  desc "Build all documentation (mkdocs and YARD)"
+  task all: [:mkdocs, :yard]
 end
 
 task default: :test

data/db/migrate/001_enable_extensions.rb

@@ -3,5 +3,6 @@
 class EnableExtensions < ActiveRecord::Migration[7.0]
   def change
     enable_extension "vector" unless extension_enabled?("vector")
+    enable_extension "pg_trgm" unless extension_enabled?("pg_trgm")
   end
 end

data/db/migrate/002_create_sources.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+class CreateSources < ActiveRecord::Migration[7.0]
+  def change
+    create_table :fact_db_sources, comment: "Stores immutable source content from which facts are extracted" do |t|
+      t.string :content_hash, null: false, limit: 64,
+               comment: "SHA-256 hash of content for deduplication and integrity verification"
+      t.string :kind, null: false, limit: 50,
+               comment: "Classification of content origin (e.g., email, document, webpage, transcript)"
+
+      t.text :content, null: false,
+             comment: "Original unmodified text content, preserved for audit and re-extraction"
+      t.string :title, limit: 500,
+               comment: "Human-readable title or subject line of the content"
+
+      t.text :source_uri,
+             comment: "URI identifying the original source location (URL, file path, message ID)"
+      t.jsonb :metadata, null: false, default: {},
+              comment: "Flexible metadata about the source (author, date, headers, etc.)"
+
+      t.vector :embedding, limit: 1536,
+               comment: "Vector embedding for semantic similarity search (OpenAI ada-002 compatible)"
+
+      t.timestamptz :captured_at, null: false,
+                    comment: "When the content was originally captured or received"
+      t.timestamps
+    end
+
+    add_index :fact_db_sources, :content_hash, unique: true
+    add_index :fact_db_sources, :captured_at
+    add_index :fact_db_sources, :kind
+    add_index :fact_db_sources, :metadata, using: :gin
+
+    # Full-text search index
+    execute <<-SQL
+      CREATE INDEX idx_sources_fulltext ON fact_db_sources
+      USING gin(to_tsvector('english', content));
+    SQL
+
+    # HNSW index for vector similarity search
+    execute <<-SQL
+      CREATE INDEX idx_sources_embedding ON fact_db_sources
+      USING hnsw (embedding vector_cosine_ops);
+    SQL
+
+    execute "COMMENT ON COLUMN fact_db_sources.created_at IS 'When this record was created in the database';"
+    execute "COMMENT ON COLUMN fact_db_sources.updated_at IS 'When this record was last modified';"
+  end
+end