fact_db 0.0.2 → 0.0.3

data/docs/guides/configuration.md

@@ -1,48 +1,89 @@
 # Configuration
 
-FactDb uses the `anyway_config` gem for flexible configuration via environment variables, YAML files, or Ruby code.
+FactDb uses the `anyway_config` gem for flexible configuration via environment variables, YAML files, or Ruby code. Configuration uses **nested sections** for better organization.
+
+## Configuration Sources
+
+Configuration is loaded from multiple sources (lowest to highest priority):
+
+1. **Bundled defaults** - `lib/fact_db/config/defaults.yml` (ships with gem)
+2. **XDG user config** - `~/.config/fact_db/fact_db.yml`
+3. **Project config** - `./config/fact_db.yml`
+4. **Local overrides** - `./config/fact_db.local.yml` (gitignored)
+5. **Environment variables** - `FDB_*`
+6. **Ruby configure block** - `FactDb.configure { |c| ... }`
+
+## Configuration Access Pattern
+
+FactDb uses nested configuration sections:
+
+```ruby
+# Nested access
+FactDb.config.database.url
+FactDb.config.database.pool_size
+FactDb.config.llm.provider
+FactDb.config.llm.model
+FactDb.config.ranking.ts_rank_weight
+```
 
 ## Configuration Methods
 
 ### Environment Variables
 
-All settings can be configured via environment variables with the `EVENT_CLOCK_` prefix:
+All settings use the `FDB_` prefix with double underscores for nested values:
 
 ```bash
-export EVENT_CLOCK_DATABASE_URL="postgresql://localhost/fact_db"
-export EVENT_CLOCK_DATABASE_POOL_SIZE=10
-export EVENT_CLOCK_LLM_PROVIDER="openai"
-export EVENT_CLOCK_LLM_MODEL="gpt-4o-mini"
-export EVENT_CLOCK_LLM_API_KEY="sk-..."
-export EVENT_CLOCK_FUZZY_MATCH_THRESHOLD=0.85
+# Database settings
+export FDB_DATABASE__URL="postgresql://localhost/fact_db"
+export FDB_DATABASE__POOL_SIZE=10
+export FDB_DATABASE__TIMEOUT=30000
+
+# LLM settings
+export FDB_LLM__PROVIDER="openai"
+export FDB_LLM__MODEL="gpt-4o-mini"
+export FDB_LLM__API_KEY="sk-..."
+
+# Top-level settings
+export FDB_FUZZY_MATCH_THRESHOLD=0.85
+export FDB_DEFAULT_EXTRACTOR="llm"
+export FDB_LOG_LEVEL="debug"
 ```
 
 ### YAML Configuration
 
-Create `config/fact_db.yml`:
+Create `config/fact_db.yml` with nested sections:
 
 ```yaml
 # Database
-database_url: postgresql://localhost/fact_db
-database_pool_size: 10
-database_timeout: 30000
+database:
+  url: postgresql://localhost/fact_db
+  pool_size: 10
+  timeout: 30000
 
 # Embeddings
-embedding_dimensions: 1536
+embedding:
+  dimensions: 1536
 
 # LLM
-llm_provider: openai
-llm_model: gpt-4o-mini
-llm_api_key: <%= ENV['OPENAI_API_KEY'] %>
-
-# Extraction
+llm:
+  provider: openai
+  model: gpt-4o-mini
+  api_key: <%= ENV['OPENAI_API_KEY'] %>
+
+# Ranking weights (should sum to 1.0)
+ranking:
+  ts_rank_weight: 0.25
+  vector_similarity_weight: 0.25
+  entity_mention_weight: 0.15
+  direct_answer_weight: 0.15
+  term_overlap_weight: 0.10
+  relationship_match_weight: 0.05
+  confidence_weight: 0.05
+
+# Top-level settings
 default_extractor: manual
-
-# Entity Resolution
 fuzzy_match_threshold: 0.85
 auto_merge_threshold: 0.95
-
-# Logging
 log_level: info
 ```
 
@@ -51,21 +92,21 @@ log_level: info
 ```ruby
 FactDb.configure do |config|
   # Database
-  config.database_url = "postgresql://localhost/fact_db"
-  config.database_pool_size = 10
-  config.database_timeout = 30_000
+  config.database.url = "postgresql://localhost/fact_db"
+  config.database.pool_size = 10
+  config.database.timeout = 30_000
 
   # Embeddings
-  config.embedding_dimensions = 1536
+  config.embedding.dimensions = 1536
   config.embedding_generator = ->(text) {
     # Your embedding generation logic
     OpenAI::Client.new.embeddings(input: text)
   }
 
-  # LLM
-  config.llm_provider = :openai
-  config.llm_model = "gpt-4o-mini"
-  config.llm_api_key = ENV['OPENAI_API_KEY']
+  # LLM (nested access)
+  config.llm.provider = :openai
+  config.llm.model = "gpt-4o-mini"
+  config.llm.api_key = ENV['OPENAI_API_KEY']
 
   # Or provide a pre-configured client
   config.llm_client = FactDb::LLM::Adapter.new(
@@ -73,10 +114,12 @@ FactDb.configure do |config|
     model: "claude-sonnet-4-20250514"
   )
 
-  # Extraction
-  config.default_extractor = :llm
+  # Ranking weights
+  config.ranking.ts_rank_weight = 0.30
+  config.ranking.vector_similarity_weight = 0.25
 
-  # Entity Resolution
+  # Top-level settings
+  config.default_extractor = :llm
   config.fuzzy_match_threshold = 0.85
   config.auto_merge_threshold = 0.95
 
@@ -90,46 +133,62 @@ end
 
 ### Database Settings
 
+Access: `FactDb.config.database.*`
+
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `database_url` | String | nil | PostgreSQL connection URL (required) |
-| `database_pool_size` | Integer | 5 | Connection pool size |
-| `database_timeout` | Integer | 30000 | Query timeout in milliseconds |
+| `url` | String | nil | PostgreSQL connection URL |
+| `host` | String | localhost | Database host |
+| `port` | Integer | 5432 | Database port |
+| `name` | String | nil | Database name |
+| `user` | String | nil | Database user |
+| `password` | String | nil | Database password |
+| `pool_size` | Integer | 5 | Connection pool size |
+| `timeout` | Integer | 30000 | Query timeout in milliseconds |
 
 ### Embedding Settings
 
+Access: `FactDb.config.embedding.*`
+
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `embedding_dimensions` | Integer | 1536 | Vector dimensions (match your model) |
-| `embedding_generator` | Proc | nil | Custom embedding generation function |
+| `dimensions` | Integer | 1536 | Vector dimensions (match your model) |
+| `generator` | Proc | nil | Custom embedding generation function |
 
 ### LLM Settings
 
+Access: `FactDb.config.llm.*`
+
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `llm_client` | Object | nil | Pre-configured LLM client |
-| `llm_provider` | Symbol | nil | Provider name (:openai, :anthropic, etc.) |
-| `llm_model` | String | varies | Model name |
-| `llm_api_key` | String | nil | API key |
+| `client` | Object | nil | Pre-configured LLM client |
+| `provider` | Symbol | nil | Provider name (:openai, :anthropic, etc.) |
+| `model` | String | varies | Model name |
+| `api_key` | String | nil | API key |
+
+### Ranking Settings
 
-### Extraction Settings
+Access: `FactDb.config.ranking.*`
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `default_extractor` | Symbol | :manual | Default extraction method |
+| `ts_rank_weight` | Float | 0.25 | PostgreSQL full-text search weight |
+| `vector_similarity_weight` | Float | 0.25 | Semantic similarity weight |
+| `entity_mention_weight` | Float | 0.15 | Entity mentions weight |
+| `direct_answer_weight` | Float | 0.15 | Direct answer pattern weight |
+| `term_overlap_weight` | Float | 0.10 | Query word matches weight |
+| `relationship_match_weight` | Float | 0.05 | Relationship words weight |
+| `confidence_weight` | Float | 0.05 | Stored confidence score weight |
+
+**Note:** Weights should sum to approximately 1.0.
 
-### Resolution Settings
+### Top-Level Settings
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
+| `default_extractor` | Symbol | :manual | Default extraction method |
 | `fuzzy_match_threshold` | Float | 0.85 | Minimum similarity for fuzzy matching |
 | `auto_merge_threshold` | Float | 0.95 | Similarity threshold for auto-merge |
-
-### Logging Settings
-
-| Option | Type | Default | Description |
-|--------|------|---------|-------------|
-| `logger` | Logger | STDOUT | Logger instance |
 | `log_level` | Symbol | :info | Log level |
 
 ## LLM Provider Configuration
@@ -138,9 +197,9 @@ end
 
 ```ruby
 FactDb.configure do |config|
-  config.llm_provider = :openai
-  config.llm_model = "gpt-4o-mini"  # or "gpt-4o", "gpt-4-turbo"
-  config.llm_api_key = ENV['OPENAI_API_KEY']
+  config.llm.provider = :openai
+  config.llm.model = "gpt-4o-mini"  # or "gpt-4o", "gpt-4-turbo"
+  config.llm.api_key = ENV['OPENAI_API_KEY']
 end
 ```
 
@@ -148,9 +207,9 @@ end
 
 ```ruby
 FactDb.configure do |config|
-  config.llm_provider = :anthropic
-  config.llm_model = "claude-sonnet-4-20250514"
-  config.llm_api_key = ENV['ANTHROPIC_API_KEY']
+  config.llm.provider = :anthropic
+  config.llm.model = "claude-sonnet-4-20250514"
+  config.llm.api_key = ENV['ANTHROPIC_API_KEY']
 end
 ```
 
@@ -158,9 +217,9 @@ end
 
 ```ruby
 FactDb.configure do |config|
-  config.llm_provider = :gemini
-  config.llm_model = "gemini-2.0-flash"
-  config.llm_api_key = ENV['GEMINI_API_KEY']
+  config.llm.provider = :gemini
+  config.llm.model = "gemini-2.0-flash"
+  config.llm.api_key = ENV['GEMINI_API_KEY']
 end
 ```
 
@@ -168,8 +227,8 @@ end
 
 ```ruby
 FactDb.configure do |config|
-  config.llm_provider = :ollama
-  config.llm_model = "llama3.2"
+  config.llm.provider = :ollama
+  config.llm.model = "llama3.2"
   # No API key needed for local Ollama
 end
 ```
@@ -178,8 +237,8 @@ end
 
 ```ruby
 FactDb.configure do |config|
-  config.llm_provider = :bedrock
-  config.llm_model = "claude-sonnet-4"
+  config.llm.provider = :bedrock
+  config.llm.model = "claude-sonnet-4"
   # Uses AWS credentials from environment
 end
 ```
@@ -188,45 +247,58 @@ end
 
 ```ruby
 FactDb.configure do |config|
-  config.llm_provider = :openrouter
-  config.llm_model = "anthropic/claude-sonnet-4"
-  config.llm_api_key = ENV['OPENROUTER_API_KEY']
+  config.llm.provider = :openrouter
+  config.llm.model = "anthropic/claude-sonnet-4"
+  config.llm.api_key = ENV['OPENROUTER_API_KEY']
 end
 ```
 
+## XDG User Configuration
+
+FactDb supports XDG Base Directory Specification for user-level configuration:
+
+- `~/.config/fact_db/fact_db.yml` (Linux/macOS)
+- `~/Library/Application Support/fact_db/fact_db.yml` (macOS)
+- `$XDG_CONFIG_HOME/fact_db/fact_db.yml` (if XDG_CONFIG_HOME is set)
+
+This allows you to set personal defaults that apply across all projects.
+
 ## Environment-Specific Configuration
 
-Use YAML anchors for shared settings:
+The bundled defaults support environment-specific overrides:
 
 ```yaml
 # config/fact_db.yml
-defaults: &defaults
-  embedding_dimensions: 1536
+defaults:
+  embedding:
+    dimensions: 1536
   fuzzy_match_threshold: 0.85
 
 development:
-  <<: *defaults
-  database_url: postgresql://localhost/fact_db_dev
+  database:
+    name: fact_db_development
   log_level: debug
 
 test:
-  <<: *defaults
-  database_url: postgresql://localhost/fact_db_test
+  database:
+    name: fact_db_test
   log_level: warn
 
 production:
-  <<: *defaults
-  database_url: <%= ENV['DATABASE_URL'] %>
+  database:
+    pool_size: 25
   log_level: info
 ```
 
+Environment is detected from: `FDB_ENV` > `RAILS_ENV` > `RACK_ENV` > `'development'`
+
 ## Validation
 
 Validate configuration at startup:
 
 ```ruby
 FactDb.configure do |config|
-  config.database_url = ENV['DATABASE_URL']
+  config.database.url = ENV['DATABASE_URL']
 end
 
 # Raises ConfigurationError if invalid
@@ -241,3 +313,12 @@ For testing, reset configuration between tests:
 # In test setup
 FactDb.reset_configuration!
 ```
+
+## Environment Helpers
+
+```ruby
+FactDb.config.test?        # true if FDB_ENV == 'test'
+FactDb.config.development? # true if FDB_ENV == 'development'
+FactDb.config.production?  # true if FDB_ENV == 'production'
+FactDb.config.environment  # returns current environment string
+```

data/docs/guides/entity-management.md

@@ -85,7 +85,7 @@ facts.entity_service.add_alias(
 
 ```ruby
 entity.entity_aliases.each do |alias_record|
-  puts "#{alias_record.alias_text} (#{alias_record.alias_type})"
+  puts "#{alias_record.name} (#{alias_record.type})"
   puts "  Confidence: #{alias_record.confidence}"
 end
 ```
@@ -142,7 +142,7 @@ results = facts.batch_resolve_entities(names)
 results.each do |result|
   status = result[:status]  # :resolved, :not_found, :error
   entity = result[:entity]
-  puts "#{result[:name]}: #{status} -> #{entity&.canonical_name}"
+  puts "#{result[:name]}: #{status} -> #{entity&.name}"
 end
 ```
 
@@ -157,13 +157,13 @@ facts.entity_service.merge(entity1.id, entity2.id)
 # After merge:
 entity2.reload
 entity2.resolution_status  # => "merged"
-entity2.merged_into_id     # => entity1.id
+entity2.canonical_id     # => entity1.id
 ```
 
 ### What Happens on Merge
 
 1. Entity2's status changes to "merged"
-2. Entity2 points to entity1 via `merged_into_id`
+2. Entity2 points to entity1 via `canonical_id`
 3. Entity2's aliases are copied to entity1
 4. All facts mentioning entity2 now also reference entity1
 
@@ -184,7 +184,7 @@ end
 ```ruby
 facts.entity_service.update(
   entity.id,
-  canonical_name: "Paula M. Chen"
+  name: "Paula M. Chen"
 )
 ```
 
@@ -203,7 +203,7 @@ facts.entity_service.update(
 # Reclassify entity type
 facts.entity_service.update(
   entity.id,
-  entity_type: :organization
+  type: :organization
 )
 ```
 
@@ -250,15 +250,15 @@ entities = facts.entity_service.search("Paula")
 
 ```ruby
 people = FactDb::Models::Entity
-  .where(entity_type: 'person')
+  .where(type: 'person')
   .where.not(resolution_status: 'merged')
 ```
 
-### Find Entities in Content
+### Find Entities in Source
 
 ```ruby
-# Find all entities mentioned in a content
-entities = facts.entity_service.in_content(content.id)
+# Find all entities mentioned in a source
+entities = facts.entity_service.in_source(source.id)
 ```
 
 ### Find Related Entities
@@ -327,9 +327,9 @@ unresolved = FactDb::Models::Entity
 
 unresolved.each do |entity|
   # Try to find duplicates
-  similar = facts.entity_service.search(entity.canonical_name)
+  similar = facts.entity_service.search(entity.name)
   if similar.count > 1
-    puts "Potential duplicate: #{entity.canonical_name}"
+    puts "Potential duplicate: #{entity.name}"
   end
 end
 ```
@@ -343,8 +343,8 @@ active_entities = FactDb::Models::Entity
 
 # Or follow the merge chain
 def canonical_entity(entity)
-  while entity.merged_into_id
-    entity = FactDb::Models::Entity.find(entity.merged_into_id)
+  while entity.canonical_id
+    entity = FactDb::Models::Entity.find(entity.canonical_id)
   end
   entity
 end

data/docs/guides/extracting-facts.md

@@ -24,7 +24,7 @@ fact = facts.fact_service.create(
     { entity: microsoft, role: "organization", text: "Microsoft" }
   ],
   sources: [
-    { content: content, type: "primary", excerpt: "...accepted the offer..." }
+    { source: source, type: "primary", excerpt: "...accepted the offer..." }
   ]
 )
 ```
@@ -36,19 +36,19 @@ Use AI to automatically extract facts:
 ```ruby
 # Configure LLM
 FactDb.configure do |config|
-  config.llm_provider = :openai
-  config.llm_api_key = ENV['OPENAI_API_KEY']
+  config.llm.provider = :openai
+  config.llm.api_key = ENV['OPENAI_API_KEY']
 end
 
 facts = FactDb.new
 
-# Extract facts from content
-extracted = facts.extract_facts(content.id, extractor: :llm)
+# Extract facts from source
+extracted = facts.extract_facts(source.id, extractor: :llm)
 
 extracted.each do |fact|
-  puts fact.fact_text
+  puts fact.text
   puts "  Valid from: #{fact.valid_at}"
-  puts "  Entities: #{fact.entity_mentions.map(&:entity).map(&:canonical_name)}"
+  puts "  Entities: #{fact.entity_mentions.map(&:entity).map(&:name)}"
 end
 ```
 
@@ -57,7 +57,7 @@ end
 Use regex patterns for structured content:
 
 ```ruby
-extracted = facts.extract_facts(content.id, extractor: :rule_based)
+extracted = facts.extract_facts(source.id, extractor: :rule_based)
 ```
 
 The rule-based extractor includes patterns for:
@@ -76,7 +76,7 @@ FactDb.configure do |config|
 end
 
 # Uses configured default
-extracted = facts.extract_facts(content.id)
+extracted = facts.extract_facts(source.id)
 ```
 
 ## Fact Structure
@@ -85,8 +85,8 @@ Every extracted fact includes:
 
 ```ruby
 fact = Models::Fact.new(
-  fact_text: "Paula Chen is Principal Engineer at Microsoft",
-  fact_hash: "sha256...",           # For deduplication
+  text: "Paula Chen is Principal Engineer at Microsoft",
+  digest: "sha256...",           # For deduplication
   valid_at: Time.parse("2024-01-10"),
   invalid_at: nil,                   # nil = currently valid
   status: "canonical",               # canonical, superseded, corroborated, synthesized
@@ -127,7 +127,7 @@ Facts link to source content:
 
 ```ruby
 fact.add_source(
-  content: email_content,
+  source: email_source,
   type: "primary",
   excerpt: "Paula has accepted our offer to join as Principal Engineer...",
   confidence: 0.95
@@ -147,16 +147,16 @@ fact.add_source(
 Process multiple content items:
 
 ```ruby
-content_ids = [content1.id, content2.id, content3.id]
+source_ids = [source1.id, source2.id, source3.id]
 
 # Sequential processing
-results = facts.batch_extract(content_ids, parallel: false)
+results = facts.batch_extract(source_ids, parallel: false)
 
 # Parallel processing (default)
-results = facts.batch_extract(content_ids, parallel: true)
+results = facts.batch_extract(source_ids, parallel: true)
 
 results.each do |result|
-  puts "Content #{result[:content_id]}:"
+  puts "Source #{result[:source_id]}:"
   puts "  Facts: #{result[:facts].count}"
   puts "  Error: #{result[:error]}" if result[:error]
 end
@@ -168,11 +168,11 @@ Create custom extractors by extending the base class:
 
 ```ruby
 class MyExtractor < FactDb::Extractors::Base
-  def extract(content)
+  def extract(source)
     extracted = []
 
     # Your extraction logic here
-    # Parse content.raw_text
+    # Parse source.content
     # Create fact records
 
     extracted
@@ -180,8 +180,8 @@ class MyExtractor < FactDb::Extractors::Base
 end
 
 # Register and use
-facts.fact_service.extract_from_content(
-  content.id,
+facts.fact_service.extract_from_source(
+  source.id,
   extractor: MyExtractor.new(config)
 )
 ```
@@ -217,7 +217,7 @@ After extraction, you may want to:
 ### Resolve Entities
 
 ```ruby
-extracted = facts.extract_facts(content.id, extractor: :llm)
+extracted = facts.extract_facts(source.id, extractor: :llm)
 
 extracted.each do |fact|
   fact.entity_mentions.each do |mention|
@@ -239,8 +239,8 @@ conflicts = facts.fact_service.resolver.find_conflicts(
 
 conflicts.each do |conflict|
   puts "Conflict between:"
-  puts "  #{conflict[:fact1].fact_text}"
-  puts "  #{conflict[:fact2].fact_text}"
+  puts "  #{conflict[:fact1].text}"
+  puts "  #{conflict[:fact2].text}"
 end
 ```
 
@@ -248,7 +248,7 @@ end
 
 ```ruby
 # If multiple sources say the same thing
-if fact1.fact_text.similar_to?(fact2.fact_text)
+if fact1.text.similar_to?(fact2.text)
   facts.fact_service.resolver.corroborate(fact1.id, fact2.id)
 end
 ```
@@ -258,7 +258,7 @@ end
 ### 1. Review LLM Extractions
 
 ```ruby
-extracted = facts.extract_facts(content.id, extractor: :llm)
+extracted = facts.extract_facts(source.id, extractor: :llm)
 
 extracted.select { |f| f.confidence < 0.8 }.each do |fact|
   # Flag for human review
@@ -282,7 +282,7 @@ end
 fact = facts.fact_service.create(
   "Important fact",
   valid_at: Date.today,
-  sources: [{ content: source_content, type: "primary" }]
+  sources: [{ source: source_record, type: "primary" }]
 )
 ```
 
@@ -290,10 +290,10 @@ fact = facts.fact_service.create(
 
 ```ruby
 begin
-  extracted = facts.extract_facts(content.id, extractor: :llm)
+  extracted = facts.extract_facts(source.id, extractor: :llm)
 rescue FactDb::ExtractionError => e
   logger.error "Extraction failed: #{e.message}"
   # Fall back to manual or rule-based
-  extracted = facts.extract_facts(content.id, extractor: :rule_based)
+  extracted = facts.extract_facts(source.id, extractor: :rule_based)
 end
 ```