smart_rag 0.1.0 → 0.2.0

data/config/llm_config.yml

@@ -1,6 +1,5 @@
 adapters:
   openai: OpenAIAdapter
-logger_file: ./log/prompt.log
 llms:
   OllamaEmbedding:
     adapter: openai
@@ -10,6 +9,9 @@ llms:
   SiliconFlow:
     adapter: openai
     url: https://api.siliconflow.cn/v1/
-    api_key: ""
+    api_key: ENV["SILICONFLOW_API_KEY"]
+# Worker files are loaded by SmartRAG services directly from the gem's
+# workers/ directory. These paths are only fallbacks for standalone
+# SmartPrompt usage and must stay non-empty.
 worker_path: "./workers"
 template_path: "./templates"

data/config/smart_rag.yml

@@ -29,7 +29,7 @@ embedding:
   api_key: <%= ENV['EMBEDDING_API_KEY'] || 'ollama-local' %>
   endpoint: <%= ENV['EMBEDDING_ENDPOINT'] || 'http://localhost:11434/v1/embeddings' %>
   model: <%= ENV['EMBEDDING_MODEL'] || 'qwen3-embedding' %>
-  dimensions: <%= ENV['EMBEDDING_DIMENSIONS'] || 1024 %>
+  dimensions: <%= ENV['EMBEDDING_DIMENSIONS'] || 4096 %>
   # Request timeout in seconds
   timeout: 30
   # Batch size for embedding generation
@@ -130,6 +130,50 @@ document:
   # Download timeout for URLs
   download_timeout: <%= ENV['DOWNLOAD_TIMEOUT'] || 30 %>
 
+# Media ingestion, extraction, storage, and asynchronous jobs
+media:
+  max_file_size_mb: <%= ENV['MEDIA_MAX_FILE_SIZE_MB'] || 50 %>
+  max_duration_ms: <%= ENV['MEDIA_MAX_DURATION_MS'] || 7200000 %>
+  command_timeout_seconds: <%= ENV['MEDIA_COMMAND_TIMEOUT'] || 120 %>
+  allow_private_urls: <%= ENV['MEDIA_ALLOW_PRIVATE_URLS'] == 'true' %>
+  allowed_hosts: <%= ENV.fetch('MEDIA_ALLOWED_HOSTS', '').split(',').map(&:strip).reject(&:empty?) %>
+  content_store:
+    enabled: <%= ENV['MEDIA_CONTENT_STORE_ENABLED'] == 'true' %>
+    provider: <%= ENV['MEDIA_CONTENT_STORE_PROVIDER'] || 'local' %>
+    root: <%= ENV['MEDIA_CONTENT_STORE_ROOT'] || File.expand_path('../storage/media', __dir__) %>
+    bucket: <%= ENV['MEDIA_S3_BUCKET'] %>
+    region: <%= ENV['MEDIA_S3_REGION'] || 'us-east-1' %>
+    prefix: <%= ENV['MEDIA_S3_PREFIX'] || 'smart-rag/media' %>
+    endpoint: <%= ENV['MEDIA_S3_ENDPOINT'] %>
+    force_path_style: <%= ENV['MEDIA_S3_FORCE_PATH_STYLE'] == 'true' %>
+    access_key_id: <%= ENV['MEDIA_S3_ACCESS_KEY_ID'] %>
+    secret_access_key: <%= ENV['MEDIA_S3_SECRET_ACCESS_KEY'] %>
+  openai:
+    base_url: <%= ENV['MEDIA_OPENAI_BASE_URL'] || 'https://api.openai.com' %>
+    api_key: <%= ENV['MEDIA_OPENAI_API_KEY'] || ENV['OPENAI_API_KEY'] %>
+    vision_model: <%= ENV['MEDIA_VISION_MODEL'] %>
+    transcription_model: <%= ENV['MEDIA_TRANSCRIPTION_MODEL'] %>
+    timeout_seconds: <%= ENV['MEDIA_MODEL_TIMEOUT'] || 120 %>
+    language: <%= ENV['MEDIA_TRANSCRIPTION_LANGUAGE'] %>
+  ocr:
+    provider: <%= ENV['MEDIA_OCR_PROVIDER'] %>
+    language: <%= ENV['MEDIA_OCR_LANGUAGE'] %>
+    timeout_seconds: <%= ENV['MEDIA_OCR_TIMEOUT'] || 60 %>
+  async:
+    max_attempts: <%= ENV['MEDIA_JOB_MAX_ATTEMPTS'] || 3 %>
+    stale_after_seconds: <%= ENV['MEDIA_JOB_STALE_AFTER_SECONDS'] || 900 %>
+    retention_seconds: <%= ENV['MEDIA_JOB_RETENTION_SECONDS'] || 604800 %>
+    heartbeat_interval_seconds: <%= ENV['MEDIA_JOB_HEARTBEAT_INTERVAL_SECONDS'] || 30 %>
+
+# Optional HTTP Bearer authentication and shared PostgreSQL quotas.
+http_auth:
+  enabled: <%= ENV['SMARTRAG_HTTP_AUTH_ENABLED'] == 'true' %>
+  principal: <%= ENV['SMARTRAG_HTTP_PRINCIPAL'] || 'default' %>
+  token: <%= ENV['SMARTRAG_HTTP_TOKEN'] %>
+  quota:
+    requests_per_minute: <%= ENV['SMARTRAG_REQUESTS_PER_MINUTE'] || 0 %>
+    upload_bytes_per_day: <%= ENV['SMARTRAG_UPLOAD_BYTES_PER_DAY'] || 0 %>
+
 # Logging Configuration
 logging:
   # Log level: debug, info, warn, error, fatal

data/config.ru

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path('lib', __dir__))
+
+require 'smart_rag'
+require 'smart_rag/http_app'
+require 'smart_rag/http_access_policy'
+
+config_path = ENV['SMARTRAG_CONFIG_PATH']
+rag = SmartRAG::SmartRAG.new(SmartRAG::Config.load(config_path))
+access_policy = SmartRAG::HttpAccessPolicy.new(db: SmartRAG.db, config: rag.config[:http_auth] || {})
+
+# Applications can replace this file and inject server-side OCR, vision, and
+# transcription adapters through HttpApp's extractors: option.
+run SmartRAG::HttpApp.new(rag: rag, access_policy: access_policy)

data/db/migrations/006_create_text_search_configs.rb

@@ -10,11 +10,12 @@ Sequel.migration do
     add_index :text_search_configs, :config_name
 
     # Seed initial language configurations
-    # Note: These assume pg_jieba extension is installed for Chinese support
+    # Note: Chinese uses pg_catalog.simple when pg_jieba is not installed.
+    # Set 'zh' to 'jiebacfg' after installing the pg_jieba extension for real segmentation.
     from(:text_search_configs).multi_insert(
       [
         { language_code: 'en', config_name: 'pg_catalog.english' },
-        { language_code: 'zh', config_name: 'jiebacfg' },
+        { language_code: 'zh', config_name: 'pg_catalog.simple' },
         { language_code: 'ja', config_name: 'pg_catalog.simple' },
         { language_code: 'ko', config_name: 'pg_catalog.simple' },
         { language_code: 'default', config_name: 'pg_catalog.simple' }

data/db/migrations/008_create_embeddings.rb

@@ -7,13 +7,14 @@ Sequel.migration do
       primary_key :id
       foreign_key :source_id, :source_sections, null: false, on_delete: :cascade
       # Vector dimension size (adjust based on embedding model)
-      column :vector, 'vector(1024)', null: false
+      column :vector, 'vector(4096)', null: false
       DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
     end
 
-    # Create IVFFLAT index for approximate nearest neighbor search
-    # IVFFLAT provides good accuracy with fast search
-    run 'CREATE INDEX idx_embedding_vector ON embeddings USING ivfflat (vector vector_cosine_ops) WITH (lists = 100)'
+    # NOTE: no vector index is created here because pgvector's ivfflat/hnsw
+    # indexes only support up to 2000 dimensions, while qwen3-embedding
+    # produces 4096-dimension vectors. Similarity search falls back to exact
+    # (sequential) scan, which is fine for small to medium datasets.
 
     # Composite index for source_id lookups
     add_index :embeddings, :source_id

data/db/migrations/012_add_metadata_to_source_sections.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  up do
+    add_column :source_sections, :metadata, :jsonb, default: '{}', null: false
+    add_index :source_sections, :metadata, type: :gin
+  end
+
+  down do
+    drop_index :source_sections, :metadata
+    drop_column :source_sections, :metadata
+  end
+end

data/db/migrations/013_create_media_jobs.rb

@@ -0,0 +1,25 @@
+Sequel.migration do
+  up do
+    create_table :media_jobs do
+      primary_key :id
+      String :operation, null: false, size: 32
+      String :source, text: true, null: false
+      column :options, :jsonb, default: '{}', null: false
+      String :status, null: false, size: 20, default: 'queued'
+      Integer :attempts, null: false, default: 0
+      Integer :max_attempts, null: false, default: 3
+      column :result, :jsonb
+      String :error, text: true
+      DateTime :available_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :started_at
+      DateTime :finished_at
+      DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+      index [:status, :available_at]
+    end
+  end
+
+  down do
+    drop_table :media_jobs
+  end
+end

data/db/migrations/014_add_media_job_operations_indexes.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  up do
+    add_index :media_jobs, [:status, :started_at]
+    add_index :media_jobs, [:status, :finished_at]
+  end
+
+  down do
+    drop_index :media_jobs, [:status, :finished_at]
+    drop_index :media_jobs, [:status, :started_at]
+  end
+end

data/db/migrations/015_add_media_leases_and_objects.rb

@@ -0,0 +1,80 @@
+Sequel.migration do
+  up do
+    alter_table :media_jobs do
+      add_column :idempotency_key, String, size: 128
+      add_column :principal, String, size: 128, null: false, default: 'system'
+      add_column :lease_token, String, size: 64
+      add_column :heartbeat_at, DateTime
+      add_index [:principal, :idempotency_key], unique: true
+      add_index [:status, :heartbeat_at]
+    end
+
+    create_table :media_objects do
+      primary_key :id
+      String :content_hash, null: false, size: 128, unique: true
+      String :storage_uri, text: true, null: false
+      Integer :byte_size, null: false
+      Integer :reference_count, null: false, default: 0
+      DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+      index :reference_count
+    end
+
+    create_table :media_object_references do
+      primary_key :id
+      foreign_key :media_object_id, :media_objects, null: false, on_delete: :cascade
+      foreign_key :document_id, :source_documents, null: false, on_delete: :cascade
+      DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+      index [:media_object_id, :document_id], unique: true, name: :media_object_document_unique
+      index :document_id
+    end
+
+
+    create_table :api_rate_limits do
+      primary_key :id
+      String :principal, null: false, size: 128
+      String :period, null: false, size: 16
+      DateTime :window_start, null: false
+      Integer :request_count, null: false, default: 0
+      Bignum :byte_count, null: false, default: 0
+      DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+      index [:principal, :period, :window_start], unique: true, name: :api_rate_limit_window_unique
+    end
+
+    run <<~SQL
+      INSERT INTO media_objects (content_hash, storage_uri, byte_size, reference_count)
+      SELECT metadata->>'content_hash', metadata->>'storage_uri',
+             COALESCE((metadata->>'file_size')::bigint, 0), 0
+      FROM source_documents
+      WHERE metadata ? 'storage_uri' AND metadata ? 'content_hash'
+      ON CONFLICT (content_hash) DO NOTHING
+    SQL
+    run <<~SQL
+      INSERT INTO media_object_references (media_object_id, document_id)
+      SELECT mo.id, sd.id
+      FROM source_documents sd
+      JOIN media_objects mo ON mo.content_hash = sd.metadata->>'content_hash'
+      WHERE sd.metadata ? 'storage_uri'
+      ON CONFLICT DO NOTHING
+    SQL
+    run <<~SQL
+      UPDATE media_objects mo SET reference_count = refs.count
+      FROM (SELECT media_object_id, COUNT(*) AS count FROM media_object_references GROUP BY media_object_id) refs
+      WHERE refs.media_object_id = mo.id
+    SQL
+  end
+
+  down do
+    drop_table :api_rate_limits
+    drop_table :media_object_references
+    drop_table :media_objects
+    alter_table :media_jobs do
+      drop_index [:status, :heartbeat_at]
+      drop_index [:principal, :idempotency_key]
+      drop_column :heartbeat_at
+      drop_column :lease_token
+      drop_column :idempotency_key
+      drop_column :principal
+    end
+  end
+end

data/db/migrations/016_add_document_principals_and_staging_references.rb

@@ -0,0 +1,38 @@
+Sequel.migration do
+  up do
+    alter_table :source_documents do
+      add_column :principal, String, size: 128, null: false, default: 'system'
+      add_index :principal
+      add_index [:principal, :created_at]
+    end
+
+    alter_table :media_jobs do
+      add_foreign_key :staging_media_object_id, :media_objects, on_delete: :set_null
+      add_index :staging_media_object_id
+    end
+
+    run <<~SQL
+      UPDATE source_documents
+      SET principal = COALESCE(NULLIF(metadata->>'principal', ''), 'system')
+    SQL
+    run <<~SQL
+      UPDATE media_jobs mj
+      SET staging_media_object_id = mo.id
+      FROM media_objects mo
+      WHERE mj.source = mo.storage_uri
+        AND mj.staging_media_object_id IS NULL
+    SQL
+  end
+
+  down do
+    alter_table :media_jobs do
+      drop_index :staging_media_object_id
+      drop_column :staging_media_object_id
+    end
+    alter_table :source_documents do
+      drop_index [:principal, :created_at]
+      drop_index :principal
+      drop_column :principal
+    end
+  end
+end

data/db/migrations/017_add_media_job_request_fingerprint.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'json'
+
+Sequel.migration do
+  up do
+    alter_table :media_jobs do
+      add_column :request_fingerprint, String, size: 64
+    end
+
+    self[:media_jobs].select(:id, :operation, :source, :options).each do |job|
+      options = begin
+        JSON.parse(job[:options].to_s)
+      rescue JSON::ParserError
+        job[:options].to_s
+      end
+      canonicalize = lambda do |value|
+        case value
+        when Hash
+          value.keys.map(&:to_s).sort.to_h do |key|
+            original_key = value.key?(key) ? key : value.keys.find { |candidate| candidate.to_s == key }
+            [key, canonicalize.call(value[original_key])]
+          end
+        when Array
+          value.map { |item| canonicalize.call(item) }
+        else
+          value
+        end
+      end
+      payload = { 'operation' => job[:operation].to_s, 'source' => job[:source].to_s,
+                  'options' => canonicalize.call(options) }
+      self[:media_jobs].where(id: job[:id]).update(
+        request_fingerprint: Digest::SHA256.hexdigest(JSON.generate(payload))
+      )
+    end
+
+    alter_table :media_jobs do
+      set_column_not_null :request_fingerprint
+    end
+  end
+
+  down do
+    alter_table :media_jobs do
+      drop_column :request_fingerprint
+    end
+  end
+end

data/db/seeds/text_search_configs.sql

@@ -9,9 +9,9 @@ INSERT INTO text_search_configs (language_code, config_name, is_installed) VALUE
 ('en', 'pg_catalog.english', true),
 ('en_us', 'pg_catalog.english', true),
 ('en_gb', 'pg_catalog.english', true),
-('zh', 'jiebacfg', true),
-('zh_cn', 'jiebacfg', true),
-('zh_tw', 'jiebacfg', true),
+('zh', 'pg_catalog.simple', true),
+('zh_cn', 'pg_catalog.simple', true),
+('zh_tw', 'pg_catalog.simple', true),
 ('ja', 'pg_catalog.simple', true),
 ('ja_jp', 'pg_catalog.simple', true),
 ('ko', 'pg_catalog.simple', true),