RubyGems - smart_rag - Versions diffs - 0.1.0 → 0.2.1 - Mend

smart_rag 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (90) hide show

checksums.yaml +4 -4
data/.env.example +252 -0
data/.rspec +2 -0
data/AGENTS.md +33 -0
data/API_DOCUMENTATION.md +828 -0
data/CHANGELOG.md +16 -1
data/ER-diagram.mmd +144 -0
data/Gemfile +50 -0
data/Gemfile.lock +398 -0
data/Hybrid_Reranking.md +171 -0
data/README.en.md +420 -28
data/README.md +534 -63
data/Rakefile +268 -0
data/SETUP_GUIDE.md +650 -0
data/SmartChunking.md +180 -0
data/USAGE_EXAMPLES.md +1002 -0
data/config/llm_config.yml +4 -2
data/config/smart_rag.yml +45 -1
data/config.ru +15 -0
data/db/migrations/006_create_text_search_configs.rb +3 -2
data/db/migrations/008_create_embeddings.rb +5 -4
data/db/migrations/012_add_metadata_to_source_sections.rb +11 -0
data/db/migrations/013_create_media_jobs.rb +25 -0
data/db/migrations/014_add_media_job_operations_indexes.rb +11 -0
data/db/migrations/015_add_media_leases_and_objects.rb +80 -0
data/db/migrations/016_add_document_principals_and_staging_references.rb +38 -0
data/db/migrations/017_add_media_job_request_fingerprint.rb +48 -0
data/db/seeds/text_search_configs.sql +3 -3
data/design.md +1057 -0
data/docs/API_DOCUMENTATION.md +838 -0
data/docs/DOCUMENTATION_INDEX.en.md +60 -0
data/docs/DOCUMENTATION_INDEX.md +65 -0
data/docs/FIX_SUMMARY.md +256 -0
data/docs/FIX_SUMMARY_COMPLETE.md +273 -0
data/docs/Hybrid_Reranking.md +171 -0
data/docs/MIGRATION_GUIDE.md +151 -0
data/docs/PERFORMANCE_GUIDE.md +58 -0
data/docs/SETUP_GUIDE.md +659 -0
data/docs/SmartChunking.md +180 -0
data/docs/USAGE_EXAMPLES.md +1008 -0
data/docs/design.md +1057 -0
data/docs/evidence_pack.md +211 -0
data/docs/requirements.md +376 -0
data/docs/retrieval_plan.md +251 -0
data/docs/smartrag_improvement_plan.md +201 -0
data/docs/smartrag_refactor.md +216 -0
data/docs/todo.md +931 -0
data/examples/common.rb +1 -1
data/exe/smart-rag-db +163 -0
data/exe/smart-rag-media-worker +34 -0
data/lib/smart_rag/config.rb +12 -0
data/lib/smart_rag/core/document_processor.rb +80 -16
data/lib/smart_rag/core/local_content_store.rb +51 -0
data/lib/smart_rag/core/media_extractors.rb +140 -0
data/lib/smart_rag/core/media_job_queue.rb +353 -0
data/lib/smart_rag/core/media_metadata_extractor.rb +188 -0
data/lib/smart_rag/core/media_object_registry.rb +79 -0
data/lib/smart_rag/core/media_processor.rb +228 -0
data/lib/smart_rag/core/media_safety_policy.rb +61 -0
data/lib/smart_rag/core/s3_content_store.rb +78 -0
data/lib/smart_rag/core/transcript_normalizer.rb +44 -0
data/lib/smart_rag/core/video_semantic_extractor.rb +130 -0
data/lib/smart_rag/http_access_policy.rb +86 -0
data/lib/smart_rag/http_app.rb +188 -0
data/lib/smart_rag/models/embedding.rb +1 -1
data/lib/smart_rag/models/research_topic.rb +1 -1
data/lib/smart_rag/models/research_topic_section.rb +5 -0
data/lib/smart_rag/models/research_topic_tag.rb +5 -0
data/lib/smart_rag/models/search_log.rb +1 -1
data/lib/smart_rag/models/section_fts.rb +5 -0
data/lib/smart_rag/models/section_tag.rb +5 -0
data/lib/smart_rag/models/source_document.rb +1 -1
data/lib/smart_rag/models/source_section.rb +1 -1
data/lib/smart_rag/models/tag.rb +1 -1
data/lib/smart_rag/models/text_search_config.rb +5 -0
data/lib/smart_rag/retrieve.rb +72 -1
data/lib/smart_rag/services/embedding_service.rb +1 -1
data/lib/smart_rag/services/fulltext_search_service.rb +11 -13
data/lib/smart_rag/services/hybrid_search_service.rb +15 -11
data/lib/smart_rag/services/summarization_service.rb +1 -1
data/lib/smart_rag/services/tag_service.rb +1 -1
data/lib/smart_rag/version.rb +1 -1
data/lib/smart_rag.rb +264 -30
data/patch_language.rb +27 -0
data/requirements.md +376 -0
data/source_documents_export.json +11072 -0
data/todo.md +931 -0
data/workers/analyze_content.rb +6 -2
data/workers/get_embedding.rb +1 -1
metadata +151 -38

data/workers/analyze_content.rb CHANGED Viewed

@@ -1,6 +1,10 @@
 SmartPrompt.define_worker :analyze_content do
-  use "SiliconFlow"
-  model "Qwen/Qwen3-8B"
+  # Use local Ollama by default for content analysis/tagging.
+  # To use SiliconFlow instead, replace with:
+  #   use "SiliconFlow"
+  #   model "Qwen/Qwen3-8B"
+  use "OllamaEmbedding"
+  model ENV["LLM_MODEL"] || "qwen3"
   prompt params[:content]
   send_msg
 end

data/workers/get_embedding.rb CHANGED Viewed

@@ -3,5 +3,5 @@ SmartPrompt.define_worker :get_embedding do
   use "OllamaEmbedding"
   model ENV["EMBEDDING_MODEL"] || "qwen3-embedding"
   prompt params[:text]
-  embeddings(1024)
+  embeddings(ENV.fetch("EMBEDDING_DIMENSIONS", "4096").to_i)
 end

metadata CHANGED Viewed

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_rag
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - SmartRAG Team
@@ -37,20 +37,74 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: smart_prompt
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.5.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.5.4
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -65,20 +119,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '4.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -164,7 +246,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '6.0'
 - !ruby/object:Gem::Dependency
-  name: database_cleaner-sequel
+  name: database_cleaner
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -177,46 +259,33 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: faker
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.14'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.14'
 description: SmartRAG provides intelligent document processing, vector embeddings,
   full-text search, and hybrid retrieval capabilities for enhanced information retrieval
   and question answering.
 email:
 - team@smartrag.com
-executables: []
+executables:
+- smart-rag-db
+- smart-rag-media-worker
 extensions: []
 extra_rdoc_files: []
 files:
+- ".env.example"
+- ".rspec"
+- AGENTS.md
+- API_DOCUMENTATION.md
 - CHANGELOG.md
+- ER-diagram.mmd
+- Gemfile
+- Gemfile.lock
+- Hybrid_Reranking.md
 - README.en.md
 - README.md
+- Rakefile
+- SETUP_GUIDE.md
+- SmartChunking.md
+- USAGE_EXAMPLES.md
+- config.ru
 - config/database.yml
 - config/fulltext_search.yml
 - config/llm_config.yml
@@ -233,8 +302,33 @@ files:
 - db/migrations/009_create_search_logs.rb
 - db/migrations/010_add_metadata_to_source_documents.rb
 - db/migrations/011_add_source_fields_to_source_documents.rb
+- db/migrations/012_add_metadata_to_source_sections.rb
+- db/migrations/013_create_media_jobs.rb
+- db/migrations/014_add_media_job_operations_indexes.rb
+- db/migrations/015_add_media_leases_and_objects.rb
+- db/migrations/016_add_document_principals_and_staging_references.rb
+- db/migrations/017_add_media_job_request_fingerprint.rb
 - db/rebuild_fts_complete.sql
 - db/seeds/text_search_configs.sql
+- design.md
+- docs/API_DOCUMENTATION.md
+- docs/DOCUMENTATION_INDEX.en.md
+- docs/DOCUMENTATION_INDEX.md
+- docs/FIX_SUMMARY.md
+- docs/FIX_SUMMARY_COMPLETE.md
+- docs/Hybrid_Reranking.md
+- docs/MIGRATION_GUIDE.md
+- docs/PERFORMANCE_GUIDE.md
+- docs/SETUP_GUIDE.md
+- docs/SmartChunking.md
+- docs/USAGE_EXAMPLES.md
+- docs/design.md
+- docs/evidence_pack.md
+- docs/requirements.md
+- docs/retrieval_plan.md
+- docs/smartrag_improvement_plan.md
+- docs/smartrag_refactor.md
+- docs/todo.md
 - examples/01_quick_start.rb
 - examples/02_document_management.rb
 - examples/03_search_operations.rb
@@ -243,15 +337,29 @@ files:
 - examples/06_error_handling_and_retry.rb
 - examples/README.md
 - examples/common.rb
+- exe/smart-rag-db
+- exe/smart-rag-media-worker
 - lib/smart_rag.rb
 - lib/smart_rag/chunker/markdown_chunker.rb
 - lib/smart_rag/config.rb
 - lib/smart_rag/core/document_processor.rb
 - lib/smart_rag/core/embedding.rb
 - lib/smart_rag/core/fulltext_manager.rb
+- lib/smart_rag/core/local_content_store.rb
 - lib/smart_rag/core/markitdown_bridge.rb
+- lib/smart_rag/core/media_extractors.rb
+- lib/smart_rag/core/media_job_queue.rb
+- lib/smart_rag/core/media_metadata_extractor.rb
+- lib/smart_rag/core/media_object_registry.rb
+- lib/smart_rag/core/media_processor.rb
+- lib/smart_rag/core/media_safety_policy.rb
 - lib/smart_rag/core/query_processor.rb
+- lib/smart_rag/core/s3_content_store.rb
+- lib/smart_rag/core/transcript_normalizer.rb
+- lib/smart_rag/core/video_semantic_extractor.rb
 - lib/smart_rag/errors.rb
+- lib/smart_rag/http_access_policy.rb
+- lib/smart_rag/http_app.rb
 - lib/smart_rag/models.rb
 - lib/smart_rag/models/embedding.rb
 - lib/smart_rag/models/model_base.rb
@@ -282,12 +390,17 @@ files:
 - lib/smart_rag/smart_chunking/structure_detector.rb
 - lib/smart_rag/smart_chunking/tokenizer.rb
 - lib/smart_rag/version.rb
+- patch_language.rb
+- requirements.md
+- source_documents_export.json
+- todo.md
 - workers/analyze_content.rb
 - workers/get_embedding.rb
 homepage: https://github.com/smartrag/smartrag
 licenses:
 - MIT
 metadata:
+  homepage_uri: https://github.com/smartrag/smartrag
   source_code_uri: https://github.com/smartrag/smartrag
   changelog_uri: https://github.com/smartrag/smartrag/blob/main/CHANGELOG.md
 rdoc_options: []
@@ -304,7 +417,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.16
 specification_version: 4
 summary: A hybrid RAG (Retrieval-Augmented Generation) system with vector and full-text
   search