smart_rag 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ed27c04c401121b0c7f8fc5650abf78602e3f930914e48bafb2749894c3f3c0
-  data.tar.gz: e7a4f56e7a9a0cc7b9930fb39e097898c812ba9a3e6ce702a28a4b51da915551
+  metadata.gz: 126c6c21c530311732722fc39ddad72632a9fb64c6dcdb8a467665d92871f44e
+  data.tar.gz: '09cab17c6e5aa7cef64548ef95833e74af40ca45659e6142a4125a680c89426d'
 SHA512:
-  metadata.gz: 8eb13a7acab801fa0db31dc74d7a86caa9bee9cb423fa8646e44581673e6122d7f3e3a58deaf74635393fe763b48208f84976fa2931b1eab5a814a45f4134d5a
-  data.tar.gz: cf1857fc42076bfaab4a687364ed21172badef86c366e2919a4b3aad748ee4f45946e8c4549b221f69769aa22b8e9e828939c3c7228741b97e1f49c34d2afc8a
+  metadata.gz: 30102eda862fa9d04aec069b1a0454d2b7fdef78eb5a9684e82509f096af40369e4a20982b63d9cb2672635ba3be00401e90a7beebca36dbdd76825ef453e005
+  data.tar.gz: 6b2b5b5f58d4168349e36f9622aa36c12948c3cdc77b9194db2eb2badb98b3a4900ee009f668b4979c334bdea051ae3d59e79d71c2b1d610f89a90f846b2a548

data/lib/smart_rag/core/document_processor.rb

@@ -321,8 +321,15 @@ module SmartRAG
         raise e
       end
 
+      # Read a text file as UTF-8.
+      #
+      # File.binread returns an ASCII-8BIT string, and transcoding that to UTF-8
+      # with `undef: :replace, replace: ''` silently deletes every high byte —
+      # i.e. it deletes all non-ASCII text (CJK included). Declare the encoding
+      # of the bytes instead of transcoding from binary; `scrub` then replaces
+      # only genuinely invalid byte sequences.
       def read_text_file(file_path)
-        normalize_text_content(File.binread(file_path))
+        normalize_text_content(File.read(file_path, mode: 'rb').force_encoding(Encoding::UTF_8))
       rescue StandardError => e
         @logger.error "Failed to read text file #{file_path}: #{e.message}"
         raise e
@@ -333,10 +340,16 @@ module SmartRAG
 
         normalized = content.is_a?(String) ? content.dup : content.to_s
 
-        begin
-          normalized = normalized.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
-        rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
-          normalized = normalized.force_encoding(Encoding::UTF_8).scrub
+        if normalized.encoding == Encoding::ASCII_8BIT
+          # Bytes of unknown encoding: reinterpret as UTF-8 instead of dropping
+          # everything above 0x7F.
+          normalized = normalized.force_encoding(Encoding::UTF_8)
+        elsif normalized.encoding != Encoding::UTF_8
+          begin
+            normalized = normalized.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
+          rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+            normalized = normalized.force_encoding(Encoding::UTF_8)
+          end
         end
 
         normalized.scrub

data/lib/smart_rag/retrieve.rb

@@ -19,6 +19,7 @@ module SmartRAG
     def execute(plan:)
       @document_metadata_cache = {}
       @section_topic_cache = {}
+      @document_record_cache = {}
       normalized_plan = normalize_plan(plan)
       validate_plan!(normalized_plan)
       principal = normalized_plan.delete(:_principal)
@@ -459,6 +460,15 @@ module SmartRAG
         return section.document.url if section.document.respond_to?(:url)
       end
 
+      # Not every search backend returns source metadata: vector search results
+      # carry only similarity/rank/section, so this returned nil for them. A nil
+      # URI matches no `source_uri_prefix`, so every candidate was rejected by
+      # the scope filter and semantic/hybrid plans always came back empty. The
+      # document row is authoritative.
+      document = document_record_for(extract_document_id(result, section))
+      stored_uri = document && document[:source_uri]
+      return stored_uri unless stored_uri.to_s.empty?
+
       nil
     end
 
@@ -471,6 +481,12 @@ module SmartRAG
         end
       end
 
+      # Same reasoning as extract_source_uri: prefer what was stored on the
+      # document over guessing from a URI a backend may not have returned.
+      document = document_record_for(extract_document_id(result, extract_section(result)))
+      stored_type = document && document[:source_type]
+      return stored_type unless stored_type.to_s.empty?
+
       return 'url' if source_uri.to_s.start_with?('http://', 'https://')
       return 'file' if source_uri.to_s.start_with?('file://', '/')
 
@@ -578,6 +594,21 @@ module SmartRAG
       nil
     end
 
+    # Document rows are the authoritative source for scope metadata; search
+    # backends do not all echo it back to the caller.
+    def document_record_for(document_id)
+      return nil if document_id.nil?
+      return nil unless defined?(::SmartRAG) && ::SmartRAG.respond_to?(:db) && ::SmartRAG.db
+
+      @document_record_cache ||= {}
+      key = document_id.to_i
+      return @document_record_cache[key] if @document_record_cache.key?(key)
+
+      @document_record_cache[key] = ::SmartRAG.db[:source_documents].where(id: key).first
+    rescue StandardError
+      nil
+    end
+
     def extract_vector_score(result, mode)
       return numeric_or_nil(result[:vector_score]) if result.is_a?(Hash) && result.key?(:vector_score)
       return numeric_or_nil(result[:similarity]) if mode == 'semantic' && result.is_a?(Hash) && result.key?(:similarity)

data/lib/smart_rag/version.rb

@@ -1,3 +1,3 @@
 module SmartRAG
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_rag
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - SmartRAG Team
@@ -417,7 +417,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: A hybrid RAG (Retrieval-Augmented Generation) system with vector and full-text
   search