smart_rag 0.2.2 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ed27c04c401121b0c7f8fc5650abf78602e3f930914e48bafb2749894c3f3c0
-  data.tar.gz: e7a4f56e7a9a0cc7b9930fb39e097898c812ba9a3e6ce702a28a4b51da915551
+  metadata.gz: b8c4c7c6f458001f016b74e56d5050632bc819d23c25e6b810c7900605b1d401
+  data.tar.gz: 3793d26b5fd103ca5ef0e32ff708f7fb235fbd5888d2f306db8f39416ab832f3
 SHA512:
-  metadata.gz: 8eb13a7acab801fa0db31dc74d7a86caa9bee9cb423fa8646e44581673e6122d7f3e3a58deaf74635393fe763b48208f84976fa2931b1eab5a814a45f4134d5a
-  data.tar.gz: cf1857fc42076bfaab4a687364ed21172badef86c366e2919a4b3aad748ee4f45946e8c4549b221f69769aa22b8e9e828939c3c7228741b97e1f49c34d2afc8a
+  metadata.gz: 45d530acad3be58c9126d61bc426780873ab03dd9129cbb4b8220b8d9655012ab4e2ae9304b813ebd9a334f0dc444c237c4ab46166edb182582f39db4ceefbc9
+  data.tar.gz: ed4c245e61190954e33460679faacc13b2d2ac6c8d23b296f09b7325e86aa899a515c4b1147bf1b9d56750b7616115292a12d3f198cf004c917ffd415fe04418

data/Gemfile.lock

@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    smart_rag (0.2.2)
+    smart_rag (0.2.3)
       aws-sdk-s3 (~> 1.0)
       concurrent-ruby (~> 1.0)
       dotenv (~> 2.8)
@@ -381,7 +381,7 @@ CHECKSUMS
   simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246
   simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428
   smart_prompt (0.5.4) sha256=81d5e052cf05d63051a68e946e0ba25bcd280dca5e43ce96664551a33656e5bc
-  smart_rag (0.2.2)
+  smart_rag (0.2.3)
   standardwebhooks (1.1.0) sha256=356ba711c14def4fb27310e4163b26bd5cccf325212c3440cabadd825e4be11f
   stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1
   tiktoken_ruby (0.0.17-x64-mingw-ucrt) sha256=8bd2f8e2c38ee9052a8cd4fc83fcb41362beb476d8885ac121125cd4dd065596

data/lib/smart_rag/core/document_processor.rb

@@ -321,8 +321,15 @@ module SmartRAG
         raise e
       end
 
+      # Read a text file as UTF-8.
+      #
+      # File.binread returns an ASCII-8BIT string, and transcoding that to UTF-8
+      # with `undef: :replace, replace: ''` silently deletes every high byte —
+      # i.e. it deletes all non-ASCII text (CJK included). Declare the encoding
+      # of the bytes instead of transcoding from binary; `scrub` then replaces
+      # only genuinely invalid byte sequences.
       def read_text_file(file_path)
-        normalize_text_content(File.binread(file_path))
+        normalize_text_content(File.read(file_path, mode: 'rb').force_encoding(Encoding::UTF_8))
       rescue StandardError => e
         @logger.error "Failed to read text file #{file_path}: #{e.message}"
         raise e
@@ -333,10 +340,16 @@ module SmartRAG
 
         normalized = content.is_a?(String) ? content.dup : content.to_s
 
-        begin
-          normalized = normalized.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
-        rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
-          normalized = normalized.force_encoding(Encoding::UTF_8).scrub
+        if normalized.encoding == Encoding::ASCII_8BIT
+          # Bytes of unknown encoding: reinterpret as UTF-8 instead of dropping
+          # everything above 0x7F.
+          normalized = normalized.force_encoding(Encoding::UTF_8)
+        elsif normalized.encoding != Encoding::UTF_8
+          begin
+            normalized = normalized.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
+          rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+            normalized = normalized.force_encoding(Encoding::UTF_8)
+          end
         end
 
         normalized.scrub

data/lib/smart_rag/core/fulltext_manager.rb

@@ -28,6 +28,13 @@ module SmartRAG
         @query_parser = options[:query_parser] || Parsers::QueryParser.new
         @logger = options[:logger] || Logger.new(STDOUT)
         @config = DEFAULT_CONFIG.merge(options)
+
+        # Callers either hand over the whole configuration (with a nested
+        # `fulltext_search` section) or the section itself. Surface the nested
+        # keys either way so options like `enable_jieba` are actually visible.
+        nested = @config[:fulltext_search] || @config['fulltext_search'] ||
+                 @config[:fulltext] || @config['fulltext']
+        @config = @config.merge(nested) if nested.is_a?(Hash)
       end
 
       # Store or update full-text index for a section
@@ -345,29 +352,67 @@ module SmartRAG
         dataset
       end
 
+      # Text search configurations shipped by the pg_jieba extension: upstream
+      # installs `jiebacfg` (mix segmentation, recommended) plus `jiebaqry`,
+      # `jiebamp` and `jiebahmm`. No released version provides a configuration
+      # literally named `jieba` — probing that name always failed, which silently
+      # downgraded Chinese search to `simple` even with the extension installed.
+      JIEBA_CONFIG = 'jiebacfg'
+      JIEBA_CONFIG_NAMES = %w[jieba jiebacfg jiebaqry jiebamp jiebahmm].freeze
+
       # Get text search configuration
       def get_text_search_config(language)
         config = Models::TextSearchConfig.first(language_code: language.to_s)&.config_name
         return 'pg_catalog.simple' unless config
+        return config unless jieba_config?(config)
+        # `fulltext_search.enable_jieba` (on unless explicitly disabled) is the
+        # switch for Chinese segmentation.
+        return 'pg_catalog.simple' unless jieba_enabled?
 
-        # For development/test environments, always fall back to simple if pg_jieba is not available
-        if config == 'jieba'
-          begin
-            # Test if pg_jieba is available in a separate transaction
-            db.fetch("SELECT to_tsvector('jieba', 'test')").first
-            return 'jieba'
-          rescue StandardError => e
-            @logger.warn "pg_jieba extension not available, falling back to simple: #{e.message}"
-            return 'pg_catalog.simple'
-          end
-        end
-
-        config
+        jieba_config_name(config) || 'pg_catalog.simple'
       rescue StandardError => e
         @logger.warn "Failed to get text search config for #{language}: #{e.message}, using simple"
         'pg_catalog.simple'
       end
 
+      # Does this configuration name belong to pg_jieba?
+      def jieba_config?(name)
+        JIEBA_CONFIG_NAMES.include?(name.to_s.sub(/\Apg_catalog\./, ''))
+      end
+
+      # `enable_jieba` defaults to on; only an explicit false switches it off.
+      def jieba_enabled?
+        value = config_value(:enable_jieba)
+        return true if value.nil?
+
+        value != false && value.to_s != 'false'
+      end
+
+      # Return a configuration the server actually has: prefer the name the
+      # database row specifies, then the name pg_jieba really installs.
+      #
+      # This asks the catalogue instead of probing with `to_tsvector(name, ...)`:
+      # a probe that raises would abort any enclosing transaction (PostgreSQL
+      # refuses further statements until it is rolled back), which made every
+      # lookup after the first failure fall back to simple.
+      def jieba_config_name(preferred)
+        candidates = [preferred.to_s, JIEBA_CONFIG].uniq
+        present = db[:pg_ts_config].where(cfgname: candidates).select_map(:cfgname)
+        name = candidates.find { |candidate| present.include?(candidate) }
+        return name if name
+
+        @logger.warn "pg_jieba text search configuration unavailable " \
+                     "(tried: #{candidates.join(', ')}), falling back to simple"
+        nil
+      end
+
+      def config_value(key)
+        return @config[key] if @config.key?(key)
+        return @config[key.to_s] if @config.key?(key.to_s)
+
+        nil
+      end
+
       # Set weight for tsvector (helper method)
       def setweight(vector, weight)
         return '' if vector.nil? || vector.to_s.strip.empty?

data/lib/smart_rag/retrieve.rb

@@ -19,6 +19,7 @@ module SmartRAG
     def execute(plan:)
       @document_metadata_cache = {}
       @section_topic_cache = {}
+      @document_record_cache = {}
       normalized_plan = normalize_plan(plan)
       validate_plan!(normalized_plan)
       principal = normalized_plan.delete(:_principal)
@@ -459,6 +460,15 @@ module SmartRAG
         return section.document.url if section.document.respond_to?(:url)
       end
 
+      # Not every search backend returns source metadata: vector search results
+      # carry only similarity/rank/section, so this returned nil for them. A nil
+      # URI matches no `source_uri_prefix`, so every candidate was rejected by
+      # the scope filter and semantic/hybrid plans always came back empty. The
+      # document row is authoritative.
+      document = document_record_for(extract_document_id(result, section))
+      stored_uri = document && document[:source_uri]
+      return stored_uri unless stored_uri.to_s.empty?
+
       nil
     end
 
@@ -471,6 +481,12 @@ module SmartRAG
         end
       end
 
+      # Same reasoning as extract_source_uri: prefer what was stored on the
+      # document over guessing from a URI a backend may not have returned.
+      document = document_record_for(extract_document_id(result, extract_section(result)))
+      stored_type = document && document[:source_type]
+      return stored_type unless stored_type.to_s.empty?
+
       return 'url' if source_uri.to_s.start_with?('http://', 'https://')
       return 'file' if source_uri.to_s.start_with?('file://', '/')
 
@@ -578,6 +594,21 @@ module SmartRAG
       nil
     end
 
+    # Document rows are the authoritative source for scope metadata; search
+    # backends do not all echo it back to the caller.
+    def document_record_for(document_id)
+      return nil if document_id.nil?
+      return nil unless defined?(::SmartRAG) && ::SmartRAG.respond_to?(:db) && ::SmartRAG.db
+
+      @document_record_cache ||= {}
+      key = document_id.to_i
+      return @document_record_cache[key] if @document_record_cache.key?(key)
+
+      @document_record_cache[key] = ::SmartRAG.db[:source_documents].where(id: key).first
+    rescue StandardError
+      nil
+    end
+
     def extract_vector_score(result, mode)
       return numeric_or_nil(result[:vector_score]) if result.is_a?(Hash) && result.key?(:vector_score)
       return numeric_or_nil(result[:similarity]) if mode == 'semantic' && result.is_a?(Hash) && result.key?(:similarity)

data/lib/smart_rag/version.rb

@@ -1,3 +1,3 @@
 module SmartRAG
-  VERSION = "0.2.2"
+  VERSION = "0.2.4"
 end

data/lib/smart_rag.rb

@@ -1026,7 +1026,13 @@ module SmartRAG
         db_connection = ::SmartRAG.db
 
         embedding_manager = ::SmartRAG::Core::Embedding.new(@config[:database])
-        fulltext_manager = ::SmartRAG::Core::FulltextManager.new(db_connection, (@config[:fulltext] || {}).merge(logger: @logger))
+        # `fulltext_search` is the section name used by the configuration files;
+        # it used to be read as `fulltext`, so the whole section (including
+        # `enable_jieba`) never reached FulltextManager.
+        fulltext_section = @config[:fulltext_search] || @config[:fulltext] || {}
+        fulltext_manager = ::SmartRAG::Core::FulltextManager.new(
+          db_connection, fulltext_section.merge(logger: @logger)
+        )
 
         @query_processor = ::SmartRAG::Core::QueryProcessor.new(
           config: @config,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_rag
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.4
 platform: ruby
 authors:
 - SmartRAG Team
@@ -417,7 +417,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: A hybrid RAG (Retrieval-Augmented Generation) system with vector and full-text
   search