smart_rag 0.1.0 → 0.2.0

data/lib/smart_rag/http_app.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'rack'
+require 'fileutils'
+require 'tmpdir'
+require 'securerandom'
+require_relative 'http_access_policy'
+require_relative 'core/media_job_queue'
+
+module SmartRAG
+  class HttpApp
+    OPERATIONS = %w[add_document add_media add_image add_audio add_video].freeze
+
+    def initialize(rag:, extractors: {}, access_policy: nil)
+      @rag = rag
+      @extractors = symbolize_keys(extractors)
+      @access_policy = access_policy
+    end
+
+    def call(env)
+      request = Rack::Request.new(env)
+      return health if request.get? && request.path == '/healthz'
+      principal = authorize(request)
+      request.env['smart_rag.principal'] = principal
+      return retrieve(request) if request.post? && request.path == '/v1/retrieve'
+      return ingest(request) if request.post? && request.path == '/v1/media'
+      return list_media_jobs(request) if request.get? && request.path == '/v1/media/jobs'
+      if request.get? && request.path == '/v1/media/jobs/stats'
+        return json(200, access_policy ? rag.media_job_statistics(principal: principal) : rag.media_job_statistics)
+      end
+      if request.post? && (match = request.path.match(%r{\A/v1/media/jobs/(\d+)/(cancel|retry)\z}))
+        return mutate_media_job(match[1], match[2], principal)
+      end
+      if request.get? && request.path.match?(%r{\A/v1/media/jobs/\d+\z})
+        job_id = request.path.split('/').last
+        job = access_policy ? rag.media_job(job_id, principal: principal) : rag.media_job(job_id)
+        return job ? json(200, job) : json(404, error: 'media job not found')
+      end
+
+      json(404, error: 'not found')
+    rescue JSON::ParserError => e
+      json(400, error: "invalid JSON: #{e.message}")
+    rescue ArgumentError => e
+      json(400, error: e.message)
+    rescue HttpAccessPolicy::Unauthorized => e
+      json(401, error: e.message)
+    rescue HttpAccessPolicy::QuotaExceeded => e
+      json(429, error: e.message)
+    rescue Core::MediaJobQueue::IdempotencyConflict => e
+      json(409, error: e.message, code: 'idempotency_conflict')
+    rescue StandardError => e
+      json(500, error: "#{e.class}: #{e.message}")
+    end
+
+    private
+
+    attr_reader :rag, :extractors, :access_policy
+
+    def authorize(request)
+      return 'anonymous' unless access_policy
+      principal = access_policy.authorize!(request)
+      bytes = request.post? && request.path == '/v1/media' ? request.content_length.to_i : 0
+      access_policy.consume!(principal, request_bytes: bytes)
+      principal
+    end
+
+    def health
+      payload = { status: 'ok' }
+      payload[:media_jobs] = rag.media_job_statistics if rag.respond_to?(:media_job_statistics)
+      json(200, payload)
+    rescue StandardError => e
+      json(503, status: 'degraded', error: "media job queue unavailable: #{e.message}")
+    end
+
+    def list_media_jobs(request)
+      options = { status: request.params['status'], limit: request.params.fetch('limit', 20),
+                  offset: request.params.fetch('offset', 0) }
+      options[:principal] = request.env['smart_rag.principal'] if access_policy
+      json(200, rag.media_jobs(**options))
+    end
+
+    def mutate_media_job(job_id, action, principal)
+      result = if action == 'cancel'
+                 access_policy ? rag.cancel_media_job(job_id, principal: principal) : rag.cancel_media_job(job_id)
+               else
+                 access_policy ? rag.retry_media_job(job_id, principal: principal) : rag.retry_media_job(job_id)
+               end
+      return json(404, error: 'media job not found') unless result
+      return json(409, result) if result[:transition_error]
+
+      json(200, result)
+    end
+
+    def retrieve(request)
+      body = json_body(request)
+      plan = symbolize_keys(body.fetch('plan', body))
+      plan[:_principal] = request.env['smart_rag.principal'] if access_policy
+      json(200, rag.retrieve(plan: plan))
+    end
+
+    def ingest(request)
+      payload, source, cleanup_path = if request.media_type.to_s.start_with?('multipart/form-data')
+                                        multipart_payload(request)
+                                      else
+                                        json_payload(request)
+                                      end
+      operation = payload.fetch(:operation).to_s
+      raise ArgumentError, "unsupported operation: #{operation}" unless OPERATIONS.include?(operation)
+      raise ArgumentError, 'source is required' if source.to_s.empty?
+
+      options = symbolize_keys(payload[:options] || {}).merge(extractors)
+      options[:principal] = request.env['smart_rag.principal'] if access_policy
+      if options.delete(:async)
+        source, cleanup_path = persist_async_upload(source, cleanup_path)
+        staged = rag.stage_media_upload(source) if File.file?(source) && rag.respond_to?(:stage_media_upload)
+        if staged
+          File.delete(source) if File.file?(source)
+          source = staged[:storage_uri]
+          options[:staging_media_object_id] = staged[:media_object_id]
+        else
+          options[:delete_source_after] = true if File.file?(source)
+        end
+        options[:idempotency_key] ||= request.get_header('HTTP_IDEMPOTENCY_KEY')
+        return json(202, rag.enqueue_media(source, options.merge(operation: operation.to_sym)))
+      end
+      json(200, rag.public_send(operation, source, options))
+    ensure
+      File.delete(cleanup_path) if cleanup_path && File.exist?(cleanup_path)
+    end
+
+    def json_payload(request)
+      payload = symbolize_keys(json_body(request))
+      [payload, payload[:source], nil]
+    end
+
+    def multipart_payload(request)
+      params = request.params
+      upload = params['file']
+      tempfile = upload.is_a?(Hash) && (upload[:tempfile] || upload['tempfile'])
+      filename = upload.is_a?(Hash) && (upload[:filename] || upload['filename'])
+      raise ArgumentError, 'multipart file is required' unless tempfile
+
+      extension = File.extname(filename.to_s)
+      source = tempfile.path
+      unless extension.empty? || File.extname(source).downcase == extension.downcase
+        preserved = File.join(Dir.tmpdir, "smart-rag-upload-#{Process.pid}-#{object_id}#{extension}")
+        FileUtils.cp(source, preserved)
+        source = preserved
+      end
+
+      payload = {
+        operation: params['operation'],
+        options: params['options'].to_s.empty? ? {} : JSON.parse(params['options'])
+      }
+      [symbolize_keys(payload), source, source == tempfile.path ? nil : source]
+    end
+
+    def json_body(request)
+      content = request.body.read
+      content.empty? ? {} : JSON.parse(content)
+    end
+
+    def persist_async_upload(source, cleanup_path)
+      return [source, cleanup_path] unless File.file?(source)
+
+      directory = ENV.fetch('SMARTRAG_MEDIA_JOB_UPLOAD_DIR', File.join(Dir.tmpdir, 'smart-rag-media-jobs'))
+      FileUtils.mkdir_p(directory)
+      extension = File.extname(source)
+      destination = File.join(directory, "#{SecureRandom.uuid}#{extension}")
+      FileUtils.cp(source, destination)
+      [destination, cleanup_path]
+    end
+
+    def symbolize_keys(value)
+      return value.map { |item| symbolize_keys(item) } if value.is_a?(Array)
+      return value unless value.is_a?(Hash)
+
+      value.each_with_object({}) do |(key, item), result|
+        result[key.to_sym] = symbolize_keys(item)
+      end
+    end
+
+    def json(status, payload)
+      [status, { 'content-type' => 'application/json' }, [JSON.generate(payload)]]
+    end
+  end
+end

data/lib/smart_rag/models/embedding.rb

@@ -11,7 +11,7 @@ module SmartRAG
 
       # Set dataset after database is connected
       def self.set_dataset_from_db
-        set_dataset(:embeddings)
+        set_dataset(Sequel::Model.db[:embeddings])
       end
 
       # Add bang methods for FactoryBot compatibility

data/lib/smart_rag/models/research_topic.rb

@@ -7,7 +7,7 @@ module SmartRAG
     class ResearchTopic < Sequel::Model
       # Set dataset after database is connected
       def self.set_dataset_from_db
-        set_dataset(:research_topics)
+        set_dataset(Sequel::Model.db[:research_topics])
       end
       include FactoryBotHelpers
       plugin :validation_helpers

data/lib/smart_rag/models/research_topic_section.rb

@@ -5,6 +5,11 @@ module SmartRAG
   module Models
     # ResearchTopicSection model for many-to-many relationship
     class ResearchTopicSection < Sequel::Model(:research_topic_sections)
+      # Set dataset after database is connected
+      def self.set_dataset_from_db
+        set_dataset(Sequel::Model.db[:research_topic_sections])
+      end
+
       include FactoryBotHelpers
       plugin :validation_helpers
       plugin :timestamps, update_on_create: false

data/lib/smart_rag/models/research_topic_tag.rb

@@ -5,6 +5,11 @@ module SmartRAG
   module Models
     # ResearchTopicTag model for many-to-many relationship between topics and tags
     class ResearchTopicTag < Sequel::Model(:research_topic_tags)
+      # Set dataset after database is connected
+      def self.set_dataset_from_db
+        set_dataset(Sequel::Model.db[:research_topic_tags])
+      end
+
       include FactoryBotHelpers
       plugin :validation_helpers
       plugin :timestamps, update_on_create: false

data/lib/smart_rag/models/search_log.rb

@@ -7,7 +7,7 @@ module SmartRAG
     class SearchLog < Sequel::Model
       # Set dataset after database is connected
       def self.set_dataset_from_db
-        set_dataset(:search_logs)
+        set_dataset(Sequel::Model.db[:search_logs])
       end
       include FactoryBotHelpers
       plugin :validation_helpers

data/lib/smart_rag/models/section_fts.rb

@@ -5,6 +5,11 @@ module SmartRAG
   module Models
     # SectionFts model for full-text search optimization
     class SectionFts < Sequel::Model(:section_fts)
+      # Set dataset after database is connected
+      def self.set_dataset_from_db
+        set_dataset(Sequel::Model.db[:section_fts])
+      end
+
       include FactoryBotHelpers
       plugin :validation_helpers
       plugin :timestamps, update_on_create: true

data/lib/smart_rag/models/section_tag.rb

@@ -5,6 +5,11 @@ module SmartRAG
   module Models
     # SectionTag model for many-to-many relationship between sections and tags
     class SectionTag < Sequel::Model(:section_tags)
+      # Set dataset after database is connected
+      def self.set_dataset_from_db
+        set_dataset(Sequel::Model.db[:section_tags])
+      end
+
       include FactoryBotHelpers
       plugin :validation_helpers
       plugin :timestamps, update_on_create: false

data/lib/smart_rag/models/source_document.rb

@@ -7,7 +7,7 @@ module SmartRAG
     class SourceDocument < Sequel::Model
       # Set dataset after database is connected
       def self.set_dataset_from_db
-        set_dataset(:source_documents)
+        set_dataset(Sequel::Model.db[:source_documents])
       end
       plugin :validation_helpers
       plugin :timestamps, update_on_create: true

data/lib/smart_rag/models/source_section.rb

@@ -7,7 +7,7 @@ module SmartRAG
     class SourceSection < Sequel::Model
       # Set dataset after database is connected
       def self.set_dataset_from_db
-        set_dataset(:source_sections)
+        set_dataset(Sequel::Model.db[:source_sections])
       end
       include FactoryBotHelpers
       plugin :validation_helpers

data/lib/smart_rag/models/tag.rb

@@ -7,7 +7,7 @@ module SmartRAG
     class Tag < Sequel::Model
       # Set dataset after database is connected
       def self.set_dataset_from_db
-        set_dataset(:tags)
+        set_dataset(Sequel::Model.db[:tags])
       end
       include FactoryBotHelpers
       plugin :validation_helpers

data/lib/smart_rag/models/text_search_config.rb

@@ -5,6 +5,11 @@ module SmartRAG
   module Models
     # TextSearchConfig model for language-specific search configurations
     class TextSearchConfig < Sequel::Model(:text_search_configs)
+      # Set dataset after database is connected
+      def self.set_dataset_from_db
+        set_dataset(Sequel::Model.db[:text_search_configs])
+      end
+
       include FactoryBotHelpers
       plugin :validation_helpers
       # This model uses text_search_configs table with language_code as PK

data/lib/smart_rag/retrieve.rb

@@ -17,8 +17,12 @@ module SmartRAG
     end
 
     def execute(plan:)
+      @document_metadata_cache = {}
+      @section_topic_cache = {}
       normalized_plan = normalize_plan(plan)
       validate_plan!(normalized_plan)
+      principal = normalized_plan.delete(:_principal)
+      normalized_plan = scope_plan_to_principal(normalized_plan, principal) if principal
 
       started_at = monotonic_now
       generated_at = Time.now.utc.iso8601
@@ -57,7 +61,7 @@ module SmartRAG
 
       pack = {
         version: normalized_plan[:version] || '0.1',
-        plan: normalized_plan,
+        plan: public_plan(normalized_plan),
         plan_id: plan_id,
         request_id: request_id,
         generated_at: generated_at,
@@ -345,6 +349,9 @@ module SmartRAG
       if filters[:source_type]
         applied[:source_type] = Array(filters[:source_type]).map(&:to_s)
       end
+      if filters[:media_type]
+        applied[:media_type] = Array(filters[:media_type]).map(&:to_s)
+      end
       if filters[:source_uri_prefix]
         applied[:source_uri_prefix] = Array(filters[:source_uri_prefix]).map(&:to_s)
       end
@@ -479,6 +486,12 @@ module SmartRAG
         return false unless allowed.include?(actual)
       end
 
+      if filters[:media_type]
+        allowed = Array(filters[:media_type]).map { |value| value.to_s.downcase }
+        actual = candidate.dig(:metadata, :media_type) || candidate.dig(:metadata, 'media_type')
+        return false unless allowed.include?(actual.to_s.downcase)
+      end
+
       if filters[:source_uri_prefix]
         prefixes = Array(filters[:source_uri_prefix]).map(&:to_s)
         uri = candidate[:source_uri].to_s
@@ -491,6 +504,11 @@ module SmartRAG
         return false if required_topic_ids.any? && (required_topic_ids & section_topics).empty?
       end
 
+      if filters[:principal]
+        actual = document_principal_for(candidate[:document_id])
+        return false unless actual == filters[:principal].to_s
+      end
+
       true
     end
 
@@ -520,9 +538,46 @@ module SmartRAG
 
       metadata[:section_id] ||= section_id if section_id
       metadata[:document_id] ||= document_id if document_id
+      section_metadata = extract_section_metadata(section)
+      metadata = document_metadata_for(document_id).merge(metadata).merge(section_metadata)
       metadata
     end
 
+    def extract_section_metadata(section)
+      raw = if section.is_a?(Hash)
+              section[:metadata] || section['metadata']
+            elsif section.respond_to?(:metadata)
+              section.metadata
+            end
+      parsed = raw.is_a?(String) ? JSON.parse(raw) : raw
+      parsed.is_a?(Hash) ? symbolize_keys(parsed) : {}
+    rescue JSON::ParserError
+      {}
+    end
+
+    def document_metadata_for(document_id)
+      return {} if document_id.nil?
+      return {} unless defined?(::SmartRAG) && ::SmartRAG.respond_to?(:db) && ::SmartRAG.db
+
+      @document_metadata_cache ||= {}
+      return @document_metadata_cache[document_id] if @document_metadata_cache.key?(document_id)
+
+      raw = ::SmartRAG.db[:source_documents].where(id: document_id).get(:metadata)
+      parsed = raw.is_a?(String) ? JSON.parse(raw) : raw
+      @document_metadata_cache[document_id] = parsed.is_a?(Hash) ? symbolize_keys(parsed) : {}
+    rescue StandardError
+      {}
+    end
+
+    def document_principal_for(document_id)
+      return nil if document_id.nil? || !::SmartRAG.db
+      @document_principal_cache ||= {}
+      return @document_principal_cache[document_id] if @document_principal_cache.key?(document_id)
+      @document_principal_cache[document_id] = ::SmartRAG.db[:source_documents].where(id: document_id).get(:principal)
+    rescue StandardError
+      nil
+    end
+
     def extract_vector_score(result, mode)
       return numeric_or_nil(result[:vector_score]) if result.is_a?(Hash) && result.key?(:vector_score)
       return numeric_or_nil(result[:similarity]) if mode == 'semantic' && result.is_a?(Hash) && result.key?(:similarity)
@@ -575,6 +630,22 @@ module SmartRAG
       deep_symbolize(plan)
     end
 
+    def scope_plan_to_principal(plan, principal)
+      allowed_ids = ::SmartRAG.db[:source_documents].where(principal: principal.to_s).select_map(:id)
+      requested = Array(plan.dig(:global_filters, :document_ids)).map(&:to_i)
+      allowed_ids &= requested unless requested.empty?
+      filters = symbolize_keys(plan[:global_filters] || {}).merge(
+        document_ids: allowed_ids,
+        principal: principal.to_s
+      )
+      plan.merge(global_filters: filters)
+    end
+
+    def public_plan(plan)
+      filters = symbolize_keys(plan[:global_filters] || {}).reject { |key, _| key == :principal }
+      plan.merge(global_filters: filters)
+    end
+
     def validate_plan!(plan)
       raise ArgumentError, 'Retrieval plan must be a hash' unless plan.is_a?(Hash)
       raise ArgumentError, 'Retrieval plan requires queries' unless plan[:queries].is_a?(Array) && !plan[:queries].empty?

data/lib/smart_rag/services/embedding_service.rb

@@ -24,7 +24,7 @@ module SmartRAG
         Dir.glob(File.join(workers_dir, '*.rb')).each { |file| require file }
 
         # Initialize SmartPrompt engine
-        config_path = @config[:config_path] || 'config/llm_config.yml'
+        config_path = @config[:config_path] || File.expand_path('../../../config/llm_config.yml', __dir__)
         @smart_prompt_engine = SmartPrompt::Engine.new(config_path)
       rescue StandardError => e
         log_error('Failed to initialize SmartPrompt engine', e)

data/lib/smart_rag/services/fulltext_search_service.rb

@@ -319,7 +319,8 @@ module SmartRAG
                                      Sequel[:source_documents][:author],
                                      Sequel[:source_documents][:publication_date],
                                      Sequel[:source_sections][:section_number],
-                                     Sequel[:source_documents][:metadata]
+                                     Sequel[:source_documents][:metadata].as(:document_metadata),
+                                     Sequel[:source_sections][:metadata].as(:section_metadata)
                                    )
 
         result = dataset.first
@@ -333,21 +334,18 @@ module SmartRAG
           section_number: result[:section_number]
         }
 
-        if result[:metadata]
-          if result[:metadata].is_a?(String)
-            begin
-              parsed = JSON.parse(result[:metadata])
-              metadata.merge!(parsed) if parsed.is_a?(Hash)
-            rescue JSON::ParserError
-              # Ignore malformed metadata strings
-            end
-          elsif result[:metadata].is_a?(Hash)
-            metadata.merge!(result[:metadata])
-          end
-        end
+        metadata.merge!(parse_metadata(result[:document_metadata]))
+        metadata.merge!(parse_metadata(result[:section_metadata]))
         metadata
       end
 
+      def parse_metadata(value)
+        parsed = value.is_a?(String) ? JSON.parse(value) : value
+        parsed.is_a?(Hash) ? parsed : {}
+      rescue JSON::ParserError
+        {}
+      end
+
       # Simplify result for quick search
       def simplify_result(result)
         {

data/lib/smart_rag/services/hybrid_search_service.rb

@@ -702,6 +702,12 @@ module SmartRAG
                                 document_id: document_id
                               }
                             end
+            section_metadata = if section.is_a?(Hash)
+                                 section[:metadata] || section['metadata']
+                               elsif section.respond_to?(:metadata)
+                                 section.metadata
+                               end
+            base_metadata.merge!(parse_metadata(section_metadata))
 
             if document_id && document_id != ''
               begin
@@ -715,18 +721,9 @@ module SmartRAG
                   # Merge document metadata (may contain category, author, etc.)
                   if doc[:metadata]
                     @logger.debug "Document metadata found: #{doc[:metadata].inspect}"
-                    parsed_metadata = if doc[:metadata].is_a?(String)
-                                        begin
-                                          JSON.parse(doc[:metadata])
-                                        rescue StandardError
-                                          {}
-                                        end
-                                      else
-                                        doc[:metadata]
-                                      end
-                    parsed_metadata = symbolize_keys(parsed_metadata) if parsed_metadata.is_a?(Hash)
+                    parsed_metadata = parse_metadata(doc[:metadata])
                     @logger.debug "Parsed metadata: #{parsed_metadata.inspect}"
-                    base_metadata.merge!(parsed_metadata) if parsed_metadata.is_a?(Hash)
+                    base_metadata = parsed_metadata.merge(base_metadata)
                   else
                     @logger.debug 'Document has no metadata field or is nil'
                   end
@@ -753,6 +750,13 @@ module SmartRAG
         end
       end
 
+      def parse_metadata(value)
+        parsed = value.is_a?(String) ? JSON.parse(value) : value
+        parsed.is_a?(Hash) ? symbolize_keys(parsed) : {}
+      rescue JSON::ParserError
+        {}
+      end
+
       def calculate_score_stats(results)
         return {} if results.empty?
 

data/lib/smart_rag/services/summarization_service.rb

@@ -26,7 +26,7 @@ module SmartRAG
         Dir.glob(File.join(workers_dir, '*.rb')).each { |file| require file }
 
         # Initialize SmartPrompt engine
-        config_path = @config[:config_path] || "config/llm_config.yml"
+        config_path = @config[:config_path] || File.expand_path("../../../config/llm_config.yml", __dir__)
         @smart_prompt_engine = SmartPrompt::Engine.new(config_path)
 
         @logger.info "SummarizationService initialized"

data/lib/smart_rag/services/tag_service.rb

@@ -27,7 +27,7 @@ module SmartRAG
         Dir.glob(File.join(workers_dir, "*.rb")).each { |file| require file }
 
         # Initialize SmartPrompt engine
-        config_path = @config[:config_path] || "config/llm_config.yml"
+        config_path = @config[:config_path] || File.expand_path("../../../config/llm_config.yml", __dir__)
         @smart_prompt_engine = SmartPrompt::Engine.new(config_path)
       rescue StandardError => e
         log_error("Failed to initialize TagService", e)

data/lib/smart_rag/version.rb

@@ -1,3 +1,3 @@
 module SmartRAG
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end