legion-data 1.6.18 → 1.6.20

data/lib/legion/data/spool.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
 require 'json'
 require 'fileutils'
 require 'securerandom'
@@ -31,6 +32,8 @@ module Legion
       end
 
       class ScopedSpool
+        include Legion::Logging::Helper
+
         def initialize(extension_module, spool_root)
           @extension_dir = File.join(spool_root, Spool.send(:extension_path, extension_module))
         end
@@ -40,25 +43,43 @@ module Legion
           FileUtils.mkdir_p(dir)
           filename = "#{Time.now.strftime('%s%9N')}-#{SecureRandom.uuid}.json"
           path = File.join(dir, filename)
-          File.write(path, ::JSON.generate(payload))
-          Legion::Logging.debug "Spool write: #{sub_namespace} -> #{filename}" if defined?(Legion::Logging)
+          temp_path = temp_path_for(dir, filename)
+          File.binwrite(temp_path, ::JSON.generate(payload))
+          File.rename(temp_path, path)
+          log.info "Spool write: #{sub_namespace} -> #{filename}"
           path
+        rescue StandardError => e
+          File.delete(temp_path) if defined?(temp_path) && temp_path && File.exist?(temp_path)
+          handle_exception(e, level: :error, handled: false, operation: :spool_write, sub_namespace: sub_namespace)
+          raise
         end
 
         def read(sub_namespace)
-          sorted_files(sub_namespace).map { |f| ::JSON.parse(File.read(f), symbolize_names: true) }
+          sorted_files(sub_namespace).each_with_object([]) do |path, events|
+            event = load_event_file(path, sub_namespace)
+            events << event if event
+          end
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: false, operation: :spool_read, sub_namespace: sub_namespace)
+          raise
         end
 
         def flush(sub_namespace)
           count = 0
+          path = nil
           sorted_files(sub_namespace).each do |path|
-            event = ::JSON.parse(File.read(path), symbolize_names: true)
+            event = load_event_file(path, sub_namespace)
+            next unless event
+
             yield event
             File.delete(path)
             count += 1
           end
-          Legion::Logging.info "Spool drained #{count} item(s) from #{sub_namespace}" if defined?(Legion::Logging) && count.positive?
+          log.info "Spool drained #{count} item(s) from #{sub_namespace}" if count.positive?
           count
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: false, operation: :spool_flush, sub_namespace: sub_namespace, path: path)
+          raise
         end
 
         def count(sub_namespace)
@@ -70,6 +91,10 @@ module Legion
           return unless Dir.exist?(dir)
 
           Dir[File.join(dir, '*.json')].each { |f| File.delete(f) }
+          log.info "Spool cleared #{sub_namespace}"
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: false, operation: :spool_clear, sub_namespace: sub_namespace)
+          raise
         end
 
         private
@@ -82,7 +107,45 @@ module Legion
           dir = sub_dir(sub_namespace)
           return [] unless Dir.exist?(dir)
 
-          Dir[File.join(dir, '*.json')]
+          Dir.glob(File.join(dir, '*.json'), sort: true)
+        end
+
+        def load_event_file(path, sub_namespace)
+          ::JSON.parse(File.binread(path), symbolize_names: true)
+        rescue Errno::ENOENT
+          nil
+        rescue ::JSON::ParserError, EOFError, ArgumentError => e
+          quarantine_corrupt_file(path, sub_namespace, e)
+          nil
+        end
+
+        def quarantine_corrupt_file(path, sub_namespace, error)
+          return unless File.exist?(path)
+
+          quarantine_dir = File.join(sub_dir(sub_namespace), 'quarantine')
+          FileUtils.mkdir_p(quarantine_dir)
+          quarantine_path = unique_quarantine_path(quarantine_dir, File.basename(path))
+          File.rename(path, quarantine_path)
+          handle_exception(
+            error,
+            level:           :warn,
+            handled:         true,
+            operation:       :spool_quarantine,
+            sub_namespace:   sub_namespace,
+            path:            path,
+            quarantine_path: quarantine_path
+          )
+        end
+
+        def unique_quarantine_path(quarantine_dir, basename)
+          path = File.join(quarantine_dir, "#{basename}.corrupt")
+          return path unless File.exist?(path)
+
+          File.join(quarantine_dir, "#{basename}.#{SecureRandom.uuid}.corrupt")
+        end
+
+        def temp_path_for(dir, filename)
+          File.join(dir, ".#{filename}.tmp-#{SecureRandom.uuid}")
         end
       end
     end

data/lib/legion/data/storage_tiers.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 module Legion
   module Data
     module StorageTiers
       TIERS = { hot: 0, warm: 1, cold: 2 }.freeze
 
       class << self
+        include Legion::Logging::Helper
+
         def archive_to_warm(table:, age_days: 90, batch_size: 1000)
           return { archived: 0, reason: 'no_connection' } unless Legion::Data.connection
           return { archived: 0, reason: 'no_archive_table' } unless Legion::Data.connection.table_exists?(:data_archive)
@@ -28,8 +32,11 @@ module Legion
             Legion::Data.connection[table].where(id: ids).delete
           end
 
-          Legion::Logging.info "Archived #{records.size} row(s) from #{table} to warm tier" if defined?(Legion::Logging)
+          log.info "Archived #{records.size} row(s) from #{table} to warm tier"
           { archived: records.size, table: table.to_s }
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: false, operation: :archive_to_warm, table: table, age_days: age_days, batch_size: batch_size)
+          raise
         end
 
         def export_to_cold(age_days: 365, batch_size: 5000)
@@ -44,14 +51,20 @@ module Legion
 
           ids = records.map { |r| r[:id] }
           Legion::Data.connection[:data_archive].where(id: ids).update(tier: TIERS[:cold])
-          Legion::Logging.info "Exported #{records.size} row(s) to cold tier" if defined?(Legion::Logging)
+          log.info "Exported #{records.size} row(s) to cold tier"
           { exported: records.size, data: records }
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: false, operation: :export_to_cold, age_days: age_days, batch_size: batch_size)
+          raise
         end
 
         def stats
           return {} unless Legion::Data.connection&.table_exists?(:data_archive)
 
           { warm: count_tier(:warm), cold: count_tier(:cold) }
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: :storage_tiers_stats)
+          {}
         end
 
         private
@@ -59,7 +72,7 @@ module Legion
         def count_tier(tier)
           Legion::Data.connection[:data_archive].where(tier: TIERS[tier]).count
         rescue StandardError => e
-          Legion::Logging.debug("StorageTiers#count_tier failed for #{tier}: #{e.message}") if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, handled: true, operation: :storage_tiers_count, tier: tier)
           0
         end
       end

data/lib/legion/data/vector.rb

@@ -1,16 +1,20 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 module Legion
   module Data
     module Vector
       class << self
+        include Legion::Logging::Helper
+
         def available?
           return false unless Legion::Data.connection
           return false unless Legion::Data.connection.adapter_scheme == :postgres
 
           Legion::Data.connection.fetch("SELECT 1 FROM pg_extension WHERE extname = 'vector'").any?
         rescue StandardError => e
-          Legion::Logging.debug("Vector#available? check failed: #{e.message}") if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, handled: true, operation: :vector_available?)
           false
         end
 
@@ -18,17 +22,17 @@ module Legion
           return false unless Legion::Data.connection&.adapter_scheme == :postgres
 
           Legion::Data.connection.run('CREATE EXTENSION IF NOT EXISTS vector')
-          Legion::Logging.info 'pgvector extension enabled' if defined?(Legion::Logging)
+          log.info 'pgvector extension enabled'
           true
         rescue StandardError => e
-          Legion::Logging.warn("pgvector extension creation failed: #{e.message}") if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, handled: true, operation: :ensure_vector_extension)
           false
         end
 
         def cosine_search(table:, column:, query_vector:, limit: 10, min_similarity: 0.0)
           return [] unless available?
 
-          Legion::Logging.debug "Vector cosine_search: table=#{table} column=#{column} limit=#{limit}" if defined?(Legion::Logging)
+          log.debug "Vector cosine_search: table=#{table} column=#{column} limit=#{limit}"
           vec_literal = vector_literal(query_vector)
           ds = Legion::Data.connection[table]
                            .select_all
@@ -43,7 +47,7 @@ module Legion
         def l2_search(table:, column:, query_vector:, limit: 10)
           return [] unless available?
 
-          Legion::Logging.debug "Vector l2_search: table=#{table} column=#{column} limit=#{limit}" if defined?(Legion::Logging)
+          log.debug "Vector l2_search: table=#{table} column=#{column} limit=#{limit}"
           vec_literal = vector_literal(query_vector)
           Legion::Data.connection[table]
                       .select_all

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.6.18'
+    VERSION = '1.6.20'
   end
 end

data/lib/legion/data.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
 require 'legion/data/version'
 require 'legion/data/settings'
 require 'sequel'
@@ -16,16 +17,38 @@ require_relative 'data/rls'
 require_relative 'data/extract'
 require_relative 'data/audit_record'
 
+unless Legion::Logging::Helper.method_defined?(:handle_exception)
+  module Legion
+    module Logging
+      module Helper
+        def handle_exception(exception, task_id: nil, level: :error, handled: true, **opts)
+          context = opts.map { |key, value| "#{key}=#{value.inspect}" }.join(' ')
+          message = "#{exception.class}: #{exception.message}"
+          message = "#{message} task_id=#{task_id}" if task_id
+          message = "#{message} handled=#{handled}"
+          message = "#{message} #{context}" unless context.empty?
+          warn("[#{level}] #{message}")
+        rescue StandardError => e
+          warn("handle_exception fallback failed: #{e.class}: #{e.message}")
+        end
+      end
+    end
+  end
+end
+
 module Legion
   module Data
     class << self
+      include Legion::Logging::Helper
+
       def setup
+        log.info 'Legion::Data setup starting'
         connection_setup
         migrate
         load_models
         setup_cache
         setup_local
-        Legion::Logging.info 'Legion::Data setup complete' if defined?(Legion::Logging)
+        log.info 'Legion::Data setup complete'
       end
 
       def connection_setup
@@ -59,7 +82,8 @@ module Legion
 
       def connected?
         Legion::Settings[:data][:connected] == true
-      rescue StandardError
+      rescue StandardError => e
+        handle_exception(e, level: :debug, handled: true, operation: :connected?)
         false
       end
 
@@ -75,7 +99,8 @@ module Legion
         @write_privileges[table_name] = connection
                                         .fetch("SELECT has_table_privilege(current_user, ?, 'INSERT') AS can", table_name.to_s)
                                         .first[:can] == true
-      rescue StandardError
+      rescue StandardError => e
+        handle_exception(e, level: :warn, handled: true, operation: :can_write?, table: table_name)
         @write_privileges[table_name] = false if @write_privileges
         false
       end
@@ -92,7 +117,8 @@ module Legion
         @read_privileges[table_name] = connection
                                        .fetch("SELECT has_table_privilege(current_user, ?, 'SELECT') AS can", table_name.to_s)
                                        .first[:can] == true
-      rescue StandardError
+      rescue StandardError => e
+        handle_exception(e, level: :warn, handled: true, operation: :can_read?, table: table_name)
         @read_privileges[table_name] = false if @read_privileges
         false
       end
@@ -111,17 +137,18 @@ module Legion
       def setup_static_cache
         [Model::Extension, Model::Runner, Model::Function].each do |model|
           model.plugin :static_cache
-          Legion::Logging.debug("StaticCache enabled for #{model}") if defined?(Legion::Logging)
+          log.debug("StaticCache enabled for #{model}")
         rescue StandardError => e
-          Legion::Logging.warn("StaticCache failed for #{model}: #{e.message}") if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, operation: :setup_static_cache, model: model.to_s)
         end
-        Legion::Logging.info 'Legion::Data static cache loaded' if defined?(Legion::Logging)
+        log.info 'Legion::Data static cache loaded'
       end
 
       def reload_static_cache
         [Model::Extension, Model::Runner, Model::Function].each do |model|
           model.load_cache if model.respond_to?(:load_cache)
         end
+        log.info 'Legion::Data static cache reloaded'
       end
 
       def setup_external_cache
@@ -132,17 +159,17 @@ module Legion
           Model::Setting      => ttl
         }.each do |model, model_ttl|
           model.plugin :caching, ::Legion::Cache, ttl: model_ttl
-          Legion::Logging.debug("Caching enabled for #{model} (ttl: #{model_ttl})") if defined?(Legion::Logging)
+          log.debug("Caching enabled for #{model} (ttl: #{model_ttl})")
         rescue StandardError => e
-          Legion::Logging.warn("Caching failed for #{model}: #{e.message}") if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, operation: :setup_external_cache, model: model.to_s, ttl: model_ttl)
         end
-        Legion::Logging.info 'Legion::Data external cache connected' if defined?(Legion::Logging)
+        log.info 'Legion::Data external cache connected'
       end
 
       def shutdown
         Legion::Data::Local.shutdown if defined?(Legion::Data::Local) && Legion::Data::Local.connected?
         Legion::Data::Connection.shutdown
-        Legion::Logging.info 'Legion::Data shutdown complete' if defined?(Legion::Logging)
+        log.info 'Legion::Data shutdown complete'
       end
 
       private
@@ -152,7 +179,7 @@ module Legion
 
         Legion::Data::Local.setup
       rescue StandardError => e
-        Legion::Logging.warn "Legion::Data::Local failed to setup: #{e.message}" if defined?(Legion::Logging)
+        handle_exception(e, level: :warn, operation: :setup_local)
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.6.18
+  version: 1.6.20
 platform: ruby
 authors:
 - Esity
@@ -29,28 +29,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.8
+        version: 1.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.8
+        version: 1.5.0
 - !ruby/object:Gem::Dependency
   name: legion-settings
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.12
+        version: 1.3.26
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.12
+        version: 1.3.26
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement