legion-data 1.6.17 → 1.6.19

data/lib/legion/data/extract/handlers/vtt.rb

@@ -32,6 +32,7 @@ module Legion
               }
             }
           rescue StandardError => e
+            handle_exception(e, level: :warn, handled: true, operation: :extract_vtt)
             { text: nil, error: e.message }
           end
 

data/lib/legion/data/extract/handlers/xlsx.rb

@@ -25,9 +25,11 @@ module Legion
             end
             text = sheets.join("\n\n")
             { text: text, metadata: { sheets: workbook.worksheets.size } }
-          rescue LoadError
+          rescue LoadError => e
+            handle_exception(e, level: :warn, handled: true, operation: :extract_xlsx, gem: gem_name)
             { text: nil, error: :gem_not_installed, gem: gem_name }
           rescue StandardError => e
+            handle_exception(e, level: :warn, handled: true, operation: :extract_xlsx)
             { text: nil, error: e.message }
           end
         end

data/lib/legion/data/extract.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
 require_relative 'extract/type_detector'
 require_relative 'extract/handlers/base'
 
@@ -7,6 +8,8 @@ module Legion
   module Data
     module Extract
       class << self
+        include Legion::Logging::Helper
+
         def extract(source, type: :auto)
           detected_type = type == :auto ? TypeDetector.detect(source) : type&.to_sym
           return { success: false, text: nil, error: :unknown_type } unless detected_type
@@ -19,13 +22,17 @@ module Legion
                      gem: handler.gem_name, type: detected_type }
           end
 
+          log.info "Extract starting type=#{detected_type} handler=#{handler.name}"
           result = handler.extract(source)
           if result[:text]
+            log.info "Extract succeeded type=#{detected_type}"
             { success: true, text: result[:text], metadata: result[:metadata], type: detected_type }
           else
+            log.warn "Extract failed type=#{detected_type} error=#{result[:error]}"
             { success: false, text: nil, error: result[:error], type: detected_type }
           end
         rescue StandardError => e
+          handle_exception(e, level: :error, handled: true, operation: :extract, type: detected_type)
           { success: false, text: nil, error: e.message, type: detected_type }
         end
 

data/lib/legion/data/helper.rb

@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 module Legion
   module Data
     module Helper
+      include Legion::Logging::Helper
+
       def data_path
         @data_path ||= "#{full_path}/data"
       end
@@ -39,7 +43,8 @@ module Legion
 
       def data_adapter
         Legion::Data::Connection.adapter
-      rescue StandardError
+      rescue StandardError => e
+        handle_exception(e, level: :warn, handled: true, operation: :data_adapter)
         :unknown
       end
 
@@ -47,7 +52,8 @@ module Legion
         return {} unless data_connected?
 
         Legion::Data::Connection.pool_stats
-      rescue StandardError
+      rescue StandardError => e
+        handle_exception(e, level: :warn, handled: true, operation: :data_pool_stats)
         {}
       end
 
@@ -55,7 +61,8 @@ module Legion
         return {} unless data_connected?
 
         Legion::Data.stats
-      rescue StandardError
+      rescue StandardError => e
+        handle_exception(e, level: :warn, handled: true, operation: :data_stats)
         {}
       end
 
@@ -63,7 +70,8 @@ module Legion
         return {} unless local_data_connected?
 
         Legion::Data::Local.stats
-      rescue StandardError
+      rescue StandardError => e
+        handle_exception(e, level: :warn, handled: true, operation: :local_data_stats)
         {}
       end
 
@@ -71,13 +79,15 @@ module Legion
 
       def data_can_read?(table_name)
         Legion::Data.can_read?(table_name)
-      rescue StandardError
+      rescue StandardError => e
+        handle_exception(e, level: :warn, handled: true, operation: :data_can_read, table: table_name)
         false
       end
 
       def data_can_write?(table_name)
         Legion::Data.can_write?(table_name)
-      rescue StandardError
+      rescue StandardError => e
+        handle_exception(e, level: :warn, handled: true, operation: :data_can_write, table: table_name)
         false
       end
     end

data/lib/legion/data/local.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 require 'sequel'
 require 'sequel/extensions/migration'
 
@@ -7,6 +9,8 @@ module Legion
   module Data
     module Local
       class << self
+        include Legion::Logging::Helper
+
         attr_reader :connection, :db_path
 
         def setup(database: nil, **)
@@ -26,14 +30,21 @@ module Legion
           if local_settings[:query_log]
             log_path = File.join(Legion::Data::Connection::QUERY_LOG_DIR, 'data-local-query.log')
             @query_file_logger = Legion::Data::Connection::QueryFileLogger.new(log_path)
-            opts[:logger]        = @query_file_logger
-            opts[:sql_log_level] = :debug
+            opts[:logger]          = @query_file_logger
+            opts[:sql_log_level]   = :debug
+          elsif data[:log] && defined?(Legion::Logging)
+            opts[:logger]          = build_local_logger
+            opts[:sql_log_level]   = resolved_sql_log_level
+            opts[:log_warn_duration] = resolved_log_warn_duration
           end
 
           @connection = ::Sequel.connect(opts)
           @connected = true
           run_migrations
-          Legion::Logging.info "Legion::Data::Local connected to #{db_file}" if defined?(Legion::Logging)
+          log.info "Legion::Data::Local connected to #{db_file}"
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: false, operation: :local_setup, database: db_file)
+          raise
         end
 
         def shutdown
@@ -82,7 +93,8 @@ module Legion
              wal_autocheckpoint cache_size busy_timeout].each do |pragma|
             val = begin
               @connection.fetch("PRAGMA #{pragma}").single_value
-            rescue StandardError
+            rescue StandardError => e
+              handle_exception(e, level: :warn, handled: true, operation: :local_stats_pragma, pragma: pragma)
               nil
             end
             stats[pragma.to_sym] = val unless val.nil?
@@ -92,6 +104,7 @@ module Legion
 
           stats
         rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: :local_stats, database: @db_path)
           { connected: connected?, error: e.message }
         end
 
@@ -119,7 +132,7 @@ module Legion
           table = :"schema_migrations_#{name}"
           ::Sequel::TimestampMigrator.new(@connection, path, table: table).run
         rescue StandardError => e
-          Legion::Logging.warn "Local migration failed for #{path}: #{e.message}" if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, handled: true, operation: :local_migration, name: name, path: path)
         end
 
         def local_settings
@@ -127,6 +140,41 @@ module Legion
 
           Legion::Settings[:data]&.dig(:local) || {}
         end
+
+        def build_local_logger
+          tagged = if defined?(Legion::Logging::TaggedLogger) && respond_to?(:tagged_logger_settings, true)
+                     Legion::Logging::TaggedLogger.new(
+                       segments: %w[data local],
+                       **send(:tagged_logger_settings)
+                     )
+                   else
+                     Legion::Data::Connection::SegmentedTaggedLogger.new(segments: %w[data local])
+                   end
+          Legion::Data::Connection::SlowQueryLogger.new(tagged)
+        rescue StandardError => e
+          if respond_to?(:handle_exception, true)
+            handle_exception(e, level: :warn, handled: true, operation: :build_local_logger)
+          else
+            log.warn("build_local_logger failed: #{e.class}: #{e.message}")
+          end
+          Legion::Data::Connection::SlowQueryLogger.new(
+            Legion::Data::Connection::SegmentedTaggedLogger.new(segments: %w[data local], logger: log)
+          )
+        end
+
+        def resolved_sql_log_level
+          (local_settings[:sql_log_level] || Legion::Settings[:data][:sql_log_level] || 'debug').to_sym
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: :resolved_sql_log_level)
+          :debug
+        end
+
+        def resolved_log_warn_duration
+          local_settings[:log_warn_duration] || Legion::Settings[:data][:log_warn_duration]
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: :resolved_log_warn_duration)
+          nil
+        end
       end
     end
   end

data/lib/legion/data/migration.rb

@@ -1,16 +1,21 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 require 'sequel/extensions/migration'
 
 module Legion
   module Data
     module Migration
       class << self
+        include Legion::Logging::Helper
+
         def migrate(connection = Legion::Data.connection, path = "#{__dir__}/migrations", **)
           Legion::Settings[:data][:migrations][:version] = Sequel::Migrator.run(connection, path, **)
-          Legion::Logging.info("Legion::Data::Migration ran successfully to version #{Legion::Settings[:data][:migrations][:version]}")
+          log.info("Legion::Data::Migration ran successfully to version #{Legion::Settings[:data][:migrations][:version]}")
           Legion::Settings[:data][:migrations][:ran] = true
         rescue Sequel::DatabaseError => e
+          handle_exception(e, level: :error, handled: false, operation: :migrate, path: path)
           if e.message.include?('InsufficientPrivilege') || e.message.include?('permission denied')
             raise Sequel::DatabaseError,
                   "#{e.message}\n  Hint: the database user lacks CREATE on schema public " \

data/lib/legion/data/migrations/044_expand_memory_traces.rb

@@ -29,7 +29,10 @@ Sequel.migration do
 
     indexes = begin
       db.indexes(:memory_traces).keys
-    rescue StandardError
+    rescue StandardError => e
+      if defined?(Legion::Data) && Legion::Data.respond_to?(:handle_exception)
+        Legion::Data.handle_exception(e, level: :warn, handled: true, operation: :migration_044_indexes)
+      end
       []
     end
 

data/lib/legion/data/migrations/061_add_versioning_and_expiry.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    next unless adapter_scheme == :postgres
+    next unless table_exists?(:apollo_entries)
+
+    existing_columns = schema(:apollo_entries).map(&:first)
+
+    alter_table(:apollo_entries) do
+      add_column :parent_knowledge_id, :uuid, null: true unless existing_columns.include?(:parent_knowledge_id)
+      add_column :is_latest, :boolean, null: false, default: true unless existing_columns.include?(:is_latest)
+      add_column :supersession_type, String, size: 20, null: true unless existing_columns.include?(:supersession_type)
+      add_column :expires_at, :timestamptz, null: true unless existing_columns.include?(:expires_at)
+      add_column :forget_reason, String, size: 255, null: true unless existing_columns.include?(:forget_reason)
+      add_column :is_inference, :boolean, null: false, default: false unless existing_columns.include?(:is_inference)
+    end
+
+    add_index :apollo_entries, :parent_knowledge_id, name: :idx_apollo_parent_knowledge, if_not_exists: true
+    add_index :apollo_entries, %i[parent_knowledge_id is_latest],
+              name:          :idx_apollo_version_chain,
+              where:         Sequel.lit('is_latest = true'),
+              if_not_exists: true
+    add_index :apollo_entries, :expires_at,
+              name:          :idx_apollo_expiry,
+              where:         Sequel.lit("expires_at IS NOT NULL AND status != 'archived'"),
+              if_not_exists: true
+    add_index :apollo_entries, :is_inference,
+              name:          :idx_apollo_inference,
+              where:         Sequel.lit('is_inference = true'),
+              if_not_exists: true
+  end
+
+  down do
+    next unless adapter_scheme == :postgres
+    next unless table_exists?(:apollo_entries)
+
+    existing_columns = schema(:apollo_entries).map(&:first)
+
+    alter_table(:apollo_entries) do
+      drop_column :parent_knowledge_id if existing_columns.include?(:parent_knowledge_id)
+      drop_column :is_latest if existing_columns.include?(:is_latest)
+      drop_column :supersession_type if existing_columns.include?(:supersession_type)
+      drop_column :expires_at if existing_columns.include?(:expires_at)
+      drop_column :forget_reason if existing_columns.include?(:forget_reason)
+      drop_column :is_inference if existing_columns.include?(:is_inference)
+    end
+  end
+end

data/lib/legion/data/model.rb

@@ -1,9 +1,13 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 module Legion
   module Data
     module Models
       class << self
+        include Legion::Logging::Helper
+
         attr_reader :loaded_models
 
         def models
@@ -13,7 +17,7 @@ module Legion
         end
 
         def load
-          Legion::Logging.info 'Loading Legion::Data::Models'
+          log.info 'Loading Legion::Data::Models'
           @loaded_models ||= []
           require_sequel_models(models)
           Legion::Settings[:data][:models][:loaded] = true
@@ -25,13 +29,13 @@ module Legion
         end
 
         def load_sequel_model(model)
-          Legion::Logging.debug("Trying to load #{model}.rb")
+          log.debug("Trying to load #{model}.rb")
           require_relative "models/#{model}"
           @loaded_models << model
-          Legion::Logging.debug("Successfully loaded #{model}")
+          log.debug("Successfully loaded #{model}")
           model
         rescue LoadError => e
-          Legion::Logging.fatal("Failed to load #{model}")
+          handle_exception(e, level: :fatal, operation: :load_sequel_model, model: model)
           raise e unless Legion::Settings[:data][:models][:continue_on_fail]
         end
       end

data/lib/legion/data/models/audit_log.rb

@@ -1,9 +1,13 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 module Legion
   module Data
     module Model
       class AuditLog < Sequel::Model(:audit_log)
+        include Legion::Logging::Helper
+
         VALID_EVENT_TYPES = %w[runner_execution lifecycle_transition].freeze
         VALID_STATUSES    = %w[success failure denied].freeze
 
@@ -18,7 +22,7 @@ module Legion
 
           Legion::JSON.load(detail)
         rescue StandardError => e
-          Legion::Logging.warn("AuditLog#parsed_detail JSON parse failed: #{e.message}") if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, handled: true, operation: :parsed_detail, id: self[:id])
           nil
         end
 

data/lib/legion/data/models/audit_record.rb

@@ -1,9 +1,13 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 module Legion
   module Data
     module Model
       class AuditRecord < Sequel::Model(:audit_records)
+        include Legion::Logging::Helper
+
         # Enforce append-only semantics at the application layer.
         # PostgreSQL enforces this at the DB layer via rules (migration 058);
         # the application guard covers SQLite and MySQL.
@@ -21,7 +25,7 @@ module Legion
 
           Legion::JSON.load(metadata)
         rescue StandardError => e
-          Legion::Logging.warn "AuditRecord#parsed_metadata failed: #{e.message}" if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, handled: true, operation: :parsed_metadata, id: self[:id])
           {}
         end
       end

data/lib/legion/data/models/function.rb

@@ -1,9 +1,13 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 module Legion
   module Data
     module Model
       class Function < Sequel::Model
+        include Legion::Logging::Helper
+
         many_to_one :runner
         one_to_many :trigger_relationships, class: 'Legion::Data::Model::Relationship', key: :trigger_id
         one_to_many :action_relationships, class: 'Legion::Data::Model::Relationship', key: :action_id
@@ -13,7 +17,7 @@ module Legion
 
           ::JSON.parse(embedding)
         rescue ::JSON::ParserError => e
-          Legion::Logging.debug("Function#embedding_vector JSON parse failed: #{e.message}") if defined?(Legion::Logging)
+          handle_exception(e, level: :debug, handled: true, operation: :embedding_vector, id: self[:id])
           nil
         end
 

data/lib/legion/data/models/node.rb

@@ -1,9 +1,13 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 module Legion
   module Data
     module Model
       class Node < Sequel::Model
+        include Legion::Logging::Helper
+
         # one_to_many :task_log
 
         def parsed_metrics
@@ -11,7 +15,7 @@ module Legion
 
           Legion::JSON.load(metrics)
         rescue StandardError => e
-          Legion::Logging.debug("Node#parsed_metrics JSON parse failed: #{e.message}") if defined?(Legion::Logging)
+          handle_exception(e, level: :debug, handled: true, operation: :parsed_metrics, id: self[:id])
           nil
         end
 
@@ -20,7 +24,7 @@ module Legion
 
           Legion::JSON.load(hosted_worker_ids)
         rescue StandardError => e
-          Legion::Logging.debug("Node#parsed_hosted_worker_ids JSON parse failed: #{e.message}") if defined?(Legion::Logging)
+          handle_exception(e, level: :debug, handled: true, operation: :parsed_hosted_worker_ids, id: self[:id])
           []
         end
       end

data/lib/legion/data/partition_manager.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 module Legion
   module Data
     module PartitionManager
       NOT_POSTGRES = { skipped: true, reason: 'not_postgres' }.freeze
 
       class << self
+        include Legion::Logging::Helper
+
         def ensure_partitions(table:, months_ahead: 3)
           return NOT_POSTGRES unless postgres?
 
@@ -28,16 +32,17 @@ module Legion
             after_count = partition_names_for(table).size
 
             if after_count > before_count
-              log_info("Created partition #{partition}") if logging?
+              log.info("Created partition #{partition}")
               created << partition
             else
               existing << partition
             end
           end
 
+          log.info "PartitionManager ensure_partitions table=#{table} created=#{created.size} existing=#{existing.size}"
           { created: created, existing: existing }
         rescue StandardError => e
-          log_warn("ensure_partitions failed for #{table}: #{e.message}") if logging?
+          handle_exception(e, level: :warn, handled: true, operation: :ensure_partitions, table: table, months_ahead: months_ahead)
           { created: [], existing: [], error: e.message }
         end
 
@@ -54,16 +59,17 @@ module Legion
 
             if part_date < cutoff
               Legion::Data.connection.run("DROP TABLE #{part}")
-              log_info("Dropped partition #{part}") if logging?
+              log.info("Dropped partition #{part}")
               dropped << part
             else
               retained << part
             end
           end
 
+          log.info "PartitionManager drop_old_partitions table=#{table} dropped=#{dropped.size} retained=#{retained.size}"
           { dropped: dropped, retained: retained }
         rescue StandardError => e
-          log_warn("drop_old_partitions failed for #{table}: #{e.message}") if logging?
+          handle_exception(e, level: :warn, handled: true, operation: :drop_old_partitions, table: table, retention_months: retention_months)
           { dropped: [], retained: [], error: e.message }
         end
 
@@ -80,12 +86,14 @@ module Legion
             ORDER  BY c.relname
           SQL
 
-          Legion::Data.connection.fetch(sql).map do |row|
+          partitions = Legion::Data.connection.fetch(sql).map do |row|
             from_val, to_val = parse_bound(row[:bound])
             { name: row[:name], from: from_val, to: to_val }
           end
+          log.info "PartitionManager list_partitions table=#{table} count=#{partitions.size}"
+          partitions
         rescue StandardError => e
-          log_warn("list_partitions failed for #{table}: #{e.message}") if logging?
+          handle_exception(e, level: :warn, handled: true, operation: :list_partitions, table: table)
           []
         end
 
@@ -95,18 +103,6 @@ module Legion
           Legion::Data::Connection.adapter == :postgres
         end
 
-        def logging?
-          defined?(Legion::Logging)
-        end
-
-        def log_info(msg)
-          Legion::Logging.info(msg)
-        end
-
-        def log_warn(msg)
-          Legion::Logging.warn(msg)
-        end
-
         def partition_name(table, date)
           "#{table}_y#{date.strftime('%Y')}m#{date.strftime('%m')}"
         end
@@ -136,7 +132,7 @@ module Legion
 
           Legion::Data.connection.fetch(sql).map { |row| row[:name] }
         rescue StandardError => e
-          log_warn("partition_names_for #{table} failed: #{e.message}") if logging?
+          handle_exception(e, level: :warn, handled: true, operation: :partition_names_for, table: table)
           []
         end
 

data/lib/legion/data/retention.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
 require_relative 'archival/policy'
 
 module Legion
@@ -9,6 +10,8 @@ module Legion
       DEFAULT_ARCHIVE_AFTER_DAYS = 90
 
       class << self
+        include Legion::Logging::Helper
+
         def archive_old_records(table:, date_column: nil, archive_after_days: DEFAULT_ARCHIVE_AFTER_DAYS)
           db = Legion::Data.connection
           return { archived: 0, table: table } unless db
@@ -29,8 +32,19 @@ module Legion
             end
           end
 
-          Legion::Logging.info "Archived #{count} row(s) from #{table}" if defined?(Legion::Logging) && count.positive?
+          log.info "Archived #{count} row(s) from #{table}" if count.positive?
           { archived: count, table: table }
+        rescue StandardError => e
+          handle_exception(
+            e,
+            level:              :error,
+            handled:            false,
+            operation:          :archive_old_records,
+            table:              table,
+            date_column:        date_column,
+            archive_after_days: archive_after_days
+          )
+          raise
         end
 
         def purge_expired_records(table:, date_column: nil, retention_years: DEFAULT_RETENTION_YEARS)
@@ -43,9 +57,20 @@ module Legion
           expired = db[archive_table].where(Sequel.identifier(date_column) < cutoff)
           count = expired.count
           expired.delete if count.positive?
-          Legion::Logging.info "Purged #{count} expired row(s) from #{archive_table}" if defined?(Legion::Logging) && count.positive?
+          log.info "Purged #{count} expired row(s) from #{archive_table}" if count.positive?
 
           { purged: count, table: table }
+        rescue StandardError => e
+          handle_exception(
+            e,
+            level:           :error,
+            handled:         false,
+            operation:       :purge_expired_records,
+            table:           table,
+            date_column:     date_column,
+            retention_years: retention_years
+          )
+          raise
         end
 
         def retention_status(table:, date_column: nil)
@@ -67,6 +92,9 @@ module Legion
             oldest_active:   oldest_active,
             oldest_archived: oldest_archived
           }
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: false, operation: :retention_status, table: table, date_column: date_column)
+          raise
         end
 
         def archive_table_name(table)
@@ -90,6 +118,7 @@ module Legion
 
           source_schema = db.schema(source_table).to_h
 
+          log.info "Creating archive table #{archive_table} from #{source_table}"
           db.create_table(archive_table) do
             source_schema.each do |col_name, col_info|
               column col_name, col_info[:db_type]

data/lib/legion/data/rls.rb

@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 module Legion
   module Data
     module Rls
+      extend Legion::Logging::Helper
+
       RLS_TABLES = %i[
         tasks digital_workers audit_log memory_traces extensions
         functions runners nodes settings value_metrics
@@ -14,7 +18,8 @@ module Legion
         return false unless Legion::Settings[:data][:connected]
 
         Legion::Data.connection.adapter_scheme == :postgres
-      rescue StandardError
+      rescue StandardError => e
+        handle_exception(e, level: :warn, handled: true, operation: :rls_enabled)
         false
       end
 
@@ -30,7 +35,8 @@ module Legion
         return nil unless rls_enabled?
 
         Legion::Data.connection.fetch('SHOW app.current_tenant').first&.values&.first
-      rescue Sequel::DatabaseError
+      rescue Sequel::DatabaseError => e
+        handle_exception(e, level: :warn, handled: true, operation: :current_tenant)
         nil
       end