legion-data 1.10.28 → 1.11.7

data/lib/legion/data/storage_tiers.rb

@@ -10,56 +10,62 @@ module Legion
       class << self
         include Legion::Logging::Helper
 
-        def archive_to_warm(table:, age_days: 90, batch_size: 1000)
-          return { archived: 0, reason: 'no_connection' } unless Legion::Data.connection
-          return { archived: 0, reason: 'no_archive_table' } unless Legion::Data.connection.table_exists?(:data_archive)
+        def archive_to_warm(model:, age_days: 90, batch_size: 1000)
+          return { archived: 0, reason: 'no_connection' } unless Legion::Data.connected?
+
+          table = model.table_name
+          db = model.db
+          raise ArgumentError, 'source model must use the DataArchive database' unless db.equal?(Legion::Data::Model::DataArchive.db)
 
           cutoff = Time.now - (age_days * 86_400)
-          records = Legion::Data.connection[table].where { created_at < cutoff }.limit(batch_size).all
-          return { archived: 0 } if records.empty?
+          archived = db.transaction(savepoint: true, **(db.database_type == :sqlite ? { mode: :immediate } : {})) do
+            candidates = model.where { created_at < cutoff }.order(:id).limit(batch_size)
+            candidates = candidates.for_update unless db.database_type == :sqlite
+            records = candidates.all
+            next 0 if records.empty?
 
-          Legion::Data.connection.transaction do
             records.each do |record|
-              Legion::Data.connection[:data_archive].insert(
+              Legion::Data::Model::DataArchive.create(
                 source_table: table.to_s, source_id: record[:id],
-                data:         Legion::JSON.dump(record),
+                data:         Legion::JSON.dump(record.values),
                 tier:         TIERS[:warm],
                 archived_at:  Time.now.utc
               )
             end
 
             ids = records.map { |r| r[:id] }
-            Legion::Data.connection[table].where(id: ids).delete
+            model.where(id: ids).delete
           end
+          return { archived: 0 } if archived.zero?
 
-          log.info "Archived #{records.size} row(s) from #{table} to warm tier"
-          { archived: records.size, table: table.to_s }
+          log.info "Archived #{archived} row(s) from #{table} to warm tier"
+          { archived: archived, table: table.to_s }
         rescue StandardError => e
           handle_exception(e, level: :error, handled: false, operation: :archive_to_warm, table: table, age_days: age_days, batch_size: batch_size)
           raise
         end
 
         def export_to_cold(age_days: 365, batch_size: 5000)
-          return { exported: 0 } unless Legion::Data.connection&.table_exists?(:data_archive)
+          return { exported: 0 } unless Legion::Data.connected?
 
           cutoff = Time.now - (age_days * 86_400)
-          records = Legion::Data.connection[:data_archive]
-                                .where(tier: TIERS[:warm])
-                                .where { archived_at < cutoff }
-                                .limit(batch_size).all
+          records = Legion::Data::Model::DataArchive
+                    .where(tier: TIERS[:warm])
+                    .where { archived_at < cutoff }
+                    .limit(batch_size).all
           return { exported: 0 } if records.empty?
 
           ids = records.map { |r| r[:id] }
-          Legion::Data.connection[:data_archive].where(id: ids).update(tier: TIERS[:cold])
+          Legion::Data::Model::DataArchive.where(id: ids).update(tier: TIERS[:cold])
           log.info "Exported #{records.size} row(s) to cold tier"
-          { exported: records.size, data: records }
+          { exported: records.size, data: records.map(&:values) }
         rescue StandardError => e
           handle_exception(e, level: :error, handled: false, operation: :export_to_cold, age_days: age_days, batch_size: batch_size)
           raise
         end
 
         def stats
-          return {} unless Legion::Data.connection&.table_exists?(:data_archive)
+          return {} unless Legion::Data.connected?
 
           { warm: count_tier(:warm), cold: count_tier(:cold) }
         rescue StandardError => e
@@ -70,7 +76,7 @@ module Legion
         private
 
         def count_tier(tier)
-          Legion::Data.connection[:data_archive].where(tier: TIERS[tier]).count
+          Legion::Data::Model::DataArchive.where(tier: TIERS[tier]).count
         rescue StandardError => e
           handle_exception(e, level: :warn, handled: true, operation: :storage_tiers_count, tier: tier)
           0

data/lib/legion/data/task/status.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Task
+      module Status
+        STATUSES = {
+          scheduled: 'task.scheduled', queued: 'task.queued', running: 'task.running', completed: 'task.completed',
+          exception: 'task.exception', failed: 'task.failed', skipped: 'task.skipped',
+          cancelled: 'task.cancelled', expired: 'task.expired', delayed: 'task.delayed',
+          dispatching: 'task.dispatching', multiplied: 'task.multiplied', retry_pending: 'task.retry_pending',
+          unknown: 'task.unknown', conditioner_queued: 'conditioner.queued', conditioner_failed: 'conditioner.failed',
+          conditioner_exception: 'conditioner.exception', transformer_queued: 'transformer.queued',
+          transformer_succeeded: 'transformer.succeeded', transformer_validation_failed: 'transformer.validation_failed',
+          transformer_failed: 'transformer.failed', transformer_exception: 'transformer.exception', blocked: 'task.blocked'
+        }.freeze
+        TERMINAL_STATUSES = STATUSES.values_at(:completed, :failed, :exception, :skipped, :cancelled, :expired, :multiplied,
+                                               :conditioner_failed, :conditioner_exception, :transformer_validation_failed,
+                                               :transformer_failed, :transformer_exception, :unknown, :blocked).freeze
+        LEGACY_TERMINAL_STATUSES = %w[completed complete failed error cancelled canceled timeout timed_out exception skipped expired multiplied].freeze
+
+        module Helper
+          def task_status(name:)
+            raise ArgumentError, 'Task status name must be a String or Symbol' unless name.is_a?(String) || name.is_a?(Symbol)
+
+            STATUSES.fetch(name.to_sym) { raise ArgumentError, "Unknown task status name #{name.inspect}" }
+          end
+
+          def task_terminal_status?(status:)
+            TERMINAL_STATUSES.include?(status) || LEGACY_TERMINAL_STATUSES.include?(status)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.10.28'
+    VERSION = '1.11.7'
   end
 end

data/lib/legion/data.rb

@@ -17,6 +17,7 @@ require_relative 'data/rls'
 require_relative 'data/extract'
 require_relative 'data/audit_record'
 require_relative 'data/audit_log_hash_chain'
+require_relative 'data/task/status'
 
 unless Legion::Logging::Helper.method_defined?(:handle_exception)
   module Legion
@@ -83,6 +84,8 @@ module Legion
       end
 
       def setup_cache
+        return unless Legion::Settings[:data][:models][:loaded]
+
         cache_settings = Legion::Settings[:data][:cache]
         setup_static_cache if cache_settings[:static_cache]
         setup_external_cache if cache_settings[:auto_enable] && defined?(::Legion::Cache)
@@ -209,7 +212,7 @@ module Legion
       end
 
       def setup_external_cache
-        ttl = Legion::Settings[:data][:cache][:ttl] || 60
+        ttl = Legion::Settings[:data][:cache][:ttl]
         {
           Model::Relationship => 10,
           Model::Node         => 10,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.10.28
+  version: 1.11.7
 platform: ruby
 authors:
 - Esity
@@ -57,14 +57,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.26
+        version: 1.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.26
+        version: 1.4.5
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -126,7 +126,6 @@ files:
 - lib/legion/data/audit_record.rb
 - lib/legion/data/auth_failure_handler.rb
 - lib/legion/data/connection.rb
-- lib/legion/data/connection_pool.rb
 - lib/legion/data/database_lock.rb
 - lib/legion/data/encryption/cipher.rb
 - lib/legion/data/encryption/key_provider.rb
@@ -288,6 +287,7 @@ files:
 - lib/legion/data/migrations/136_drop_llm_messages_seq_unique.rb
 - lib/legion/data/migrations/137_preserve_task_archive_fields.rb
 - lib/legion/data/migrations/138_add_append_claims.rb
+- lib/legion/data/migrations/139_create_task_logs.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo/access_log.rb
 - lib/legion/data/models/apollo/entries.rb
@@ -299,13 +299,16 @@ files:
 - lib/legion/data/models/apollo_entry.rb
 - lib/legion/data/models/apollo_expertise.rb
 - lib/legion/data/models/apollo_relation.rb
+- lib/legion/data/models/append_claim.rb
 - lib/legion/data/models/audit_log.rb
 - lib/legion/data/models/audit_record.rb
 - lib/legion/data/models/chain.rb
+- lib/legion/data/models/data_archive.rb
 - lib/legion/data/models/digital_worker.rb
 - lib/legion/data/models/extension.rb
 - lib/legion/data/models/extract_step_timing.rb
 - lib/legion/data/models/function.rb
+- lib/legion/data/models/governance_event.rb
 - lib/legion/data/models/identity/audit_log.rb
 - lib/legion/data/models/identity/group.rb
 - lib/legion/data/models/identity/group_memberships.rb
@@ -346,6 +349,7 @@ files:
 - lib/legion/data/settings.rb
 - lib/legion/data/spool.rb
 - lib/legion/data/storage_tiers.rb
+- lib/legion/data/task/status.rb
 - lib/legion/data/vector.rb
 - lib/legion/data/version.rb
 - scripts/pre-commit-rubocop.sh

data/lib/legion/data/connection_pool.rb

@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-
-require 'legion/logging/helper'
-
-module Legion
-  module Data
-    module ConnectionPool
-      include Legion::Logging::Helper
-
-      def self.extended(pool)
-        pool.instance_variable_set(:@shutdown_mutex, Mutex.new)
-        pool.instance_variable_set(:@shutdown_condition, ConditionVariable.new)
-        pool.instance_variable_set(:@shutdown_owners, {}.compare_by_identity)
-        pool.instance_variable_set(:@shutdown_state, :open)
-      end
-
-      def hold(server = :default)
-        owner = Sequel.current
-        @shutdown_mutex.synchronize do
-          raise Sequel::Error, 'Database pool is shutting down or closed' unless @shutdown_state == :open || @shutdown_owners.key?(owner)
-
-          @shutdown_owners[owner] = @shutdown_owners.fetch(owner, 0) + 1
-        end
-        begin
-          super
-        ensure
-          finish = @shutdown_mutex.synchronize do
-            @shutdown_owners[owner] -= 1
-            @shutdown_owners.delete(owner) if @shutdown_owners[owner].zero?
-            if @shutdown_state == :draining && @shutdown_owners.empty?
-              @shutdown_state = :closing
-              true
-            end
-          end
-          finish_shutdown if finish
-        end
-      end
-
-      def shutdown(timeout:, &on_closed)
-        deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout
-        finish = @shutdown_mutex.synchronize do
-          @shutdown_callback ||= on_closed
-          @shutdown_state = :draining if %i[open failed].include?(@shutdown_state)
-          if @shutdown_state == :draining && @shutdown_owners.empty?
-            @shutdown_state = :closing
-            true
-          end
-        end
-        finish_shutdown if finish
-        @shutdown_mutex.synchronize do
-          until @shutdown_state == :closed
-            remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-            return false if remaining <= 0 || @shutdown_state == :failed || @shutdown_owners.key?(Sequel.current)
-
-            @shutdown_condition.wait(@shutdown_mutex, remaining)
-          end
-        end
-        true
-      end
-
-      def shutdown_status
-        @shutdown_mutex.synchronize { { state: @shutdown_state, active_requests: @shutdown_owners.size } }
-      end
-
-      private
-
-      def finish_shutdown
-        disconnect
-        @shutdown_callback.call
-        @shutdown_mutex.synchronize do
-          @shutdown_state = :closed
-          @shutdown_condition.broadcast
-        end
-      rescue StandardError => e
-        @shutdown_mutex.synchronize do
-          @shutdown_state = :failed
-          @shutdown_condition.broadcast
-        end
-        handle_exception(e, level: :error, handled: true, operation: :pool_shutdown)
-      end
-    end
-  end
-end