legion-data 1.4.9 → 1.4.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ed390611755acbac0c9701c31ee783d322f0ade227aaf13b2c9c74802db412e
-  data.tar.gz: d2c0a758edca0c1a2add30ca3e86a3fa949fe7bd1f79d71cfb48f7f6fb97e77a
+  metadata.gz: b4189d8ca05d1a51425ae559b6467e5be87f33b6189c21fd48501abc0102e7d8
+  data.tar.gz: 3aae8b800f4026cdff398d068111dc96f95c6ff465929ceb9634af514b46310f
 SHA512:
-  metadata.gz: 3e6f7e9fe8bfb5d699b51409bf7feee684014fe90639e4ecc0cfd620d8ffff6c0004611ef1ef442406abdf003f410cc30f45d5187855c75634237ace91bc5b2f
-  data.tar.gz: fbca2c2acf2862fa51018c42b200c4056d2a4cf6108620a5991c878ce279b2df5945439b7397740dfef3f20ad4d56f0f41d6f7ac5bd99fc1480eecb0825200a1
+  metadata.gz: 937412b3403fbcfb47c27d2f4a82ea8d0c8e02f7eec0f0f0540cbdae67ea79cd390e4cd5d721ab80d47197adcd73cce5b92997ea5414bed1c053d03bd076fd11
+  data.tar.gz: 11a1b127dedd48eb11d3b0479d1c3055a5183b32619dfcd17f842b42aa39d8bec327bc0a6e99fab805e256c1c24c7e4b23a98c5a81f625a82d425c6f0bc5a7b2

data/.rubocop.yml

@@ -49,3 +49,9 @@ Style/FrozenStringLiteralComment:
 
 Naming/FileName:
   Enabled: false
+
+Naming/VariableNumber:
+  Enabled: false
+
+Metrics/ParameterLists:
+  Max: 8

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Legion::Data Changelog
 
+## v1.4.11
+
+### Added
+- Read replica support: `read_replica_url` and `replicas` settings, `Connection.connect_with_replicas` via Sequel `server_block` extension, `read_server` and `replica_servers` class methods for read/write splitting
+- `PartitionManager`: PostgreSQL range partitioning helper — `ensure_partitions`, `drop_old_partitions`, `list_partitions` for monthly table partitioning
+- `Archiver`: cold storage archival pipeline — batch export to JSONL+gzip, SHA-256 manifest, pluggable upload backends (S3, Azure, local tmpdir)
+- Migration 034: `archive_manifest` table (PostgreSQL only) for tracking archived batches
+- Archival settings: `retention_days`, `batch_size`, `storage_backend` defaults
+- 58 new specs (257 total, 0 failures)
+
+## v1.4.10
+
+### Added
+- TLS support for PostgreSQL connections: `sslmode`, `sslrootcert`, `sslcert`, `sslkey`
+- TLS support for MySQL connections: `ssl_mode`, `sslca`, `sslcert`, `sslkey`
+- `Connection.merge_tls_creds` resolves TLS config via `Legion::Crypt::TLS.resolve`
+- SQLite connections skip TLS entirely (local file, no network)
+
 ## v1.4.8
 
 ### Fixed

data/lib/legion/data/archiver.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'fileutils'
+require 'json'
+require 'securerandom'
+require 'tmpdir'
+require 'zlib'
+
+module Legion
+  module Data
+    module Archiver
+      class UploadError < StandardError; end
+
+      class << self
+        def archive_table(table:, retention_days: 90, batch_size: 1000, storage_backend: nil)
+          return { skipped: true, reason: 'not_postgres' } unless postgres?
+
+          conn = Legion::Data.connection
+          cutoff = Time.now - (retention_days * 86_400)
+          now = Time.now.utc
+
+          batches = 0
+          total_rows = 0
+          paths = []
+          batch_n = 0
+
+          loop do
+            batch_n += 1
+            rows = conn[table].where { created_at < cutoff }.limit(batch_size).all
+            break if rows.empty?
+
+            ids = rows.map { |r| r[:id] }
+            jsonl = serialize_rows(rows)
+            compressed = gzip_compress(jsonl)
+            checksum = Digest::SHA256.hexdigest(compressed)
+            batch_id = SecureRandom.uuid
+
+            path = upload_batch(
+              data:    compressed,
+              table:   table.to_s,
+              year:    now.year,
+              month:   now.month,
+              batch_n: batch_n,
+              backend: storage_backend
+            )
+
+            conn.transaction do
+              conn[:archive_manifest].insert(
+                batch_id:     batch_id,
+                source_table: table.to_s,
+                row_count:    rows.size,
+                checksum:     checksum,
+                storage_path: path,
+                archived_at:  now
+              )
+              conn[table].where(id: ids).delete
+            end
+
+            batches += 1
+            total_rows += rows.size
+            paths << path
+          end
+
+          { batches: batches, total_rows: total_rows, paths: paths }
+        end
+
+        def upload_batch(data:, table:, year:, month:, batch_n:, backend:)
+          case backend
+          when :s3
+            upload_s3(data: data, table: table, year: year, month: month, batch_n: batch_n)
+          when :azure
+            upload_azure(data: data, table: table, year: year, month: month, batch_n: batch_n)
+          else
+            upload_tmpdir(data: data, table: table, year: year, month: month, batch_n: batch_n)
+          end
+        end
+
+        def manifest_stats
+          return {} unless postgres?
+          return {} unless Legion::Data.connection.table_exists?(:archive_manifest)
+
+          Legion::Data.connection[:archive_manifest]
+                      .group_and_count(:source_table)
+                      .select_append(
+                        Sequel.function(:sum, :row_count).as(:total_rows),
+                        Sequel.function(:min, :archived_at).as(:earliest),
+                        Sequel.function(:max, :archived_at).as(:latest)
+                      )
+                      .all
+                      .to_h do |row|
+            [row[:source_table], {
+              batches:    row[:count],
+              total_rows: row[:total_rows].to_i,
+              earliest:   row[:earliest],
+              latest:     row[:latest]
+            }]
+          end
+        end
+
+        private
+
+        def postgres?
+          Legion::Data::Connection.adapter == :postgres
+        end
+
+        def serialize_rows(rows)
+          rows.map { |row| json_dump(row) }.join("\n")
+        end
+
+        def json_dump(obj)
+          if defined?(Legion::JSON)
+            Legion::JSON.dump(obj)
+          else
+            ::JSON.generate(obj)
+          end
+        end
+
+        def gzip_compress(data)
+          output = StringIO.new
+          output.binmode
+          gz = Zlib::GzipWriter.new(output)
+          gz.write(data)
+          gz.close
+          output.string
+        end
+
+        def upload_s3(data:, table:, year:, month:, batch_n:)
+          raise UploadError, 'S3 backend not available: Legion::Extensions::S3::Runners::Put not defined' unless defined?(Legion::Extensions::S3::Runners::Put)
+
+          key = "legion-archive/#{table}/#{year}/#{month}/batch_#{batch_n}.jsonl.gz"
+          Legion::Extensions::S3::Runners::Put.run(key: key, body: data)
+          "s3://#{key}"
+        rescue UploadError
+          raise
+        rescue StandardError => e
+          raise UploadError, "S3 upload failed: #{e.message}"
+        end
+
+        def upload_azure(data:, table:, year:, month:, batch_n:)
+          unless defined?(Legion::Extensions::AzureStorage::Runners::Upload)
+            raise UploadError, 'Azure backend not available: Legion::Extensions::AzureStorage::Runners::Upload not defined'
+          end
+
+          blob_name = "legion-archive/#{table}/#{year}/#{month}/batch_#{batch_n}.jsonl.gz"
+          Legion::Extensions::AzureStorage::Runners::Upload.run(blob_name: blob_name, data: data)
+          "azure://#{blob_name}"
+        rescue UploadError
+          raise
+        rescue StandardError => e
+          raise UploadError, "Azure upload failed: #{e.message}"
+        end
+
+        def upload_tmpdir(data:, table:, year:, month:, batch_n:)
+          dir = File.join(Dir.tmpdir, 'legion-archive', table.to_s, year.to_s, month.to_s)
+          FileUtils.mkdir_p(dir)
+          path = File.join(dir, "batch_#{batch_n}.jsonl.gz")
+          File.binwrite(path, data)
+          "file://#{path}"
+        rescue StandardError => e
+          raise UploadError, "Tmpdir upload failed: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/connection.rb

@@ -34,6 +34,7 @@ module Legion
                     end
           Legion::Settings[:data][:connected] = true
           configure_logging
+          connect_with_replicas
         end
 
         def shutdown
@@ -41,10 +42,67 @@ module Legion
           Legion::Settings[:data][:connected] = false
         end
 
+        def connect_with_replicas
+          return unless adapter == :postgres
+
+          replica_url  = Legion::Settings[:data][:read_replica_url]
+          replica_list = Array(Legion::Settings[:data][:replicas]).dup
+
+          replica_list.prepend(replica_url) if replica_url && !replica_url.empty?
+          replica_list.uniq!
+          replica_list.compact!
+
+          return if replica_list.empty?
+
+          @sequel.extension(:server_block)
+
+          replica_list.each_with_index do |url, idx|
+            @sequel.add_servers("read_#{idx}": url)
+          end
+
+          @replica_servers = replica_list.each_with_index.map { |_, idx| :"read_#{idx}" }
+        end
+
+        def read_server
+          return :default if @replica_servers.nil? || @replica_servers.empty?
+
+          :read_0
+        end
+
+        def replica_servers
+          @replica_servers || []
+        end
+
+        def merge_tls_creds(creds, adapter:, port:)
+          return creds if adapter == :sqlite
+          return creds unless defined?(Legion::Crypt::TLS)
+
+          tls = Legion::Crypt::TLS.resolve(data_tls_settings, port: port)
+          return creds unless tls[:enabled]
+
+          case adapter
+          when :postgres
+            creds[:sslmode]     = tls[:verify] == :none ? 'require' : 'verify-full'
+            creds[:sslrootcert] = tls[:ca] if tls[:ca]
+            creds[:sslcert]     = tls[:cert] if tls[:cert]
+            creds[:sslkey]      = tls[:key] if tls[:key]
+          when :mysql2
+            creds[:ssl_mode] = tls[:verify] == :none ? 'required' : 'verify_identity'
+            creds[:sslca]    = tls[:ca] if tls[:ca]
+            creds[:sslcert]  = tls[:cert] if tls[:cert]
+            creds[:sslkey]   = tls[:key] if tls[:key]
+          end
+
+          creds
+        end
+
         def creds_builder(final_creds = {})
           final_creds.merge! Legion::Data::Settings.creds(adapter)
           final_creds.merge! Legion::Settings[:data][:creds] if Legion::Settings[:data][:creds].is_a? Hash
 
+          port = final_creds[:port]
+          merge_tls_creds(final_creds, adapter: adapter, port: port)
+
           return final_creds if Legion::Settings[:vault].nil?
 
           if Legion::Settings[:vault][:connected] && ::Vault.sys.mounts.key?(:database)
@@ -58,6 +116,14 @@ module Legion
 
         private
 
+        def data_tls_settings
+          return {} unless defined?(Legion::Settings)
+
+          Legion::Settings[:data][:tls] || {}
+        rescue StandardError
+          {}
+        end
+
         def dev_fallback?
           data_settings = Legion::Settings[:data]
           data_settings[:dev_mode] == true && data_settings[:dev_fallback] != false

data/lib/legion/data/migrations/034_add_archive_manifest.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    next unless [:postgres].include?(adapter_scheme)
+    next if table_exists?(:archive_manifest)
+
+    create_table(:archive_manifest) do
+      primary_key :id
+      String :batch_id, null: false, unique: true
+      String :source_table, null: false
+      Integer :row_count, null: false
+      String :checksum, null: false
+      String :storage_path, null: false
+      DateTime :archived_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+      column :metadata, :jsonb
+
+      index :source_table
+      index :archived_at
+    end
+  end
+
+  down do
+    next unless [:postgres].include?(adapter_scheme)
+
+    drop_table(:archive_manifest) if table_exists?(:archive_manifest)
+  end
+end

data/lib/legion/data/partition_manager.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module PartitionManager
+      NOT_POSTGRES = { skipped: true, reason: 'not_postgres' }.freeze
+
+      class << self
+        def ensure_partitions(table:, months_ahead: 3)
+          return NOT_POSTGRES unless postgres?
+
+          created = []
+          existing = []
+          base = Date.today
+
+          months_ahead.times do |i|
+            target = advance_months(base, i)
+            partition = partition_name(table, target)
+            from_str  = target.strftime('%Y-%m-%d')
+            to_str    = advance_months(target, 1).strftime('%Y-%m-%d')
+
+            ddl = "CREATE TABLE IF NOT EXISTS #{partition} " \
+                  "PARTITION OF #{table} " \
+                  "FOR VALUES FROM ('#{from_str}') TO ('#{to_str}')"
+
+            before_count = partition_names_for(table).size
+            Legion::Data.connection.run(ddl)
+            after_count = partition_names_for(table).size
+
+            if after_count > before_count
+              log_info("Created partition #{partition}") if logging?
+              created << partition
+            else
+              existing << partition
+            end
+          end
+
+          { created: created, existing: existing }
+        rescue StandardError => e
+          log_warn("ensure_partitions failed for #{table}: #{e.message}") if logging?
+          { created: [], existing: [], error: e.message }
+        end
+
+        def drop_old_partitions(table:, retention_months: 24)
+          return NOT_POSTGRES unless postgres?
+
+          cutoff = advance_months(Date.today, -retention_months)
+          dropped = []
+          retained = []
+
+          partition_names_for(table).each do |part|
+            part_date = parse_partition_date(part)
+            next unless part_date
+
+            if part_date < cutoff
+              Legion::Data.connection.run("DROP TABLE #{part}")
+              log_info("Dropped partition #{part}") if logging?
+              dropped << part
+            else
+              retained << part
+            end
+          end
+
+          { dropped: dropped, retained: retained }
+        rescue StandardError => e
+          log_warn("drop_old_partitions failed for #{table}: #{e.message}") if logging?
+          { dropped: [], retained: [], error: e.message }
+        end
+
+        def list_partitions(table:)
+          return NOT_POSTGRES unless postgres?
+
+          sql = <<~SQL
+            SELECT c.relname AS name,
+                   pg_get_expr(c.relpartbound, c.oid) AS bound
+            FROM   pg_inherits i
+            JOIN   pg_class    p ON p.oid = i.inhparent
+            JOIN   pg_class    c ON c.oid = i.inhrelid
+            WHERE  p.relname = '#{table}'
+            ORDER  BY c.relname
+          SQL
+
+          Legion::Data.connection.fetch(sql).map do |row|
+            from_val, to_val = parse_bound(row[:bound])
+            { name: row[:name], from: from_val, to: to_val }
+          end
+        rescue StandardError => e
+          log_warn("list_partitions failed for #{table}: #{e.message}") if logging?
+          []
+        end
+
+        private
+
+        def postgres?
+          Legion::Data::Connection.adapter == :postgres
+        end
+
+        def logging?
+          defined?(Legion::Logging)
+        end
+
+        def log_info(msg)
+          Legion::Logging.info(msg)
+        end
+
+        def log_warn(msg)
+          Legion::Logging.warn(msg)
+        end
+
+        def partition_name(table, date)
+          "#{table}_y#{date.strftime('%Y')}m#{date.strftime('%m')}"
+        end
+
+        def advance_months(date, months)
+          year  = date.year
+          month = date.month + months
+          while month > 12
+            month -= 12
+            year  += 1
+          end
+          while month < 1
+            month += 12
+            year  -= 1
+          end
+          Date.new(year, month, 1)
+        end
+
+        def partition_names_for(table)
+          sql = <<~SQL
+            SELECT c.relname AS name
+            FROM   pg_inherits i
+            JOIN   pg_class    p ON p.oid = i.inhparent
+            JOIN   pg_class    c ON c.oid = i.inhrelid
+            WHERE  p.relname = '#{table}'
+          SQL
+
+          Legion::Data.connection.fetch(sql).map { |row| row[:name] }
+        rescue StandardError
+          []
+        end
+
+        def parse_partition_date(partition_name)
+          match = partition_name.match(/y(\d{4})m(\d{2})$/)
+          return nil unless match
+
+          Date.new(match[1].to_i, match[2].to_i, 1)
+        end
+
+        def parse_bound(expr)
+          return [nil, nil] unless expr
+
+          matches = expr.scan(/'([^']+)'/)
+          from_val = matches[0]&.first
+          to_val   = matches[1]&.first
+          [from_val, to_val]
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/settings.rb

@@ -35,7 +35,10 @@ module Legion
           local:            local,
           dev_mode:         false,
           dev_fallback:     true,
-          connect_on_start: true
+          connect_on_start: true,
+          read_replica_url: nil,
+          replicas:         [],
+          archival:         archival
         }
       end
 
@@ -79,6 +82,14 @@ module Legion
         CREDS.fetch(adapter, CREDS[:sqlite]).dup
       end
 
+      def self.archival
+        {
+          retention_days:  90,
+          batch_size:      1000,
+          storage_backend: nil
+        }
+      end
+
       def self.cache
         {
           connected:   false,

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.4.9'
+    VERSION = '1.4.11'
   end
 end

data/lib/legion/data.rb

@@ -9,6 +9,8 @@ require 'legion/data/model'
 require 'legion/data/migration'
 require_relative 'data/local'
 require_relative 'data/spool'
+require_relative 'data/partition_manager'
+require_relative 'data/archiver'
 
 module Legion
   module Data

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.4.9
+  version: 1.4.11
 platform: ruby
 authors:
 - Esity
@@ -88,6 +88,7 @@ files:
 - lib/legion/data.rb
 - lib/legion/data/archival.rb
 - lib/legion/data/archival/policy.rb
+- lib/legion/data/archiver.rb
 - lib/legion/data/connection.rb
 - lib/legion/data/encryption/cipher.rb
 - lib/legion/data/encryption/key_provider.rb
@@ -129,6 +130,7 @@ files:
 - lib/legion/data/migrations/031_add_task_depth.rb
 - lib/legion/data/migrations/032_add_task_cancelled_at.rb
 - lib/legion/data/migrations/033_add_task_delay.rb
+- lib/legion/data/migrations/034_add_archive_manifest.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo_access_log.rb
 - lib/legion/data/models/apollo_entry.rb
@@ -147,6 +149,7 @@ files:
 - lib/legion/data/models/setting.rb
 - lib/legion/data/models/task.rb
 - lib/legion/data/models/task_log.rb
+- lib/legion/data/partition_manager.rb
 - lib/legion/data/retention.rb
 - lib/legion/data/settings.rb
 - lib/legion/data/spool.rb