legion-data 1.2.0 → 1.3.7

data/legion-data.gemspec

@@ -6,28 +6,28 @@ Gem::Specification.new do |spec|
   spec.name = 'legion-data'
   spec.version       = Legion::Data::VERSION
   spec.authors       = ['Esity']
-  spec.email         = %w[matthewdiverson@gmail.com ruby@optum.com]
+  spec.email         = ['matthewdiverson@gmail.com']
 
   spec.summary       = 'Manages the connects to the backend database'
   spec.description   = 'A LegionIO gem to connect to a persistent data store'
-  spec.homepage      = 'https://github.com/Optum/legion-data'
+  spec.homepage      = 'https://github.com/LegionIO/legion-data'
   spec.license       = 'Apache-2.0'
-  spec.required_ruby_version = '>= 2.5'
+  spec.required_ruby_version = '>= 3.4'
   spec.require_paths = ['lib']
   spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.test_files        = spec.files.select { |p| p =~ %r{^test/.*_test.rb} }
-  spec.extra_rdoc_files  = %w[README.md LICENSE CHANGELOG.md]
+  spec.extra_rdoc_files = %w[README.md LICENSE CHANGELOG.md]
   spec.metadata = {
-    'bug_tracker_uri' => 'https://github.com/Optum/legion-data/issues',
-    'changelog_uri' => 'https://github.com/Optum/legion-data/src/main/CHANGELOG.md',
-    'documentation_uri' => 'https://github.com/Optum/legion-data',
-    'homepage_uri' => 'https://github.com/Optum/LegionIO',
-    'source_code_uri' => 'https://github.com/Optum/legion-data',
-    'wiki_uri' => 'https://github.com/Optum/legion-data/wiki'
+    'bug_tracker_uri'       => 'https://github.com/LegionIO/legion-data/issues',
+    'changelog_uri'         => 'https://github.com/LegionIO/legion-data/blob/main/CHANGELOG.md',
+    'documentation_uri'     => 'https://github.com/LegionIO/legion-data',
+    'homepage_uri'          => 'https://github.com/LegionIO/LegionIO',
+    'source_code_uri'       => 'https://github.com/LegionIO/legion-data',
+    'wiki_uri'              => 'https://github.com/LegionIO/legion-data/wiki',
+    'rubygems_mfa_required' => 'true'
   }
 
   spec.add_dependency 'legion-logging'
   spec.add_dependency 'legion-settings'
-  spec.add_dependency 'mysql2'
-  spec.add_dependency 'sequel'
+  spec.add_dependency 'sequel', '>= 5.70'
+  spec.add_dependency 'sqlite3', '>= 2.0'
 end

data/lib/legion/data/connection.rb

@@ -1,27 +1,39 @@
+# frozen_string_literal: true
+
 require 'sequel'
 
 module Legion
   module Data
     module Connection
+      ADAPTERS = %i[sqlite mysql2 postgres].freeze
+
       class << self
         attr_accessor :sequel
 
         def adapter
-          @adapter ||= RUBY_ENGINE == 'jruby' ? :jdbc : :mysql2
+          @adapter ||= Legion::Settings[:data][:adapter]&.to_sym || :sqlite
         end
 
         def setup
-          @sequel = if adapter == :mysql2
-                      ::Sequel.connect(adapter: adapter, **creds_builder)
+          @sequel = if adapter == :sqlite
+                      ::Sequel.sqlite(sqlite_path)
                     else
-                      ::Sequel.connect("jdbc:mysql://#{creds_builder[:host]}:#{creds_builder[:port]}/#{creds_builder[:database]}?user=#{creds_builder[:username]}&password=#{creds_builder[:password]}&serverTimezone=UTC") # rubocop:disable Layout/LineLength
+                      begin
+                        ::Sequel.connect(adapter: adapter, **creds_builder)
+                      rescue StandardError => e
+                        raise unless dev_fallback?
+
+                        if defined?(Legion::Logging)
+                          Legion::Logging.warn(
+                            "Shared DB unreachable (#{e.message}), dev_mode fallback to SQLite"
+                          )
+                        end
+                        @adapter = :sqlite
+                        ::Sequel.sqlite(sqlite_path)
+                      end
                     end
           Legion::Settings[:data][:connected] = true
-          return if Legion::Settings[:data][:connection].nil? || Legion::Settings[:data][:connection][:log].nil?
-
-          @sequel.logger = Legion::Logging
-          @sequel.sql_log_level = Legion::Settings[:data][:connection][:sql_log_level]
-          @sequel.log_warn_duration = Legion::Settings[:data][:connection][:log_warn_duration]
+          configure_logging
         end
 
         def shutdown
@@ -30,15 +42,9 @@ module Legion
         end
 
         def creds_builder(final_creds = {})
-          final_creds.merge! Legion::Data::Settings.creds
+          final_creds.merge! Legion::Data::Settings.creds(adapter)
           final_creds.merge! Legion::Settings[:data][:creds] if Legion::Settings[:data][:creds].is_a? Hash
 
-          # if Legion::Settings[:data][:connection][:max_connections].is_a? Integer
-          #   final_creds[:max_connections] = Legion::Settings[:data][:connection][:max_connections]
-          # end
-
-          # final_creds[:preconnect] = :concurrently if Legion::Settings[:data][:connection][:preconnect]
-
           return final_creds if Legion::Settings[:vault].nil?
 
           if Legion::Settings[:vault][:connected] && ::Vault.sys.mounts.key?(:database)
@@ -50,15 +56,23 @@ module Legion
           final_creds
         end
 
-        def default_creds
-          {
-            host: '127.0.0.1',
-            port: 3306,
-            username: 'legion',
-            password: 'legion',
-            database: 'legion',
-            max_connections: 4
-          }
+        private
+
+        def dev_fallback?
+          data_settings = Legion::Settings[:data]
+          data_settings[:dev_mode] == true && data_settings[:dev_fallback] != false
+        end
+
+        def sqlite_path
+          Legion::Settings[:data][:creds][:database] || 'legionio.db'
+        end
+
+        def configure_logging
+          return if Legion::Settings[:data][:connection].nil? || Legion::Settings[:data][:connection][:log].nil?
+
+          @sequel.logger = Legion::Logging
+          @sequel.sql_log_level = Legion::Settings[:data][:connection][:sql_log_level]
+          @sequel.log_warn_duration = Legion::Settings[:data][:connection][:log_warn_duration]
         end
       end
     end

data/lib/legion/data/encryption/cipher.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'openssl'
+
+module Legion
+  module Data
+    module Encryption
+      module Cipher
+        VERSION_BYTE = "\x01".b.freeze
+        IV_LENGTH = 12
+        TAG_LENGTH = 16
+
+        class << self
+          def encrypt(plaintext, key:, aad: '')
+            cipher = OpenSSL::Cipher.new('aes-256-gcm').encrypt
+            iv = OpenSSL::Random.random_bytes(IV_LENGTH)
+            cipher.key = key
+            cipher.iv = iv
+            cipher.auth_data = aad
+
+            ciphertext = cipher.update(plaintext.to_s) + cipher.final
+            tag = cipher.auth_tag(TAG_LENGTH)
+
+            VERSION_BYTE + iv + ciphertext + tag
+          end
+
+          def decrypt(blob, key:, aad: '')
+            raise ArgumentError, 'data too short' if blob.bytesize < 1 + IV_LENGTH + TAG_LENGTH
+
+            version = blob.byteslice(0, 1)
+            raise ArgumentError, "unsupported version: #{version.unpack1('C')}" unless version == VERSION_BYTE
+
+            iv = blob.byteslice(1, IV_LENGTH)
+            tag = blob.byteslice(-TAG_LENGTH, TAG_LENGTH)
+            ciphertext = blob.byteslice(1 + IV_LENGTH, blob.bytesize - 1 - IV_LENGTH - TAG_LENGTH)
+
+            cipher = OpenSSL::Cipher.new('aes-256-gcm').decrypt
+            cipher.key = key
+            cipher.iv = iv
+            cipher.auth_tag = tag
+            cipher.auth_data = aad
+
+            cipher.update(ciphertext) + cipher.final
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/encryption/key_provider.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'openssl'
+
+module Legion
+  module Data
+    module Encryption
+      class KeyProvider
+        def initialize(mode: :auto)
+          @mode = mode
+          @key_cache = {}
+        end
+
+        def key_for(tenant_id: nil)
+          cache_key = tenant_id || '__default__'
+          @key_cache[cache_key] ||= derive_key(tenant_id)
+        end
+
+        def clear_cache!
+          @key_cache.clear
+        end
+
+        private
+
+        def derive_key(tenant_id)
+          if tenant_id && crypt_available?
+            Legion::Crypt::PartitionKeys.derive(tenant_id: tenant_id)
+          elsif crypt_available?
+            Legion::Crypt.default_encryption_key
+          else
+            local_key
+          end
+        end
+
+        def crypt_available?
+          defined?(Legion::Crypt::PartitionKeys)
+        end
+
+        def local_key
+          OpenSSL::Digest.digest('SHA256', 'legion-dev-encryption-key')
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/encryption/sequel_plugin.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative 'cipher'
+require_relative 'key_provider'
+
+module Legion
+  module Data
+    module Encryption
+      module SequelPlugin
+        module ClassMethods
+          def encrypted_columns
+            @encrypted_columns ||= {}
+          end
+
+          def encrypted_column(name, key_scope: :default)
+            col_scope = key_scope
+            encrypted_columns[name] = { key_scope: col_scope }
+
+            define_method(name) do
+              raw = super()
+              return nil if raw.nil?
+
+              provider = self.class.encryption_key_provider
+              tenant = col_scope == :tenant ? self[:tenant_id] : nil
+              key = provider.key_for(tenant_id: tenant)
+              aad = "#{self.class.table_name}:#{pk}:#{name}"
+              Legion::Data::Encryption::Cipher.decrypt(raw.b, key: key, aad: aad)
+            end
+
+            define_method(:"#{name}=") do |value|
+              if value.nil?
+                super(nil)
+              else
+                provider = self.class.encryption_key_provider
+                tenant = col_scope == :tenant ? self[:tenant_id] : nil
+                key = provider.key_for(tenant_id: tenant)
+                aad = "#{self.class.table_name}:#{pk || 0}:#{name}"
+                encrypted = Legion::Data::Encryption::Cipher.encrypt(value.to_s, key: key, aad: aad)
+                super(Sequel.blob(encrypted))
+              end
+            end
+          end
+
+          def encryption_key_provider
+            @encryption_key_provider ||= KeyProvider.new
+          end
+        end
+
+        module InstanceMethods
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/event_store/projection.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module EventStore
+      class Projection
+        attr_reader :state
+
+        def initialize
+          @state = {}
+        end
+
+        def apply(_event)
+          raise NotImplementedError, "#{self.class} must implement #apply"
+        end
+
+        def self.build_from(stream, since: nil)
+          projection = new
+          events = EventStore.read_stream(stream, since: since)
+          events.each { |e| projection.apply(e) }
+          projection
+        end
+      end
+
+      class ConsentState < Projection
+        def apply(event)
+          scope = event.dig(:data, :scope)
+          return unless scope
+
+          case event[:type]
+          when 'consent.granted', 'consent.modified'
+            @state[scope] = event.dig(:data, :tier)
+          when 'consent.revoked'
+            @state.delete(scope)
+          end
+        end
+      end
+
+      class GovernanceTimeline < Projection
+        def initialize
+          super
+          @state = []
+        end
+
+        def apply(event)
+          @state << {
+            type:   event[:type],
+            stream: event[:stream],
+            at:     event[:created_at],
+            data:   event[:data]
+          }
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/event_store.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Legion
+  module Data
+    module EventStore
+      GOVERNANCE_EVENT_TYPES = %w[
+        consent.granted consent.revoked consent.modified
+        extinction.triggered extinction.resolved
+        worker.registered worker.retired worker.transferred
+        scope.approved scope.violated scope.reconciled
+        audit.retention_applied audit.exported
+      ].freeze
+
+      class << self
+        def append(stream:, type:, data: {}, metadata: {})
+          return { error: 'db unavailable' } unless db_ready?
+
+          conn = Legion::Data.connection
+          conn.transaction do
+            last = conn[:governance_events]
+                   .where(stream_id: stream)
+                   .order(Sequel.desc(:sequence_number))
+                   .first
+
+            seq = (last&.[](:sequence_number) || 0) + 1
+            prev_hash = last&.[](:event_hash) || ('0' * 64)
+
+            data_json = Legion::JSON.dump(data)
+            metadata_json = Legion::JSON.dump(metadata)
+            event_hash = compute_hash(stream, seq, type, data_json, prev_hash)
+
+            conn[:governance_events].insert(
+              stream_id:       stream,
+              event_type:      type,
+              sequence_number: seq,
+              data_json:       data_json,
+              metadata_json:   metadata_json,
+              event_hash:      event_hash,
+              previous_hash:   prev_hash,
+              created_at:      Time.now
+            )
+
+            { stream: stream, sequence: seq, hash: event_hash }
+          end
+        end
+
+        def read_stream(stream, since: nil)
+          return [] unless db_ready?
+
+          ds = Legion::Data.connection[:governance_events].where(stream_id: stream)
+          ds = ds.where { created_at >= since } if since
+          ds.order(:sequence_number).all.map { |e| deserialize(e) }
+        end
+
+        def read_by_type(type, since: nil, limit: 100)
+          return [] unless db_ready?
+
+          ds = Legion::Data.connection[:governance_events].where(event_type: type)
+          ds = ds.where { created_at >= since } if since
+          ds.order(Sequel.desc(:created_at)).limit(limit).all.map { |e| deserialize(e) }
+        end
+
+        def verify_chain(stream)
+          return { valid: false, error: 'db unavailable' } unless db_ready?
+
+          events = Legion::Data.connection[:governance_events]
+                               .where(stream_id: stream)
+                               .order(:sequence_number)
+                               .all
+
+          prev_hash = '0' * 64
+          events.each do |e|
+            expected = compute_hash(stream, e[:sequence_number], e[:event_type], e[:data_json], prev_hash)
+            return { valid: false, broken_at: e[:sequence_number] } unless e[:event_hash] == expected
+            return { valid: false, broken_at: e[:sequence_number] } unless e[:previous_hash] == prev_hash
+
+            prev_hash = e[:event_hash]
+          end
+
+          { valid: true, length: events.size }
+        end
+
+        private
+
+        def compute_hash(stream, seq, type, data_json, prev_hash)
+          Digest::SHA256.hexdigest("#{stream}:#{seq}:#{type}:#{data_json}:#{prev_hash}")
+        end
+
+        def deserialize(event)
+          {
+            id:         event[:id],
+            stream:     event[:stream_id],
+            type:       event[:event_type],
+            sequence:   event[:sequence_number],
+            data:       Legion::JSON.load(event[:data_json] || '{}'),
+            metadata:   Legion::JSON.load(event[:metadata_json] || '{}'),
+            hash:       event[:event_hash],
+            created_at: event[:created_at]
+          }
+        end
+
+        def db_ready?
+          defined?(Legion::Data) && Legion::Data.connection&.table_exists?(:governance_events)
+        rescue StandardError
+          false
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/local.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'sequel'
+require 'sequel/extensions/migration'
+
+module Legion
+  module Data
+    module Local
+      class << self
+        attr_reader :connection, :db_path
+
+        def setup(database: nil, **)
+          return if @connected
+
+          db_file = database || local_settings[:database] || 'legionio_local.db'
+          @db_path = db_file
+          @connection = ::Sequel.sqlite(db_file)
+          @connected = true
+          run_migrations
+          Legion::Logging.info "Legion::Data::Local connected to #{db_file}" if defined?(Legion::Logging)
+        end
+
+        def shutdown
+          @connection&.disconnect
+          @connection = nil
+          @connected = false
+        end
+
+        def connected?
+          @connected == true
+        end
+
+        def register_migrations(name:, path:)
+          @registered_migrations ||= {}
+          @registered_migrations[name] = path
+        end
+
+        def registered_migrations
+          @registered_migrations || {}
+        end
+
+        def model(table_name)
+          raise 'Legion::Data::Local not connected' unless connected?
+
+          ::Sequel::Model(connection[table_name])
+        end
+
+        def reset!
+          @connection = nil
+          @connected = false
+          @db_path = nil
+          @registered_migrations = nil
+        end
+
+        private
+
+        def run_migrations
+          return unless local_settings.dig(:migrations, :auto_migrate) != false
+
+          registered_migrations.each_value do |path|
+            next unless File.directory?(path)
+
+            ::Sequel::TimestampMigrator.new(@connection, path).run
+          rescue StandardError => e
+            Legion::Logging.warn "Local migration failed for #{path}: #{e.message}" if defined?(Legion::Logging)
+          end
+        end
+
+        def local_settings
+          return {} unless defined?(Legion::Settings)
+
+          Legion::Settings[:data]&.dig(:local) || {}
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/migration.rb

@@ -1,12 +1,14 @@
+# frozen_string_literal: true
+
 require 'sequel/extensions/migration'
 
 module Legion
   module Data
     module Migration
       class << self
-        def migrate(connection = Legion::Data.connection, path = "#{__dir__}/migrations", **opts)
-          Legion::Settings[:data][:migrations][:version] = Sequel::Migrator.run(connection, path, **opts)
-          Legion::Logging.info("Legion::Data::Migration ran successfully to version #{Legion::Settings[:data][:migrations][:version]}") # rubocop:disable Layout/LineLength
+        def migrate(connection = Legion::Data.connection, path = "#{__dir__}/migrations", **)
+          Legion::Settings[:data][:migrations][:version] = Sequel::Migrator.run(connection, path, **)
+          Legion::Logging.info("Legion::Data::Migration ran successfully to version #{Legion::Settings[:data][:migrations][:version]}")
           Legion::Settings[:data][:migrations][:ran] = true
         end
       end

data/lib/legion/data/migrations/001_add_schema_columns.rb

@@ -1,10 +1,16 @@
+# frozen_string_literal: true
+
 require 'sequel/extensions/migration'
 
 Sequel.migration do
   up do
-    run 'ALTER TABLE `schema_info` ADD `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP AFTER `version`;'
-    run 'ALTER TABLE `schema_info` ADD `updated_at` TIMESTAMP  NULL  ON UPDATE CURRENT_TIMESTAMP  AFTER `created_at`;'
-    run 'ALTER TABLE `schema_info` ADD `catalog` VARCHAR(255)  NULL  DEFAULT NULL  AFTER `version`;'
+    alter_table(:schema_info) do
+      # SQLite does not support non-constant defaults in ALTER TABLE ADD COLUMN,
+      # so we omit the default here and let the application set timestamps.
+      add_column :created_at, DateTime, null: true
+      add_column :updated_at, DateTime, null: true
+      add_column :catalog, String, size: 255, null: true
+    end
   end
 
   down do

data/lib/legion/data/migrations/002_add_nodes.rb

@@ -1,17 +1,15 @@
+# frozen_string_literal: true
+
 Sequel.migration do
   up do
-    run "CREATE TABLE `nodes` (
-      `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
-      `name` varchar(128) NOT NULL DEFAULT '',
-      `status` varchar(255) NOT NULL DEFAULT 'unknown',
-      `active` tinyint(1) unsigned NOT NULL DEFAULT '1',
-      `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-      `updated` timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
-      PRIMARY KEY (`id`),
-      UNIQUE KEY `name` (`name`),
-      KEY `active` (`active`),
-      KEY `status` (`status`)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;"
+    create_table(:nodes) do
+      primary_key :id
+      String :name, size: 128, null: false, default: '', unique: true
+      String :status, size: 255, null: false, default: 'unknown', index: true
+      TrueClass :active, null: false, default: true, index: true
+      DateTime :created, null: false, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated, null: true
+    end
   end
 
   down do

data/lib/legion/data/migrations/003_add_settings.rb

@@ -1,15 +1,15 @@
+# frozen_string_literal: true
+
 Sequel.migration do
   up do
-    run "CREATE TABLE `settings` (
-      `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
-      `key` varchar(128) NOT NULL,
-      `value` varchar(256) NOT NULL,
-      `encrypted` tinyint(1) unsigned NOT NULL DEFAULT '0',
-      `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-      `updated` timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
-      PRIMARY KEY (`id`),
-      UNIQUE KEY `key` (`key`)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;"
+    create_table(:settings) do
+      primary_key :id
+      String :key, size: 128, null: false, unique: true
+      String :value, size: 256, null: false
+      TrueClass :encrypted, null: false, default: false
+      DateTime :created, null: false, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated, null: true
+    end
   end
 
   down do

data/lib/legion/data/migrations/004_add_extensions.rb

@@ -1,22 +1,20 @@
+# frozen_string_literal: true
+
 Sequel.migration do
   up do
-    run "CREATE TABLE `extensions` (
-      `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
-      `active` tinyint(1) unsigned NOT NULL DEFAULT '1',
-      `name` varchar(128) NOT NULL,
-      `namespace` varchar(128) NOT NULL DEFAULT '',
-      `exchange` varchar(255) DEFAULT NULL,
-      `uri` varchar(256) DEFAULT NULL,
-      `schema_version` int(11) unsigned NOT NULL DEFAULT 0,
-      `updated` timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
-      `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-      PRIMARY KEY (`id`),
-      UNIQUE KEY `name_namespace` (`name`,`namespace`),
-      KEY `active` (`active`),
-      KEY `name` (`name`),
-      KEY `namespace` (`namespace`),
-      key `schema_version` (`schema_version`)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;"
+    create_table(:extensions) do
+      primary_key :id
+      TrueClass :active, null: false, default: true, index: true
+      String :name, size: 128, null: false, index: true
+      String :namespace, size: 128, null: false, default: '', index: true
+      String :exchange, size: 255, null: true
+      String :uri, size: 256, null: true
+      Integer :schema_version, null: false, default: 0, index: true
+      DateTime :updated, null: true
+      DateTime :created, null: false, default: Sequel::CURRENT_TIMESTAMP
+
+      unique %i[name namespace]
+    end
   end
 
   down do