umbrellio-sequel-plugins 0.16.0.239 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7cd4b57e64e2977befdd1b94060f80af00be347813ea90f487b1c8226ae1c19
-  data.tar.gz: 70f2b9ca0809f895972c98d067f4626863ab61c09b68c7b9c79c04299017f963
+  metadata.gz: b7572a3f551bf5409ea0cca6442366ec84ae4cf7776e379f3e020f099fe41f4c
+  data.tar.gz: f3dc5382d920669c883e7ae4d77e340f97df35c3d9bffbac3b258b03653b9744
 SHA512:
-  metadata.gz: 999b1d353436eaa1c1a45c4210981ee3ff1ee2081bd68a8a384364704e44f50bc846ea448f08df601a237837b65c6a6d272431a7f392d0029025f5a9d2959699
-  data.tar.gz: e9befb06abf11993d79a7e01fb52813901c2f8cfc4567444f11c79a52e83d52b0c0813b1e4922bf152dcfc57a028dfddea533510dd510f24105654adf397a3b4
+  metadata.gz: c0381a1a25eae8e70624c830a3c09554efe042efb97c8218ef5137a7fc04056ebac42d5f976eb425c6aee9c5ccf711a945811213833e0e7acabcb608315e31f9
+  data.tar.gz: 640655214164e3f23290c4b3d2f60df248638ecdadb98687dc68f85d49276af31e768bbde3c1dfdbf11b29f8589427ade0a44ae39c4d6d0a461427dbf4ea544b

data/.github/workflows/ci.yml

@@ -57,9 +57,6 @@ jobs:
       id-token: write
       contents: write
 
-    env:
-      PUBLISH_JOB: true
-
     # Run on push to master branch
     if: github.event_name == 'push' && github.ref == 'refs/heads/master'
 

data/.gitignore

@@ -12,4 +12,3 @@
 
 log
 *.gemspec
-Gemfile.lock

data/Gemfile

@@ -4,13 +4,13 @@ source "https://rubygems.org"
 
 gemspec
 
-gem "async", "~> 1.31.0"
-gem "money", "~> 6.19.0"
-gem "pg", "~> 1.5.7"
-gem "pry", "~> 0.14.2"
-gem "rake", "~> 13.2.1"
-gem "rspec", "~> 3.13.0"
-gem "rubocop-config-umbrellio", "~> 1.63.0.93"
-gem "simplecov", "~> 0.22.0"
-gem "simplecov-lcov", "~> 0.8.0"
-gem "symbiont-ruby", "~> 0.7.0"
+gem "async"
+gem "money"
+gem "pg"
+gem "pry"
+gem "rake"
+gem "rspec"
+gem "rubocop-config-umbrellio"
+gem "simplecov"
+gem "simplecov-lcov"
+gem "symbiont-ruby"

data/Gemfile.lock

@@ -0,0 +1,153 @@
+PATH
+  remote: .
+  specs:
+    umbrellio-sequel-plugins (0.17.0)
+      sequel
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (7.1.4)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      mutex_m
+      tzinfo (~> 2.0)
+    ast (2.4.2)
+    async (1.32.1)
+      console (~> 1.10)
+      nio4r (~> 2.3)
+      timers (~> 4.1)
+    base64 (0.2.0)
+    bigdecimal (3.1.8)
+    coderay (1.1.3)
+    concurrent-ruby (1.3.4)
+    connection_pool (2.4.1)
+    console (1.24.0)
+      fiber-annotation
+      fiber-local
+      json
+    diff-lcs (1.5.1)
+    docile (1.4.1)
+    drb (2.2.1)
+    fiber-annotation (0.2.0)
+    fiber-local (1.0.0)
+    i18n (1.14.5)
+      concurrent-ruby (~> 1.0)
+    json (2.7.2)
+    language_server-protocol (3.17.0.3)
+    method_source (1.1.0)
+    minitest (5.25.1)
+    money (6.19.0)
+      i18n (>= 0.6.4, <= 2)
+    mutex_m (0.2.0)
+    nio4r (2.7.3)
+    parallel (1.26.3)
+    parser (3.3.5.0)
+      ast (~> 2.4.1)
+      racc
+    pg (1.5.8)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    racc (1.8.1)
+    rack (3.1.7)
+    rainbow (3.1.1)
+    rake (13.2.1)
+    regexp_parser (2.9.2)
+    rexml (3.3.7)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.1)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
+    rubocop (1.63.5)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.32.3)
+      parser (>= 3.3.1.0)
+    rubocop-capybara (2.21.0)
+      rubocop (~> 1.41)
+    rubocop-config-umbrellio (1.63.0.93)
+      rubocop (~> 1.63.0)
+      rubocop-performance (~> 1.21.0)
+      rubocop-rails (~> 2.24.0)
+      rubocop-rake (~> 0.6.0)
+      rubocop-rspec (~> 2.29.0)
+      rubocop-sequel (~> 0.3.3)
+    rubocop-factory_bot (2.26.1)
+      rubocop (~> 1.61)
+    rubocop-performance (1.21.1)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-rails (2.24.1)
+      activesupport (>= 4.2.0)
+      rack (>= 1.1)
+      rubocop (>= 1.33.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    rubocop-rspec (2.29.2)
+      rubocop (~> 1.40)
+      rubocop-capybara (~> 2.17)
+      rubocop-factory_bot (~> 2.22)
+      rubocop-rspec_rails (~> 2.28)
+    rubocop-rspec_rails (2.29.1)
+      rubocop (~> 1.61)
+    rubocop-sequel (0.3.4)
+      rubocop (~> 1.0)
+    ruby-progressbar (1.13.0)
+    sequel (5.84.0)
+      bigdecimal
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.13.1)
+    simplecov-lcov (0.8.0)
+    simplecov_json_formatter (0.1.4)
+    symbiont-ruby (0.7.0)
+    timers (4.3.5)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.6.0)
+
+PLATFORMS
+  arm64-darwin-23
+  ruby
+
+DEPENDENCIES
+  async
+  money
+  pg
+  pry
+  rake
+  rspec
+  rubocop-config-umbrellio
+  simplecov
+  simplecov-lcov
+  symbiont-ruby
+  umbrellio-sequel-plugins!
+
+BUNDLED WITH
+   2.5.18

data/README.md

@@ -485,6 +485,130 @@ Overrides Rails default `dbconsole` and `db` commands. In order to use it, you h
 require "umbrellio_sequel_plugins/rails_db_command"
 ```
 
+# Database Tasks for ClickHouse and Sequel
+
+## ClickHouse Rake Tasks
+
+We have added a set of Rake tasks to manage ClickHouse database migrations and database operations. These tasks are located in the `namespace :ch`.
+
+### Task: `ch:create`
+
+Creates a ClickHouse database in the specified cluster.
+
+```bash
+rake ch:create
+```
+This task will create a ClickHouse database as defined in the configuration file with the option to specify the cluster using the ClickHouse.config.database.
+
+Example: 
+```ruby
+ClickHouse.config do |config|
+  config.assign Rails.application.config_for(:clickhouse)
+end
+```
+
+### Task: `ch:create_migration_table`
+
+Creates a migration tracking table for ClickHouse in PostgreSQL. This table will be used to track applied migrations for the ClickHouse database.
+
+```bash
+rake ch:create_migration_table
+```
+
+### Task: `ch:drop`
+
+Drops the ClickHouse database and truncates the migration tracking table.
+
+```bash
+rake ch:drop
+```
+
+### Task: `ch:migrate`
+
+Runs the migrations for the ClickHouse database from the db/migrate/clickhouse directory.
+
+```bash
+rake ch:migrate
+```
+
+You can specify a version to migrate to using the VERSION environment variable.
+
+### Task: `ch:rollback`
+
+Rollbacks the migrations for the ClickHouse database to a specified version.
+
+```bash
+rake ch:rollback VERSION=<version_number>
+```
+
+If no version is provided, it rolls back the last migration.
+
+### Task: `ch:reset`
+
+Drops, recreates, and runs all migrations for the ClickHouse database. This is useful for resetting the entire ClickHouse setup.
+
+```bash
+rake ch:reset
+```
+
+### Task: `ch:rollback_missing_migrations`
+
+Rollbacks any missing migrations for the ClickHouse database by comparing applied migrations to the available migration files.
+
+```bash
+rake ch:rollback_missing_migrations
+```
+
+### Sequel Rake Tasks
+
+Several tasks have been added under the namespace :sequel to provide better management of migrations and rollbacks in Sequel. These tasks help in managing PostgreSQL and ClickHouse migrations.
+
+### Task: `sequel:archive_migrations`
+
+Archives migration source code into a PostgreSQL table for tracking purposes. This task can now accept custom paths for migrations and source tables.
+
+```bash
+rake sequel:archive_migrations[migrations_path, migration_table_source]
+```
+
+- `migrations_path`: Path to the migration files (default is `db/migrate/*.rb`).
+- `migration_table_source`: Table to store migration source code (default is `:schema_migrations_sources`).
+
+### `Task: sequel:rollback_archived_migrations`
+
+Rollbacks migrations that were applied but are no longer present in the current release. The task supports additional options such as custom migration paths, tables, and transaction settings.
+
+```bash
+rake sequel:rollback_archived_migrations[migrations_path, migration_table, migration_table_source, use_transactions]
+```
+
+- `migrations_path`: Path to the migration files (default is `db/migrate/*.rb`).
+- `migration_table`: Table used to track applied migrations (default is `:schema_migrations`).
+- `migration_table_source`: Table storing migration source code (default is `:schema_migrations_sources`).
+- `use_transactions`: Whether to use transactions for rolling back (default is `false`).
+
+### Task: `sequel:rollback_missing_migrations`
+
+Rollbacks migrations that are absent in the current release when deploying to staging or production. This task helps ensure consistency between different versions.
+
+```bash
+rake sequel:rollback_missing_migrations[table, use_transactions]
+```
+
+- `table`: The table used to track migrations (optional).
+- `use_transactions`: Whether to use transactions during rollback (default is `false`).
+
+### Task: `sequel:rollback_missing_migrations`
+
+This task specifically helps during deployment by rolling back any migrations that are not present in the current release.
+
+```bash
+rake sequel:rollback_missing_migrations[table, use_transactions]
+```
+
+- `table`: The table used to track migrations (optional).
+- `use_transactions`: Whether or not to use transactions when rolling back (optional).
+
 ## License
 
 Released under MIT License.

data/lib/clickhouse/migrator.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# :nocov:
+module Clickhouse
+  module Migrator
+    module_function
+
+    def migrate(to: nil)
+      if to.present?
+        migrator(target: to.to_i).run
+      else
+        migrator.run
+      end
+    end
+
+    def rollback(to: nil)
+      target = to || migrator.applied_migrations.reverse[1]
+      migrator(target: target.to_i).run
+    end
+
+    def migrator(**opts)
+      Sequel::TimestampMigrator.new(
+        DB,
+        Rails.root.join("db/migrate/clickhouse"),
+        table: :clickhouse_migrations,
+        use_transactions: false,
+        **opts,
+      )
+    end
+  end
+end
+# :nocov:

data/lib/sequel/extensions/fibered_connection_pool.rb

@@ -77,14 +77,8 @@ module Sequel::ConnectionPoolPatch
   end
 end
 
-# NOTE: Ruby 2.7 DOES NOT SUPPORT class methods prepend in this way
-# https://bugs.ruby-lang.org/issues/17423
-if RUBY_VERSION > "3"
-  Sequel::ConnectionPool::ClassMethods.prepend(Sequel::ConnectionPoolPatch)
-else
-  class Sequel::ConnectionPool
-    class << self
-      prepend Sequel::ConnectionPoolPatch
-    end
+class Sequel::ConnectionPool
+  class << self
+    prepend Sequel::ConnectionPoolPatch
   end
 end

data/lib/tasks/clickhouse.rake

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "clickhouse/migrator"
+
+namespace :ch do
+  desc "Create a ClickHouse database in the specified cluster"
+  task create: :environment do
+    CH.create_database(ClickHouse.config.database, cluster: "click_cluster")
+  end
+
+  desc "Create a migration tracking table for ClickHouse in PostgreSQL"
+  task create_migration_table: :environment do
+    DB.create_table Sequel[:public][:clickhouse_migrations] do
+      column :filename, :text, null: false, primary_key: true
+    end
+  end
+
+  desc "Drop the ClickHouse database and truncate the migration tracking table"
+  task drop: :environment do
+    CH.drop_database(ClickHouse.config.database, cluster: "click_cluster")
+    DB.from(Sequel[:public][:clickhouse_migrations]).truncate
+    DB.from(Sequel[:public][:clickhouse_migrations_sources]).truncate
+  end
+
+  desc "Run migrations for the ClickHouse database"
+  task migrate: :environment do
+    path = "db/migrate/clickhouse/*.rb"
+    migrations_table = :clickhouse_migrations
+    migrations_sources_table = :clickhouse_migrations_sources
+    use_transactions = "false"
+
+    Rake::Task["sequel:archive_migrations"].reenable
+    Rake::Task["sequel:archive_migrations"].invoke(path, migrations_sources_table)
+
+    Rake::Task["sequel:rollback_archived_migrations"].reenable
+    Rake::Task["sequel:rollback_archived_migrations"]
+      .invoke(path, migrations_table, migrations_sources_table, use_transactions)
+
+    Clickhouse::Migrator.migrate(to: ENV.fetch("VERSION", nil))
+  end
+
+  desc "Rollback migrations for the ClickHouse database"
+  task rollback: :environment do
+    Clickhouse::Migrator.rollback(to: ENV.fetch("VERSION", nil))
+  end
+
+  desc "Reset the ClickHouse database: drop, recreate, and run all migrations"
+  task reset: :environment do
+    Rake::Task["ch:drop"].invoke
+    Rake::Task["ch:create"].invoke
+    Rake::Task["ch:migrate"].invoke
+  end
+
+  desc "Rollback any missing migrations for ClickHouse"
+  task rollback_missing_migrations: :environment do
+    Rake::Task["sequel:rollback_missing_migrations"].reenable
+    Rake::Task["sequel:rollback_missing_migrations"].invoke(:clickhouse_migrations, "false")
+  end
+end

data/lib/tasks/sequel/archive_migrations.rake

@@ -2,14 +2,18 @@
 
 namespace :sequel do
   desc "Archive migrations source code"
-  task archive_migrations: :environment do
-    DB.create_table?(:schema_migrations_sources) do
+  task :archive_migrations,
+       [:migrations_path, :migration_table_source] => :environment do |_t, args|
+    migrations_path = args[:migrations_path] || "db/migrate/*.rb"
+    migration_table_source = args[:migration_table_source]&.to_sym || :schema_migrations_sources
+
+    DB.create_table?(migration_table_source) do
       column :version, "numeric", primary_key: true
       column :filename, "text", null: false
       column :source, "text", null: false
     end
 
-    migrations = Rails.root.glob("db/migrate/*.rb").map do |file|
+    migrations = Rails.root.glob(migrations_path).map do |file|
       filename = file.basename.to_s
       { version: filename.to_i, filename: filename, source: file.read }
     end
@@ -19,6 +23,6 @@ namespace :sequel do
       update: { filename: Sequel[:excluded][:filename], source: Sequel[:excluded][:source] },
     }
 
-    DB[:schema_migrations_sources].insert_conflict(**conflict_options).multi_insert(migrations)
+    DB[migration_table_source].insert_conflict(**conflict_options).multi_insert(migrations)
   end
 end

data/lib/tasks/sequel/rollback_archived_migrations.rake

@@ -4,19 +4,30 @@ require "sequel/timestamp_migrator_undo_extension"
 
 namespace :sequel do
   desc "Rollback migrations that were applied earlier but are not present in current release"
-  task rollback_archived_migrations: :environment do
+  task :rollback_archived_migrations,
+       [:migrations_path, :migration_table, :migration_table_source,
+        :use_transactions] => :environment do |_t, args|
+    migrations_path = args[:migrations_path] || "db/migrate/*.rb"
+    migration_table_source = args[:migration_table_source]&.to_sym || :schema_migrations_sources
+    use_transactions = args[:use_transactions].nil? ? nil : args[:use_transactions] == "true"
+
     DB.log_info("Finding applied migrations not present in current release...")
 
     Dir.mktmpdir do |tmpdir|
-      DB[:schema_migrations_sources].each do |migration|
+      DB[migration_table_source].each do |migration|
         path = File.join(tmpdir, migration.fetch(:filename))
         File.write(path, migration.fetch(:source))
       end
 
-      migrator = Sequel::TimestampMigrator.new(DB, tmpdir, allow_missing_migration_files: true)
+      migrator_args = {
+        table: args[:migration_table],
+        use_transactions: use_transactions,
+        allow_missing_migration_files: false,
+      }.compact
+      migrator = Sequel::TimestampMigrator.new(DB, tmpdir, migrator_args)
 
       applied_migrations = migrator.applied_migrations.map(&:to_i)
-      filesystem_migrations = Rails.root.glob("db/migrate/*.rb").map { |x| File.basename(x).to_i }
+      filesystem_migrations = Rails.root.glob(migrations_path).map { |x| File.basename(x).to_i }
       missing_migrations = applied_migrations - filesystem_migrations
 
       if missing_migrations.any?

data/lib/tasks/sequel/rollback_missing_migrations.rake

@@ -4,7 +4,9 @@ require "sequel/timestamp_migrator_undo_extension"
 
 namespace :sequel do
   desc "Rollback migrations that are absent in revision when deploying on staging"
-  task rollback_missing_migrations: :environment do
+  task :rollback_missing_migrations, [:table, :use_transactions] => :environment do |_t, args|
+    use_transactions = args[:use_transactions].nil? ? nil : args[:use_transactions] == "true"
+
     extract_migrations = lambda do |path|
       Dir.glob("#{path}/db/migrate/*.rb").map { |filename| File.basename(filename).to_i }
     end
@@ -19,7 +21,12 @@ namespace :sequel do
     puts migrations_to_rollback
 
     path = Rails.root.join("db/migrate")
-    migrator = Sequel::TimestampMigrator.new(DB, path, allow_missing_migration_files: true)
+    migrator_args = {
+      table: args[:table],
+      use_transactions: use_transactions,
+      allow_missing_migration_files: false,
+    }.compact
+    migrator = Sequel::TimestampMigrator.new(DB, path, migrator_args)
     applied_migrations = migrator.applied_migrations.map(&:to_i)
     migrations = applied_migrations.select { |m| m.in?(migrations_to_rollback) }.sort.reverse
 

data/umbrellio-sequel-plugins.gemspec

@@ -4,14 +4,8 @@ lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
-  gem_version = "0.16.0"
-
-  if ENV.fetch("PUBLISH_JOB", nil)
-    release_version = "#{gem_version}.#{ENV.fetch("GITHUB_RUN_NUMBER")}"
-  end
-
   spec.name = "umbrellio-sequel-plugins"
-  spec.version = release_version || gem_version
+  spec.version = "0.17.0"
   spec.required_ruby_version = ">= 3.0"
 
   spec.authors = ["Team Umbrellio"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: umbrellio-sequel-plugins
 version: !ruby/object:Gem::Version
-  version: 0.16.0.239
+  version: 0.17.0
 platform: ruby
 authors:
 - Team Umbrellio
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-09-13 00:00:00.000000000 Z
+date: 2024-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -36,11 +36,13 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - Gemfile
+- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
+- lib/clickhouse/migrator.rb
 - lib/sequel/extensions/currency_rates.rb
 - lib/sequel/extensions/deferrable_foreign_keys.rb
 - lib/sequel/extensions/fibered_connection_pool.rb
@@ -60,6 +62,7 @@ files:
 - lib/sequel/plugins/upsert.rb
 - lib/sequel/plugins/with_lock.rb
 - lib/sequel/timestamp_migrator_undo_extension.rb
+- lib/tasks/clickhouse.rake
 - lib/tasks/sequel/archive_migrations.rake
 - lib/tasks/sequel/rollback_archived_migrations.rake
 - lib/tasks/sequel/rollback_missing_migrations.rake