switchman 3.2.1 → 4.0.0

data/lib/tasks/switchman.rake

@@ -1,40 +1,30 @@
 # frozen_string_literal: true
 
-# In rails 7.0+ if you have only 1 db in the env it doesn't try to do explicit activation
-# (and for rails purposes we only have one db per env because each database server is a separate env)
-if Rails.version < '7.0'
-  task_prefix = ::Rake::Task.task_defined?('app:db:migrate') ? 'app:db' : 'db'
-  ::Rake::Task["#{task_prefix}:migrate"].clear_actions.enhance do
-    ::ActiveRecord::Tasks::DatabaseTasks.migrate
-    # Ensure this doesn't blow up when running inside the dummy app
-    Rake::Task["#{task_prefix}:_dump"].invoke
-  end
-end
-
 module Switchman
   module Rake
-    def self.filter_database_servers(&block)
-      chain = filter_database_servers_chain # use a local variable so that the current chain is closed over in the following lambda
-      @filter_database_servers_chain = ->(servers) { block.call(servers, chain) }
+    def self.filter_database_servers
+      # use a local variable so that the current chain is closed over in the following lambda
+      chain = filter_database_servers_chain
+      @filter_database_servers_chain = ->(servers) { yield(servers, chain) }
     end
 
     def self.scope(base_scope = Shard,
-                   database_server: ENV['DATABASE_SERVER'],
-                   shard: ENV['SHARD'])
+                   database_server: ENV.fetch("DATABASE_SERVER", nil),
+                   shard: ENV.fetch("SHARD", nil))
       servers = DatabaseServer.all
 
       if database_server
         servers = database_server
-        if servers.first == '-'
+        if servers.first == "-"
           negative = true
           servers = servers[1..]
         end
-        servers = servers.split(',')
-        open = servers.delete('open')
+        servers = servers.split(",")
+        open = servers.delete("open")
 
-        servers = servers.map { |server| DatabaseServer.find(server) }.compact
+        servers = servers.filter_map { |server| DatabaseServer.find(server) }
         if open
-          open_servers = DatabaseServer.all.select { |server| server.config[:open] }
+          open_servers = DatabaseServer.select { |server| server.config[:open] }
           servers.concat(open_servers)
           servers << DatabaseServer.find(nil) if open_servers.empty?
           servers.uniq!
@@ -42,9 +32,22 @@ module Switchman
         servers = DatabaseServer.all - servers if negative
       end
 
+      ENV["REGION"]&.split(",")&.each do |region|
+        method = :select!
+        if region[0] == "-"
+          method = :reject!
+          region = region[1..]
+        end
+        if region == "self"
+          servers.send(method, &:in_current_region?)
+        else
+          servers.send(method) { |server| server.in_region?(region) }
+        end
+      end
+
       servers = filter_database_servers_chain.call(servers)
 
-      scope = base_scope.order(::Arel.sql('database_server_id IS NOT NULL, database_server_id, id'))
+      scope = base_scope.order(::Arel.sql("database_server_id IS NOT NULL, database_server_id, id"))
       if servers != DatabaseServer.all
         database_server_ids = servers.map(&:id)
         database_server_ids << nil if servers.include?(Shard.default.database_server)
@@ -57,36 +60,81 @@ module Switchman
     end
 
     def self.options
-      { parallel: ENV['PARALLEL'].to_i }
+      { exception: (ENV["FAIL_FAST"] == "0") ? :defer : :raise, parallel: ENV["PARALLEL"].to_i }
     end
 
     # classes - an array or proc, to activate as the current shard during the
     # task.
     def self.shardify_task(task_name, classes: [::ActiveRecord::Base])
+      log_format = ENV.fetch("LOG_FORMAT", nil)
       old_task = ::Rake::Task[task_name]
       old_actions = old_task.actions.dup
       old_task.actions.clear
 
       old_task.enhance do |*task_args|
         if ::Rails.env.test?
-          require 'switchman/test_helper'
+          require "switchman/test_helper"
           TestHelper.recreate_persistent_test_shards(dont_create: true)
         end
 
         ::GuardRail.activate(:deploy) do
           Shard.default.database_server.unguard do
             classes = classes.call if classes.respond_to?(:call)
-            Shard.with_each_shard(scope, classes, **options) do
+
+            # We don't want the shard status messages to be wrapped using a custom log transfomer
+            original_stderr = $stderr
+            original_stdout = $stdout
+            output = if log_format == "json"
+                       lambda { |msg|
+                         JSON.dump(shard: Shard.current.id,
+                                   database_server: Shard.current.database_server.id,
+                                   type: "log",
+                                   message: msg)
+                       }
+                     else
+                       nil
+                     end
+            Shard.with_each_shard(scope, classes, output: output, **options) do
               shard = Shard.current
-              puts "#{shard.id}: #{shard.description}"
+
+              if log_format == "json"
+                original_stdout.puts JSON.dump(
+                  shard: shard.id,
+                  database_server: shard.database_server.id,
+                  type: "started"
+                )
+              else
+                original_stdout.puts "#{shard.id}: #{shard.description}"
+              end
 
               shard.database_server.unguard do
                 old_actions.each { |action| action.call(*task_args) }
               end
+
+              if log_format == "json"
+                original_stdout.puts JSON.dump(
+                  shard: shard.id,
+                  database_server: shard.database_server.id,
+                  type: "completed"
+                )
+              end
               nil
+            rescue => e
+              if log_format == "json"
+                original_stderr.puts JSON.dump(
+                  shard: shard.id,
+                  database_server: shard.database_server.id,
+                  type: "failed",
+                  message: e.full_message
+                )
+              end
+
+              raise
             end
           rescue => e
-            warn "Exception from #{e.current_shard.id}: #{e.current_shard.description}:\n#{e.full_message}" if options[:parallel] != 0
+            if options[:parallel] != 0
+              warn "Exception from #{e.current_shard.id}: #{e.current_shard.description}:\n#{e.full_message}"
+            end
             raise
           end
         end
@@ -98,7 +146,7 @@ module Switchman
     end
 
     def self.shard_scope(scope, raw_shard_ids)
-      raw_shard_ids = raw_shard_ids.split(',')
+      raw_shard_ids = raw_shard_ids.split(",")
 
       shard_ids = []
       negative_shard_ids = []
@@ -108,13 +156,13 @@ module Switchman
 
       raw_shard_ids.each do |id|
         case id
-        when 'default'
+        when "default"
           shard_ids << Shard.default.id
-        when '-default'
+        when "-default"
           negative_shard_ids << Shard.default.id
-        when 'primary'
+        when "primary"
           shard_ids.concat(Shard.primary.pluck(:id))
-        when '-primary'
+        when "-primary"
           negative_shard_ids.concat(Shard.primary.pluck(:id))
         when /^(-?)(\d+)?\.\.(\.)?(\d+)?$/
           negative, start, open, finish = $1.present?, $2, $3.present?, $4
@@ -122,8 +170,8 @@ module Switchman
 
           range = []
           range << "id>=#{start}" if start
-          range << "id<#{'=' unless open}#{finish}" if finish
-          (negative ? negative_ranges : ranges) << "(#{range.join(' AND ')})"
+          range << "id<#{"=" unless open}#{finish}" if finish
+          (negative ? negative_ranges : ranges) << "(#{range.join(" AND ")})"
         when /^-(\d+)$/
           negative_shard_ids << $1.to_i
         when /^\d+$/
@@ -151,21 +199,21 @@ module Switchman
           select = []
           if index != 1
             subscope = subscope.offset(per_chunk * (index - 1))
-            select << 'MIN(id) AS min_id'
+            select << "MIN(id) AS min_id"
           end
           if index != denominator
             subscope = subscope.limit(per_chunk)
-            select << 'MAX(id) AS max_id'
+            select << "MAX(id) AS max_id"
           end
 
-          result = Shard.from(subscope).select(select.join(', ')).to_a.first
+          result = Shard.from(subscope).select(select.join(", ")).to_a.first
           range = case index
                   when 1
-                    "id<=#{result['max_id']}"
+                    "id<=#{result["max_id"]}"
                   when denominator
-                    "id>=#{result['min_id']}"
+                    "id>=#{result["min_id"]}"
                   else
-                    "(id>=#{result['min_id']} AND id<=#{result['max_id']})"
+                    "(id>=#{result["min_id"]} AND id<=#{result["max_id"]})"
                   end
 
           (numerator.negative? ? negative_ranges : ranges) << range
@@ -186,16 +234,16 @@ module Switchman
 
       conditions = []
       positive_queries = []
-      positive_queries << ranges.join(' OR ') unless ranges.empty?
+      positive_queries << ranges.join(" OR ") unless ranges.empty?
       unless shard_ids.empty?
-        positive_queries << 'id IN (?)'
+        positive_queries << "id IN (?)"
         conditions << shard_ids
       end
-      positive_query = positive_queries.join(' OR ')
+      positive_query = positive_queries.join(" OR ")
       scope = scope.where(positive_query, *conditions) unless positive_queries.empty?
 
-      scope = scope.where("NOT (#{negative_ranges.join(' OR')})") unless negative_ranges.empty?
-      scope = scope.where('id NOT IN (?)', negative_shard_ids) unless negative_shard_ids.empty?
+      scope = scope.where("NOT (#{negative_ranges.join(" OR")})") unless negative_ranges.empty?
+      scope = scope.where("id NOT IN (?)", negative_shard_ids) unless negative_shard_ids.empty?
       scope
     end
 
@@ -206,21 +254,9 @@ module Switchman
 
   module ActiveRecord
     module PostgreSQLDatabaseTasks
-      def structure_dump(filename, extra_flags = nil)
-        set_psql_env
-        args = ['--schema-only', '--no-privileges', '--no-owner', '--file', filename]
-        args.concat(Array(extra_flags)) if extra_flags
-        shard = Shard.current.name
-        serialized_search_path = shard
-        args << "--schema=#{Shellwords.escape(shard)}"
-
-        ignore_tables = ::ActiveRecord::SchemaDumper.ignore_tables
-        args += ignore_tables.flat_map { |table| ['-T', table] } if ignore_tables.any?
-
-        args << db_config.database
-        run_cmd('pg_dump', args, 'dumping')
-        remove_sql_header_comments(filename)
-        File.open(filename, 'a') { |f| f << "SET search_path TO #{serialized_search_path};\n\n" }
+      def structure_dump(...)
+        ::ActiveRecord.dump_schemas = Switchman::Shard.current.name
+        super
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: switchman
 version: !ruby/object:Gem::Version
-  version: 3.2.1
+  version: 4.0.0
 platform: ruby
 authors:
 - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-22 00:00:00.000000000 Z
+date: 2025-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -18,20 +18,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.1.4
+        version: '7.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '7.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.1.4
+        version: '7.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '7.2'
 - !ruby/object:Gem::Dependency
   name: guardrail
   requirement: !ruby/object:Gem::Requirement
@@ -66,48 +66,34 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '7.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '7.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '7.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '7.2'
 - !ruby/object:Gem::Dependency
-  name: appraisal
+  name: debug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: '1.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.3.0
-- !ruby/object:Gem::Dependency
-  name: byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -122,20 +108,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -170,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -192,6 +164,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rubocop-inst
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rubocop-rake
   requirement: !ruby/object:Gem::Requirement
@@ -255,6 +241,7 @@ files:
 - lib/switchman/active_record/attribute_methods.rb
 - lib/switchman/active_record/base.rb
 - lib/switchman/active_record/calculations.rb
+- lib/switchman/active_record/connection_handler.rb
 - lib/switchman/active_record/connection_pool.rb
 - lib/switchman/active_record/database_configurations.rb
 - lib/switchman/active_record/database_configurations/database_config.rb
@@ -262,6 +249,7 @@ files:
 - lib/switchman/active_record/log_subscriber.rb
 - lib/switchman/active_record/migration.rb
 - lib/switchman/active_record/model_schema.rb
+- lib/switchman/active_record/pending_migration_connection.rb
 - lib/switchman/active_record/persistence.rb
 - lib/switchman/active_record/postgresql_adapter.rb
 - lib/switchman/active_record/predicate_builder.rb
@@ -290,6 +278,7 @@ files:
 - lib/switchman/rails.rb
 - lib/switchman/shard.rb
 - lib/switchman/sharded_instrumenter.rb
+- lib/switchman/shared_schema_cache.rb
 - lib/switchman/standard_error.rb
 - lib/switchman/test_helper.rb
 - lib/switchman/unsharded_record.rb
@@ -300,6 +289,7 @@ licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
+  source_code_uri: https://github.com/instructure/switchman
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -308,14 +298,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key: 
 specification_version: 4
 summary: Rails sharding magic