switchman-inst-jobs 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5cc218f034bd73745f6e3baf68bfd2a582deac1e
+  data.tar.gz: 1360f305062ef01a6142da72e9e0703db33ef8dc
+SHA512:
+  metadata.gz: 428cf01ae0ade7147f4f574f766003fd6be59bf6d353d469ee56111c379ff3ea3f6ef25dd8b3644770776f5b8f1f1ba9b03fb82cfa812076e05f6435bcaf91fb
+  data.tar.gz: d12f3089fadf8bc28d4c86b5548badfc47de993e20353afb21e738d456031e852b92be9f5b280bcc15b938c3a3d157de0508f2ad0f52292d333464e44f319410

data/db/migrate/20170308045400_add_shard_id_to_delayed_jobs.rb

@@ -0,0 +1,33 @@
+class AddShardIdToDelayedJobs < ActiveRecord::Migration
+  disable_ddl_transaction!
+
+  def connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def up
+    add_column :delayed_jobs, :shard_id, :integer, limit: 8
+    add_index :delayed_jobs, :shard_id, algorithm: :concurrently
+
+    add_column :failed_jobs, :shard_id, :integer, limit: 8
+    add_index :failed_jobs, :shard_id, algorithm: :concurrently
+
+    add_column :switchman_shards, :delayed_jobs_shard_id, :integer, limit: 8
+    add_foreign_key(
+      :switchman_shards,
+      :switchman_shards,
+      column: :delayed_jobs_shard_id
+    )
+  end
+
+  def down
+    remove_foreign_key :switchman_shards, column: :delayed_jobs_shard_id
+    remove_column :switchman_shards, :delayed_jobs_shard_id
+
+    remove_index :failed_jobs, :shard_id
+    remove_column :failed_jobs, :shard_id
+
+    remove_index :delayed_jobs, :shard_id
+    remove_column :delayed_jobs, :shard_id
+  end
+end

data/lib/switchman_inst_jobs.rb

@@ -0,0 +1,46 @@
+require 'inst-jobs'
+require 'rails/railtie'
+require 'switchman'
+
+module SwitchmanInstJobs
+  def self.initialize_active_record
+    ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(
+      ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+    )
+  end
+
+  def self.initialize_inst_jobs
+    ::Delayed::Backend::ActiveRecord::Job.prepend(
+      Delayed::Backend::Base
+    )
+    ::Delayed::Backend::Redis::Job.prepend(
+      Delayed::Backend::Base
+    )
+    ::Delayed::Backend::Redis::Job.column :shard_id, :integer
+    ::Delayed::Pool.prepend Delayed::Pool
+    ::Delayed::Worker.prepend Delayed::Worker
+    ::Object.include Delayed::MessageSending
+  end
+
+  def self.initialize_shackles
+    ::Shackles.singleton_class.prepend Shackles::ClassMethods
+  end
+
+  def self.initialize_switchman
+    ::Switchman::DatabaseServer.prepend Switchman::DatabaseServer
+    ::Switchman::DefaultShard.prepend Switchman::DefaultShard
+    ::Switchman::Shard.prepend Switchman::Shard
+  end
+end
+
+require 'switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter'
+require 'switchman_inst_jobs/active_record/migration'
+require 'switchman_inst_jobs/delayed/backend/base'
+require 'switchman_inst_jobs/delayed/message_sending'
+require 'switchman_inst_jobs/delayed/pool'
+require 'switchman_inst_jobs/delayed/worker'
+require 'switchman_inst_jobs/railtie'
+require 'switchman_inst_jobs/shackles'
+require 'switchman_inst_jobs/switchman/database_server'
+require 'switchman_inst_jobs/switchman/default_shard'
+require 'switchman_inst_jobs/switchman/shard'

data/lib/switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter.rb

@@ -0,0 +1,14 @@
+module SwitchmanInstJobs
+  module ActiveRecord
+    module ConnectionAdapters
+      module PostgreSQLAdapter
+        def set_search_path(function, args = '()', path = ::Switchman::Shard.current.name)
+          execute <<-SQL
+            ALTER FUNCTION #{quote_table_name(function)}#{args}
+              SET search_path TO #{path}
+          SQL
+        end
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/active_record/migration.rb

@@ -0,0 +1,27 @@
+module SwitchmanInstJobs
+  module ActiveRecord
+    module Migration
+      module ClassMethods
+        def self.included(klass)
+          klass.send(:attr_writer, :open_migrations)
+        end
+
+        def open_migrations
+          @open_migrations ||= 0
+        end
+      end
+
+      def migrate(direction)
+        ::ActiveRecord::Migration.open_migrations += 1
+        super
+      ensure
+        ::ActiveRecord::Migration.open_migrations -= 1
+      end
+    end
+  end
+end
+
+ActiveRecord::Migration.prepend SwitchmanInstJobs::ActiveRecord::Migration
+ActiveRecord::Migration.singleton_class.include(
+  SwitchmanInstJobs::ActiveRecord::Migration::ClassMethods
+)

data/lib/switchman_inst_jobs/delayed/backend/base.rb

@@ -0,0 +1,54 @@
+module SwitchmanInstJobs
+  module Delayed
+    module Backend
+      module Base
+        module ClassMethods
+          def enqueue(object, options = {})
+            enqueue_options = options.merge(
+              current_shard: ::Switchman::Shard.current
+            )
+
+            if ::ActiveRecord::Migration.open_migrations.positive?
+              ::Switchman::Shard.current.delayed_jobs_shard.activate(:delayed_jobs) do
+                super(object, enqueue_options)
+              end
+            else
+              super(object, enqueue_options)
+            end
+          end
+        end
+
+        def self.prepended(base)
+          base.singleton_class.prepend(ClassMethods)
+          base.shard_category = :delayed_jobs if base.name == 'Delayed::Backend::ActiveRecord::Job'
+        end
+
+        def current_shard
+          @current_shard ||= ::Switchman::Shard.lookup(shard_id)
+        end
+
+        def current_shard=(shard)
+          self.shard_id = shard.id
+          self.shard_id = nil if shard.is_a?(::Switchman::DefaultShard)
+        end
+
+        def invoke_job
+          current_shard.activate { super }
+        end
+
+        def deserialize(source)
+          raise "Shard not found: #{shard_id}" unless current_shard
+          current_shard.activate { super }
+        rescue ::Switchman::ConnectionError, PG::ConnectionBad, PG::UndefinedTable
+          # likely a missing shard with a stale cache
+          current_shard.send(:clear_cache)
+          ::Switchman::Shard.clear_cache
+          unless ::Switchman::Shard.where(id: shard_id).exists?
+            raise "Shard not found: #{shard_id}"
+          end
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/delayed/message_sending.rb

@@ -0,0 +1,10 @@
+module SwitchmanInstJobs
+  module Delayed
+    module MessageSending
+      def send_later_enqueue_args(method, _enqueue_args = {}, *args)
+        return send(method, *args) if ::Switchman::DatabaseServer.creating_new_shard
+        super
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/delayed/pool.rb

@@ -0,0 +1,29 @@
+module SwitchmanInstJobs
+  module Delayed
+    module Pool
+      def unlock_orphaned_jobs(worker = nil, pid = nil)
+        if worker
+          shards = [worker.shard]
+        else
+          # Since we're not unlocking for a specific worker, look through
+          # the config for all shards this pool has workers for, and unlock
+          # on each shard found.
+          #
+          # If this host used to have workers for shard X, and then it died
+          # ungracefully at the same time that all workers for shard X were
+          # removed, we won't properly unlock those jobs here. That's an
+          # acceptable edge case though.
+          #
+          # We purposely don't .compact to remove nils here, since if any
+          # workers are on the default jobs shard we want to unlock against
+          # that shard too.
+          shard_ids = @config[:workers].map { |c| c[:shard] }.uniq
+          shards = shard_ids.map { |shard_id| ::Delayed::Worker.shard(shard_id) }
+        end
+        ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs]) do
+          super
+        end
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/delayed/worker.rb

@@ -0,0 +1,34 @@
+module SwitchmanInstJobs
+  module Delayed
+    module Worker
+      def self.prepended(base)
+        base.singleton_class.prepend(ClassMethods)
+      end
+
+      def start
+        shard.activate(:delayed_jobs) { super }
+      end
+
+      # Worker#run is usually only called from Worker#start, but if the worker
+      # is called directly from the console, we want to make sure it still gets
+      # the right shard activated.
+      def run
+        shard.activate(:delayed_jobs) { super }
+      end
+
+      def shard
+        self.class.shard(@config[:shard])
+      end
+
+      module ClassMethods
+        def shard(shard_id)
+          if shard_id
+            shard = ::Switchman::Shard.lookup(shard_id)
+            return shard if shard
+          end
+          ::Switchman::Shard.default.delayed_jobs_shard
+        end
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/railtie.rb

@@ -0,0 +1,25 @@
+module SwitchmanInstJobs
+  class Railtie < Rails::Railtie
+    initializer 'sharding.active_record',
+                after: 'switchman.extend_connection_adapters' do
+      SwitchmanInstJobs.initialize_active_record
+    end
+
+    initializer 'sharding.delayed' do
+      SwitchmanInstJobs.initialize_inst_jobs
+    end
+
+    initializer 'sharding.shackles',
+                after: 'switchman.extend_shackles' do
+      SwitchmanInstJobs.initialize_shackles
+    end
+
+    initializer 'sharding.switchman' do
+      SwitchmanInstJobs.initialize_switchman
+    end
+
+    config.after_initialize do
+      ::Switchman::Shard.default.delayed_jobs_shard.activate!(:delayed_jobs)
+    end
+  end
+end

data/lib/switchman_inst_jobs/shackles.rb

@@ -0,0 +1,13 @@
+module SwitchmanInstJobs
+  module Shackles
+    module ClassMethods
+      def activate(env, &block)
+        if ::ActiveRecord::Migration.open_migrations.positive?
+          yield
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/switchman/database_server.rb

@@ -0,0 +1,16 @@
+module SwitchmanInstJobs
+  module Switchman
+    module DatabaseServer
+      def delayed_jobs_shard(shard = nil)
+        return shard if config[:delayed_jobs_shard] == 'self'
+        dj_shard =
+          config[:delayed_jobs_shard] &&
+          ::Switchman::Shard.lookup(config[:delayed_jobs_shard])
+        # have to avoid recursion for the default shard asking for the default
+        # shard's delayed_jobs_shard
+        dj_shard ||= shard if shard.default?
+        dj_shard || ::Switchman::Shard.default.delayed_jobs_shard
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/switchman/default_shard.rb

@@ -0,0 +1,9 @@
+module SwitchmanInstJobs
+  module Switchman
+    module DefaultShard
+      def delayed_jobs_shard
+        self
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/switchman/shard.rb

@@ -0,0 +1,57 @@
+module SwitchmanInstJobs
+  module Switchman
+    module Shard
+      def self.prepended(base)
+        base.singleton_class.prepend(ClassMethods)
+      end
+
+      def clear_cache
+        self.class.connection.after_transaction_commit { super }
+      end
+
+      def delayed_jobs_shard
+        if read_attribute(:delayed_jobs_shard_id)
+          shard = ::Switchman::Shard.lookup(delayed_jobs_shard_id)
+          return shard if shard
+        end
+        database_server.try(:delayed_jobs_shard, self)
+      end
+
+      module ClassMethods
+        def current(category = :primary)
+          if category == :delayed_jobs
+            active_shards[category] || super(:primary).delayed_jobs_shard
+          else
+            super
+          end
+        end
+
+        def activate!(categories)
+          if !@skip_delayed_job_auto_activation &&
+             !categories[:delayed_jobs] &&
+             categories[:primary] &&
+             categories[:primary] != active_shards[:primary]
+            skip_delayed_job_auto_activation do
+              categories[:delayed_jobs] =
+                categories[:primary].delayed_jobs_shard
+            end
+          end
+          super
+        end
+
+        def skip_delayed_job_auto_activation
+          was = @skip_delayed_job_auto_activation
+          @skip_delayed_job_auto_activation = true
+          yield
+        ensure
+          @skip_delayed_job_auto_activation = was
+        end
+
+        def create
+          db = ::Switchman::DatabaseServer.server_for_new_shard
+          db.create_new_shard
+        end
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/version.rb

@@ -0,0 +1,3 @@
+module SwitchmanInstJobs
+  VERSION = '1.0.0'.freeze
+end

metadata

@@ -0,0 +1,238 @@
+--- !ruby/object:Gem::Specification
+name: switchman-inst-jobs
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Bryan Petty
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-04-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: inst-jobs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: switchman
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.7
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: wwtd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+description: 
+email:
+- bpetty@instructure.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- db/migrate/20170308045400_add_shard_id_to_delayed_jobs.rb
+- lib/switchman_inst_jobs.rb
+- lib/switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter.rb
+- lib/switchman_inst_jobs/active_record/migration.rb
+- lib/switchman_inst_jobs/delayed/backend/base.rb
+- lib/switchman_inst_jobs/delayed/message_sending.rb
+- lib/switchman_inst_jobs/delayed/pool.rb
+- lib/switchman_inst_jobs/delayed/worker.rb
+- lib/switchman_inst_jobs/railtie.rb
+- lib/switchman_inst_jobs/shackles.rb
+- lib/switchman_inst_jobs/switchman/database_server.rb
+- lib/switchman_inst_jobs/switchman/default_shard.rb
+- lib/switchman_inst_jobs/switchman/shard.rb
+- lib/switchman_inst_jobs/version.rb
+homepage: https://github.com/instructure/switchman-inst-jobs
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Switchman and Instructure Jobs compatibility gem.
+test_files: []