canvas-jobs 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 727a0655ca96cd696cca764d7cc6abbf4386fbb9
+  data.tar.gz: f85f4112dd1f605ff176b250d8cc92c330fe721b
+SHA512:
+  metadata.gz: bcd41802f4198d324e7f3ca295f21bcb0a1fc9f1f7bff25490b0c8f156feedd50b7eac05d510bc778831584ac235d99c72b357381d42be7a04a56968a583f2ea
+  data.tar.gz: a4254c78e7dc10866675584f954aa08a3f729951e4f20d425cfea3a0e3e205323fb58d8cd1ac36c9d7a3f70665410941b6a3b157091d2fe12c57f9ee093ff913

data/db/migrate/20101216224513_create_delayed_jobs.rb

@@ -0,0 +1,40 @@
+class CreateDelayedJobs < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    raise("#{connection.adapter_name} is not supported for delayed jobs queue") unless connection.adapter_name == 'PostgreSQL'
+
+    create_table :delayed_jobs do |table|
+      # Allows some jobs to jump to the front of the queue
+      table.integer  :priority, :default => 0
+      # Provides for retries, but still fail eventually.
+      table.integer  :attempts, :default => 0
+      # YAML-encoded string of the object that will do work
+      table.text     :handler, :limit => (500 * 1024)
+      # reason for last failure (See Note below)
+      table.text     :last_error
+      # The queue that this job is in
+      table.string   :queue, :default => nil
+      # When to run.
+      # Could be Time.zone.now for immediately, or sometime in the future.
+      table.datetime :run_at
+      # Set when a client is working on this object
+      table.datetime :locked_at
+      # Set when all retries have failed
+      table.datetime :failed_at
+      # Who is working on this object (if locked)
+      table.string   :locked_by
+
+      table.timestamps
+    end
+
+    add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
+    add_index :delayed_jobs, [:queue], :name => 'delayed_jobs_queue'
+  end
+
+  def self.down
+    drop_table :delayed_jobs
+  end
+end

data/db/migrate/20110208031356_add_delayed_jobs_tag.rb

@@ -0,0 +1,14 @@
+class AddDelayedJobsTag < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    add_column :delayed_jobs, :tag, :string
+    add_index :delayed_jobs, [:tag]
+  end
+
+  def self.down
+    remove_column :delayed_jobs, :tag
+  end
+end

data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb

@@ -0,0 +1,13 @@
+class AddDelayedJobsMaxAttempts < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    add_column :delayed_jobs, :max_attempts, :integer
+  end
+
+  def self.down
+    remove_column :delayed_jobs, :max_attempts
+  end
+end

data/db/migrate/20110516225834_add_delayed_jobs_strand.rb

@@ -0,0 +1,14 @@
+class AddDelayedJobsStrand < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    add_column :delayed_jobs, :strand, :string
+    add_index :delayed_jobs, :strand
+  end
+
+  def self.down
+    remove_column :delayed_jobs, :strand
+  end
+end

data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb

@@ -0,0 +1,26 @@
+class CleanupDelayedJobsIndexes < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    case connection.adapter_name
+    when 'PostgreSQL'
+      # "nulls first" syntax is postgresql specific, and allows for more
+      # efficient querying for the next job
+      connection.execute("CREATE INDEX get_delayed_jobs_index ON delayed_jobs (priority, run_at, failed_at nulls first, locked_at nulls first, queue)")
+    else
+      add_index :delayed_jobs, %w(priority run_at locked_at failed_at queue), :name => 'get_delayed_jobs_index'
+    end
+
+    # unused indexes
+    remove_index :delayed_jobs, :name => 'delayed_jobs_queue'
+    remove_index :delayed_jobs, :name => 'delayed_jobs_priority'
+  end
+
+  def self.down
+    remove_index :delayed_jobs, :name => 'get_delayed_jobs_index'
+    add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
+    add_index :delayed_jobs, [:queue], :name => 'delayed_jobs_queue'
+  end
+end

data/db/migrate/20110610213249_optimize_delayed_jobs.rb

@@ -0,0 +1,40 @@
+class OptimizeDelayedJobs < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    create_table :failed_jobs do |t|
+      t.integer  "priority",    :default => 0
+      t.integer  "attempts",    :default => 0
+      t.string   "handler",     :limit => 512000
+      t.integer  "original_id", :limit => 8
+      t.text     "last_error"
+      t.string   "queue"
+      t.datetime "run_at"
+      t.datetime "locked_at"
+      t.datetime "failed_at"
+      t.string   "locked_by"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+      t.string   "tag"
+      t.integer  "max_attempts"
+      t.string   "strand"
+    end
+
+    remove_index :delayed_jobs, :name => 'get_delayed_jobs_index'
+    remove_index :delayed_jobs, [:strand]
+
+    add_index :delayed_jobs, %w(run_at queue locked_at strand priority), :name => 'index_delayed_jobs_for_get_next'
+    add_index :delayed_jobs, %w(strand id), :name => 'index_delayed_jobs_on_strand'
+
+    # move all failed jobs to the new failed table
+    Delayed::Backend::ActiveRecord::Job.where("failed_at IS NOT NULL").find_each do |job|
+      job.fail! unless job.on_hold?
+    end
+  end
+
+  def self.down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb

@@ -0,0 +1,52 @@
+class AddDelayedJobsNextInStrand < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    remove_index :delayed_jobs, :name => 'index_delayed_jobs_for_get_next'
+
+    add_column :delayed_jobs, :next_in_strand, :boolean, :default => true, :null => false
+
+    # create the new index
+    connection.execute("CREATE INDEX get_delayed_jobs_index ON delayed_jobs (priority, run_at, queue) WHERE locked_at IS NULL AND next_in_strand = 't'")
+
+    # create the insert trigger
+    execute(<<-CODE)
+    CREATE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+    BEGIN
+      LOCK delayed_jobs IN SHARE ROW EXCLUSIVE MODE;
+      IF (SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand LIMIT 1) = 1 THEN
+        NEW.next_in_strand := 'f';
+      END IF;
+      RETURN NEW;
+    END;
+    $$ LANGUAGE plpgsql;
+    CODE
+    execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON delayed_jobs FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE delayed_jobs_before_insert_row_tr_fn()")
+
+    # create the delete trigger
+    execute(<<-CODE)
+    CREATE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+    BEGIN
+      UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1);
+      RETURN OLD;
+    END;
+    $$ LANGUAGE plpgsql;
+    CODE
+    execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON delayed_jobs FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE delayed_jobs_after_delete_row_tr_fn()")
+
+    execute(%{UPDATE delayed_jobs SET next_in_strand = 'f' WHERE strand IS NOT NULL AND id <> (SELECT id FROM delayed_jobs j2 WHERE j2.strand = delayed_jobs.strand ORDER BY j2.strand, j2.id ASC LIMIT 1)})
+  end
+
+  def self.down
+    execute %{DROP TRIGGER delayed_jobs_before_insert_row_tr ON delayed_jobs}
+    execute %{DROP FUNCTION delayed_jobs_before_insert_row_tr_fn()}
+    execute %{DROP TRIGGER delayed_jobs_after_delete_row_tr ON delayed_jobs}
+    execute %{DROP FUNCTION delayed_jobs_after_delete_row_tr_fn()}
+
+    remove_column :delayed_jobs, :next_in_strand
+    remove_index :delayed_jobs, :name => 'get_delayed_jobs_index'
+    add_index :delayed_jobs, %w(run_at queue locked_at strand priority), :name => 'index_delayed_jobs_for_get_next'
+  end
+end

data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb

@@ -0,0 +1,31 @@
+class DelayedJobsDeleteTriggerLockForUpdate < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    if connection.adapter_name == 'PostgreSQL'
+      execute(<<-CODE)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      BEGIN
+        UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1 FOR UPDATE);
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+      CODE
+    end
+  end
+
+  def self.down
+    if connection.adapter_name == 'PostgreSQL'
+      execute(<<-CODE)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      BEGIN
+        UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1);
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+      CODE
+    end
+  end
+end

data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb

@@ -0,0 +1,15 @@
+class DropPsqlJobsPopFn < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    if connection.adapter_name == 'PostgreSQL'
+      connection.execute("DROP FUNCTION IF EXISTS pop_from_delayed_jobs(varchar, varchar, integer, integer, timestamp without time zone)")
+    end
+  end
+
+  def self.down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb

@@ -0,0 +1,80 @@
+class DelayedJobsUseAdvisoryLocks < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    # use an advisory lock based on the name of the strand, instead of locking the whole table
+    # note that we're using half of the md5, so collisions are possible, but we don't really
+    # care because that would just be the old behavior, whereas for the most part locking will
+    # be much smaller
+    if connection.adapter_name == 'PostgreSQL'
+      execute(<<-CODE)
+      CREATE FUNCTION half_md5_as_bigint(strand varchar) RETURNS bigint AS $$
+      DECLARE
+        strand_md5 bytea;
+      BEGIN
+        strand_md5 := decode(md5(strand), 'hex');
+        RETURN (CAST(get_byte(strand_md5, 0) AS bigint) << 56) +
+                                  (CAST(get_byte(strand_md5, 1) AS bigint) << 48) +
+                                  (CAST(get_byte(strand_md5, 2) AS bigint) << 40) +
+                                  (CAST(get_byte(strand_md5, 3) AS bigint) << 32) +
+                                  (CAST(get_byte(strand_md5, 4) AS bigint) << 24) +
+                                  (get_byte(strand_md5, 5) << 16) +
+                                  (get_byte(strand_md5, 6) << 8) +
+                                   get_byte(strand_md5, 7);
+      END;
+      $$ LANGUAGE plpgsql;
+      CODE
+
+      execute(<<-CODE)
+      CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+      BEGIN
+        PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+        IF (SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand LIMIT 1) = 1 THEN
+          NEW.next_in_strand := 'f';
+        END IF;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql;
+      CODE
+
+      execute(<<-CODE)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      BEGIN
+        PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+        UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1 FOR UPDATE);
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+      CODE
+    end
+  end
+
+  def self.down
+    if connection.adapter_name == 'PostgreSQL'
+      execute(<<-CODE)
+      CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+      BEGIN
+        LOCK delayed_jobs IN SHARE ROW EXCLUSIVE MODE;
+        IF (SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand LIMIT 1) = 1 THEN
+          NEW.next_in_strand := 'f';
+        END IF;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql;
+      CODE
+
+      execute(<<-CODE)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      BEGIN
+        UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1 FOR UPDATE);
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+      CODE
+
+      execute('DROP FUNCTION half_md5_as_bigint(varchar)')
+    end
+  end
+end

data/db/migrate/20120607181141_index_jobs_on_locked_by.rb

@@ -0,0 +1,15 @@
+class IndexJobsOnLockedBy < ActiveRecord::Migration
+  disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
+
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    add_index :delayed_jobs, :locked_by, :algorithm => :concurrently, :where => "locked_by IS NOT NULL"
+  end
+
+  def self.down
+    remove_index :delayed_jobs, :locked_by
+  end
+end

data/db/migrate/20120608191051_add_jobs_run_at_index.rb

@@ -0,0 +1,15 @@
+class AddJobsRunAtIndex < ActiveRecord::Migration
+  disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
+
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    add_index :delayed_jobs, %w[run_at tag], :algorithm => :concurrently
+  end
+
+  def self.down
+    remove_index :delayed_jobs, :name => "index_delayed_jobs_on_run_at_and_tag"
+  end
+end

data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb

@@ -0,0 +1,13 @@
+class ChangeDelayedJobsHandlerToText < ActiveRecord::Migration
+  def self.connection
+    Delayed::Job.connection
+  end
+
+  def self.up
+    change_column :delayed_jobs, :handler, :text
+  end
+
+  def self.down
+    change_column :delayed_jobs, :handler, :string, :limit => 500.kilobytes
+  end
+end

data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb

@@ -0,0 +1,13 @@
+class AddFailedJobsOriginalJobId < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    add_column :failed_jobs, :original_job_id, :integer, limit: 8
+  end
+
+  def self.down
+    remove_column :failed_jobs, :original_job_id
+  end
+end

data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb

@@ -0,0 +1,13 @@
+class CopyFailedJobsOriginalId < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    # this is a smaller, less frequently accessed table, so we just update all at once
+    Delayed::Backend::ActiveRecord::Job::Failed.where("original_job_id is null").update_all("original_job_id = original_id")
+  end
+
+  def self.down
+  end
+end

data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb

@@ -0,0 +1,13 @@
+class DropFailedJobsOriginalId < ActiveRecord::Migration
+  def self.connection
+    Delayed::Backend::ActiveRecord::Job.connection
+  end
+
+  def self.up
+    remove_column :failed_jobs, :original_id
+  end
+
+  def self.down
+    add_column :failed_jobs, :original_id, :integer, limit: 8
+  end
+end