postjob 0.1.1

data/lib/postjob.rb

@@ -0,0 +1,170 @@
+require "expectation"
+require "simple/sql"
+
+require_relative "postjob/version"
+require_relative "postjob/workflow"
+require_relative "postjob/registry"
+require_relative "postjob/job"
+require_relative "postjob/error"
+require_relative "postjob/queue"
+require_relative "postjob/runner"
+
+module Postjob
+  attr_accessor :logger
+  extend self
+
+  # In fast mode <tt>Postjob.run</tt> doesn't wait that long between retrying
+  # failed jobs. This mode is enabled by default during tests; and it can be
+  # enabled via "postjob run --fast"
+  #
+  # Note that fast mode should only be used during development and tests.
+  attr_accessor :fast_mode
+  self.fast_mode = false
+
+  def enqueue!(workflow, *args, queue: nil,
+               parent_id: nil,
+               max_attempts: nil,
+               timeout: nil,
+               version: nil,
+               tags: nil)
+    expect! queue => [nil, String]
+    expect! workflow => String
+    expect! parent_id => [nil, Integer]
+    expect! max_attempts => [nil, Integer]
+    expect! timeout => [nil, Numeric]
+    expect! tags => [nil, Hash]
+
+    tags = stringify_hash(tags) if tags
+    job = Queue.enqueue_job workflow, *args, queue: queue,
+                                             parent_id: parent_id,
+                                             max_attempts: max_attempts,
+                                             timeout: timeout,
+                                             tags: tags,
+                                             version: version
+    logger.info "Generated process #{job}"
+    job.id
+  end
+
+  private
+
+  def stringify_hash(hsh)
+    hsh.inject({}) do |r, (k, v)|
+      k = k.to_s if k.is_a?(Symbol)
+      r.update k => v
+    end
+  end
+
+  public
+
+  # process all waiting jobs.
+  #
+  # This method starts processing jobs, as long as there are some. It returns
+  # once no runnable jobs can be found anymore.
+  #
+  # Note that this method is not limited to the set of runnable jobs present
+  # when calling it; if running a job results in newly created runnable jobs
+  # these jobs will be processed as well.
+  #
+  # This method returns the number of processed jobs.
+  def process_all
+    run do |job|
+      !job.nil?
+    end
+  end
+
+  # processes many jobs.
+  #
+  # This method starts processing jobs, as long as there are some. If no
+  # jobs can be found this method waits until a job becomes available.
+  #
+  # After processing each job is yielded into the passed in block.
+  #
+  # This method continues until:
+  # a) the requested number of jobs (via the count: argument) was processed (note:
+  #    repeated job executions due to rerunning jobs that slept or errored count
+  #    multiple times), or
+  # b) the block yielded into returns false.
+  #
+  # This method returns the number of processed jobs.
+  def run(count: nil, &block)
+    # to run 10^12 jobs that would take 1 msecs each we would need, at least,
+    # 760 years - so this default should be fine. Also, someone should update
+    # the machine in the meantime :)
+    count ||= 1_000_000_000_000
+
+    processed_jobs_count = 0
+
+    loop do
+      processed_job = Postjob.step
+      processed_jobs_count += 1 if processed_job
+
+      break if processed_jobs_count >= count
+      break if block && (yield(processed_job) == false)
+
+      next if processed_job
+
+      Queue::Notifications.wait_for_new_job
+    end
+
+    processed_jobs_count
+  end
+
+  # Runs a single job
+  #
+  # This method tries to check out a runnable job. If it finds one the
+  # job is processed (via Postjob.process_job) and returned. If not,
+  # this method just returns nil.
+  def step
+    Queue.checkout_runnable do |job|
+      process_job job
+    end
+  end
+
+  private
+
+  # This method is called from tests. Otherwise it is supposed to be private.
+  # rubocop:disable Metrics/CyclomaticComplexity
+  def process_job(job) # :nodoc:
+    expect! job => Job
+
+    version, status, value = if job.timed_out
+                               [job.workflow_version, :timeout, nil]
+                             else
+                               Runner.process_job(job)
+                             end
+
+    if job.workflow_version != "" && version != job.workflow_version
+      raise "Integrity check failed: job's workflow version changed (from #{job.workflow_version} to #{version})"
+    end
+
+    expect! version => String
+
+    case status
+    when :failed    then Queue.set_job_error job, *value, status: :failed, version: version
+    when :err       then Queue.set_job_error job, *value, status: :err, version: version
+    when :timeout   then Queue.set_job_error job, "Timeout", "Timeout", status: :timeout, version: version
+    when :pending   then Queue.set_job_pending job, version: version
+    when :ok        then Queue.set_job_result job, value, version: version
+    else            raise ArgumentError, "Invalid status #{status.inspect}"
+    end
+
+    [status, value]
+  end
+
+  public
+
+  def resolve(token:, result:)
+    job = Queue.find_job_by_token(token)
+    Queue.set_job_result job, result, version: nil
+  end
+
+  def register_workflow(workflow, options = {})
+    expect! options => {
+      version: [nil, /^\d+(\.\d+)*/]
+    }
+
+    workflow.include Postjob::Workflow
+    workflow.send(:set_workflow_version, options[:version] || "0.0")
+    Registry.register workflow, options
+  end
+end

data/spec/postjob/enqueue_spec.rb

@@ -0,0 +1,86 @@
+# rubocop:disable Metrics/BlockLength
+
+require "spec_helper"
+
+describe "Postjob.enqueue!" do
+  include TestHelper
+
+  context "with valid arguments" do
+    it "creates a job" do
+      Postjob.enqueue! "FooishWorkflow"
+      expect(TestHelper.jobs_count).to eq(1)
+    end
+
+    it "returns the created job" do
+      id = Postjob.enqueue! "FooishWorkflow"
+      newest_job = TestHelper.newest_job
+      expect(newest_job.id).to eq(id)
+    end
+
+    it "uses the parent id and sets a full id" do
+      id1 = Postjob.enqueue! "FooishWorkflow"
+      id2 = Postjob.enqueue! "FooishWorkflow", parent_id: id1
+      id3 = Postjob.enqueue! "FooishWorkflow", parent_id: id2
+
+      job1 = load_job id1
+      job2 = load_job id2
+      job3 = load_job id3
+
+      expect(job3.full_id).to eq("#{job1.id}.#{job2.id}.#{job3.id}")
+      expect(job3.full_id).to match(/\d+\.\d+\.\d+/)
+    end
+
+    it "sets arguments" do
+      id1 = Postjob.enqueue! "FooishWorkflow", 1, "two", "three"
+      job1 = load_job id1
+      expect(job1.args).to eq([1, "two", "three"])
+
+      id2 = Postjob.enqueue! "FooishWorkflow"
+      job2 = load_job id2
+      expect(job2.args).to eq([])
+    end
+
+    it "sets queue" do
+      id1 = Postjob.enqueue! "FooishWorkflow"
+      job1 = load_job id1
+      expect(job1.queue).to eq("q")
+
+      id1 = Postjob.enqueue! "FooishWorkflow", queue: "bla"
+      job1 = load_job id1
+      expect(job1.queue).to eq("bla")
+    end
+
+    it "sets max_attempts" do
+      id1 = Postjob.enqueue! "FooishWorkflow"
+      job1 = load_job id1
+      expect(job1.max_attempts).to eq(5)
+
+      id1 = Postjob.enqueue! "FooishWorkflow", max_attempts: 2
+      job1 = load_job id1
+      expect(job1.max_attempts).to eq(2)
+    end
+
+    it "sets the status to ready" do
+      id1 = Postjob.enqueue! "FooishWorkflow", 1, "two", "three"
+      job1 = load_job id1
+      expect(job1.status).to eq("ready")
+    end
+  end
+
+  context "with invalid arguments" do
+    it "raises an error with missing args" do
+      expect { Postjob.enqueue! queue: "q" }.to raise_error(ArgumentError)
+      expect { Postjob.enqueue! workflow: "q" }.to raise_error(ArgumentError)
+    end
+
+    it "raises an error with invalid args" do
+      expect { Postjob.enqueue! queue: 1, workflow: 13 }.to raise_error(ArgumentError)
+    end
+
+    it "raises an error with invalid parent_id" do
+      id1 = Postjob.enqueue! "FooishWorkflow"
+
+      expect { Postjob.enqueue! "FooishWorkflow", parent_id: id1 - 1 }.to raise_error(PG::ForeignKeyViolation)
+    end
+  end
+end

data/spec/postjob/full_workflow_spec.rb

@@ -0,0 +1,86 @@
+require "spec_helper"
+
+#
+# This module simulates the MPX services interfaces, with a certain
+# change not to be able to run (see operational_error?)
+module MPX
+  module Impl
+    extend self
+
+    def self.load_users(user_ids)
+      user_ids.map do |user_id|
+        {
+          "type" => "User",
+          "id" => user_id
+        }
+      end
+    end
+
+    def self.clone_group(group_id)
+      group_id + 1000
+    end
+
+    def self.send_email(_user, _cloned_group_id)
+      "sent"
+    end
+  end
+
+  def self.method_missing(sym, *args)
+    operational_error?
+    Impl.send sym, *args
+  end
+
+  # The operational_error method introduces a certain chance of not being able
+  # to run one of the MPX.* methods, simulating a network timeout or other such
+  # error condition.
+  def self.operational_error?
+    raise "please rerun" if rand < 0.1
+  end
+end
+
+module RecommendGroupWorkflow
+  Postjob.register_workflow self
+
+  def self.clone_group(group_id)
+    MPX.clone_group(group_id)
+  end
+
+  def self.load_users(user_ids)
+    MPX.load_users(user_ids)
+  end
+
+  def self.send_email(user, cloned_group_id)
+    MPX.send_email user, cloned_group_id
+  end
+
+  def self.run(group_id, user_ids)
+    users = await :load_users, user_ids
+
+    cloned_group_id = await :clone_group, group_id
+    users.each do |user|
+      async :send_email, user, cloned_group_id
+    end
+    await :all
+    users.count
+  end
+end
+
+describe "Sub Workflows" do
+  include TestHelper
+
+  let!(:users_count) { 100 }
+  let!(:user_ids) { 100.upto(100 + users_count - 1).to_a }
+  let!(:id) { Postjob.enqueue! "RecommendGroupWorkflow", 1, user_ids }
+
+  it "runs the job returning the result" do
+    expect(MPX::Impl).to receive(:clone_group).exactly(1).times.and_call_original
+    expect(MPX::Impl).to receive(:load_users).exactly(1).times.and_call_original
+    expect(MPX::Impl).to receive(:send_email).exactly(users_count).times.and_call_original
+
+    Postjob.process_all
+
+    print_jobs
+    processed_job = load_job(id)
+    expect(processed_job.status).to eq("ok")
+  end
+end

data/spec/postjob/job_control/manual_spec.rb

@@ -0,0 +1,45 @@
+require "spec_helper"
+
+module ManualWorkflow
+  def self.run
+    job = async :manual, timeout: 10
+    _token = workflow_token job
+    manual_result = await job
+    "manual-result:#{manual_result}"
+  end
+
+  Postjob.register_workflow self
+end
+
+describe "manual processing" do
+  let!(:id) { Postjob.enqueue! "ManualWorkflow" }
+
+  include TestHelper
+
+  before do
+    Postjob.process_all
+  end
+
+  def load_child_job
+    TestHelper.load_job "SELECT * FROM postjobs WHERE parent_id=$1", id
+  end
+
+  def token
+    load_token(load_child_job)
+  end
+
+  it "creates a token" do
+    expect! token => /[A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[a-fA-F0-9]{4}-[A-F0-9]{12}/i
+  end
+
+  it "the token can be used to resolve the load_child_job" do
+    Postjob.resolve token: token, result: "foobar"
+    expect(load_child_job.status).to eq("ok")
+    expect(load_child_job.result).to eq("foobar")
+
+    Postjob.process_all
+    job = load_job id
+    expect(job.status).to eq("ok")
+    expect(job.result).to eq("manual-result:foobar")
+  end
+end

data/spec/postjob/job_control/max_attempts_spec.rb

@@ -0,0 +1,70 @@
+# rubocop:disable Metrics/BlockLength
+
+require "spec_helper"
+
+module MaxAttemptWorkflow
+  module ChildWorker
+    def self.run
+      raise "Nono nono!"
+    end
+
+    Postjob.register_workflow self
+  end
+
+  def self.run(max_attempts)
+    set_workflow_status "starting up"
+    await async ChildWorker, max_attempts: max_attempts
+    set_workflow_status "starting finished"
+  end
+
+  Postjob.register_workflow self
+end
+
+describe "max_attempts" do
+  before do
+    Simple::SQL.all("DELETE FROM postjob.postjobs")
+  end
+
+  def load_child_job
+    TestHelper.load_job "SELECT * FROM postjobs WHERE parent_id=$1", id
+  end
+
+  def load_job
+    TestHelper.load_job(id)
+  end
+
+  def load_child_job
+    TestHelper.load_job "SELECT * FROM postjobs WHERE parent_id=$1", id
+  end
+
+  context "when running with max_attempts 1" do
+    let!(:id) { Postjob.enqueue!("MaxAttemptWorkflow", 1, max_attempts: 1) }
+
+    it "fails the childjob with a timeout" do
+      while Postjob.process_all > 0
+        sleep 0.03
+      end
+
+      expect(load_child_job.failed_attempts).to eq(1)
+      expect(load_child_job.status).to eq("failed")
+
+      expect(load_job.status).to eq("failed")
+    end
+  end
+
+  context "when running with max_attempts 5" do
+    let!(:id) { Postjob.enqueue!("MaxAttemptWorkflow", 5, max_attempts: 1) }
+
+    it "fails the childjob with a timeout" do
+      while Postjob.process_all > 0
+        sleep 0.03
+      end
+
+      expect(load_child_job.failed_attempts).to eq(5)
+      expect(load_child_job.status).to eq("failed")
+
+      # The main job now failed. It only ran with max_attempts: 1
+      expect(load_job.status).to eq("failed")
+    end
+  end
+end

data/spec/postjob/job_control/timeout_spec.rb

@@ -0,0 +1,31 @@
+require "spec_helper"
+
+module FastFailWorkflow
+  def self.run
+    await async :manual, timeout: 0.000001
+  end
+
+  Postjob.register_workflow self
+end
+
+describe "timeout" do
+  context "with an instant timeout" do
+    let!(:id) { Postjob.enqueue! "FastFailWorkflow" }
+
+    before do
+      Postjob.process_all
+    end
+
+    it "fails the childjob with a timeout" do
+      manual_job = TestHelper.load_job "SELECT * FROM postjobs WHERE parent_id=$1", id
+      expect(manual_job.status).to eq("timeout")
+      expect { manual_job.resolve }.to raise_error(Timeout::Error)
+    end
+
+    xit "fails the mainjob" do
+      job = TestHelper.load_job(id)
+      expect(job.status).to eq("err")
+      expect { job.resolve }.to raise_error(Postjob::Error)
+    end
+  end
+end

data/spec/postjob/job_control/workflow_status_spec.rb

@@ -0,0 +1,52 @@
+require "spec_helper"
+
+module WorkflowStatusWorkflow
+  def self.run
+    set_workflow_status "starting-up"
+
+    job = async :manual, timeout: 10
+    set_workflow_status "created-manual"
+
+    _token = workflow_token job
+    set_workflow_status "created-manual-token"
+
+    manual_result = await job
+    set_workflow_status "got-manual-token"
+
+    "manual-result:#{manual_result}"
+  end
+
+  Postjob.register_workflow self
+end
+
+describe "workflow_status" do
+  let!(:id) { Postjob.enqueue! "WorkflowStatusWorkflow" }
+
+  include TestHelper
+
+  before do
+    Postjob.process_all
+  end
+
+  def load_child_job
+    TestHelper.load_job "SELECT * FROM postjobs WHERE parent_id=$1", id
+  end
+
+  def token
+    load_token(load_child_job)
+  end
+
+  it "creates a token" do
+    expect! token => /[A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[a-fA-F0-9]{4}-[A-F0-9]{12}/i
+  end
+
+  it "the token can be used to resolve the load_child_job" do
+    Postjob.resolve token: token, result: "foobar"
+    expect(load_child_job.status).to eq("ok")
+    expect(load_job(id).workflow_status).to eq("created-manual-token")
+
+    Postjob.process_all
+    expect(load_job(id).status).to eq("ok")
+    expect(load_job(id).workflow_status).to eq("got-manual-token")
+  end
+end

data/spec/postjob/process_job_spec.rb

@@ -0,0 +1,25 @@
+require "spec_helper"
+
+module BarWorkflow
+  def self.run
+    "Foo"
+  end
+
+  Postjob.register_workflow self, version: "1.2"
+end
+
+describe "Postjob.process_job" do
+  include TestHelper
+
+  let!(:job) { Postjob.enqueue! "BarWorkflow" }
+
+  it "runs the job returning the result" do
+    expect(Postjob.process_job(newest_job)).to eq([:ok, "Foo"])
+  end
+
+  it "updates the job version" do
+    expect(newest_job.workflow_version).to eq("")
+    Postjob.process_job(newest_job)
+    expect(newest_job.workflow_version).to eq("1.2")
+  end
+end

data/spec/postjob/queue/encoder_spec.rb

@@ -0,0 +1,46 @@
+require "spec_helper"
+
+module EncodeSpecWorkflow
+  def self.run(_inp, out)
+    out
+  end
+
+  Postjob.register_workflow self
+end
+
+describe "Postjob::Queue::Encoder" do
+  def run_workflow(input: "dummy input", output: "dummy output")
+    id = Postjob.enqueue! "EncodeSpecWorkflow", input, output
+    Postjob.process_all
+    job = TestHelper.load_job(id)
+    job.result
+  end
+
+  it "allows encoding of numbers" do
+    expect(run_workflow(input: 10, output: 12)).to eq(12)
+  end
+
+  it "allows encoding of booleans" do
+    expect(run_workflow(input: true, output: false)).to eq(false)
+  end
+
+  it "allows encoding of strings" do
+    expect(run_workflow(input: "foo", output: "bar")).to eq("bar")
+  end
+
+  it "allows encoding of arrays" do
+    expect(run_workflow(input: ["foo", "foo"], output: ["bar", "baz"])).to eq(["bar", "baz"])
+  end
+
+  describe "invalid input" do
+    it "cannot encode Symbols on input" do
+      expect do
+        run_workflow input: :foo
+      end.to raise_error(ArgumentError)
+
+      expect do
+        run_workflow output: :foo
+      end.to raise_error(ArgumentError)
+    end
+  end
+end