perfectqueue 0.7.32 → 0.8.0

data/spec/spec_helper.rb

@@ -0,0 +1,44 @@
+$LOAD_PATH.unshift(File.expand_path('../lib', File.dirname(__FILE__)))
+
+require 'perfectqueue'
+
+if ENV['SIMPLE_COV']
+  require 'simplecov'
+  SimpleCov.start do
+    add_filter 'spec/'
+    add_filter 'pkg/'
+    add_filter 'vendor/'
+  end
+end
+
+require 'fileutils'
+
+def test_queue_config
+  {:type=>'rdb_compat', :url=>'sqlite://spec/test.db', :table=>'test_tasks', :processor_type=>'thread'}
+end
+
+def create_test_queue
+  FileUtils.rm_f 'spec/test.db'
+  queue = PerfectQueue.open(test_queue_config)
+
+  sql = %[
+      CREATE TABLE IF NOT EXISTS `test_tasks` (
+        id VARCHAR(256) NOT NULL,
+        timeout INT NOT NULL,
+        data BLOB NOT NULL,
+        created_at INT,
+        resource VARCHAR(256),
+        PRIMARY KEY (id)
+      );]
+
+  queue.client.backend.db.run sql
+
+  return queue
+end
+
+def get_test_queue
+  PerfectQueue.open(test_queue_config)
+end
+
+include PerfectQueue
+

data/spec/worker_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+
+class TestHandler < PerfectQueue::Application::Base
+  def run
+    puts "TestHandler: #{task}"
+    if task.data['raise_error']
+      raise "expected error test"
+    end
+    if num = task.data['sleep']
+      sleep num
+    end
+    puts "Task finished"
+  end
+
+  def kill(reason)
+    puts "kill: #{reason.class}: #{reason}"
+  end
+end
+
+class RegexpHandler < PerfectQueue::Application::Base
+  def run
+    puts "RegexpHandler: #{task}"
+  end
+end
+
+class TestApp < PerfectQueue::Application::Dispatch
+  route 'test' => TestHandler
+  route /reg.*/ => RegexpHandler
+end
+
+describe Worker do
+  before do
+    create_test_queue.close
+    @worker = Worker.new(TestApp, test_queue_config)
+    @thread = Thread.new {
+      @worker.run
+    }
+  end
+
+  after do
+    @worker.stop(true)
+    @thread.join
+  end
+
+  def submit(*args)
+    queue = get_test_queue
+    queue.submit(*args)
+    queue.close
+  end
+
+  it 'route' do
+    TestHandler.any_instance.should_receive(:run).once
+    RegexpHandler.any_instance.should_receive(:run).once
+    submit('task01', 'test', {})
+    submit('task02', 'reg01', {})
+    sleep 1
+  end
+
+  it 'term signal' do
+    sleep 1
+    Process.kill(:TERM, Process.pid)
+    puts "finish expected..."
+    @thread.join
+  end
+
+  it 'quit signal' do
+    sleep 1
+    Process.kill(:QUIT, Process.pid)
+    puts "finish expected..."
+    @thread.join
+  end
+
+  it 'kill reason' do
+    TestHandler.any_instance.should_receive(:kill).once #.with(kind_of(PerfectQueue::CancelRequestedError))  # FIXME 'with' dead locks
+    submit('task01', 'test', {'sleep'=>4})
+    sleep 2
+    Process.kill(:TERM, Process.pid)
+    @thread.join
+  end
+end
+

metadata

@@ -1,89 +1,142 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.7.32
+  version: 0.8.0
+  prerelease: 
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-22 00:00:00.000000000 Z
+date: 2012-05-23 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70295369852020 !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 3.48.0
+        version: 3.26.0
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  version_requirements: *70295369852020
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70295369850460 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: *70295369850460
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70295369849140 !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 3.48.0
-description: 
+        version: 2.10.0
+  type: :development
+  prerelease: false
+  version_requirements: *70295369849140
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: &70295369848300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :development
+  prerelease: false
+  version_requirements: *70295369848300
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: &70295369847400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.3
+  type: :development
+  prerelease: false
+  version_requirements: *70295369847400
+description: Highly available distributed cron built on RDBMS
 email: frsyuki@gmail.com
 executables:
 - perfectqueue
 extensions: []
-extra_rdoc_files:
-- ChangeLog
-- README.rdoc
+extra_rdoc_files: []
 files:
+- .gitignore
 - ChangeLog
-- README.rdoc
+- Gemfile
+- README.md
+- Rakefile
 - bin/perfectqueue
 - lib/perfectqueue.rb
+- lib/perfectqueue/application.rb
+- lib/perfectqueue/application/base.rb
+- lib/perfectqueue/application/dispatch.rb
+- lib/perfectqueue/application/router.rb
 - lib/perfectqueue/backend.rb
-- lib/perfectqueue/backend/null.rb
-- lib/perfectqueue/backend/rdb.rb
-- lib/perfectqueue/backend/simpledb.rb
+- lib/perfectqueue/backend/rdb_compat.rb
+- lib/perfectqueue/blocking_flag.rb
+- lib/perfectqueue/client.rb
 - lib/perfectqueue/command/perfectqueue.rb
+- lib/perfectqueue/daemons_logger.rb
 - lib/perfectqueue/engine.rb
+- lib/perfectqueue/error.rb
+- lib/perfectqueue/model.rb
+- lib/perfectqueue/multiprocess.rb
+- lib/perfectqueue/multiprocess/child_process.rb
+- lib/perfectqueue/multiprocess/child_process_monitor.rb
+- lib/perfectqueue/multiprocess/fork_processor.rb
+- lib/perfectqueue/multiprocess/thread_processor.rb
+- lib/perfectqueue/queue.rb
+- lib/perfectqueue/runner.rb
+- lib/perfectqueue/signal_queue.rb
+- lib/perfectqueue/task.rb
+- lib/perfectqueue/task_metadata.rb
+- lib/perfectqueue/task_monitor.rb
+- lib/perfectqueue/task_status.rb
 - lib/perfectqueue/version.rb
 - lib/perfectqueue/worker.rb
-- test/backend_test.rb
-- test/cat.sh
-- test/echo.sh
-- test/exec_test.rb
-- test/fail.sh
-- test/huge.sh
-- test/stress.rb
-- test/success.sh
-- test/test_helper.rb
+- perfectqueue.gemspec
+- spec/queue_spec.rb
+- spec/spec_helper.rb
+- spec/worker_spec.rb
 homepage: https://github.com/treasure-data/perfectqueue
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 1.8.12
 signing_key: 
-specification_version: 4
-summary: Highly available distributed queue built on RDBMS or SimpleDB
+specification_version: 3
+summary: Highly available distributed cron built on RDBMS
 test_files:
-- test/backend_test.rb
-- test/exec_test.rb
-- test/stress.rb
-- test/test_helper.rb
-- test/cat.sh
-- test/echo.sh
-- test/fail.sh
-- test/huge.sh
-- test/success.sh
+- spec/queue_spec.rb
+- spec/spec_helper.rb
+- spec/worker_spec.rb
+has_rdoc: false

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: f80914c5d9cacd678348735b7bb64343153ee3c0
-  data.tar.gz: 5e5c88e7748b4d8ad13e5b65ceb569935434936c
-SHA512:
-  metadata.gz: 0f4d15f8e6ac83846de46a1363a92a260f30568480117f3536a154fdd06ffc874837f11c8df030ccadbcc827562d7448e82ae6463eb9c44dc4e37a50bedbe3d0
-  data.tar.gz: 3d433c82757132a7866be3cb13ec6d626fe049b760c43ef52e5f55b11b6dce9d6ed35229c6a117a67f2da7623ac8a3cac303c2877ba3d3ff673e7ff818ee5ca0

data/README.rdoc

@@ -1,224 +0,0 @@
-= PerfectQueue
-
-Highly available distributed queue.
-
-It provides exactly-once semantics unless backend database fails. Pushed tasks are surely retried by another worker node even if a worker fails. And it never delivers finished/canceled tasks.
-
-Backend database is pluggable. You can use any databases that supports CAS (compare-and-swap) operation. PerfectQueue supports RDBMS and Amazon SimpleDB for now.
-
-
-== Architecture
-
-PerfectQueue uses following database schema:
-
-  (
-    id:string        -- unique identifier of the task
-    data:blob        -- additional attributes of the task
-    created_at:int   -- unix time when the task is created (or null for canceled tasks)
-    timeout:int
-  )
-
-1. list: lists tasks whose timeout column is old enough.
-2. lock: updates timeout column of the first task
-3. run: executes a command
-   * if the task takes long time, updates the timeout column. this is repeated until the task is finished
-   * if the task takes more long time, kills the process
-4. remove: if it succeeded, removes the row from the backend database
-5. or leave: if it failed, leave the row and expect to be retried
-
-
-== Usage
-
-=== Submitting a task
-
-*Using* *command* *line:*
-
-  # RDBMS
-  $ perfectqueue \
-        --database mysql://user:password@localhost/mydb \
-        --table perfectqueue \
-        --push unique-key-id '{"any":"data"}'
-
-  # SimpleDB
-  $ perfectqueue \
-        --simpledb your-simpledb-domain-name \
-        -k AWS_KEY_ID \
-        -s AWS_SECRET_KEY \
-        --push unique-key-id '{"any":"data"}'
-
-*Using* *PerfectQueue* *library:*
-
-  require 'perfectqueue'
-  
-  # RDBMS
-  require 'perfectqueue/backend/rdb'
-  queue = PerfectQueue::RDBBackend.new(
-         'mysql://user:password@localhost/mydb', table='perfectqueue')
-
-  # SimpleDB
-  require 'perfectqueue/backend/simpledb'
-  queue = PerfectQueue::SimpleDBBackend.new(
-         'AWS_KEY_ID', 'AWS_SECRET_KEY', 'your-simpledb-domain-name')
-  
-  queue.submit('unique-key-id', '{"any":"data"}')
-
-
-Alternatively, you can insert a row into the backend database directly.
-
-*RDBMS:*
-
-  > CREATE TABLE IF NOT EXISTS perfectqueue (
-        id VARCHAR(256) NOT NULL,
-        timeout INT NOT NULL,
-        data BLOB NOT NULL,
-        created_at INT,
-        PRIMARY KEY (id)
-      );
-  > SET @now = UNIX_TIMESTAMP();
-  > INSERT INTO perfectqueue (id, timeout, data, created_at)
-      VALUES ('unique-task-id', @now, '{"any":"data"}', @now);
-
-*SimpleDB:*
-
-  require 'aws'  # gem install aws-sdk
-  queue = AWS::SimpleDB.new
-  domain = queue.domains['your-simpledb-domain-name']
-  
-  now = "%08x" % Time.now.to_i
-  domain.items['unique-task-id'].attributes.replace(
-      'timeout'=>now, 'data'=>'{"any":"data"}', 'created_at'=>now,
-      :unless=>'timeout')
-
-
-=== Canceling a queued task
-
-*Using* *command* *line:*
-
-  $ perfectqueue ... --cancel unique-key-id
-
-*Using* *PerfectQueue* *library:*
-
-  queue.cancel('unique-key-id')
-
-
-Alternatively, you can delete a row from the backend database directly.
-
-*RDBMS:*
-
-  > DELETE FROM perfectqueue WHERE id='unique-key-id';
-
-*SimpleDB:*
-
-  domain.items['unique-task-id'].delete
-
-
-=== Running worker node
-
-Use _perfectqueue_ command to execute a command.
-
-  Usage: perfectqueue [options] [-- <ARGV-for-exec-or-run>]
-  
-          --push <ID> <DATA>           Push a task to the queue
-          --list                       Show queued tasks
-          --cancel <ID>                Cancel a queued task
-          --configure <PATH.yaml>      Write configuration file
-  
-          --exec <COMMAND>             Execute command
-          --run <SCRIPT.rb>            Run method named 'run' defined in the script
-  
-      -f, --file <PATH.yaml>           Read configuration file
-      -C, --run-class                  Class name for --run (default: ::Run)
-      -t, --timeout <SEC>              Time for another worker to take over a task when this worker goes down (default: 600)
-      -b, --heartbeat-interval <SEC>   Threshold time to extend the timeout (heartbeat interval) (default: timeout * 3/4)
-      -x, --kill-timeout <SEC>         Threshold time to kill a task process (default: timeout * 10)
-      -X, --kill-interval <SEC>        Threshold time to retry killing a task process (default: 60)
-      -i, --poll-interval <SEC>        Polling interval (default: 1)
-      -r, --retry-wait <SEC>           Time to retry a task when it is failed (default: same as timeout)
-      -e, --expire <SEC>               Threshold time to expire a task (default: 345601 (4days))
-  
-          --database <URI>             Use RDBMS for the backend database (e.g.: mysql://user:password@localhost/mydb)
-          --table <NAME>               backend: name of the table (default: perfectqueue)
-          --simpledb <DOMAIN>          Use Amazon SimpleDB for the backend database (e.g.: --simpledb mydomain -k KEY_ID -s SEC_KEY)
-      -k, --key-id <ID>                AWS Access Key ID
-      -s, --secret-key <KEY>           AWS Secret Access Key
-  
-      -w, --worker <NUM>               Number of worker threads (default: 1)
-      -d, --daemon <PIDFILE>           Daemonize (default: foreground)
-          --env K=V                    Set environment variable
-      -o, --log <PATH>                 log file path
-      -v, --verbose                    verbose mode
-
-
-==== --exec
-
-Execute a command when a task is received. The the data column is passed to the stdin and the id column passed to the last argument. The command have to exit with status code 0 when it succeeded.
-
-*Example:*
-
-  #!/usr/bin/env ruby
-  
-  require 'json'
-  js = JSON.load(STDIN.read)
-  puts "received: id=#{ARGV.last} #{js.inspect}"
-  
-  #$ perfectqueue --database sqlite://test.db \
-     --exec ./this_file -- my cmd args
-
-When the kill timeout (-x, --kill-timeout) is elapsed, SIGTERM signal will be sent to the child process. The signal will be repeated every few seconds (-X, --kill-interval).
-
-
-==== --run
-
-This is same as 'exec' except that it creates a instance of a class named 'Run' defined in the file. The class should has 'initialize(task)', 'run' and 'kill' methods. You can get data column and id column of the task from the argument of the initialize method. It is assumed it succeeded if the method doesn't raise any errors.
-
-*Example:*
-
-  require 'json'
-  
-  class Run
-    def initialize(task)
-      @task = task
-    end
-  
-    def run
-      js = JSON.load(@task.data)
-      puts "received: id=#{@task.id} #{js.inspect}"
-    end
-  
-    def kill
-      puts "kill!"
-    end
-  end
-  
-  #$ perfectqueue --database sqlite://test.db \
-     --run ./this_file.rb -- my cmd args
-
-When the kill timeout (-x, --klill-timeout) is elapsed, Run#kill method will be called (if it is defined). It will be repeated every few seconds (-X, --kill-retry).
-
-
-==== --configure
-
-Write configuration file and exit. Written configuration file can be used with -f option:
-
-*Example:*
-
-  ## create myqueue.yaml file
-  $ perfectqueue --database mysql://root:my@localhost/mydb \
-    --run myrun.rb -- my cmd args \
-    --configure myqueue.yaml
-
-  ## run perfectqueue using the configuration file
-  $ perfectqueue -f myqueue.yaml
-
-
-==== --list
-
-Show queued tasks.
-
-*Example:*
-
-  $ perfectqueue --database sqlite://test.db --list
-                          id                 created_at                    timeout  data
-                       task1  2011-08-23 23:07:45 +0900  2011-08-23 23:07:45 +0900  {"attr1":"val1","attr":"val2"}
-  1 entries.
-