abid 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea0a29c738c86aebaf241507a7b0acab89dc3cf7
-  data.tar.gz: c708d6813dae9bd21cd8e1c2d4396d3a055dc50f
+  metadata.gz: 7cc7876b6fa9c07f29b893a57ffca6222569385c
+  data.tar.gz: fabd1296f57c4ad14b58564882250b71a917fe68
 SHA512:
-  metadata.gz: 845024d893bf17650b189eabb90280840827f3e03910fb61e76c35e0446bceb9755ce8d79c46397aa0245ac26c54ecbc6c78bc98172594a2304bf850e35ff25d
-  data.tar.gz: 3e18c50195b334d8bebb1137ff45417198ae57466a3ef50aad14d630c23467bb423572047e9f1647f16ae3a14e9245fae9764479b4cb6bdf5c43002c2178485d
+  metadata.gz: 1fcf95c39e0b23ee3bb8116d5e227d27ff9439fc83bbcbd2cbff7bd86b0f8a4ec837d430af09b6dd0e71643328367a7daee483157bfdc53cc868dc2050876190
+  data.tar.gz: 57b0276deeaef0a06f3e35aa57674f2cc7d8eb15494e9b14b2fab717b9397e23bd74f460854779beba8afe48e397fc64a0104c414740cf5d6331a59f78340acd

data/README.md

@@ -41,8 +41,8 @@ play :fetch_source do
   param :date, type: :date
 
   def run
-    mkdir 'out'
     open('http://example.com') do |f|
+      FileUtils.makedirs "out/#{date.strftime('%Y-%m-%d')}"
       File.write("out/#{date.strftime('%Y-%m-%d')}/example.com", f.read)
     end
   end
@@ -51,7 +51,7 @@ end
 play :count do
   param :date, type: :date
 
-  def setup
+  setup do
     needs 'fetch_source', date: date
   end
 
@@ -67,11 +67,313 @@ Then you can invoke the task:
 $ bundle exec abid count date=2016-01-01
 ```
 
-This Abidfile has two tasks: `fetch_source` and `count`. They can be treated as a normal rake task, but they have some additional features:
+This Abidfile has two tasks: `fetch_source` and `count`. They are kinds of rake tasks, but they have some additional features:
 
 * A play can take parameters. They are declared with `param` keyword, and passed via environment variables.
 * All play results are saved to the external database. If a play is invoked twice with same parameters, it will be ignored.
-* Depending tasks can be declared in `setup` method. If a depending task is a play task, parameters can be passed.
+* Depending tasks can be declared in `setup` block. If a depending task is a play task, parameters can be specified.
+
+
+## Execution Model
+
+When a play is invoked, its parameters and results are saved in a database by default.
+If the play has been executed with same parameters and successed, it will not be executed any more.
+
+```ruby
+# Abidfile.rb
+play :test do
+  params :name, type: :string
+  def run
+    puts name
+  end
+end
+
+$ abid test name=apple #=> "apple"
+$ abid test name=apple # nothing happens
+$ abid test name=orange #=> "orange"
+```
+
+Normal rake task results are not stored in DB.
+They are always executed even if they have been executed and successed.
+These tasks are called "volatile".
+
+If prerequisites tasks have been failed, the subsequent task also fails.
+When prerequisites failed, you have to manually fix a problem and re-execute them.
+
+```ruby
+# Abidfile.rb
+play :query do
+  def run
+    result = `mysql -e 'SELECT COUNT(*) FROM users'`
+    File.write(result, 'result.txt')
+  end
+end
+play :report do
+  setup { needs :query }
+  def run
+    `cat result.txt | sendmail all@example.com`
+  end
+end
+
+$ abid query   #=> Failed because of MySQL server down
+$ abid report  #=> Fails because prerequisites failed
+$ ...          # restart MySQL server
+$ abid query   #=> ok
+$ abid report  #=> ok
+
+```
+
+### Volatile plays
+
+Abid plays can also be volatile.
+
+```
+play :voaltile_play do
+  set :volatile, true
+  def run
+    ...
+  end
+end
+```
+
+These plays are not stored in DB and will be always executed.
+
+### Repair mode
+
+When abid is executed with `--repair` flag, failed prerequisites are re-executed and successed tasks are executed only when their prerequisites are executed.
+
+```
+$ abid report           #=> Failed because of MySQL server down
+$ ...                   # restart MySQL server
+$ abid --repair report  #=> :query and :report tasks are executed
+```
+
+### Parallel execution
+
+All tasks are executed in a thread pool.
+
+By default, the thread pool size is 1, i.e. all tasks are executed in single thread. When `-m` option is given, the thread pool size is decided from CPU size. You can specify the thread pool size by `-j` option.
+
+Abid supports multiple thread pools.
+Each tasks can be executed in different thread pools.
+
+```ruby
+define_worker :copy, 2
+define_worker :hive, 4
+
+play :copy_source_1 do
+  set :worker, :copy
+  def run
+    ...
+  end
+end
+
+play :hive_query_1 do
+  set :worker, :hive
+  def run
+    ...
+  end
+end
+```
+
+Two thread pools `copy` and `hive` are defined in above example, each thread pool sizes are 2 and 4.
+`:copy_source_1` is executed in `copy` thread pool, and `:hive_query_1` is executed in `hive` thread pool.
+
+## Plays detail
+
+### Params
+
+```ruby
+play :sample do
+  param :name, type: :string
+  param :date, type: :date, default: Date.today - 1
+
+  def run
+    date #=> #<Date: ????-??-?? ((0000000j,0s,0n),+0s,0000000j)>
+  end
+end
+```
+
+Each parameters are initialized by corresponding environment variables. If no environment variable with same name is found, default will be used.
+
+Abid supports following types:
+
+- `boolean`
+- `int`
+- `float`
+- `string`
+- `date`
+- `datetime`
+- `time`
+
+### Settings
+
+```ruby
+play :count do
+  set(:file_path, './sql/count.sql')
+  set(:query) { File.read(file_path) }
+
+  def run
+    query #=> the contents of './sql/count.sql'
+  end
+end
+```
+
+Plays settings can be declared by `set` and referenced in `run` method.
+If block given, it is evaluated in the same context as `run` method.
+The block is called only once and its result is cached.
+
+Following settings are used in abid core:
+- `worker`
+- `volatile`
+
+### Dependencies
+
+```ruby
+play :sample do
+  param :name, type: :string  
+  setup do
+    needs "parent_task:#{name}"
+  end
+
+  def run
+    # executed after `parent_task`
+  end
+end
+```
+
+All prerequisites must be declared in `setup` block.
+You can refer parameters and settings in `setup` block.
+
+### Callbacks
+
+```ruby
+play :sample do
+  def run
+    ...
+  end
+
+  before do
+    # executed before running
+  end
+
+  after do
+    # executed when the task successed
+  end
+
+  around do |body|
+    body.call # `run` method is called
+  ensure
+    ...       # executed even if the task failed
+  end
+end
+```
+
+### Extending plays
+
+You can extend plays in object-oriented style.
+All parameters, settings and methods are inherited.
+
+```ruby
+play :abstract_count do
+  def run
+    `hive -f #{file_path}`
+  end
+end
+
+play :count, extends: :abstract_count do
+  set :file_path, 'sql/count.sql'
+end
+
+---
+
+$ abid count #=> hive -f sql/count.sql
+```
+
+Common base play can be defined by `play_base` keyword:
+
+```
+play_base do
+  param :date, type: :date
+end
+```
+
+All plays inherit the `play_base` definition.
+
+### Plays internal
+
+The play implementation can be illustrated as below:
+
+```ruby
+play :sample do
+  param :date, type: :date
+  set(:file_path, 'sql/count.sql')
+  set(:query) { File.read(file_path) }
+  def run
+    ...
+  end
+end
+
+# <=>
+
+class Sample < Abid::Play
+  attr_reader :date
+  def initialize(date)
+    @date = Date.parse(date)
+  end
+
+  def file_path
+    'sql/count.sql'
+  end
+  def query
+    @query ||= File.read(file_path)
+  end
+
+  def run
+    ...
+  end
+end
+
+task :sample do
+  Sample.new(ENV['date']).run
+end
+```
+
+When play is defined, new subclass of Avid::Play is created and play body is evaluated in that new class context. So, any class goodies can be put in play's body, i.e. including modules, `attr_reader` / `attr_writer`, method definitions, etc..
+
+## Built-in tasks
+
+### `state:list`
+
+```
+$ abid state:list started_after="2000-01-01 00:00:00" started_before="2000=01-02 00:00:00"
+```
+
+Display plays current states.
+
+### `state:revoke`
+
+```
+$ abid state:revoke[id]
+```
+
+Remove the play recored from DB.
+
+### `state:assume`
+
+```
+$ abid state:assume[task_name] date=2000-01-01
+```
+
+Insert a record that the play successed into DB.
+
+### `db:migrate`
+
+```
+$ abid db:migrate
+```
+
+Initialize or update DB.
 
 ## Development
 

data/abid.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "minitest"
+  spec.add_development_dependency "pry-byebug"
 
   spec.add_dependency "rake", "~> 10.0"
   spec.add_dependency "concurrent-ruby-ext"

data/lib/Abidfile.rb

@@ -1,80 +1 @@
-namespace :state do
-  task default: :list
-
-  desc 'Show play histories'
-  play :list do
-    set :volatile, true
-
-    param :started_after, type: :time, default: nil
-    param :started_before, type: :time, default: nil
-
-    def run
-      states = Abid::State.list(
-        started_after: started_after,
-        started_before: started_before
-      )
-
-      table = states.map do |state|
-        params = state[:params].map do |k, v|
-          v.to_s =~ /\s/ ? "#{k}='#{v}'" : "#{k}=#{v}"
-        end.join(' ')
-        [
-          state[:id].to_s,
-          state[:state].to_s,
-          state[:name],
-          params,
-          state[:start_time].to_s,
-          state[:end_time].to_s
-        ]
-      end
-
-      header = %w(id state name params start_time end_time)
-
-      tab_width = header.each_with_index.map do |c, i|
-        [c.length, table.map { |row| row[i].length }.max || 0].max
-      end
-
-      header.each_with_index do |c, i|
-        print c.ljust(tab_width[i] + 2)
-      end
-      puts
-
-      header.each_with_index do |_, i|
-        print '-' * (tab_width[i] + 2)
-      end
-      puts
-
-      table.map do |row|
-        row.each_with_index do |v, i|
-          print v.ljust(tab_width[i] + 2)
-        end
-        puts
-      end
-    end
-  end
-end
-
-namespace :db do
-  desc 'Delete play history'
-  task :revoke, [:task] do |_t, args|
-    task = Rake.application[args[:task]]
-    state = Abid::State.find(task)
-    state.revoke
-  end
-
-  desc 'Run migrations'
-  task :migrate, [:version] do |_t, args|
-    migrations_path = File.expand_path('../../migrations', __FILE__)
-
-    require 'sequel'
-    Sequel.extension :migration
-    db = Rake.application.database
-    if args[:version]
-      puts "Migrating to version #{args[:version]}"
-      Sequel::Migrator.run(db, migrations_path, target: args[:version].to_i)
-    else
-      puts 'Migrating to latest'
-      Sequel::Migrator.run(db, migrations_path)
-    end
-  end
-end
+# allows the built-in tasks to load without a abidfile

data/lib/abid.rb

@@ -1,8 +1,9 @@
 require 'rake'
 require 'date'
 require 'digest/md5'
-require 'english'
+require 'English'
 require 'forwardable'
+require 'monitor'
 require 'time'
 require 'yaml'
 require 'concurrent'
@@ -12,6 +13,7 @@ require 'sqlite3'
 require 'sequel'
 
 require 'abid/rake_extensions'
+require 'abid/concurrent_extention'
 require 'abid/version'
 require 'abid/waiter'
 require 'abid/worker'