abid 0.3.0.pre.alpha.2 → 0.3.0.pre.alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,48 @@
1
+ require 'monitor'
2
+
3
+ module Abid
4
+ class Environment
5
+ def initialize
6
+ @mon = Monitor.new
7
+ end
8
+
9
+ def application
10
+ @application ||= Abid::Application.new(self)
11
+ end
12
+ attr_writer :application
13
+
14
+ def options
15
+ application.options
16
+ end
17
+
18
+ def config
19
+ @mon.synchronize do
20
+ @cofig ||= Config.new
21
+ end
22
+ end
23
+
24
+ def job_manager
25
+ @mon.synchronize do
26
+ @job_manager ||= JobManager.new(self)
27
+ end
28
+ end
29
+
30
+ def process_manager
31
+ @mon.synchronize do
32
+ @process_manager ||= Engine::ProcessManager.new(self)
33
+ end
34
+ end
35
+
36
+ def worker_manager
37
+ @mon.synchronize do
38
+ @worker_manager ||= Engine::WorkerManager.new(self)
39
+ end
40
+ end
41
+
42
+ def db
43
+ @mon.synchronize do
44
+ @db ||= StateManager::Database.new(self)
45
+ end
46
+ end
47
+ end
48
+ end
data/lib/abid/job.rb CHANGED
@@ -1,13 +1,27 @@
1
+ require 'forwardable'
2
+ require 'monitor'
3
+
1
4
  module Abid
2
- # Job instance that is consists of a task name and params.
5
+ # Job is an aggregation object of components around the task.
3
6
  class Job
4
- attr_reader :name, :params
7
+ attr_reader :name, :params, :env
8
+
9
+ class << self
10
+ extend Forwardable
11
+ def_delegators 'Abid.global.job_manager', :[], :find_by_task
12
+ end
5
13
 
6
- # @param name [String] task name
7
- # @param params [Hash] task params
8
- def initialize(name, params)
14
+ # @!visibility private
15
+ def initialize(env, name, params)
16
+ @env = env
9
17
  @name = name
10
18
  @params = params.sort.to_h.freeze
19
+ @mon = Monitor.new
20
+ end
21
+
22
+ def invoke(*args)
23
+ Engine::Scheduler.invoke(self, *args)
24
+ process.wait
11
25
  end
12
26
 
13
27
  def params_str
@@ -18,8 +32,36 @@ module Abid
18
32
  @digest ||= Digest::MD5.hexdigest(name + "\n" + params_str)
19
33
  end
20
34
 
21
- def assume(force: false)
22
- State.assume(self, force: force)
35
+ def task
36
+ @task ||= Abid.application[name, nil, params]
37
+ end
38
+
39
+ def state
40
+ @state ||= StateManager::StateProxy.new(self)
41
+ end
42
+
43
+ def prerequisites
44
+ task.prerequisite_tasks.map do |preq_task|
45
+ env.job_manager.find_by_task(preq_task)
46
+ end
47
+ end
48
+
49
+ def process
50
+ @mon.synchronize do
51
+ @process ||= env.process_manager.create
52
+ end
53
+ end
54
+
55
+ def worker
56
+ env.worker_manager[task.worker]
57
+ end
58
+
59
+ def volatile?
60
+ task.volatile? || env.options.disable_state
61
+ end
62
+
63
+ def dryrun?
64
+ env.options.dryrun || env.options.preview
23
65
  end
24
66
  end
25
67
  end
@@ -0,0 +1,22 @@
1
+ require 'monitor'
2
+
3
+ module Abid
4
+ class JobManager
5
+ def initialize(env)
6
+ @env = env
7
+ @cache = {}
8
+ @mon = Monitor.new
9
+ end
10
+
11
+ def [](name, params = {})
12
+ @mon.synchronize do
13
+ key = [name, params.sort.freeze].freeze
14
+ @cache[key] ||= Job.new(@env, name, params)
15
+ end
16
+ end
17
+
18
+ def find_by_task(task)
19
+ self[task.name, task.params]
20
+ end
21
+ end
22
+ end
@@ -17,12 +17,12 @@ module Abid
17
17
  end
18
18
 
19
19
  def execute(_args = nil)
20
- fail 'mixin is not executable'
20
+ raise 'mixin is not executable'
21
21
  end
22
22
 
23
23
  class <<self
24
24
  def define_mixin(*args, &block) # :nodoc:
25
- Rake.application.define_mixin(self, *args, &block)
25
+ Abid.application.define_mixin(self, *args, &block)
26
26
  end
27
27
  end
28
28
  end
@@ -1,6 +1,14 @@
1
- module Abid
2
- module RakeExtensions
3
- require 'abid/rake_extensions/task'
4
- Rake::Task.include Task
1
+ require 'abid/rake_extensions/task'
2
+
3
+ # Delegate Rake.application to Abid.global.application
4
+ class << Rake
5
+ def application
6
+ Abid.global.application
7
+ end
8
+
9
+ def application=(app)
10
+ Abid.global.application = app
5
11
  end
6
12
  end
13
+
14
+ Rake::Task.include Abid::RakeExtensions::Task
@@ -9,18 +9,12 @@ module Abid
9
9
  :default
10
10
  end
11
11
 
12
- def session
13
- @session ||= Session.new(self).tap do |session|
14
- session.add_observer do |_, _, reason|
15
- if session.successed? || session.failed?
16
- call_hooks(:after_invoke, reason)
17
- end
18
- end
19
- end
12
+ def job
13
+ Job[name, defined?(params) ? params : {}]
20
14
  end
21
15
 
22
- def state
23
- State.find(self)
16
+ def params
17
+ {}
24
18
  end
25
19
 
26
20
  def name_with_params
@@ -42,113 +36,6 @@ module Abid
42
36
  def call_hooks(tag, *args)
43
37
  hooks[tag].each { |h| h.call(*args) }
44
38
  end
45
-
46
- def async_invoke(*args)
47
- task_args = Rake::TaskArguments.new(arg_names, args)
48
- async_invoke_with_call_chain(task_args, Rake::InvocationChain::EMPTY)
49
- end
50
-
51
- def async_invoke_with_call_chain(task_args, invocation_chain)
52
- session.enter do
53
- session.capture_exception do
54
- new_chain = Rake::InvocationChain.append(self, invocation_chain)
55
-
56
- unless concerned?
57
- session.skip
58
- break
59
- end
60
-
61
- call_hooks(:before_invoke)
62
-
63
- async_invoke_prerequisites(task_args, new_chain)
64
-
65
- async_execute_after_prerequisites(task_args)
66
- end
67
- end
68
- end
69
-
70
- def async_invoke_prerequisites(task_args, invocation_chain)
71
- prerequisite_tasks.each do |t|
72
- args = task_args.new_scope(t.arg_names)
73
- t.async_invoke_with_call_chain(args, invocation_chain)
74
- end
75
- end
76
-
77
- def async_execute_after_prerequisites(task_args)
78
- if prerequisite_tasks.empty?
79
- async_execute(task_args)
80
- else
81
- counter = Concurrent::DependencyCounter.new(prerequisite_tasks.size) do
82
- session.capture_exception do
83
- async_execute(task_args)
84
- end
85
- end
86
- prerequisite_tasks.each { |t| t.session.add_observer counter }
87
- end
88
- end
89
-
90
- def async_execute(task_args)
91
- failed_task = prerequisite_tasks.find do |t|
92
- t.session.failed? || t.session.canceled?
93
- end
94
- if failed_task
95
- session.cancel(failed_task.session.error)
96
- return
97
- end
98
-
99
- async_post(worker) do
100
- if !needed?
101
- session.skip
102
- elsif session.lock
103
- call_hooks(:before_execute)
104
-
105
- execute(task_args)
106
-
107
- session.success
108
- else
109
- async_wait_external
110
- end
111
- end
112
- end
113
-
114
- def async_wait_external
115
- unless application.options.wait_external_task
116
- fail "task #{name_with_params} already running"
117
- end
118
-
119
- application.trace "** Wait #{name_with_params}" if application.options.trace
120
-
121
- async_post(:waiter) do
122
- interval = application.options.wait_external_task_interval || 10
123
- timeout = application.options.wait_external_task_timeout || 3600
124
- timeout_tm = Time.now.to_f + timeout
125
-
126
- loop do
127
- state.reload
128
- if !state.running?
129
- session.success
130
- break
131
- elsif Time.now.to_f >= timeout_tm
132
- fail "#{name} -- timeout exceeded"
133
- else
134
- sleep interval
135
- end
136
- end
137
- end
138
- end
139
-
140
- def async_post(worker_name, &block)
141
- application.worker[worker_name].post do
142
- session.capture_exception do
143
- begin
144
- block.call
145
- finished = true
146
- ensure
147
- fail 'thread killed' if $ERROR_INFO.nil? && !finished
148
- end
149
- end
150
- end
151
- end
152
39
  end
153
40
  end
154
41
  end
@@ -1,35 +1,39 @@
1
1
  require 'sequel/plugins/serialization'
2
+ require 'abid/state_manager/state'
3
+ require 'abid/state_manager/state_proxy'
2
4
 
3
5
  module Abid
4
6
  module StateManager
5
- module Database
7
+ class Database
6
8
  Sequel.extension :migration
7
9
 
8
- # Create a new database object.
9
- #
10
- # Abid.config['database'] is used Sequel.connect params.
10
+ MIGRATIONS_PATH = File.expand_path('../../../../migrations', __FILE__)
11
+
12
+ # Creates a new database object and checks schema version.
11
13
  #
12
14
  # @return [Sequel::Database] database object
13
- def self.connect
14
- db = connect!
15
- Sequel::Migrator.check_current(db, migrations_path)
15
+ # @see Sequel.connect
16
+ def self.connect(*args)
17
+ db = Sequel.connect(*args)
18
+ Sequel::Migrator.check_current(db, MIGRATIONS_PATH)
16
19
  db
17
20
  rescue Sequel::Migrator::NotCurrentError
18
21
  raise Error, 'current schema is out of date'
19
22
  end
20
23
 
21
- # Connect to database without schema version check
22
- #
23
- # @return [Sequel::Database] database object
24
- def self.connect!
25
- # symbolize keys
26
- params = {}
27
- Abid.config.database.each { |k, v| params[k.to_sym] = v }
28
- Sequel.connect(**params)
24
+ def initialize(env)
25
+ @env = env
26
+ @mon = Monitor.new
27
+ end
28
+
29
+ def connection
30
+ @connection ||= self.class.connect(@env.config.database)
29
31
  end
30
32
 
31
- def self.migrations_path
32
- File.expand_path('../../../../migrations', __FILE__)
33
+ def states
34
+ @mon.synchronize do
35
+ @states ||= Class.new(State.Model(connection[:states]))
36
+ end
33
37
  end
34
38
  end
35
39
  end
@@ -1,7 +1,11 @@
1
1
  module Abid
2
2
  module StateManager
3
+ State = Class.new(Sequel::Model)
4
+
3
5
  # O/R Mapper for `states` table.
4
- class State < Sequel::Model(StateManager.database)
6
+ #
7
+ # Use #init_by_job to initialize a state object.
8
+ class State
5
9
  RUNNING = 1
6
10
  SUCCESSED = 2
7
11
  FAILED = 3
@@ -42,9 +46,55 @@ module Abid
42
46
  ).first
43
47
  end
44
48
 
45
- def self.find_or_initialize_by_job(job)
46
- find_by_job(job) || \
47
- new(name: job.name, params: job.params_str, digest: job.digest)
49
+ # Initialize a state by a job.
50
+ #
51
+ # @param job [Job] job
52
+ # @return [State] state object
53
+ def self.init_by_job(job)
54
+ new(
55
+ name: job.name,
56
+ params: job.params_str,
57
+ digest: job.digest
58
+ )
59
+ end
60
+
61
+ # Find or initialize a state by a job.
62
+ #
63
+ # @param job [Job] job
64
+ # @return [State] state object
65
+ def self.find_or_init_by_job(job)
66
+ find_by_job(job) || init_by_job(job)
67
+ end
68
+
69
+ # Update the state to RUNNING.
70
+ #
71
+ # @param job [Job] job
72
+ def self.start(job)
73
+ db.transaction do
74
+ state = find_or_init_by_job(job)
75
+ state.check_running!
76
+ state.state = RUNNING
77
+ state.start_time = Time.now
78
+ state.end_time = nil
79
+ state.save
80
+ end
81
+ end
82
+
83
+ # Update the state to SUCCESSED or FAILED.
84
+ #
85
+ # If error is given, the state will be FAILED.
86
+ #
87
+ # @param job [Job] job
88
+ # @param error [Error] error object
89
+ def self.finish(job, error = nil)
90
+ db.transaction do
91
+ state = find_or_init_by_job(job)
92
+ return unless state.running?
93
+
94
+ state.state = error ? FAILED : SUCCESSED
95
+ state.end_time = Time.now
96
+ state.save
97
+ end
48
98
  end
49
99
 
50
100
  # Assume the job to be successed
@@ -54,11 +104,10 @@ module Abid
54
104
  #
55
105
  # @param job [Job] job
56
106
  # @param force [Boolean] force update the state
57
- # @return [State] state object
107
+ # @return [void]
58
108
  def self.assume(job, force: false)
59
- StateManager.database.transaction do
60
- state = find_or_initialize_by_job(job)
61
-
109
+ db.transaction do
110
+ state = find_or_init_by_job(job)
62
111
  return state if state.successed?
63
112
  state.check_running! unless force
64
113
 
@@ -72,15 +121,16 @@ module Abid
72
121
 
73
122
  # Delete the state.
74
123
  #
124
+ # @param state_id [Integer] State ID
75
125
  # @param force [Boolean] If true, delete the state even if running
76
126
  # @return [void]
77
- def revoke(force: false)
78
- StateManager.database.transaction do
79
- unless force
80
- refresh
81
- check_running!
82
- end
83
- delete
127
+ def self.revoke(state_id, force: false)
128
+ db.transaction do
129
+ state = self[state_id]
130
+ return false if state.nil?
131
+ state.check_running! unless force
132
+ state.delete
133
+ true
84
134
  end
85
135
  end
86
136