crono_trigger 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -2
- data/README.md +40 -0
- data/Rakefile +17 -0
- data/crono_trigger.gemspec +4 -1
- data/exe/crono_trigger-web +33 -0
- data/lib/crono_trigger.rb +20 -2
- data/lib/crono_trigger/cli.rb +8 -3
- data/lib/crono_trigger/models/signal.rb +52 -0
- data/lib/crono_trigger/models/worker.rb +16 -0
- data/lib/crono_trigger/polling_thread.rb +58 -20
- data/lib/crono_trigger/railtie.rb +15 -0
- data/lib/crono_trigger/schedulable.rb +69 -17
- data/lib/crono_trigger/version.rb +1 -1
- data/lib/crono_trigger/web.rb +163 -0
- data/lib/crono_trigger/worker.rb +118 -8
- data/lib/generators/crono_trigger/install/install_generator.rb +16 -0
- data/lib/generators/crono_trigger/install/templates/install.rb +23 -0
- data/lib/generators/crono_trigger/migration/templates/create_table_migration.rb +1 -0
- data/lib/generators/crono_trigger/migration/templates/migration.rb +1 -0
- data/web/app/.gitignore +21 -0
- data/web/app/README.md +2448 -0
- data/web/app/images.d.ts +3 -0
- data/web/app/package-lock.json +12439 -0
- data/web/app/package.json +36 -0
- data/web/app/public/favicon.ico +0 -0
- data/web/app/public/index.html +45 -0
- data/web/app/public/manifest.json +8 -0
- data/web/app/src/App.css +5 -0
- data/web/app/src/App.test.tsx +9 -0
- data/web/app/src/App.tsx +91 -0
- data/web/app/src/Models.tsx +61 -0
- data/web/app/src/SchedulableRecord.tsx +208 -0
- data/web/app/src/SchedulableRecordTableCell.tsx +19 -0
- data/web/app/src/SchedulableRecords.tsx +110 -0
- data/web/app/src/Signal.tsx +21 -0
- data/web/app/src/Signals.tsx +74 -0
- data/web/app/src/Worker.tsx +106 -0
- data/web/app/src/Workers.tsx +78 -0
- data/web/app/src/index.css +5 -0
- data/web/app/src/index.tsx +15 -0
- data/web/app/src/interfaces.ts +77 -0
- data/web/app/tsconfig.json +30 -0
- data/web/app/tsconfig.prod.json +3 -0
- data/web/app/tsconfig.test.json +6 -0
- data/web/app/tslint.json +13 -0
- data/web/public/asset-manifest.json +6 -0
- data/web/public/favicon.ico +0 -0
- data/web/public/manifest.json +8 -0
- data/web/public/service-worker.js +1 -0
- data/web/public/static/css/main.0f826673.css +2 -0
- data/web/public/static/css/main.0f826673.css.map +1 -0
- data/web/public/static/js/main.1413dc51.js +2 -0
- data/web/public/static/js/main.1413dc51.js.map +1 -0
- data/web/views/index.erb +1 -0
- data/web/views/signals.erb +9 -0
- data/web/views/workers.erb +9 -0
- metadata +89 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 035751ab4333491a769367b5d28fbace76bcb70806400cc10f6338c89b65e394
|
4
|
+
data.tar.gz: '0917037bd6064a166b2ff0a95d7ffa094e44a7ed54c8b08f120e593df41660bc'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 27bd18f70a271450c1c316d3cdf10b837f1b0322026a72788373633be3743d20719654e40e2dba528f2b2f178deee893930a6a208984709bafcf3e40d9bb7ae7
|
7
|
+
data.tar.gz: 142f7e3a5fc634c81142bb254dae96e20a20d6efb6fb6a425ca01f7d77af2a89f43073296ce901127c4fef6fab9a4c5b6b679fd8033b0d019c3a79c8e490ad82
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -25,6 +25,28 @@ Or install it yourself as:
|
|
25
25
|
|
26
26
|
$ gem install crono_trigger
|
27
27
|
|
28
|
+
## Update from v0.3.x
|
29
|
+
|
30
|
+
### Create crono_trigger system tables
|
31
|
+
```
|
32
|
+
$ rails g crono_trigger:install # => create migrations
|
33
|
+
$ rake db:migrate
|
34
|
+
```
|
35
|
+
|
36
|
+
### Add `locked_by:string` column to CronoTrigger::Schedulable model
|
37
|
+
```
|
38
|
+
$ rails g migration add_locked_by_column_to_your_model
|
39
|
+
$ rake db:migrate
|
40
|
+
```
|
41
|
+
|
42
|
+
```ruby
|
43
|
+
class AddLockedByColumnToYourModel < ActiveRecord::Migration[5.2]
|
44
|
+
def change
|
45
|
+
add_column :your_models, :locked_by, :string
|
46
|
+
end
|
47
|
+
end
|
48
|
+
```
|
49
|
+
|
28
50
|
## Usage
|
29
51
|
|
30
52
|
#### Execute `crono_trigger:model` generator.
|
@@ -154,6 +176,24 @@ Usage: crono_trigger [options] MODEL [MODEL..]
|
|
154
176
|
You can rename some columns.
|
155
177
|
ex. `crono_trigger_options[:next_execute_at_column_name] = "next_time"`
|
156
178
|
|
179
|
+
## Admin Web
|
180
|
+
|
181
|
+

|
182
|
+
|
183
|
+
### Standalone mode
|
184
|
+
|
185
|
+
```
|
186
|
+
$ crono_trigger-web --rails
|
187
|
+
```
|
188
|
+
|
189
|
+
### Mount as Rack app
|
190
|
+
|
191
|
+
```ruby
|
192
|
+
# config/routes.rb
|
193
|
+
require "crono_trigger/web"
|
194
|
+
mount CronoTrigger::Web => '/crono_trigger'
|
195
|
+
```
|
196
|
+
|
157
197
|
## Rollbar integration
|
158
198
|
This gem has rollbar plugin.
|
159
199
|
If `crono_trigger/rollbar` is required, Add Rollbar logging process to `CronoTrigger.config.error_handlers`
|
data/Rakefile
CHANGED
@@ -9,6 +9,23 @@ pwd = File.expand_path('../', __FILE__)
|
|
9
9
|
|
10
10
|
gemfiles = Dir.glob(File.join(pwd, "gemfiles", "*.gemfile")).map { |f| File.basename(f, ".*") }
|
11
11
|
|
12
|
+
namespace :js do
|
13
|
+
task :clean do
|
14
|
+
rm_r(File.join(pwd, "web", "app", "build")) if File.exist?(File.join(pwd, "web", "app", "build"))
|
15
|
+
rm_r(File.join(pwd, "web", "public"))
|
16
|
+
end
|
17
|
+
|
18
|
+
task build: [:clean] do
|
19
|
+
Dir.chdir(File.join(pwd, "web", "app"))
|
20
|
+
sh({"PUBLIC_URL" => "<%= URI.parse(url('/')).path.chop %>"}, "npm run build") do |ok, res|
|
21
|
+
raise "failed to build JS" unless ok
|
22
|
+
|
23
|
+
mv(File.join(pwd, "web", "app", "build"), File.join(pwd, "web", "public"))
|
24
|
+
mv(File.join(pwd, "web", "public", "index.html"), File.join(pwd, "web", "views", "index.erb"))
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
12
29
|
namespace :spec do
|
13
30
|
gemfiles.each do |gemfile|
|
14
31
|
desc "Run Tests by #{gemfile}.gemfile"
|
data/crono_trigger.gemspec
CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
|
|
15
15
|
spec.license = "MIT"
|
16
16
|
|
17
17
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
-
f.match(%r{^(test|spec|features)/})
|
18
|
+
f.match(%r{^(test|spec|features|screenshots)/})
|
19
19
|
end
|
20
20
|
spec.bindir = "exe"
|
21
21
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
@@ -26,6 +26,9 @@ Gem::Specification.new do |spec|
|
|
26
26
|
spec.add_dependency "serverengine"
|
27
27
|
spec.add_dependency "concurrent-ruby"
|
28
28
|
spec.add_dependency "tzinfo"
|
29
|
+
spec.add_dependency "sinatra"
|
30
|
+
spec.add_dependency "rack-contrib"
|
31
|
+
spec.add_dependency "oj"
|
29
32
|
spec.add_dependency "activerecord", ">= 4.2"
|
30
33
|
|
31
34
|
spec.add_development_dependency "sqlite3"
|
@@ -0,0 +1,33 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "optparse"
|
4
|
+
require "crono_trigger/web"
|
5
|
+
|
6
|
+
options = {
|
7
|
+
bind: "0.0.0.0",
|
8
|
+
port: 9292
|
9
|
+
}
|
10
|
+
|
11
|
+
opt_parser = OptionParser.new do |o|
|
12
|
+
o.on('-p', '--port=PORT', 'Set port number (Default: 9292)') { |port| options[:port] = port.to_i }
|
13
|
+
o.on('-b', '--bind=ADRESS', 'Set address to bind (Default: 0.0.0.0)') { |addr| options[:bind] = addr }
|
14
|
+
o.on('-s', '--server=SERVER', 'Set Rack server handler name') { |handler| options[:server] = handler }
|
15
|
+
o.on('--rails', 'Require rails environment') do
|
16
|
+
pwd = Dir.pwd
|
17
|
+
loader = -> do
|
18
|
+
begin
|
19
|
+
require File.join(pwd, "config", "environment")
|
20
|
+
Rails.application.eager_load!
|
21
|
+
rescue LoadError
|
22
|
+
raise if pwd == "/"
|
23
|
+
pwd = File.expand_path("..", pwd)
|
24
|
+
loader.call
|
25
|
+
end
|
26
|
+
end
|
27
|
+
loader.call
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
opt_parser.parse!
|
32
|
+
|
33
|
+
CronoTrigger::Web.run!(options)
|
data/lib/crono_trigger.rb
CHANGED
@@ -1,18 +1,22 @@
|
|
1
1
|
require "crono_trigger/version"
|
2
2
|
|
3
3
|
require "ostruct"
|
4
|
+
require "socket"
|
4
5
|
require "active_record"
|
5
6
|
require "concurrent"
|
7
|
+
require "crono_trigger/models/worker"
|
8
|
+
require "crono_trigger/models/signal"
|
6
9
|
require "crono_trigger/worker"
|
7
10
|
require "crono_trigger/polling_thread"
|
8
11
|
require "crono_trigger/schedulable"
|
9
12
|
|
10
13
|
module CronoTrigger
|
11
14
|
@config = OpenStruct.new(
|
12
|
-
|
15
|
+
worker_id: Socket.ip_address_list.detect { |info| !info.ipv4_loopback? && !info.ipv6_loopback? }.ip_address,
|
16
|
+
polling_thread: nil,
|
13
17
|
polling_interval: 5,
|
14
18
|
executor_thread: 25,
|
15
|
-
model_names:
|
19
|
+
model_names: nil,
|
16
20
|
error_handlers: [],
|
17
21
|
)
|
18
22
|
|
@@ -24,12 +28,26 @@ module CronoTrigger
|
|
24
28
|
yield config
|
25
29
|
end
|
26
30
|
|
31
|
+
def self.reloader
|
32
|
+
@reloader
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.reloader=(reloader)
|
36
|
+
@reloader = reloader
|
37
|
+
end
|
38
|
+
|
39
|
+
self.reloader = proc { |&block| block.call }
|
40
|
+
|
27
41
|
def self.load_config(yml, environment = nil)
|
28
42
|
config = YAML.load_file(yml)[environment || "default"]
|
29
43
|
config.each do |k, v|
|
30
44
|
@config[k] = v
|
31
45
|
end
|
32
46
|
end
|
47
|
+
|
48
|
+
def self.workers
|
49
|
+
CronoTrigger::Models::Worker.alive_workers
|
50
|
+
end
|
33
51
|
end
|
34
52
|
|
35
53
|
if defined?(Rails)
|
data/lib/crono_trigger/cli.rb
CHANGED
@@ -8,7 +8,11 @@ options = {
|
|
8
8
|
}
|
9
9
|
|
10
10
|
opt_parser = OptionParser.new do |opts|
|
11
|
-
opts.banner = "Usage: crono_trigger [options]
|
11
|
+
opts.banner = "Usage: crono_trigger [options] [MODEL..]\n If MODEL is not given, Search classes including CronoTrigger::Schedulable module automatically."
|
12
|
+
|
13
|
+
opts.on("-w", "--worker-id=ID", "Worker ID (default: First local ip address which is not loopback") do |id|
|
14
|
+
options[:worker_id] = id
|
15
|
+
end
|
12
16
|
|
13
17
|
opts.on("-f", "--config-file=CONFIG", "Config file (ex. ./crono_trigger.rb)") do |cfg|
|
14
18
|
options[:config] = cfg
|
@@ -18,7 +22,7 @@ opt_parser = OptionParser.new do |opts|
|
|
18
22
|
options[:env] = env
|
19
23
|
end
|
20
24
|
|
21
|
-
opts.on("-p", "--polling-thread=SIZE", Integer, "Polling thread size (Default:
|
25
|
+
opts.on("-p", "--polling-thread=SIZE", Integer, "Polling thread size (Default: Min of (target model count or processor_count)") do |i|
|
22
26
|
options[:polling_thread] = i
|
23
27
|
end
|
24
28
|
|
@@ -56,6 +60,7 @@ opt_parser.parse!
|
|
56
60
|
|
57
61
|
begin
|
58
62
|
require "rails"
|
63
|
+
require "crono_trigger/railtie"
|
59
64
|
require File.expand_path("./config/environment", Rails.root)
|
60
65
|
rescue LoadError
|
61
66
|
end
|
@@ -66,7 +71,7 @@ CronoTrigger.load_config(options[:config], options[:env]) if options[:config]
|
|
66
71
|
CronoTrigger.config[name] = options[name] if options[name]
|
67
72
|
end
|
68
73
|
|
69
|
-
CronoTrigger.config.model_names
|
74
|
+
CronoTrigger.config.model_names = ARGV
|
70
75
|
|
71
76
|
se = ServerEngine.create(nil, CronoTrigger::Worker, {
|
72
77
|
daemonize: options[:daemonize],
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module CronoTrigger
|
2
|
+
module Models
|
3
|
+
class Signal < ActiveRecord::Base
|
4
|
+
self.table_name = "crono_trigger_signals"
|
5
|
+
|
6
|
+
IGNORE_THRESHOLD = 300
|
7
|
+
|
8
|
+
enum signal: {TERM: "TERM", USR1: "USR1", CONT: "CONT", TSTP: "TSTP"}
|
9
|
+
|
10
|
+
scope :sent_to_me, proc {
|
11
|
+
raise "Must set worker_id" unless CronoTrigger.config.worker_id
|
12
|
+
|
13
|
+
where(arel_table[:sent_at].gteq(Time.current - IGNORE_THRESHOLD))
|
14
|
+
.where(worker_id: CronoTrigger.config.worker_id)
|
15
|
+
.where(received_at: nil)
|
16
|
+
.order(:sent_at)
|
17
|
+
}
|
18
|
+
|
19
|
+
class << self
|
20
|
+
def send_signal(signal, worker_id)
|
21
|
+
create!(signal: signal, worker_id: worker_id, sent_at: Time.current)
|
22
|
+
end
|
23
|
+
|
24
|
+
def send_term(worker_id)
|
25
|
+
send_signal("TERM", worker_id)
|
26
|
+
end
|
27
|
+
|
28
|
+
def send_usr1(worker_id)
|
29
|
+
send_signal("USR1", worker_id)
|
30
|
+
end
|
31
|
+
|
32
|
+
def send_cont(worker_id)
|
33
|
+
send_signal("CONT", worker_id)
|
34
|
+
end
|
35
|
+
|
36
|
+
def send_tstp(worker_id)
|
37
|
+
send_signal("TSTP", worker_id)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def kill_me(to_supervisor: true)
|
42
|
+
if update(received_at: Time.current)
|
43
|
+
if to_supervisor && Process.ppid != 1
|
44
|
+
Process.kill(signal, Process.ppid)
|
45
|
+
else
|
46
|
+
Process.kill(signal, Process.pid)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require "crono_trigger/worker"
|
2
|
+
|
3
|
+
module CronoTrigger
|
4
|
+
module Models
|
5
|
+
class Worker < ActiveRecord::Base
|
6
|
+
self.table_name = "crono_trigger_workers"
|
7
|
+
|
8
|
+
ALIVE_THRESHOLD = CronoTrigger::Worker::HEARTBEAT_INTERVAL * 5
|
9
|
+
|
10
|
+
enum executor_status: {running: "running", quiet: "quiet", shuttingdown: "shuttingdown", shutdown: "shutdown"}
|
11
|
+
serialize :polling_model_names, JSON
|
12
|
+
|
13
|
+
scope :alive_workers, proc { where(arel_table[:last_heartbeated_at].gteq(Time.current - ALIVE_THRESHOLD)) }
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -1,25 +1,32 @@
|
|
1
1
|
module CronoTrigger
|
2
2
|
class PollingThread
|
3
|
-
def initialize(model_queue, stop_flag, logger, executor)
|
3
|
+
def initialize(model_queue, stop_flag, logger, executor, execution_counter)
|
4
4
|
@model_queue = model_queue
|
5
5
|
@stop_flag = stop_flag
|
6
6
|
@logger = logger
|
7
7
|
@executor = executor
|
8
|
+
@execution_counter = execution_counter
|
9
|
+
@quiet = Concurrent::AtomicBoolean.new(false)
|
8
10
|
end
|
9
11
|
|
10
12
|
def run
|
11
13
|
@thread = Thread.start do
|
12
14
|
@logger.info "(polling-thread-#{Thread.current.object_id}) Start polling thread"
|
13
15
|
until @stop_flag.wait_for_set(CronoTrigger.config.polling_interval)
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
16
|
+
next if quiet?
|
17
|
+
|
18
|
+
CronoTrigger.reloader.call do
|
19
|
+
begin
|
20
|
+
model_name = @model_queue.pop(true)
|
21
|
+
model = model_name.classify.constantize
|
22
|
+
poll(model)
|
23
|
+
rescue ThreadError => e
|
24
|
+
@logger.error(e) unless e.message == "queue empty"
|
25
|
+
rescue => e
|
26
|
+
@logger.error(e)
|
27
|
+
ensure
|
28
|
+
@model_queue << model_name if model_name
|
29
|
+
end
|
23
30
|
end
|
24
31
|
end
|
25
32
|
end
|
@@ -29,29 +36,60 @@ module CronoTrigger
|
|
29
36
|
@thread.join
|
30
37
|
end
|
31
38
|
|
39
|
+
def quiet
|
40
|
+
@quiet.make_true
|
41
|
+
end
|
42
|
+
|
43
|
+
def quiet?
|
44
|
+
@quiet.true?
|
45
|
+
end
|
46
|
+
|
47
|
+
def alive?
|
48
|
+
@thread.alive?
|
49
|
+
end
|
50
|
+
|
32
51
|
def poll(model)
|
33
52
|
@logger.debug "(polling-thread-#{Thread.current.object_id}) Poll #{model}"
|
34
53
|
records = []
|
35
|
-
|
54
|
+
overflowed_record_ids = []
|
55
|
+
|
36
56
|
begin
|
37
57
|
model.connection_pool.with_connection do
|
38
|
-
records = model.executables_with_lock
|
39
|
-
primary_key_offset = records.last && records.last.id
|
58
|
+
records = model.executables_with_lock
|
40
59
|
end
|
41
60
|
|
42
61
|
records.each do |record|
|
43
|
-
|
44
|
-
|
45
|
-
@
|
62
|
+
begin
|
63
|
+
@executor.post do
|
64
|
+
@execution_counter.increment
|
46
65
|
begin
|
47
|
-
|
48
|
-
|
49
|
-
|
66
|
+
model.connection_pool.with_connection do
|
67
|
+
@logger.info "(executor-thread-#{Thread.current.object_id}) Execute #{record.class}-#{record.id}"
|
68
|
+
begin
|
69
|
+
record.do_execute
|
70
|
+
rescue Exception => e
|
71
|
+
@logger.error(e)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
ensure
|
75
|
+
@execution_counter.decrement
|
50
76
|
end
|
51
77
|
end
|
78
|
+
rescue Concurrent::RejectedExecutionError
|
79
|
+
overflowed_record_ids << record.id
|
52
80
|
end
|
53
81
|
end
|
54
|
-
|
82
|
+
unlock_overflowed_records(model, overflowed_record_ids)
|
83
|
+
end while overflowed_record_ids.empty? && records.any?
|
84
|
+
end
|
85
|
+
|
86
|
+
private def unlock_overflowed_records(model, overflowed_record_ids)
|
87
|
+
model.connection_pool.with_connection do
|
88
|
+
model.where(id: overflowed_record_ids).crono_trigger_unlock_all!
|
89
|
+
end
|
90
|
+
rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::LockWaitTimeout, ActiveRecord::StatementTimeout, ActiveRecord::Deadlocked
|
91
|
+
sleep 1
|
92
|
+
retry
|
55
93
|
end
|
56
94
|
end
|
57
95
|
end
|
@@ -1,4 +1,19 @@
|
|
1
1
|
module CronoTrigger
|
2
2
|
class Railtie < ::Rails::Railtie
|
3
|
+
config.after_initialize do
|
4
|
+
CronoTrigger.reloader = CronoTrigger::Railtie::Reloader.new
|
5
|
+
end
|
6
|
+
|
7
|
+
class Reloader
|
8
|
+
def initialize(app = ::Rails.application)
|
9
|
+
@app = app
|
10
|
+
end
|
11
|
+
|
12
|
+
def call
|
13
|
+
@app.reloader.wrap do
|
14
|
+
yield
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
3
18
|
end
|
4
19
|
end
|