crono_trigger 0.3.2 → 0.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -2
  3. data/README.md +40 -0
  4. data/Rakefile +17 -0
  5. data/crono_trigger.gemspec +4 -1
  6. data/exe/crono_trigger-web +33 -0
  7. data/lib/crono_trigger.rb +20 -2
  8. data/lib/crono_trigger/cli.rb +8 -3
  9. data/lib/crono_trigger/models/signal.rb +52 -0
  10. data/lib/crono_trigger/models/worker.rb +16 -0
  11. data/lib/crono_trigger/polling_thread.rb +58 -20
  12. data/lib/crono_trigger/railtie.rb +15 -0
  13. data/lib/crono_trigger/schedulable.rb +69 -17
  14. data/lib/crono_trigger/version.rb +1 -1
  15. data/lib/crono_trigger/web.rb +163 -0
  16. data/lib/crono_trigger/worker.rb +118 -8
  17. data/lib/generators/crono_trigger/install/install_generator.rb +16 -0
  18. data/lib/generators/crono_trigger/install/templates/install.rb +23 -0
  19. data/lib/generators/crono_trigger/migration/templates/create_table_migration.rb +1 -0
  20. data/lib/generators/crono_trigger/migration/templates/migration.rb +1 -0
  21. data/web/app/.gitignore +21 -0
  22. data/web/app/README.md +2448 -0
  23. data/web/app/images.d.ts +3 -0
  24. data/web/app/package-lock.json +12439 -0
  25. data/web/app/package.json +36 -0
  26. data/web/app/public/favicon.ico +0 -0
  27. data/web/app/public/index.html +45 -0
  28. data/web/app/public/manifest.json +8 -0
  29. data/web/app/src/App.css +5 -0
  30. data/web/app/src/App.test.tsx +9 -0
  31. data/web/app/src/App.tsx +91 -0
  32. data/web/app/src/Models.tsx +61 -0
  33. data/web/app/src/SchedulableRecord.tsx +208 -0
  34. data/web/app/src/SchedulableRecordTableCell.tsx +19 -0
  35. data/web/app/src/SchedulableRecords.tsx +110 -0
  36. data/web/app/src/Signal.tsx +21 -0
  37. data/web/app/src/Signals.tsx +74 -0
  38. data/web/app/src/Worker.tsx +106 -0
  39. data/web/app/src/Workers.tsx +78 -0
  40. data/web/app/src/index.css +5 -0
  41. data/web/app/src/index.tsx +15 -0
  42. data/web/app/src/interfaces.ts +77 -0
  43. data/web/app/tsconfig.json +30 -0
  44. data/web/app/tsconfig.prod.json +3 -0
  45. data/web/app/tsconfig.test.json +6 -0
  46. data/web/app/tslint.json +13 -0
  47. data/web/public/asset-manifest.json +6 -0
  48. data/web/public/favicon.ico +0 -0
  49. data/web/public/manifest.json +8 -0
  50. data/web/public/service-worker.js +1 -0
  51. data/web/public/static/css/main.0f826673.css +2 -0
  52. data/web/public/static/css/main.0f826673.css.map +1 -0
  53. data/web/public/static/js/main.1413dc51.js +2 -0
  54. data/web/public/static/js/main.1413dc51.js.map +1 -0
  55. data/web/views/index.erb +1 -0
  56. data/web/views/signals.erb +9 -0
  57. data/web/views/workers.erb +9 -0
  58. metadata +89 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8d12624dcc2f27a3f0bc7c83544ae735490c8d4294587fe6fce6b16661d934c1
4
- data.tar.gz: 931bd6e1dcba7a0a77f644235d74f762f3587e65b17a36a1bb9bf962b8e27a9e
3
+ metadata.gz: 035751ab4333491a769367b5d28fbace76bcb70806400cc10f6338c89b65e394
4
+ data.tar.gz: '0917037bd6064a166b2ff0a95d7ffa094e44a7ed54c8b08f120e593df41660bc'
5
5
  SHA512:
6
- metadata.gz: aee050331778f7db89628d332efc7c4cfe7965dc2a8b19f6bd1eaa678045f5fe5b64042ba3b7511bee542f9b310debdd65728694e2c4f07b94bd1c362a80370a
7
- data.tar.gz: ef25f44ecdc3afc9dab964ffa9d82ff455d02fe26d96614951df75b764a9067cbc177083bbbb4f39071e6d3208a99cd924613b04b26866d09506eb56f8360d9e
6
+ metadata.gz: 27bd18f70a271450c1c316d3cdf10b837f1b0322026a72788373633be3743d20719654e40e2dba528f2b2f178deee893930a6a208984709bafcf3e40d9bb7ae7
7
+ data.tar.gz: 142f7e3a5fc634c81142bb254dae96e20a20d6efb6fb6a425ca01f7d77af2a89f43073296ce901127c4fef6fab9a4c5b6b679fd8033b0d019c3a79c8e490ad82
@@ -2,10 +2,9 @@ sudo: false
2
2
  language: ruby
3
3
  cache: bundler
4
4
  rvm:
5
- - 2.5.0
5
+ - 2.5.1
6
6
  - 2.4.3
7
7
  gemfile:
8
- - gemfiles/activerecord-50.gemfile
9
8
  - gemfiles/activerecord-51.gemfile
10
9
  - gemfiles/activerecord-52.gemfile
11
10
  before_install:
data/README.md CHANGED
@@ -25,6 +25,28 @@ Or install it yourself as:
25
25
 
26
26
  $ gem install crono_trigger
27
27
 
28
+ ## Update from v0.3.x
29
+
30
+ ### Create crono_trigger system tables
31
+ ```
32
+ $ rails g crono_trigger:install # => create migrations
33
+ $ rake db:migrate
34
+ ```
35
+
36
+ ### Add `locked_by:string` column to CronoTrigger::Schedulable model
37
+ ```
38
+ $ rails g migration add_locked_by_column_to_your_model
39
+ $ rake db:migrate
40
+ ```
41
+
42
+ ```ruby
43
+ class AddLockedByColumnToYourModel < ActiveRecord::Migration[5.2]
44
+ def change
45
+ add_column :your_models, :locked_by, :string
46
+ end
47
+ end
48
+ ```
49
+
28
50
  ## Usage
29
51
 
30
52
  #### Execute `crono_trigger:model` generator.
@@ -154,6 +176,24 @@ Usage: crono_trigger [options] MODEL [MODEL..]
154
176
  You can rename some columns.
155
177
  ex. `crono_trigger_options[:next_execute_at_column_name] = "next_time"`
156
178
 
179
+ ## Admin Web
180
+
181
+ ![screenshots/crono_trigger_web.jpg](screenshots/crono_trigger_web.jpg)
182
+
183
+ ### Standalone mode
184
+
185
+ ```
186
+ $ crono_trigger-web --rails
187
+ ```
188
+
189
+ ### Mount as Rack app
190
+
191
+ ```ruby
192
+ # config/routes.rb
193
+ require "crono_trigger/web"
194
+ mount CronoTrigger::Web => '/crono_trigger'
195
+ ```
196
+
157
197
  ## Rollbar integration
158
198
  This gem has rollbar plugin.
159
199
  If `crono_trigger/rollbar` is required, Add Rollbar logging process to `CronoTrigger.config.error_handlers`
data/Rakefile CHANGED
@@ -9,6 +9,23 @@ pwd = File.expand_path('../', __FILE__)
9
9
 
10
10
  gemfiles = Dir.glob(File.join(pwd, "gemfiles", "*.gemfile")).map { |f| File.basename(f, ".*") }
11
11
 
12
+ namespace :js do
13
+ task :clean do
14
+ rm_r(File.join(pwd, "web", "app", "build")) if File.exist?(File.join(pwd, "web", "app", "build"))
15
+ rm_r(File.join(pwd, "web", "public"))
16
+ end
17
+
18
+ task build: [:clean] do
19
+ Dir.chdir(File.join(pwd, "web", "app"))
20
+ sh({"PUBLIC_URL" => "<%= URI.parse(url('/')).path.chop %>"}, "npm run build") do |ok, res|
21
+ raise "failed to build JS" unless ok
22
+
23
+ mv(File.join(pwd, "web", "app", "build"), File.join(pwd, "web", "public"))
24
+ mv(File.join(pwd, "web", "public", "index.html"), File.join(pwd, "web", "views", "index.erb"))
25
+ end
26
+ end
27
+ end
28
+
12
29
  namespace :spec do
13
30
  gemfiles.each do |gemfile|
14
31
  desc "Run Tests by #{gemfile}.gemfile"
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
15
15
  spec.license = "MIT"
16
16
 
17
17
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
- f.match(%r{^(test|spec|features)/})
18
+ f.match(%r{^(test|spec|features|screenshots)/})
19
19
  end
20
20
  spec.bindir = "exe"
21
21
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -26,6 +26,9 @@ Gem::Specification.new do |spec|
26
26
  spec.add_dependency "serverengine"
27
27
  spec.add_dependency "concurrent-ruby"
28
28
  spec.add_dependency "tzinfo"
29
+ spec.add_dependency "sinatra"
30
+ spec.add_dependency "rack-contrib"
31
+ spec.add_dependency "oj"
29
32
  spec.add_dependency "activerecord", ">= 4.2"
30
33
 
31
34
  spec.add_development_dependency "sqlite3"
@@ -0,0 +1,33 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "optparse"
4
+ require "crono_trigger/web"
5
+
6
+ options = {
7
+ bind: "0.0.0.0",
8
+ port: 9292
9
+ }
10
+
11
+ opt_parser = OptionParser.new do |o|
12
+ o.on('-p', '--port=PORT', 'Set port number (Default: 9292)') { |port| options[:port] = port.to_i }
13
+ o.on('-b', '--bind=ADRESS', 'Set address to bind (Default: 0.0.0.0)') { |addr| options[:bind] = addr }
14
+ o.on('-s', '--server=SERVER', 'Set Rack server handler name') { |handler| options[:server] = handler }
15
+ o.on('--rails', 'Require rails environment') do
16
+ pwd = Dir.pwd
17
+ loader = -> do
18
+ begin
19
+ require File.join(pwd, "config", "environment")
20
+ Rails.application.eager_load!
21
+ rescue LoadError
22
+ raise if pwd == "/"
23
+ pwd = File.expand_path("..", pwd)
24
+ loader.call
25
+ end
26
+ end
27
+ loader.call
28
+ end
29
+ end
30
+
31
+ opt_parser.parse!
32
+
33
+ CronoTrigger::Web.run!(options)
@@ -1,18 +1,22 @@
1
1
  require "crono_trigger/version"
2
2
 
3
3
  require "ostruct"
4
+ require "socket"
4
5
  require "active_record"
5
6
  require "concurrent"
7
+ require "crono_trigger/models/worker"
8
+ require "crono_trigger/models/signal"
6
9
  require "crono_trigger/worker"
7
10
  require "crono_trigger/polling_thread"
8
11
  require "crono_trigger/schedulable"
9
12
 
10
13
  module CronoTrigger
11
14
  @config = OpenStruct.new(
12
- polling_thread: 1,
15
+ worker_id: Socket.ip_address_list.detect { |info| !info.ipv4_loopback? && !info.ipv6_loopback? }.ip_address,
16
+ polling_thread: nil,
13
17
  polling_interval: 5,
14
18
  executor_thread: 25,
15
- model_names: [],
19
+ model_names: nil,
16
20
  error_handlers: [],
17
21
  )
18
22
 
@@ -24,12 +28,26 @@ module CronoTrigger
24
28
  yield config
25
29
  end
26
30
 
31
+ def self.reloader
32
+ @reloader
33
+ end
34
+
35
+ def self.reloader=(reloader)
36
+ @reloader = reloader
37
+ end
38
+
39
+ self.reloader = proc { |&block| block.call }
40
+
27
41
  def self.load_config(yml, environment = nil)
28
42
  config = YAML.load_file(yml)[environment || "default"]
29
43
  config.each do |k, v|
30
44
  @config[k] = v
31
45
  end
32
46
  end
47
+
48
+ def self.workers
49
+ CronoTrigger::Models::Worker.alive_workers
50
+ end
33
51
  end
34
52
 
35
53
  if defined?(Rails)
@@ -8,7 +8,11 @@ options = {
8
8
  }
9
9
 
10
10
  opt_parser = OptionParser.new do |opts|
11
- opts.banner = "Usage: crono_trigger [options] MODEL [MODEL..]"
11
+ opts.banner = "Usage: crono_trigger [options] [MODEL..]\n If MODEL is not given, Search classes including CronoTrigger::Schedulable module automatically."
12
+
13
+ opts.on("-w", "--worker-id=ID", "Worker ID (default: First local ip address which is not loopback") do |id|
14
+ options[:worker_id] = id
15
+ end
12
16
 
13
17
  opts.on("-f", "--config-file=CONFIG", "Config file (ex. ./crono_trigger.rb)") do |cfg|
14
18
  options[:config] = cfg
@@ -18,7 +22,7 @@ opt_parser = OptionParser.new do |opts|
18
22
  options[:env] = env
19
23
  end
20
24
 
21
- opts.on("-p", "--polling-thread=SIZE", Integer, "Polling thread size (Default: 1)") do |i|
25
+ opts.on("-p", "--polling-thread=SIZE", Integer, "Polling thread size (Default: Min of (target model count or processor_count)") do |i|
22
26
  options[:polling_thread] = i
23
27
  end
24
28
 
@@ -56,6 +60,7 @@ opt_parser.parse!
56
60
 
57
61
  begin
58
62
  require "rails"
63
+ require "crono_trigger/railtie"
59
64
  require File.expand_path("./config/environment", Rails.root)
60
65
  rescue LoadError
61
66
  end
@@ -66,7 +71,7 @@ CronoTrigger.load_config(options[:config], options[:env]) if options[:config]
66
71
  CronoTrigger.config[name] = options[name] if options[name]
67
72
  end
68
73
 
69
- CronoTrigger.config.model_names.concat(ARGV)
74
+ CronoTrigger.config.model_names = ARGV
70
75
 
71
76
  se = ServerEngine.create(nil, CronoTrigger::Worker, {
72
77
  daemonize: options[:daemonize],
@@ -0,0 +1,52 @@
1
+ module CronoTrigger
2
+ module Models
3
+ class Signal < ActiveRecord::Base
4
+ self.table_name = "crono_trigger_signals"
5
+
6
+ IGNORE_THRESHOLD = 300
7
+
8
+ enum signal: {TERM: "TERM", USR1: "USR1", CONT: "CONT", TSTP: "TSTP"}
9
+
10
+ scope :sent_to_me, proc {
11
+ raise "Must set worker_id" unless CronoTrigger.config.worker_id
12
+
13
+ where(arel_table[:sent_at].gteq(Time.current - IGNORE_THRESHOLD))
14
+ .where(worker_id: CronoTrigger.config.worker_id)
15
+ .where(received_at: nil)
16
+ .order(:sent_at)
17
+ }
18
+
19
+ class << self
20
+ def send_signal(signal, worker_id)
21
+ create!(signal: signal, worker_id: worker_id, sent_at: Time.current)
22
+ end
23
+
24
+ def send_term(worker_id)
25
+ send_signal("TERM", worker_id)
26
+ end
27
+
28
+ def send_usr1(worker_id)
29
+ send_signal("USR1", worker_id)
30
+ end
31
+
32
+ def send_cont(worker_id)
33
+ send_signal("CONT", worker_id)
34
+ end
35
+
36
+ def send_tstp(worker_id)
37
+ send_signal("TSTP", worker_id)
38
+ end
39
+ end
40
+
41
+ def kill_me(to_supervisor: true)
42
+ if update(received_at: Time.current)
43
+ if to_supervisor && Process.ppid != 1
44
+ Process.kill(signal, Process.ppid)
45
+ else
46
+ Process.kill(signal, Process.pid)
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,16 @@
1
+ require "crono_trigger/worker"
2
+
3
+ module CronoTrigger
4
+ module Models
5
+ class Worker < ActiveRecord::Base
6
+ self.table_name = "crono_trigger_workers"
7
+
8
+ ALIVE_THRESHOLD = CronoTrigger::Worker::HEARTBEAT_INTERVAL * 5
9
+
10
+ enum executor_status: {running: "running", quiet: "quiet", shuttingdown: "shuttingdown", shutdown: "shutdown"}
11
+ serialize :polling_model_names, JSON
12
+
13
+ scope :alive_workers, proc { where(arel_table[:last_heartbeated_at].gteq(Time.current - ALIVE_THRESHOLD)) }
14
+ end
15
+ end
16
+ end
@@ -1,25 +1,32 @@
1
1
  module CronoTrigger
2
2
  class PollingThread
3
- def initialize(model_queue, stop_flag, logger, executor)
3
+ def initialize(model_queue, stop_flag, logger, executor, execution_counter)
4
4
  @model_queue = model_queue
5
5
  @stop_flag = stop_flag
6
6
  @logger = logger
7
7
  @executor = executor
8
+ @execution_counter = execution_counter
9
+ @quiet = Concurrent::AtomicBoolean.new(false)
8
10
  end
9
11
 
10
12
  def run
11
13
  @thread = Thread.start do
12
14
  @logger.info "(polling-thread-#{Thread.current.object_id}) Start polling thread"
13
15
  until @stop_flag.wait_for_set(CronoTrigger.config.polling_interval)
14
- begin
15
- model = @model_queue.pop(true)
16
- poll(model)
17
- rescue ThreadError => e
18
- @logger.error(e) unless e.message == "queue empty"
19
- rescue => e
20
- @logger.error(e)
21
- ensure
22
- @model_queue << model if model
16
+ next if quiet?
17
+
18
+ CronoTrigger.reloader.call do
19
+ begin
20
+ model_name = @model_queue.pop(true)
21
+ model = model_name.classify.constantize
22
+ poll(model)
23
+ rescue ThreadError => e
24
+ @logger.error(e) unless e.message == "queue empty"
25
+ rescue => e
26
+ @logger.error(e)
27
+ ensure
28
+ @model_queue << model_name if model_name
29
+ end
23
30
  end
24
31
  end
25
32
  end
@@ -29,29 +36,60 @@ module CronoTrigger
29
36
  @thread.join
30
37
  end
31
38
 
39
+ def quiet
40
+ @quiet.make_true
41
+ end
42
+
43
+ def quiet?
44
+ @quiet.true?
45
+ end
46
+
47
+ def alive?
48
+ @thread.alive?
49
+ end
50
+
32
51
  def poll(model)
33
52
  @logger.debug "(polling-thread-#{Thread.current.object_id}) Poll #{model}"
34
53
  records = []
35
- primary_key_offset = nil
54
+ overflowed_record_ids = []
55
+
36
56
  begin
37
57
  model.connection_pool.with_connection do
38
- records = model.executables_with_lock(primary_key_offset: primary_key_offset)
39
- primary_key_offset = records.last && records.last.id
58
+ records = model.executables_with_lock
40
59
  end
41
60
 
42
61
  records.each do |record|
43
- @executor.post do
44
- model.connection_pool.with_connection do
45
- @logger.info "(executor-thread-#{Thread.current.object_id}) Execute #{record.class}-#{record.id}"
62
+ begin
63
+ @executor.post do
64
+ @execution_counter.increment
46
65
  begin
47
- record.do_execute
48
- rescue Exception => e
49
- @logger.error(e)
66
+ model.connection_pool.with_connection do
67
+ @logger.info "(executor-thread-#{Thread.current.object_id}) Execute #{record.class}-#{record.id}"
68
+ begin
69
+ record.do_execute
70
+ rescue Exception => e
71
+ @logger.error(e)
72
+ end
73
+ end
74
+ ensure
75
+ @execution_counter.decrement
50
76
  end
51
77
  end
78
+ rescue Concurrent::RejectedExecutionError
79
+ overflowed_record_ids << record.id
52
80
  end
53
81
  end
54
- end while records.any?
82
+ unlock_overflowed_records(model, overflowed_record_ids)
83
+ end while overflowed_record_ids.empty? && records.any?
84
+ end
85
+
86
+ private def unlock_overflowed_records(model, overflowed_record_ids)
87
+ model.connection_pool.with_connection do
88
+ model.where(id: overflowed_record_ids).crono_trigger_unlock_all!
89
+ end
90
+ rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::LockWaitTimeout, ActiveRecord::StatementTimeout, ActiveRecord::Deadlocked
91
+ sleep 1
92
+ retry
55
93
  end
56
94
  end
57
95
  end
@@ -1,4 +1,19 @@
1
1
  module CronoTrigger
2
2
  class Railtie < ::Rails::Railtie
3
+ config.after_initialize do
4
+ CronoTrigger.reloader = CronoTrigger::Railtie::Reloader.new
5
+ end
6
+
7
+ class Reloader
8
+ def initialize(app = ::Rails.application)
9
+ @app = app
10
+ end
11
+
12
+ def call
13
+ @app.reloader.wrap do
14
+ yield
15
+ end
16
+ end
17
+ end
3
18
  end
4
19
  end