fort_ci-worker 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 2fbeed40e5bde29d66fa1dd479c6a9e824cd50f0
4
+ data.tar.gz: c3d247842162aee2357ca7a1c353fefc0e088a2b
5
+ SHA512:
6
+ metadata.gz: 3f26aef3e8dc1f6c85a2006b905cd1c9874f745d81cb40cc2fada3ebbdaf0c2145ce3563c817a34e33b906e9cef8842a9f6fa2540a09bbd40fd329f5d8c6cef1
7
+ data.tar.gz: 54f2bb0a9d0450f4c052c01b27527278f534c7e2d9ddf2176bd1659dee65551dbe2fa14c77112a14f68fb476585145cba51f8ea100ad56d09fec0e526c620da1
data/.gitignore ADDED
@@ -0,0 +1,10 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ .idea/
data/Gemfile ADDED
@@ -0,0 +1,8 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in worker.gemspec
4
+ gemspec
5
+
6
+ gem 'sqlite3'
7
+ gem 'mysql2'
8
+ gem 'activerecord'
data/README.md ADDED
@@ -0,0 +1,36 @@
1
+ # Worker
2
+
3
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/worker`. To experiment with that code, run `bin/console` for an interactive prompt.
4
+
5
+ TODO: Delete this and the text above, and describe your gem
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'worker'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install worker
22
+
23
+ ## Usage
24
+
25
+ TODO: Write usage instructions here
26
+
27
+ ## Development
28
+
29
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
30
+
31
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
32
+
33
+ ## Contributing
34
+
35
+ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/worker.
36
+
data/Rakefile ADDED
@@ -0,0 +1,11 @@
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+
4
+ Rake::TestTask.new(:test) do |t|
5
+ t.libs << "test"
6
+ t.libs << "lib"
7
+ t.test_files = FileList['test/**/*_test.rb']
8
+ t.warning = false
9
+ end
10
+
11
+ task :default => :test
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "fort_ci/worker"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
data/lib/fort_ci.rb ADDED
@@ -0,0 +1,2 @@
1
+ module FortCI
2
+ end
@@ -0,0 +1,12 @@
1
+ require "fort_ci/worker/version"
2
+ require "fort_ci/worker/queue"
3
+ require "fort_ci/worker/job"
4
+ require "logger"
5
+
6
+ module FortCI
7
+ module Worker
8
+ class << self
9
+ attr_accessor :queue, :logger, :db
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,62 @@
1
+ require "json"
2
+ require "active_support"
3
+ require "active_support/hash_with_indifferent_access"
4
+
5
+ module FortCI
6
+ module Worker
7
+ class Job
8
+
9
+ def initialize(data={})
10
+ @data = data
11
+ end
12
+
13
+ def self.queue(name=nil)
14
+ @queue = name if name
15
+ @queue || 'default'
16
+ end
17
+
18
+ def self.priority(num=nil)
19
+ @priority = num if num
20
+ @priority || 0
21
+ end
22
+
23
+ def queue
24
+ self.class.queue
25
+ end
26
+
27
+ def priority
28
+ self.class.priority
29
+ end
30
+
31
+ def self.deserialize(string)
32
+ self.new(Hash[*JSON.parse(string).map { |k, v| [k.to_sym, v] }.flatten])
33
+ end
34
+
35
+ def before
36
+ end
37
+
38
+ def on_success
39
+ end
40
+
41
+ def on_failure
42
+ end
43
+
44
+ def perform
45
+ sleep 5
46
+ puts "performing"
47
+ end
48
+
49
+ def run_at
50
+ Time.now
51
+ end
52
+
53
+ def serialize
54
+ JSON.generate(Hash[*instance_variables.map { |attr| [attr.to_s.sub('@', ''), instance_variable_get(attr)] }.flatten])
55
+ end
56
+
57
+ def enqueue
58
+ Worker.queue.enqueue(self)
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,180 @@
1
+ require "sequel"
2
+ require "logger"
3
+
4
+
5
+ module FortCI
6
+ module Worker
7
+ class Queue
8
+ attr_reader :id
9
+ attr_accessor :jobs_per_worker, :max_job_time, :queue_name, :max_attempts, :logger, :poll_interval, :log_backtrace
10
+
11
+ def initialize(opts={})
12
+ @logger = Logger.new(STDOUT)
13
+
14
+ db_config = opts[:db_config] || Worker.db
15
+ db_config.merge(logger: logger) if opts[:log_queries]
16
+
17
+ @db = Sequel.connect(db_config)
18
+
19
+ name = `hostname`.chomp("\n")
20
+
21
+ @id = "#{name}.#{Process.pid}.#{rand(0..1000)}-#{opts[:queue_name] || 'all'}"
22
+
23
+ @jobs_per_worker = opts[:jobs_per_worker] || 5
24
+ @max_job_time = opts[:max_job_time] || 5 * 60 # in seconds
25
+ @queue_name = opts[:queue_name]
26
+ @max_attempts = opts[:max_attempts] || 20
27
+ @poll_interval = opts[:poll_interval] || 5 # 5 second polling
28
+ @log_backtrace = !!opts[:log_backtrace]
29
+
30
+ @db_adapter = db_config[:adapter]
31
+
32
+ create_table
33
+ end
34
+
35
+ def create_table
36
+ db.create_table? :worker_jobs do
37
+ primary_key :id
38
+ Time :locked_at, null: true
39
+ Time :run_at, null: true
40
+ String :locked_by, null: true
41
+ String :handler, null: false, text: true
42
+ String :last_error, null: true, text: true
43
+ String :queue, null: false, default: 'default'
44
+ String :job_class, null: false
45
+ Integer :attempts, null: false, default: 0
46
+ Integer :priority, null: false, default: 0
47
+ end
48
+ end
49
+
50
+ def set_error(job, error)
51
+ db[:worker_jobs]
52
+ .where(id: job[:id])
53
+ .update(last_error: error.backtrace.join("\n"), run_at: Time.now + 10)
54
+ end
55
+
56
+ def remove(id)
57
+ db[:worker_jobs].where(id: id).delete
58
+ end
59
+
60
+ # cleans locked by jobs that have been dormant for a while
61
+ def clean_locks
62
+ db[:worker_jobs].where('locked_at < ?', Time.now - max_job_time).update(locked_by: nil)
63
+ end
64
+
65
+ # locks a set of jobs for this worker
66
+ def dequeue
67
+
68
+ if @db_adapter == 'mysql2' || @db_adapter == 'mysql'
69
+ query = db[:worker_jobs]
70
+ .limit(jobs_per_worker)
71
+ .where('`run_at` < ?', Time.now)
72
+ .where('`locked_by` IS NULL')
73
+ .reverse_order('priority')
74
+ else
75
+ query = db[:worker_jobs]
76
+ .where(
77
+ '`run_at` < ? AND `locked_by` IS NULL AND `id` in (SELECT `id` FROM `worker_jobs` ORDER BY `priority` DESC LIMIT ?)',
78
+ Time.now, jobs_per_worker)
79
+ end
80
+
81
+ if queue_name
82
+ query = query.where(queue: queue_name)
83
+ end
84
+
85
+ query.update(locked_by: id, locked_at: Time.now)
86
+ db[:worker_jobs].where(locked_by: id)
87
+ end
88
+
89
+ def enqueue(job)
90
+ db[:worker_jobs].insert(
91
+ job_class: job.class.name,
92
+ queue: job.queue,
93
+ priority: job.priority,
94
+ handler: job.serialize,
95
+ run_at: job.run_at,
96
+ )
97
+ end
98
+
99
+ def unlock_all
100
+ db[:worker_jobs].where(locked_by: id).update(locked_by: nil)
101
+ end
102
+
103
+ def db
104
+ @db
105
+ end
106
+
107
+ def run(number_to_run=nil)
108
+ begin
109
+ while true
110
+
111
+ logger.debug("Polling")
112
+
113
+ t1 = Time.now
114
+ failures = 0
115
+ successes = 0
116
+
117
+ dequeue.each do |item|
118
+
119
+ if number_to_run
120
+ number_to_run -= 1
121
+ end
122
+
123
+ logger.info("Performing #{item[:job_class]}.#{item[:id]} attempts=#{item[:attempts]} priority=#{item[:priority]} run_at=#{item[:run_at]}")
124
+
125
+ job = nil
126
+ begin
127
+ job = eval(item[:job_class]).deserialize(item[:handler])
128
+
129
+ Timeout::timeout(max_job_time) do
130
+ job.before
131
+ job.perform
132
+ job.on_success
133
+ end
134
+
135
+ remove(item[:id])
136
+
137
+ successes += 1
138
+ logger.info("Successful #{item[:job_class]}.#{item[:id]}")
139
+
140
+ rescue Exception => e
141
+
142
+ job.on_failure if job
143
+
144
+ if item[:attempts] + 1 > max_attempts
145
+ remove(item[:id])
146
+ else
147
+ set_error(item, e)
148
+ end
149
+
150
+ failures += 1
151
+ logger.info("Failed #{item[:job_class]}.#{item[:id]} err: #{e.class.name} #{e.message}")
152
+ if log_backtrace
153
+ puts e.backtrace
154
+ end
155
+
156
+ end
157
+ end
158
+
159
+ t2 = Time.now
160
+
161
+ if failures + successes > 0
162
+ logger.info("Performed #{failures + successes} jobs in #{t2 - t1} seconds")
163
+ end
164
+
165
+ return if number_to_run && number_to_run <= 0
166
+ sleep(poll_interval)
167
+ end
168
+ rescue StandardError => e
169
+ logger.warn("Exiting due to #{e}")
170
+ return
171
+ ensure
172
+ logger.info("Unlocking")
173
+ unlock_all
174
+ end
175
+
176
+ end
177
+
178
+ end
179
+ end
180
+ end
@@ -0,0 +1,5 @@
1
+ module FortCI
2
+ module Worker
3
+ VERSION = "0.1.3"
4
+ end
5
+ end
data/worker.gemspec ADDED
@@ -0,0 +1,27 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'fort_ci'
5
+ require 'fort_ci/worker/version'
6
+
7
+ Gem::Specification.new do |spec|
8
+ spec.name = "fort_ci-worker"
9
+ spec.version = FortCI::Worker::VERSION
10
+ spec.authors = ["Colin Walker"]
11
+ spec.email = ["colinwalker270@gmail.com"]
12
+
13
+ spec.summary = %q{Database based job execution. Rails independent.}
14
+ spec.homepage = "https://github.com/coldog/fort.ci"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
17
+ spec.bindir = "exe"
18
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_development_dependency "bundler", "~> 1.12"
22
+ spec.add_development_dependency "rake", "~> 10.0"
23
+ spec.add_development_dependency "minitest", "~> 5.0"
24
+
25
+ spec.add_dependency "sequel", "~> 4.40"
26
+ spec.add_dependency "activesupport", ">= 4.0"
27
+ end
metadata ADDED
@@ -0,0 +1,125 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fort_ci-worker
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.3
5
+ platform: ruby
6
+ authors:
7
+ - Colin Walker
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2016-11-17 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.12'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.12'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: minitest
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '5.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '5.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: sequel
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '4.40'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '4.40'
69
+ - !ruby/object:Gem::Dependency
70
+ name: activesupport
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '4.0'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '4.0'
83
+ description:
84
+ email:
85
+ - colinwalker270@gmail.com
86
+ executables: []
87
+ extensions: []
88
+ extra_rdoc_files: []
89
+ files:
90
+ - ".gitignore"
91
+ - Gemfile
92
+ - README.md
93
+ - Rakefile
94
+ - bin/console
95
+ - bin/setup
96
+ - lib/fort_ci.rb
97
+ - lib/fort_ci/worker.rb
98
+ - lib/fort_ci/worker/job.rb
99
+ - lib/fort_ci/worker/queue.rb
100
+ - lib/fort_ci/worker/version.rb
101
+ - worker.gemspec
102
+ homepage: https://github.com/coldog/fort.ci
103
+ licenses: []
104
+ metadata: {}
105
+ post_install_message:
106
+ rdoc_options: []
107
+ require_paths:
108
+ - lib
109
+ required_ruby_version: !ruby/object:Gem::Requirement
110
+ requirements:
111
+ - - ">="
112
+ - !ruby/object:Gem::Version
113
+ version: '0'
114
+ required_rubygems_version: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - ">="
117
+ - !ruby/object:Gem::Version
118
+ version: '0'
119
+ requirements: []
120
+ rubyforge_project:
121
+ rubygems_version: 2.5.1
122
+ signing_key:
123
+ specification_version: 4
124
+ summary: Database based job execution. Rails independent.
125
+ test_files: []