dronejob 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +3 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +133 -0
- data/README.md +9 -0
- data/bin/dronejob +17 -0
- data/dronejob.gemspec +31 -0
- data/lib/active_job/queue_adapters/pub_sub_queue_adapter.rb +76 -0
- data/lib/dronejob/base.rb +21 -0
- data/lib/dronejob/command.rb +63 -0
- data/lib/dronejob/loader.rb +24 -0
- data/lib/dronejob/logger.rb +25 -0
- data/lib/dronejob/modules/attr_store.rb +41 -0
- data/lib/dronejob/modules/core.rb +63 -0
- data/lib/dronejob/modules/git.rb +61 -0
- data/lib/dronejob/modules/log.rb +24 -0
- data/lib/dronejob/modules/params.rb +47 -0
- data/lib/dronejob/modules/phases.rb +49 -0
- data/lib/dronejob/modules/workspace.rb +25 -0
- data/lib/dronejob/version.rb +3 -0
- data/lib/dronejob.rb +9 -0
- metadata +220 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: d88b29b5d14a2265c6995aadc9b1311a60ec75b1
|
4
|
+
data.tar.gz: 098e871db099aac491e1597aa291f725426f2bd6
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d2e247b70efef090e89a2772360ef4acb56cb0c8a6ea1d2d504ef9af61446850bbc7e2b2ae275dc92b3726f7c2bf89c0202914e9598f65d40eea4ab625457d0f
|
7
|
+
data.tar.gz: 06a0097b2444fa76547a11f2f511ef3a23f9672f706dec4518c5ed4b35fae32308eef4e50fc87edfb4a1de67308ee952ff5ca15af9bf67d588bcbd97859ca689
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,133 @@
|
|
1
|
+
GIT
|
2
|
+
remote: git://github.com/GoogleCloudPlatform/gcloud-ruby.git
|
3
|
+
revision: bde4915fe7baffae8326ed93b4ffcf3dc2de7c54
|
4
|
+
branch: master
|
5
|
+
specs:
|
6
|
+
gcloud (0.6.1)
|
7
|
+
beefcake (~> 1.0)
|
8
|
+
digest-crc (~> 0.4)
|
9
|
+
google-api-client (~> 0.8.3)
|
10
|
+
mime-types (~> 2.4)
|
11
|
+
zonefile (~> 1.04)
|
12
|
+
|
13
|
+
PATH
|
14
|
+
remote: .
|
15
|
+
specs:
|
16
|
+
dronejob (1.0.1)
|
17
|
+
activejob (~> 4.2)
|
18
|
+
bundler (~> 1.11)
|
19
|
+
gcloud (~> 0.6)
|
20
|
+
git (~> 1.2)
|
21
|
+
google-api-client (~> 0.8)
|
22
|
+
logging (~> 2.0)
|
23
|
+
logging-google-cloud (~> 1.0)
|
24
|
+
rake (~> 10.5)
|
25
|
+
thor (~> 0.19)
|
26
|
+
|
27
|
+
GEM
|
28
|
+
remote: http://rubygems.org/
|
29
|
+
specs:
|
30
|
+
activejob (4.2.5.1)
|
31
|
+
activesupport (= 4.2.5.1)
|
32
|
+
globalid (>= 0.3.0)
|
33
|
+
activesupport (4.2.5.1)
|
34
|
+
i18n (~> 0.7)
|
35
|
+
json (~> 1.7, >= 1.7.7)
|
36
|
+
minitest (~> 5.1)
|
37
|
+
thread_safe (~> 0.3, >= 0.3.4)
|
38
|
+
tzinfo (~> 1.1)
|
39
|
+
addressable (2.4.0)
|
40
|
+
autoparse (0.3.3)
|
41
|
+
addressable (>= 2.3.1)
|
42
|
+
extlib (>= 0.9.15)
|
43
|
+
multi_json (>= 1.0.0)
|
44
|
+
beefcake (1.1.0)
|
45
|
+
coderay (1.1.0)
|
46
|
+
diff-lcs (1.2.5)
|
47
|
+
digest-crc (0.4.1)
|
48
|
+
extlib (0.9.16)
|
49
|
+
faraday (0.9.2)
|
50
|
+
multipart-post (>= 1.2, < 3)
|
51
|
+
git (1.2.9.1)
|
52
|
+
globalid (0.3.6)
|
53
|
+
activesupport (>= 4.1.0)
|
54
|
+
google-api-client (0.8.6)
|
55
|
+
activesupport (>= 3.2)
|
56
|
+
addressable (~> 2.3)
|
57
|
+
autoparse (~> 0.3)
|
58
|
+
extlib (~> 0.9)
|
59
|
+
faraday (~> 0.9)
|
60
|
+
googleauth (~> 0.3)
|
61
|
+
launchy (~> 2.4)
|
62
|
+
multi_json (~> 1.10)
|
63
|
+
retriable (~> 1.4)
|
64
|
+
signet (~> 0.6)
|
65
|
+
googleauth (0.5.1)
|
66
|
+
faraday (~> 0.9)
|
67
|
+
jwt (~> 1.4)
|
68
|
+
logging (~> 2.0)
|
69
|
+
memoist (~> 0.12)
|
70
|
+
multi_json (~> 1.11)
|
71
|
+
os (~> 0.9)
|
72
|
+
signet (~> 0.7)
|
73
|
+
i18n (0.7.0)
|
74
|
+
json (1.8.3)
|
75
|
+
jwt (1.5.2)
|
76
|
+
launchy (2.4.3)
|
77
|
+
addressable (~> 2.3)
|
78
|
+
little-plugger (1.1.4)
|
79
|
+
logging (2.0.0)
|
80
|
+
little-plugger (~> 1.1)
|
81
|
+
multi_json (~> 1.10)
|
82
|
+
logging-google-cloud (1.0.2)
|
83
|
+
gcloud (~> 0.6)
|
84
|
+
logging (~> 2.0)
|
85
|
+
memoist (0.14.0)
|
86
|
+
method_source (0.8.2)
|
87
|
+
mime-types (2.99)
|
88
|
+
minitest (5.8.4)
|
89
|
+
multi_json (1.11.2)
|
90
|
+
multipart-post (2.0.0)
|
91
|
+
os (0.9.6)
|
92
|
+
pry (0.10.3)
|
93
|
+
coderay (~> 1.1.0)
|
94
|
+
method_source (~> 0.8.1)
|
95
|
+
slop (~> 3.4)
|
96
|
+
rake (10.5.0)
|
97
|
+
retriable (1.4.1)
|
98
|
+
rspec (3.4.0)
|
99
|
+
rspec-core (~> 3.4.0)
|
100
|
+
rspec-expectations (~> 3.4.0)
|
101
|
+
rspec-mocks (~> 3.4.0)
|
102
|
+
rspec-core (3.4.2)
|
103
|
+
rspec-support (~> 3.4.0)
|
104
|
+
rspec-expectations (3.4.0)
|
105
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
106
|
+
rspec-support (~> 3.4.0)
|
107
|
+
rspec-mocks (3.4.1)
|
108
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
109
|
+
rspec-support (~> 3.4.0)
|
110
|
+
rspec-support (3.4.1)
|
111
|
+
signet (0.7.2)
|
112
|
+
addressable (~> 2.3)
|
113
|
+
faraday (~> 0.9)
|
114
|
+
jwt (~> 1.5)
|
115
|
+
multi_json (~> 1.10)
|
116
|
+
slop (3.6.0)
|
117
|
+
thor (0.19.1)
|
118
|
+
thread_safe (0.3.5)
|
119
|
+
tzinfo (1.2.2)
|
120
|
+
thread_safe (~> 0.1)
|
121
|
+
zonefile (1.04)
|
122
|
+
|
123
|
+
PLATFORMS
|
124
|
+
ruby
|
125
|
+
|
126
|
+
DEPENDENCIES
|
127
|
+
dronejob!
|
128
|
+
gcloud!
|
129
|
+
pry (~> 0.10)
|
130
|
+
rspec (~> 3.3)
|
131
|
+
|
132
|
+
BUNDLED WITH
|
133
|
+
1.11.2
|
data/README.md
ADDED
data/bin/dronejob
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
$:.push("lib/")
|
3
|
+
# require "pry"
|
4
|
+
require "thor"
|
5
|
+
|
6
|
+
# initialize activejob
|
7
|
+
require "dronejob"
|
8
|
+
require "active_job"
|
9
|
+
require "active_job/queue_adapters/pub_sub_queue_adapter"
|
10
|
+
ActiveJob::Base.queue_adapter = :pub_sub_queue
|
11
|
+
ActiveJob::Base.logger = Dronejob::Logger.logger
|
12
|
+
|
13
|
+
# initialize dronejob
|
14
|
+
Dronejob::Loader.init
|
15
|
+
|
16
|
+
require "dronejob/command"
|
17
|
+
Dronejob::Command.start
|
data/dronejob.gemspec
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
lib = File.expand_path('../lib', __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require 'dronejob/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "dronejob"
|
7
|
+
s.version = Dronejob::VERSION
|
8
|
+
s.licenses = ["BSD-3-Clause"]
|
9
|
+
s.platform = Gem::Platform::RUBY
|
10
|
+
s.authors = ["Tobias Strebitzer"]
|
11
|
+
s.email = ["tobias.strebitzer@magloft.com"]
|
12
|
+
s.homepage = "https://github.com/MagLoft/dronejob"
|
13
|
+
s.summary = "DroneJob Drone Factory"
|
14
|
+
s.description = "Scalable worker factory for ruby"
|
15
|
+
s.required_ruby_version = '~> 2.0'
|
16
|
+
s.required_rubygems_version = '~> 2.4'
|
17
|
+
s.add_runtime_dependency "bundler", "~> 1.11"
|
18
|
+
s.add_runtime_dependency "rake", "~> 10.5"
|
19
|
+
s.add_runtime_dependency "git", "~> 1.2"
|
20
|
+
s.add_runtime_dependency "logging", "~> 2.0"
|
21
|
+
s.add_runtime_dependency "thor", "~> 0.19"
|
22
|
+
s.add_runtime_dependency "activejob", "~> 4.2"
|
23
|
+
s.add_runtime_dependency "logging-google-cloud", "~> 1.0"
|
24
|
+
s.add_runtime_dependency "gcloud", "~> 0.6"
|
25
|
+
s.add_runtime_dependency "google-api-client", "~> 0.8"
|
26
|
+
s.add_development_dependency "rspec", "~> 3.3"
|
27
|
+
s.add_development_dependency "pry", "~> 0.10"
|
28
|
+
s.files = `git ls-files`.split("\n")
|
29
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
|
30
|
+
s.require_path = 'lib'
|
31
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
module ActiveJob
|
2
|
+
module QueueAdapters
|
3
|
+
class PubSubQueueAdapter
|
4
|
+
|
5
|
+
def self.project_id
|
6
|
+
@@project_id
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.project_id=(project_id)
|
10
|
+
@@project_id = project_id
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.queue_topic
|
14
|
+
@@queue_topic ||= "dronejob_queue"
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.queue_topic=(queue_topic)
|
18
|
+
@@queue_topic = queue_topic
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.status_topic
|
22
|
+
@@status_topic ||= "dronejob_status"
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.status_topic=(status_topic)
|
26
|
+
@@status_topic = status_topic
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.pubsub
|
30
|
+
gcloud = Gcloud.new(self.project_id)
|
31
|
+
gcloud.pubsub
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.enqueue(job)
|
35
|
+
Dronejob::Logger.log("info", "Enqueuing worker #{job.class.name} with job id #{job.job_id}")
|
36
|
+
topic = pubsub.topic(self.queue_topic)
|
37
|
+
topic.publish(job.serialize.to_json)
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.listen(job, &block)
|
41
|
+
subscribe(self.status_topic, true) do |data|
|
42
|
+
block.call(data["detail"]) if data["job_id"] == job.job_id
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.run_worker!
|
47
|
+
subscribe(self.queue_topic, true) do |data|
|
48
|
+
job = ActiveJob::Base.deserialize(data)
|
49
|
+
Dronejob::Logger.log("info", "Processing worker #{job.class.name} with job id #{job.job_id}")
|
50
|
+
job.perform_now()
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.notify(job, detail)
|
55
|
+
topic = pubsub.topic(self.status_topic)
|
56
|
+
data = {job_id: job.job_id, detail: detail}
|
57
|
+
topic.publish(data.to_json)
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def self.subscribe(topic_name, autoack=true, &block)
|
63
|
+
Dronejob::Logger.log("info", "Preparing subscription to #{topic_name}")
|
64
|
+
topic = pubsub.topic(topic_name)
|
65
|
+
subscription = topic.subscription(topic_name)
|
66
|
+
topic.subscribe(topic_name) if subscription.nil? or !subscription.exists?
|
67
|
+
Dronejob::Logger.log("info", "Subscribing to #{topic}")
|
68
|
+
subscription.listen autoack: autoack do |message|
|
69
|
+
data = JSON.parse(message.data)
|
70
|
+
block.call(data)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'dronejob/modules/attr_store'
|
2
|
+
require 'dronejob/modules/core'
|
3
|
+
require 'dronejob/modules/git'
|
4
|
+
require 'dronejob/modules/log'
|
5
|
+
require 'dronejob/modules/params'
|
6
|
+
require 'dronejob/modules/phases'
|
7
|
+
require 'dronejob/modules/workspace'
|
8
|
+
|
9
|
+
module Dronejob
|
10
|
+
class Base < ActiveJob::Base
|
11
|
+
include Modules::AttrStore
|
12
|
+
include Modules::Core
|
13
|
+
include Modules::Git
|
14
|
+
include Modules::Log
|
15
|
+
include Modules::Params
|
16
|
+
include Modules::Phases
|
17
|
+
include Modules::Workspace
|
18
|
+
|
19
|
+
ActiveSupport.run_load_hooks(:dronejob, self)
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
module Dronejob
|
2
|
+
class Command < Thor
|
3
|
+
|
4
|
+
Dronejob::Loader.workers.each do |identifier, worker|
|
5
|
+
desc identifier, "Queue #{identifier} worker"
|
6
|
+
|
7
|
+
# Pub/Sub options
|
8
|
+
method_option(:project_id, type: :string, required: true)
|
9
|
+
method_option(:queue_topic, type: :string, required: true, default: "dronejob_queue")
|
10
|
+
method_option(:status_topic, type: :string, required: true, default: "dronejob_status")
|
11
|
+
|
12
|
+
# Worker options
|
13
|
+
method_option(:run, aliases: "-r", type: :boolean, default: false)
|
14
|
+
method_option(:break, aliases: "-b", type: :boolean, default: false)
|
15
|
+
method_option(:skip, aliases: "-s", type: :array, default: [])
|
16
|
+
method_option(:from, aliases: "-f", type: :string)
|
17
|
+
method_option(:listen, aliases: "-l", type: :boolean, default: false)
|
18
|
+
|
19
|
+
# Worker options
|
20
|
+
worker.params.each do |key, config|
|
21
|
+
method_option(key, required: config[:required], type: config[:type], default: config[:default])
|
22
|
+
end
|
23
|
+
|
24
|
+
define_method(identifier) do
|
25
|
+
# Set up adapter
|
26
|
+
ActiveJob::QueueAdapters::PubSubQueueAdapter.project_id = options.project_id
|
27
|
+
ActiveJob::QueueAdapters::PubSubQueueAdapter.queue_topic = options.queue_topic
|
28
|
+
ActiveJob::QueueAdapters::PubSubQueueAdapter.status_topic = options.status_topic
|
29
|
+
|
30
|
+
# Execute / queue job
|
31
|
+
if options.run
|
32
|
+
worker.perform_now(options.to_hash)
|
33
|
+
else
|
34
|
+
job = worker.perform_later(options.to_hash)
|
35
|
+
|
36
|
+
# Listen for notifications
|
37
|
+
if options.listen
|
38
|
+
ActiveJob::QueueAdapters::PubSubQueueAdapter.listen(job) do |detail|
|
39
|
+
Dronejob::Logger.log("info", "status: #{detail.to_json}")
|
40
|
+
abort if detail["phase"] == "complete"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
desc :server, "Start Dronejob Server"
|
49
|
+
method_option(:project_id, type: :string, required: true)
|
50
|
+
method_option(:queue_topic, type: :string, required: true, default: "dronejob_queue")
|
51
|
+
method_option(:status_topic, type: :string, required: true, default: "dronejob_status")
|
52
|
+
def server
|
53
|
+
# Set up adapter
|
54
|
+
ActiveJob::QueueAdapters::PubSubQueueAdapter.project_id = options.project_id
|
55
|
+
ActiveJob::QueueAdapters::PubSubQueueAdapter.queue_topic = options.queue_topic
|
56
|
+
ActiveJob::QueueAdapters::PubSubQueueAdapter.status_topic = options.status_topic
|
57
|
+
|
58
|
+
# Run worker
|
59
|
+
ActiveJob::QueueAdapters::PubSubQueueAdapter.run_worker!
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Dronejob
|
2
|
+
module Loader
|
3
|
+
@@workers = {}
|
4
|
+
|
5
|
+
def self.init(path="workers")
|
6
|
+
require "active_support/all"
|
7
|
+
Dir["#{path}/*.rb"].each do |file|
|
8
|
+
require "./#{file}"
|
9
|
+
identifier = File.basename(file).gsub(/\.rb$/, "")
|
10
|
+
klassname = "Workers::#{identifier.camelcase}"
|
11
|
+
klass = klassname.constantize
|
12
|
+
@@workers[identifier] = klass
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.identifier_for(worker)
|
17
|
+
self.workers.key(worker.class)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.workers
|
21
|
+
@@workers
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require "logging"
|
2
|
+
require "logging/appenders/gcl"
|
3
|
+
|
4
|
+
module Dronejob
|
5
|
+
module Logger
|
6
|
+
@@logger = nil
|
7
|
+
|
8
|
+
def self.logger
|
9
|
+
if @@logger.nil?
|
10
|
+
Logging.init(Logging::Appenders::GoogleCloudLogging::SEVERITY_NAMES)
|
11
|
+
Logging.color_scheme("bright", levels: { debug: :blue, info: :green, warn: :yellow, error: :red, fatal: [:white, :on_red] }, date: :blue, ndc: :blue, logger: :cyan, message: :black)
|
12
|
+
Logging.appenders.stdout("stdout", layout: Logging.layouts.pattern( pattern: '[%d] %-5l %c{1} %x %m\n', color_scheme: 'bright'))
|
13
|
+
Logging.appenders.gcl("gcl", project_id: "typeshot-01", log_name: "dronejob", resource_type: "gce_instance", resource_labels: {instance_id: "drone-05", zone: "us-central1-b"}, buffer_size: '3', immediate_at: 'error, fatal')
|
14
|
+
@@logger = Logging::Logger.new("Dronejob")
|
15
|
+
@@logger.add_appenders("stdout") # gcl
|
16
|
+
end
|
17
|
+
@@logger
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.log(level, message)
|
21
|
+
logger.send(level, message)
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require "psych"
|
2
|
+
|
3
|
+
module Dronejob
|
4
|
+
module Modules
|
5
|
+
module AttrStore
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
included do
|
9
|
+
attr_accessor :variables
|
10
|
+
end
|
11
|
+
|
12
|
+
module ClassMethods
|
13
|
+
def attr_store(*args)
|
14
|
+
@attr_stores ||= []
|
15
|
+
@attr_stores.push(*args)
|
16
|
+
end
|
17
|
+
|
18
|
+
def attr_stores
|
19
|
+
@attr_stores
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def save_variables
|
24
|
+
variables = {}
|
25
|
+
self.class.attr_stores.each do |key|
|
26
|
+
variables[key] = instance_variable_get("@#{key}")
|
27
|
+
end
|
28
|
+
file_write(".dronejob-variables", Psych.dump(variables))
|
29
|
+
end
|
30
|
+
|
31
|
+
def load_variables
|
32
|
+
yaml_file = File.join(working_dir, ".dronejob-variables")
|
33
|
+
variables = File.exists?(yaml_file) ? Psych.load_file(yaml_file) : {}
|
34
|
+
variables.each do |key, value|
|
35
|
+
instance_variable_set("@#{key}", value)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
module Dronejob
|
2
|
+
module Modules
|
3
|
+
module Core
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
included do
|
7
|
+
attr_reader :commits, :config
|
8
|
+
end
|
9
|
+
|
10
|
+
def perform(params)
|
11
|
+
@params = params
|
12
|
+
validate_parameters!
|
13
|
+
create_working_dir
|
14
|
+
load_variables
|
15
|
+
git_init(@working_dir, param(:from))
|
16
|
+
|
17
|
+
# Run through phases
|
18
|
+
each_phase do |phase, config|
|
19
|
+
begin
|
20
|
+
phase_result = public_send(phase)
|
21
|
+
rescue Exception => e
|
22
|
+
error!(e)
|
23
|
+
end
|
24
|
+
notify(phase_result) if config[:notify]
|
25
|
+
git_commit(phase)
|
26
|
+
end
|
27
|
+
@phase = "complete"
|
28
|
+
notify(true)
|
29
|
+
end
|
30
|
+
|
31
|
+
def uuid
|
32
|
+
@uuid ||= "#{self.class.name.split('::').last.underscore}_#{SecureRandom.uuid}"
|
33
|
+
end
|
34
|
+
|
35
|
+
def notify(result=nil)
|
36
|
+
ActiveJob::QueueAdapters::PubSubQueueAdapter.notify(self, {phase: @phase, result: result})
|
37
|
+
end
|
38
|
+
|
39
|
+
def shell
|
40
|
+
@shell ||= Thor::Shell::Color.new
|
41
|
+
end
|
42
|
+
|
43
|
+
def ask(*args)
|
44
|
+
shell.ask(*args)
|
45
|
+
end
|
46
|
+
|
47
|
+
def say(*args)
|
48
|
+
shell.say(*args)
|
49
|
+
end
|
50
|
+
|
51
|
+
def breakpoint
|
52
|
+
if param(:break)
|
53
|
+
say(" ", [Thor::Shell::Color::WHITE, Thor::Shell::Color::ON_BLACK], true)
|
54
|
+
say(" BREAKPOINT ACTIVATED ", [Thor::Shell::Color::WHITE, Thor::Shell::Color::ON_BLACK, Thor::Shell::Color::BOLD], true)
|
55
|
+
say(" PRESS RETURN TO CONTINUE ", [Thor::Shell::Color::WHITE, Thor::Shell::Color::ON_BLACK], true)
|
56
|
+
say(" ", [Thor::Shell::Color::WHITE, Thor::Shell::Color::ON_BLACK], false)
|
57
|
+
ask("")
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require "git"
|
2
|
+
|
3
|
+
module Dronejob
|
4
|
+
module Modules
|
5
|
+
module Git
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
included do
|
9
|
+
attr_accessor :from
|
10
|
+
attr_reader :git
|
11
|
+
end
|
12
|
+
|
13
|
+
def git_init(path, commit=nil)
|
14
|
+
@git = ::Git.init(path)
|
15
|
+
git_collect_commits
|
16
|
+
git_commit("start")
|
17
|
+
git_reset(commit)
|
18
|
+
git_collect_commits
|
19
|
+
git_clean
|
20
|
+
end
|
21
|
+
|
22
|
+
def git_commit(commit)
|
23
|
+
if !@commits.keys.include?(commit.to_s)
|
24
|
+
save_variables()
|
25
|
+
@git.add(all: true, force: true)
|
26
|
+
@git.commit(commit.to_s, allow_empty: true)
|
27
|
+
commit = @git.log(1).last
|
28
|
+
@commits[commit] = commit.to_s
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def git_collect_commits
|
33
|
+
@commits = {}
|
34
|
+
begin
|
35
|
+
@git.log(100).each do |c|
|
36
|
+
@commits[c.message] = c.to_s
|
37
|
+
end
|
38
|
+
rescue Exception => e
|
39
|
+
# skip if log does not yet exist
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def git_clean
|
44
|
+
@git.clean(force: true, d: true)
|
45
|
+
end
|
46
|
+
|
47
|
+
def git_reset(commit=nil)
|
48
|
+
if commit
|
49
|
+
log("resetting to phase '#{commit}'")
|
50
|
+
from_index = @commits.keys.index(commit)
|
51
|
+
error!("phase '#{commit}' not found!") if from_index.nil? or !@commits.keys[from_index]
|
52
|
+
previous = @commits.keys[from_index]
|
53
|
+
@git.reset_hard(@commits[previous])
|
54
|
+
else
|
55
|
+
@git.reset_hard()
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Dronejob
|
2
|
+
module Modules
|
3
|
+
module Log
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
def log(message, level=:info)
|
7
|
+
begin
|
8
|
+
Dronejob::Logger.log(level, message)
|
9
|
+
rescue Exception => e
|
10
|
+
puts "LOGGING ERROR: #{e}"
|
11
|
+
e.backtrace.each do |line|
|
12
|
+
puts("▸ #{line}")
|
13
|
+
end
|
14
|
+
abort "EXIT"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def error!(message)
|
19
|
+
log(message, level=:error)
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Dronejob
|
2
|
+
module Modules
|
3
|
+
module Params
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
module ClassMethods
|
7
|
+
def param(key, options)
|
8
|
+
@params ||= {}
|
9
|
+
@params[key] = options
|
10
|
+
end
|
11
|
+
|
12
|
+
def params
|
13
|
+
@params
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def param(key)
|
18
|
+
@params[key.to_s]
|
19
|
+
end
|
20
|
+
|
21
|
+
def validate_parameters!
|
22
|
+
# Fill default parameters
|
23
|
+
self.class.params.each do |param_name, param_config|
|
24
|
+
param_value = @params[param_name.to_s]
|
25
|
+
if param_value.nil? and param_config[:default]
|
26
|
+
@params[param_name.to_s] = param_config[:default]
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
# Validate parameters
|
31
|
+
type_map = {string: [String], numeric: [Fixnum, Integer, Float], array: [Array], boolean: [TrueClass, FalseClass]}
|
32
|
+
self.class.params.each do |param_name, param_config|
|
33
|
+
param_value = @params[param_name.to_s]
|
34
|
+
if param_value.nil?
|
35
|
+
fail ArgumentError, "Missing required parameter '#{param_name}' (#{param_config[:type]})" if param_config[:required]
|
36
|
+
else
|
37
|
+
allowed_classes = type_map[param_config[:type]]
|
38
|
+
if !allowed_classes.include?(param_value.class)
|
39
|
+
fail ArgumentError, "Invalid value '#{param_value}' for parameter '#{param_name}' (expected #{param_config[:type]}, got #{param_value.class.name})"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module Dronejob
|
2
|
+
module Modules
|
3
|
+
module Phases
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
included do
|
7
|
+
attr_accessor :phase
|
8
|
+
end
|
9
|
+
|
10
|
+
module ClassMethods
|
11
|
+
def phase(key, options)
|
12
|
+
@phases ||= {}
|
13
|
+
@phases[key] = options
|
14
|
+
end
|
15
|
+
|
16
|
+
def phases
|
17
|
+
@phases
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def each_phase(&block)
|
22
|
+
self.class.phases.each do |phase, config|
|
23
|
+
fail "Phase not found: '#{phase}'" unless self.respond_to?(phase)
|
24
|
+
@phase = phase
|
25
|
+
before_phase(phase, config) if self.respond_to?(:before_phase)
|
26
|
+
Logging.ndc.push("#{uuid}##{phase}")
|
27
|
+
if completed_phase?(phase) and !config[:always_run]
|
28
|
+
log("already completed")
|
29
|
+
elsif skip_phase?(phase)
|
30
|
+
log("skipping...")
|
31
|
+
else
|
32
|
+
block.call(phase, config)
|
33
|
+
end
|
34
|
+
Logging.ndc.pop
|
35
|
+
after_phase(phase, config) if self.respond_to?(:after_phase)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def completed_phase?(phase)
|
40
|
+
@commits.include?(phase.to_s)
|
41
|
+
end
|
42
|
+
|
43
|
+
def skip_phase?(phase)
|
44
|
+
param(:skip) and param(:skip).include?(phase.to_s)
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Dronejob
|
2
|
+
module Modules
|
3
|
+
module Workspace
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
included do
|
7
|
+
attr_accessor :working_dir
|
8
|
+
end
|
9
|
+
|
10
|
+
def create_working_dir()
|
11
|
+
@working_dir = File.join("jobs", uuid)
|
12
|
+
FileUtils.mkdir_p(@working_dir)
|
13
|
+
end
|
14
|
+
|
15
|
+
def file_write(target_path, contents)
|
16
|
+
full_path = File.join(working_dir, target_path)
|
17
|
+
open(full_path, "wb") do |file|
|
18
|
+
file << contents
|
19
|
+
end
|
20
|
+
full_path
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/lib/dronejob.rb
ADDED
metadata
ADDED
@@ -0,0 +1,220 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: dronejob
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Tobias Strebitzer
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-02-15 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.11'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.11'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.5'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.5'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: git
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ~>
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.2'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.2'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: logging
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '2.0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ~>
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '2.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: thor
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ~>
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0.19'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ~>
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0.19'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: activejob
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ~>
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '4.2'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ~>
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '4.2'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: logging-google-cloud
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ~>
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '1.0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ~>
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '1.0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: gcloud
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ~>
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0.6'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ~>
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0.6'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: google-api-client
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ~>
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0.8'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ~>
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0.8'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: rspec
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ~>
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '3.3'
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ~>
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '3.3'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: pry
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ~>
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0.10'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ~>
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0.10'
|
167
|
+
description: Scalable worker factory for ruby
|
168
|
+
email:
|
169
|
+
- tobias.strebitzer@magloft.com
|
170
|
+
executables:
|
171
|
+
- dronejob
|
172
|
+
extensions: []
|
173
|
+
extra_rdoc_files: []
|
174
|
+
files:
|
175
|
+
- .gitignore
|
176
|
+
- Gemfile
|
177
|
+
- Gemfile.lock
|
178
|
+
- README.md
|
179
|
+
- bin/dronejob
|
180
|
+
- dronejob.gemspec
|
181
|
+
- lib/active_job/queue_adapters/pub_sub_queue_adapter.rb
|
182
|
+
- lib/dronejob.rb
|
183
|
+
- lib/dronejob/base.rb
|
184
|
+
- lib/dronejob/command.rb
|
185
|
+
- lib/dronejob/loader.rb
|
186
|
+
- lib/dronejob/logger.rb
|
187
|
+
- lib/dronejob/modules/attr_store.rb
|
188
|
+
- lib/dronejob/modules/core.rb
|
189
|
+
- lib/dronejob/modules/git.rb
|
190
|
+
- lib/dronejob/modules/log.rb
|
191
|
+
- lib/dronejob/modules/params.rb
|
192
|
+
- lib/dronejob/modules/phases.rb
|
193
|
+
- lib/dronejob/modules/workspace.rb
|
194
|
+
- lib/dronejob/version.rb
|
195
|
+
homepage: https://github.com/MagLoft/dronejob
|
196
|
+
licenses:
|
197
|
+
- BSD-3-Clause
|
198
|
+
metadata: {}
|
199
|
+
post_install_message:
|
200
|
+
rdoc_options: []
|
201
|
+
require_paths:
|
202
|
+
- lib
|
203
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
204
|
+
requirements:
|
205
|
+
- - ~>
|
206
|
+
- !ruby/object:Gem::Version
|
207
|
+
version: '2.0'
|
208
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
209
|
+
requirements:
|
210
|
+
- - ~>
|
211
|
+
- !ruby/object:Gem::Version
|
212
|
+
version: '2.4'
|
213
|
+
requirements: []
|
214
|
+
rubyforge_project:
|
215
|
+
rubygems_version: 2.5.1
|
216
|
+
signing_key:
|
217
|
+
specification_version: 4
|
218
|
+
summary: DroneJob Drone Factory
|
219
|
+
test_files: []
|
220
|
+
has_rdoc:
|