tq 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gems +3 -3
- data/lib/tq.rb +250 -3
- data/lib/version.rb +1 -1
- data/test/helper.rb +37 -93
- data/test/suite.rb +1 -1
- data/test/test_auth.rb +7 -39
- data/test/test_logger.rb +34 -29
- data/test/test_run.rb +55 -186
- metadata +11 -15
- data/lib/tq/app.rb +0 -176
- data/lib/tq/queue.rb +0 -161
- data/lib/tq/shell.rb +0 -104
- data/test/test_shell.rb +0 -161
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Eric Gjertsen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-01-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: google-api-client
|
@@ -16,43 +16,43 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - '='
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.19.3
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - '='
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.
|
26
|
+
version: 0.19.3
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: minitest
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - '='
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 5.
|
33
|
+
version: 5.11.1
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - '='
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 5.
|
40
|
+
version: 5.11.1
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: parallel
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - '='
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 1.
|
47
|
+
version: 1.12.1
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - '='
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 1.
|
55
|
-
description:
|
54
|
+
version: 1.12.1
|
55
|
+
description: A simple framework for writing task worker processes
|
56
56
|
email: ericgj72@gmail.com
|
57
57
|
executables: []
|
58
58
|
extensions: []
|
@@ -60,17 +60,13 @@ extra_rdoc_files: []
|
|
60
60
|
files:
|
61
61
|
- ".gems"
|
62
62
|
- lib/tq.rb
|
63
|
-
- lib/tq/app.rb
|
64
63
|
- lib/tq/logger.rb
|
65
|
-
- lib/tq/queue.rb
|
66
|
-
- lib/tq/shell.rb
|
67
64
|
- lib/version.rb
|
68
65
|
- test/helper.rb
|
69
66
|
- test/suite.rb
|
70
67
|
- test/test_auth.rb
|
71
68
|
- test/test_logger.rb
|
72
69
|
- test/test_run.rb
|
73
|
-
- test/test_shell.rb
|
74
70
|
homepage: https://github.com/ericgj/tq
|
75
71
|
licenses:
|
76
72
|
- MIT
|
@@ -91,8 +87,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
91
87
|
version: '0'
|
92
88
|
requirements: []
|
93
89
|
rubyforge_project:
|
94
|
-
rubygems_version: 2.
|
90
|
+
rubygems_version: 2.6.13
|
95
91
|
signing_key:
|
96
92
|
specification_version: 4
|
97
|
-
summary: Ruby client for Google
|
93
|
+
summary: Ruby client for Google Cloud Tasks (REST API v2beta2)
|
98
94
|
test_files: []
|
data/lib/tq/app.rb
DELETED
@@ -1,176 +0,0 @@
|
|
1
|
-
require 'google/api_client'
|
2
|
-
require 'google/api_client/client_secrets'
|
3
|
-
require 'google/api_client/auth/file_storage'
|
4
|
-
require 'google/api_client/auth/installed_app'
|
5
|
-
require 'parallel'
|
6
|
-
|
7
|
-
require_relative 'queue'
|
8
|
-
|
9
|
-
TASKQUEUE_API = 'taskqueue'
|
10
|
-
TASKQUEUE_API_VERSION = 'v1beta2'
|
11
|
-
TASKQUEUE_API_SCOPES = ['https://www.googleapis.com/auth/taskqueue']
|
12
|
-
|
13
|
-
module TQ
|
14
|
-
|
15
|
-
DEFAULT_OPTIONS = {
|
16
|
-
'concurrency' => 2,
|
17
|
-
'log' => {
|
18
|
-
'file' => $stderr
|
19
|
-
},
|
20
|
-
'env' => {}
|
21
|
-
}
|
22
|
-
|
23
|
-
class App
|
24
|
-
|
25
|
-
attr_reader :id, :worker
|
26
|
-
def initialize(id, worker, options={})
|
27
|
-
@id = id; @worker = worker
|
28
|
-
@options = DEFAULT_OPTIONS.merge(options)
|
29
|
-
end
|
30
|
-
|
31
|
-
def options(_)
|
32
|
-
App.new @id, @worker, @options.merge(_)
|
33
|
-
end
|
34
|
-
|
35
|
-
def project(_)
|
36
|
-
options({'project' => _})
|
37
|
-
end
|
38
|
-
|
39
|
-
def log(_)
|
40
|
-
options({'log' => @options['log'].merge(_)})
|
41
|
-
end
|
42
|
-
|
43
|
-
def logger(_)
|
44
|
-
options({'logger' => _})
|
45
|
-
end
|
46
|
-
|
47
|
-
def env(_)
|
48
|
-
options({'env' => @options['env'].merge(_)})
|
49
|
-
end
|
50
|
-
|
51
|
-
def stdin(_)
|
52
|
-
return stdin({'name' => _}) if String === _
|
53
|
-
options({'stdin' => _})
|
54
|
-
end
|
55
|
-
|
56
|
-
def stdout(_)
|
57
|
-
return stdout({'name' => _}) if String === _
|
58
|
-
options({'stdout' => _})
|
59
|
-
end
|
60
|
-
|
61
|
-
def stderr(_)
|
62
|
-
return stderr({'name' => _}) if String === _
|
63
|
-
options({'stderr' => _})
|
64
|
-
end
|
65
|
-
|
66
|
-
def service_run!(issuer, p12_file)
|
67
|
-
setup_logger!
|
68
|
-
_run *(_queues( TQ::Queue.new( *(service_auth!(issuer, p12_file)) ).project(@options['project']) ) )
|
69
|
-
end
|
70
|
-
|
71
|
-
def run!(secrets_file=nil, store_file=nil)
|
72
|
-
setup_logger!
|
73
|
-
_run *(_queues( TQ::Queue.new( *(auth!(secrets_file, store_file)) ).project(@options['project']) ) )
|
74
|
-
end
|
75
|
-
|
76
|
-
# Note issuer is not a file name but the service account email address
|
77
|
-
def service_auth!(issuer, p12_file)
|
78
|
-
key = Google::APIClient::KeyUtils.load_from_pkcs12(p12_file, 'notasecret')
|
79
|
-
client.authorization = Signet::OAuth2::Client.new(
|
80
|
-
:token_credential_uri => 'https://accounts.google.com/o/oauth2/token',
|
81
|
-
:audience => 'https://accounts.google.com/o/oauth2/token',
|
82
|
-
:scope => TASKQUEUE_API_SCOPES,
|
83
|
-
:issuer => issuer,
|
84
|
-
:signing_key => key)
|
85
|
-
client.authorization.fetch_access_token!
|
86
|
-
|
87
|
-
api = client.discovered_api(TASKQUEUE_API, TASKQUEUE_API_VERSION)
|
88
|
-
|
89
|
-
return client, api
|
90
|
-
end
|
91
|
-
|
92
|
-
def auth!(secrets_file=nil, store_file=nil)
|
93
|
-
if store_file.nil? || (cred_store = credentials_store(store_file)).authorization.nil?
|
94
|
-
client_secrets = Google::APIClient::ClientSecrets.load(secrets_file)
|
95
|
-
flow = Google::APIClient::InstalledAppFlow.new(
|
96
|
-
:client_id => client_secrets.client_id,
|
97
|
-
:client_secret => client_secrets.client_secret,
|
98
|
-
:scope => TASKQUEUE_API_SCOPES
|
99
|
-
)
|
100
|
-
client.authorization = store_file.nil? ?
|
101
|
-
flow.authorize :
|
102
|
-
flow.authorize(cred_store)
|
103
|
-
else
|
104
|
-
client.authorization = cred_store.authorization
|
105
|
-
end
|
106
|
-
|
107
|
-
api = client.discovered_api(TASKQUEUE_API, TASKQUEUE_API_VERSION)
|
108
|
-
|
109
|
-
return client, api
|
110
|
-
end
|
111
|
-
|
112
|
-
def application_name
|
113
|
-
@id.split('/')[0]
|
114
|
-
end
|
115
|
-
|
116
|
-
def application_version
|
117
|
-
@id.split('/')[1] || '0.0.0'
|
118
|
-
end
|
119
|
-
|
120
|
-
private
|
121
|
-
|
122
|
-
def setup_logger!
|
123
|
-
if logger = @options['logger']
|
124
|
-
else
|
125
|
-
if (log = @options['log']) && (file = log['file'])
|
126
|
-
logger = Logger.new(file)
|
127
|
-
if level = log['level']
|
128
|
-
logger.level = level
|
129
|
-
end
|
130
|
-
end
|
131
|
-
end
|
132
|
-
(Google::APIClient.logger = logger) if logger
|
133
|
-
end
|
134
|
-
|
135
|
-
def client
|
136
|
-
@client ||= Google::APIClient.new(
|
137
|
-
:application_name => application_name,
|
138
|
-
:application_version => application_version
|
139
|
-
)
|
140
|
-
end
|
141
|
-
|
142
|
-
def credentials_store(file)
|
143
|
-
Google::APIClient::FileStorage.new(file)
|
144
|
-
end
|
145
|
-
|
146
|
-
def _queues(q)
|
147
|
-
qin = @options['stdin'] && q.options(@options['stdin'])
|
148
|
-
qout = @options['stdout'] && q.options(@options['stdout'])
|
149
|
-
qerr = @options['stderr'] && q.options(@options['stderr'])
|
150
|
-
return qin, qout, qerr
|
151
|
-
end
|
152
|
-
|
153
|
-
# TODO handle uncaught worker errors by qerr.push!(err) and qin.finish!(task)
|
154
|
-
# TODO raise if not qin
|
155
|
-
def _run(qin, qout, qerr)
|
156
|
-
tasks = qin.lease!
|
157
|
-
Parallel.each(tasks, :in_threads => @options['concurrency']) do |task|
|
158
|
-
if task.try?
|
159
|
-
@worker.new(qin, qout, qerr, inherited_env).call(task)
|
160
|
-
else
|
161
|
-
qin.finish!(task)
|
162
|
-
end
|
163
|
-
end
|
164
|
-
end
|
165
|
-
|
166
|
-
# default log/logger options into env
|
167
|
-
def inherited_env
|
168
|
-
env = @options['env']
|
169
|
-
log = @options['log']
|
170
|
-
logger = @options['logger']
|
171
|
-
{'log' => log, 'logger' => logger}.merge(env)
|
172
|
-
end
|
173
|
-
|
174
|
-
end
|
175
|
-
|
176
|
-
end
|
data/lib/tq/queue.rb
DELETED
@@ -1,161 +0,0 @@
|
|
1
|
-
require 'json'
|
2
|
-
require 'base64'
|
3
|
-
|
4
|
-
module TQ
|
5
|
-
|
6
|
-
class Queue
|
7
|
-
|
8
|
-
DEFAULT_OPTIONS = {
|
9
|
-
'lease_secs' => 60,
|
10
|
-
'num_tasks' => 1,
|
11
|
-
'max_tries' => -1
|
12
|
-
}
|
13
|
-
|
14
|
-
attr_reader :client, :api
|
15
|
-
def initialize(client, api, options={})
|
16
|
-
@client, @api = client, api
|
17
|
-
@options = DEFAULT_OPTIONS.merge(options)
|
18
|
-
end
|
19
|
-
|
20
|
-
def options(_)
|
21
|
-
Queue.new @client, @api, @options.merge(_)
|
22
|
-
end
|
23
|
-
|
24
|
-
def project(_)
|
25
|
-
options({'project' => _})
|
26
|
-
end
|
27
|
-
|
28
|
-
def name(_)
|
29
|
-
options({'name' => _})
|
30
|
-
end
|
31
|
-
|
32
|
-
def option(key)
|
33
|
-
@options[key]
|
34
|
-
end
|
35
|
-
|
36
|
-
def lease!(opts={})
|
37
|
-
opts = @options.merge(opts)
|
38
|
-
results = client.execute!(
|
39
|
-
:api_method => api.tasks.lease,
|
40
|
-
:parameters => { :leaseSecs => opts['lease_secs'],
|
41
|
-
:project => opts['project'],
|
42
|
-
:taskqueue => opts['name'],
|
43
|
-
:numTasks => opts['num_tasks']
|
44
|
-
}
|
45
|
-
)
|
46
|
-
items = (results.data && results.data['items']) || []
|
47
|
-
items.map {|t| new_task(t) }
|
48
|
-
end
|
49
|
-
|
50
|
-
# note: does not currently work; filed bug report https://code.google.com/p/googleappengine/issues/detail?id=11838
|
51
|
-
def extend!(task, secs=nil)
|
52
|
-
secs = secs.nil? ? @options['lease_secs'] : secs
|
53
|
-
opts = @options
|
54
|
-
results = client.execute!(
|
55
|
-
:api_method => api.tasks.update,
|
56
|
-
:parameters => { :newLeaseSeconds => secs,
|
57
|
-
:project => opts['project'],
|
58
|
-
:taskqueue => opts['name'],
|
59
|
-
:task => task.id
|
60
|
-
}
|
61
|
-
)
|
62
|
-
new_task(results.data)
|
63
|
-
end
|
64
|
-
|
65
|
-
def push!(payload, tag=nil)
|
66
|
-
opts = @options
|
67
|
-
body = { 'queueName' => opts['name'],
|
68
|
-
'payloadBase64' => encode(payload)
|
69
|
-
}
|
70
|
-
body['tag'] = tag if tag
|
71
|
-
|
72
|
-
results = client.execute!(
|
73
|
-
:api_method => api.tasks.insert,
|
74
|
-
:parameters => { :project => opts['project'],
|
75
|
-
:taskqueue => opts['name']
|
76
|
-
},
|
77
|
-
:body_object => body
|
78
|
-
)
|
79
|
-
new_task(results.data)
|
80
|
-
end
|
81
|
-
|
82
|
-
# note: you must have previously leased given task
|
83
|
-
def finish!(task)
|
84
|
-
opts = @options
|
85
|
-
client.execute!( :api_method => api.tasks.delete,
|
86
|
-
:parameters => { :project => opts['project'],
|
87
|
-
:taskqueue => opts['name'],
|
88
|
-
:task => task.id
|
89
|
-
}
|
90
|
-
)
|
91
|
-
return
|
92
|
-
end
|
93
|
-
|
94
|
-
private
|
95
|
-
|
96
|
-
def new_task(t)
|
97
|
-
Task.new(
|
98
|
-
self,
|
99
|
-
t['id'],
|
100
|
-
timestamp_time(t['leaseTimestamp']),
|
101
|
-
t['retry_count'],
|
102
|
-
t['tag'],
|
103
|
-
decode(t.payloadBase64),
|
104
|
-
t
|
105
|
-
)
|
106
|
-
end
|
107
|
-
|
108
|
-
def timestamp_time(t)
|
109
|
-
Time.at( t / 1000000 )
|
110
|
-
end
|
111
|
-
|
112
|
-
def encode(obj)
|
113
|
-
Base64.urlsafe_encode64(JSON.dump(obj))
|
114
|
-
end
|
115
|
-
|
116
|
-
def decode(str)
|
117
|
-
JSON.load(Base64.urlsafe_decode64(str))
|
118
|
-
end
|
119
|
-
|
120
|
-
end
|
121
|
-
|
122
|
-
class Task < Struct.new(:queue, :id, :expires, :tries, :tag, :payload, :raw)
|
123
|
-
|
124
|
-
def initialize(*args)
|
125
|
-
super
|
126
|
-
@clock = Time
|
127
|
-
end
|
128
|
-
|
129
|
-
def finish!
|
130
|
-
self.queue.finish!(self)
|
131
|
-
end
|
132
|
-
|
133
|
-
def extend!(secs=nil)
|
134
|
-
self.queue.extend!(self, secs)
|
135
|
-
end
|
136
|
-
|
137
|
-
def clock!(_)
|
138
|
-
@clock = _; return self
|
139
|
-
end
|
140
|
-
|
141
|
-
def reset_clock!
|
142
|
-
@clock = Time; return self
|
143
|
-
end
|
144
|
-
|
145
|
-
def lease_remaining
|
146
|
-
self.expires - @clock.now
|
147
|
-
end
|
148
|
-
|
149
|
-
def lease_expired?
|
150
|
-
self.expires < @clock.now
|
151
|
-
end
|
152
|
-
|
153
|
-
def try?
|
154
|
-
max = self.queue.option('max_tries')
|
155
|
-
return (max == -1 or self.tries < max)
|
156
|
-
end
|
157
|
-
|
158
|
-
end
|
159
|
-
|
160
|
-
end
|
161
|
-
|
data/lib/tq/shell.rb
DELETED
@@ -1,104 +0,0 @@
|
|
1
|
-
require 'optparse'
|
2
|
-
require 'json'
|
3
|
-
require_relative '../version'
|
4
|
-
|
5
|
-
module TQ
|
6
|
-
|
7
|
-
class Shell
|
8
|
-
|
9
|
-
DEFAULT_OPTIONS = {
|
10
|
-
app: {},
|
11
|
-
config_file: './tq-app.json'
|
12
|
-
}
|
13
|
-
|
14
|
-
def initialize(app, logger=nil)
|
15
|
-
@app = app
|
16
|
-
@logger = logger
|
17
|
-
@summary = []
|
18
|
-
end
|
19
|
-
|
20
|
-
def banner(_)
|
21
|
-
@banner = _; return self
|
22
|
-
end
|
23
|
-
|
24
|
-
def summary(*_)
|
25
|
-
@summary = _; return self
|
26
|
-
end
|
27
|
-
|
28
|
-
def call(argv=ARGV)
|
29
|
-
|
30
|
-
progname = File.basename(__FILE__,'.rb')
|
31
|
-
|
32
|
-
opts = parse_args(argv)
|
33
|
-
@logger.debug(progname) { "Configuration for #{@app.id}: #{opts.inspect}" } if @logger
|
34
|
-
|
35
|
-
@app = @app.options( opts[:app] )
|
36
|
-
@app = @app.logger(@logger) if @logger
|
37
|
-
|
38
|
-
@logger.info(progname) { "Running #{@app.id} using worker #{@app.worker}" } if @logger
|
39
|
-
|
40
|
-
secrets, store = opts[:auth_secrets_file], opts[:auth_store_file]
|
41
|
-
issuer, p12 = opts[:service_auth_issuer_file], opts[:service_auth_p12_file]
|
42
|
-
|
43
|
-
if secrets
|
44
|
-
@app.run!(secrets, store)
|
45
|
-
elsif issuer && p12
|
46
|
-
@app.service_run!(File.read(issuer).chomp, p12)
|
47
|
-
else
|
48
|
-
raise ArgumentError, "You must provide either OAuth2 secrets and credentials store, " +
|
49
|
-
"or service-account issuer and p12 files."
|
50
|
-
end
|
51
|
-
|
52
|
-
@logger.info(progname) { "Completed #{@app.id}" } if @logger
|
53
|
-
|
54
|
-
end
|
55
|
-
|
56
|
-
private
|
57
|
-
|
58
|
-
def parse_args(argv)
|
59
|
-
opts = {}.merge(DEFAULT_OPTIONS)
|
60
|
-
|
61
|
-
OptionParser.new do |shell|
|
62
|
-
|
63
|
-
(shell.banner = @banner) if @banner
|
64
|
-
@summary.each do |line|
|
65
|
-
shell.separator line
|
66
|
-
end
|
67
|
-
|
68
|
-
shell.on('-a', '--auth-secrets [FILE]', "Google OAuth2 secrets file") do |given|
|
69
|
-
opts[:auth_secrets_file] = given
|
70
|
-
end
|
71
|
-
|
72
|
-
shell.on('-s', '--auth-store [FILE]', "Google OAuth2 credentials storage file") do |given|
|
73
|
-
opts[:auth_store_file] = given
|
74
|
-
end
|
75
|
-
|
76
|
-
shell.on('-i', '--service-auth-issuer [FILE]', "Google service account issuer file") do |given|
|
77
|
-
opts[:service_auth_issuer_file] = given
|
78
|
-
end
|
79
|
-
|
80
|
-
shell.on('-p', '--service-auth-p12 [FILE]', "Google service account p12 file") do |given|
|
81
|
-
opts[:service_auth_p12_file] = given
|
82
|
-
end
|
83
|
-
|
84
|
-
shell.on('-c', '--config [FILE]', "Application config file (json)") do |given|
|
85
|
-
opts[:config_file] = given
|
86
|
-
opts[:app] = JSON.load( File.open(given, 'r') )
|
87
|
-
end
|
88
|
-
|
89
|
-
shell.on('-h', '--help', "Prints this help") do |given|
|
90
|
-
puts shell; exit
|
91
|
-
end
|
92
|
-
|
93
|
-
shell.on('-v', '--version', "Prints TQ version") do |given|
|
94
|
-
puts TQ::VERSION; exit
|
95
|
-
end
|
96
|
-
|
97
|
-
end.parse(argv)
|
98
|
-
|
99
|
-
return opts
|
100
|
-
end
|
101
|
-
|
102
|
-
end
|
103
|
-
|
104
|
-
end
|