tq 0.2.1 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tq
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Eric Gjertsen
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-08-04 00:00:00.000000000 Z
11
+ date: 2018-01-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: google-api-client
@@ -16,43 +16,43 @@ dependencies:
16
16
  requirements:
17
17
  - - '='
18
18
  - !ruby/object:Gem::Version
19
- version: 0.8.4
19
+ version: 0.19.3
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - '='
25
25
  - !ruby/object:Gem::Version
26
- version: 0.8.4
26
+ version: 0.19.3
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: minitest
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - '='
32
32
  - !ruby/object:Gem::Version
33
- version: 5.5.1
33
+ version: 5.11.1
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - '='
39
39
  - !ruby/object:Gem::Version
40
- version: 5.5.1
40
+ version: 5.11.1
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: parallel
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - '='
46
46
  - !ruby/object:Gem::Version
47
- version: 1.4.1
47
+ version: 1.12.1
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - '='
53
53
  - !ruby/object:Gem::Version
54
- version: 1.4.1
55
- description: Provides a simple framework for writing task worker processes
54
+ version: 1.12.1
55
+ description: A simple framework for writing task worker processes
56
56
  email: ericgj72@gmail.com
57
57
  executables: []
58
58
  extensions: []
@@ -60,17 +60,13 @@ extra_rdoc_files: []
60
60
  files:
61
61
  - ".gems"
62
62
  - lib/tq.rb
63
- - lib/tq/app.rb
64
63
  - lib/tq/logger.rb
65
- - lib/tq/queue.rb
66
- - lib/tq/shell.rb
67
64
  - lib/version.rb
68
65
  - test/helper.rb
69
66
  - test/suite.rb
70
67
  - test/test_auth.rb
71
68
  - test/test_logger.rb
72
69
  - test/test_run.rb
73
- - test/test_shell.rb
74
70
  homepage: https://github.com/ericgj/tq
75
71
  licenses:
76
72
  - MIT
@@ -91,8 +87,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
91
87
  version: '0'
92
88
  requirements: []
93
89
  rubyforge_project:
94
- rubygems_version: 2.5.1
90
+ rubygems_version: 2.6.13
95
91
  signing_key:
96
92
  specification_version: 4
97
- summary: Ruby client for Google App Engine TaskQueue (REST API)
93
+ summary: Ruby client for Google Cloud Tasks (REST API v2beta2)
98
94
  test_files: []
@@ -1,176 +0,0 @@
1
- require 'google/api_client'
2
- require 'google/api_client/client_secrets'
3
- require 'google/api_client/auth/file_storage'
4
- require 'google/api_client/auth/installed_app'
5
- require 'parallel'
6
-
7
- require_relative 'queue'
8
-
9
- TASKQUEUE_API = 'taskqueue'
10
- TASKQUEUE_API_VERSION = 'v1beta2'
11
- TASKQUEUE_API_SCOPES = ['https://www.googleapis.com/auth/taskqueue']
12
-
13
- module TQ
14
-
15
- DEFAULT_OPTIONS = {
16
- 'concurrency' => 2,
17
- 'log' => {
18
- 'file' => $stderr
19
- },
20
- 'env' => {}
21
- }
22
-
23
- class App
24
-
25
- attr_reader :id, :worker
26
- def initialize(id, worker, options={})
27
- @id = id; @worker = worker
28
- @options = DEFAULT_OPTIONS.merge(options)
29
- end
30
-
31
- def options(_)
32
- App.new @id, @worker, @options.merge(_)
33
- end
34
-
35
- def project(_)
36
- options({'project' => _})
37
- end
38
-
39
- def log(_)
40
- options({'log' => @options['log'].merge(_)})
41
- end
42
-
43
- def logger(_)
44
- options({'logger' => _})
45
- end
46
-
47
- def env(_)
48
- options({'env' => @options['env'].merge(_)})
49
- end
50
-
51
- def stdin(_)
52
- return stdin({'name' => _}) if String === _
53
- options({'stdin' => _})
54
- end
55
-
56
- def stdout(_)
57
- return stdout({'name' => _}) if String === _
58
- options({'stdout' => _})
59
- end
60
-
61
- def stderr(_)
62
- return stderr({'name' => _}) if String === _
63
- options({'stderr' => _})
64
- end
65
-
66
- def service_run!(issuer, p12_file)
67
- setup_logger!
68
- _run *(_queues( TQ::Queue.new( *(service_auth!(issuer, p12_file)) ).project(@options['project']) ) )
69
- end
70
-
71
- def run!(secrets_file=nil, store_file=nil)
72
- setup_logger!
73
- _run *(_queues( TQ::Queue.new( *(auth!(secrets_file, store_file)) ).project(@options['project']) ) )
74
- end
75
-
76
- # Note issuer is not a file name but the service account email address
77
- def service_auth!(issuer, p12_file)
78
- key = Google::APIClient::KeyUtils.load_from_pkcs12(p12_file, 'notasecret')
79
- client.authorization = Signet::OAuth2::Client.new(
80
- :token_credential_uri => 'https://accounts.google.com/o/oauth2/token',
81
- :audience => 'https://accounts.google.com/o/oauth2/token',
82
- :scope => TASKQUEUE_API_SCOPES,
83
- :issuer => issuer,
84
- :signing_key => key)
85
- client.authorization.fetch_access_token!
86
-
87
- api = client.discovered_api(TASKQUEUE_API, TASKQUEUE_API_VERSION)
88
-
89
- return client, api
90
- end
91
-
92
- def auth!(secrets_file=nil, store_file=nil)
93
- if store_file.nil? || (cred_store = credentials_store(store_file)).authorization.nil?
94
- client_secrets = Google::APIClient::ClientSecrets.load(secrets_file)
95
- flow = Google::APIClient::InstalledAppFlow.new(
96
- :client_id => client_secrets.client_id,
97
- :client_secret => client_secrets.client_secret,
98
- :scope => TASKQUEUE_API_SCOPES
99
- )
100
- client.authorization = store_file.nil? ?
101
- flow.authorize :
102
- flow.authorize(cred_store)
103
- else
104
- client.authorization = cred_store.authorization
105
- end
106
-
107
- api = client.discovered_api(TASKQUEUE_API, TASKQUEUE_API_VERSION)
108
-
109
- return client, api
110
- end
111
-
112
- def application_name
113
- @id.split('/')[0]
114
- end
115
-
116
- def application_version
117
- @id.split('/')[1] || '0.0.0'
118
- end
119
-
120
- private
121
-
122
- def setup_logger!
123
- if logger = @options['logger']
124
- else
125
- if (log = @options['log']) && (file = log['file'])
126
- logger = Logger.new(file)
127
- if level = log['level']
128
- logger.level = level
129
- end
130
- end
131
- end
132
- (Google::APIClient.logger = logger) if logger
133
- end
134
-
135
- def client
136
- @client ||= Google::APIClient.new(
137
- :application_name => application_name,
138
- :application_version => application_version
139
- )
140
- end
141
-
142
- def credentials_store(file)
143
- Google::APIClient::FileStorage.new(file)
144
- end
145
-
146
- def _queues(q)
147
- qin = @options['stdin'] && q.options(@options['stdin'])
148
- qout = @options['stdout'] && q.options(@options['stdout'])
149
- qerr = @options['stderr'] && q.options(@options['stderr'])
150
- return qin, qout, qerr
151
- end
152
-
153
- # TODO handle uncaught worker errors by qerr.push!(err) and qin.finish!(task)
154
- # TODO raise if not qin
155
- def _run(qin, qout, qerr)
156
- tasks = qin.lease!
157
- Parallel.each(tasks, :in_threads => @options['concurrency']) do |task|
158
- if task.try?
159
- @worker.new(qin, qout, qerr, inherited_env).call(task)
160
- else
161
- qin.finish!(task)
162
- end
163
- end
164
- end
165
-
166
- # default log/logger options into env
167
- def inherited_env
168
- env = @options['env']
169
- log = @options['log']
170
- logger = @options['logger']
171
- {'log' => log, 'logger' => logger}.merge(env)
172
- end
173
-
174
- end
175
-
176
- end
@@ -1,161 +0,0 @@
1
- require 'json'
2
- require 'base64'
3
-
4
- module TQ
5
-
6
- class Queue
7
-
8
- DEFAULT_OPTIONS = {
9
- 'lease_secs' => 60,
10
- 'num_tasks' => 1,
11
- 'max_tries' => -1
12
- }
13
-
14
- attr_reader :client, :api
15
- def initialize(client, api, options={})
16
- @client, @api = client, api
17
- @options = DEFAULT_OPTIONS.merge(options)
18
- end
19
-
20
- def options(_)
21
- Queue.new @client, @api, @options.merge(_)
22
- end
23
-
24
- def project(_)
25
- options({'project' => _})
26
- end
27
-
28
- def name(_)
29
- options({'name' => _})
30
- end
31
-
32
- def option(key)
33
- @options[key]
34
- end
35
-
36
- def lease!(opts={})
37
- opts = @options.merge(opts)
38
- results = client.execute!(
39
- :api_method => api.tasks.lease,
40
- :parameters => { :leaseSecs => opts['lease_secs'],
41
- :project => opts['project'],
42
- :taskqueue => opts['name'],
43
- :numTasks => opts['num_tasks']
44
- }
45
- )
46
- items = (results.data && results.data['items']) || []
47
- items.map {|t| new_task(t) }
48
- end
49
-
50
- # note: does not currently work; filed bug report https://code.google.com/p/googleappengine/issues/detail?id=11838
51
- def extend!(task, secs=nil)
52
- secs = secs.nil? ? @options['lease_secs'] : secs
53
- opts = @options
54
- results = client.execute!(
55
- :api_method => api.tasks.update,
56
- :parameters => { :newLeaseSeconds => secs,
57
- :project => opts['project'],
58
- :taskqueue => opts['name'],
59
- :task => task.id
60
- }
61
- )
62
- new_task(results.data)
63
- end
64
-
65
- def push!(payload, tag=nil)
66
- opts = @options
67
- body = { 'queueName' => opts['name'],
68
- 'payloadBase64' => encode(payload)
69
- }
70
- body['tag'] = tag if tag
71
-
72
- results = client.execute!(
73
- :api_method => api.tasks.insert,
74
- :parameters => { :project => opts['project'],
75
- :taskqueue => opts['name']
76
- },
77
- :body_object => body
78
- )
79
- new_task(results.data)
80
- end
81
-
82
- # note: you must have previously leased given task
83
- def finish!(task)
84
- opts = @options
85
- client.execute!( :api_method => api.tasks.delete,
86
- :parameters => { :project => opts['project'],
87
- :taskqueue => opts['name'],
88
- :task => task.id
89
- }
90
- )
91
- return
92
- end
93
-
94
- private
95
-
96
- def new_task(t)
97
- Task.new(
98
- self,
99
- t['id'],
100
- timestamp_time(t['leaseTimestamp']),
101
- t['retry_count'],
102
- t['tag'],
103
- decode(t.payloadBase64),
104
- t
105
- )
106
- end
107
-
108
- def timestamp_time(t)
109
- Time.at( t / 1000000 )
110
- end
111
-
112
- def encode(obj)
113
- Base64.urlsafe_encode64(JSON.dump(obj))
114
- end
115
-
116
- def decode(str)
117
- JSON.load(Base64.urlsafe_decode64(str))
118
- end
119
-
120
- end
121
-
122
- class Task < Struct.new(:queue, :id, :expires, :tries, :tag, :payload, :raw)
123
-
124
- def initialize(*args)
125
- super
126
- @clock = Time
127
- end
128
-
129
- def finish!
130
- self.queue.finish!(self)
131
- end
132
-
133
- def extend!(secs=nil)
134
- self.queue.extend!(self, secs)
135
- end
136
-
137
- def clock!(_)
138
- @clock = _; return self
139
- end
140
-
141
- def reset_clock!
142
- @clock = Time; return self
143
- end
144
-
145
- def lease_remaining
146
- self.expires - @clock.now
147
- end
148
-
149
- def lease_expired?
150
- self.expires < @clock.now
151
- end
152
-
153
- def try?
154
- max = self.queue.option('max_tries')
155
- return (max == -1 or self.tries < max)
156
- end
157
-
158
- end
159
-
160
- end
161
-
@@ -1,104 +0,0 @@
1
- require 'optparse'
2
- require 'json'
3
- require_relative '../version'
4
-
5
- module TQ
6
-
7
- class Shell
8
-
9
- DEFAULT_OPTIONS = {
10
- app: {},
11
- config_file: './tq-app.json'
12
- }
13
-
14
- def initialize(app, logger=nil)
15
- @app = app
16
- @logger = logger
17
- @summary = []
18
- end
19
-
20
- def banner(_)
21
- @banner = _; return self
22
- end
23
-
24
- def summary(*_)
25
- @summary = _; return self
26
- end
27
-
28
- def call(argv=ARGV)
29
-
30
- progname = File.basename(__FILE__,'.rb')
31
-
32
- opts = parse_args(argv)
33
- @logger.debug(progname) { "Configuration for #{@app.id}: #{opts.inspect}" } if @logger
34
-
35
- @app = @app.options( opts[:app] )
36
- @app = @app.logger(@logger) if @logger
37
-
38
- @logger.info(progname) { "Running #{@app.id} using worker #{@app.worker}" } if @logger
39
-
40
- secrets, store = opts[:auth_secrets_file], opts[:auth_store_file]
41
- issuer, p12 = opts[:service_auth_issuer_file], opts[:service_auth_p12_file]
42
-
43
- if secrets
44
- @app.run!(secrets, store)
45
- elsif issuer && p12
46
- @app.service_run!(File.read(issuer).chomp, p12)
47
- else
48
- raise ArgumentError, "You must provide either OAuth2 secrets and credentials store, " +
49
- "or service-account issuer and p12 files."
50
- end
51
-
52
- @logger.info(progname) { "Completed #{@app.id}" } if @logger
53
-
54
- end
55
-
56
- private
57
-
58
- def parse_args(argv)
59
- opts = {}.merge(DEFAULT_OPTIONS)
60
-
61
- OptionParser.new do |shell|
62
-
63
- (shell.banner = @banner) if @banner
64
- @summary.each do |line|
65
- shell.separator line
66
- end
67
-
68
- shell.on('-a', '--auth-secrets [FILE]', "Google OAuth2 secrets file") do |given|
69
- opts[:auth_secrets_file] = given
70
- end
71
-
72
- shell.on('-s', '--auth-store [FILE]', "Google OAuth2 credentials storage file") do |given|
73
- opts[:auth_store_file] = given
74
- end
75
-
76
- shell.on('-i', '--service-auth-issuer [FILE]', "Google service account issuer file") do |given|
77
- opts[:service_auth_issuer_file] = given
78
- end
79
-
80
- shell.on('-p', '--service-auth-p12 [FILE]', "Google service account p12 file") do |given|
81
- opts[:service_auth_p12_file] = given
82
- end
83
-
84
- shell.on('-c', '--config [FILE]', "Application config file (json)") do |given|
85
- opts[:config_file] = given
86
- opts[:app] = JSON.load( File.open(given, 'r') )
87
- end
88
-
89
- shell.on('-h', '--help', "Prints this help") do |given|
90
- puts shell; exit
91
- end
92
-
93
- shell.on('-v', '--version', "Prints TQ version") do |given|
94
- puts TQ::VERSION; exit
95
- end
96
-
97
- end.parse(argv)
98
-
99
- return opts
100
- end
101
-
102
- end
103
-
104
- end