opener-daemons 2.5.6 → 2.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (10) hide show
  1. checksums.yaml +5 -5
  2. data/lib/opener/daemons.rb +3 -1
  3. data/lib/opener/daemons/configuration.rb +6 -2
  4. data/lib/opener/daemons/ext/zlib.rb +5 -0
  5. data/lib/opener/daemons/mapper.rb +2 -1
  6. data/lib/opener/daemons/version.rb +1 -1
  7. data/lib/opener/daemons/worker.rb +60 -40
  8. data/schema/inline_sqs_input.json +34 -0
  9. data/schema/sqs_input.json +4 -0
  10. metadata +36 -34
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 6424114dde7c03f70745dbbaeea26d0aa72da637
4
- data.tar.gz: 373a582d47a426c51b13d7d53c05a16d160d453e
2
+ SHA256:
3
+ metadata.gz: 5b078bce8070e4160f3be157039fe606a4b74b0f903287cd8a6badb783ebbfab
4
+ data.tar.gz: a182c4cb68d0771f00269ce735dd09b2c0a59b2e96ed9c17ef1a2977fff826fd
5
5
  SHA512:
6
- metadata.gz: d405f6c1e74fb7556004da83439f52b3d93b03fd4a12365e33a7708588d1ed9c232902c452b7d3ef3b3534f4f49476cff28358e9282aa55beed0e1152df85121
7
- data.tar.gz: 4e6c58c945a45807a515899244a9db23feabe629a0ffa3d1a9dc3d204882ad64b896faed9c34042c59889300f5a779e76f5c696b20246a8b9ce7cce4e1e79a65
6
+ metadata.gz: e52d4c74559798c937435c2f205804bb663ed26b8b9fc2fd3a9d0507c99e47d4f3af2b89bdb15e5252217fa2a23255e71b9cfbe2bc0cf2887a0862462244d7f7
7
+ data.tar.gz: 2b1da739e1e44f76657438259a20654825499ba29d0a8d70a555a1ee789260b0cf65a966fdc6c2cfbac445f423443305e56d072812fa407f8897196dbe785225
data/lib/opener/daemons.rb CHANGED
@@ -15,6 +15,8 @@ require 'opener/core'
15
15
  require 'new_relic/control'
16
16
  require 'rollbar'
17
17
 
18
+ require_relative 'daemons/ext/zlib'
19
+
18
20
  require_relative 'daemons/version'
19
21
  require_relative 'daemons/daemons'
20
22
  require_relative 'daemons/option_parser'
@@ -27,6 +29,6 @@ require_relative 'daemons/transaction'
27
29
 
28
30
  require_relative 'daemons/minio'
29
31
 
30
- require_relative 'daemons/mapper'
31
32
  require_relative 'daemons/worker'
33
+ require_relative 'daemons/mapper'
32
34
  require_relative 'daemons/daemon'
data/lib/opener/daemons/configuration.rb CHANGED
@@ -19,8 +19,11 @@ module Opener
19
19
  # @return [Hash]
20
20
  #
21
21
  class Configuration
22
- attr_reader :component, :component_options, :input_url, :callbacks,
23
- :metadata
22
+
23
+ attr_reader :component, :component_options
24
+
25
+ attr_reader :input, :input_url
26
+ attr_reader :callbacks, :metadata
24
27
 
25
28
  ##
26
29
  # @param [Class] component The component to use.
@@ -63,6 +66,7 @@ module Opener
63
66
  def component_instance
64
67
  return component.new(component_options)
65
68
  end
69
+
66
70
  end # Configuration
67
71
  end # Daemons
68
72
  end # Opener
data/lib/opener/daemons/ext/zlib.rb ADDED
@@ -0,0 +1,5 @@
1
+ if RUBY_VERSION < '2.4.0'
2
+ require 'rubygems/util'
3
+ Zlib.send :define_singleton_method, :gzip, &Gem::Util.method(:gzip)
4
+ Zlib.send :define_singleton_method, :gunzip, &Gem::Util.method(:gunzip)
5
+ end
data/lib/opener/daemons/mapper.rb CHANGED
@@ -25,7 +25,8 @@ module Opener
25
25
  #
26
26
  # @return [String]
27
27
  #
28
- INPUT_SCHEMA = File.join(SCHEMA_DIRECTORY, 'sqs_input.json')
28
+ INPUT_SCHEMA_FILE = if Worker::INLINE_IO then 'inline_sqs_input.json' else 'sqs_input.json' end
29
+ INPUT_SCHEMA = File.join SCHEMA_DIRECTORY, INPUT_SCHEMA_FILE
29
30
 
30
31
  ##
31
32
  # @param [Class] component
data/lib/opener/daemons/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  module Opener
2
2
  module Daemons
3
- VERSION = '2.5.6'
3
+ VERSION = '2.6.0'
4
4
  end # Daemons
5
5
  end # Opener
data/lib/opener/daemons/worker.rb CHANGED
@@ -20,6 +20,8 @@ module Opener
20
20
  class Worker < Oni::Worker
21
21
  attr_reader :config, :uploader, :downloader, :callback_handler
22
22
 
23
+ INLINE_IO = !!ENV['INLINE_IO']
24
+
23
25
  include NewRelic::Agent::Instrumentation::ControllerInstrumentation
24
26
  include NewRelic::Agent::MethodTracer
25
27
 
@@ -31,6 +33,8 @@ module Opener
31
33
  @downloader = Downloader.new
32
34
  @uploader = Uploader.new
33
35
  @callback_handler = CallbackHandler.new
36
+ @input = nil
37
+ @output = nil
34
38
  end
35
39
 
36
40
  ##
@@ -42,10 +46,10 @@ module Opener
42
46
  add_transaction_attributes
43
47
 
44
48
  begin
45
- output = run_component
46
- object = upload_output(output)
47
-
48
- submit_callbacks(object)
49
+ process_input
50
+ run_component
51
+ process_output
52
+ submit_callbacks
49
53
 
50
54
  # Unsupported languages are handled in a different manner as they can
51
55
  # occur quite often. In these cases we _do_ want the data to be sent
@@ -56,28 +60,33 @@ module Opener
56
60
  end
57
61
  end
58
62
 
63
+ ##
64
+ #
65
+ def process_input
66
+ if config.input
67
+ @input = Zlib.gunzip Base64.decode64 config.input
68
+ else
69
+ @input = downloader.download config.input_url
70
+ end
71
+ end
72
+
59
73
  ##
60
74
  # @return [String]
61
75
  #
62
76
  def run_component
63
- input = downloader.download(config.input_url)
64
-
65
- return config.component_instance.run(input)
77
+ @output = config.component_instance.run @input
66
78
  end
67
79
 
68
80
  ##
69
81
  # @param [String] output
70
82
  # @return [Aws::S3::Object]
71
83
  #
72
- def upload_output(output)
73
- object = uploader.upload(config.identifier, output, config.metadata)
74
-
75
- Core::Syslog.info(
76
- "Wrote output to s3://#{Daemons.output_bucket}/#{object.key}",
77
- config.metadata
78
- )
79
-
80
- return object
84
+ def process_output
85
+ if INLINE_IO
86
+ @next_input = Base64.encode64 Zlib.gzip @output
87
+ else
88
+ @object = uploader.upload config.identifier, @output, config.metadata
89
+ end
81
90
  end
82
91
 
83
92
  ##
@@ -85,17 +94,14 @@ module Opener
85
94
  #
86
95
  # @param [Aws::S3::Object] object
87
96
  #
88
- def submit_callbacks(object)
89
- urls = config.callbacks.dup
90
- next_url = urls.shift
91
- input_url = object.public_url.to_s
92
-
93
- callback_handler.post(
94
- next_url,
95
- :input_url => input_url,
96
- :identifier => config.identifier,
97
- :callbacks => urls,
98
- :metadata => config.metadata
97
+ def submit_callbacks
98
+ urls = config.callbacks.dup
99
+ next_url = urls.shift
100
+
101
+ callback_handler.post next_url, next_input_params.merge(
102
+ identifier: config.identifier,
103
+ callbacks: urls,
104
+ metadata: config.metadata,
99
105
  )
100
106
 
101
107
  Core::Syslog.info("Submitted response to #{next_url}", config.metadata)
@@ -107,11 +113,9 @@ module Opener
107
113
  def handle_unsupported_language
108
114
  last_url = config.callbacks.last
109
115
 
110
- callback_handler.post(
111
- last_url,
112
- :input_url => config.input_url,
113
- :identifier => config.identifier,
114
- :metadata => config.metadata
116
+ callback_handler.post last_url, input_params.merge(
117
+ identifier: config.identifier,
118
+ metadata: config.metadata,
115
119
  )
116
120
 
117
121
  Core::Syslog.info(
@@ -122,21 +126,37 @@ module Opener
122
126
 
123
127
  private
124
128
 
129
+ def input_params
130
+ if INLINE_IO
131
+ {input: config.input}
132
+ else
133
+ {input_url: config.input_url}
134
+ end
135
+ end
136
+
137
+ def next_input_params
138
+ if INLINE_IO
139
+ {input: @next_input}
140
+ else
141
+ {input_url: @object.public_url.to_s}
142
+ end
143
+ end
144
+
125
145
  def add_transaction_attributes
126
146
  Transaction.current.add_parameters(
127
- :input_url => config.input_url,
128
- :identifier => config.identifier,
129
- :callbacks => config.callbacks,
130
- :metadata => config.metadata
147
+ input_url: config.input_url,
148
+ identifier: config.identifier,
149
+ callbacks: config.callbacks,
150
+ metadata: config.metadata,
131
151
  )
132
152
  end
133
153
 
134
154
  if Daemons.newrelic?
135
- add_transaction_tracer(:process, :category => :task)
155
+ add_transaction_tracer :process, category: :task
136
156
 
137
- add_method_tracer(:run_component)
138
- add_method_tracer(:upload_output)
139
- add_method_tracer(:submit_callbacks)
157
+ add_method_tracer :run_component
158
+ add_method_tracer :process_output
159
+ add_method_tracer :submit_callbacks
140
160
  end
141
161
  end # Worker
142
162
  end # Daemons
data/schema/inline_sqs_input.json ADDED
@@ -0,0 +1,34 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-04/schema#",
3
+ "title": "opener-daemons",
4
+ "description": "Schema for SQS input messages",
5
+ "type": "object",
6
+ "properties": {
7
+ "input": {
8
+ "description": "A Base64 representation of the gzip compressed input",
9
+ "type": "string"
10
+ },
11
+ "input_url": {
12
+ "description": "URL to the KAF document to process",
13
+ "type": "string"
14
+ },
15
+
16
+ "identifier": {
17
+ "description": "A unique identifier to associate with the document",
18
+ "type": "string"
19
+ },
20
+
21
+ "callbacks": {
22
+ "description": "An Array of URLs to submit the result to",
23
+ "type": "array",
24
+ "minItems": 1
25
+ },
26
+
27
+ "metadata": {
28
+ "description": "Extra metadata to pass along to each output URL",
29
+ "type": "object",
30
+ "additionalProperties": true
31
+ }
32
+ },
33
+ "required": ["input", "callbacks"]
34
+ }
data/schema/sqs_input.json CHANGED
@@ -8,6 +8,10 @@
8
8
  "description": "URL to the KAF document to process",
9
9
  "type": "string"
10
10
  },
11
+ "input": {
12
+ "description": "A Base64 representation of the gzip compressed input",
13
+ "type": "string"
14
+ },
11
15
 
12
16
  "identifier": {
13
17
  "description": "A unique identifier to associate with the document",
metadata CHANGED
@@ -1,144 +1,143 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: opener-daemons
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.5.6
4
+ version: 2.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Wilco van Duinkerken
8
8
  - Olery
9
- autorequire:
9
+ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2018-02-21 00:00:00.000000000 Z
12
+ date: 2018-03-27 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
- name: aws-sdk
16
15
  requirement: !ruby/object:Gem::Requirement
17
16
  requirements:
18
17
  - - "~>"
19
18
  - !ruby/object:Gem::Version
20
19
  version: '2.0'
21
- type: :runtime
20
+ name: aws-sdk
22
21
  prerelease: false
22
+ type: :runtime
23
23
  version_requirements: !ruby/object:Gem::Requirement
24
24
  requirements:
25
25
  - - "~>"
26
26
  - !ruby/object:Gem::Version
27
27
  version: '2.0'
28
28
  - !ruby/object:Gem::Dependency
29
- name: slop
30
29
  requirement: !ruby/object:Gem::Requirement
31
30
  requirements:
32
31
  - - "~>"
33
32
  - !ruby/object:Gem::Version
34
33
  version: '3.0'
35
- type: :runtime
34
+ name: slop
36
35
  prerelease: false
36
+ type: :runtime
37
37
  version_requirements: !ruby/object:Gem::Requirement
38
38
  requirements:
39
39
  - - "~>"
40
40
  - !ruby/object:Gem::Version
41
41
  version: '3.0'
42
42
  - !ruby/object:Gem::Dependency
43
- name: opener-callback-handler
44
43
  requirement: !ruby/object:Gem::Requirement
45
44
  requirements:
46
45
  - - "~>"
47
46
  - !ruby/object:Gem::Version
48
47
  version: '1.1'
49
- type: :runtime
48
+ name: opener-callback-handler
50
49
  prerelease: false
50
+ type: :runtime
51
51
  version_requirements: !ruby/object:Gem::Requirement
52
52
  requirements:
53
53
  - - "~>"
54
54
  - !ruby/object:Gem::Version
55
55
  version: '1.1'
56
56
  - !ruby/object:Gem::Dependency
57
- name: opener-core
58
57
  requirement: !ruby/object:Gem::Requirement
59
58
  requirements:
60
59
  - - "~>"
61
60
  - !ruby/object:Gem::Version
62
61
  version: '2.3'
63
- type: :runtime
62
+ name: opener-core
64
63
  prerelease: false
64
+ type: :runtime
65
65
  version_requirements: !ruby/object:Gem::Requirement
66
66
  requirements:
67
67
  - - "~>"
68
68
  - !ruby/object:Gem::Version
69
69
  version: '2.3'
70
70
  - !ruby/object:Gem::Dependency
71
- name: newrelic_rpm
72
71
  requirement: !ruby/object:Gem::Requirement
73
72
  requirements:
74
73
  - - ">="
75
74
  - !ruby/object:Gem::Version
76
75
  version: '0'
77
- type: :runtime
76
+ name: newrelic_rpm
78
77
  prerelease: false
78
+ type: :runtime
79
79
  version_requirements: !ruby/object:Gem::Requirement
80
80
  requirements:
81
81
  - - ">="
82
82
  - !ruby/object:Gem::Version
83
83
  version: '0'
84
84
  - !ruby/object:Gem::Dependency
85
- name: json-schema
86
85
  requirement: !ruby/object:Gem::Requirement
87
86
  requirements:
88
87
  - - ">="
89
88
  - !ruby/object:Gem::Version
90
89
  version: '0'
91
- type: :runtime
90
+ name: json-schema
92
91
  prerelease: false
92
+ type: :runtime
93
93
  version_requirements: !ruby/object:Gem::Requirement
94
94
  requirements:
95
95
  - - ">="
96
96
  - !ruby/object:Gem::Version
97
97
  version: '0'
98
98
  - !ruby/object:Gem::Dependency
99
- name: rollbar
100
99
  requirement: !ruby/object:Gem::Requirement
101
100
  requirements:
102
101
  - - "~>"
103
102
  - !ruby/object:Gem::Version
104
103
  version: '1.0'
105
- type: :runtime
104
+ name: rollbar
106
105
  prerelease: false
106
+ type: :runtime
107
107
  version_requirements: !ruby/object:Gem::Requirement
108
108
  requirements:
109
109
  - - "~>"
110
110
  - !ruby/object:Gem::Version
111
111
  version: '1.0'
112
112
  - !ruby/object:Gem::Dependency
113
- name: oni
114
113
  requirement: !ruby/object:Gem::Requirement
115
114
  requirements:
116
115
  - - "~>"
117
116
  - !ruby/object:Gem::Version
118
117
  version: '4.0'
119
- type: :runtime
118
+ name: oni
120
119
  prerelease: false
120
+ type: :runtime
121
121
  version_requirements: !ruby/object:Gem::Requirement
122
122
  requirements:
123
123
  - - "~>"
124
124
  - !ruby/object:Gem::Version
125
125
  version: '4.0'
126
126
  - !ruby/object:Gem::Dependency
127
- name: oga
128
127
  requirement: !ruby/object:Gem::Requirement
129
128
  requirements:
130
129
  - - "~>"
131
130
  - !ruby/object:Gem::Version
132
131
  version: '1.0'
133
- type: :runtime
132
+ name: oga
134
133
  prerelease: false
134
+ type: :runtime
135
135
  version_requirements: !ruby/object:Gem::Requirement
136
136
  requirements:
137
137
  - - "~>"
138
138
  - !ruby/object:Gem::Version
139
139
  version: '1.0'
140
140
  - !ruby/object:Gem::Dependency
141
- name: httpclient
142
141
  requirement: !ruby/object:Gem::Requirement
143
142
  requirements:
144
143
  - - "~>"
@@ -147,8 +146,9 @@ dependencies:
147
146
  - - ">="
148
147
  - !ruby/object:Gem::Version
149
148
  version: 2.5.3.3
150
- type: :runtime
149
+ name: httpclient
151
150
  prerelease: false
151
+ type: :runtime
152
152
  version_requirements: !ruby/object:Gem::Requirement
153
153
  requirements:
154
154
  - - "~>"
@@ -158,56 +158,56 @@ dependencies:
158
158
  - !ruby/object:Gem::Version
159
159
  version: 2.5.3.3
160
160
  - !ruby/object:Gem::Dependency
161
- name: pry
162
161
  requirement: !ruby/object:Gem::Requirement
163
162
  requirements:
164
163
  - - ">="
165
164
  - !ruby/object:Gem::Version
166
165
  version: '0'
167
- type: :development
166
+ name: pry
168
167
  prerelease: false
168
+ type: :development
169
169
  version_requirements: !ruby/object:Gem::Requirement
170
170
  requirements:
171
171
  - - ">="
172
172
  - !ruby/object:Gem::Version
173
173
  version: '0'
174
174
  - !ruby/object:Gem::Dependency
175
- name: bundler
176
175
  requirement: !ruby/object:Gem::Requirement
177
176
  requirements:
178
177
  - - ">="
179
178
  - !ruby/object:Gem::Version
180
179
  version: '0'
181
- type: :development
180
+ name: bundler
182
181
  prerelease: false
182
+ type: :development
183
183
  version_requirements: !ruby/object:Gem::Requirement
184
184
  requirements:
185
185
  - - ">="
186
186
  - !ruby/object:Gem::Version
187
187
  version: '0'
188
188
  - !ruby/object:Gem::Dependency
189
- name: rake
190
189
  requirement: !ruby/object:Gem::Requirement
191
190
  requirements:
192
191
  - - ">="
193
192
  - !ruby/object:Gem::Version
194
193
  version: '0'
195
- type: :development
194
+ name: rake
196
195
  prerelease: false
196
+ type: :development
197
197
  version_requirements: !ruby/object:Gem::Requirement
198
198
  requirements:
199
199
  - - ">="
200
200
  - !ruby/object:Gem::Version
201
201
  version: '0'
202
202
  - !ruby/object:Gem::Dependency
203
- name: rspec
204
203
  requirement: !ruby/object:Gem::Requirement
205
204
  requirements:
206
205
  - - ">="
207
206
  - !ruby/object:Gem::Version
208
207
  version: '0'
209
- type: :development
208
+ name: rspec
210
209
  prerelease: false
210
+ type: :development
211
211
  version_requirements: !ruby/object:Gem::Requirement
212
212
  requirements:
213
213
  - - ">="
@@ -230,6 +230,7 @@ files:
230
230
  - lib/opener/daemons/daemon.rb
231
231
  - lib/opener/daemons/daemons.rb
232
232
  - lib/opener/daemons/downloader.rb
233
+ - lib/opener/daemons/ext/zlib.rb
233
234
  - lib/opener/daemons/mapper.rb
234
235
  - lib/opener/daemons/minio.rb
235
236
  - lib/opener/daemons/option_parser.rb
@@ -239,12 +240,13 @@ files:
239
240
  - lib/opener/daemons/version.rb
240
241
  - lib/opener/daemons/worker.rb
241
242
  - opener-daemons.gemspec
243
+ - schema/inline_sqs_input.json
242
244
  - schema/sqs_input.json
243
245
  homepage: http://opener-project.github.io
244
246
  licenses:
245
247
  - Apache 2.0
246
248
  metadata: {}
247
- post_install_message:
249
+ post_install_message:
248
250
  rdoc_options: []
249
251
  require_paths:
250
252
  - lib
@@ -259,9 +261,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
259
261
  - !ruby/object:Gem::Version
260
262
  version: '0'
261
263
  requirements: []
262
- rubyforge_project:
264
+ rubyforge_project:
263
265
  rubygems_version: 2.6.13
264
- signing_key:
266
+ signing_key:
265
267
  specification_version: 4
266
268
  summary: Toolkit for turning OpeNER components into daemons
267
269
  test_files: []