opener-daemons 2.5.6 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. checksums.yaml +5 -5
  2. data/lib/opener/daemons.rb +3 -1
  3. data/lib/opener/daemons/configuration.rb +6 -2
  4. data/lib/opener/daemons/ext/zlib.rb +5 -0
  5. data/lib/opener/daemons/mapper.rb +2 -1
  6. data/lib/opener/daemons/version.rb +1 -1
  7. data/lib/opener/daemons/worker.rb +60 -40
  8. data/schema/inline_sqs_input.json +34 -0
  9. data/schema/sqs_input.json +4 -0
  10. metadata +36 -34
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 6424114dde7c03f70745dbbaeea26d0aa72da637
4
- data.tar.gz: 373a582d47a426c51b13d7d53c05a16d160d453e
2
+ SHA256:
3
+ metadata.gz: 5b078bce8070e4160f3be157039fe606a4b74b0f903287cd8a6badb783ebbfab
4
+ data.tar.gz: a182c4cb68d0771f00269ce735dd09b2c0a59b2e96ed9c17ef1a2977fff826fd
5
5
  SHA512:
6
- metadata.gz: d405f6c1e74fb7556004da83439f52b3d93b03fd4a12365e33a7708588d1ed9c232902c452b7d3ef3b3534f4f49476cff28358e9282aa55beed0e1152df85121
7
- data.tar.gz: 4e6c58c945a45807a515899244a9db23feabe629a0ffa3d1a9dc3d204882ad64b896faed9c34042c59889300f5a779e76f5c696b20246a8b9ce7cce4e1e79a65
6
+ metadata.gz: e52d4c74559798c937435c2f205804bb663ed26b8b9fc2fd3a9d0507c99e47d4f3af2b89bdb15e5252217fa2a23255e71b9cfbe2bc0cf2887a0862462244d7f7
7
+ data.tar.gz: 2b1da739e1e44f76657438259a20654825499ba29d0a8d70a555a1ee789260b0cf65a966fdc6c2cfbac445f423443305e56d072812fa407f8897196dbe785225
data/lib/opener/daemons.rb CHANGED
@@ -15,6 +15,8 @@ require 'opener/core'
15
15
  require 'new_relic/control'
16
16
  require 'rollbar'
17
17
 
18
+ require_relative 'daemons/ext/zlib'
19
+
18
20
  require_relative 'daemons/version'
19
21
  require_relative 'daemons/daemons'
20
22
  require_relative 'daemons/option_parser'
@@ -27,6 +29,6 @@ require_relative 'daemons/transaction'
27
29
 
28
30
  require_relative 'daemons/minio'
29
31
 
30
- require_relative 'daemons/mapper'
31
32
  require_relative 'daemons/worker'
33
+ require_relative 'daemons/mapper'
32
34
  require_relative 'daemons/daemon'
data/lib/opener/daemons/configuration.rb CHANGED
@@ -19,8 +19,11 @@ module Opener
19
19
  # @return [Hash]
20
20
  #
21
21
  class Configuration
22
- attr_reader :component, :component_options, :input_url, :callbacks,
23
- :metadata
22
+
23
+ attr_reader :component, :component_options
24
+
25
+ attr_reader :input, :input_url
26
+ attr_reader :callbacks, :metadata
24
27
 
25
28
  ##
26
29
  # @param [Class] component The component to use.
@@ -63,6 +66,7 @@ module Opener
63
66
  def component_instance
64
67
  return component.new(component_options)
65
68
  end
69
+
66
70
  end # Configuration
67
71
  end # Daemons
68
72
  end # Opener
data/lib/opener/daemons/ext/zlib.rb ADDED
@@ -0,0 +1,5 @@
1
+ if RUBY_VERSION < '2.4.0'
2
+ require 'rubygems/util'
3
+ Zlib.send :define_singleton_method, :gzip, &Gem::Util.method(:gzip)
4
+ Zlib.send :define_singleton_method, :gunzip, &Gem::Util.method(:gunzip)
5
+ end
data/lib/opener/daemons/mapper.rb CHANGED
@@ -25,7 +25,8 @@ module Opener
25
25
  #
26
26
  # @return [String]
27
27
  #
28
- INPUT_SCHEMA = File.join(SCHEMA_DIRECTORY, 'sqs_input.json')
28
+ INPUT_SCHEMA_FILE = if Worker::INLINE_IO then 'inline_sqs_input.json' else 'sqs_input.json' end
29
+ INPUT_SCHEMA = File.join SCHEMA_DIRECTORY, INPUT_SCHEMA_FILE
29
30
 
30
31
  ##
31
32
  # @param [Class] component
data/lib/opener/daemons/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  module Opener
2
2
  module Daemons
3
- VERSION = '2.5.6'
3
+ VERSION = '2.6.0'
4
4
  end # Daemons
5
5
  end # Opener
data/lib/opener/daemons/worker.rb CHANGED
@@ -20,6 +20,8 @@ module Opener
20
20
  class Worker < Oni::Worker
21
21
  attr_reader :config, :uploader, :downloader, :callback_handler
22
22
 
23
+ INLINE_IO = !!ENV['INLINE_IO']
24
+
23
25
  include NewRelic::Agent::Instrumentation::ControllerInstrumentation
24
26
  include NewRelic::Agent::MethodTracer
25
27
 
@@ -31,6 +33,8 @@ module Opener
31
33
  @downloader = Downloader.new
32
34
  @uploader = Uploader.new
33
35
  @callback_handler = CallbackHandler.new
36
+ @input = nil
37
+ @output = nil
34
38
  end
35
39
 
36
40
  ##
@@ -42,10 +46,10 @@ module Opener
42
46
  add_transaction_attributes
43
47
 
44
48
  begin
45
- output = run_component
46
- object = upload_output(output)
47
-
48
- submit_callbacks(object)
49
+ process_input
50
+ run_component
51
+ process_output
52
+ submit_callbacks
49
53
 
50
54
  # Unsupported languages are handled in a different manner as they can
51
55
  # occur quite often. In these cases we _do_ want the data to be sent
@@ -56,28 +60,33 @@ module Opener
56
60
  end
57
61
  end
58
62
 
63
+ ##
64
+ #
65
+ def process_input
66
+ if config.input
67
+ @input = Zlib.gunzip Base64.decode64 config.input
68
+ else
69
+ @input = downloader.download config.input_url
70
+ end
71
+ end
72
+
59
73
  ##
60
74
  # @return [String]
61
75
  #
62
76
  def run_component
63
- input = downloader.download(config.input_url)
64
-
65
- return config.component_instance.run(input)
77
+ @output = config.component_instance.run @input
66
78
  end
67
79
 
68
80
  ##
69
81
  # @param [String] output
70
82
  # @return [Aws::S3::Object]
71
83
  #
72
- def upload_output(output)
73
- object = uploader.upload(config.identifier, output, config.metadata)
74
-
75
- Core::Syslog.info(
76
- "Wrote output to s3://#{Daemons.output_bucket}/#{object.key}",
77
- config.metadata
78
- )
79
-
80
- return object
84
+ def process_output
85
+ if INLINE_IO
86
+ @next_input = Base64.encode64 Zlib.gzip @output
87
+ else
88
+ @object = uploader.upload config.identifier, @output, config.metadata
89
+ end
81
90
  end
82
91
 
83
92
  ##
@@ -85,17 +94,14 @@ module Opener
85
94
  #
86
95
  # @param [Aws::S3::Object] object
87
96
  #
88
- def submit_callbacks(object)
89
- urls = config.callbacks.dup
90
- next_url = urls.shift
91
- input_url = object.public_url.to_s
92
-
93
- callback_handler.post(
94
- next_url,
95
- :input_url => input_url,
96
- :identifier => config.identifier,
97
- :callbacks => urls,
98
- :metadata => config.metadata
97
+ def submit_callbacks
98
+ urls = config.callbacks.dup
99
+ next_url = urls.shift
100
+
101
+ callback_handler.post next_url, next_input_params.merge(
102
+ identifier: config.identifier,
103
+ callbacks: urls,
104
+ metadata: config.metadata,
99
105
  )
100
106
 
101
107
  Core::Syslog.info("Submitted response to #{next_url}", config.metadata)
@@ -107,11 +113,9 @@ module Opener
107
113
  def handle_unsupported_language
108
114
  last_url = config.callbacks.last
109
115
 
110
- callback_handler.post(
111
- last_url,
112
- :input_url => config.input_url,
113
- :identifier => config.identifier,
114
- :metadata => config.metadata
116
+ callback_handler.post last_url, input_params.merge(
117
+ identifier: config.identifier,
118
+ metadata: config.metadata,
115
119
  )
116
120
 
117
121
  Core::Syslog.info(
@@ -122,21 +126,37 @@ module Opener
122
126
 
123
127
  private
124
128
 
129
+ def input_params
130
+ if INLINE_IO
131
+ {input: config.input}
132
+ else
133
+ {input_url: config.input_url}
134
+ end
135
+ end
136
+
137
+ def next_input_params
138
+ if INLINE_IO
139
+ {input: @next_input}
140
+ else
141
+ {input_url: @object.public_url.to_s}
142
+ end
143
+ end
144
+
125
145
  def add_transaction_attributes
126
146
  Transaction.current.add_parameters(
127
- :input_url => config.input_url,
128
- :identifier => config.identifier,
129
- :callbacks => config.callbacks,
130
- :metadata => config.metadata
147
+ input_url: config.input_url,
148
+ identifier: config.identifier,
149
+ callbacks: config.callbacks,
150
+ metadata: config.metadata,
131
151
  )
132
152
  end
133
153
 
134
154
  if Daemons.newrelic?
135
- add_transaction_tracer(:process, :category => :task)
155
+ add_transaction_tracer :process, category: :task
136
156
 
137
- add_method_tracer(:run_component)
138
- add_method_tracer(:upload_output)
139
- add_method_tracer(:submit_callbacks)
157
+ add_method_tracer :run_component
158
+ add_method_tracer :process_output
159
+ add_method_tracer :submit_callbacks
140
160
  end
141
161
  end # Worker
142
162
  end # Daemons
data/schema/inline_sqs_input.json ADDED
@@ -0,0 +1,34 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-04/schema#",
3
+ "title": "opener-daemons",
4
+ "description": "Schema for SQS input messages",
5
+ "type": "object",
6
+ "properties": {
7
+ "input": {
8
+ "description": "A Base64 representation of the gzip compressed input",
9
+ "type": "string"
10
+ },
11
+ "input_url": {
12
+ "description": "URL to the KAF document to process",
13
+ "type": "string"
14
+ },
15
+
16
+ "identifier": {
17
+ "description": "A unique identifier to associate with the document",
18
+ "type": "string"
19
+ },
20
+
21
+ "callbacks": {
22
+ "description": "An Array of URLs to submit the result to",
23
+ "type": "array",
24
+ "minItems": 1
25
+ },
26
+
27
+ "metadata": {
28
+ "description": "Extra metadata to pass along to each output URL",
29
+ "type": "object",
30
+ "additionalProperties": true
31
+ }
32
+ },
33
+ "required": ["input", "callbacks"]
34
+ }
data/schema/sqs_input.json CHANGED
@@ -8,6 +8,10 @@
8
8
  "description": "URL to the KAF document to process",
9
9
  "type": "string"
10
10
  },
11
+ "input": {
12
+ "description": "A Base64 representation of the gzip compressed input",
13
+ "type": "string"
14
+ },
11
15
 
12
16
  "identifier": {
13
17
  "description": "A unique identifier to associate with the document",
metadata CHANGED
@@ -1,144 +1,143 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: opener-daemons
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.5.6
4
+ version: 2.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Wilco van Duinkerken
8
8
  - Olery
9
- autorequire:
9
+ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2018-02-21 00:00:00.000000000 Z
12
+ date: 2018-03-27 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
- name: aws-sdk
16
15
  requirement: !ruby/object:Gem::Requirement
17
16
  requirements:
18
17
  - - "~>"
19
18
  - !ruby/object:Gem::Version
20
19
  version: '2.0'
21
- type: :runtime
20
+ name: aws-sdk
22
21
  prerelease: false
22
+ type: :runtime
23
23
  version_requirements: !ruby/object:Gem::Requirement
24
24
  requirements:
25
25
  - - "~>"
26
26
  - !ruby/object:Gem::Version
27
27
  version: '2.0'
28
28
  - !ruby/object:Gem::Dependency
29
- name: slop
30
29
  requirement: !ruby/object:Gem::Requirement
31
30
  requirements:
32
31
  - - "~>"
33
32
  - !ruby/object:Gem::Version
34
33
  version: '3.0'
35
- type: :runtime
34
+ name: slop
36
35
  prerelease: false
36
+ type: :runtime
37
37
  version_requirements: !ruby/object:Gem::Requirement
38
38
  requirements:
39
39
  - - "~>"
40
40
  - !ruby/object:Gem::Version
41
41
  version: '3.0'
42
42
  - !ruby/object:Gem::Dependency
43
- name: opener-callback-handler
44
43
  requirement: !ruby/object:Gem::Requirement
45
44
  requirements:
46
45
  - - "~>"
47
46
  - !ruby/object:Gem::Version
48
47
  version: '1.1'
49
- type: :runtime
48
+ name: opener-callback-handler
50
49
  prerelease: false
50
+ type: :runtime
51
51
  version_requirements: !ruby/object:Gem::Requirement
52
52
  requirements:
53
53
  - - "~>"
54
54
  - !ruby/object:Gem::Version
55
55
  version: '1.1'
56
56
  - !ruby/object:Gem::Dependency
57
- name: opener-core
58
57
  requirement: !ruby/object:Gem::Requirement
59
58
  requirements:
60
59
  - - "~>"
61
60
  - !ruby/object:Gem::Version
62
61
  version: '2.3'
63
- type: :runtime
62
+ name: opener-core
64
63
  prerelease: false
64
+ type: :runtime
65
65
  version_requirements: !ruby/object:Gem::Requirement
66
66
  requirements:
67
67
  - - "~>"
68
68
  - !ruby/object:Gem::Version
69
69
  version: '2.3'
70
70
  - !ruby/object:Gem::Dependency
71
- name: newrelic_rpm
72
71
  requirement: !ruby/object:Gem::Requirement
73
72
  requirements:
74
73
  - - ">="
75
74
  - !ruby/object:Gem::Version
76
75
  version: '0'
77
- type: :runtime
76
+ name: newrelic_rpm
78
77
  prerelease: false
78
+ type: :runtime
79
79
  version_requirements: !ruby/object:Gem::Requirement
80
80
  requirements:
81
81
  - - ">="
82
82
  - !ruby/object:Gem::Version
83
83
  version: '0'
84
84
  - !ruby/object:Gem::Dependency
85
- name: json-schema
86
85
  requirement: !ruby/object:Gem::Requirement
87
86
  requirements:
88
87
  - - ">="
89
88
  - !ruby/object:Gem::Version
90
89
  version: '0'
91
- type: :runtime
90
+ name: json-schema
92
91
  prerelease: false
92
+ type: :runtime
93
93
  version_requirements: !ruby/object:Gem::Requirement
94
94
  requirements:
95
95
  - - ">="
96
96
  - !ruby/object:Gem::Version
97
97
  version: '0'
98
98
  - !ruby/object:Gem::Dependency
99
- name: rollbar
100
99
  requirement: !ruby/object:Gem::Requirement
101
100
  requirements:
102
101
  - - "~>"
103
102
  - !ruby/object:Gem::Version
104
103
  version: '1.0'
105
- type: :runtime
104
+ name: rollbar
106
105
  prerelease: false
106
+ type: :runtime
107
107
  version_requirements: !ruby/object:Gem::Requirement
108
108
  requirements:
109
109
  - - "~>"
110
110
  - !ruby/object:Gem::Version
111
111
  version: '1.0'
112
112
  - !ruby/object:Gem::Dependency
113
- name: oni
114
113
  requirement: !ruby/object:Gem::Requirement
115
114
  requirements:
116
115
  - - "~>"
117
116
  - !ruby/object:Gem::Version
118
117
  version: '4.0'
119
- type: :runtime
118
+ name: oni
120
119
  prerelease: false
120
+ type: :runtime
121
121
  version_requirements: !ruby/object:Gem::Requirement
122
122
  requirements:
123
123
  - - "~>"
124
124
  - !ruby/object:Gem::Version
125
125
  version: '4.0'
126
126
  - !ruby/object:Gem::Dependency
127
- name: oga
128
127
  requirement: !ruby/object:Gem::Requirement
129
128
  requirements:
130
129
  - - "~>"
131
130
  - !ruby/object:Gem::Version
132
131
  version: '1.0'
133
- type: :runtime
132
+ name: oga
134
133
  prerelease: false
134
+ type: :runtime
135
135
  version_requirements: !ruby/object:Gem::Requirement
136
136
  requirements:
137
137
  - - "~>"
138
138
  - !ruby/object:Gem::Version
139
139
  version: '1.0'
140
140
  - !ruby/object:Gem::Dependency
141
- name: httpclient
142
141
  requirement: !ruby/object:Gem::Requirement
143
142
  requirements:
144
143
  - - "~>"
@@ -147,8 +146,9 @@ dependencies:
147
146
  - - ">="
148
147
  - !ruby/object:Gem::Version
149
148
  version: 2.5.3.3
150
- type: :runtime
149
+ name: httpclient
151
150
  prerelease: false
151
+ type: :runtime
152
152
  version_requirements: !ruby/object:Gem::Requirement
153
153
  requirements:
154
154
  - - "~>"
@@ -158,56 +158,56 @@ dependencies:
158
158
  - !ruby/object:Gem::Version
159
159
  version: 2.5.3.3
160
160
  - !ruby/object:Gem::Dependency
161
- name: pry
162
161
  requirement: !ruby/object:Gem::Requirement
163
162
  requirements:
164
163
  - - ">="
165
164
  - !ruby/object:Gem::Version
166
165
  version: '0'
167
- type: :development
166
+ name: pry
168
167
  prerelease: false
168
+ type: :development
169
169
  version_requirements: !ruby/object:Gem::Requirement
170
170
  requirements:
171
171
  - - ">="
172
172
  - !ruby/object:Gem::Version
173
173
  version: '0'
174
174
  - !ruby/object:Gem::Dependency
175
- name: bundler
176
175
  requirement: !ruby/object:Gem::Requirement
177
176
  requirements:
178
177
  - - ">="
179
178
  - !ruby/object:Gem::Version
180
179
  version: '0'
181
- type: :development
180
+ name: bundler
182
181
  prerelease: false
182
+ type: :development
183
183
  version_requirements: !ruby/object:Gem::Requirement
184
184
  requirements:
185
185
  - - ">="
186
186
  - !ruby/object:Gem::Version
187
187
  version: '0'
188
188
  - !ruby/object:Gem::Dependency
189
- name: rake
190
189
  requirement: !ruby/object:Gem::Requirement
191
190
  requirements:
192
191
  - - ">="
193
192
  - !ruby/object:Gem::Version
194
193
  version: '0'
195
- type: :development
194
+ name: rake
196
195
  prerelease: false
196
+ type: :development
197
197
  version_requirements: !ruby/object:Gem::Requirement
198
198
  requirements:
199
199
  - - ">="
200
200
  - !ruby/object:Gem::Version
201
201
  version: '0'
202
202
  - !ruby/object:Gem::Dependency
203
- name: rspec
204
203
  requirement: !ruby/object:Gem::Requirement
205
204
  requirements:
206
205
  - - ">="
207
206
  - !ruby/object:Gem::Version
208
207
  version: '0'
209
- type: :development
208
+ name: rspec
210
209
  prerelease: false
210
+ type: :development
211
211
  version_requirements: !ruby/object:Gem::Requirement
212
212
  requirements:
213
213
  - - ">="
@@ -230,6 +230,7 @@ files:
230
230
  - lib/opener/daemons/daemon.rb
231
231
  - lib/opener/daemons/daemons.rb
232
232
  - lib/opener/daemons/downloader.rb
233
+ - lib/opener/daemons/ext/zlib.rb
233
234
  - lib/opener/daemons/mapper.rb
234
235
  - lib/opener/daemons/minio.rb
235
236
  - lib/opener/daemons/option_parser.rb
@@ -239,12 +240,13 @@ files:
239
240
  - lib/opener/daemons/version.rb
240
241
  - lib/opener/daemons/worker.rb
241
242
  - opener-daemons.gemspec
243
+ - schema/inline_sqs_input.json
242
244
  - schema/sqs_input.json
243
245
  homepage: http://opener-project.github.io
244
246
  licenses:
245
247
  - Apache 2.0
246
248
  metadata: {}
247
- post_install_message:
249
+ post_install_message:
248
250
  rdoc_options: []
249
251
  require_paths:
250
252
  - lib
@@ -259,9 +261,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
259
261
  - !ruby/object:Gem::Version
260
262
  version: '0'
261
263
  requirements: []
262
- rubyforge_project:
264
+ rubyforge_project:
263
265
  rubygems_version: 2.6.13
264
- signing_key:
266
+ signing_key:
265
267
  specification_version: 4
266
268
  summary: Toolkit for turning OpeNER components into daemons
267
269
  test_files: []