opener-chained-daemon 1.0.2 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 529591ceec3fc6900d961d97e5064647afacf3c6216d59a36d339b4b17f86ce6
4
- data.tar.gz: cfb6977e833c25e30644684cc60b2c6a487f1e2e5fbd3a5076d6fd0f29b4ffa5
3
+ metadata.gz: fe6f23e32f84727ec3ea06338fc95913cbdf08c91c70d5095a0dc9d5395f2f04
4
+ data.tar.gz: 11e53fde615b379336bda63c3198c05cb8ec9ebbdabd999013df1cf31799c047
5
5
  SHA512:
6
- metadata.gz: b06587c11fcedbc69802bbed5c823504456e21b69c0c86dc5083bffcde95a9b77c0201d778668da4c47e21d705421e37c8aa1d60626aa8c6a1ab004a1f4d3f33
7
- data.tar.gz: 541dbb5b3e944cea173b1bed70561fb4b3aae1a9f8146a8f797b2fc0c6d2a5a83567a5ddce06a2910d9e5c5d242baa4e615960a81b1c24db0c557cf5b98ccd79
6
+ metadata.gz: 196281f0e1e89b3d9272d568f2a4d67034a08b3819e4cc9748873c601cab8fab87facf7523989b0f9fca2d3ab08cfbbe0561d01c7015e8963ef9c03481bf8b3e
7
+ data.tar.gz: 736a7790ede035c155278adb8b67ec1fa89e3bdf9b48a7bb7349e64c44a3b749ecbc15bd8d9129f9fd167405819627ad8056a113ed6b964500cd3cd50251033f
@@ -0,0 +1,41 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler/setup'
4
+ require_relative '../lib/opener/chained_daemon'
5
+ require_relative '../../kaf-processor/lib/kaf_processor/exts/oga'
6
+ require_relative '../../kaf-processor/lib/kaf_processor/services/string_cleaner'
7
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/document'
8
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/property'
9
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/sentiment'
10
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/term'
11
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/opinion'
12
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/word_form'
13
+ require_relative '../../kaf-processor/lib/kaf_processor/configuration'
14
+ require_relative '../../kaf-processor/lib/kaf_processor/processor/review_opinions'
15
+ require 'csv'
16
+
17
+ if ARGV.size != 1
18
+ puts 'USAGE: chained-daemon-csv file.csv'
19
+ exit
20
+ end
21
+
22
+
23
+ headers = [:review_id, :property, :polarity, :strength, :comment]
24
+ out_csv = CSV.open('/tmp/opinions.csv', 'w', write_headers: true, headers: headers)
25
+
26
+ CSV.foreach(ARGV.first, headers: true).each do |review|
27
+ #puts review['comment']
28
+ cd = Opener::ChainedDaemon.new
29
+ output = cd.run(review['comment'])
30
+ document = KafProcessor::KAF::Document.from_xml output
31
+ config = KafProcessor::Configuration.new identifier: review['id']
32
+ opinions = KafProcessor::Processor::ReviewOpinions.new(config, document).generate_rows
33
+ opinions.each do |o|
34
+ out_csv << [o[:review_id], o[:properties].join(', '), o[:polarity], o[:strength], review['comment']]
35
+ end
36
+ if opinions.empty?
37
+ out_csv << [review['id'], '', '', '', review['comment']]
38
+ end
39
+ end
40
+
41
+ out_csv.close
@@ -0,0 +1,13 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler/setup'
4
+ require_relative '../lib/opener/chained_daemon'
5
+
6
+ begin
7
+ require 'awesome_print'
8
+ require 'pry'
9
+ binding.pry
10
+ rescue LoadError
11
+ require 'irb'
12
+ IRB.start
13
+ end
@@ -1,12 +1,13 @@
1
1
  require 'active_support/all'
2
2
  require 'oga'
3
+ require 'hashie'
4
+ require 'google/cloud/translate'
3
5
 
4
6
  require 'opener/daemons'
5
7
 
6
8
  require 'opener/language_identifier'
7
9
  require 'opener/tokenizer'
8
10
  # require ner before pos_tagger for compatible opennlp
9
- require 'opener/ner'
10
11
  require 'opener/pos_tagger'
11
12
  require 'opener/polarity_tagger'
12
13
  require 'opener/property_tagger'
@@ -14,3 +15,5 @@ require 'opener/opinion_detector_basic'
14
15
 
15
16
  require_relative 'chained_daemon/chained_daemon'
16
17
  require_relative 'chained_daemon/cli'
18
+ require_relative 'chained_daemon/microsoft_translator'
19
+
@@ -7,27 +7,87 @@ module Opener
7
7
  def initialize options = {}
8
8
  @options = DEFAULT_OPTIONS.merge options
9
9
  @queue_map = {
10
- 'opener-language-identifier': Opener::LanguageIdentifier.new,
11
- 'opener-tokenizer': Opener::Tokenizer.new,
12
- 'opener-pos-tagger': Opener::POSTagger.new,
13
- 'opener-polarity-tagger': Opener::PolarityTagger.new,
14
- 'opener-property-tagger': Opener::PropertyTagger.new,
15
- 'opener-ner': Opener::Ner.new,
16
- 'opener-opinion-detector-basic': Opener::OpinionDetectorBasic.new,
17
- }
10
+ 'opener-language-identifier': Opener::LanguageIdentifier.new,
11
+ 'opener-tokenizer': Opener::Tokenizer.new,
12
+ 'opener-pos-tagger': Opener::POSTagger.new,
13
+ 'opener-polarity-tagger': Opener::PolarityTagger.new,
14
+ 'opener-property-tagger': Opener::PropertyTagger.new,
15
+ 'opener-opinion-detector-basic': Opener::OpinionDetectorBasic.new,
16
+ }
18
17
  end
19
18
 
20
- def run input
21
- output = nil
19
+ def run input, params = {}
20
+ params ||= {}
21
+ params.symbolize_keys!
22
+ params[:translate_languages] ||= []
23
+
24
+ lang = nil
25
+ output = nil
22
26
  @queue_map.each do |queue, component|
23
- File.write "input-#{queue}", input if ENV['DEBUG']
27
+ debug_print queue, input if ENV['DEBUG']
28
+
24
29
  output = component.run input
25
30
  input = output
31
+
32
+ rescue Core::UnsupportedLanguageError
33
+ xml = Nokogiri.parse input
34
+ lang = xml.root.attr('xml:lang')
35
+ raise unless lang.in? params[:translate_languages]
36
+
37
+ input = translate xml, params
38
+ retry
39
+ end
40
+
41
+ if lang
42
+ # put back original language
43
+ xml = Nokogiri.parse output
44
+ xml.root.attributes['lang'].value = lang
45
+ output = xml.to_s
26
46
  end
47
+
27
48
  output
49
+
28
50
  rescue Core::UnsupportedLanguageError
51
+ puts "Error on unsupported language: #{input}" if ENV['DEBUG']
29
52
  output
30
53
  end
31
54
 
55
+ def translate xml, params
56
+ raw = xml.at :raw
57
+ case translate_service params
58
+ when :google
59
+ raw.content = google_translator.translate raw.content, to: :en
60
+ when :microsoft
61
+ raw.content = microsoft_translator.translate raw.content, to: :en
62
+ else
63
+ raw.content = google_translator.translate raw.content, to: :en
64
+ end
65
+
66
+ xml.root.attributes['lang'].value = 'en'
67
+ xml.to_s
68
+ end
69
+
70
+ protected
71
+
72
+ def translate_service params
73
+ params[:translate_service]&.to_sym || :google
74
+ end
75
+
76
+ def google_translator
77
+ @google_translator ||= Google::Cloud.new.translate ENV['GOOGLE_TRANSLATE_TOKEN']
78
+ end
79
+
80
+ def microsoft_translator
81
+ @microsoft_translator ||= MicrosoftTranslator.new
82
+ end
83
+
84
+ private
85
+
86
+ def debug_print queue, input
87
+ return unless ENV['DEBUG']
88
+ File.write "input-#{queue}", input if ENV['DEBUG']
89
+ puts input
90
+ end
91
+
32
92
  end
33
93
  end
@@ -47,8 +47,9 @@ Example:
47
47
  run do |opts, args|
48
48
  daemon = ChainedDaemon.new args: args
49
49
  input = STDIN.tty? ? nil : STDIN.read
50
+ params = JSON.parse ENV['PARAMS'] if ENV['PARAMS']
50
51
 
51
- puts daemon.run input
52
+ puts daemon.run input, params || {}
52
53
  end
53
54
  end
54
55
  end
@@ -0,0 +1,27 @@
1
+ module Opener
2
+ class ChainedDaemon
3
+ class MicrosoftTranslator
4
+
5
+ URL = 'https://api.cognitive.microsofttranslator.com/translate?api-version=3.0&to=%{to}'
6
+ TOKEN = ENV['MICROSOFT_TRANSLATE_TOKEN']
7
+
8
+ def initialize
9
+ @http = HTTPClient.new
10
+ end
11
+
12
+ def translate text, to: :en
13
+ url = URL % {to: to}
14
+ resp = @http.post url,
15
+ body: [{Text: text}].to_json,
16
+ header: {'Ocp-Apim-Subscription-Key' => TOKEN, 'Content-Type' => 'application/json'}
17
+
18
+ data = JSON.parse resp.body
19
+ raise data['error']['message'] if data.is_a? Hash and data['error']
20
+
21
+ data = Hashie::Mash.new data[0]
22
+ data&.translations&.first&.text
23
+ end
24
+
25
+ end
26
+ end
27
+ end
@@ -1,7 +1,7 @@
1
1
  module Opener
2
2
  class ChainedDaemon
3
3
 
4
- VERSION = '1.0.2'
4
+ VERSION = '2.2.0'
5
5
 
6
6
  end
7
7
  end
@@ -21,20 +21,24 @@ Gem::Specification.new do |spec|
21
21
  spec.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
22
22
 
23
23
  spec.add_dependency 'activesupport'
24
- spec.add_dependency 'opener-daemons'
24
+ spec.add_dependency 'google-cloud-translate', '~> 1.0'
25
+ spec.add_dependency 'httpclient'
26
+ spec.add_dependency 'hashie'
27
+
28
+ spec.add_dependency 'opener-daemons', '~> 2.7.1'
25
29
  spec.add_dependency 'opener-callback-handler', '~> 1.0'
26
30
 
27
31
  spec.add_dependency 'opener-language-identifier'
28
32
  spec.add_dependency 'opener-tokenizer'
29
33
  spec.add_dependency 'opener-pos-tagger'
30
- spec.add_dependency 'opener-polarity-tagger'
31
- spec.add_dependency 'opener-property-tagger'
32
- spec.add_dependency 'opener-ner'
34
+ spec.add_dependency 'opener-polarity-tagger', '>= 3.1.0'
35
+ spec.add_dependency 'opener-property-tagger', '>= 3.2.0'
33
36
  spec.add_dependency 'opener-opinion-detector-basic'
34
37
 
35
38
  spec.add_development_dependency 'bundler', '~> 1.3'
36
39
  spec.add_development_dependency 'rake'
37
40
  spec.add_development_dependency 'pry'
41
+ spec.add_development_dependency 'awesome_print'
38
42
  spec.add_development_dependency 'rspec'
39
43
  spec.add_development_dependency 'rack-test'
40
44
  end
metadata CHANGED
@@ -1,220 +1,262 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: opener-chained-daemon
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 2.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - development@olery.com
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-06-07 00:00:00.000000000 Z
11
+ date: 2020-09-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ name: activesupport
14
15
  requirement: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - ">="
17
18
  - !ruby/object:Gem::Version
18
19
  version: '0'
19
- name: activesupport
20
- prerelease: false
21
20
  type: :runtime
21
+ prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
+ name: google-cloud-translate
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: httpclient
28
43
  requirement: !ruby/object:Gem::Requirement
29
44
  requirements:
30
45
  - - ">="
31
46
  - !ruby/object:Gem::Version
32
47
  version: '0'
33
- name: opener-daemons
48
+ type: :runtime
34
49
  prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: hashie
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
35
62
  type: :runtime
63
+ prerelease: false
36
64
  version_requirements: !ruby/object:Gem::Requirement
37
65
  requirements:
38
66
  - - ">="
39
67
  - !ruby/object:Gem::Version
40
68
  version: '0'
41
69
  - !ruby/object:Gem::Dependency
70
+ name: opener-daemons
42
71
  requirement: !ruby/object:Gem::Requirement
43
72
  requirements:
44
73
  - - "~>"
45
74
  - !ruby/object:Gem::Version
46
- version: '1.0'
47
- name: opener-callback-handler
48
- prerelease: false
75
+ version: 2.7.1
49
76
  type: :runtime
77
+ prerelease: false
50
78
  version_requirements: !ruby/object:Gem::Requirement
51
79
  requirements:
52
80
  - - "~>"
53
81
  - !ruby/object:Gem::Version
54
- version: '1.0'
82
+ version: 2.7.1
55
83
  - !ruby/object:Gem::Dependency
84
+ name: opener-callback-handler
56
85
  requirement: !ruby/object:Gem::Requirement
57
86
  requirements:
58
- - - ">="
87
+ - - "~>"
59
88
  - !ruby/object:Gem::Version
60
- version: '0'
61
- name: opener-language-identifier
62
- prerelease: false
89
+ version: '1.0'
63
90
  type: :runtime
91
+ prerelease: false
64
92
  version_requirements: !ruby/object:Gem::Requirement
65
93
  requirements:
66
- - - ">="
94
+ - - "~>"
67
95
  - !ruby/object:Gem::Version
68
- version: '0'
96
+ version: '1.0'
69
97
  - !ruby/object:Gem::Dependency
98
+ name: opener-language-identifier
70
99
  requirement: !ruby/object:Gem::Requirement
71
100
  requirements:
72
101
  - - ">="
73
102
  - !ruby/object:Gem::Version
74
103
  version: '0'
75
- name: opener-tokenizer
76
- prerelease: false
77
104
  type: :runtime
105
+ prerelease: false
78
106
  version_requirements: !ruby/object:Gem::Requirement
79
107
  requirements:
80
108
  - - ">="
81
109
  - !ruby/object:Gem::Version
82
110
  version: '0'
83
111
  - !ruby/object:Gem::Dependency
112
+ name: opener-tokenizer
84
113
  requirement: !ruby/object:Gem::Requirement
85
114
  requirements:
86
115
  - - ">="
87
116
  - !ruby/object:Gem::Version
88
117
  version: '0'
89
- name: opener-pos-tagger
90
- prerelease: false
91
118
  type: :runtime
119
+ prerelease: false
92
120
  version_requirements: !ruby/object:Gem::Requirement
93
121
  requirements:
94
122
  - - ">="
95
123
  - !ruby/object:Gem::Version
96
124
  version: '0'
97
125
  - !ruby/object:Gem::Dependency
126
+ name: opener-pos-tagger
98
127
  requirement: !ruby/object:Gem::Requirement
99
128
  requirements:
100
129
  - - ">="
101
130
  - !ruby/object:Gem::Version
102
131
  version: '0'
103
- name: opener-polarity-tagger
104
- prerelease: false
105
132
  type: :runtime
133
+ prerelease: false
106
134
  version_requirements: !ruby/object:Gem::Requirement
107
135
  requirements:
108
136
  - - ">="
109
137
  - !ruby/object:Gem::Version
110
138
  version: '0'
111
139
  - !ruby/object:Gem::Dependency
140
+ name: opener-polarity-tagger
112
141
  requirement: !ruby/object:Gem::Requirement
113
142
  requirements:
114
143
  - - ">="
115
144
  - !ruby/object:Gem::Version
116
- version: '0'
117
- name: opener-property-tagger
118
- prerelease: false
145
+ version: 3.1.0
119
146
  type: :runtime
147
+ prerelease: false
120
148
  version_requirements: !ruby/object:Gem::Requirement
121
149
  requirements:
122
150
  - - ">="
123
151
  - !ruby/object:Gem::Version
124
- version: '0'
152
+ version: 3.1.0
125
153
  - !ruby/object:Gem::Dependency
154
+ name: opener-property-tagger
126
155
  requirement: !ruby/object:Gem::Requirement
127
156
  requirements:
128
157
  - - ">="
129
158
  - !ruby/object:Gem::Version
130
- version: '0'
131
- name: opener-ner
132
- prerelease: false
159
+ version: 3.2.0
133
160
  type: :runtime
161
+ prerelease: false
134
162
  version_requirements: !ruby/object:Gem::Requirement
135
163
  requirements:
136
164
  - - ">="
137
165
  - !ruby/object:Gem::Version
138
- version: '0'
166
+ version: 3.2.0
139
167
  - !ruby/object:Gem::Dependency
168
+ name: opener-opinion-detector-basic
140
169
  requirement: !ruby/object:Gem::Requirement
141
170
  requirements:
142
171
  - - ">="
143
172
  - !ruby/object:Gem::Version
144
173
  version: '0'
145
- name: opener-opinion-detector-basic
146
- prerelease: false
147
174
  type: :runtime
175
+ prerelease: false
148
176
  version_requirements: !ruby/object:Gem::Requirement
149
177
  requirements:
150
178
  - - ">="
151
179
  - !ruby/object:Gem::Version
152
180
  version: '0'
153
181
  - !ruby/object:Gem::Dependency
182
+ name: bundler
154
183
  requirement: !ruby/object:Gem::Requirement
155
184
  requirements:
156
185
  - - "~>"
157
186
  - !ruby/object:Gem::Version
158
187
  version: '1.3'
159
- name: bundler
160
- prerelease: false
161
188
  type: :development
189
+ prerelease: false
162
190
  version_requirements: !ruby/object:Gem::Requirement
163
191
  requirements:
164
192
  - - "~>"
165
193
  - !ruby/object:Gem::Version
166
194
  version: '1.3'
167
195
  - !ruby/object:Gem::Dependency
196
+ name: rake
168
197
  requirement: !ruby/object:Gem::Requirement
169
198
  requirements:
170
199
  - - ">="
171
200
  - !ruby/object:Gem::Version
172
201
  version: '0'
173
- name: rake
174
- prerelease: false
175
202
  type: :development
203
+ prerelease: false
176
204
  version_requirements: !ruby/object:Gem::Requirement
177
205
  requirements:
178
206
  - - ">="
179
207
  - !ruby/object:Gem::Version
180
208
  version: '0'
181
209
  - !ruby/object:Gem::Dependency
210
+ name: pry
182
211
  requirement: !ruby/object:Gem::Requirement
183
212
  requirements:
184
213
  - - ">="
185
214
  - !ruby/object:Gem::Version
186
215
  version: '0'
187
- name: pry
188
- prerelease: false
189
216
  type: :development
217
+ prerelease: false
190
218
  version_requirements: !ruby/object:Gem::Requirement
191
219
  requirements:
192
220
  - - ">="
193
221
  - !ruby/object:Gem::Version
194
222
  version: '0'
195
223
  - !ruby/object:Gem::Dependency
224
+ name: awesome_print
196
225
  requirement: !ruby/object:Gem::Requirement
197
226
  requirements:
198
227
  - - ">="
199
228
  - !ruby/object:Gem::Version
200
229
  version: '0'
201
- name: rspec
202
- prerelease: false
203
230
  type: :development
231
+ prerelease: false
204
232
  version_requirements: !ruby/object:Gem::Requirement
205
233
  requirements:
206
234
  - - ">="
207
235
  - !ruby/object:Gem::Version
208
236
  version: '0'
209
237
  - !ruby/object:Gem::Dependency
238
+ name: rspec
210
239
  requirement: !ruby/object:Gem::Requirement
211
240
  requirements:
212
241
  - - ">="
213
242
  - !ruby/object:Gem::Version
214
243
  version: '0'
215
- name: rack-test
244
+ type: :development
216
245
  prerelease: false
246
+ version_requirements: !ruby/object:Gem::Requirement
247
+ requirements:
248
+ - - ">="
249
+ - !ruby/object:Gem::Version
250
+ version: '0'
251
+ - !ruby/object:Gem::Dependency
252
+ name: rack-test
253
+ requirement: !ruby/object:Gem::Requirement
254
+ requirements:
255
+ - - ">="
256
+ - !ruby/object:Gem::Version
257
+ version: '0'
217
258
  type: :development
259
+ prerelease: false
218
260
  version_requirements: !ruby/object:Gem::Requirement
219
261
  requirements:
220
262
  - - ">="
@@ -225,15 +267,20 @@ email:
225
267
  executables:
226
268
  - chained-daemon
227
269
  - chained-daemon-daemon
270
+ - chained-daemon-csv
271
+ - console
228
272
  extensions: []
229
273
  extra_rdoc_files: []
230
274
  files:
231
275
  - bin/chained-daemon
276
+ - bin/chained-daemon-csv
232
277
  - bin/chained-daemon-daemon
278
+ - bin/console
233
279
  - exec/chained-daemon.rb
234
280
  - lib/opener/chained_daemon.rb
235
281
  - lib/opener/chained_daemon/chained_daemon.rb
236
282
  - lib/opener/chained_daemon/cli.rb
283
+ - lib/opener/chained_daemon/microsoft_translator.rb
237
284
  - lib/opener/chained_daemon/version.rb
238
285
  - opener-chained-daemon.gemspec
239
286
  homepage:
@@ -256,7 +303,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
256
303
  version: '0'
257
304
  requirements: []
258
305
  rubyforge_project:
259
- rubygems_version: 2.6.13
306
+ rubygems_version: 2.7.8
260
307
  signing_key:
261
308
  specification_version: 4
262
309
  summary: OpeNER daemon for processing multiple queues at once