opener-chained-daemon 1.0.2 → 2.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 529591ceec3fc6900d961d97e5064647afacf3c6216d59a36d339b4b17f86ce6
4
- data.tar.gz: cfb6977e833c25e30644684cc60b2c6a487f1e2e5fbd3a5076d6fd0f29b4ffa5
3
+ metadata.gz: fe6f23e32f84727ec3ea06338fc95913cbdf08c91c70d5095a0dc9d5395f2f04
4
+ data.tar.gz: 11e53fde615b379336bda63c3198c05cb8ec9ebbdabd999013df1cf31799c047
5
5
  SHA512:
6
- metadata.gz: b06587c11fcedbc69802bbed5c823504456e21b69c0c86dc5083bffcde95a9b77c0201d778668da4c47e21d705421e37c8aa1d60626aa8c6a1ab004a1f4d3f33
7
- data.tar.gz: 541dbb5b3e944cea173b1bed70561fb4b3aae1a9f8146a8f797b2fc0c6d2a5a83567a5ddce06a2910d9e5c5d242baa4e615960a81b1c24db0c557cf5b98ccd79
6
+ metadata.gz: 196281f0e1e89b3d9272d568f2a4d67034a08b3819e4cc9748873c601cab8fab87facf7523989b0f9fca2d3ab08cfbbe0561d01c7015e8963ef9c03481bf8b3e
7
+ data.tar.gz: 736a7790ede035c155278adb8b67ec1fa89e3bdf9b48a7bb7349e64c44a3b749ecbc15bd8d9129f9fd167405819627ad8056a113ed6b964500cd3cd50251033f
@@ -0,0 +1,41 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler/setup'
4
+ require_relative '../lib/opener/chained_daemon'
5
+ require_relative '../../kaf-processor/lib/kaf_processor/exts/oga'
6
+ require_relative '../../kaf-processor/lib/kaf_processor/services/string_cleaner'
7
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/document'
8
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/property'
9
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/sentiment'
10
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/term'
11
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/opinion'
12
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/word_form'
13
+ require_relative '../../kaf-processor/lib/kaf_processor/configuration'
14
+ require_relative '../../kaf-processor/lib/kaf_processor/processor/review_opinions'
15
+ require 'csv'
16
+
17
+ if ARGV.size != 1
18
+ puts 'USAGE: chained-daemon-csv file.csv'
19
+ exit
20
+ end
21
+
22
+
23
+ headers = [:review_id, :property, :polarity, :strength, :comment]
24
+ out_csv = CSV.open('/tmp/opinions.csv', 'w', write_headers: true, headers: headers)
25
+
26
+ CSV.foreach(ARGV.first, headers: true).each do |review|
27
+ #puts review['comment']
28
+ cd = Opener::ChainedDaemon.new
29
+ output = cd.run(review['comment'])
30
+ document = KafProcessor::KAF::Document.from_xml output
31
+ config = KafProcessor::Configuration.new identifier: review['id']
32
+ opinions = KafProcessor::Processor::ReviewOpinions.new(config, document).generate_rows
33
+ opinions.each do |o|
34
+ out_csv << [o[:review_id], o[:properties].join(', '), o[:polarity], o[:strength], review['comment']]
35
+ end
36
+ if opinions.empty?
37
+ out_csv << [review['id'], '', '', '', review['comment']]
38
+ end
39
+ end
40
+
41
+ out_csv.close
@@ -0,0 +1,13 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler/setup'
4
+ require_relative '../lib/opener/chained_daemon'
5
+
6
+ begin
7
+ require 'awesome_print'
8
+ require 'pry'
9
+ binding.pry
10
+ rescue LoadError
11
+ require 'irb'
12
+ IRB.start
13
+ end
@@ -1,12 +1,13 @@
1
1
  require 'active_support/all'
2
2
  require 'oga'
3
+ require 'hashie'
4
+ require 'google/cloud/translate'
3
5
 
4
6
  require 'opener/daemons'
5
7
 
6
8
  require 'opener/language_identifier'
7
9
  require 'opener/tokenizer'
8
10
  # require ner before pos_tagger for compatible opennlp
9
- require 'opener/ner'
10
11
  require 'opener/pos_tagger'
11
12
  require 'opener/polarity_tagger'
12
13
  require 'opener/property_tagger'
@@ -14,3 +15,5 @@ require 'opener/opinion_detector_basic'
14
15
 
15
16
  require_relative 'chained_daemon/chained_daemon'
16
17
  require_relative 'chained_daemon/cli'
18
+ require_relative 'chained_daemon/microsoft_translator'
19
+
@@ -7,27 +7,87 @@ module Opener
7
7
  def initialize options = {}
8
8
  @options = DEFAULT_OPTIONS.merge options
9
9
  @queue_map = {
10
- 'opener-language-identifier': Opener::LanguageIdentifier.new,
11
- 'opener-tokenizer': Opener::Tokenizer.new,
12
- 'opener-pos-tagger': Opener::POSTagger.new,
13
- 'opener-polarity-tagger': Opener::PolarityTagger.new,
14
- 'opener-property-tagger': Opener::PropertyTagger.new,
15
- 'opener-ner': Opener::Ner.new,
16
- 'opener-opinion-detector-basic': Opener::OpinionDetectorBasic.new,
17
- }
10
+ 'opener-language-identifier': Opener::LanguageIdentifier.new,
11
+ 'opener-tokenizer': Opener::Tokenizer.new,
12
+ 'opener-pos-tagger': Opener::POSTagger.new,
13
+ 'opener-polarity-tagger': Opener::PolarityTagger.new,
14
+ 'opener-property-tagger': Opener::PropertyTagger.new,
15
+ 'opener-opinion-detector-basic': Opener::OpinionDetectorBasic.new,
16
+ }
18
17
  end
19
18
 
20
- def run input
21
- output = nil
19
+ def run input, params = {}
20
+ params ||= {}
21
+ params.symbolize_keys!
22
+ params[:translate_languages] ||= []
23
+
24
+ lang = nil
25
+ output = nil
22
26
  @queue_map.each do |queue, component|
23
- File.write "input-#{queue}", input if ENV['DEBUG']
27
+ debug_print queue, input if ENV['DEBUG']
28
+
24
29
  output = component.run input
25
30
  input = output
31
+
32
+ rescue Core::UnsupportedLanguageError
33
+ xml = Nokogiri.parse input
34
+ lang = xml.root.attr('xml:lang')
35
+ raise unless lang.in? params[:translate_languages]
36
+
37
+ input = translate xml, params
38
+ retry
39
+ end
40
+
41
+ if lang
42
+ # put back original language
43
+ xml = Nokogiri.parse output
44
+ xml.root.attributes['lang'].value = lang
45
+ output = xml.to_s
26
46
  end
47
+
27
48
  output
49
+
28
50
  rescue Core::UnsupportedLanguageError
51
+ puts "Error on unsupported language: #{input}" if ENV['DEBUG']
29
52
  output
30
53
  end
31
54
 
55
+ def translate xml, params
56
+ raw = xml.at :raw
57
+ case translate_service params
58
+ when :google
59
+ raw.content = google_translator.translate raw.content, to: :en
60
+ when :microsoft
61
+ raw.content = microsoft_translator.translate raw.content, to: :en
62
+ else
63
+ raw.content = google_translator.translate raw.content, to: :en
64
+ end
65
+
66
+ xml.root.attributes['lang'].value = 'en'
67
+ xml.to_s
68
+ end
69
+
70
+ protected
71
+
72
+ def translate_service params
73
+ params[:translate_service]&.to_sym || :google
74
+ end
75
+
76
+ def google_translator
77
+ @google_translator ||= Google::Cloud.new.translate ENV['GOOGLE_TRANSLATE_TOKEN']
78
+ end
79
+
80
+ def microsoft_translator
81
+ @microsoft_translator ||= MicrosoftTranslator.new
82
+ end
83
+
84
+ private
85
+
86
+ def debug_print queue, input
87
+ return unless ENV['DEBUG']
88
+ File.write "input-#{queue}", input if ENV['DEBUG']
89
+ puts input
90
+ end
91
+
32
92
  end
33
93
  end
@@ -47,8 +47,9 @@ Example:
47
47
  run do |opts, args|
48
48
  daemon = ChainedDaemon.new args: args
49
49
  input = STDIN.tty? ? nil : STDIN.read
50
+ params = JSON.parse ENV['PARAMS'] if ENV['PARAMS']
50
51
 
51
- puts daemon.run input
52
+ puts daemon.run input, params || {}
52
53
  end
53
54
  end
54
55
  end
@@ -0,0 +1,27 @@
1
+ module Opener
2
+ class ChainedDaemon
3
+ class MicrosoftTranslator
4
+
5
+ URL = 'https://api.cognitive.microsofttranslator.com/translate?api-version=3.0&to=%{to}'
6
+ TOKEN = ENV['MICROSOFT_TRANSLATE_TOKEN']
7
+
8
+ def initialize
9
+ @http = HTTPClient.new
10
+ end
11
+
12
+ def translate text, to: :en
13
+ url = URL % {to: to}
14
+ resp = @http.post url,
15
+ body: [{Text: text}].to_json,
16
+ header: {'Ocp-Apim-Subscription-Key' => TOKEN, 'Content-Type' => 'application/json'}
17
+
18
+ data = JSON.parse resp.body
19
+ raise data['error']['message'] if data.is_a? Hash and data['error']
20
+
21
+ data = Hashie::Mash.new data[0]
22
+ data&.translations&.first&.text
23
+ end
24
+
25
+ end
26
+ end
27
+ end
@@ -1,7 +1,7 @@
1
1
  module Opener
2
2
  class ChainedDaemon
3
3
 
4
- VERSION = '1.0.2'
4
+ VERSION = '2.2.0'
5
5
 
6
6
  end
7
7
  end
@@ -21,20 +21,24 @@ Gem::Specification.new do |spec|
21
21
  spec.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
22
22
 
23
23
  spec.add_dependency 'activesupport'
24
- spec.add_dependency 'opener-daemons'
24
+ spec.add_dependency 'google-cloud-translate', '~> 1.0'
25
+ spec.add_dependency 'httpclient'
26
+ spec.add_dependency 'hashie'
27
+
28
+ spec.add_dependency 'opener-daemons', '~> 2.7.1'
25
29
  spec.add_dependency 'opener-callback-handler', '~> 1.0'
26
30
 
27
31
  spec.add_dependency 'opener-language-identifier'
28
32
  spec.add_dependency 'opener-tokenizer'
29
33
  spec.add_dependency 'opener-pos-tagger'
30
- spec.add_dependency 'opener-polarity-tagger'
31
- spec.add_dependency 'opener-property-tagger'
32
- spec.add_dependency 'opener-ner'
34
+ spec.add_dependency 'opener-polarity-tagger', '>= 3.1.0'
35
+ spec.add_dependency 'opener-property-tagger', '>= 3.2.0'
33
36
  spec.add_dependency 'opener-opinion-detector-basic'
34
37
 
35
38
  spec.add_development_dependency 'bundler', '~> 1.3'
36
39
  spec.add_development_dependency 'rake'
37
40
  spec.add_development_dependency 'pry'
41
+ spec.add_development_dependency 'awesome_print'
38
42
  spec.add_development_dependency 'rspec'
39
43
  spec.add_development_dependency 'rack-test'
40
44
  end
metadata CHANGED
@@ -1,220 +1,262 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: opener-chained-daemon
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 2.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - development@olery.com
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-06-07 00:00:00.000000000 Z
11
+ date: 2020-09-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ name: activesupport
14
15
  requirement: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - ">="
17
18
  - !ruby/object:Gem::Version
18
19
  version: '0'
19
- name: activesupport
20
- prerelease: false
21
20
  type: :runtime
21
+ prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
+ name: google-cloud-translate
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: httpclient
28
43
  requirement: !ruby/object:Gem::Requirement
29
44
  requirements:
30
45
  - - ">="
31
46
  - !ruby/object:Gem::Version
32
47
  version: '0'
33
- name: opener-daemons
48
+ type: :runtime
34
49
  prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: hashie
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
35
62
  type: :runtime
63
+ prerelease: false
36
64
  version_requirements: !ruby/object:Gem::Requirement
37
65
  requirements:
38
66
  - - ">="
39
67
  - !ruby/object:Gem::Version
40
68
  version: '0'
41
69
  - !ruby/object:Gem::Dependency
70
+ name: opener-daemons
42
71
  requirement: !ruby/object:Gem::Requirement
43
72
  requirements:
44
73
  - - "~>"
45
74
  - !ruby/object:Gem::Version
46
- version: '1.0'
47
- name: opener-callback-handler
48
- prerelease: false
75
+ version: 2.7.1
49
76
  type: :runtime
77
+ prerelease: false
50
78
  version_requirements: !ruby/object:Gem::Requirement
51
79
  requirements:
52
80
  - - "~>"
53
81
  - !ruby/object:Gem::Version
54
- version: '1.0'
82
+ version: 2.7.1
55
83
  - !ruby/object:Gem::Dependency
84
+ name: opener-callback-handler
56
85
  requirement: !ruby/object:Gem::Requirement
57
86
  requirements:
58
- - - ">="
87
+ - - "~>"
59
88
  - !ruby/object:Gem::Version
60
- version: '0'
61
- name: opener-language-identifier
62
- prerelease: false
89
+ version: '1.0'
63
90
  type: :runtime
91
+ prerelease: false
64
92
  version_requirements: !ruby/object:Gem::Requirement
65
93
  requirements:
66
- - - ">="
94
+ - - "~>"
67
95
  - !ruby/object:Gem::Version
68
- version: '0'
96
+ version: '1.0'
69
97
  - !ruby/object:Gem::Dependency
98
+ name: opener-language-identifier
70
99
  requirement: !ruby/object:Gem::Requirement
71
100
  requirements:
72
101
  - - ">="
73
102
  - !ruby/object:Gem::Version
74
103
  version: '0'
75
- name: opener-tokenizer
76
- prerelease: false
77
104
  type: :runtime
105
+ prerelease: false
78
106
  version_requirements: !ruby/object:Gem::Requirement
79
107
  requirements:
80
108
  - - ">="
81
109
  - !ruby/object:Gem::Version
82
110
  version: '0'
83
111
  - !ruby/object:Gem::Dependency
112
+ name: opener-tokenizer
84
113
  requirement: !ruby/object:Gem::Requirement
85
114
  requirements:
86
115
  - - ">="
87
116
  - !ruby/object:Gem::Version
88
117
  version: '0'
89
- name: opener-pos-tagger
90
- prerelease: false
91
118
  type: :runtime
119
+ prerelease: false
92
120
  version_requirements: !ruby/object:Gem::Requirement
93
121
  requirements:
94
122
  - - ">="
95
123
  - !ruby/object:Gem::Version
96
124
  version: '0'
97
125
  - !ruby/object:Gem::Dependency
126
+ name: opener-pos-tagger
98
127
  requirement: !ruby/object:Gem::Requirement
99
128
  requirements:
100
129
  - - ">="
101
130
  - !ruby/object:Gem::Version
102
131
  version: '0'
103
- name: opener-polarity-tagger
104
- prerelease: false
105
132
  type: :runtime
133
+ prerelease: false
106
134
  version_requirements: !ruby/object:Gem::Requirement
107
135
  requirements:
108
136
  - - ">="
109
137
  - !ruby/object:Gem::Version
110
138
  version: '0'
111
139
  - !ruby/object:Gem::Dependency
140
+ name: opener-polarity-tagger
112
141
  requirement: !ruby/object:Gem::Requirement
113
142
  requirements:
114
143
  - - ">="
115
144
  - !ruby/object:Gem::Version
116
- version: '0'
117
- name: opener-property-tagger
118
- prerelease: false
145
+ version: 3.1.0
119
146
  type: :runtime
147
+ prerelease: false
120
148
  version_requirements: !ruby/object:Gem::Requirement
121
149
  requirements:
122
150
  - - ">="
123
151
  - !ruby/object:Gem::Version
124
- version: '0'
152
+ version: 3.1.0
125
153
  - !ruby/object:Gem::Dependency
154
+ name: opener-property-tagger
126
155
  requirement: !ruby/object:Gem::Requirement
127
156
  requirements:
128
157
  - - ">="
129
158
  - !ruby/object:Gem::Version
130
- version: '0'
131
- name: opener-ner
132
- prerelease: false
159
+ version: 3.2.0
133
160
  type: :runtime
161
+ prerelease: false
134
162
  version_requirements: !ruby/object:Gem::Requirement
135
163
  requirements:
136
164
  - - ">="
137
165
  - !ruby/object:Gem::Version
138
- version: '0'
166
+ version: 3.2.0
139
167
  - !ruby/object:Gem::Dependency
168
+ name: opener-opinion-detector-basic
140
169
  requirement: !ruby/object:Gem::Requirement
141
170
  requirements:
142
171
  - - ">="
143
172
  - !ruby/object:Gem::Version
144
173
  version: '0'
145
- name: opener-opinion-detector-basic
146
- prerelease: false
147
174
  type: :runtime
175
+ prerelease: false
148
176
  version_requirements: !ruby/object:Gem::Requirement
149
177
  requirements:
150
178
  - - ">="
151
179
  - !ruby/object:Gem::Version
152
180
  version: '0'
153
181
  - !ruby/object:Gem::Dependency
182
+ name: bundler
154
183
  requirement: !ruby/object:Gem::Requirement
155
184
  requirements:
156
185
  - - "~>"
157
186
  - !ruby/object:Gem::Version
158
187
  version: '1.3'
159
- name: bundler
160
- prerelease: false
161
188
  type: :development
189
+ prerelease: false
162
190
  version_requirements: !ruby/object:Gem::Requirement
163
191
  requirements:
164
192
  - - "~>"
165
193
  - !ruby/object:Gem::Version
166
194
  version: '1.3'
167
195
  - !ruby/object:Gem::Dependency
196
+ name: rake
168
197
  requirement: !ruby/object:Gem::Requirement
169
198
  requirements:
170
199
  - - ">="
171
200
  - !ruby/object:Gem::Version
172
201
  version: '0'
173
- name: rake
174
- prerelease: false
175
202
  type: :development
203
+ prerelease: false
176
204
  version_requirements: !ruby/object:Gem::Requirement
177
205
  requirements:
178
206
  - - ">="
179
207
  - !ruby/object:Gem::Version
180
208
  version: '0'
181
209
  - !ruby/object:Gem::Dependency
210
+ name: pry
182
211
  requirement: !ruby/object:Gem::Requirement
183
212
  requirements:
184
213
  - - ">="
185
214
  - !ruby/object:Gem::Version
186
215
  version: '0'
187
- name: pry
188
- prerelease: false
189
216
  type: :development
217
+ prerelease: false
190
218
  version_requirements: !ruby/object:Gem::Requirement
191
219
  requirements:
192
220
  - - ">="
193
221
  - !ruby/object:Gem::Version
194
222
  version: '0'
195
223
  - !ruby/object:Gem::Dependency
224
+ name: awesome_print
196
225
  requirement: !ruby/object:Gem::Requirement
197
226
  requirements:
198
227
  - - ">="
199
228
  - !ruby/object:Gem::Version
200
229
  version: '0'
201
- name: rspec
202
- prerelease: false
203
230
  type: :development
231
+ prerelease: false
204
232
  version_requirements: !ruby/object:Gem::Requirement
205
233
  requirements:
206
234
  - - ">="
207
235
  - !ruby/object:Gem::Version
208
236
  version: '0'
209
237
  - !ruby/object:Gem::Dependency
238
+ name: rspec
210
239
  requirement: !ruby/object:Gem::Requirement
211
240
  requirements:
212
241
  - - ">="
213
242
  - !ruby/object:Gem::Version
214
243
  version: '0'
215
- name: rack-test
244
+ type: :development
216
245
  prerelease: false
246
+ version_requirements: !ruby/object:Gem::Requirement
247
+ requirements:
248
+ - - ">="
249
+ - !ruby/object:Gem::Version
250
+ version: '0'
251
+ - !ruby/object:Gem::Dependency
252
+ name: rack-test
253
+ requirement: !ruby/object:Gem::Requirement
254
+ requirements:
255
+ - - ">="
256
+ - !ruby/object:Gem::Version
257
+ version: '0'
217
258
  type: :development
259
+ prerelease: false
218
260
  version_requirements: !ruby/object:Gem::Requirement
219
261
  requirements:
220
262
  - - ">="
@@ -225,15 +267,20 @@ email:
225
267
  executables:
226
268
  - chained-daemon
227
269
  - chained-daemon-daemon
270
+ - chained-daemon-csv
271
+ - console
228
272
  extensions: []
229
273
  extra_rdoc_files: []
230
274
  files:
231
275
  - bin/chained-daemon
276
+ - bin/chained-daemon-csv
232
277
  - bin/chained-daemon-daemon
278
+ - bin/console
233
279
  - exec/chained-daemon.rb
234
280
  - lib/opener/chained_daemon.rb
235
281
  - lib/opener/chained_daemon/chained_daemon.rb
236
282
  - lib/opener/chained_daemon/cli.rb
283
+ - lib/opener/chained_daemon/microsoft_translator.rb
237
284
  - lib/opener/chained_daemon/version.rb
238
285
  - opener-chained-daemon.gemspec
239
286
  homepage:
@@ -256,7 +303,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
256
303
  version: '0'
257
304
  requirements: []
258
305
  rubyforge_project:
259
- rubygems_version: 2.6.13
306
+ rubygems_version: 2.7.8
260
307
  signing_key:
261
308
  specification_version: 4
262
309
  summary: OpeNER daemon for processing multiple queues at once