opener-chained-daemon 1.0.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 529591ceec3fc6900d961d97e5064647afacf3c6216d59a36d339b4b17f86ce6
4
- data.tar.gz: cfb6977e833c25e30644684cc60b2c6a487f1e2e5fbd3a5076d6fd0f29b4ffa5
3
+ metadata.gz: acaa68db270b274f2cf20783d7407b1baeb967ba956ee3d1122aaf5f72156fb8
4
+ data.tar.gz: 336b2d8cf72c6a7b011acae5335de015bc29158042706c4e5c0729ca845feab9
5
5
  SHA512:
6
- metadata.gz: b06587c11fcedbc69802bbed5c823504456e21b69c0c86dc5083bffcde95a9b77c0201d778668da4c47e21d705421e37c8aa1d60626aa8c6a1ab004a1f4d3f33
7
- data.tar.gz: 541dbb5b3e944cea173b1bed70561fb4b3aae1a9f8146a8f797b2fc0c6d2a5a83567a5ddce06a2910d9e5c5d242baa4e615960a81b1c24db0c557cf5b98ccd79
6
+ metadata.gz: dad04d51c0b89e08c928a41c6c45c88174303f47d457f7576294e31b4b04341add667d4f2390859b120d20026728f66e3b8272551b35eef434f8292f2a1e9bf7
7
+ data.tar.gz: 4468af5b3d8e96ecf7df73cf0cf2d109b0c8af92236f51597403ff3fb78ba07c741d652e8f2ab4457d70b4219ed477e90ba9d09edb2bd13eae475efdda50c6ee
@@ -0,0 +1,41 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler/setup'
4
+ require_relative '../lib/opener/chained_daemon'
5
+ require_relative '../../kaf-processor/lib/kaf_processor/exts/oga'
6
+ require_relative '../../kaf-processor/lib/kaf_processor/services/string_cleaner'
7
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/document'
8
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/property'
9
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/sentiment'
10
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/term'
11
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/opinion'
12
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/word_form'
13
+ require_relative '../../kaf-processor/lib/kaf_processor/configuration'
14
+ require_relative '../../kaf-processor/lib/kaf_processor/processor/review_opinions'
15
+ require 'csv'
16
+
17
+ if ARGV.size != 1
18
+ puts 'USAGE: chained-daemon-csv file.csv'
19
+ exit
20
+ end
21
+
22
+
23
+ headers = [:review_id, :property, :polarity, :strength, :comment]
24
+ out_csv = CSV.open('/tmp/opinions.csv', 'w', write_headers: true, headers: headers)
25
+
26
+ CSV.foreach(ARGV.first, headers: true).each do |review|
27
+ #puts review['comment']
28
+ cd = Opener::ChainedDaemon.new
29
+ output = cd.run(review['comment'])
30
+ document = KafProcessor::KAF::Document.from_xml output
31
+ config = KafProcessor::Configuration.new identifier: review['id']
32
+ opinions = KafProcessor::Processor::ReviewOpinions.new(config, document).generate_rows
33
+ opinions.each do |o|
34
+ out_csv << [o[:review_id], o[:properties].join(', '), o[:polarity], o[:strength], review['comment']]
35
+ end
36
+ if opinions.empty?
37
+ out_csv << [review['id'], '', '', '', review['comment']]
38
+ end
39
+ end
40
+
41
+ out_csv.close
@@ -7,27 +7,81 @@ module Opener
7
7
  def initialize options = {}
8
8
  @options = DEFAULT_OPTIONS.merge options
9
9
  @queue_map = {
10
- 'opener-language-identifier': Opener::LanguageIdentifier.new,
11
- 'opener-tokenizer': Opener::Tokenizer.new,
12
- 'opener-pos-tagger': Opener::POSTagger.new,
13
- 'opener-polarity-tagger': Opener::PolarityTagger.new,
14
- 'opener-property-tagger': Opener::PropertyTagger.new,
15
- 'opener-ner': Opener::Ner.new,
16
- 'opener-opinion-detector-basic': Opener::OpinionDetectorBasic.new,
17
- }
10
+ 'opener-language-identifier': Opener::LanguageIdentifier.new,
11
+ 'opener-tokenizer': Opener::Tokenizer.new,
12
+ 'opener-pos-tagger': Opener::POSTagger.new,
13
+ 'opener-polarity-tagger': Opener::PolarityTagger.new,
14
+ 'opener-property-tagger': Opener::PropertyTagger.new,
15
+ 'opener-ner': Opener::Ner.new,
16
+ 'opener-opinion-detector-basic': Opener::OpinionDetectorBasic.new,
17
+ }
18
18
  end
19
19
 
20
- def run input
21
- output = nil
20
+ def run input, params = {}
21
+ params.symbolize_keys!
22
+ params[:translate_languages] ||= []
23
+
24
+ lang = nil
25
+ output = nil
22
26
  @queue_map.each do |queue, component|
23
- File.write "input-#{queue}", input if ENV['DEBUG']
27
+ debug_print queue, input if ENV['DEBUG']
28
+
24
29
  output = component.run input
25
30
  input = output
31
+
32
+ rescue Core::UnsupportedLanguageError
33
+ xml = Nokogiri.parse input
34
+ lang = xml.root.attr('xml:lang')
35
+ raise unless lang.in? params[:translate_languages]
36
+
37
+ input = translate xml, params
38
+ retry
26
39
  end
40
+
41
+ if lang
42
+ # put back original language
43
+ xml = Nokogiri.parse output
44
+ xml.root.attributes['lang'].value = lang
45
+ output = xml.to_s
46
+ end
47
+
27
48
  output
49
+
28
50
  rescue Core::UnsupportedLanguageError
51
+ puts "Error on unsupported language: #{input}" if ENV['DEBUG']
29
52
  output
30
53
  end
31
54
 
55
+ def translate xml, params
56
+ raw = xml.at :raw
57
+ case translate_service params
58
+ when :google
59
+ raw.content = google_translator.translate raw.content, to: :en
60
+ else
61
+ raw.content = google_translator.translate raw.content, to: :en
62
+ end
63
+
64
+ xml.root.attributes['lang'].value = 'en'
65
+ xml.to_s
66
+ end
67
+
68
+ protected
69
+
70
+ def translate_service params
71
+ params[:translate_service]&.to_sym || :google
72
+ end
73
+
74
+ def google_translator
75
+ @google_translator ||= Google::Cloud.new.translate ENV['GOOGLE_TRANSLATE_TOKEN']
76
+ end
77
+
78
+ private
79
+
80
+ def debug_print queue, input
81
+ return unless ENV['DEBUG']
82
+ File.write "input-#{queue}", input if ENV['DEBUG']
83
+ puts input
84
+ end
85
+
32
86
  end
33
87
  end
@@ -47,8 +47,9 @@ Example:
47
47
  run do |opts, args|
48
48
  daemon = ChainedDaemon.new args: args
49
49
  input = STDIN.tty? ? nil : STDIN.read
50
+ params = JSON.parse ENV['PARAMS'] if ENV['PARAMS']
50
51
 
51
- puts daemon.run input
52
+ puts daemon.run input, params || {}
52
53
  end
53
54
  end
54
55
  end
@@ -1,7 +1,7 @@
1
1
  module Opener
2
2
  class ChainedDaemon
3
3
 
4
- VERSION = '1.0.2'
4
+ VERSION = '2.0.0'
5
5
 
6
6
  end
7
7
  end
@@ -1,5 +1,6 @@
1
1
  require 'active_support/all'
2
2
  require 'oga'
3
+ require 'google/cloud/translate'
3
4
 
4
5
  require 'opener/daemons'
5
6
 
@@ -21,7 +21,9 @@ Gem::Specification.new do |spec|
21
21
  spec.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
22
22
 
23
23
  spec.add_dependency 'activesupport'
24
- spec.add_dependency 'opener-daemons'
24
+ spec.add_dependency 'google-cloud-translate', '~> 1.0'
25
+
26
+ spec.add_dependency 'opener-daemons', '>= 2.7'
25
27
  spec.add_dependency 'opener-callback-handler', '~> 1.0'
26
28
 
27
29
  spec.add_dependency 'opener-language-identifier'
metadata CHANGED
@@ -1,234 +1,250 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: opener-chained-daemon
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 2.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - development@olery.com
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-06-07 00:00:00.000000000 Z
11
+ date: 2020-03-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ name: activesupport
14
15
  requirement: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - ">="
17
18
  - !ruby/object:Gem::Version
18
19
  version: '0'
19
- name: activesupport
20
- prerelease: false
21
20
  type: :runtime
21
+ prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
+ name: google-cloud-translate
28
29
  requirement: !ruby/object:Gem::Requirement
29
30
  requirements:
30
- - - ">="
31
+ - - "~>"
31
32
  - !ruby/object:Gem::Version
32
- version: '0'
33
- name: opener-daemons
33
+ version: '1.0'
34
+ type: :runtime
34
35
  prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: opener-daemons
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '2.7'
35
48
  type: :runtime
49
+ prerelease: false
36
50
  version_requirements: !ruby/object:Gem::Requirement
37
51
  requirements:
38
52
  - - ">="
39
53
  - !ruby/object:Gem::Version
40
- version: '0'
54
+ version: '2.7'
41
55
  - !ruby/object:Gem::Dependency
56
+ name: opener-callback-handler
42
57
  requirement: !ruby/object:Gem::Requirement
43
58
  requirements:
44
59
  - - "~>"
45
60
  - !ruby/object:Gem::Version
46
61
  version: '1.0'
47
- name: opener-callback-handler
48
- prerelease: false
49
62
  type: :runtime
63
+ prerelease: false
50
64
  version_requirements: !ruby/object:Gem::Requirement
51
65
  requirements:
52
66
  - - "~>"
53
67
  - !ruby/object:Gem::Version
54
68
  version: '1.0'
55
69
  - !ruby/object:Gem::Dependency
70
+ name: opener-language-identifier
56
71
  requirement: !ruby/object:Gem::Requirement
57
72
  requirements:
58
73
  - - ">="
59
74
  - !ruby/object:Gem::Version
60
75
  version: '0'
61
- name: opener-language-identifier
62
- prerelease: false
63
76
  type: :runtime
77
+ prerelease: false
64
78
  version_requirements: !ruby/object:Gem::Requirement
65
79
  requirements:
66
80
  - - ">="
67
81
  - !ruby/object:Gem::Version
68
82
  version: '0'
69
83
  - !ruby/object:Gem::Dependency
84
+ name: opener-tokenizer
70
85
  requirement: !ruby/object:Gem::Requirement
71
86
  requirements:
72
87
  - - ">="
73
88
  - !ruby/object:Gem::Version
74
89
  version: '0'
75
- name: opener-tokenizer
76
- prerelease: false
77
90
  type: :runtime
91
+ prerelease: false
78
92
  version_requirements: !ruby/object:Gem::Requirement
79
93
  requirements:
80
94
  - - ">="
81
95
  - !ruby/object:Gem::Version
82
96
  version: '0'
83
97
  - !ruby/object:Gem::Dependency
98
+ name: opener-pos-tagger
84
99
  requirement: !ruby/object:Gem::Requirement
85
100
  requirements:
86
101
  - - ">="
87
102
  - !ruby/object:Gem::Version
88
103
  version: '0'
89
- name: opener-pos-tagger
90
- prerelease: false
91
104
  type: :runtime
105
+ prerelease: false
92
106
  version_requirements: !ruby/object:Gem::Requirement
93
107
  requirements:
94
108
  - - ">="
95
109
  - !ruby/object:Gem::Version
96
110
  version: '0'
97
111
  - !ruby/object:Gem::Dependency
112
+ name: opener-polarity-tagger
98
113
  requirement: !ruby/object:Gem::Requirement
99
114
  requirements:
100
115
  - - ">="
101
116
  - !ruby/object:Gem::Version
102
117
  version: '0'
103
- name: opener-polarity-tagger
104
- prerelease: false
105
118
  type: :runtime
119
+ prerelease: false
106
120
  version_requirements: !ruby/object:Gem::Requirement
107
121
  requirements:
108
122
  - - ">="
109
123
  - !ruby/object:Gem::Version
110
124
  version: '0'
111
125
  - !ruby/object:Gem::Dependency
126
+ name: opener-property-tagger
112
127
  requirement: !ruby/object:Gem::Requirement
113
128
  requirements:
114
129
  - - ">="
115
130
  - !ruby/object:Gem::Version
116
131
  version: '0'
117
- name: opener-property-tagger
118
- prerelease: false
119
132
  type: :runtime
133
+ prerelease: false
120
134
  version_requirements: !ruby/object:Gem::Requirement
121
135
  requirements:
122
136
  - - ">="
123
137
  - !ruby/object:Gem::Version
124
138
  version: '0'
125
139
  - !ruby/object:Gem::Dependency
140
+ name: opener-ner
126
141
  requirement: !ruby/object:Gem::Requirement
127
142
  requirements:
128
143
  - - ">="
129
144
  - !ruby/object:Gem::Version
130
145
  version: '0'
131
- name: opener-ner
132
- prerelease: false
133
146
  type: :runtime
147
+ prerelease: false
134
148
  version_requirements: !ruby/object:Gem::Requirement
135
149
  requirements:
136
150
  - - ">="
137
151
  - !ruby/object:Gem::Version
138
152
  version: '0'
139
153
  - !ruby/object:Gem::Dependency
154
+ name: opener-opinion-detector-basic
140
155
  requirement: !ruby/object:Gem::Requirement
141
156
  requirements:
142
157
  - - ">="
143
158
  - !ruby/object:Gem::Version
144
159
  version: '0'
145
- name: opener-opinion-detector-basic
146
- prerelease: false
147
160
  type: :runtime
161
+ prerelease: false
148
162
  version_requirements: !ruby/object:Gem::Requirement
149
163
  requirements:
150
164
  - - ">="
151
165
  - !ruby/object:Gem::Version
152
166
  version: '0'
153
167
  - !ruby/object:Gem::Dependency
168
+ name: bundler
154
169
  requirement: !ruby/object:Gem::Requirement
155
170
  requirements:
156
171
  - - "~>"
157
172
  - !ruby/object:Gem::Version
158
173
  version: '1.3'
159
- name: bundler
160
- prerelease: false
161
174
  type: :development
175
+ prerelease: false
162
176
  version_requirements: !ruby/object:Gem::Requirement
163
177
  requirements:
164
178
  - - "~>"
165
179
  - !ruby/object:Gem::Version
166
180
  version: '1.3'
167
181
  - !ruby/object:Gem::Dependency
182
+ name: rake
168
183
  requirement: !ruby/object:Gem::Requirement
169
184
  requirements:
170
185
  - - ">="
171
186
  - !ruby/object:Gem::Version
172
187
  version: '0'
173
- name: rake
174
- prerelease: false
175
188
  type: :development
189
+ prerelease: false
176
190
  version_requirements: !ruby/object:Gem::Requirement
177
191
  requirements:
178
192
  - - ">="
179
193
  - !ruby/object:Gem::Version
180
194
  version: '0'
181
195
  - !ruby/object:Gem::Dependency
196
+ name: pry
182
197
  requirement: !ruby/object:Gem::Requirement
183
198
  requirements:
184
199
  - - ">="
185
200
  - !ruby/object:Gem::Version
186
201
  version: '0'
187
- name: pry
188
- prerelease: false
189
202
  type: :development
203
+ prerelease: false
190
204
  version_requirements: !ruby/object:Gem::Requirement
191
205
  requirements:
192
206
  - - ">="
193
207
  - !ruby/object:Gem::Version
194
208
  version: '0'
195
209
  - !ruby/object:Gem::Dependency
210
+ name: rspec
196
211
  requirement: !ruby/object:Gem::Requirement
197
212
  requirements:
198
213
  - - ">="
199
214
  - !ruby/object:Gem::Version
200
215
  version: '0'
201
- name: rspec
202
- prerelease: false
203
216
  type: :development
217
+ prerelease: false
204
218
  version_requirements: !ruby/object:Gem::Requirement
205
219
  requirements:
206
220
  - - ">="
207
221
  - !ruby/object:Gem::Version
208
222
  version: '0'
209
223
  - !ruby/object:Gem::Dependency
224
+ name: rack-test
210
225
  requirement: !ruby/object:Gem::Requirement
211
226
  requirements:
212
227
  - - ">="
213
228
  - !ruby/object:Gem::Version
214
229
  version: '0'
215
- name: rack-test
216
- prerelease: false
217
230
  type: :development
231
+ prerelease: false
218
232
  version_requirements: !ruby/object:Gem::Requirement
219
233
  requirements:
220
234
  - - ">="
221
235
  - !ruby/object:Gem::Version
222
236
  version: '0'
223
237
  description: OpeNER daemon for processing multiple queues at once
224
- email:
238
+ email:
225
239
  executables:
226
240
  - chained-daemon
227
241
  - chained-daemon-daemon
242
+ - chained-daemon-csv
228
243
  extensions: []
229
244
  extra_rdoc_files: []
230
245
  files:
231
246
  - bin/chained-daemon
247
+ - bin/chained-daemon-csv
232
248
  - bin/chained-daemon-daemon
233
249
  - exec/chained-daemon.rb
234
250
  - lib/opener/chained_daemon.rb
@@ -236,11 +252,11 @@ files:
236
252
  - lib/opener/chained_daemon/cli.rb
237
253
  - lib/opener/chained_daemon/version.rb
238
254
  - opener-chained-daemon.gemspec
239
- homepage:
255
+ homepage:
240
256
  licenses:
241
257
  - Apache 2.0
242
258
  metadata: {}
243
- post_install_message:
259
+ post_install_message:
244
260
  rdoc_options: []
245
261
  require_paths:
246
262
  - lib
@@ -255,9 +271,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
255
271
  - !ruby/object:Gem::Version
256
272
  version: '0'
257
273
  requirements: []
258
- rubyforge_project:
259
- rubygems_version: 2.6.13
260
- signing_key:
274
+ rubyforge_project:
275
+ rubygems_version: 2.7.8
276
+ signing_key:
261
277
  specification_version: 4
262
278
  summary: OpeNER daemon for processing multiple queues at once
263
279
  test_files: []