opener-chained-daemon 1.0.2 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/bin/chained-daemon-csv +41 -0
  3. data/lib/opener/chained_daemon/chained_daemon.rb +65 -11
  4. data/lib/opener/chained_daemon/cli.rb +2 -1
  5. data/lib/opener/chained_daemon/version.rb +1 -1
  6. data/lib/opener/chained_daemon.rb +1 -0
  7. data/opener-chained-daemon.gemspec +3 -1
  8. metadata +57 -41
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 529591ceec3fc6900d961d97e5064647afacf3c6216d59a36d339b4b17f86ce6
4
- data.tar.gz: cfb6977e833c25e30644684cc60b2c6a487f1e2e5fbd3a5076d6fd0f29b4ffa5
3
+ metadata.gz: acaa68db270b274f2cf20783d7407b1baeb967ba956ee3d1122aaf5f72156fb8
4
+ data.tar.gz: 336b2d8cf72c6a7b011acae5335de015bc29158042706c4e5c0729ca845feab9
5
5
  SHA512:
6
- metadata.gz: b06587c11fcedbc69802bbed5c823504456e21b69c0c86dc5083bffcde95a9b77c0201d778668da4c47e21d705421e37c8aa1d60626aa8c6a1ab004a1f4d3f33
7
- data.tar.gz: 541dbb5b3e944cea173b1bed70561fb4b3aae1a9f8146a8f797b2fc0c6d2a5a83567a5ddce06a2910d9e5c5d242baa4e615960a81b1c24db0c557cf5b98ccd79
6
+ metadata.gz: dad04d51c0b89e08c928a41c6c45c88174303f47d457f7576294e31b4b04341add667d4f2390859b120d20026728f66e3b8272551b35eef434f8292f2a1e9bf7
7
+ data.tar.gz: 4468af5b3d8e96ecf7df73cf0cf2d109b0c8af92236f51597403ff3fb78ba07c741d652e8f2ab4457d70b4219ed477e90ba9d09edb2bd13eae475efdda50c6ee
data/bin/chained-daemon-csv ADDED
@@ -0,0 +1,41 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler/setup'
4
+ require_relative '../lib/opener/chained_daemon'
5
+ require_relative '../../kaf-processor/lib/kaf_processor/exts/oga'
6
+ require_relative '../../kaf-processor/lib/kaf_processor/services/string_cleaner'
7
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/document'
8
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/property'
9
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/sentiment'
10
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/term'
11
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/opinion'
12
+ require_relative '../../kaf-processor/lib/kaf_processor/kaf/word_form'
13
+ require_relative '../../kaf-processor/lib/kaf_processor/configuration'
14
+ require_relative '../../kaf-processor/lib/kaf_processor/processor/review_opinions'
15
+ require 'csv'
16
+
17
+ if ARGV.size != 1
18
+ puts 'USAGE: chained-daemon-csv file.csv'
19
+ exit
20
+ end
21
+
22
+
23
+ headers = [:review_id, :property, :polarity, :strength, :comment]
24
+ out_csv = CSV.open('/tmp/opinions.csv', 'w', write_headers: true, headers: headers)
25
+
26
+ CSV.foreach(ARGV.first, headers: true).each do |review|
27
+ #puts review['comment']
28
+ cd = Opener::ChainedDaemon.new
29
+ output = cd.run(review['comment'])
30
+ document = KafProcessor::KAF::Document.from_xml output
31
+ config = KafProcessor::Configuration.new identifier: review['id']
32
+ opinions = KafProcessor::Processor::ReviewOpinions.new(config, document).generate_rows
33
+ opinions.each do |o|
34
+ out_csv << [o[:review_id], o[:properties].join(', '), o[:polarity], o[:strength], review['comment']]
35
+ end
36
+ if opinions.empty?
37
+ out_csv << [review['id'], '', '', '', review['comment']]
38
+ end
39
+ end
40
+
41
+ out_csv.close
data/lib/opener/chained_daemon/chained_daemon.rb CHANGED
@@ -7,27 +7,81 @@ module Opener
7
7
  def initialize options = {}
8
8
  @options = DEFAULT_OPTIONS.merge options
9
9
  @queue_map = {
10
- 'opener-language-identifier': Opener::LanguageIdentifier.new,
11
- 'opener-tokenizer': Opener::Tokenizer.new,
12
- 'opener-pos-tagger': Opener::POSTagger.new,
13
- 'opener-polarity-tagger': Opener::PolarityTagger.new,
14
- 'opener-property-tagger': Opener::PropertyTagger.new,
15
- 'opener-ner': Opener::Ner.new,
16
- 'opener-opinion-detector-basic': Opener::OpinionDetectorBasic.new,
17
- }
10
+ 'opener-language-identifier': Opener::LanguageIdentifier.new,
11
+ 'opener-tokenizer': Opener::Tokenizer.new,
12
+ 'opener-pos-tagger': Opener::POSTagger.new,
13
+ 'opener-polarity-tagger': Opener::PolarityTagger.new,
14
+ 'opener-property-tagger': Opener::PropertyTagger.new,
15
+ 'opener-ner': Opener::Ner.new,
16
+ 'opener-opinion-detector-basic': Opener::OpinionDetectorBasic.new,
17
+ }
18
18
  end
19
19
 
20
- def run input
21
- output = nil
20
+ def run input, params = {}
21
+ params.symbolize_keys!
22
+ params[:translate_languages] ||= []
23
+
24
+ lang = nil
25
+ output = nil
22
26
  @queue_map.each do |queue, component|
23
- File.write "input-#{queue}", input if ENV['DEBUG']
27
+ debug_print queue, input if ENV['DEBUG']
28
+
24
29
  output = component.run input
25
30
  input = output
31
+
32
+ rescue Core::UnsupportedLanguageError
33
+ xml = Nokogiri.parse input
34
+ lang = xml.root.attr('xml:lang')
35
+ raise unless lang.in? params[:translate_languages]
36
+
37
+ input = translate xml, params
38
+ retry
26
39
  end
40
+
41
+ if lang
42
+ # put back original language
43
+ xml = Nokogiri.parse output
44
+ xml.root.attributes['lang'].value = lang
45
+ output = xml.to_s
46
+ end
47
+
27
48
  output
49
+
28
50
  rescue Core::UnsupportedLanguageError
51
+ puts "Error on unsupported language: #{input}" if ENV['DEBUG']
29
52
  output
30
53
  end
31
54
 
55
+ def translate xml, params
56
+ raw = xml.at :raw
57
+ case translate_service params
58
+ when :google
59
+ raw.content = google_translator.translate raw.content, to: :en
60
+ else
61
+ raw.content = google_translator.translate raw.content, to: :en
62
+ end
63
+
64
+ xml.root.attributes['lang'].value = 'en'
65
+ xml.to_s
66
+ end
67
+
68
+ protected
69
+
70
+ def translate_service params
71
+ params[:translate_service]&.to_sym || :google
72
+ end
73
+
74
+ def google_translator
75
+ @google_translator ||= Google::Cloud.new.translate ENV['GOOGLE_TRANSLATE_TOKEN']
76
+ end
77
+
78
+ private
79
+
80
+ def debug_print queue, input
81
+ return unless ENV['DEBUG']
82
+ File.write "input-#{queue}", input if ENV['DEBUG']
83
+ puts input
84
+ end
85
+
32
86
  end
33
87
  end
data/lib/opener/chained_daemon/cli.rb CHANGED
@@ -47,8 +47,9 @@ Example:
47
47
  run do |opts, args|
48
48
  daemon = ChainedDaemon.new args: args
49
49
  input = STDIN.tty? ? nil : STDIN.read
50
+ params = JSON.parse ENV['PARAMS'] if ENV['PARAMS']
50
51
 
51
- puts daemon.run input
52
+ puts daemon.run input, params || {}
52
53
  end
53
54
  end
54
55
  end
data/lib/opener/chained_daemon/version.rb CHANGED
@@ -1,7 +1,7 @@
1
1
  module Opener
2
2
  class ChainedDaemon
3
3
 
4
- VERSION = '1.0.2'
4
+ VERSION = '2.0.0'
5
5
 
6
6
  end
7
7
  end
data/lib/opener/chained_daemon.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  require 'active_support/all'
2
2
  require 'oga'
3
+ require 'google/cloud/translate'
3
4
 
4
5
  require 'opener/daemons'
5
6
 
data/opener-chained-daemon.gemspec CHANGED
@@ -21,7 +21,9 @@ Gem::Specification.new do |spec|
21
21
  spec.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
22
22
 
23
23
  spec.add_dependency 'activesupport'
24
- spec.add_dependency 'opener-daemons'
24
+ spec.add_dependency 'google-cloud-translate', '~> 1.0'
25
+
26
+ spec.add_dependency 'opener-daemons', '>= 2.7'
25
27
  spec.add_dependency 'opener-callback-handler', '~> 1.0'
26
28
 
27
29
  spec.add_dependency 'opener-language-identifier'
metadata CHANGED
@@ -1,234 +1,250 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: opener-chained-daemon
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 2.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - development@olery.com
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-06-07 00:00:00.000000000 Z
11
+ date: 2020-03-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ name: activesupport
14
15
  requirement: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - ">="
17
18
  - !ruby/object:Gem::Version
18
19
  version: '0'
19
- name: activesupport
20
- prerelease: false
21
20
  type: :runtime
21
+ prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
+ name: google-cloud-translate
28
29
  requirement: !ruby/object:Gem::Requirement
29
30
  requirements:
30
- - - ">="
31
+ - - "~>"
31
32
  - !ruby/object:Gem::Version
32
- version: '0'
33
- name: opener-daemons
33
+ version: '1.0'
34
+ type: :runtime
34
35
  prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: opener-daemons
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '2.7'
35
48
  type: :runtime
49
+ prerelease: false
36
50
  version_requirements: !ruby/object:Gem::Requirement
37
51
  requirements:
38
52
  - - ">="
39
53
  - !ruby/object:Gem::Version
40
- version: '0'
54
+ version: '2.7'
41
55
  - !ruby/object:Gem::Dependency
56
+ name: opener-callback-handler
42
57
  requirement: !ruby/object:Gem::Requirement
43
58
  requirements:
44
59
  - - "~>"
45
60
  - !ruby/object:Gem::Version
46
61
  version: '1.0'
47
- name: opener-callback-handler
48
- prerelease: false
49
62
  type: :runtime
63
+ prerelease: false
50
64
  version_requirements: !ruby/object:Gem::Requirement
51
65
  requirements:
52
66
  - - "~>"
53
67
  - !ruby/object:Gem::Version
54
68
  version: '1.0'
55
69
  - !ruby/object:Gem::Dependency
70
+ name: opener-language-identifier
56
71
  requirement: !ruby/object:Gem::Requirement
57
72
  requirements:
58
73
  - - ">="
59
74
  - !ruby/object:Gem::Version
60
75
  version: '0'
61
- name: opener-language-identifier
62
- prerelease: false
63
76
  type: :runtime
77
+ prerelease: false
64
78
  version_requirements: !ruby/object:Gem::Requirement
65
79
  requirements:
66
80
  - - ">="
67
81
  - !ruby/object:Gem::Version
68
82
  version: '0'
69
83
  - !ruby/object:Gem::Dependency
84
+ name: opener-tokenizer
70
85
  requirement: !ruby/object:Gem::Requirement
71
86
  requirements:
72
87
  - - ">="
73
88
  - !ruby/object:Gem::Version
74
89
  version: '0'
75
- name: opener-tokenizer
76
- prerelease: false
77
90
  type: :runtime
91
+ prerelease: false
78
92
  version_requirements: !ruby/object:Gem::Requirement
79
93
  requirements:
80
94
  - - ">="
81
95
  - !ruby/object:Gem::Version
82
96
  version: '0'
83
97
  - !ruby/object:Gem::Dependency
98
+ name: opener-pos-tagger
84
99
  requirement: !ruby/object:Gem::Requirement
85
100
  requirements:
86
101
  - - ">="
87
102
  - !ruby/object:Gem::Version
88
103
  version: '0'
89
- name: opener-pos-tagger
90
- prerelease: false
91
104
  type: :runtime
105
+ prerelease: false
92
106
  version_requirements: !ruby/object:Gem::Requirement
93
107
  requirements:
94
108
  - - ">="
95
109
  - !ruby/object:Gem::Version
96
110
  version: '0'
97
111
  - !ruby/object:Gem::Dependency
112
+ name: opener-polarity-tagger
98
113
  requirement: !ruby/object:Gem::Requirement
99
114
  requirements:
100
115
  - - ">="
101
116
  - !ruby/object:Gem::Version
102
117
  version: '0'
103
- name: opener-polarity-tagger
104
- prerelease: false
105
118
  type: :runtime
119
+ prerelease: false
106
120
  version_requirements: !ruby/object:Gem::Requirement
107
121
  requirements:
108
122
  - - ">="
109
123
  - !ruby/object:Gem::Version
110
124
  version: '0'
111
125
  - !ruby/object:Gem::Dependency
126
+ name: opener-property-tagger
112
127
  requirement: !ruby/object:Gem::Requirement
113
128
  requirements:
114
129
  - - ">="
115
130
  - !ruby/object:Gem::Version
116
131
  version: '0'
117
- name: opener-property-tagger
118
- prerelease: false
119
132
  type: :runtime
133
+ prerelease: false
120
134
  version_requirements: !ruby/object:Gem::Requirement
121
135
  requirements:
122
136
  - - ">="
123
137
  - !ruby/object:Gem::Version
124
138
  version: '0'
125
139
  - !ruby/object:Gem::Dependency
140
+ name: opener-ner
126
141
  requirement: !ruby/object:Gem::Requirement
127
142
  requirements:
128
143
  - - ">="
129
144
  - !ruby/object:Gem::Version
130
145
  version: '0'
131
- name: opener-ner
132
- prerelease: false
133
146
  type: :runtime
147
+ prerelease: false
134
148
  version_requirements: !ruby/object:Gem::Requirement
135
149
  requirements:
136
150
  - - ">="
137
151
  - !ruby/object:Gem::Version
138
152
  version: '0'
139
153
  - !ruby/object:Gem::Dependency
154
+ name: opener-opinion-detector-basic
140
155
  requirement: !ruby/object:Gem::Requirement
141
156
  requirements:
142
157
  - - ">="
143
158
  - !ruby/object:Gem::Version
144
159
  version: '0'
145
- name: opener-opinion-detector-basic
146
- prerelease: false
147
160
  type: :runtime
161
+ prerelease: false
148
162
  version_requirements: !ruby/object:Gem::Requirement
149
163
  requirements:
150
164
  - - ">="
151
165
  - !ruby/object:Gem::Version
152
166
  version: '0'
153
167
  - !ruby/object:Gem::Dependency
168
+ name: bundler
154
169
  requirement: !ruby/object:Gem::Requirement
155
170
  requirements:
156
171
  - - "~>"
157
172
  - !ruby/object:Gem::Version
158
173
  version: '1.3'
159
- name: bundler
160
- prerelease: false
161
174
  type: :development
175
+ prerelease: false
162
176
  version_requirements: !ruby/object:Gem::Requirement
163
177
  requirements:
164
178
  - - "~>"
165
179
  - !ruby/object:Gem::Version
166
180
  version: '1.3'
167
181
  - !ruby/object:Gem::Dependency
182
+ name: rake
168
183
  requirement: !ruby/object:Gem::Requirement
169
184
  requirements:
170
185
  - - ">="
171
186
  - !ruby/object:Gem::Version
172
187
  version: '0'
173
- name: rake
174
- prerelease: false
175
188
  type: :development
189
+ prerelease: false
176
190
  version_requirements: !ruby/object:Gem::Requirement
177
191
  requirements:
178
192
  - - ">="
179
193
  - !ruby/object:Gem::Version
180
194
  version: '0'
181
195
  - !ruby/object:Gem::Dependency
196
+ name: pry
182
197
  requirement: !ruby/object:Gem::Requirement
183
198
  requirements:
184
199
  - - ">="
185
200
  - !ruby/object:Gem::Version
186
201
  version: '0'
187
- name: pry
188
- prerelease: false
189
202
  type: :development
203
+ prerelease: false
190
204
  version_requirements: !ruby/object:Gem::Requirement
191
205
  requirements:
192
206
  - - ">="
193
207
  - !ruby/object:Gem::Version
194
208
  version: '0'
195
209
  - !ruby/object:Gem::Dependency
210
+ name: rspec
196
211
  requirement: !ruby/object:Gem::Requirement
197
212
  requirements:
198
213
  - - ">="
199
214
  - !ruby/object:Gem::Version
200
215
  version: '0'
201
- name: rspec
202
- prerelease: false
203
216
  type: :development
217
+ prerelease: false
204
218
  version_requirements: !ruby/object:Gem::Requirement
205
219
  requirements:
206
220
  - - ">="
207
221
  - !ruby/object:Gem::Version
208
222
  version: '0'
209
223
  - !ruby/object:Gem::Dependency
224
+ name: rack-test
210
225
  requirement: !ruby/object:Gem::Requirement
211
226
  requirements:
212
227
  - - ">="
213
228
  - !ruby/object:Gem::Version
214
229
  version: '0'
215
- name: rack-test
216
- prerelease: false
217
230
  type: :development
231
+ prerelease: false
218
232
  version_requirements: !ruby/object:Gem::Requirement
219
233
  requirements:
220
234
  - - ">="
221
235
  - !ruby/object:Gem::Version
222
236
  version: '0'
223
237
  description: OpeNER daemon for processing multiple queues at once
224
- email:
238
+ email:
225
239
  executables:
226
240
  - chained-daemon
227
241
  - chained-daemon-daemon
242
+ - chained-daemon-csv
228
243
  extensions: []
229
244
  extra_rdoc_files: []
230
245
  files:
231
246
  - bin/chained-daemon
247
+ - bin/chained-daemon-csv
232
248
  - bin/chained-daemon-daemon
233
249
  - exec/chained-daemon.rb
234
250
  - lib/opener/chained_daemon.rb
@@ -236,11 +252,11 @@ files:
236
252
  - lib/opener/chained_daemon/cli.rb
237
253
  - lib/opener/chained_daemon/version.rb
238
254
  - opener-chained-daemon.gemspec
239
- homepage:
255
+ homepage:
240
256
  licenses:
241
257
  - Apache 2.0
242
258
  metadata: {}
243
- post_install_message:
259
+ post_install_message:
244
260
  rdoc_options: []
245
261
  require_paths:
246
262
  - lib
@@ -255,9 +271,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
255
271
  - !ruby/object:Gem::Version
256
272
  version: '0'
257
273
  requirements: []
258
- rubyforge_project:
259
- rubygems_version: 2.6.13
260
- signing_key:
274
+ rubyforge_project:
275
+ rubygems_version: 2.7.8
276
+ signing_key:
261
277
  specification_version: 4
262
278
  summary: OpeNER daemon for processing multiple queues at once
263
279
  test_files: []