opener-chained-daemon 1.0.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/chained-daemon-csv +41 -0
- data/lib/opener/chained_daemon/chained_daemon.rb +65 -11
- data/lib/opener/chained_daemon/cli.rb +2 -1
- data/lib/opener/chained_daemon/version.rb +1 -1
- data/lib/opener/chained_daemon.rb +1 -0
- data/opener-chained-daemon.gemspec +3 -1
- metadata +57 -41
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: acaa68db270b274f2cf20783d7407b1baeb967ba956ee3d1122aaf5f72156fb8
|
4
|
+
data.tar.gz: 336b2d8cf72c6a7b011acae5335de015bc29158042706c4e5c0729ca845feab9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dad04d51c0b89e08c928a41c6c45c88174303f47d457f7576294e31b4b04341add667d4f2390859b120d20026728f66e3b8272551b35eef434f8292f2a1e9bf7
|
7
|
+
data.tar.gz: 4468af5b3d8e96ecf7df73cf0cf2d109b0c8af92236f51597403ff3fb78ba07c741d652e8f2ab4457d70b4219ed477e90ba9d09edb2bd13eae475efdda50c6ee
|
@@ -0,0 +1,41 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require_relative '../lib/opener/chained_daemon'
|
5
|
+
require_relative '../../kaf-processor/lib/kaf_processor/exts/oga'
|
6
|
+
require_relative '../../kaf-processor/lib/kaf_processor/services/string_cleaner'
|
7
|
+
require_relative '../../kaf-processor/lib/kaf_processor/kaf/document'
|
8
|
+
require_relative '../../kaf-processor/lib/kaf_processor/kaf/property'
|
9
|
+
require_relative '../../kaf-processor/lib/kaf_processor/kaf/sentiment'
|
10
|
+
require_relative '../../kaf-processor/lib/kaf_processor/kaf/term'
|
11
|
+
require_relative '../../kaf-processor/lib/kaf_processor/kaf/opinion'
|
12
|
+
require_relative '../../kaf-processor/lib/kaf_processor/kaf/word_form'
|
13
|
+
require_relative '../../kaf-processor/lib/kaf_processor/configuration'
|
14
|
+
require_relative '../../kaf-processor/lib/kaf_processor/processor/review_opinions'
|
15
|
+
require 'csv'
|
16
|
+
|
17
|
+
if ARGV.size != 1
|
18
|
+
puts 'USAGE: chained-daemon-csv file.csv'
|
19
|
+
exit
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
headers = [:review_id, :property, :polarity, :strength, :comment]
|
24
|
+
out_csv = CSV.open('/tmp/opinions.csv', 'w', write_headers: true, headers: headers)
|
25
|
+
|
26
|
+
CSV.foreach(ARGV.first, headers: true).each do |review|
|
27
|
+
#puts review['comment']
|
28
|
+
cd = Opener::ChainedDaemon.new
|
29
|
+
output = cd.run(review['comment'])
|
30
|
+
document = KafProcessor::KAF::Document.from_xml output
|
31
|
+
config = KafProcessor::Configuration.new identifier: review['id']
|
32
|
+
opinions = KafProcessor::Processor::ReviewOpinions.new(config, document).generate_rows
|
33
|
+
opinions.each do |o|
|
34
|
+
out_csv << [o[:review_id], o[:properties].join(', '), o[:polarity], o[:strength], review['comment']]
|
35
|
+
end
|
36
|
+
if opinions.empty?
|
37
|
+
out_csv << [review['id'], '', '', '', review['comment']]
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
out_csv.close
|
@@ -7,27 +7,81 @@ module Opener
|
|
7
7
|
def initialize options = {}
|
8
8
|
@options = DEFAULT_OPTIONS.merge options
|
9
9
|
@queue_map = {
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
10
|
+
'opener-language-identifier': Opener::LanguageIdentifier.new,
|
11
|
+
'opener-tokenizer': Opener::Tokenizer.new,
|
12
|
+
'opener-pos-tagger': Opener::POSTagger.new,
|
13
|
+
'opener-polarity-tagger': Opener::PolarityTagger.new,
|
14
|
+
'opener-property-tagger': Opener::PropertyTagger.new,
|
15
|
+
'opener-ner': Opener::Ner.new,
|
16
|
+
'opener-opinion-detector-basic': Opener::OpinionDetectorBasic.new,
|
17
|
+
}
|
18
18
|
end
|
19
19
|
|
20
|
-
def run input
|
21
|
-
|
20
|
+
def run input, params = {}
|
21
|
+
params.symbolize_keys!
|
22
|
+
params[:translate_languages] ||= []
|
23
|
+
|
24
|
+
lang = nil
|
25
|
+
output = nil
|
22
26
|
@queue_map.each do |queue, component|
|
23
|
-
|
27
|
+
debug_print queue, input if ENV['DEBUG']
|
28
|
+
|
24
29
|
output = component.run input
|
25
30
|
input = output
|
31
|
+
|
32
|
+
rescue Core::UnsupportedLanguageError
|
33
|
+
xml = Nokogiri.parse input
|
34
|
+
lang = xml.root.attr('xml:lang')
|
35
|
+
raise unless lang.in? params[:translate_languages]
|
36
|
+
|
37
|
+
input = translate xml, params
|
38
|
+
retry
|
26
39
|
end
|
40
|
+
|
41
|
+
if lang
|
42
|
+
# put back original language
|
43
|
+
xml = Nokogiri.parse output
|
44
|
+
xml.root.attributes['lang'].value = lang
|
45
|
+
output = xml.to_s
|
46
|
+
end
|
47
|
+
|
27
48
|
output
|
49
|
+
|
28
50
|
rescue Core::UnsupportedLanguageError
|
51
|
+
puts "Error on unsupported language: #{input}" if ENV['DEBUG']
|
29
52
|
output
|
30
53
|
end
|
31
54
|
|
55
|
+
def translate xml, params
|
56
|
+
raw = xml.at :raw
|
57
|
+
case translate_service params
|
58
|
+
when :google
|
59
|
+
raw.content = google_translator.translate raw.content, to: :en
|
60
|
+
else
|
61
|
+
raw.content = google_translator.translate raw.content, to: :en
|
62
|
+
end
|
63
|
+
|
64
|
+
xml.root.attributes['lang'].value = 'en'
|
65
|
+
xml.to_s
|
66
|
+
end
|
67
|
+
|
68
|
+
protected
|
69
|
+
|
70
|
+
def translate_service params
|
71
|
+
params[:translate_service]&.to_sym || :google
|
72
|
+
end
|
73
|
+
|
74
|
+
def google_translator
|
75
|
+
@google_translator ||= Google::Cloud.new.translate ENV['GOOGLE_TRANSLATE_TOKEN']
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
def debug_print queue, input
|
81
|
+
return unless ENV['DEBUG']
|
82
|
+
File.write "input-#{queue}", input if ENV['DEBUG']
|
83
|
+
puts input
|
84
|
+
end
|
85
|
+
|
32
86
|
end
|
33
87
|
end
|
@@ -47,8 +47,9 @@ Example:
|
|
47
47
|
run do |opts, args|
|
48
48
|
daemon = ChainedDaemon.new args: args
|
49
49
|
input = STDIN.tty? ? nil : STDIN.read
|
50
|
+
params = JSON.parse ENV['PARAMS'] if ENV['PARAMS']
|
50
51
|
|
51
|
-
puts daemon.run input
|
52
|
+
puts daemon.run input, params || {}
|
52
53
|
end
|
53
54
|
end
|
54
55
|
end
|
@@ -21,7 +21,9 @@ Gem::Specification.new do |spec|
|
|
21
21
|
spec.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
|
22
22
|
|
23
23
|
spec.add_dependency 'activesupport'
|
24
|
-
spec.add_dependency '
|
24
|
+
spec.add_dependency 'google-cloud-translate', '~> 1.0'
|
25
|
+
|
26
|
+
spec.add_dependency 'opener-daemons', '>= 2.7'
|
25
27
|
spec.add_dependency 'opener-callback-handler', '~> 1.0'
|
26
28
|
|
27
29
|
spec.add_dependency 'opener-language-identifier'
|
metadata
CHANGED
@@ -1,234 +1,250 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: opener-chained-daemon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- development@olery.com
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-03-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
14
15
|
requirement: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - ">="
|
17
18
|
- !ruby/object:Gem::Version
|
18
19
|
version: '0'
|
19
|
-
name: activesupport
|
20
|
-
prerelease: false
|
21
20
|
type: :runtime
|
21
|
+
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
+
name: google-cloud-translate
|
28
29
|
requirement: !ruby/object:Gem::Requirement
|
29
30
|
requirements:
|
30
|
-
- - "
|
31
|
+
- - "~>"
|
31
32
|
- !ruby/object:Gem::Version
|
32
|
-
version: '0'
|
33
|
-
|
33
|
+
version: '1.0'
|
34
|
+
type: :runtime
|
34
35
|
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: opener-daemons
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '2.7'
|
35
48
|
type: :runtime
|
49
|
+
prerelease: false
|
36
50
|
version_requirements: !ruby/object:Gem::Requirement
|
37
51
|
requirements:
|
38
52
|
- - ">="
|
39
53
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
54
|
+
version: '2.7'
|
41
55
|
- !ruby/object:Gem::Dependency
|
56
|
+
name: opener-callback-handler
|
42
57
|
requirement: !ruby/object:Gem::Requirement
|
43
58
|
requirements:
|
44
59
|
- - "~>"
|
45
60
|
- !ruby/object:Gem::Version
|
46
61
|
version: '1.0'
|
47
|
-
name: opener-callback-handler
|
48
|
-
prerelease: false
|
49
62
|
type: :runtime
|
63
|
+
prerelease: false
|
50
64
|
version_requirements: !ruby/object:Gem::Requirement
|
51
65
|
requirements:
|
52
66
|
- - "~>"
|
53
67
|
- !ruby/object:Gem::Version
|
54
68
|
version: '1.0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
70
|
+
name: opener-language-identifier
|
56
71
|
requirement: !ruby/object:Gem::Requirement
|
57
72
|
requirements:
|
58
73
|
- - ">="
|
59
74
|
- !ruby/object:Gem::Version
|
60
75
|
version: '0'
|
61
|
-
name: opener-language-identifier
|
62
|
-
prerelease: false
|
63
76
|
type: :runtime
|
77
|
+
prerelease: false
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
65
79
|
requirements:
|
66
80
|
- - ">="
|
67
81
|
- !ruby/object:Gem::Version
|
68
82
|
version: '0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
84
|
+
name: opener-tokenizer
|
70
85
|
requirement: !ruby/object:Gem::Requirement
|
71
86
|
requirements:
|
72
87
|
- - ">="
|
73
88
|
- !ruby/object:Gem::Version
|
74
89
|
version: '0'
|
75
|
-
name: opener-tokenizer
|
76
|
-
prerelease: false
|
77
90
|
type: :runtime
|
91
|
+
prerelease: false
|
78
92
|
version_requirements: !ruby/object:Gem::Requirement
|
79
93
|
requirements:
|
80
94
|
- - ">="
|
81
95
|
- !ruby/object:Gem::Version
|
82
96
|
version: '0'
|
83
97
|
- !ruby/object:Gem::Dependency
|
98
|
+
name: opener-pos-tagger
|
84
99
|
requirement: !ruby/object:Gem::Requirement
|
85
100
|
requirements:
|
86
101
|
- - ">="
|
87
102
|
- !ruby/object:Gem::Version
|
88
103
|
version: '0'
|
89
|
-
name: opener-pos-tagger
|
90
|
-
prerelease: false
|
91
104
|
type: :runtime
|
105
|
+
prerelease: false
|
92
106
|
version_requirements: !ruby/object:Gem::Requirement
|
93
107
|
requirements:
|
94
108
|
- - ">="
|
95
109
|
- !ruby/object:Gem::Version
|
96
110
|
version: '0'
|
97
111
|
- !ruby/object:Gem::Dependency
|
112
|
+
name: opener-polarity-tagger
|
98
113
|
requirement: !ruby/object:Gem::Requirement
|
99
114
|
requirements:
|
100
115
|
- - ">="
|
101
116
|
- !ruby/object:Gem::Version
|
102
117
|
version: '0'
|
103
|
-
name: opener-polarity-tagger
|
104
|
-
prerelease: false
|
105
118
|
type: :runtime
|
119
|
+
prerelease: false
|
106
120
|
version_requirements: !ruby/object:Gem::Requirement
|
107
121
|
requirements:
|
108
122
|
- - ">="
|
109
123
|
- !ruby/object:Gem::Version
|
110
124
|
version: '0'
|
111
125
|
- !ruby/object:Gem::Dependency
|
126
|
+
name: opener-property-tagger
|
112
127
|
requirement: !ruby/object:Gem::Requirement
|
113
128
|
requirements:
|
114
129
|
- - ">="
|
115
130
|
- !ruby/object:Gem::Version
|
116
131
|
version: '0'
|
117
|
-
name: opener-property-tagger
|
118
|
-
prerelease: false
|
119
132
|
type: :runtime
|
133
|
+
prerelease: false
|
120
134
|
version_requirements: !ruby/object:Gem::Requirement
|
121
135
|
requirements:
|
122
136
|
- - ">="
|
123
137
|
- !ruby/object:Gem::Version
|
124
138
|
version: '0'
|
125
139
|
- !ruby/object:Gem::Dependency
|
140
|
+
name: opener-ner
|
126
141
|
requirement: !ruby/object:Gem::Requirement
|
127
142
|
requirements:
|
128
143
|
- - ">="
|
129
144
|
- !ruby/object:Gem::Version
|
130
145
|
version: '0'
|
131
|
-
name: opener-ner
|
132
|
-
prerelease: false
|
133
146
|
type: :runtime
|
147
|
+
prerelease: false
|
134
148
|
version_requirements: !ruby/object:Gem::Requirement
|
135
149
|
requirements:
|
136
150
|
- - ">="
|
137
151
|
- !ruby/object:Gem::Version
|
138
152
|
version: '0'
|
139
153
|
- !ruby/object:Gem::Dependency
|
154
|
+
name: opener-opinion-detector-basic
|
140
155
|
requirement: !ruby/object:Gem::Requirement
|
141
156
|
requirements:
|
142
157
|
- - ">="
|
143
158
|
- !ruby/object:Gem::Version
|
144
159
|
version: '0'
|
145
|
-
name: opener-opinion-detector-basic
|
146
|
-
prerelease: false
|
147
160
|
type: :runtime
|
161
|
+
prerelease: false
|
148
162
|
version_requirements: !ruby/object:Gem::Requirement
|
149
163
|
requirements:
|
150
164
|
- - ">="
|
151
165
|
- !ruby/object:Gem::Version
|
152
166
|
version: '0'
|
153
167
|
- !ruby/object:Gem::Dependency
|
168
|
+
name: bundler
|
154
169
|
requirement: !ruby/object:Gem::Requirement
|
155
170
|
requirements:
|
156
171
|
- - "~>"
|
157
172
|
- !ruby/object:Gem::Version
|
158
173
|
version: '1.3'
|
159
|
-
name: bundler
|
160
|
-
prerelease: false
|
161
174
|
type: :development
|
175
|
+
prerelease: false
|
162
176
|
version_requirements: !ruby/object:Gem::Requirement
|
163
177
|
requirements:
|
164
178
|
- - "~>"
|
165
179
|
- !ruby/object:Gem::Version
|
166
180
|
version: '1.3'
|
167
181
|
- !ruby/object:Gem::Dependency
|
182
|
+
name: rake
|
168
183
|
requirement: !ruby/object:Gem::Requirement
|
169
184
|
requirements:
|
170
185
|
- - ">="
|
171
186
|
- !ruby/object:Gem::Version
|
172
187
|
version: '0'
|
173
|
-
name: rake
|
174
|
-
prerelease: false
|
175
188
|
type: :development
|
189
|
+
prerelease: false
|
176
190
|
version_requirements: !ruby/object:Gem::Requirement
|
177
191
|
requirements:
|
178
192
|
- - ">="
|
179
193
|
- !ruby/object:Gem::Version
|
180
194
|
version: '0'
|
181
195
|
- !ruby/object:Gem::Dependency
|
196
|
+
name: pry
|
182
197
|
requirement: !ruby/object:Gem::Requirement
|
183
198
|
requirements:
|
184
199
|
- - ">="
|
185
200
|
- !ruby/object:Gem::Version
|
186
201
|
version: '0'
|
187
|
-
name: pry
|
188
|
-
prerelease: false
|
189
202
|
type: :development
|
203
|
+
prerelease: false
|
190
204
|
version_requirements: !ruby/object:Gem::Requirement
|
191
205
|
requirements:
|
192
206
|
- - ">="
|
193
207
|
- !ruby/object:Gem::Version
|
194
208
|
version: '0'
|
195
209
|
- !ruby/object:Gem::Dependency
|
210
|
+
name: rspec
|
196
211
|
requirement: !ruby/object:Gem::Requirement
|
197
212
|
requirements:
|
198
213
|
- - ">="
|
199
214
|
- !ruby/object:Gem::Version
|
200
215
|
version: '0'
|
201
|
-
name: rspec
|
202
|
-
prerelease: false
|
203
216
|
type: :development
|
217
|
+
prerelease: false
|
204
218
|
version_requirements: !ruby/object:Gem::Requirement
|
205
219
|
requirements:
|
206
220
|
- - ">="
|
207
221
|
- !ruby/object:Gem::Version
|
208
222
|
version: '0'
|
209
223
|
- !ruby/object:Gem::Dependency
|
224
|
+
name: rack-test
|
210
225
|
requirement: !ruby/object:Gem::Requirement
|
211
226
|
requirements:
|
212
227
|
- - ">="
|
213
228
|
- !ruby/object:Gem::Version
|
214
229
|
version: '0'
|
215
|
-
name: rack-test
|
216
|
-
prerelease: false
|
217
230
|
type: :development
|
231
|
+
prerelease: false
|
218
232
|
version_requirements: !ruby/object:Gem::Requirement
|
219
233
|
requirements:
|
220
234
|
- - ">="
|
221
235
|
- !ruby/object:Gem::Version
|
222
236
|
version: '0'
|
223
237
|
description: OpeNER daemon for processing multiple queues at once
|
224
|
-
email:
|
238
|
+
email:
|
225
239
|
executables:
|
226
240
|
- chained-daemon
|
227
241
|
- chained-daemon-daemon
|
242
|
+
- chained-daemon-csv
|
228
243
|
extensions: []
|
229
244
|
extra_rdoc_files: []
|
230
245
|
files:
|
231
246
|
- bin/chained-daemon
|
247
|
+
- bin/chained-daemon-csv
|
232
248
|
- bin/chained-daemon-daemon
|
233
249
|
- exec/chained-daemon.rb
|
234
250
|
- lib/opener/chained_daemon.rb
|
@@ -236,11 +252,11 @@ files:
|
|
236
252
|
- lib/opener/chained_daemon/cli.rb
|
237
253
|
- lib/opener/chained_daemon/version.rb
|
238
254
|
- opener-chained-daemon.gemspec
|
239
|
-
homepage:
|
255
|
+
homepage:
|
240
256
|
licenses:
|
241
257
|
- Apache 2.0
|
242
258
|
metadata: {}
|
243
|
-
post_install_message:
|
259
|
+
post_install_message:
|
244
260
|
rdoc_options: []
|
245
261
|
require_paths:
|
246
262
|
- lib
|
@@ -255,9 +271,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
255
271
|
- !ruby/object:Gem::Version
|
256
272
|
version: '0'
|
257
273
|
requirements: []
|
258
|
-
rubyforge_project:
|
259
|
-
rubygems_version: 2.
|
260
|
-
signing_key:
|
274
|
+
rubyforge_project:
|
275
|
+
rubygems_version: 2.7.8
|
276
|
+
signing_key:
|
261
277
|
specification_version: 4
|
262
278
|
summary: OpeNER daemon for processing multiple queues at once
|
263
279
|
test_files: []
|