opener-chained-daemon 3.1.1 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/opener/chained_daemon.rb +2 -0
- data/lib/opener/chained_daemon/chained_daemon.rb +11 -9
- data/lib/opener/chained_daemon/cli.rb +1 -6
- data/lib/opener/chained_daemon/version.rb +1 -1
- data/lib/opener/stanza/tokenizer_pos.rb +19 -10
- data/lib/opener/sym_mash.rb +14 -0
- data/opener-chained-daemon.gemspec +2 -2
- metadata +11 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ccbd3019511ea9dd574ac013aeadb58f5c480e5a71a641320b047f6220c5288a
|
4
|
+
data.tar.gz: 5dfa19e0fc0d176bd9bfe5cdbc04740ebf1ee2c15165f99454552d6686f751fe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3d2b23910cdb755ac043ea9143716a18fa305f7dfcb381914dc8387fd94ce72de7bf4a6bca163280c5b2ef14298191aa21604ae0fca01a7e5f77153c0cf63fa6
|
7
|
+
data.tar.gz: 8c56eb7b23ad95e7d6a50f059154f870fd3b6cd16a3f4ffa2532ea5898e2ff52f903b71138eba3e06e679ce77a03c03972bb1df1cb5238305ddff764d4918274
|
@@ -11,17 +11,19 @@ module Opener
|
|
11
11
|
'stanza-tokenizer-pos': Stanza::TokenizerPos.new, # replace this tokenizer-pos with both below with you dont have a stanza server
|
12
12
|
#'opener-tokenizer': Opener::Tokenizer.new,
|
13
13
|
#'opener-pos-tagger': Opener::POSTagger.new,
|
14
|
-
'opener-polarity-tagger': Opener::PolarityTagger.new,
|
15
14
|
'opener-property-tagger': Opener::PropertyTagger.new,
|
15
|
+
'opener-polarity-tagger': Opener::PolarityTagger.new,
|
16
16
|
'opener-opinion-detector-basic': Opener::OpinionDetectorBasic.new,
|
17
17
|
}
|
18
18
|
end
|
19
19
|
|
20
|
-
def run input,
|
21
|
-
params
|
22
|
-
params.
|
23
|
-
params
|
24
|
-
params
|
20
|
+
def run input, _params = {}
|
21
|
+
params = SymMash.new _params
|
22
|
+
params.translate_languages ||= []
|
23
|
+
params.cache_keys = SymMash.new params.cache_keys&.to_h&.sort&.to_h || {}
|
24
|
+
if params.filter_vertical and params.property_type.present?
|
25
|
+
params.cache_keys.property_type = params.property_type
|
26
|
+
end
|
25
27
|
|
26
28
|
lang = nil
|
27
29
|
output = nil
|
@@ -34,7 +36,7 @@ module Opener
|
|
34
36
|
rescue Core::UnsupportedLanguageError
|
35
37
|
xml = Nokogiri.parse input
|
36
38
|
lang = xml.root.attr('xml:lang')
|
37
|
-
raise unless lang.in? params
|
39
|
+
raise unless lang.in? params.translate_languages
|
38
40
|
|
39
41
|
input = translate xml, params
|
40
42
|
retry
|
@@ -47,7 +49,7 @@ module Opener
|
|
47
49
|
output = xml.to_s
|
48
50
|
end
|
49
51
|
|
50
|
-
output = pretty_print output if params
|
52
|
+
output = pretty_print output if params.cache_keys&.environment == 'staging'
|
51
53
|
output
|
52
54
|
|
53
55
|
rescue Core::UnsupportedLanguageError
|
@@ -84,7 +86,7 @@ module Opener
|
|
84
86
|
protected
|
85
87
|
|
86
88
|
def translate_service params
|
87
|
-
params
|
89
|
+
params.translate_service&.to_sym || :google
|
88
90
|
end
|
89
91
|
|
90
92
|
def google_translator
|
@@ -47,12 +47,7 @@ Example:
|
|
47
47
|
run do |opts, args|
|
48
48
|
daemon = ChainedDaemon.new args: args
|
49
49
|
input = STDIN.tty? ? nil : STDIN.read
|
50
|
-
|
51
|
-
params = if ENV['PARAMS']
|
52
|
-
JSON.parse ENV['PARAMS']
|
53
|
-
else
|
54
|
-
{}
|
55
|
-
end
|
50
|
+
params = if ENV['PARAMS'] then JSON.parse ENV['PARAMS'] else {} end
|
56
51
|
# Set environment as staging from console for testing purposes
|
57
52
|
params[:cache_keys] = { environment: 'staging', merged: true }
|
58
53
|
|
@@ -43,33 +43,42 @@ module Opener
|
|
43
43
|
raise Core::UnsupportedLanguageError.new kaf.language
|
44
44
|
end
|
45
45
|
|
46
|
-
|
46
|
+
input = kaf.raw
|
47
|
+
input = input.gsub(/\,[^\ ]/, ', ')
|
48
|
+
response = Faraday.post BASE_URL, {lang: kaf.language, input: input}.to_query
|
47
49
|
raise Core::UnsupportedLanguageError, kaf.language if response.status == 406
|
48
50
|
raise response.body if response.status >= 400
|
49
51
|
tokens = JSON.parse response.body
|
50
52
|
|
51
53
|
w_index = 0
|
52
54
|
|
55
|
+
miscs = {}
|
56
|
+
tokens.each_with_index do |t, i|
|
57
|
+
miscs[i] = {}
|
58
|
+
t.each do |word|
|
59
|
+
word['id'].is_a?(Array) && word['id'].each { |id| miscs[i][id] = word['misc'] }
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
53
63
|
tokens.map{ |t| t.reverse! } if RTL_LANGUAGES.include? kaf.language
|
54
64
|
tokens.each_with_index do |sentence, s_index|
|
55
|
-
miscs = {}
|
56
65
|
sentence.each_with_index do |word|
|
57
66
|
w_index += 1
|
58
67
|
# save misc for later usase in a MWT case
|
59
|
-
if word['id'].is_a? Array
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
68
|
+
next if word['id'].is_a? Array
|
69
|
+
|
70
|
+
misc = word['misc'] || miscs[s_index][word['id']]
|
71
|
+
|
72
|
+
Rollbar.scoped({ input: input, params: params, tokens: tokens, word: word }) do
|
73
|
+
raise 'Missing misc'
|
74
|
+
end if misc.nil?
|
64
75
|
|
65
76
|
offset = misc.match(/start_char=(\d+)|/)[1].to_i
|
66
77
|
length = misc.match(/end_char=(\d+)/)[1].to_i - offset
|
67
78
|
|
68
79
|
u_pos = word['upos']
|
69
80
|
pos = POS[u_pos]
|
70
|
-
|
71
|
-
raise "Didn't find a map for #{u_pos}"
|
72
|
-
end if pos.nil?
|
81
|
+
raise "Didn't find a map for #{u_pos}" if pos.nil?
|
73
82
|
type = if POS_OPEN.include? pos then 'open' else 'close' end
|
74
83
|
|
75
84
|
params = {
|
@@ -35,8 +35,8 @@ Gem::Specification.new do |spec|
|
|
35
35
|
spec.add_dependency 'opener-language-identifier', '>= 4.4.0'
|
36
36
|
spec.add_dependency 'opener-tokenizer', '>= 2.2.0'
|
37
37
|
spec.add_dependency 'opener-pos-tagger', '>= 3.2.0'
|
38
|
-
spec.add_dependency 'opener-
|
39
|
-
spec.add_dependency 'opener-
|
38
|
+
spec.add_dependency 'opener-property-tagger', '>= 3.3.6'
|
39
|
+
spec.add_dependency 'opener-polarity-tagger', '>= 3.3.0'
|
40
40
|
spec.add_dependency 'opener-opinion-detector-basic', '>= 3.2.3'
|
41
41
|
|
42
42
|
spec.add_development_dependency 'bundler', '~> 1.3'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: opener-chained-daemon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- development@olery.com
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -179,33 +179,33 @@ dependencies:
|
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: 3.2.0
|
181
181
|
- !ruby/object:Gem::Dependency
|
182
|
-
name: opener-
|
182
|
+
name: opener-property-tagger
|
183
183
|
requirement: !ruby/object:Gem::Requirement
|
184
184
|
requirements:
|
185
185
|
- - ">="
|
186
186
|
- !ruby/object:Gem::Version
|
187
|
-
version: 3.
|
187
|
+
version: 3.3.6
|
188
188
|
type: :runtime
|
189
189
|
prerelease: false
|
190
190
|
version_requirements: !ruby/object:Gem::Requirement
|
191
191
|
requirements:
|
192
192
|
- - ">="
|
193
193
|
- !ruby/object:Gem::Version
|
194
|
-
version: 3.
|
194
|
+
version: 3.3.6
|
195
195
|
- !ruby/object:Gem::Dependency
|
196
|
-
name: opener-
|
196
|
+
name: opener-polarity-tagger
|
197
197
|
requirement: !ruby/object:Gem::Requirement
|
198
198
|
requirements:
|
199
199
|
- - ">="
|
200
200
|
- !ruby/object:Gem::Version
|
201
|
-
version: 3.3.
|
201
|
+
version: 3.3.0
|
202
202
|
type: :runtime
|
203
203
|
prerelease: false
|
204
204
|
version_requirements: !ruby/object:Gem::Requirement
|
205
205
|
requirements:
|
206
206
|
- - ">="
|
207
207
|
- !ruby/object:Gem::Version
|
208
|
-
version: 3.3.
|
208
|
+
version: 3.3.0
|
209
209
|
- !ruby/object:Gem::Dependency
|
210
210
|
name: opener-opinion-detector-basic
|
211
211
|
requirement: !ruby/object:Gem::Requirement
|
@@ -308,9 +308,9 @@ description: OpeNER daemon for processing multiple queues at once
|
|
308
308
|
email:
|
309
309
|
executables:
|
310
310
|
- chained-daemon
|
311
|
-
- chained-daemon-daemon
|
312
311
|
- chained-daemon-csv
|
313
312
|
- console
|
313
|
+
- chained-daemon-daemon
|
314
314
|
extensions: []
|
315
315
|
extra_rdoc_files: []
|
316
316
|
files:
|
@@ -331,6 +331,7 @@ files:
|
|
331
331
|
- lib/opener/kaf/term.rb
|
332
332
|
- lib/opener/kaf/text.rb
|
333
333
|
- lib/opener/stanza/tokenizer_pos.rb
|
334
|
+
- lib/opener/sym_mash.rb
|
334
335
|
- opener-chained-daemon.gemspec
|
335
336
|
homepage:
|
336
337
|
licenses:
|
@@ -352,7 +353,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
352
353
|
version: '0'
|
353
354
|
requirements: []
|
354
355
|
rubyforge_project:
|
355
|
-
rubygems_version: 2.7.
|
356
|
+
rubygems_version: 2.7.8
|
356
357
|
signing_key:
|
357
358
|
specification_version: 4
|
358
359
|
summary: OpeNER daemon for processing multiple queues at once
|