opener-chained-daemon 2.2.2 → 3.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb437fa8aeeb2133629c9545254252aa07145cbe9c81a9804703b168cc32c098
-  data.tar.gz: 47c6e1b0462a52710100b201c41b7a3feddaa89d9e80649efc323eaa4546db5e
+  metadata.gz: 1f37e21fff5f35b6736a17e4d4efcad5d496c1c2ec63e74c7b8907db45210770
+  data.tar.gz: 6e3b80ba665a950407b1c4bcba9e0f6074db41db86b2da233b21c1db638c792f
 SHA512:
-  metadata.gz: ff6d5446a98856a6d3981f1610a776374a50b5ae0f78bd3a9040e8f58fc704ce6a2f453475c88ecadc13cd4c0adb872403d60e827c4b2a95fd2ff11391fecb37
-  data.tar.gz: 25e202e8df0be31a8c03a6588b68f30d7a82faec24e1c461302ca8d7369de81a1ee87468df9a7b28cb472cb0aa5c476adbc4b004c525230fbff3aa56b7b841cb
+  metadata.gz: 5738bfe0a4dbf5ff66010a9aa54ef7de4e951e7728d753000b99effbb0f4c17884791a8eb6c58fcfcafd51f0ba94520148f6d3699dd592c74145766e2c06917f
+  data.tar.gz: 7100f63ac6fd859b98f28eac22dd73e60419f21fc40e3b59c34c6550915182ca0598eb74f2ec9eddaa870fd4b91f94b31a694f70fc62ed20307b1415c0e82804

data/config/puma.rb

@@ -0,0 +1,11 @@
+APP_NAME     = 'opener'
+deploy_to    = "/var/www/#{APP_NAME}"
+run_path     = "/var/run/#{APP_NAME}"
+log_path     = "/var/log/#{APP_NAME}"
+
+workers    ENV['WORKERS']&.to_i || 1  unless RUBY_ENGINE == 'jruby'
+threads 0, ENV['THREADS']&.to_i || 16
+
+bind 'tcp://0.0.0.0:9292'
+#bind "unix://#{run_path}/chained-daemon-server.sock"
+

data/exec/chained-daemon.rb

@@ -4,5 +4,8 @@ require 'opener/daemons'
 
 require_relative '../lib/opener/chained_daemon'
 
+webservice = Rack::Server.new app: Opener::ChainedDaemon::Webservice.freeze.app
+Thread.new{ webservice.start }
+
 daemon = Opener::Daemons::Daemon.new Opener::ChainedDaemon
 daemon.start

data/lib/opener/chained_daemon.rb

@@ -5,15 +5,20 @@ require 'google/cloud/translate'
 
 require 'opener/daemons'
 
+require_relative 'chained_daemon/languages_cache'
 require 'opener/language_identifier'
 require 'opener/tokenizer'
-# require ner before pos_tagger for compatible opennlp
-require 'opener/pos_tagger'
+require 'opener/pos_tagger' if RUBY_ENGINE == 'jruby'
 require 'opener/polarity_tagger'
 require 'opener/property_tagger'
 require 'opener/opinion_detector_basic'
+require 'opener/stanza/tokenizer_pos'
 
 require_relative 'chained_daemon/chained_daemon'
 require_relative 'chained_daemon/cli'
 require_relative 'chained_daemon/microsoft_translator'
+require_relative 'chained_daemon/webservice'
+
+require_relative 'kaf/document'
+require_relative 'kaf/term'
 

data/lib/opener/chained_daemon/chained_daemon.rb

@@ -8,8 +8,9 @@ module Opener
       @options   = DEFAULT_OPTIONS.merge options
       @queue_map = {
         'opener-language-identifier':    Opener::LanguageIdentifier.new,
-        'opener-tokenizer':              Opener::Tokenizer.new,
-        'opener-pos-tagger':             Opener::POSTagger.new,
+        'stanza-tokenizer-pos':          Stanza::TokenizerPos.new, # replace this tokenizer-pos with both below with you dont have a stanza server
+        #'opener-tokenizer':              Opener::Tokenizer.new,
+        #'opener-pos-tagger':             Opener::POSTagger.new,
         'opener-polarity-tagger':        Opener::PolarityTagger.new,
         'opener-property-tagger':        Opener::PropertyTagger.new,
         'opener-opinion-detector-basic': Opener::OpinionDetectorBasic.new,
@@ -18,15 +19,16 @@ module Opener
 
     def run input, params = {}
       params ||= {}
-      params.symbolize_keys!
+      params.deep_symbolize_keys!
       params[:translate_languages] ||= []
+      params[:cache_keys] = params[:cache_keys]&.sort&.to_h || {}
 
       lang     = nil
       output   = nil
       @queue_map.each do |queue, component|
         debug_print queue, input if ENV['DEBUG']
 
-        output = component.run input
+        output = component.run input, params
         input  = output
 
       rescue Core::UnsupportedLanguageError

data/lib/opener/chained_daemon/languages_cache.rb

@@ -0,0 +1,45 @@
+module Opener
+  class ChainedDaemon
+    class LanguagesCache
+
+      include MonitorMixin
+
+      UPDATE_INTERVAL = (ENV['CACHE_EXPIRE_MINS']&.to_i || 5).minutes
+
+      def initialize
+        super #MonitorMixin
+
+        @url          = ENV['SUPPORTED_LANGUAGES_URL']
+        @cache        = []
+        @last_updated = nil
+      end
+
+      def get
+        synchronize do
+          break @cache if @last_updated and @last_updated > UPDATE_INTERVAL.ago
+          cache_update
+        end
+        @cache
+      end
+
+      def cache_update
+        puts "loading supported languages from url #{@url}" if ENV['DEBUG']
+
+        languages     = JSON.parse http.get(@url).body
+        @cache        = languages['data'].map { |l| l['code'] }
+        @last_updated = Time.now
+      end
+
+      def http
+        return @http if @http
+
+        @http = HTTPClient.new
+        @http.send_timeout    = 120
+        @http.receive_timeout = 120
+        @http.connect_timeout = 120
+        @http
+      end
+
+    end
+  end
+end

data/lib/opener/chained_daemon/version.rb

@@ -1,7 +1,7 @@
 module Opener
   class ChainedDaemon
 
-    VERSION = '2.2.2'
+    VERSION = '3.0.7'
 
   end
 end

data/lib/opener/chained_daemon/webservice.rb

@@ -0,0 +1,29 @@
+require 'roda'
+
+module Opener
+  class ChainedDaemon
+    class Webservice < Roda
+
+      TOKEN = ENV['PRIVATE_TOKEN']
+
+      class_attribute :processor
+      self.processor = ChainedDaemon.new
+
+      route do |r|
+        response.status = 403 and r.halt if r.params['auth_token'] != TOKEN
+
+        sentiment = -> do
+          r.params.deep_symbolize_keys!
+          kaf = processor.run r.params[:input], **r.params.except(:input)
+
+          response['Content-Type'] = 'text/xml'
+          kaf
+        end
+
+        r.get  'sentiment.kaf', &sentiment
+        r.post 'sentiment.kaf', &sentiment
+      end
+
+    end
+  end
+end

data/lib/opener/kaf/document.rb

@@ -0,0 +1,89 @@
+require_relative 'term'
+
+module Opener
+  module KAF
+    class Document
+
+      attr_reader :document
+      attr_reader :lexicons
+
+      attr_accessor :map
+
+      def initialize xml
+        @document = xml
+      end
+
+      def self.from_xml xml
+        new Nokogiri::XML xml
+      end
+
+      def language
+        @language ||= @document.at_xpath('KAF').attr 'xml:lang'
+      end
+
+      def terms
+        @terms ||= collection 'KAF/terms/term', Term
+      end
+
+      def texts
+        @texts ||= collection 'KAF/texts/wf', Text
+      end
+
+      def raw
+        @document.at('raw').text
+      end
+
+      def add_linguistic_processor name, version, layer, timestamp: false
+        header  = @document.at('kafHeader') || @document.root.add_child('<kafHeader/>').first
+        procs   = header.css('linguisticProcessors').find{ |l| l.attr(:layer) == layer }
+        procs ||= header.add_child("<linguisticProcessors layer='#{layer}'/>").first
+        lp      = procs.add_child('<lp/>')
+        lp.attr(
+          timestamp: if timestamp then Time.now.iso8601 else '*' end,
+          version:   version,
+          name:      name,
+        )
+        lp
+      end
+
+      def add_word_form params
+        text = @document.at('text') || @document.root.add_child('<text/>').first
+        wf   = text.add_child("<wf>#{params[:text]}</wf>")
+        attrs = {
+          wid:    "w#{params[:wid]}",
+          sent:   params[:sid],
+          para:   params[:para],
+          offset: params[:offset],
+          length: params[:length],
+        }
+        wf.attr attrs
+      end
+
+      def add_term params
+        text  = @document.at('terms') || @document.root.add_child('<terms/>').first
+        term  = text.add_child("<term/>")
+        attrs = {
+          tid:        "t#{params[:tid]}",
+          type:       params[:type],
+          lemma:      params[:lemma],
+          text:       params[:text],
+          pos:        params[:pos],
+          morphofeat: params[:morphofeat],
+        }
+        term.attr attrs
+        term.first.add_child("<span><target id='w#{params[:wid]}' /></span>")
+      end
+
+      def to_xml
+        @document.to_xml indent: 2
+      end
+
+      protected
+
+      def collection query, wrapper
+        @document.xpath(query).map{ |node| wrapper.new self, node }
+      end
+
+    end
+  end
+end

data/lib/opener/kaf/term.rb

@@ -0,0 +1,35 @@
+module Opener
+  module KAF
+    class Term
+
+      attr_reader :document
+      attr_reader :node
+
+      def initialize document, node
+        @document = document
+        @node     = node
+      end
+
+      def id
+        @id ||= @node.attr :tid
+      end
+
+      def lemma
+        @node.attr :lemma
+      end
+
+      def pos
+        @node.attr :pos
+      end
+
+      def setPolarity attrs, polarity_pos
+        #In case there is no pos info, we use the polarityPos
+        @node[:pos] = polarity_pos if !pos and polarity_pos
+
+        sentiment = @node.add_child('<sentiment/>')
+        sentiment.attr attrs
+      end
+
+    end
+  end
+end

data/lib/opener/kaf/text.rb

@@ -0,0 +1,30 @@
+module Opener
+  module KAF
+    class WordForm
+      def initialize(document, xml_node)
+        @document = document
+        @xml_node = xml_node
+      end
+
+      def id
+        return @id ||= @xml_node.attr('wid')
+      end
+
+      def text
+        return @text ||= @xml_node.text
+      end
+
+      def length
+        return @length ||= @xml_node.attr('length').to_i
+      end
+
+      def offset
+        return @offset ||= @xml_node.attr('offset').to_i
+      end
+
+      def paragraph
+        return @paragraph ||= @xml_node.attr('para').to_i
+      end
+    end
+  end
+end

data/lib/opener/stanza/tokenizer_pos.rb

@@ -0,0 +1,87 @@
+module Opener
+  module Stanza
+    class TokenizerPos
+
+      DESC            = 'Tokenizer / POS by Stanza'
+      VERSION         = '1.0'
+
+      BASE_URL        = ENV['STANZA_SERVER']
+      LANGUAGES_CACHE = Opener::ChainedDaemon::LanguagesCache.new
+
+      POS             = {
+        'DET'   => 'D',
+        'ADJ'   => 'G',
+        'NOUN'  => 'N',
+        'VERB'  => 'V',
+        'AUX'   => 'V',
+        'ADV'   => 'A',
+        'CCONJ' => 'J',
+        'PUNCT' => '.',
+        'ADP'   => 'P',
+        'PRON'  => 'Q',
+        'PROPN' => 'R',
+        'PART'  => 'P',
+        'NUM'   => 'O',
+        'X'     => 'O',
+        'SYM'   => 'O',
+        'SCONJ' => 'P',
+        'INTJ'  => 'O',
+      }
+
+      POS_OPEN        = %w[N R G V A O]
+
+      def run input, params
+        raise 'missing Stanza server' if ENV['STANZA_SERVER'].blank?
+
+        kaf      = KAF::Document.from_xml input
+
+        prod     = params[:cache_keys][:environment] != 'staging'
+        if prod and !LANGUAGES_CACHE.get.include?(kaf.language)
+          raise Core::UnsupportedLanguageError.new kaf.language
+        end
+
+        response = Faraday.post BASE_URL, {lang: kaf.language, input: kaf.raw}.to_query
+        raise Core::UnsupportedLanguageError, kaf.language if response.status == 406
+        raise response.body if response.status >= 400
+        tokens   = JSON.parse response.body
+
+        w_index = 0
+        tokens.each_with_index do |sentence, s_index|
+          sentence.each_with_index do |word|
+            w_index += 1
+            misc   = word['misc']
+            offset = misc.match(/start_char=(\d+)|/)[1].to_i
+            length = misc.match(/end_char=(\d+)/)[1].to_i - offset
+
+            u_pos  = word['upos']
+            pos    = POS[u_pos]
+            raise "Didn't find a map for #{u_pos}" if pos.nil?
+            type   = if POS_OPEN.include? pos then 'open' else 'close' end
+
+            params = {
+              wid:        w_index,
+              sid:        s_index + 1,
+              tid:        w_index,
+              para:       1,
+              offset:     offset,
+              length:     length,
+              text:       word['text'],
+              lemma:      word['lemma'],
+              morphofeat: u_pos,
+              pos:        pos,
+              type:       type,
+            }
+
+            kaf.add_word_form params
+            kaf.add_term params
+          end
+        end
+
+        kaf.add_linguistic_processor DESC, "#{VERSION}", 'text', timestamp: true
+
+        kaf.to_xml
+      end
+
+    end
+  end
+end

data/opener-chained-daemon.gemspec

@@ -10,6 +10,7 @@ Gem::Specification.new do |spec|
   spec.license = 'Apache 2.0'
 
   spec.files = Dir.glob([
+    'config/**/*',
     'exec/**/*',
     'lib/**/*',
     '*.gemspec',
@@ -18,22 +19,25 @@ Gem::Specification.new do |spec|
   ]).select{ |file| File.file? file }
 
   spec.bindir = 'bin'
-  spec.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
+  spec.executables = Dir.glob('bin/*').map{ |file| File.basename(file) }
 
   spec.add_dependency 'activesupport'
   spec.add_dependency 'google-cloud-translate', '~> 1.0'
   spec.add_dependency 'httpclient'
   spec.add_dependency 'hashie'
 
-  spec.add_dependency 'opener-daemons', '~> 2.7.1'
+  spec.add_dependency 'roda'
+  spec.add_dependency 'rack-timeout'
+  spec.add_dependency 'faraday'
+  spec.add_dependency 'opener-daemons', '~> 2.7.2'
   spec.add_dependency 'opener-callback-handler', '~> 1.0'
 
-  spec.add_dependency 'opener-language-identifier'
-  spec.add_dependency 'opener-tokenizer'
-  spec.add_dependency 'opener-pos-tagger'
-  spec.add_dependency 'opener-polarity-tagger', '>= 3.1.3'
-  spec.add_dependency 'opener-property-tagger', '>= 3.2.1'
-  spec.add_dependency 'opener-opinion-detector-basic'
+  spec.add_dependency 'opener-language-identifier', '>= 4.4.0'
+  spec.add_dependency 'opener-tokenizer',  '>= 2.2.0'
+  spec.add_dependency 'opener-pos-tagger', '>= 3.2.0'
+  spec.add_dependency 'opener-polarity-tagger', '>= 3.2.7'
+  spec.add_dependency 'opener-property-tagger', '>= 3.3.5'
+  spec.add_dependency 'opener-opinion-detector-basic', '>= 3.2.3'
 
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'rake'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opener-chained-daemon
 version: !ruby/object:Gem::Version
-  version: 2.2.2
+  version: 3.0.7
 platform: ruby
 authors:
 - development@olery.com
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-23 00:00:00.000000000 Z
+date: 2020-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -66,20 +66,62 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: roda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-timeout
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: opener-daemons
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.7.1
+        version: 2.7.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.7.1
+        version: 2.7.2
 - !ruby/object:Gem::Dependency
   name: opener-callback-handler
   requirement: !ruby/object:Gem::Requirement
@@ -100,84 +142,84 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.4.0
 - !ruby/object:Gem::Dependency
   name: opener-tokenizer
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: opener-pos-tagger
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.2.0
 - !ruby/object:Gem::Dependency
   name: opener-polarity-tagger
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.1.3
+        version: 3.2.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.1.3
+        version: 3.2.7
 - !ruby/object:Gem::Dependency
   name: opener-property-tagger
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.2.1
+        version: 3.3.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.2.1
+        version: 3.3.5
 - !ruby/object:Gem::Dependency
   name: opener-opinion-detector-basic
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.2.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -276,12 +318,19 @@ files:
 - bin/chained-daemon-csv
 - bin/chained-daemon-daemon
 - bin/console
+- config/puma.rb
 - exec/chained-daemon.rb
 - lib/opener/chained_daemon.rb
 - lib/opener/chained_daemon/chained_daemon.rb
 - lib/opener/chained_daemon/cli.rb
+- lib/opener/chained_daemon/languages_cache.rb
 - lib/opener/chained_daemon/microsoft_translator.rb
 - lib/opener/chained_daemon/version.rb
+- lib/opener/chained_daemon/webservice.rb
+- lib/opener/kaf/document.rb
+- lib/opener/kaf/term.rb
+- lib/opener/kaf/text.rb
+- lib/opener/stanza/tokenizer_pos.rb
 - opener-chained-daemon.gemspec
 homepage:
 licenses:
@@ -303,7 +352,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.7.8
+rubygems_version: 2.7.6.2
 signing_key:
 specification_version: 4
 summary: OpeNER daemon for processing multiple queues at once