embulk-filter-azure_text_analytics 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cac630a471bf5e67125d08e80f149d46eeccfcc3
-  data.tar.gz: f1194a27bd82b07d1571fc555045e04c2abc96d2
+  metadata.gz: 0c87e0a6e5f569a09564b23881c78ef7022e3605
+  data.tar.gz: 057800f35d38cefed68ed8dee262f4cd4cd0ff2e
 SHA512:
-  metadata.gz: fdc16277d95fad436d178c78cc391d89e948ebf39236440fa9263ee6c992e4e3ee953ddcd5aa3f414bf168e31edbcbea02d077cd3008246b51d21a09f9c157bf
-  data.tar.gz: 73f376ee3f09ec3ad0f355108f7c5a922d28d8d28b19ae84c9843164532aa55e22aa93159fa97e0e2bcf202ebe8d7d650dda48504d9588846e6d70129c4146f4
+  metadata.gz: 7051bbf8881458e3c40270c770c414c05334d75bca6e0bac18dad8fa68ddd4c12121cb7eef9021c4c7ae6bc4f99c4d5ff2826f4d344cb13d7418f6fcfa22307f
+  data.tar.gz: 238cd7a23073962212d7370e041b762eb0134c371f3748bb5724b938db533a32d1605babade73856f07b0a175b30cd8d5a54fc0f84de135cfb211b9ebdc5d925

data/.ruby-version

@@ -1 +1 @@
-jruby-9.1.5.0
+jruby-9.1.5.0

data/README.md

@@ -28,7 +28,6 @@ Azure Text Analytics filter plugin for Embulk.
 ### sentiment
 
 ```yaml
-  # en,es,fr,pt
   - type: azure_text_analytics
     api_type: sentiment
     key_name: target_key
@@ -71,7 +70,11 @@ Azure Text Analytics filter plugin for Embulk.
 ### keyPhrases
 
 ```yaml
-    # en,es,fr,pt
+exec:
+  max_threads: 1
+  min_output_tasks: 1
+
+filters:
   - type: azure_text_analytics_topics
     out_key_name: _parsed
     key_name: pr
@@ -82,6 +85,7 @@ Azure Text Analytics filter plugin for Embulk.
 
 ```
 * required, over 100 documents.
+* en,es,fr,pt support
 
 ## Build
 

data/embulk-filter-azure_text_analytics.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-filter-azure_text_analytics"
-  spec.version       = "0.1.0"
+  spec.version       = "0.2.0"
   spec.authors       = ["toyama0919"]
   spec.summary       = "Azure Text Analytics filter plugin for Embulk"
   spec.description   = "Azure Text Analytics"
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency 'embulk', ['>= 0.8.15']
+  spec.add_development_dependency 'embulk', ['>= 0.8.16']
   spec.add_development_dependency 'bundler', ['>= 1.10.6']
   spec.add_development_dependency 'rake', ['>= 10.0']
 end

data/lib/embulk/filter/azure_text_analytics_topics.rb

@@ -18,14 +18,19 @@ module Embulk
           "body_params" => config.param("body_params", :hash, default: {}),
           "params" => config.param("params", :hash, default: {}),
           "subscription_key" => config.param("subscription_key", :string),
+          "operation_id" => config.param("operation_id", :string, default: nil),
           "stop_words" => config.param("stop_words", :array, default: nil),
           "stop_phrases" => config.param("stop_phrases", :array, default: nil),
+          "id_format" => config.param("id_format", :string, default: nil),
+          "id_keys" => config.param("id_keys", :array, default: []),
         }
 
-        out_columns = [
-          Column.new(nil, task["out_key_name"], :json)
+        add_columns = [
+          Column.new(nil, task['out_key_name'], :json)
         ]
 
+        out_columns = in_schema + add_columns
+
         yield(task, out_columns)
       end
 
@@ -37,6 +42,11 @@ module Embulk
         @out_key_name = task['out_key_name']
         @stop_words = task['stop_words']
         @stop_phrases = task['stop_phrases']
+        @id_format = task['id_format']
+        @id_keys = task['id_keys']
+        @operation_location = if task['operation_id']
+          "https://westus.api.cognitive.microsoft.com/text/analytics/v2.0/operations/" + task['operation_id']
+        end
 
         uri_string = "#{ENDPOINT_PREFIX}/topics"
         @uri = URI.parse(uri_string)
@@ -72,7 +82,11 @@ module Embulk
         documents = []
         records.each do |record|
           document = {}
-          document_id = SecureRandom.uuid
+          document_id = if (@id_format && @id_keys)
+            generate_id(@id_format, record, @id_keys)
+          else
+            SecureRandom.uuid
+          end
           document["language"] = @language if @language
           document["id"] = document_id
           document["text"] = record[@key_name]
@@ -86,43 +100,66 @@ module Embulk
         @request.body = request_body_hash.to_json
         Embulk.logger.debug "request body => #{@request.body}"
         response_hash = @http.start do |h|
-          proc_http(h)
+          @operation_location ? proc_topics(h, @operation_location) : proc_http(h)
+        end
+        if response_hash.key?('innerError')
+          raise "ERROR => #{response_hash}"
         end
-
         topics = response_hash["operationProcessingResult"]["topics"]
+        topics_assignments = response_hash["operationProcessingResult"]["topicAssignments"]
+
+        topics_assignments_merged = topics_assignments.map do |topics_assignment|
+          selected_topics = topics.select { |topic| topics_assignment['topicId'] == topic['id'] }
+          result = topics_assignment.merge(selected_topics.first)
+          result.delete('id')
+          result
+        end
 
-        topics.each do |data|
-          page_builder.add([data])
+        documents.each_with_index do |document, i|
+          record = records[i]
+          selected_topics_assignments = topics_assignments_merged.select{ |topics_assignment|
+            topics_assignment.delete('topicId')
+            topics_assignment['documentId'] == document['id']
+          }
+          page_builder.add(record.values + [selected_topics_assignments])
         end
       end
 
+      def generate_id(template, record, id_keys)
+        template % id_keys.map { |key| record[key] }
+      end
+
       def proc_http(h)
         response = h.request(@request)
         operation_location = response['operation-location']
 
         if operation_location
-          Embulk.logger.info "operation_location => #{operation_location}"
-          topics_request = Net::HTTP::Get.new(operation_location)
-          topics_request['Ocp-Apim-Subscription-Key'] = @subscription_key
-          loop do
-            topics_response = h.request(topics_request)
-            topics_response_body = topics_response.body
-            topics_response_hash = JSON.parse(topics_response_body)
-            status = topics_response_hash['status']
-            Embulk.logger.info "status => #{status}"
-            if status == 'Succeeded'
-              Embulk.logger.debug "topics_response_hash => #{topics_response_hash}"
-              return topics_response_hash
-            end
-            if status == 'Failed'
-              raise "topics_response_hash => #{topics_response_hash}"
-            end
-            sleep 60
-          end
+          return proc_topics(h, operation_location)
         end
 
         JSON.parse(response.body)
       end
+
+      def proc_topics(h, operation_location)
+        Embulk.logger.info "operation_location => #{operation_location}"
+        topics_request = Net::HTTP::Get.new(operation_location)
+        topics_request['Ocp-Apim-Subscription-Key'] = @subscription_key
+        loop do
+          topics_response = h.request(topics_request)
+          topics_response_body = topics_response.body
+          topics_response_hash = JSON.parse(topics_response_body)
+          status = topics_response_hash['status']
+          Embulk.logger.info "status => #{status}"
+          if status == 'Succeeded'
+            Embulk.logger.debug "topics_response_hash => #{topics_response_hash}"
+            return topics_response_hash
+          end
+          if status == 'Failed'
+            raise "topics_response_hash => #{topics_response_hash}"
+          end
+          sleep 60
+        end
+      end
     end
   end
 end

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-azure_text_analytics
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-10 00:00:00.000000000 Z
+date: 2017-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.15
+        version: 0.8.16
   name: embulk
   prerelease: false
   type: :development
@@ -23,7 +23,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.15
+        version: 0.8.16
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: