es_importer 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f5139f9596f419bb2e9be23229becb9ff26c180c38e8f2f4a1b3bcaf8f2fb977
-  data.tar.gz: a3ee1f507b0ffeb95369300baad2cde7af60e10aff67fe1deae74984690c8349
+  metadata.gz: c535d2850816bfd5e36c616057c6db755cbe27e3e99474b6149cbc74de188bc4
+  data.tar.gz: 984636ea2d8aaa849dfab4cdf4d156632eefd468ff4ea67a6a96228a73b9a1b3
 SHA512:
-  metadata.gz: 1ffe0e7c95b2c3f79eb1a97218c478863a32bd4bcd97d1fcff7db61ecec987cbb747dd53ed00d72458be77f40e03b2003fc8d44cabb2b76fde323d55841f75e3
-  data.tar.gz: '084ebc7fbac313071e5ebd9f054d1c133407260d56518bfd20aeb633369f4f944fd81543240662a1e95d7c1d634a5d64d148733cd2f4f1cce67a03b598a6fe56'
+  metadata.gz: 0ad7fb8717ef68d7f61d40d09634fe1bd9df9b998e648602c9b6937efec17166dd5fea8ebdf52654ce7fc18a28e058b6668115d2813ad3fabd43e18ddd530897
+  data.tar.gz: 3a1194226e3063a58b66de11f4e803141fb669efc3e589815b9c282350d076c232e84b17e231a4520a56f2352eac6d19f91b22258dd2d341ba4fdf40bc6fe0b6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    es_importer (0.3.0)
+    es_importer (0.4.0)
       aws-sdk
       elasticsearch
       faraday
@@ -12,13 +12,13 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.11.28)
-      aws-sdk-resources (= 2.11.28)
-    aws-sdk-core (2.11.28)
+    aws-sdk (2.11.33)
+      aws-sdk-resources (= 2.11.33)
+    aws-sdk-core (2.11.33)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.11.28)
-      aws-sdk-core (= 2.11.28)
+    aws-sdk-resources (2.11.33)
+      aws-sdk-core (= 2.11.33)
     aws-sigv4 (1.0.2)
     diff-lcs (1.3)
     elasticsearch (6.0.2)
@@ -39,7 +39,7 @@ GEM
       aws-sdk-resources (>= 2, < 3)
       faraday (~> 0.9)
     ffi (1.9.23)
-    jmespath (1.3.1)
+    jmespath (1.4.0)
     multi_json (1.13.1)
     multipart-post (2.0.0)
     rake (10.5.0)

data/README.md

@@ -25,7 +25,7 @@ See inline coments:
 require 'es_importer'
 
 # configure client
-EsImporter.configure('http://localhost:9200')
+EsImporter.configure('http://localhost:9200', logger: Logger.new($stdout))
 
 # generate some users
 users = (1..100).to_a.map do |i|
@@ -97,9 +97,18 @@ EsImporter.add_importer(importer)
 # create index
 EsImporter.create_index!(:users)
 
-# import users
+# import single users
+EsImporter.import(:users, users[0])
+
+# import users sequentially
 EsImporter.import(:users, users)
 
+# or import in bulk
+EsImporter.import_in_bulk(:users, users)
+
+# or just transform a document without importing it
+EsImporter.transform_document(:users, users[0])
+
 #  delete index
 EsImporter.delete_index!(:users)
 
@@ -107,7 +116,6 @@ EsImporter.delete_index!(:users)
 
 AWS elastic instance is also supported, region is extracted from url and credentials are set form ruby SDK.
 
-
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/es_importer.rb

@@ -46,7 +46,7 @@ module EsImporter
 
   # create index
   def create_index!(index)
-    puts "Creating #{index} index at #{@es_uri} ..."
+    @logger&.debug("Creating #{index} index at #{@es_uri} ...")
 
     mapping   = @importers.dig(index, :mapping)
     keywords  = @importers.dig(index, :keywords)
@@ -64,7 +64,7 @@ module EsImporter
             if v.kind_of?(Symbol)
               field_def = {type: v}
             # field with ull def, ie. mapping: {user_id: {type: :text, analyzer: :my_analyzer}}
-          elsif v.kind_of?(Hash)
+            elsif v.kind_of?(Hash)
               field_def = v
             end
 
@@ -86,80 +86,126 @@ module EsImporter
     @client.indices.create index: index, body: body
 
     rescue => error
-      puts "Error creating #{index} index. #{error.class}: #{error.message}"
+      @logger&.debug("Error creating #{index} index. #{error.class}: #{error.message}")
       raise
   end
 
   # delete index
   def delete_index!(index)
-    puts "Deleting #{index} index at #{@es_uri} ..."
+    @logger&.debug("Deleting #{index} index at #{@es_uri} ...")
     @client.indices.delete index: index
     rescue => error
-      puts "Error deleting #{index} index. #{error.class}: #{error.message}"
+      @logger&.debug("Error deleting #{index} index. #{error.class}: #{error.message}")
       raise
   end
 
+  # transform document using converters
+  def transform_document(index, document)
+
+    # convert all keys to strings
+    document =  _deep_transform_keys_in_object(document, &:to_s)
+
+    # convert keys or add new ones
+    @importers.dig(index, :converters)&.each do |keys, converter|
+      keys = keys.split('.')
+
+      # transform existing key
+      if value = document.dig(*keys)
+        document[keys.first] = converter.call(value, document) if keys.size == 1 # lvl 1
+        keys.first(keys.size-1).reduce(document, :fetch)[keys.last] = converter.call(value, document) if keys.size > 1 # lvl > 1
+
+      # add new key
+      else
+        missing_key_index = nil
+        keys.each_with_index do |key, i|
+          missing_key_index = i and break unless document.dig(*keys.first(i + 1))
+        end
+
+        tail_keys = keys[missing_key_index..-1]
+        tail_hash = keys[0...missing_key_index].reduce(document, :fetch)
+
+        tail_keys.each_with_index do |key, i|
+          tail_hash[tail_keys[i]] = tail_keys.size == i + 1 ? converter.call(document) : {}
+          tail_hash = tail_hash[tail_keys[i]]
+        end
+      end
+    end
+
+    # add elastic id
+    id_key            = @importers.dig(index, :id_key)
+    generated_id      = document[id_key.to_s] if id_key.is_a?(Symbol) # single key
+    generated_id      = id_key.reduce([]){|acc, key| acc << document[key.to_s]}.join('-') if id_key.is_a?(Array) # composite key
+    document['es_id'] = generated_id
+
+    # transformed document
+    document
+  end
+
   # import documents
   def import(index, documents)
 
+    # accept single document
+    documents = [documents] if documents.is_a?(Hash)
+
     # import stats init
-    start_time = Time.now
-    failed = 0; imported = 0
+    start_time  = Time.now
+    imported    = {count: 0}
+    failed      = {count: 0, items: []}
 
     # insert into elastic
     documents.each_with_index do |document, i|
 
-      # convert all keys to strings
-      document =  _deep_transform_keys_in_object(document, &:to_s)
+      # transform document
+      transformed_document = transform_document(index, document)
 
-      # generate id
-      id_key  = @importers.dig(index, :id_key)
-      id      = document[id_key.to_s] if id_key.is_a?(Symbol) # single key
-      id      = id_key.reduce([]){|acc, key| acc << document[key.to_s]}.join('-') if id_key.is_a?(Array) # composite key
+      # save it into index
+      begin
+        @client.index index: index, type: index.to_s.chomp('s'), id: transformed_document['es_id'], body: transformed_document
+        imported[:count] +=1
+      rescue => e
+        if @logger&.debug?
+          @logger.debug(e.class)
+          @logger.debug(e.message)
+        end
+        failed[:count] +=1
+        failed[:items] << {id: transformed_document['es_id'], error: e.message}
+      end
+    end
 
-      # convert keys or add new ones
-      @importers.dig(index, :converters)&.each do |keys, converter|
-        keys = keys.split('.')
+    # print import statistics
+    if @logger&.debug?
+      @logger.debug(">>> #{index} import statistics")
+      @logger.debug("Imported: #{imported}")
+      @logger.debug("Failed: #{failed}")
+      @logger.debug("Time spent: #{Time.now - start_time} sec")
+    end
 
-        # transform existing key
-        if value = document.dig(*keys)
-          document[keys.first] = converter.call(value, document) if keys.size == 1 # lvl 1
-          keys.first(keys.size-1).reduce(document, :fetch)[keys.last] = converter.call(value, document) if keys.size > 1 # lvl > 1
+    # return stats
+    {imported: imported, failed: failed}
+  end
 
-        # add new key
-        else
-          missing_key_index = nil
-          keys.each_with_index do |key, i|
-            missing_key_index = i and break unless document.dig(*keys.first(i + 1))
-          end
+  def import_in_bulk(index, documents)
+    # import stats init
+    start_time  = Time.now
 
-          tail_keys = keys[missing_key_index..-1]
-          tail_hash = keys[0...missing_key_index].reduce(document, :fetch)
+    # transform documents and build bulk payload
+    transformed_documents_for_bulk = documents.map do |document|
+      transformed_document = transform_document(index, document)
+      es_id = transformed_document.delete('es_id')
+      {index: { _index: index, _type: index.to_s.chomp('s'), _id: es_id, data: transformed_document}}
+    end
 
-          tail_keys.each_with_index do |key, i|
-            tail_hash[tail_keys[i]] = tail_keys.size == i + 1 ? converter.call(document) : {}
-            tail_hash = tail_hash[tail_keys[i]]
-          end
-        end
-      end
+    # import
+    resp = @client.bulk body: transformed_documents_for_bulk
 
-      begin
-        @client.index index: index, type: index.to_s.chomp('s'), id: id, body: document
-        puts "##{i + 1} imported #{id}" if @logger
-        imported = imported + 1
-      rescue => e
-        puts "##{i + 1} failed #{id}" if @logger
-        puts e.class; puts e.message
-        failed = failed + 1
-      end
+    # print import statistics
+    if @logger&.debug?
+      @logger.debug(">>> #{index} import statistics")
+      @logger.debug("Time spent: #{Time.now - start_time} sec")
     end
 
-    # print import statistics
-    puts; puts "#{index} import statistics"; puts '-' * 100
-    puts "Failed: #{failed}"
-    puts "Imported: #{imported}"
-    puts "Time spent: #{Time.now - start_time} sec"
-    puts
+    # return deserialized es response
+    resp
   end
 
   # taken from https://github.com/rails/rails/blob/f213e926892020f9ab6c8974612c59e2ba959253/activesupport/lib/active_support/core_ext/hash/keys.rb#L145

data/lib/es_importer/version.rb

@@ -1,3 +1,3 @@
 module EsImporter
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: es_importer
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Damir Roso
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-13 00:00:00.000000000 Z
+date: 2018-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler