es_importer 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f7c04aa828e0ae96ecf1435708b67cc8b16e23b7
-  data.tar.gz: 64c25c201532396329b75ed2e0d456ba0aa8c45d
+SHA256:
+  metadata.gz: f5139f9596f419bb2e9be23229becb9ff26c180c38e8f2f4a1b3bcaf8f2fb977
+  data.tar.gz: a3ee1f507b0ffeb95369300baad2cde7af60e10aff67fe1deae74984690c8349
 SHA512:
-  metadata.gz: 7454c639c0b9242d5709bd8ec660f54fbfcceffe0a401be6b9a9a166ac4c5c0e6917b083e5b8b51d969122286f0e6798d8e3ec644fadccb9593be5fc3337d4fe
-  data.tar.gz: 7998ff89e18e6783a86baf81aca787ebb4d4cfcbd6f6642dd24bc9a08fc34847700acc95e98fc822b39fb8d241ce65f7451d0212f1e9ff1115b7ca060f733218
+  metadata.gz: 1ffe0e7c95b2c3f79eb1a97218c478863a32bd4bcd97d1fcff7db61ecec987cbb747dd53ed00d72458be77f40e03b2003fc8d44cabb2b76fde323d55841f75e3
+  data.tar.gz: '084ebc7fbac313071e5ebd9f054d1c133407260d56518bfd20aeb633369f4f944fd81543240662a1e95d7c1d634a5d64d148733cd2f4f1cce67a03b598a6fe56'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    es_importer (0.2.0)
+    es_importer (0.3.0)
       aws-sdk
       elasticsearch
       faraday
@@ -12,13 +12,13 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.11.24)
-      aws-sdk-resources (= 2.11.24)
-    aws-sdk-core (2.11.24)
+    aws-sdk (2.11.28)
+      aws-sdk-resources (= 2.11.28)
+    aws-sdk-core (2.11.28)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.11.24)
-      aws-sdk-core (= 2.11.24)
+    aws-sdk-resources (2.11.28)
+      aws-sdk-core (= 2.11.28)
     aws-sigv4 (1.0.2)
     diff-lcs (1.3)
     elasticsearch (6.0.2)

data/README.md

@@ -42,27 +42,39 @@ users = (1..100).to_a.map do |i|
   }
 end
 
-# define elastic id, mapping and keywords for index; provide conversion procs
+# define mapping and settings for index
+# provide conversion procs to transform the document
 importer = {
 
   # name of the index
+  # NOTE: type will be set to singular form of index name, ie. users => user
   users: {
 
   # build id from single or multiple keys
   id_key: [:user_id, :created_at],
 
   # define index mapping
+  # supply a type for a field or a hash with field definition
   mapping: {
-    user_id: :text,
+    user_id: :integer,
     active: :boolean,
-    email: :text,
+    email: {type: :text, analyzer: :my_analyzer},
     created_at: :date,
     country_code: :text
   },
 
+  # shortcut to set keywords to fields
   # keyword generated is 'country_code.keyword'
   keywords: [:country_code],
 
+  # index settings - add a custom analyzer with a ngram filter
+  settings: {
+    analysis: {
+      filter: {my_filter: {type: :ngram, min_gram: 3, max_gram: 4}},
+	    analyzer: {my_analyzer: {type: :custom, tokenizer: :standard, filter: [:lowercase, :my_filter]}}
+	  }
+  },
+
   converters: {
     # downcase existing field value, second argument is document being processed
     'email' => Proc.new{|attr, _| attr.downcase},
@@ -104,7 +116,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/es_importer. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/damir/es_importer. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## License
 
@@ -112,4 +124,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the EsImporter project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/es_importer/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the EsImporter project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/damir/es_importer/blob/master/CODE_OF_CONDUCT.md).

data/lib/es_importer/version.rb

@@ -1,3 +1,3 @@
 module EsImporter
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/es_importer.rb

@@ -13,7 +13,7 @@ module EsImporter
     @client       = Elasticsearch::Client.new transport: transport
   end
 
-  # init es transport
+  # init ransport
   def transport
     # parse uri for host configuration
     es_uri = URI.parse(@es_uri)
@@ -44,51 +44,81 @@ module EsImporter
     @importers.update(importer)
   end
 
-  # create es index
-  def create_index!(es_index)
-    puts "Creating #{es_index} index at #{@es_uri} ..."
-    @client.indices.create index: es_index, body: {
+  # create index
+  def create_index!(index)
+    puts "Creating #{index} index at #{@es_uri} ..."
+
+    mapping   = @importers.dig(index, :mapping)
+    keywords  = @importers.dig(index, :keywords)
+    type_name = index.to_s.chomp('s')
+
+    body = {
+
+      # add mapping
       mappings: {
-        es_index.to_s.chomp('s') => {
+         type_name => {
           dynamic: false,
-          properties: @importers.dig(es_index, :mapping).reduce({}){|a, (k,v)| a.update({k => {type: v}.update(@importers.dig(es_index, :keywords)&.include?(k) ? {fields: {keyword: {type: :keyword}}} : {})})}
+          properties: mapping.reduce({}) do |a, (k, v)|
+
+            # field with only type def, ie. mapping: {user_id: :text}
+            if v.kind_of?(Symbol)
+              field_def = {type: v}
+            # field with ull def, ie. mapping: {user_id: {type: :text, analyzer: :my_analyzer}}
+          elsif v.kind_of?(Hash)
+              field_def = v
+            end
+
+            # optional keywords
+            field_def.update(keywords&.include?(k) ? {fields: {keyword: {type: :keyword}}} : {}) if keywords
+
+            # set field definition
+            a.update(k => field_def)
+          end
         }
       }
     }
+
+    # merge settings if its set
+    settings = @importers.dig(index, :settings)
+    body.update(settings: settings) if settings
+
+    # create index
+    @client.indices.create index: index, body: body
+
     rescue => error
-      puts "Error creating #{es_index} index. #{error.class}: #{error.message}"
+      puts "Error creating #{index} index. #{error.class}: #{error.message}"
       raise
   end
 
-  # create es index
-  def delete_index!(es_index)
-    puts "Deleting #{es_index} index at #{@es_uri} ..."
-    @client.indices.delete index: es_index
+  # delete index
+  def delete_index!(index)
+    puts "Deleting #{index} index at #{@es_uri} ..."
+    @client.indices.delete index: index
     rescue => error
-      puts "Error deleting #{es_index} index. #{error.class}: #{error.message}"
+      puts "Error deleting #{index} index. #{error.class}: #{error.message}"
       raise
   end
 
   # import documents
-  def import(es_index, documents)
+  def import(index, documents)
 
     # import stats init
     start_time = Time.now
     failed = 0; imported = 0
 
     # insert into elastic
-    documents.each_with_index do |document, index|
+    documents.each_with_index do |document, i|
 
       # convert all keys to strings
       document =  _deep_transform_keys_in_object(document, &:to_s)
 
       # generate id
-      id_key  = @importers.dig(es_index, :id_key)
+      id_key  = @importers.dig(index, :id_key)
       id      = document[id_key.to_s] if id_key.is_a?(Symbol) # single key
       id      = id_key.reduce([]){|acc, key| acc << document[key.to_s]}.join('-') if id_key.is_a?(Array) # composite key
 
       # convert keys or add new ones
-      @importers.dig(es_index, :converters)&.each do |keys, converter|
+      @importers.dig(index, :converters)&.each do |keys, converter|
         keys = keys.split('.')
 
         # transform existing key
@@ -99,33 +129,33 @@ module EsImporter
         # add new key
         else
           missing_key_index = nil
-          keys.each_with_index do |key, index|
-            missing_key_index = index and break unless document.dig(*keys.first(index + 1))
+          keys.each_with_index do |key, i|
+            missing_key_index = i and break unless document.dig(*keys.first(i + 1))
           end
 
           tail_keys = keys[missing_key_index..-1]
           tail_hash = keys[0...missing_key_index].reduce(document, :fetch)
 
-          tail_keys.each_with_index do |key, index|
-            tail_hash[tail_keys[index]] = tail_keys.size == index + 1 ? converter.call(document) : {}
-            tail_hash = tail_hash[tail_keys[index]]
+          tail_keys.each_with_index do |key, i|
+            tail_hash[tail_keys[i]] = tail_keys.size == i + 1 ? converter.call(document) : {}
+            tail_hash = tail_hash[tail_keys[i]]
           end
         end
       end
 
       begin
-        @client.index index: es_index, type: es_index.to_s.chomp('s'), id: id, body: document
-        puts "##{index + 1} imported #{id}" if @logger
+        @client.index index: index, type: index.to_s.chomp('s'), id: id, body: document
+        puts "##{i + 1} imported #{id}" if @logger
         imported = imported + 1
       rescue => e
-        puts "##{index + 1} failed #{id}" if @logger
+        puts "##{i + 1} failed #{id}" if @logger
         puts e.class; puts e.message
         failed = failed + 1
       end
     end
 
     # print import statistics
-    puts; puts "#{es_index} import statistics"; puts '-' * 100
+    puts; puts "#{index} import statistics"; puts '-' * 100
     puts "Failed: #{failed}"
     puts "Imported: #{imported}"
     puts "Time spent: #{Time.now - start_time} sec"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: es_importer
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Damir Roso
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-29 00:00:00.000000000 Z
+date: 2018-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -177,7 +177,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Transform and import JSON documents into elastic search.