rotating_es_loader 0.0.0 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 929a07d2d4a9527442a16944c5970152f8d3d0e6daf6fd1693577339ae4c8ec0
-  data.tar.gz: 836e310957e7ceb637b6896ed96435797d2955eeac9fd4e8739b99fc5a84f394
+  metadata.gz: a895e503532b76e0a807d233339a1d5bbb6d273e9f12972da776a50857eee8e2
+  data.tar.gz: a92614bd378f9d298aa51b1ae68bcf25638205092f7a6e45ffe36194246278fb
 SHA512:
-  metadata.gz: 253ed26543206be48a15ada0d9336c770fcd6a36eb824651c211ddc38c83184265caa952c5ae166d811236655dd94c5e7a5e108b192a808d650c7968169081e6
-  data.tar.gz: e8afa85a8eb6b52b79eb36e17b66e036bf1a160c4e5c478e02ac76a68d0c2e1213909e23bb020c415209286ca0b3d49398290194d0041fd336ba5f8fd17e8a5e
+  metadata.gz: 9be744ca07756790a7ed69954fa9318344e56fde8c23c91f04c2b06b8456187e6f1dca68918c686c94763ed33a92c84c11159c3ef37e6c85621b0ad0f665c5ef
+  data.tar.gz: 832b9495e7cddd079e084ea0f0249f0a47e731d44e46cf670f20f1dbaef028e5f569f9c2feba677872c6477d473fba4eabee28ba7c7ff5576e06f2df4078cc5a

data/lib/rotating_es_loader.rb

@@ -1,5 +1,204 @@
-class RotatingEsLoader
-  def self.hi
-    puts "Hello world"
+# frozen_string_literal: true
+
+require 'rotating_es_loader/es_client'
+
+# :nodoc
+class RotatingEsLoader < EsClient
+  extend Memoist
+
+  # indexs with a datestamp newer than this age will not be wiped
+  MAX_INDEX_AGE = 3
+  DEFAULT_SLICE_SIZE = 50
+
+  attr_accessor :slice_size, :es_major_version
+
+  def initialize(opts)
+    raise('no credentials provided') unless opts[:credentials]
+    raise('no url provided') unless opts[:url]
+    raise('no definitions provided') unless opts[:index_definitions].is_a?(Hash)
+    uri = URI.parse(opts[:url])
+
+    super(
+      url: opts[:url],
+      credentials: opts[:credentials]
+    )
+
+    @index_definitions = opts[:index_definitions]
+    @slice_size = opts[:slice_size] || DEFAULT_SLICE_SIZE
+
+    @logger.debug("index keys: #{index_keys}")
+    @datasources = opts[:datasources]
+
+    index_keys.each do |key|
+      raise("No datasource for #{key}") unless @index_definitions[key][:datasource].respond_to?(:each)
+    end
+
+    es_info = client.info
+    @es_major_version = es_info['version']['number'].split('.').first.to_i
+  end
+
+  def document_type_for(key)
+    raise "document type not supported for ES #{es_major_version}" \
+      unless es_major_version <= 5
+    @index_definitions[key][:type]
+  end
+
+  def index_keys
+    @index_definitions.keys
+  end
+
+  def mappings_for(key)
+    @index_definitions[key][:mappings]
+  end
+
+  def settings_for(key)
+    @index_definitions[key][:settings]
+  end
+
+  def datasource_for(key)
+    @index_definitions[key][:datasource]
+  end
+
+  def execute
+    create_indices
+    create_documents
+    swap_aliases
+    delete_old_indices
+  end
+
+  def multitype_support?
+    return es_major_version <= 5
+  end
+
+  def create_documents
+    index_keys.each do |k|
+      create_documents_for_type(
+        name: get_index_name(k),
+        data: datasource_for(k),
+        type: document_type_for(k)
+      )
+    end
+  end
+
+  def create_documents_for_type(name:, data:, type: nil)
+    @logger.info("Creating documents of in index #{name} in batches of #{@slice_size}")
+    data.lazy.each_slice(@slice_size).each_with_index do |slice, slice_num|
+      @logger.debug("batch #{slice_num}: #{slice.size} docs")
+      result = client.bulk(
+        body: slice.flat_map do |rec|
+          index_record = { index: { _index: name, _id: rec[:id] } }
+          index_record[:index].merge!(_type: type) if es_major_version == 5
+
+          [
+            index_record,
+            rec
+          ]
+        end
+      )
+
+      @logger.warn("ERRORS: #{JSON.pretty_generate(result)}") if result['errors']
+    end
+  end
+
+  def create_indices
+    index_keys.each do |k|
+      create_index(name: get_index_name(k), key: k)
+    end
+  end
+
+  def key_age(key)
+    date_str = key.split('-')[1]
+    if date_str && date_str.size == 8
+      (Date.today - Date.parse(date_str)).to_i
+    else
+      0
+    end
+  end
+
+  def get_index_name(key)
+    # TODO: make it more sequential, so that it sorts correctly
+    date_str = Date.today.to_s.gsub(/\D/, '') + '-' + Time.now.to_i.to_s + '-' + Process.pid.to_s
+    raise("provided key #{key} is not a valid index") unless index_keys.include?(key)
+    return key.to_s + '-' + date_str
+  end
+  memoize :get_index_name # otherwise time might change
+
+  def delete_old_indices
+    existing_indices = client.indices.get(index: '_all')
+
+    @logger.debug("Existing indexes: #{existing_indices.keys}")
+
+    index_keys.each do |index|
+      keys = existing_indices.keys.select { |k| k.include?(index.to_s) }.sort
+      keys_by_date = keys.group_by { |k| key_age(k) }
+      keys_to_delete = []
+
+      # delete all indexes, keeping one from each day for the last few days
+      keys_by_date.each do |age, key_list|
+        key_list.pop if age <= MAX_INDEX_AGE
+        keys_to_delete += key_list
+      end
+
+      unless keys_to_delete.empty?
+        @logger.debug("Deleting indexes #{keys_to_delete.join(', ')}")
+        client.indices.delete index: keys_to_delete
+      end
+    end
+  end
+
+  def swap_aliases
+    index_keys.each do |alias_name|
+      index_name = get_index_name(alias_name)
+
+      actions = [
+        { add: { index: index_name, alias: alias_name } }
+      ]
+
+      @logger.debug("fetching any indices attached to alias #{alias_name}")
+      begin
+        client.indices.get_alias(name: alias_name).keys.each do |index_to_remove|
+          actions.unshift(
+            remove: { index: index_to_remove, alias: alias_name }
+          )
+        end
+      rescue StandardError => e
+        @logger.warn(e)
+      end
+
+      @logger.debug('update_aliases actions: ' + actions.to_json)
+
+      client.indices.update_aliases body: {
+        actions: actions
+      }
+    end
+  end
+
+  def mappings_adjusted_for_es_version(key)
+    mapping_for_key = mappings_for(key) || @logger.warn("mappings does not contain a mapping for #{key}")
+    mappings = {}
+    if es_major_version < 6
+      mappings[key] = { properties: mapping_for_key }
+    else
+      mappings[:properties] = mapping_for_key
+    end
+
+    mappings
+  end
+
+  def create_index(name:, key:)
+    @logger.debug("creating index #{name}")
+
+    mappings = mappings_adjusted_for_es_version(key)
+
+    @logger.debug("mappings: #{mappings.to_json}")
+    @logger.debug("creating index #{name}")
+
+    client.indices.create({
+      index: name,
+      body: {
+        settings: settings_for(key),
+        mappings: mappings
+      }
+    }.tap { |x| puts JSON.pretty_generate(x) })
   end
 end

data/lib/rotating_es_loader/array_datasource.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# :nodoc
+class ArrayDatasource
+  include Enumerable
+
+  def initialize(data)
+    @data = data
+    @iter = data.each
+  end
+
+  def each(&block)
+    return to_enum(:each) unless block
+
+    @data.each(&block)
+    self
+  end
+
+  def size
+    @data.size
+  end
+end

data/lib/rotating_es_loader/es_client.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'faraday_middleware/aws_signers_v4'
+require 'faraday_middleware/gzip'
+require 'elasticsearch'
+require 'memoist'
+require 'logger'
+require 'aws-sdk'
+
+# :nodoc
+class EsClient
+  extend Memoist
+
+  def initialize(
+    url:,
+    credentials:,
+    logger: nil
+  )
+
+    raise('credentials must be an Aws::SharedCredentials') unless \
+      credentials.is_a?(Aws::SharedCredentials)
+
+    @logger = logger || Logger.new(STDOUT)
+    @url = url
+    @credentials = credentials
+    @logger.info('URL is ' + url)
+  end
+
+  def client
+    Elasticsearch::Client.new(url: @url) do |f|
+      f.use FaradayMiddleware::Gzip
+      f.request :aws_signers_v4,
+                credentials: @credentials,
+                service_name: 'es',
+                region: 'us-west-1'
+    end
+  end
+  memoize :client
+
+  def method_missing(m, *args, &block)
+    @logger.debug("Delegating #{m}")
+    client.send(m, *args, &block)
+  end
+end

data/lib/rotating_es_loader/sql_datasource.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'xmlsimple'
+
+# :nodoc
+class SqlDatasource
+  include Enumerable
+
+  def initialize(sql:, ar_connection:)
+    @sql = sql
+    @ar_connection = ar_connection
+    raise unless @sql
+  end
+
+  def normalize(o)
+    o
+  end
+
+  def data
+    queries = @sql.is_a?(String) ? [@sql] : @sql
+
+    queries.flat_map do |query|
+      records_array = @ar_connection.execute(@sql)
+      fields = records_array.fields.map(&:to_sym)
+
+      records_array.map do |row_array|
+        normalize(fields.zip(row_array).to_h)
+      end
+    end
+  end
+
+  def each(&block)
+    return to_enum(:each) unless block
+
+    data.each(&block)
+
+    self
+  end
+end

data/lib/rotating_es_loader/xml_files_datasource.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'xmlsimple'
+
+# :nodoc
+class XmlFilesDatasource
+  include Enumerable
+
+  def initialize(glob)
+    @files = Dir.glob(glob).to_a
+  end
+
+  def normalize(o)
+    o
+  end
+
+  def each(&block)
+    return to_enum(:each) unless block
+
+    @files.each do |xml_file|
+      hash = XmlSimple.xml_in(
+        xml_file,
+        ForceArray: false,
+        SuppressEmpty: ''
+      )
+      Array(normalize(hash)).each(&block)
+    end
+    
+    self
+  end
+
+  def size
+    @files.size
+  end
+end

metadata

@@ -1,15 +1,197 @@
 --- !ruby/object:Gem::Specification
 name: rotating_es_loader
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.4
 platform: ruby
 authors:
 - Mike Kowdley
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-21 00:00:00.000000000 Z
-dependencies: []
+date: 2019-09-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.11.358
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.11.358
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-resources
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.11.258
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.11.258
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.5
+- !ruby/object:Gem::Dependency
+  name: elasticsearch-extensions
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.31
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.31
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.1
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware-aws-signers-v4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.9
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+- !ruby/object:Gem::Dependency
+  name: memoist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+- !ruby/object:Gem::Dependency
+  name: xml-simple
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.5
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.8.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.74.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.74.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.35.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.35.0
 description: A base class for code that loads data into Elasticsearch
 email: mike@valuationmetricsinc.com
 executables: []
@@ -17,7 +199,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/rotating_es_loader.rb
-homepage: https://rubygems.org/gems/rotating_es_loader
+- lib/rotating_es_loader/array_datasource.rb
+- lib/rotating_es_loader/es_client.rb
+- lib/rotating_es_loader/sql_datasource.rb
+- lib/rotating_es_loader/xml_files_datasource.rb
+homepage: https://github.com/mikevm/rotating_es_loader
 licenses:
 - MIT
 metadata: {}