elasticated 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8dc9235eb397f995d6bd706184dde4d43a3a85dc
-  data.tar.gz: 10cd46beddee5efa0aaa3bf301d534880413f968
+  metadata.gz: 7828418b23833264f77949ef5c083c9ac8d5970d
+  data.tar.gz: ba7887d592b1f2643821fef99f172303168967e7
 SHA512:
-  metadata.gz: 27039469aef119a66796daaa93ab1713845a91cb61345bd3592a55e41ccd84827cc72ec9e4ebeaa450d24979f98a5f09feef6eedf1ed003be86a69ba8c0d242d
-  data.tar.gz: 1998fbde3e419a39fcf6090fc827ffce2a498237056130efb5fd7b2b152d87983026ac203fcbed5e5f9c7e739a9eb92ebb0b6f74c1b57867dd38b93832bbcda4
+  metadata.gz: 9262818290a6679e8997f3784d0de0fcbac4b282f64ee0e43b7800dba72611b5b30a2d3eb13a8c15a2a8f20aa921550dab512e4441e47a5332defb93c13b2175
+  data.tar.gz: 0a520739e691bc59e8613816a3ae038785828ff4cbc8bb0c0dc7c33ee165674711e9183496152aa93aa55e38379009abcf21f87fbebd0b8839266a8f99534af0

data/lib/elasticated.rb

@@ -90,8 +90,6 @@ require_relative 'elasticated/mapping/builder'
 require_relative 'elasticated/index_selector'
 require_relative 'elasticated/partitioned_repository'
 
-require_relative 'elasticated/repositories/monthly_partitioned_repository'
-
 
 module Elasticated
 

data/lib/elasticated/client.rb

@@ -9,11 +9,13 @@ module Elasticated
     end
 
     def index_document(document_source, opts={})
-      transport.index opts.merge body: document_source
+      response = transport.index opts.merge body: document_source
+      log.debug "Document indexed #{response.to_json}"
     end
 
     def update_document(document_source, opts={})
-      transport.update opts.merge body: { doc: document_source }
+      response = transport.update opts.merge body: { doc: document_source }
+      log.debug "Document updated #{response.to_json}"
     end
 
     def index_exists?(index_name)
@@ -28,32 +30,31 @@ module Elasticated
       body.merge!(settings: { number_of_shards: shards }) if shards
       body.merge!(mappings: mapping) if mapping
       args.merge! body: body unless body.empty?
-      log.info "Creating index #{index_name}"
+      log.info "Creating index '#{index_name}'"
       transport.indices.create args
-      log.info "Index #{index_name} created"
+      log.info "Index '#{index_name}' created"
     end
 
     def create_alias(index_name, new_alias)
-      log.info "Putting alias #{new_alias} to index #{index_name}"
       transport.indices.put_alias index: index_name, name: new_alias
-      log.info "Alias #{new_alias} for index #{index_name} created"
+      log.info "Alias '#{new_alias}' for index '#{index_name}' created"
     end
 
     def remove_aliases(index_name)
       aliases = transport.indices.get_aliases[index_name]['aliases'].map(&:first) rescue Array.new
-      aliases.each do |alias_name|
-        transport.indices.delete_alias index: index_name, name: alias_name
+      aliases.each do |index_alias|
+        remove_alias index_name, index_alias
       end
     end
 
     def remove_alias(index_name, index_alias)
-      log.info "Alias #{index_alias} removed from index #{index_name}"
       transport.indices.delete_alias index: index_name, name: index_alias
+      log.info "Alias '#{index_alias}' removed from index '#{index_name}'"
     end
 
     def refresh_index(index_name)
-      log.debug "Refreshing #{index_name}"
       transport.indices.refresh index: index_name
+      log.debug "Index '#{index_name}' refreshed"
     end
 
     def refresh
@@ -61,24 +62,32 @@ module Elasticated
     end
 
     def count(body, opts={})
-      log.debug "Elasticsearch count #{body.to_json}"
+      log_query :count, body.to_json, opts
       transport.count opts.merge body: body
     end
 
     def search(body, opts={})
-      log.debug "Elasticsearch query #{body.to_json}"
+      log_query :search, body.to_json, opts
       transport.search opts.merge body: body
     end
 
     def scroll(scroll_id, opts={})
-      log.debug "Elasticsearch scroll #{scroll_id}"
+      log_query :scroll, scroll_id, opts
       transport.scroll opts.merge scroll_id: scroll_id
     end
 
     def delete(body, opts={})
-      log.debug "Elasticsearch delete #{body.to_json}"
+      log_query :delete, body.to_json, opts
       transport.delete_by_query opts.merge body: body
     end
 
+    protected
+
+    def log_query(query_type, message_body, query_opts={})
+      message = "Elasticsearch #{query_type} #{message_body}"
+      message = "#{message} with opts #{query_opts.to_json}" unless query_opts.empty?
+      log.debug message
+    end
+
   end
 end

data/lib/elasticated/configuration.rb

@@ -3,6 +3,7 @@ module Elasticated
 
     class << self
       attr_accessor :logger
+      attr_accessor :scroll_expiration_time, :scroll_page_size, :search_page_size
     end
 
   end

data/lib/elasticated/repository.rb

@@ -90,14 +90,13 @@ module Elasticated
     end
 
     def _exec_paginated_search(query, aggregated, opts={})
-      scroll_interval = '3m'
       body = aggregated ? query.build_for_aggregated_search : query.build_for_search
       if query.limited?
         response = client.search body, opts
         Results.from_elasticsearch_response response, query
       elsif query.sorted? || query.aggregated?
         # normal pagination
-        page_size = 50
+        page_size = Configuration.scroll_page_size || 50
         current_page = 1
         loop do
           offset = page_size * (current_page - 1)
@@ -115,8 +114,10 @@ module Elasticated
         results
       else
         # scan & scroll
-        response = client.search body, opts.merge(search_type: 'scan', scroll: scroll_interval, size: 1000)
-        while response = client.scroll(response['_scroll_id'], scroll: scroll_interval) and not response['hits']['hits'].empty? do
+        scroll_expiration_time = Configuration.scroll_expiration_time || '3m'
+        page_size = Configuration.scroll_page_size || 1000
+        response = client.search body, opts.merge(search_type: 'scan', scroll: scroll_expiration_time, size: page_size)
+        while response = client.scroll(response['_scroll_id'], scroll: scroll_expiration_time) and not response['hits']['hits'].empty? do
           results = results ? results.append_results_from(response) : Results.from_elasticsearch_response(response, query)
         end
         results ? results : Results.from_elasticsearch_response(response, query)

data/lib/version.rb

@@ -1,9 +1,13 @@
 module Elasticated
-  VERSION = '1.0.0'
+  VERSION = '1.1.0'
 end
 
 # Changelog
 
+# 1.1.0
+# Se elimina MonthlyPartitionedRepository cediendo paso a PartitionedRepository
+# Ahora el scroll expiration time y los tamanos de pagina son configurables
+
 # 1.0.0
 # Cambios en Repository
 # Se elimina InsightsRepository y sus hijos

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticated
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Pablo Fernandez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-31 00:00:00.000000000 Z
+date: 2016-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -162,7 +162,6 @@ files:
 - lib/elasticated/query.rb
 - lib/elasticated/query_aggregations.rb
 - lib/elasticated/query_conditions.rb
-- lib/elasticated/repositories/monthly_partitioned_repository.rb
 - lib/elasticated/repository.rb
 - lib/elasticated/results.rb
 - lib/version.rb
@@ -175,7 +174,6 @@ files:
 - spec/elasticsearch_top_hits_response.json
 - spec/integration_spec.rb
 - spec/mapping_spec.rb
-- spec/monthly_partitioned_repository_spec.rb
 - spec/query_aggregations_spec.rb
 - spec/query_conditions_spec.rb
 - spec/query_spec.rb
@@ -216,7 +214,6 @@ test_files:
 - spec/elasticsearch_top_hits_response.json
 - spec/integration_spec.rb
 - spec/mapping_spec.rb
-- spec/monthly_partitioned_repository_spec.rb
 - spec/query_aggregations_spec.rb
 - spec/query_conditions_spec.rb
 - spec/query_spec.rb

data/lib/elasticated/repositories/monthly_partitioned_repository.rb

@@ -1,96 +0,0 @@
-module Elasticated
-  class MonthlyPartitionedRepository < Repository
-    include Configurable
-
-    attr_accessor :_index_name, :_index_alias, :_general_alias
-    attr_accessor :date_field
-    attr_accessor :dynamic_creation, :index_options
-
-    def initialize(opts={})
-      super opts
-      self._index_name = opts[:index_name] || raise('An index prefix should be specified')
-      self._index_alias = opts[:index_alias] || raise('An index alias prefix should be specified')
-      self._general_alias = opts[:general_alias] || _index_alias
-      self.date_field = opts[:date_field] || :date
-      self.index_options = opts[:index_options] # hash with mapping, shards info, etc or nil
-      self.dynamic_creation = opts[:dynamic_creation] || false
-    end
-
-    def execute(action, query, opts={})
-      index_alias = index_alias_for query
-      super action, query, opts.merge(index: index_alias)
-    # rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
-    #   action == :count ? 0 : nil
-    end
-
-    def prepare(action, document, opts={})
-      date = date_from document
-      raise("The document has not a valid '#{date_field}' field") unless date
-      check_or_create_index! date
-      super action, document, opts.merge(index: index_alias_for(date))
-    end
-
-    def date_from(document)
-      str = document.source[date_field.to_s]
-      str && Date.parse(str)
-    end
-
-    def check_or_create_index! date
-      index_alias = index_alias_for date
-      return if client.index_exists? index_alias
-      raise("Index '#{index_alias}' not found (dynamic per-month index creation is disabled)") unless dynamic_creation
-      index_name = index_name_for date
-      create_index index_name
-      create_alias index_name, index_alias
-    end
-
-    def create_index! date
-      create_index index_name_for date
-    end
-
-    def create_alias! date
-      index_name = index_name_for date
-      index_alias = index_alias_for date
-      create_alias index_name, index_alias
-    end
-
-    def index_name_for(object)
-      object.is_a?(Query) ? index_alias_for_query(object) : index_name_for_date(object)
-    end
-
-    def index_alias_for(object)
-      object.is_a?(Query) ? index_alias_for_query(object) : index_alias_for_date(object)
-    end
-
-    protected
-
-    def create_index(index_name)
-      client.create_index index_name, index_options
-      log.info "Index #{index_name} created"
-    end
-
-    def create_alias(index_name, index_alias)
-      client.create_alias index_name, index_alias
-      client.create_alias index_name, _general_alias
-      log.info "Alias #{index_alias} created over index #{index_name}"
-    end
-
-    def index_alias_for_query(query)
-      _general_alias # TODO
-    end
-
-    def index_alias_for_date(date)
-      generate_name _index_alias, date
-    end
-
-    def index_name_for_date(date)
-      generate_name _index_name, date
-    end
-
-    def generate_name(prefix, date)
-      str = date.strftime '%Y-%m'
-      "#{prefix}-#{str}" # prefix-YYYY-MM
-    end
-
-  end
-end

data/spec/monthly_partitioned_repository_spec.rb

@@ -1,99 +0,0 @@
-require_relative 'spec_helper'
-
-module Elasticated
-  describe MonthlyPartitionedRepository do
-
-    let :repository do
-      MonthlyPartitionedRepository.new index_name: 'fbinsights-v3', index_alias: 'fbinsights'
-    end
-
-    before :each do
-      Elasticated.configure do |config|
-        config.logger = double.as_null_object
-      end
-    end
-
-    it "should return the correct index name" do
-      date = Date.parse '2015-09-04'
-      name = repository.index_name_for date
-      expect(name).to eq 'fbinsights-v3-2015-09'
-    end
-
-    it "should return the correct index alias" do
-      date = Date.parse '2015-09-04'
-      name = repository.index_alias_for date
-      expect(name).to eq 'fbinsights-2015-09'
-    end
-
-    context "on a search" do
-
-      let :query do
-        Query.new # TODO date
-      end
-
-      it "should execute a search on the correct index" do
-        expect(repository).to receive(:_exec_search).with query, index: 'fbinsights' # TODO
-        repository.execute_search query
-      end
-
-      it "should execute a count on the correct index" do
-        expect(repository).to receive(:_exec_count).with query, index: 'fbinsights' # TODO
-        repository.execute_count query
-      end
-
-      it "should execute a delete on the correct index" do
-        expect(repository).to receive(:_exec_delete).with query, index: 'fbinsights' # TODO
-        repository.delete_by query
-      end
-
-    end
-
-    context "on an indexation" do
-
-      let :document do
-        Document.create source: { 'date' => '2015-09-04' }
-      end
-
-      it "should use the appropiated index alias" do
-        client = repository.client
-        expect(client).to receive(:index_exists?).with('fbinsights-2015-09').and_return true
-        expect(client).to receive(:index_document).with document.source, index: 'fbinsights-2015-09', type: 'post'
-        repository.index_document document, type: 'post'
-      end
-
-      it "should create the appropiated index (only if dynamic creation is enabled)" do
-        client = repository.client
-        expect(client).to receive(:index_exists?).with('fbinsights-2015-09').and_return false
-        expect{ repository.index_document document, type: 'post' }.to raise_error
-        expect(client).to receive(:index_exists?).with('fbinsights-2015-09').and_return false
-        expect(client).to receive(:create_index).with 'fbinsights-v3-2015-09', anything
-        repository.dynamic_creation = true
-        expect(client).to receive(:create_alias).with('fbinsights-v3-2015-09', 'fbinsights-2015-09').once
-        expect(client).to receive(:create_alias).with('fbinsights-v3-2015-09', 'fbinsights').once
-        expect(client).to receive(:index_document).with document.source, index: 'fbinsights-2015-09', type: 'post'
-        repository.index_document document, type: 'post'
-      end
-
-    end
-
-    context "the index creation method" do
-
-      let :date do
-        Date.parse '2015-09-04'
-      end
-
-      it "creates the appropiated index" do
-        expect(repository.client).to receive(:create_index).with 'fbinsights-v3-2015-09', anything
-        repository.create_index! date
-      end
-
-      it "creates the appropiated alias" do
-        expect(repository.client).to receive(:create_alias).with('fbinsights-v3-2015-09', 'fbinsights-2015-09').once
-        expect(repository.client).to receive(:create_alias).with('fbinsights-v3-2015-09', 'fbinsights').once
-        repository.create_alias! date
-      end
-
-    end
-
-  end
-end