elasticated 1.0.0

data/lib/elasticated/query_aggregations.rb

@@ -0,0 +1,71 @@
+module Elasticated
+  class QueryAggregations
+
+    include Clonable
+    include BlockEvaluation
+
+    attr_accessor :_aggregations
+
+    def initialize
+      self._aggregations = Array.new
+    end
+
+    def build
+      _aggregations.inject({}) do |ret, aggregation|
+        ret.merge aggregation.name => aggregation.build
+      end
+    end
+
+    def parse(response)
+      _aggregations.inject({}) do |hash, aggregation|
+        name = aggregation.name.to_s
+        original_name = aggregation.original_name.to_s
+        hash.merge original_name => aggregation.parse(response[name])
+      end
+    end
+
+    def merge! other_query_aggs
+      other_query_aggs._aggregations.each do |other_aggregation|
+        add_aggregation other_aggregation
+      end
+    end
+
+    def empty?
+      _aggregations.empty?
+    end
+
+    private
+
+    def get_aggregation_class(agg_name)
+      camel_case_name = Helpers.string_to_camel_case agg_name.to_s
+      self.class.const_get("::Elasticated::#{camel_case_name}Aggregation") rescue nil
+    end
+
+    def method_missing(method_name, *args, &block)
+      agg_class = get_aggregation_class method_name
+      if agg_class
+        aggregation = agg_class.new(*args, &block)
+        add_aggregation aggregation
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(name, include_private=false)
+      get_aggregation_class(name) || super
+    end
+
+    def add_aggregation(aggregation)
+      equivalent_agg = _aggregations.select do |previous_agg|
+        previous_agg == aggregation
+      end.first
+      if !equivalent_agg
+        _aggregations << aggregation
+        aggregation
+      else
+        equivalent_agg._subaggregations.merge! aggregation._subaggregations if aggregation.is_a? Subaggregated
+      end
+    end
+
+  end
+end

data/lib/elasticated/query_conditions.rb

@@ -0,0 +1,89 @@
+module Elasticated
+  class QueryConditions
+
+    include Clonable
+
+    attr_accessor :_must, :_must_not, :_should, :_minimum_should_match, :_cache
+
+    def initialize
+      self._must = BooleanClause.new
+      self._must_not = BooleanClause.new
+      self._should = BooleanClause.new
+    end
+
+    # delimiters
+
+    def fill_delimiter(field_delimiter)
+      _must.fill_delimiter field_delimiter
+    end
+
+    # conditions
+
+    def add(condition)
+      _must.add condition
+    end
+
+    include ConditionsBuilder
+
+    def must(&block)
+      _must.evaluate block
+    end
+
+    def must_not(&block)
+      _must_not.evaluate block
+    end
+
+    def should(&block)
+      _should.evaluate block
+    end
+
+    # 'must_not' conditions
+
+    def not_equal(*args)
+      _must_not.equal *args
+    end
+
+    def without(*args)
+      _must_not.with *args
+    end
+
+    def not_between(*args)
+      _must_not.between *args
+    end
+
+    # cache
+
+    def cache(value=true)
+      self._cache = value
+    end
+
+    # other methods
+
+    def minimum_should_match(value)
+      self._minimum_should_match = value
+    end
+
+    def build
+      if empty?
+        { match_all: {} }
+      elsif _must_not.empty? && _should.empty? && _must.count == 1
+        _must.build_first
+      else
+        bool = {}
+        bool.merge! must: _must.build unless _must.empty?
+        bool.merge! must_not: _must_not.build unless _must_not.empty?
+        if !_should.empty?
+          bool.merge! should: _should.build
+          bool.merge! minimum_should_match: _minimum_should_match if _minimum_should_match
+        end
+        bool.merge! _cache: _cache if _cache
+        { bool: bool }
+      end
+    end
+
+    def empty?
+      _must.empty? && _must_not.empty? && _should.empty?
+    end
+
+  end
+end

data/lib/elasticated/repositories/monthly_partitioned_repository.rb

@@ -0,0 +1,96 @@
+module Elasticated
+  class MonthlyPartitionedRepository < Repository
+    include Configurable
+
+    attr_accessor :_index_name, :_index_alias, :_general_alias
+    attr_accessor :date_field
+    attr_accessor :dynamic_creation, :index_options
+
+    def initialize(opts={})
+      super opts
+      self._index_name = opts[:index_name] || raise('An index prefix should be specified')
+      self._index_alias = opts[:index_alias] || raise('An index alias prefix should be specified')
+      self._general_alias = opts[:general_alias] || _index_alias
+      self.date_field = opts[:date_field] || :date
+      self.index_options = opts[:index_options] # hash with mapping, shards info, etc or nil
+      self.dynamic_creation = opts[:dynamic_creation] || false
+    end
+
+    def execute(action, query, opts={})
+      index_alias = index_alias_for query
+      super action, query, opts.merge(index: index_alias)
+    # rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
+    #   action == :count ? 0 : nil
+    end
+
+    def prepare(action, document, opts={})
+      date = date_from document
+      raise("The document has not a valid '#{date_field}' field") unless date
+      check_or_create_index! date
+      super action, document, opts.merge(index: index_alias_for(date))
+    end
+
+    def date_from(document)
+      str = document.source[date_field.to_s]
+      str && Date.parse(str)
+    end
+
+    def check_or_create_index! date
+      index_alias = index_alias_for date
+      return if client.index_exists? index_alias
+      raise("Index '#{index_alias}' not found (dynamic per-month index creation is disabled)") unless dynamic_creation
+      index_name = index_name_for date
+      create_index index_name
+      create_alias index_name, index_alias
+    end
+
+    def create_index! date
+      create_index index_name_for date
+    end
+
+    def create_alias! date
+      index_name = index_name_for date
+      index_alias = index_alias_for date
+      create_alias index_name, index_alias
+    end
+
+    def index_name_for(object)
+      object.is_a?(Query) ? index_alias_for_query(object) : index_name_for_date(object)
+    end
+
+    def index_alias_for(object)
+      object.is_a?(Query) ? index_alias_for_query(object) : index_alias_for_date(object)
+    end
+
+    protected
+
+    def create_index(index_name)
+      client.create_index index_name, index_options
+      log.info "Index #{index_name} created"
+    end
+
+    def create_alias(index_name, index_alias)
+      client.create_alias index_name, index_alias
+      client.create_alias index_name, _general_alias
+      log.info "Alias #{index_alias} created over index #{index_name}"
+    end
+
+    def index_alias_for_query(query)
+      _general_alias # TODO
+    end
+
+    def index_alias_for_date(date)
+      generate_name _index_alias, date
+    end
+
+    def index_name_for_date(date)
+      generate_name _index_name, date
+    end
+
+    def generate_name(prefix, date)
+      str = date.strftime '%Y-%m'
+      "#{prefix}-#{str}" # prefix-YYYY-MM
+    end
+
+  end
+end

data/lib/elasticated/repository.rb

@@ -0,0 +1,139 @@
+module Elasticated
+  class Repository
+
+    # child can implement 'execute(action, query, opts)'
+    # child can implement 'prepare(action, document, opts)'
+
+    attr_accessor :client
+
+    def initialize(opts={})
+      self.client = Client.new opts
+    end
+
+    def execute_search(query, opts={})
+      execute :search, query, opts
+    end
+
+    def execute_aggregations(query, opts={})
+      execute :aggregations, query, opts
+    end
+
+    def execute_count(query, opts={})
+      execute :count, query, opts
+    end
+
+    def execute_aggregated_search(query, opts={})
+      execute :aggregated_search, query, opts
+    end
+
+    def delete_by(query, opts={})
+      execute :delete, query, opts
+    end
+
+    def exists?(query, opts={})
+      execute_count(query, opts) > 0
+    end
+
+    def index_document(document, opts={})
+      prepare :index, document, opts
+    end
+
+    def update_document(document, opts={})
+      prepare :update, document, opts
+    end
+
+    protected
+
+    # write actions
+
+    def _exec_index(document, opts={})
+      _exec_upsert :index, document, opts
+    end
+
+    def _exec_update(document, opts={})
+      _exec_upsert :update, document, opts
+    end
+
+    def _exec_upsert(method, document, opts={})
+      opts.merge! id: document.id if document.id
+      opts.merge! type: document.type unless opts[:type]
+      opts.merge! index: document.index unless opts[:index]
+      document.index = opts[:index]
+      document.type = opts[:type]
+      client.send "#{method}_document", document.source, opts
+    end
+
+    # read actions
+
+    def _exec_aggregations(query, opts={})
+      body = query.build_for_aggregations
+      response = client.search body, opts
+      query.parse_aggregations response['aggregations']
+    end
+
+    def _exec_count(query, opts={})
+      body = query.build_for_count
+      response = client.count body, opts
+      response['count']
+    end
+
+    def _exec_search(query, opts={})
+      _exec_paginated_search query, false, opts
+    end
+
+    def _exec_aggregated_search(query, opts={})
+      _exec_paginated_search query, true, opts
+    end
+
+    def _exec_delete(query, opts={})
+      client.delete query, opts
+    end
+
+    def _exec_paginated_search(query, aggregated, opts={})
+      scroll_interval = '3m'
+      body = aggregated ? query.build_for_aggregated_search : query.build_for_search
+      if query.limited?
+        response = client.search body, opts
+        Results.from_elasticsearch_response response, query
+      elsif query.sorted? || query.aggregated?
+        # normal pagination
+        page_size = 50
+        current_page = 1
+        loop do
+          offset = page_size * (current_page - 1)
+          response = client.search body, opts.merge(size: page_size, from: offset)
+          if current_page == 1
+            results = Results.from_elasticsearch_response response, query
+          else
+            results.append_results_from response
+          end
+          total_pages = (response['hits']['total'] / page_size.to_f).ceil
+          break if current_page >= total_pages
+          current_page += 1
+          body = query.build_for_search if aggregated
+        end
+        results
+      else
+        # scan & scroll
+        response = client.search body, opts.merge(search_type: 'scan', scroll: scroll_interval, size: 1000)
+        while response = client.scroll(response['_scroll_id'], scroll: scroll_interval) and not response['hits']['hits'].empty? do
+          results = results ? results.append_results_from(response) : Results.from_elasticsearch_response(response, query)
+        end
+        results ? results : Results.from_elasticsearch_response(response, query)
+      end
+    end
+
+    # abstract methods
+
+    def execute(action, query, opts={})
+      # child's implementation here
+      send "_exec_#{action}", query, opts
+    end
+
+    def prepare(action, document, opts={})
+      # child's implementation here
+      send "_exec_#{action}", document, opts
+    end
+
+  end
+end

data/lib/elasticated/results.rb

@@ -0,0 +1,43 @@
+module Elasticated
+
+  ShardsInfo = Struct.new :total, :successful, :failed
+  HitsInfo = Struct.new :total, :max_score
+
+  class Results < Array
+
+    def self.from_elasticsearch_response(elasticsearch_response, query=nil)
+      documents = elasticsearch_response['hits']['hits'].map{ |hit| Document.from_elasticsearch_hit hit }
+      results = new documents
+      results.documents = documents
+      # cluster metadata
+      results.took = elasticsearch_response['took']
+      results.timed_out = elasticsearch_response['timed_out']
+      # shards metadata
+      shards = elasticsearch_response['_shards']
+      results.shards = ShardsInfo.new shards['total'], shards['successful'], shards['failed']
+      # search metadata
+      hits = elasticsearch_response['hits']
+      results.hits = HitsInfo.new hits['total'], hits['max_score']
+      # aggregations results
+      aggregations = elasticsearch_response['aggregations']
+      results.aggregations = query.parse_aggregations aggregations if query && aggregations
+      results
+    end
+
+    attr_accessor :took, :timed_out
+    attr_accessor :shards # methods: total, successful, failed
+    attr_accessor :hits # methods: total, max_score
+    attr_accessor :documents, :aggregations
+
+    def append_results_from(elasticsearch_response)
+      elasticsearch_response['hits']['hits'].each do |hit|
+        documents.push Document.from_elasticsearch_hit hit
+      end
+    end
+
+    def sources
+      documents.map &:source
+    end
+
+  end
+end

data/lib/version.rb

@@ -0,0 +1,92 @@
+module Elasticated
+  VERSION = '1.0.0'
+end
+
+# Changelog
+
+# 1.0.0
+# Cambios en Repository
+# Se elimina InsightsRepository y sus hijos
+# Cambios en MonthlyPartitionedRepository
+# Ahora las queries pueden devolver resultados y aggregations en un solo pedido
+# Se agrega la clase Document
+# Se agrega la clase Results
+# Se eliminan SocialContentRepository y ConversationsRepository
+# El Terms Filter ahora acepta parametros opcionales (execution y cache)
+# El Nested Filter ahora acepta el parametro opcional cache
+# Ahora se puede setear el parametro cache para el Bool Filter
+# El Range Filter ahora acepta parametros opcionales (execution y cache)
+# Ahora se admite post_filter en la query
+# Ahora se admiten bools anidados
+
+# 0.8.2
+# Ahora se puede ordenar por multiples criterios en una TermsAggregation
+
+# 0.8.1
+# Se agrego la posibilidad de recibir post_zone y time_zone al DateHistogramAggregation
+
+# 0.8.0
+# Se agrego un repositorio de conversaciones
+
+# 0.7.2
+# Se agregan las metricas ads_actions y ads_post_engagement a FacebookInsightsRepository
+
+# 0.7.1
+# Se agrega el campo sub_type al mapping de post de FacebookInsightsRepository
+
+# 0.7.0
+# Agregado YoutubeInsightsRepository
+# Agregado InstagramInsightsRepository
+# Agregada opcion para configurar todos los repositorios al mismo tiempo
+# Agregadas las condiciones greater_than, greater_equal, less_than y lower_or_equals
+# Agregada RangeAggregation
+
+# 0.6.0
+# Agregado el campo business_ads al mapping de FacebookInsightsRepository
+# Agregados los campos de la marketing api a los tipos de FacebookInsightsRepository
+# Agregadas las metricas 'embedded_media_views' y 'video_views' a TwitterInsightsRepository
+
+# 0.5.2
+# Ahora se puede "no compactar" los resultados de una AggregatedQuery
+# Fix critico al metodo '==' de FilterAggregation
+# Agregadas las metricas del tipo 'video_retention' a FacebookInsightsRepository
+# Agregada la opcion :offset a DateHistogramAggregation
+
+# 0.5.1
+# Fix critico a FacebookInsightsRepository#update
+# Actualizada la dependencia keepcon_utils a la version 0.3
+
+# 0.5.0
+# Agregadas metricas faltantes de video al mapping de FacebookInsightsRepository
+# Fix critico al metodo '==' de QueryConditions
+# Agregado MappingBuilder
+# Agregada la clase Client, wrapper de Elasticsearch::Client
+# Se elimina la dependencia de Repository hacia Elasticsearch::Client, ahora pasa por Client
+# Ahora el TermsAggregation se construye con size 0 por default
+# Se elimina AbstractQuery#over_repository
+# La opcion 'include_id' de repository.execute_query pasa a llamarse 'metadata'
+
+# 0.4.1
+# Ahora los valores numericos de InsightsRepository son del tipo long
+
+# 0.4.0
+# Agregado metodo update_document en InsightsRepository
+# Agregadas las key tipo 'calc' ademas de las preexistentes 'gen' a InsightsRepository
+# Contemplados los pares key-value de tercer nivel en InsightsRepository#prepare,restore
+# Ahora repository.execute_query acepta la opcion 'include_id'
+
+# 0.3.0
+# Agregado metodo create_test_index en SocialContentRepository
+
+# 0.2.0
+# Se agrega NestedAggregation
+# Ahora las aggregations que adminen subaggregations se "auto-mergean"
+# Agregado el filtro 'nested'
+# Agregado 'min' aggregation
+# Agregado 'page_fans_country' al mapping de fb insights
+# Agregado 'page_fans_gender_age' al mapping de fb insights
+# Los metodos 'create_index_for' y 'put_alias_for' de 'facebook_insights_repository' ahora son publicos
+# Se agrega HistogramAggregation
+# Se agrega MonthlyPartitionedRepository
+# Se agrega InsightsRepository
+# Se agrega InsightsMappingGenerator