artirix_data_models 0.5.0 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f6a90c9b7956cc31c38374094fd416d8d4e9dd09
-  data.tar.gz: 3cf3edcbf15f24b14b7beb57943b9f93bb12dbaa
+  metadata.gz: 4c953eedf5e26123d9a2136591abf12b0a6be6ff
+  data.tar.gz: a3966ad6e6704d526176554e4b9a5d332dde1264
 SHA512:
-  metadata.gz: 568c1f293797134592df3bd5bca0ba3daacf0ede0aff7770004b5a3aae4821f9ae8eb2df8a668607bff2bfcb4f31bc6ea1a8c334b213084c19763ddebe578714
-  data.tar.gz: 5e27b33a022ccb060b174fb4466fb62bea449cc8c6ff8a5b3b405482b20f0f7c14eba73eeeddd4f62b31d447b0f2999d0afa8cf2485f095eda3ff3d425dea8bc
+  metadata.gz: 2c79dbd9b140a1e584bad0d110d38a6a6ae384761d41ea296d78a58ebc81c44cea3691cbc5ed5ef00cdd9e5fb92d565549946ffeaf356748b57fac362f155677
+  data.tar.gz: afc8d927f2082134cb81a9f118f73b799882a019fccc20760886c2d34a2fc0cebfae479794523e735eccddf9b5110edecd1d6b04bd2c61fac00dcc0ebe53913c

data/.gitignore

@@ -12,4 +12,5 @@
 *.o
 *.a
 mkmf.log
-.idea
+.idea
+/manual_sandbox.rb

data/README.md

@@ -224,7 +224,30 @@ end
 
 ## Changes
 
-### v.0.5.0
+### 0.6.2
+
+*Fixed Breaking Change*: removal of `Aggregation.from_json` static method. Now back but delegating to default factory method is `aggregation_factory.aggregation_from_json` in the Aggregation Factory *instance*.
+
+- EsCollection's aggregations can now be build based on raw ElasticSearch responses, including nested aggregations. It ignores any aggregation that does not have "buckets", so that nested aggs for `global` or `filtered` are skipped and only the ones with real data are used. (TODO: write docs. In the mean time, have a look at the specs).
+- added `aggregation` method to `Aggregation::Value` class, and also the aggs to the `data_hash` if they are present.
+
+### 0.5.0
 
 - opening gem as is to the public.
-- still a lot of TODOs in the documentation
+- still a lot of TODOs in the documentation
+
+
+## Yanked versions
+
+
+### ~0.6.1~ 
+
+Yanked because of breaking change introduction: removal of `Aggregation.from_json` method
+
+- added `aggregation` method to `Aggregation::Value` class, and also the aggs to the `data_hash` if they are present.
+
+### ~v0.6.0~
+
+Yanked because of breaking change introduction: removal of `Aggregation.from_json` method
+
+- EsCollection's aggregations can now be build based on raw ElasticSearch responses, including nested aggregations. It ignores any aggregation that does not have "buckets", so that nested aggs for `global` or `filtered` are skipped and only the ones with real data are used. (TODO: write docs. In the mean time, have a look at the specs).

data/artirix_data_models.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'faraday'
   spec.add_dependency 'keyword_init', '~> 1.3'
   spec.add_dependency 'naught'
+  spec.add_dependency 'hashie'
 
   spec.add_development_dependency 'kaminari', '~> 0.16'
   spec.add_development_dependency 'will_paginate', '~> 3.0'

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'artirix_data_models'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require 'pry'
+Pry.start
+
+# require 'irb'
+# IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/artirix_data_models/aggregation.rb

@@ -6,11 +6,7 @@ module ArtirixDataModels
     delegate :each, :empty?, to: :buckets
 
     def self.from_json(definition, value_class = Value)
-      buckets = definition[:buckets].map do |bucket|
-        value_class.new definition[:name].to_sym, bucket[:name], bucket[:count]
-      end
-
-      new definition[:name].to_sym, buckets
+      DAORegistry.aggregations_factory.aggregation_from_json(definition, value_class: value_class, aggregation_class: self)
     end
 
     def pretty_name
@@ -32,7 +28,14 @@ module ArtirixDataModels
       }
     end
 
-    class Value < Struct.new(:aggregation_name, :name, :count)
+    class Value < Struct.new(:aggregation_name, :name, :count, :aggregations)
+
+      def aggregations
+        Array(super)
+      end
+
+      alias_method :nested_aggregations, :aggregations
+      alias_method :nested_aggregations=, :aggregations=
 
       def pretty_name
         tranlsation_key = "aggregations.#{aggregation_name.to_s.gsub('.', '_')}.buckets.#{name.to_s.gsub('.', '_')}"
@@ -47,7 +50,20 @@ module ArtirixDataModels
         count == 0
       end
 
+      def aggregation(name)
+        n = name.to_sym
+        aggregations.detect { |x| x.name == n }
+      end
+
       def data_hash
+        basic_data_hash.tap do |h|
+          if aggregations.present?
+            h[:aggregations] = aggregations.map(&:data_hash)
+          end
+        end
+      end
+
+      def basic_data_hash
         {
           name:  name,
           count: count

data/lib/artirix_data_models/aggregation_builder.rb

@@ -0,0 +1,47 @@
+module ArtirixDataModels
+
+  class AggregationBuilder
+    attr_reader :aggregations_factory, :definition, :value_class, :aggregation_class
+
+    def initialize(aggregations_factory:, definition:, aggregation_class: Aggregation, value_class: Aggregation::Value)
+      @aggregations_factory = aggregations_factory
+      @definition           = definition
+      @aggregation_class    = aggregation_class
+      @value_class          = value_class
+    end
+
+    def build
+      aggregation_class.new agg_name, buckets
+    end
+
+    alias_method :call, :build
+
+    private
+    def buckets
+      definition[:buckets].map do |bucket|
+        build_bucket(bucket)
+      end
+    end
+
+    def build_bucket(bucket)
+      name        = bucket[:name]
+      count       = bucket[:count]
+      nested_aggs = nested_aggs_from(bucket)
+
+      value_class.new agg_name, name, count, nested_aggs
+    end
+
+    def nested_aggs_from(bucket)
+      raw_nested_aggs = bucket.fetch(:aggregations) { [] }
+
+      raw_nested_aggs.map do |nested_agg|
+        aggregations_factory.build_from_json nested_agg, value_class
+      end
+    end
+
+    def agg_name
+      definition[:name].to_sym
+    end
+
+  end
+end

data/lib/artirix_data_models/aggregations_factory.rb

@@ -1,28 +1,18 @@
 module ArtirixDataModels
   class AggregationsFactory
-    DEFAULT_FACTORY               = ->(aggregation) { Aggregation.from_json aggregation }
     DEFAULT_COLLECTION_CLASS_NAME = ''.freeze
 
-    # singleton instance
     def initialize
       @_loaders = Hash.new { |h, k| h[k] = {} }
       setup_config
     end
 
+    # SETUP AND CONFIG MANAGEMENT
+
     def setup_config
       # To be Extended
     end
 
-    def build_from_json(aggregation, collection_class = nil)
-      get_loader(aggregation[:name], collection_class).call aggregation
-    end
-
-    def get_loader(aggregation_name, collection_class)
-      @_loaders[collection_class.to_s][aggregation_name.to_s] ||
-        @_loaders[DEFAULT_COLLECTION_CLASS_NAME][aggregation_name.to_s] ||
-        DEFAULT_FACTORY
-    end
-
     def set_loader(aggregation_name, collection_class = nil, loader = nil, &block)
       if block
         @_loaders[collection_class.to_s][aggregation_name.to_s] = block
@@ -33,13 +23,43 @@ module ArtirixDataModels
       end
     end
 
-    # static methods
-    def self.set_loader(aggregation_name, collection_class, loader = nil, &block)
-      instance.set_loader aggregation_name, collection_class, loader, &block
+    def default_loader
+      proc { |aggregation| Aggregation.from_json aggregation }
     end
 
-    def self.build_from_json(aggregation_name, collection_class)
-      instance.build_from_json aggregation_name, collection_class
+    def get_loader(aggregation_name, collection_class)
+      @_loaders[collection_class.to_s][aggregation_name.to_s] ||
+        @_loaders[DEFAULT_COLLECTION_CLASS_NAME][aggregation_name.to_s] ||
+        default_loader
     end
+
+    # AGGREGATION BUILDING
+
+    def build_from_json(aggregation, collection_class = nil)
+      get_loader(aggregation[:name], collection_class).call aggregation
+    end
+
+    def build_all_from_raw_data(raw, collection_class = nil)
+      normalised = normalise_aggregations_data(raw)
+      normalised.map { |definition| build_from_json definition, collection_class }
+    end
+
+    def aggregation_from_json(definition, value_class: Aggregation::Value, aggregation_class: Aggregation)
+      builder_params = {
+        aggregations_factory: self,
+        definition: definition,
+        aggregation_class: aggregation_class,
+        value_class: value_class,
+      }
+
+      AggregationBuilder.new(builder_params).build
+    end
+
+    private
+
+    def normalise_aggregations_data(raw_aggs)
+      RawAggregationDataNormaliser.new(self, raw_aggs).normalise
+    end
+
   end
 end

data/lib/artirix_data_models/dao_registry.rb

@@ -27,6 +27,10 @@ class ArtirixDataModels::DAORegistry
     set_loader(:model_fields) { ArtirixDataModels::ModelFieldsDAO.new gateway: get(:gateway) }
   end
 
+  def aggregations_factory
+    get :aggregations_factory
+  end
+
   def method_missing(method, *args, &block)
     if exist?(method)
       get(method)

data/lib/artirix_data_models/es_collection.rb

@@ -106,7 +106,7 @@ module ArtirixDataModels
     end
 
     def aggregations
-      @aggregations ||= response[:aggregations].to_a.map { |aggregation| aggregations_factory.build_from_json aggregation, model_class }
+      @aggregations ||= build_aggregations
     end
 
     def aggregation(name)
@@ -152,7 +152,15 @@ module ArtirixDataModels
       ].join(CACHE_KEY_SECTION_SEPARATOR)
     end
 
+    def raw_aggregations_data
+      response[:aggregations]
+    end
+
     private
+    def build_aggregations
+      aggregations_factory.build_all_from_raw_data(raw_aggregations_data)
+    end
+
     def load_results
       hits[:hits].map do |document|
         deserialize_document(document)

data/lib/artirix_data_models/raw_aggregation_data_normaliser.rb

@@ -0,0 +1,68 @@
+module ArtirixDataModels
+  class RawAggregationDataNormaliser
+
+    FIND_BUCKETS = ->(_k, v, _o) { v.respond_to?(:key?) && v.key?(:buckets) }
+
+    attr_reader :raw_aggs, :aggregations_factory
+
+    def initialize(aggregations_factory, raw_aggs)
+      @aggregations_factory = aggregations_factory
+      @raw_aggs             = raw_aggs
+    end
+
+    def normalise
+      return [] unless raw_aggs.present?
+      return raw_aggs if Array === raw_aggs
+
+      normalise_hash
+    end
+
+    alias_method :call, :normalise
+
+    private
+
+    def normalise_hash
+      with_buckets_list = deep_locate raw_aggs, FIND_BUCKETS
+
+      with_buckets_list.reduce([]) do |list, with_buckets|
+        with_buckets.each do |name, value|
+          add_normalised_element_to_list(list, name, value)
+        end
+
+        list
+      end
+    end
+
+    def add_normalised_element_to_list(list, k, v)
+      return unless Hash === v && v.key?(:buckets)
+
+      buckets = v[:buckets].map do |raw_bucket|
+        normalise_bucket(raw_bucket)
+      end
+
+      list << { name: k, buckets: buckets }
+    end
+
+    def normalise_bucket(raw_bucket)
+      basic_bucket(raw_bucket).tap do |bucket|
+        nested_aggs           = nested_aggs_from(raw_bucket)
+        bucket[:aggregations] = nested_aggs if nested_aggs.present?
+      end
+    end
+
+    def basic_bucket(raw_bucket)
+      name  = raw_bucket[:key] || raw_bucket[:name]
+      count = raw_bucket[:doc_count] || raw_bucket[:count]
+      { name: name, count: count }
+    end
+
+    # aux
+    def deep_locate(object, callable)
+      Hashie::Extensions::DeepLocate.deep_locate(callable, object).deep_dup
+    end
+
+    def nested_aggs_from(raw_bucket)
+      RawAggregationDataNormaliser.new(aggregations_factory, raw_bucket).normalise
+    end
+  end
+end

data/lib/artirix_data_models/spec_support/fake_mode.rb

@@ -1,3 +1,4 @@
+# :nocov:
 def fake_mode_for(model_name)
   before(:all) do
     SimpleConfig.for(:site) do
@@ -22,3 +23,4 @@ def fake_mode_for(model_name)
   end
 end
 
+# :nocov:

data/lib/artirix_data_models/spec_support/gateway_mock.rb

@@ -1,3 +1,4 @@
+# :nocov:
 def given_gateway_config(connection_url = nil)
   connection_url ||= 'http://example.com/other'
   before(:all) do
@@ -21,3 +22,5 @@ def mock_gateway_get_not_found_response(path:, body: nil, gateway: nil)
 
   allow(gateway).to receive(:perform_get).with(path, body).and_raise ArtirixDataModels::DataGateway::NotFound
 end
+
+# :nocov:

data/lib/artirix_data_models/spec_support/shared_examples/a_readonly_active_model_like_artirix_data_models.rb

@@ -1,3 +1,5 @@
+# :nocov:
+
 # based on https://github.com/geoffharcourt/active_model_lint-rspec/blob/master/lib/active_model_lint-rspec/an_active_model.rb
 shared_examples_for 'a ReadOnly ActiveModel like ArtirixDataModels' do
   require 'active_model/lint'
@@ -18,3 +20,4 @@ shared_examples_for 'a ReadOnly ActiveModel like ArtirixDataModels' do
   end
 end
 
+# :nocov:

data/lib/artirix_data_models/spec_support/shared_examples/an_artirix_data_model_dao.rb

@@ -1,4 +1,4 @@
-
+# :nocov:
 shared_examples_for 'an ArtirixDataModel DAO' do
   # example of needed definitions
   #
@@ -95,3 +95,4 @@ shared_examples_for 'an ArtirixDataModel DAO' do
     Then { result == partial_mode_fields_list }
   end
 end
+# :nocov:

data/lib/artirix_data_models/spec_support/shared_examples/an_artirix_data_model_model.rb

@@ -1,4 +1,4 @@
-
+# :nocov:
 shared_examples_for 'an ArtirixDataModel Model' do
 
   # Example of Config:
@@ -222,3 +222,4 @@ shared_examples_for 'an ArtirixDataModel Model' do
   end
 end
 
+# :nocov:

data/lib/artirix_data_models/spec_support/shared_examples/has_attributes.rb

@@ -1,3 +1,4 @@
+# :nocov:
 shared_examples_for 'has attributes' do
   describe 'responds to the given attribute getters and setters' do
     When(:subject) { described_class.new }
@@ -29,4 +30,6 @@ shared_examples_for 'has attributes' do
       attributes.reject { |at| subject.respond_to? "#{at}?" }.empty?
     end
   end
-end
+end
+
+# :nocov:

data/lib/artirix_data_models/version.rb

@@ -1,3 +1,3 @@
 module ArtirixDataModels
-  VERSION = '0.5.0'
+  VERSION = '0.6.2'
 end

data/lib/artirix_data_models.rb

@@ -1,4 +1,4 @@
-require "artirix_data_models/version"
+require 'artirix_data_models/version'
 
 # dependencies
 require 'active_support/all'
@@ -7,6 +7,7 @@ require 'oj'
 require 'faraday'
 require 'keyword_init'
 require 'naught'
+require 'hashie'
 
 # note DO NOT require kaminari or will_paginate, it'll be done when invoking `ArtirixDataModels::EsCollection.work_with_will_paginate`
 
@@ -14,6 +15,8 @@ require 'naught'
 # loading features
 require 'artirix_data_models/es_collection'
 require 'artirix_data_models/aggregations_factory'
+require 'artirix_data_models/raw_aggregation_data_normaliser'
+require 'artirix_data_models/aggregation_builder'
 require 'artirix_data_models/aggregation'
 require 'artirix_data_models/model'
 require 'artirix_data_models/gateways/data_gateway'

data/spec/artirix_data_models/es_collection_spec.rb

@@ -4,7 +4,7 @@ RSpec.describe ArtirixDataModels::EsCollection, type: :model do
 
   describe "with an ES response" do
 
-    Given(:model_class){
+    Given(:model_class) {
       Class.new do
         attr_reader :data
 
@@ -13,9 +13,9 @@ RSpec.describe ArtirixDataModels::EsCollection, type: :model do
         end
       end
     }
-    Given(:es_response){ ArtirixDataModels::EsCollection::EMPTY_RESPONSE }
+    Given(:es_response) { ArtirixDataModels::EsCollection::EMPTY_RESPONSE }
 
-    When(:es_collection){ ArtirixDataModels::EsCollection.new(model_class, response: es_response) }
+    When(:es_collection) { ArtirixDataModels::EsCollection.new(model_class, response: es_response) }
 
     Then { es_collection.total == 0 }
     Then { es_collection.results == [] }
@@ -23,7 +23,7 @@ RSpec.describe ArtirixDataModels::EsCollection, type: :model do
 
   describe "coerced from an array" do
 
-    Given(:model_class){
+    Given(:model_class) {
       Class.new do
         attr_reader :number
 
@@ -33,13 +33,142 @@ RSpec.describe ArtirixDataModels::EsCollection, type: :model do
       end
     }
 
-    Given(:results_array){ [model_class.new(1), model_class.new(2), model_class.new(3)] }
+    Given(:results_array) { [model_class.new(1), model_class.new(2), model_class.new(3)] }
 
-    When(:es_collection){ ArtirixDataModels::EsCollection.from_array(results_array) }
+    When(:es_collection) { ArtirixDataModels::EsCollection.from_array(results_array) }
 
     Then { es_collection.total == 3 }
     Then { es_collection.results == results_array }
     Then { es_collection.aggregations == [] }
   end
 
+  describe 'aggregations' do
+
+    Given(:model_class) do
+      Class.new do
+        attr_reader :data
+
+        def initialize(data)
+          @data = { given: data }
+        end
+      end
+    end
+
+    Given(:es_response) { Oj.load(File.read(fixture_file), symbol_keys: true) }
+
+    context 'with Data Layer conversion into array' do
+      Given(:fixture_file) { fixture_pathname('articles_search_dl.json') }
+
+      When(:es_collection) { ArtirixDataModels::EsCollection.new(model_class, response: es_response) }
+
+      Then { es_collection.total == 11477 }
+      Then { es_collection.results == [] }
+
+      Then { es_collection.aggregations.size == 2 }
+
+      Then { es_collection.aggregations.first.name == :primary_category_slug }
+      Then { es_collection.aggregations.first.buckets.size == 10 }
+      Then { es_collection.aggregations.first.buckets.first.name == 'yacht-market-intelligence' }
+      Then { es_collection.aggregations.first.buckets.first.count == 7794 }
+      Then { es_collection.aggregations.first.buckets.last.name == 'brokerage' }
+      Then { es_collection.aggregations.first.buckets.last.count == 7 }
+
+      Then { es_collection.aggregations.last.name == :location_slug }
+      Then { es_collection.aggregations.last.buckets.size == 10 }
+      Then { es_collection.aggregations.last.buckets.first.name == 'americas' }
+      Then { es_collection.aggregations.last.buckets.first.count == 3 }
+      Then { es_collection.aggregations.last.buckets.last.name == 'italy' }
+      Then { es_collection.aggregations.last.buckets.last.count == 1 }
+    end
+
+    context 'with Raw ES with single aggregation' do
+      Given(:fixture_file) { fixture_pathname('articles_search_raw_es.json') }
+
+      When(:es_collection) { ArtirixDataModels::EsCollection.new(model_class, response: es_response) }
+
+      Then { es_collection.total == 14974 }
+      Then { es_collection.results == [] }
+
+      Then { es_collection.aggregations.size == 1 }
+
+      Then { es_collection.aggregations.first.name == :disease_slug }
+      Then { es_collection.aggregations.first.buckets.size == 6 }
+      Then { es_collection.aggregations.first.buckets.first.name == 'sickle-cell-anemia' }
+      Then { es_collection.aggregations.first.buckets.first.count == 5754 }
+      Then { es_collection.aggregations.first.buckets.last.name == 'fabry-disease' }
+      Then { es_collection.aggregations.first.buckets.last.count == 742 }
+    end
+
+    context 'with Raw ES with single nested aggregation' do
+      Given(:fixture_file) { fixture_pathname('articles_search_nested_single_raw_es.json') }
+
+      When(:es_collection) { ArtirixDataModels::EsCollection.new(model_class, response: es_response) }
+
+      Then { es_collection.total == 11492 }
+      Then { es_collection.results == [] }
+
+      Then { es_collection.aggregations.size == 2 }
+
+      Then { es_collection.aggregations.first.name == :category_slug }
+      Then { es_collection.aggregations.first.buckets.size == 24 }
+      Then { es_collection.aggregations.first.buckets.first.name == 'brokerage-sales-news' }
+      Then { es_collection.aggregations.first.buckets.first.count == 7792 }
+      Then { es_collection.aggregations.first.buckets.last.name == 'why-charter-a-superyacht' }
+      Then { es_collection.aggregations.first.buckets.last.count == 1 }
+
+      Then { es_collection.aggregations.last.name == :location_slug }
+      Then { es_collection.aggregations.last.buckets.size == 19 }
+      Then { es_collection.aggregations.last.buckets.first.name == 'americas' }
+      Then { es_collection.aggregations.last.buckets.first.count == 4 }
+      Then { es_collection.aggregations.last.buckets.last.name == 'tahiti' }
+      Then { es_collection.aggregations.last.buckets.last.count == 1 }
+    end
+
+    context 'with Raw ES with nested aggregations' do
+      Given(:fixture_file) { fixture_pathname('articles_search_nested_raw_es.json') }
+
+      Given(:model_class) {
+        Class.new do
+          attr_reader :data
+
+          def initialize(data)
+            @data = { given: data }
+          end
+        end
+      }
+      Given(:es_response) { Oj.load(File.read(fixture_file), symbol_keys: true) }
+
+      When(:es_collection) { ArtirixDataModels::EsCollection.new(model_class, response: es_response) }
+
+      Then { es_collection.total == 4512 }
+      Then { es_collection.results == [] }
+
+      Then { es_collection.aggregations.size == 1 }
+
+      Then { es_collection.aggregations.first.name == :level1_taxonomy }
+      Then { es_collection.aggregations.first.buckets.size == 4 }
+
+      Then { es_collection.aggregations.first.buckets.first.name == 'Treatment' }
+      Then { es_collection.aggregations.first.buckets.first.count == 2404 }
+      Then { es_collection.aggregations.first.buckets.first.aggregations.size == 1 }
+      Then { es_collection.aggregations.first.buckets.first.aggregations.first.name == :level2_taxonomy }
+      Then { es_collection.aggregations.first.buckets.first.aggregations.first.buckets.size == 7 }
+      Then { es_collection.aggregations.first.buckets.first.aggregations.first.buckets.first.name == 'Drug Treatments' }
+      Then { es_collection.aggregations.first.buckets.first.aggregations.first.buckets.first.count == 977 }
+      Then { es_collection.aggregations.first.buckets.first.aggregations.first.buckets.last.name == 'Complementary and Alternative Therapies' }
+      Then { es_collection.aggregations.first.buckets.first.aggregations.first.buckets.last.count == 14 }
+
+
+      Then { es_collection.aggregations.first.buckets.last.name == 'Living' }
+      Then { es_collection.aggregations.first.buckets.last.count == 365 }
+      Then { es_collection.aggregations.first.buckets.last.aggregations.size == 1 }
+      Then { es_collection.aggregations.first.buckets.last.aggregations.first.name == :level2_taxonomy }
+      Then { es_collection.aggregations.first.buckets.last.aggregations.first.buckets.size == 8 }
+      Then { es_collection.aggregations.first.buckets.last.aggregations.first.buckets.first.name == 'Emotional Impact' }
+      Then { es_collection.aggregations.first.buckets.last.aggregations.first.buckets.first.count == 104 }
+      Then { es_collection.aggregations.first.buckets.last.aggregations.first.buckets.last.name == 'Cognitive Impact' }
+      Then { es_collection.aggregations.first.buckets.last.aggregations.first.buckets.last.count == 3 }
+    end
+  end
+
 end

data/spec/fixtures/articles_search_dl.json

@@ -0,0 +1,99 @@
+{
+  "hits": {
+    "total": 11477,
+    "max_score": 0.0,
+    "hits": []
+  },
+  "aggregations": [
+    {
+      "name": "primary_category_slug",
+      "buckets": [
+        {
+          "name": "yacht-market-intelligence",
+          "count": 7794
+        },
+        {
+          "name": "brokerage-sales-news",
+          "count": 7792
+        },
+        {
+          "name": "yachts",
+          "count": 2238
+        },
+        {
+          "name": "news",
+          "count": 2236
+        },
+        {
+          "name": "charter",
+          "count": 1322
+        },
+        {
+          "name": "luxury-yacht-charter-news",
+          "count": 1292
+        },
+        {
+          "name": "luxury-yacht-events",
+          "count": 110
+        },
+        {
+          "name": "luxury-yacht-events-news",
+          "count": 110
+        },
+        {
+          "name": "luxury-yacht-charter-advice",
+          "count": 15
+        },
+        {
+          "name": "brokerage",
+          "count": 7
+        }
+      ]
+    },
+    {
+      "name": "location_slug",
+      "buckets": [
+        {
+          "name": "americas",
+          "count": 3
+        },
+        {
+          "name": "central-america",
+          "count": 3
+        },
+        {
+          "name": "europe",
+          "count": 3
+        },
+        {
+          "name": "caribbean",
+          "count": 2
+        },
+        {
+          "name": "costa-rica",
+          "count": 2
+        },
+        {
+          "name": "mediterranean",
+          "count": 2
+        },
+        {
+          "name": "anegada",
+          "count": 1
+        },
+        {
+          "name": "france",
+          "count": 1
+        },
+        {
+          "name": "french-polynesia",
+          "count": 1
+        },
+        {
+          "name": "italy",
+          "count": 1
+        }
+      ]
+    }
+  ]
+}

data/spec/fixtures/articles_search_nested_raw_es.json

@@ -0,0 +1,149 @@
+{
+  "took": 16,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "failed": 0
+  },
+  "hits": {
+    "total": 4512,
+    "max_score": 0,
+    "hits": []
+  },
+  "aggregations": {
+    "taxonomy": {
+      "doc_count": 4674,
+      "level1_taxonomy": {
+        "doc_count_error_upper_bound": 0,
+        "sum_other_doc_count": 0,
+        "buckets": [
+          {
+            "key": "Treatment",
+            "doc_count": 2404,
+            "level2_taxonomy": {
+              "doc_count_error_upper_bound": 0,
+              "sum_other_doc_count": 0,
+              "buckets": [
+                {
+                  "key": "Drug Treatments",
+                  "doc_count": 977
+                },
+                {
+                  "key": "Drug Development",
+                  "doc_count": 731
+                },
+                {
+                  "key": "Other Treatments",
+                  "doc_count": 358
+                },
+                {
+                  "key": "Surgical Treatments",
+                  "doc_count": 264
+                },
+                {
+                  "key": "Assistive Devices",
+                  "doc_count": 43
+                },
+                {
+                  "key": "Palliative Care",
+                  "doc_count": 17
+                },
+                {
+                  "key": "Complementary and Alternative Therapies",
+                  "doc_count": 14
+                }
+              ]
+            }
+          },
+          {
+            "key": "Understand",
+            "doc_count": 1356,
+            "level2_taxonomy": {
+              "doc_count_error_upper_bound": 0,
+              "sum_other_doc_count": 0,
+              "buckets": [
+                {
+                  "key": "Symptoms",
+                  "doc_count": 1066
+                },
+                {
+                  "key": "Outlook",
+                  "doc_count": 143
+                },
+                {
+                  "key": "Causes",
+                  "doc_count": 123
+                },
+                {
+                  "key": "Ethics",
+                  "doc_count": 24
+                }
+              ]
+            }
+          },
+          {
+            "key": "Diagnosis",
+            "doc_count": 549,
+            "level2_taxonomy": {
+              "doc_count_error_upper_bound": 0,
+              "sum_other_doc_count": 0,
+              "buckets": [
+                {
+                  "key": "Diagnostic Techniques",
+                  "doc_count": 525
+                },
+                {
+                  "key": "Diagnostic Challenges",
+                  "doc_count": 24
+                }
+              ]
+            }
+          },
+          {
+            "key": "Living",
+            "doc_count": 365,
+            "level2_taxonomy": {
+              "doc_count_error_upper_bound": 0,
+              "sum_other_doc_count": 0,
+              "buckets": [
+                {
+                  "key": "Emotional Impact",
+                  "doc_count": 104
+                },
+                {
+                  "key": "Genetic Counseling",
+                  "doc_count": 63
+                },
+                {
+                  "key": "Diet",
+                  "doc_count": 59
+                },
+                {
+                  "key": "Fertility/Sex",
+                  "doc_count": 56
+                },
+                {
+                  "key": "Exercise",
+                  "doc_count": 48
+                },
+                {
+                  "key": "Physical Impact",
+                  "doc_count": 26
+                },
+                {
+                  "key": "Sleep",
+                  "doc_count": 6
+                },
+                {
+                  "key": "Cognitive Impact",
+                  "doc_count": 3
+                }
+              ]
+            }
+          }
+        ]
+      }
+    }
+  }
+}

data/spec/fixtures/articles_search_nested_single_raw_es.json

@@ -0,0 +1,212 @@
+{
+  "took": 2,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "failed": 0
+  },
+  "hits": {
+    "total": 11492,
+    "max_score": 0,
+    "hits": []
+  },
+  "aggregations": {
+    "global_primary_category_slug": {
+      "doc_count": 11492,
+      "global_filtered_primary_category_slug": {
+        "doc_count": 11492,
+        "category_slug": {
+          "doc_count_error_upper_bound": 0,
+          "sum_other_doc_count": 0,
+          "buckets": [
+            {
+              "key": "brokerage-sales-news",
+              "doc_count": 7792
+            },
+            {
+              "key": "news",
+              "doc_count": 2236
+            },
+            {
+              "key": "luxury-yacht-charter-news",
+              "doc_count": 1292
+            },
+            {
+              "key": "luxury-yacht-events-news",
+              "doc_count": 110
+            },
+            {
+              "key": "charter",
+              "doc_count": 14
+            },
+            {
+              "key": "brokerage",
+              "doc_count": 7
+            },
+            {
+              "key": "luxury-yacht-charter-advice",
+              "doc_count": 5
+            },
+            {
+              "key": "cruising-advice",
+              "doc_count": 4
+            },
+            {
+              "key": "luxury-yacht-life",
+              "doc_count": 4
+            },
+            {
+              "key": "safety-advice",
+              "doc_count": 4
+            },
+            {
+              "key": "design01-news",
+              "doc_count": 3
+            },
+            {
+              "key": "plan-and-book-advice",
+              "doc_count": 3
+            },
+            {
+              "key": "charter-news",
+              "doc_count": 2
+            },
+            {
+              "key": "destinations",
+              "doc_count": 2
+            },
+            {
+              "key": "superyacht-sales-data",
+              "doc_count": 2
+            },
+            {
+              "key": "testcat123456789-edit",
+              "doc_count": 2
+            },
+            {
+              "key": "yachts",
+              "doc_count": 2
+            },
+            {
+              "key": "yachts-luxury-yacht-broker-directory",
+              "doc_count": 2
+            },
+            {
+              "key": "annual-reports",
+              "doc_count": 1
+            },
+            {
+              "key": "boat-shows",
+              "doc_count": 1
+            },
+            {
+              "key": "company-news",
+              "doc_count": 1
+            },
+            {
+              "key": "new-to-the-fleet",
+              "doc_count": 1
+            },
+            {
+              "key": "superyacht1234",
+              "doc_count": 1
+            },
+            {
+              "key": "why-charter-a-superyacht",
+              "doc_count": 1
+            }
+          ]
+        }
+      }
+    },
+    "global_location_slug": {
+      "doc_count": 11492,
+      "global_filtered_location_slug": {
+        "doc_count": 11492,
+        "location_slug": {
+          "doc_count_error_upper_bound": 0,
+          "sum_other_doc_count": 0,
+          "buckets": [
+            {
+              "key": "americas",
+              "doc_count": 4
+            },
+            {
+              "key": "central-america",
+              "doc_count": 4
+            },
+            {
+              "key": "europe",
+              "doc_count": 3
+            },
+            {
+              "key": "caribbean",
+              "doc_count": 2
+            },
+            {
+              "key": "costa-rica",
+              "doc_count": 2
+            },
+            {
+              "key": "mediterranean",
+              "doc_count": 2
+            },
+            {
+              "key": "anegada",
+              "doc_count": 1
+            },
+            {
+              "key": "france",
+              "doc_count": 1
+            },
+            {
+              "key": "french-polynesia",
+              "doc_count": 1
+            },
+            {
+              "key": "italy",
+              "doc_count": 1
+            },
+            {
+              "key": "leeward-islands",
+              "doc_count": 1
+            },
+            {
+              "key": "north-america",
+              "doc_count": 1
+            },
+            {
+              "key": "pacific",
+              "doc_count": 1
+            },
+            {
+              "key": "panama",
+              "doc_count": 1
+            },
+            {
+              "key": "rest-of-the-world",
+              "doc_count": 1
+            },
+            {
+              "key": "san-blas-islands",
+              "doc_count": 1
+            },
+            {
+              "key": "spain",
+              "doc_count": 1
+            },
+            {
+              "key": "st-barths",
+              "doc_count": 1
+            },
+            {
+              "key": "tahiti",
+              "doc_count": 1
+            }
+          ]
+        }
+      }
+    }
+  }
+}

data/spec/fixtures/articles_search_raw_es.json

@@ -0,0 +1,46 @@
+{
+  "took": 5,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "failed": 0
+  },
+  "hits": {
+    "total": 14974,
+    "max_score": 0,
+    "hits": []
+  },
+  "aggregations": {
+    "disease_slug": {
+      "doc_count_error_upper_bound": 0,
+      "sum_other_doc_count": 0,
+      "buckets": [
+        {
+          "key": "sickle-cell-anemia",
+          "doc_count": 5754
+        },
+        {
+          "key": "cystic-fibrosis",
+          "doc_count": 4512
+        },
+        {
+          "key": "behcet-syndrome",
+          "doc_count": 1613
+        },
+        {
+          "key": "myasthenia-gravis",
+          "doc_count": 1402
+        },
+        {
+          "key": "cushing-syndrome",
+          "doc_count": 951
+        },
+        {
+          "key": "fabry-disease",
+          "doc_count": 742
+        }
+      ]
+    }
+  }
+}

data/spec/support/fixtures.rb

@@ -0,0 +1,5 @@
+require 'pathname'
+
+def fixture_pathname(filename)
+  Pathname(__FILE__).join("../../fixtures/#{filename}")
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: artirix_data_models
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.2
 platform: ruby
 authors:
 - Eduardo Turiño
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-07 00:00:00.000000000 Z
+date: 2015-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: kaminari
   requirement: !ruby/object:Gem::Requirement
@@ -195,7 +209,9 @@ dependencies:
 description: used in Boat International UI and Admin apps
 email:
 - eturino@artirix.com
-executables: []
+executables:
+- console
+- setup
 extensions: []
 extra_rdoc_files: []
 files:
@@ -207,8 +223,11 @@ files:
 - README.md
 - Rakefile
 - artirix_data_models.gemspec
+- bin/console
+- bin/setup
 - lib/artirix_data_models.rb
 - lib/artirix_data_models/aggregation.rb
+- lib/artirix_data_models/aggregation_builder.rb
 - lib/artirix_data_models/aggregations_factory.rb
 - lib/artirix_data_models/cache_service.rb
 - lib/artirix_data_models/cached_action_adaptor.rb
@@ -224,6 +243,7 @@ files:
 - lib/artirix_data_models/gateway_response_adaptors/model_adaptor.rb
 - lib/artirix_data_models/gateways/data_gateway.rb
 - lib/artirix_data_models/model.rb
+- lib/artirix_data_models/raw_aggregation_data_normaliser.rb
 - lib/artirix_data_models/spec_support.rb
 - lib/artirix_data_models/spec_support/fake_mode.rb
 - lib/artirix_data_models/spec_support/gateway_mock.rb
@@ -238,11 +258,16 @@ files:
 - spec/artirix_data_models/gateways/data_gateway_spec.rb
 - spec/artirix_data_models/gateways/gateway_response_adaptors/model_adaptor_spec.rb
 - spec/artirix_data_models/model_fields_dao_spec.rb
+- spec/fixtures/articles_search_dl.json
+- spec/fixtures/articles_search_nested_raw_es.json
+- spec/fixtures/articles_search_nested_single_raw_es.json
+- spec/fixtures/articles_search_raw_es.json
 - spec/spec_helper.rb
 - spec/support/.keep
 - spec/support/a_finder_enabled_ui_model_dao.rb
 - spec/support/a_search_enabled_ui_model_dao.rb
 - spec/support/artirix_data_models.rb
+- spec/support/fixtures.rb
 homepage: ''
 licenses:
 - MIT
@@ -273,9 +298,14 @@ test_files:
 - spec/artirix_data_models/gateways/data_gateway_spec.rb
 - spec/artirix_data_models/gateways/gateway_response_adaptors/model_adaptor_spec.rb
 - spec/artirix_data_models/model_fields_dao_spec.rb
+- spec/fixtures/articles_search_dl.json
+- spec/fixtures/articles_search_nested_raw_es.json
+- spec/fixtures/articles_search_nested_single_raw_es.json
+- spec/fixtures/articles_search_raw_es.json
 - spec/spec_helper.rb
 - spec/support/.keep
 - spec/support/a_finder_enabled_ui_model_dao.rb
 - spec/support/a_search_enabled_ui_model_dao.rb
 - spec/support/artirix_data_models.rb
+- spec/support/fixtures.rb
 has_rdoc: