quandl-elasticsearch 2.1.0.rc5

data/spec/lib/quandl/elasticsearch/indice_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+require 'quandl/elasticsearch/indice'
+
+describe Quandl::Elasticsearch::Indice do
+  describe '.initialize' do
+    it 'can initialize' do
+      _indice = Quandl::Elasticsearch::Indice.new
+    end
+  end
+end

data/spec/lib/quandl/elasticsearch/query_spec.rb

@@ -0,0 +1,239 @@
+require 'spec_helper'
+
+describe Quandl::Elasticsearch::Query do
+  let(:dataset_search_filter) do
+    [
+      { term: { db_hidden: false } },
+      { term: { db_exclusive: false } }
+    ]
+  end
+  describe '.initialize' do
+    it 'can initialize' do
+      query = Quandl::Elasticsearch::Query.new
+      expect(query.class).to be(Quandl::Elasticsearch::Query)
+    end
+  end
+
+  describe '.query_body' do
+    let(:expected) do
+      {
+        explain: false,
+        fields: %w(_parent code),
+        query: {
+          bool: {
+            must: {
+              bool: {
+                should: [
+                  {
+                    multi_match: {
+                      fields: ['code', 'name^1.1'],
+                      query: 'stocks', type: 'phrase', operator: 'and',
+                      zero_terms_query: 'all', slop: 10
+                    }
+                  },
+                  {
+                    has_child: {
+                      type: 'dataset', score_mode: 'max',
+                      query: {
+                        bool: {
+                          should: [
+                            { multi_match: { fields: ['name^1.1'], query: 'stocks', type: 'phrase', slop: 10 } },
+                            { multi_match: { fields: ['code', 'name^1.1'], query: 'stocks', type: 'best_fields' } },
+                            { prefix: { code: 'stocks' } }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  { prefix: { code: 'stocks' } }
+                ]
+              }
+            },
+            filter: { bool: { must: [] } }
+          }
+        },
+        from: 1,
+        size: 10
+      }
+    end
+    it 'returns the correct JSON string' do
+      expect(Quandl::Elasticsearch::Query.query_body('stocks', 1, 10)).to eq(expected)
+    end
+  end
+
+  describe '.multi_dataset_query_body' do
+    let(:expected_query) do
+      {
+        timeout: '10s',
+        fields: %w(_parent _id code),
+        query: {
+          bool: {
+            must: {
+              bool: {
+                should: [{ multi_match: { fields: ['code', 'name^1.1'], query: 'stocks', type: 'best_fields' } },
+                         { prefix: { code: 'stocks' } }]
+              }
+            },
+            filter: {
+              bool: {
+                must: [{ term: { _parent: 1 } }, { term: { is_private: false } }]
+              }
+            }
+          }
+        },
+        sort: ['_score', { to_date: { order: 'desc' } }],
+        highlight: { fields: { name: {} } },
+        size: 10
+      }
+    end
+
+    it 'returns the correct JSON string' do
+      expect(Quandl::Elasticsearch::Query.multi_dataset_query_body(1, 'stocks', 10)).to eq(index: 'quandl_index', type: 'dataset', search: expected_query)
+    end
+  end
+
+  describe '.dataset_query_body' do
+    context 'searching using a query' do
+      let(:options) { { query: 'stocks' } }
+      let(:expected_query) do
+        {
+          fields: %w(_parent code),
+          query: {
+            bool: {
+              must: {
+                bool: {
+                  should: [
+                    { multi_match: { fields: ['name^1.1'], query: 'stocks', type: 'phrase', slop: 10 } },
+                    { multi_match: { fields: ['code', 'name^1.1'], query: 'stocks', type: 'best_fields' } },
+                    { prefix: { code: 'stocks' } }
+                  ]
+                }
+              },
+              filter: {
+                bool: {
+                  must: [{ term: { is_private: false } }] + dataset_search_filter
+                }
+              }
+            }
+          },
+          highlight: { fields: { name: {} } }, from: 1, size: 10
+        }
+      end
+
+      it 'returns the correct JSON string' do
+        expect(Quandl::Elasticsearch::Query.dataset_query_body(1, 10, options)).to eq(expected_query)
+      end
+    end
+
+    context 'searching all datasets' do
+      let(:options) { {} }
+      let(:expected_query) do
+        {
+          fields: %w(_parent code),
+          query: {
+            bool: {
+              must: { match_all: {} },
+              filter: {
+                bool: {
+                  must: [{ term: { is_private: false } }] + dataset_search_filter
+                }
+              }
+            }
+          },
+          sort: [{ to_date: { order: 'desc' } }],
+          from: 1,
+          size: 10
+        }
+      end
+
+      it 'returns the correct JSON string' do
+        expect(Quandl::Elasticsearch::Query.dataset_query_body(1, 10, options)).to eq(expected_query)
+      end
+    end
+
+    context 'searching within a database' do
+      let(:options) { { query: 'stocks', database_id: 1 } }
+      let(:expected_query) do
+        {
+          fields: %w(_parent code),
+          query: {
+            bool: {
+              must: {
+                bool: {
+                  should: [
+                    { multi_match: { fields: ['name^1.1'], query: 'stocks', type: 'phrase', slop: 10 } },
+                    { multi_match: { fields: ['code', 'name^1.1'], query: 'stocks', type: 'best_fields' } },
+                    { prefix: { code: 'stocks' } }
+                  ]
+                }
+              },
+              filter: {
+                bool: { must: [{ term: { is_private: false } }, { term: { _parent: 1 } }] }
+              }
+            }
+          },
+          highlight: { fields: { name: {} } },
+          from: 1,
+          size: 10
+        }
+      end
+
+      it 'returns the correct JSON string' do
+        expect(Quandl::Elasticsearch::Query.dataset_query_body(1, 10, options)).to eq(expected_query)
+      end
+    end
+  end
+
+  context 'filter option' do
+    let(:filter) { 'sample' }
+    let(:options) { { query: 'stocks', database_id: 1, filter: filter } }
+
+    context 'filter option is set to sample' do
+      it 'filters on sample=true datasets' do
+        query_body = Quandl::Elasticsearch::Query.dataset_query_body(1, 10, options)
+        expect(query_body[:query][:bool][:filter][:bool][:must]).to include(term: { sample: true })
+      end
+    end
+
+    context 'filter option is not set to sample' do
+      let(:filter) { 'all' }
+
+      it 'does not filter on sample=true datasets' do
+        query_body = Quandl::Elasticsearch::Query.dataset_query_body(1, 10, options)
+        expect(query_body[:query][:bool][:filter][:bool][:must]).to contain_exactly({ term: { is_private: false } }, term: { _parent: 1 })
+      end
+    end
+  end
+
+  describe '.add_timeout!' do
+    let(:body) { {} }
+
+    context 'timeout configured' do
+      context 'search key body' do
+        it 'adds timeout to body' do
+          Quandl::Elasticsearch::Query.add_timeout!(body)
+          expect(body).to eq(timeout: '10s')
+        end
+      end
+
+      context 'root body' do
+        let(:body) { { search: {} } }
+
+        it 'adds timeout to search body' do
+          Quandl::Elasticsearch::Query.add_timeout!(body)
+          expect(body).to eq(search: { timeout: '10s' })
+        end
+      end
+    end
+
+    context 'timeout is not configured' do
+      before(:each) do
+        Quandl::Elasticsearch.configuration.api_timeout = nil
+      end
+      it 'does not add timeout to body' do
+        Quandl::Elasticsearch::Query.add_timeout!(body)
+        expect(body).to eq({})
+      end
+    end
+  end
+end

data/spec/lib/quandl/elasticsearch/search_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+require 'vcr'
+
+describe Quandl::Elasticsearch::Search do
+  describe '.initialize' do
+    it 'can initialize' do
+      search = Quandl::Elasticsearch::Search.new
+      expect(search.class).to be(Quandl::Elasticsearch::Search)
+    end
+  end
+
+  describe '.page_size' do
+    it 'can return default page size' do
+      expect(subject.page_size).to be(10)
+    end
+
+    it 'can change a page size via init and return it' do
+      search = Quandl::Elasticsearch::Search.new(5)
+      expect(search.page_size).to be(5)
+    end
+  end
+
+  describe '.max_datasets' do
+    it 'can return max_datasets' do
+      expect(subject.max_datasets).to be(3)
+    end
+  end
+
+  describe '.database' do
+    it 'can process raw data from ES into appropriate format' do
+      VCR.use_cassette('search_spec_database_1') do
+        response = subject.database('stocks', '', 'free', 0, false)
+        expect(response).to eq([{ '_id' => '33', 'code' => 'NSE' }, { '_id' => '460', 'code' => 'LUXSE' }, { '_id' => '442', 'code' => 'ZAGREBSE' }, { '_id' => '462', 'code' => 'BUCHARESTSE' }, { '_id' => '5628', 'code' => 'EURONEXT' }, { '_id' => '450', 'code' => 'PHILSE' }, { '_id' => '663', 'code' => 'SI' }, { '_id' => '1383', 'code' => 'SIX' }, { '_id' => '464', 'code' => 'NIKKEI' }, { '_id' => '8401', 'code' => 'CRYPTOCHART' }, { 'total' => 130, 'took' => 752 }])
+      end
+      VCR.use_cassette('search_spec_database_2') do
+        response = subject.database('stocks', '', 'premium', 0, false)
+        expect(response).to eq([{ '_id' => '12910', 'code' => 'EOD' }, { '_id' => '13091', 'code' => 'ZFA' }, { '_id' => '13122', 'code' => 'ZFB' }, { '_id' => '13065', 'code' => 'ZSEE' }, { '_id' => '12266', 'code' => 'ZEE' }, { '_id' => '13000', 'code' => 'ZET' }, { '_id' => '12985', 'code' => 'ZAR' }, { '_id' => '12908', 'code' => 'SF1' }, { '_id' => '12973', 'code' => 'ZDIV' }, { '_id' => '12991', 'code' => 'ZSS' }, { 'total' => 18, 'took' => 769 }])
+      end
+    end
+
+    it 'adds timeout to query body' do
+      VCR.use_cassette('search_spec_database_1') do
+        expect(Quandl::Elasticsearch::Query).to receive(:add_timeout!).and_call_original
+        subject.database('stocks', '', 'premium', 0, false)
+      end
+    end
+
+    context 'timeout' do
+      it 'raises timeout error' do
+        expect_any_instance_of(::Elasticsearch::Transport::Client).to receive(:search).and_return('timed_out' => true)
+        expect { subject.database('stocks', '', 'premium', 0, false) }.to raise_error(Quandl::Elasticsearch::Search::TimeoutError)
+      end
+    end
+  end
+
+  describe '.dataset' do
+    it 'can process raw data returned from ES into appropriate format' do
+      VCR.use_cassette('search_spec_dataset_1') do
+        response = subject.dataset('stocks', 'daily', 0, 10, database_id: 104_71)
+        expect(response).to eq([{ '_id' => '13147174', 'highlight' => { 'name' => ['OMX Iceland All-<em>Share</em> GI (OMXIGI)'] } }, { '_id' => '13147176', 'highlight' => { 'name' => ['OMX Iceland All-<em>Share</em> PI (OMXIPI)'] } }, { '_id' => '13147219', 'highlight' => { 'name' => ['OMX Stockholm All-<em>Share</em> Cap_GI (OMXSCAPGI)'] } }, { '_id' => '13147220', 'highlight' => { 'name' => ['OMX Stockholm All-<em>Share</em> Cap_PI (OMXSCAPPI)'] } }, { '_id' => '13147404', 'highlight' => { 'name' => ['OMXI-FO All <em>Share</em> PI (OMXIFOPI)'] } }, { '_id' => '13147405', 'highlight' => { 'name' => ['OMXI-FO All <em>Share</em> GI (OMXIFOGI)'] } }, { '_id' => '13127717', 'highlight' => { 'name' => ['NASDAQ Select Canadian Preferred <em>Share</em> Index TR (NQCAPFDT)'] } }, { '_id' => '13127718', 'highlight' => { 'name' => ['NASDAQ Select Canadian Preferred <em>Share</em> Index (NQCAPFD)'] } }, { '_id' => '13961773', 'highlight' => { 'name' => ['NASDAQ Bullet<em>Shares</em> Corporate Debt 0-3 Ladder (BSCP03)'] } }, { '_id' => '13961774', 'highlight' => { 'name' => ['NASDAQ Bullet<em>Shares</em> High Yield Debt 0-3 Ladder TR (BSHY03T)'] } }, { 'total' => 30 }])
+      end
+
+      VCR.use_cassette('search_spec_dataset_2') do
+        response = subject.dataset('mexico', nil, 0, 3, database_id: 104_71)
+        expect(response).to eq([{ '_id' => '13124809', 'highlight' => { 'name' => ['NASDAQ <em>Mexico</em> Basic Matls AUD Index (NQMX1000AUD)'] } }, { '_id' => '13124810', 'highlight' => { 'name' => ['NASDAQ <em>Mexico</em> AUD Index (NQMXAUD)'] } }, { '_id' => '13124813', 'highlight' => { 'name' => ['NASDAQ <em>Mexico</em> AUD NTR Index (NQMXAUDN)'] } }, { 'total' => 315 }])
+      end
+    end
+
+    it 'adds timout to query body' do
+      VCR.use_cassette('search_spec_dataset_1') do
+        expect(Quandl::Elasticsearch::Query).to receive(:add_timeout!).and_call_original
+        subject.dataset('stocks', 'daily', 0, 10, database_id: 104_71)
+      end
+    end
+
+    context 'timeout' do
+      it 'raises timeout error' do
+        expect_any_instance_of(::Elasticsearch::Transport::Client).to receive(:search).and_return('timed_out' => true)
+        expect { subject.dataset('stocks', 'daily', 0, 10, database_id: 104_71) }.to raise_error(Quandl::Elasticsearch::Search::TimeoutError)
+      end
+    end
+  end
+end

data/spec/lib/quandl/elasticsearch/template_spec.rb

@@ -0,0 +1,182 @@
+require 'spec_helper'
+require 'quandl/elasticsearch/indice'
+require 'active_support/all'
+
+describe Quandl::Elasticsearch::Template do
+  describe '.initialize' do
+    it 'can initialize' do
+      template = Quandl::Elasticsearch::Template.new
+      expect(template.class).to be(Quandl::Elasticsearch::Template)
+    end
+  end
+
+  describe '.main_index' do
+    it 'can return the correct string' do
+      expect(Quandl::Elasticsearch::Template.main_index.deep_stringify_keys).to match(
+        {
+          settings: {
+            index: {
+              number_of_shards: 1, number_of_replicas: 0
+            },
+            similarity: {
+              quandl_bm25: {
+                type: 'BM25', k1: 0, b: 0.75
+              }
+            },
+            analysis: {
+              analyzer: {
+                keyword_lowercase: {
+                  type: 'english', tokenizer: 'keyword', filter: 'lowercase', stopwords: '_none_'
+                },
+                quandl_index: {
+                  type: 'custom', tokenizer: 'whitespace', filter: %w(quandl_delimiter lowercase quandl_synonym_filter quandl_stop_words quandl_stemmer trim), char_filter: ['quandl_char_filter']
+                },
+                quandl_search: {
+                  type: 'custom', tokenizer: 'whitespace', filter: %w(quandl_delimiter lowercase quandl_stop_words quandl_stemmer trim), char_filter: ['quandl_char_filter']
+                }
+              },
+              filter: {
+                quandl_synonym_filter: {
+                  type: 'synonym',
+                  synonyms: be_an(Array)
+                },
+                quandl_stemmer: {
+                  type: 'stemmer',
+                  name: 'english'
+                },
+                quandl_stop_words: {
+                  type: 'stop',
+                  ignore_case: true,
+                  stopwords: be_an(Array)
+                },
+                quandl_delimiter: {
+                  type: 'word_delimiter',
+                  split_on_case_change: true,
+                  catenate_numbers: true, preserve_original: true, generate_word_parts: true, split_on_numbers: true, stem_english_prossessive: true, type_table: ['& => ALPHA', '_ => ALPHA', '$ => DIGIT', '% => DIGIT'], protected_word: ['u.s.', 'u.s.a.']
+                }
+              },
+              char_filter: {
+                quandl_char_filter: {
+                  type: 'mapping', mappings: ['\\u0060=>\\u0020', '\\u002F=>\\u0020', '\\u003A=>\\u0020', '\\u002D=>\\u0020', '\\u0028=>\\u0020', '\\u0029=>\\u0020', '\\u002C=>\\u0020', '\\u003B=>\\u0020']
+                }
+              }
+            }
+          },
+          mappings: {
+            database: {
+              properties: {
+                code: {
+                  type: 'string', store: true, norms: {
+                    enabled: false
+                  },
+                  analyzer: 'keyword_lowercase'
+                },
+                name: {
+                  type: 'string', store: true, norms: {
+                    enabled: true
+                  },
+                  analyzer: 'quandl_index', search_analyzer: 'quandl_search'
+                },
+                description: {
+                  type: 'string', store: true, norms: {
+                    enabled: true
+                  },
+                  analyzer: 'quandl_index'
+                },
+                documentation: {
+                  type: 'string', store: true, norms: {
+                    enabled: true
+                  },
+                  analyzer: 'quandl_index'
+                },
+                premium: {
+                  type: 'boolean', store: true, norms: {
+                    enabled: false
+                  }
+                },
+                hidden: {
+                  type: 'boolean', store: true, norms: {
+                    enabled: false
+                  }
+                },
+                exclusive: {
+                  type: 'boolean', store: true, norms: {
+                    enabled: false
+                  }
+                },
+                type: {
+                  type: 'string', store: true, norms: {
+                    enabled: false
+                  },
+                  index: 'not_analyzed'
+                },
+                rating: {
+                  type: 'integer', store: true, norms: {
+                    enabled: false
+                  }, index: 'not_analyzed'
+                },
+                tags: {
+                  type: 'string', store: true, norms: {
+                    enabled: false
+                  },
+                  index: 'not_analyzed'
+                }
+              }
+            },
+            dataset: {
+              _parent: {
+                type: 'database'
+              },
+              properties: {
+                code: {
+                  type: 'string', store: true, norms: {
+                    enabled: false
+                  },
+                  analyzer: 'keyword_lowercase'
+                },
+                name: {
+                  type: 'string', store: true, norms: {
+                    enabled: true
+                  },
+                  analyzer: 'quandl_index', search_analyzer: 'quandl_search'
+                },
+                is_private: {
+                  type: 'boolean', norms: {
+                    enabled: false
+                  },
+                  store: false
+                },
+                type: {
+                  type: 'string', store: true, norms: {
+                    enabled: false
+                  },
+                  index: 'not_analyzed'
+                },
+                frequency: {
+                  type: 'string', store: false, norms: {
+                    enabled: false
+                  },
+                  index: 'not_analyzed'
+                },
+                to_date: {
+                  type: 'date', format: 'date', store: false, norms: {
+                    enabled: false
+                  }
+                },
+                db_hidden: {
+                  type: 'boolean', norms: { enabled: false }, store: false
+                },
+                db_exclusive: {
+                  type: 'boolean', norms: { enabled: false }, store: false
+                },
+                sample: {
+                  type: 'boolean', norms: { enabled: false }, store: false
+                }
+              }
+            }
+          }
+        }.deep_stringify_keys
+      )
+    end
+  end
+end