quandl-elasticsearch 2.1.0.rc5

data/lib/quandl/elasticsearch/template.rb

@@ -0,0 +1,189 @@
+require_relative 'version'
+
+module Quandl
+  module Elasticsearch
+    class Template
+      class << self
+        def gem_path
+          Pathname.new(Gem::Specification.find_by_name('quandl-elasticsearch', Quandl::Elasticsearch::VERSION).gem_dir)
+        end
+
+        def main_index
+          { settings: { index: index_settings,
+                        similarity: { quandl_bm25: quandl_bm25 },
+                        analysis: { analyzer: analyzer, filter: filter, char_filter: char_filter } },
+            mappings: { database: { properties: database_mapping },
+                        dataset: { _parent: { type: 'database' }, properties: dataset_mapping } } }
+        end
+
+        private
+
+        def analyzer
+          {
+            keyword_lowercase: keyword_lowercase_analyzer,
+            quandl_index: quandl_text_analyzer,
+            quandl_search: quandl_search_analyzer
+          }
+        end
+
+        def filter
+          {
+            quandl_synonym_filter: quandl_synonym_filter,
+            quandl_stemmer: quandl_stemmer,
+            quandl_stop_words: quandl_stop_words,
+            quandl_delimiter: quandl_delimiter
+          }
+        end
+
+        def char_filter
+          {
+            quandl_char_filter: quandl_char_filter
+          }
+        end
+
+        def quandl_text_analyzer
+          {
+            type: 'custom',
+            tokenizer: 'whitespace',
+            filter: %w(quandl_delimiter lowercase quandl_synonym_filter quandl_stop_words quandl_stemmer trim),
+            char_filter: %w(quandl_char_filter)
+          }
+        end
+
+        def quandl_search_analyzer
+          ## without quandl_synonym_filte
+          {
+            type: 'custom',
+            tokenizer: 'whitespace',
+            filter: %w(quandl_delimiter lowercase quandl_stop_words quandl_stemmer trim),
+            char_filter: %w(quandl_char_filter)
+          }
+        end
+
+        def keyword_lowercase_analyzer
+          {
+            type: 'english',
+            tokenizer: 'keyword',
+            filter: 'lowercase',
+            stopwords: '_none_'
+          }
+        end
+
+        def quandl_synonym_filter
+          synonyms = []
+          File.read(gem_path.join('elasticsearch/synonyms/synonyms_english.txt')).each_line do |line|
+            clean_line = line.strip
+            next if clean_line[0] == '#' || clean_line =~ /^\s+$/
+            synonyms << clean_line
+          end
+
+          {
+            type: 'synonym',
+            synonyms: synonyms
+          }
+        end
+
+        def quandl_char_filter
+          # unicode table
+          {
+            type: 'mapping',
+            mappings: [
+              '\\u0060=>\\u0020', # `
+              '\\u002F=>\\u0020', # /
+              '\\u003A=>\\u0020', # :
+              '\\u002D=>\\u0020', # -
+              '\\u0028=>\\u0020', # (
+              '\\u0029=>\\u0020', # )
+              '\\u002C=>\\u0020', # ,
+              '\\u003B=>\\u0020'  # ;
+            ]
+          }
+        end
+
+        def quandl_stemmer
+          {
+            type: 'stemmer',
+            name: 'english'
+          }
+        end
+
+        def quandl_stop_words
+          stop_words = []
+          File.read(gem_path.join('elasticsearch/stopwords/english.txt')).each_line do |line|
+            clean_line = line.strip
+            next if clean_line[0] == '#' || clean_line =~ /^\s+$/
+            stop_words << clean_line
+          end
+
+          {
+            type: 'stop',
+            ignore_case: true,
+            stopwords: stop_words
+          }
+        end
+
+        def quandl_delimiter
+          {
+            type: 'word_delimiter',
+            split_on_case_change: true,
+            catenate_numbers: true,
+            preserve_original: true,
+            generate_word_parts: true,
+            split_on_numbers: true,
+            stem_english_prossessive: true,
+            type_table: ['& => ALPHA',
+                         '_ => ALPHA',
+                         '$ => DIGIT',
+                         '% => DIGIT'],
+            protected_word: ['u.s.', 'u.s.a.']
+          }
+        end
+
+        def database_mapping
+          {
+            code: { type: 'string', store: true, norms: { enabled: false }, analyzer: 'keyword_lowercase' },
+            name: { type: 'string', store: true, norms: { enabled: true }, analyzer: 'quandl_index', search_analyzer: 'quandl_search' },
+            updated_at: { type: 'date' },
+            description: { type: 'string', store: true, norms: { enabled: true }, analyzer: 'quandl_index' },
+            documentation: { type: 'string', store: true, norms: { enabled: true }, analyzer: 'quandl_index' },
+            premium: { type: 'boolean', store: true, norms: { enabled: false } },
+            hidden: { type: 'boolean', store: true, norms: { enabled: false } },
+            exclusive: { type: 'boolean', store: true, norms: { enabled: false } },
+            type: { type: 'string', store: true, norms: { enabled: false }, index: 'not_analyzed' },
+            rating: { type: 'integer', store: true, norms: { enabled: false }, index: 'not_analyzed' },
+            tags: { type: 'string', store: true, norms: { enabled: false }, index: 'not_analyzed' }
+          }
+        end
+
+        def dataset_mapping
+          {
+            code: { type: 'string', store: true, norms: { enabled: false }, analyzer: 'keyword_lowercase' },
+            name: { type: 'string', store: true, norms: { enabled: true }, analyzer: 'quandl_index', search_analyzer: 'quandl_search' },
+            is_private: { type: 'boolean', norms: { enabled: false }, store: false },
+            type: { type: 'string', store: true, norms: { enabled: false }, index: 'not_analyzed' },
+            frequency: { type: 'string', store: false, norms: { enabled: false }, index: 'not_analyzed' },
+            to_date: { type: 'date', format: 'date', store: false, norms: { enabled: false } },
+            db_hidden: { type: 'boolean', norms: { enabled: false }, store: false },
+            db_exclusive: { type: 'boolean', norms: { enabled: false }, store: false },
+            sample: { type: 'boolean', norms: { enabled: false }, store: false }
+          }
+        end
+
+        def index_settings
+          {
+            number_of_shards: 1,
+            number_of_replicas: 0
+          }
+        end
+
+        def quandl_bm25
+          {
+            type: 'BM25',
+            k1: 0,
+            b: 0.75
+          }
+        end
+      end
+    end
+  end
+end

data/lib/quandl/elasticsearch/utility.rb

@@ -0,0 +1,6 @@
+module Quandl
+  module Elasticsearch
+    class Utility
+    end
+  end
+end

data/lib/quandl/elasticsearch/version.rb

@@ -0,0 +1,6 @@
+module Quandl
+  ## Elasticsearch
+  module Elasticsearch
+    VERSION = '2.1.0.rc5'.freeze
+  end
+end

data/quandl

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'commander/import'
+require './tests/search.rb'
+
+program :version, '0.0.1'
+program :description, 'quandl-elasticsearch command line tools'
+default_command :search
+
+command :search do |c|
+  c.syntax = 'quandl_es search [options]'
+  c.summary = 'search v3 on staging'
+  c.description = c.summary
+  c.example 'search on query strings', './quand_es search -q "bank"'
+  c.example 'search based on query strings', './quandl_es search --query "natural gas future"'
+  c.example 'show search results for page 3.', './quandl_es search --q "bank" -p 3'
+  c.example 'show search results for page 2 and each page shows 25 results.', './quandl_es search --q "oil" -p 2 -s 25'
+  c.example 'serch for databases with term aaple and with tags USA and STOCKS', './quandl_es search --q "aapl" -g USA,STOCKS'
+
+  c.option '-q', '--query "query strings"', String, 'query strings'
+  c.option '-g', '--tags "tags list"', String, 'tag list'
+  c.option '-p', '--page PAGE_NUMBER', 'search results page number', Integer, 'page number'
+  c.option '-s', '--size PER_PAGE', 'search results per page', Integer, 'per page result'
+  c.option '-m', '--premium', String, 'only premium databases'
+  c.option '-f', '--free', String, 'only free databases'
+  c.option '-w', '--with-dataset', 'show dadatasets', String, 'show dataset'
+  c.action do |_args, options|
+    page = options.page ? options.page.to_i : 0
+    tags = options.tags ? options.tags.to_s : ''
+
+    f = options.free ? true : false
+    p = options.premium ? true : false
+
+    premium = nil
+    premium = false if f
+    premium = true if p
+
+    unless tags.empty?
+      tags.split(',').each do |t|
+        fail 'tags must be one of tag list' unless tag_list.include?(t)
+      end
+    end
+    if options.query
+      result = search(options.query, tags, options.with_dataset, premium, page)
+      db_codes = result.map { |s| s['code'] }.reject(&:nil?)
+      r = { 'databases' => result }
+      puts JSON.pretty_generate(r)
+      puts "#{db_codes.count} hit databases code: #{db_codes.join(',')}"
+    else
+      system('./quandl_es search --help')
+    end
+  end
+end
+
+command :dataset do |c|
+  c.option '-i', '--id DATABASE_id', String, 'database id'
+  c.option '-q', '--query SEARCH_QUERY', String, 'search query'
+  c.option '-p', '--page PAGE_NUMBER', 'search results page number', Integer, 'page number'
+  c.option '-s', '--size PER_PAGE', 'search results per page', Integer, 'per page result'
+  c.option '-f', '--frequency FREQUENCY_LIST', String, 'frequency list'
+  frequency_list = %w(daily weekly monthly "quarterly annual)
+
+  c.action do |_args, options|
+    frequency = options.frequency ? options.frequency : nil
+    page = options.page ? options.page.to_i : 0
+    # size = options.size ? options.size.to_i : 10
+    unless frequency.nil?
+      frequency.split(',').each do |d|
+        fail "frequency should be in #{frequency_list}" unless frequency_list.include?(d)
+      end
+    end
+    query = options.query && options.query.strip != '' ? options.query : nil
+    r = dataset_search(options.id, query, frequency, page)
+    puts JSON.pretty_generate(r)
+  end
+end

data/quandl-elasticsearch.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'quandl/elasticsearch/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'quandl-elasticsearch'
+  spec.version       = Quandl::Elasticsearch::VERSION
+  spec.authors       = ['Jun Li']
+  spec.email         = ['jun@quandl.com']
+  spec.description   = 'Elasticsearch interface for Quandl'
+  spec.summary       = 'Elasticsearch interface for Quandl'
+  spec.homepage      = 'https://github.com/quandl/quandl-elasticsearch'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'elasticsearch', '~> 1.0.17'
+  spec.add_runtime_dependency 'commander', '~> 4.3'
+  spec.add_runtime_dependency 'json', '~> 1.8.2'
+  spec.add_runtime_dependency 'quandl-config', '> 0.0.3', '< 2.0'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rubocopter'
+  spec.add_development_dependency 'activesupport'
+  spec.add_development_dependency 'vcr'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'webmock'
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+end

data/solano.yml

@@ -0,0 +1,20 @@
+environment:
+  'QUANDL_PROJECT': 'Elasticsearch Gem'
+  'SLACK_WEBHOOK': 'https://hooks.slack.com/services/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'
+  'TZ': 'UTC'
+tool_config:
+  gc: '4.6'
+ruby_version: 2.2.2
+bundler_version: 1.10.5
+rake:
+  rails_env: 'test'
+solr: false
+rabbitmq: false
+redis: false
+tests:
+    - bundle exec rubocopter --commit origin/master
+test_pattern:
+  - spec/**_spec.rb
+coverage: true
+hooks:
+  post_build: 'gem install quandl_utility && qutil ci notify_slack'

data/spec/lib/quandl/elasticsearch/database_spec.rb

@@ -0,0 +1,98 @@
+require 'spec_helper'
+require 'quandl/elasticsearch/indice'
+
+describe Quandl::Elasticsearch::Database do
+  describe '.initialize' do
+    it 'can return the correct object' do
+      record = { 'id' => 4 }
+      database = Quandl::Elasticsearch::Database.new(record)
+      expect(database).to be_a_kind_of(Quandl::Elasticsearch::Database)
+    end
+  end
+
+  describe '.id' do
+    it 'can return the correct id' do
+      record = { 'id' => 4 }
+      database = Quandl::Elasticsearch::Database.new(record)
+      expect(database.id).to eq(4)
+    end
+  end
+
+  describe '.data' do
+    it 'can return the correct data and does not flag as remove' do
+      record = { 'id' => 4, 'name' => 'Name', 'description' => 'Description',
+                 'documentation' => 'Documentation', 'premium' => true, 'rating' => 5,
+                 'deleted_at' => nil, 'type' => nil, 'hidden' => 'f',
+                 'elasticsearch_flags' => 'USA,STOCKS', 'exclusive' => true }
+      database = Quandl::Elasticsearch::Database.new(record)
+      expect(database.remove?).to eq(false)
+      expect(database.data).to eq(name: 'Name', code: nil, description: 'Description', documentation: 'Documentation',
+                                  premium: true, hidden: false, tags: %w(USA STOCKS), exclusive: true)
+    end
+  end
+
+  describe '.remove' do
+    it 'can not remove correctly if it is hidden' do
+      record = { 'hidden' => 't', 'deleted_at' => nil, 'rating' => 1, 'type' => nil }
+      database1 = Quandl::Elasticsearch::Database.new(record)
+      expect(database1.remove?).to eq(false)
+      record = { 'hidden' => 'f', 'deleted_at' => nil, 'rating' => 1, 'type' => nil }
+      database2 = Quandl::Elasticsearch::Database.new(record)
+      expect(database2.remove?).to eq(false)
+    end
+
+    it 'can remove correctly if it is deleted_at' do
+      record = { 'deleted_at' => nil, 'rating' => 5, 'type' => nil }
+      database1 = Quandl::Elasticsearch::Database.new(record)
+      expect(database1.remove?).to eq(false)
+      record = { 'deleted_at' => '2015-01-01', 'rating' => 5, 'type' => nil }
+      database2 = Quandl::Elasticsearch::Database.new(record)
+      expect(database2.remove?).to eq(true)
+    end
+
+    it 'can not remove correctly if it is rated zero or lower' do
+      record = { 'deleted_at' => nil, 'hidden' => 'f', 'rating' => 1, 'type' => nil }
+      database1 = Quandl::Elasticsearch::Database.new(record)
+      expect(database1.remove?).to eq(false)
+      record = { 'deleted_at' => nil, 'hidden' => 'f', 'rating' => 0, 'type' => nil }
+      database2 = Quandl::Elasticsearch::Database.new(record)
+      expect(database2.remove?).to eq(false)
+      record = { 'rating' => -10, 'deleted_at' => nil, 'hidden' => 'f', 'type' => nil }
+      database3 = Quandl::Elasticsearch::Database.new(record)
+      expect(database3.remove?).to eq(false)
+      record = { 'rating' => 4, 'deleted_at' => nil, 'hidden' => 'f', 'type' => nil }
+      database4 = Quandl::Elasticsearch::Database.new(record)
+      expect(database4.remove?).to eq(false)
+    end
+
+    it 'can remove correctly if it is not a Source' do
+      record = { 'type' => 'UserSource', 'rating' => 1 }
+      database1 = Quandl::Elasticsearch::Database.new(record)
+      expect(database1.remove?).to eq(true)
+      record = { 'type' => nil, 'rating' => 1 }
+      database2 = Quandl::Elasticsearch::Database.new(record)
+      expect(database2.remove?).to eq(false)
+    end
+
+    it 'can remove correctly if rating <= 0' do
+      record = { 'rating' => 1  }
+      database1 = Quandl::Elasticsearch::Database.new(record)
+      expect(database1.remove?).to eq(false)
+      record = { 'rating' => nil }
+      database2 = Quandl::Elasticsearch::Database.new(record)
+      expect(database2.remove?).to eq(false)
+      record = { 'rating' => 0 }
+      database3 = Quandl::Elasticsearch::Database.new(record)
+      expect(database3.remove?).to eq(false)
+    end
+
+    it 'can not remove correctly if it is hidden' do
+      record = { 'hidden' => 't' }
+      database1 = Quandl::Elasticsearch::Database.new(record)
+      expect(database1.remove?).to eq(false)
+      record = { 'hidden' => 'f', 'deleted_at' => nil, 'rating' => 1, 'type' => nil }
+      database2 = Quandl::Elasticsearch::Database.new(record)
+      expect(database2.remove?).to eq(false)
+    end
+  end
+end

data/spec/lib/quandl/elasticsearch/dataset_spec.rb

@@ -0,0 +1,124 @@
+require 'spec_helper'
+require 'quandl/elasticsearch/indice'
+
+describe Quandl::Elasticsearch::Dataset do
+  describe '.initialize' do
+    it 'can return the correct object' do
+      record = { 'id' => 9, 'hidden' => 'f', 'deleted_at' => nil, 'rating' => 1, 'type' => nil }
+      dataset = Quandl::Elasticsearch::Dataset.new(record)
+      expect(dataset).to be_a_kind_of(Quandl::Elasticsearch::Dataset)
+    end
+  end
+
+  describe '.id' do
+    it 'can return the correct value' do
+      record = { 'id' => 9, 'type' => nil, 'deleted_at' => nil, 'state' => 'published' }
+      dataset = Quandl::Elasticsearch::Dataset.new(record)
+      expect(dataset.id).to eq(9)
+      expect(dataset.remove?).to eq(false)
+    end
+  end
+
+  describe '.database_id' do
+    it 'can return the correct value' do
+      record = { 'id' => 9, 'database_id' => 598_393 }
+      dataset = Quandl::Elasticsearch::Dataset.new(record)
+      expect(dataset.database_id).to eq(598_393)
+    end
+  end
+
+  describe '.data' do
+    it 'can return the correct value and does not flag as remove' do
+      # {:code=>nil, :name=>nil, :to_date=>nil, :frequency=>nil}
+      record = { 'id' => 9, 'deleted_at' => nil, 'type' => nil, 'state' => 'published',
+                 'code' => 'Code', 'name' => 'Name', 'to_date' => '12122015',
+                 'is_private' => false, 'frequency' => 'daily', 'db_hidden' => true,
+                 'db_exclusive' => true, 'db_premium' => true, 'db_preview_enabled' => true,
+                 'db_preview_datasets' => '' }
+      dataset = Quandl::Elasticsearch::Dataset.new(record)
+      expect(dataset.data).to eq(code: 'Code', name: 'Name', to_date: '12122015',
+                                 is_private: false, frequency: 'daily',
+                                 db_hidden: true, db_exclusive: true, sample: true)
+      expect(dataset.remove?).to eq(false)
+    end
+  end
+
+  describe '.remove' do
+    it 'can remove correctly if it is deleted_at' do
+      record = { 'type' => nil, 'deleted_at' => nil, 'is_private' => 'f' }
+      dataset1 = Quandl::Elasticsearch::Dataset.new(record)
+      expect(dataset1.remove?).to eq(false)
+      record = { 'deleted_at' => '2015-01-01' }
+      dataset2 = Quandl::Elasticsearch::Dataset.new(record)
+      expect(dataset2.remove?).to eq(true)
+    end
+
+    it 'can remove correctly if it is not a dataset' do
+      record = { 'type' => 'Superset' }
+      dataset1 = Quandl::Elasticsearch::Dataset.new(record)
+      expect(dataset1.remove?).to eq(true)
+      record = { 'type' => nil, 'deleted_at' => nil, 'is_private' => 'f' }
+      dataset2 = Quandl::Elasticsearch::Dataset.new(record)
+      expect(dataset2.remove?).to eq(false)
+    end
+  end
+
+  describe 'sample' do
+    subject { Quandl::Elasticsearch::Dataset.new(record).data }
+
+    let(:db_preview_datasets) { 'BBB,AAA,CCC' }
+    let(:db_preview_enabled) { 't' }
+    let(:db_premium) { 't' }
+    let(:code) { 'AAA' }
+
+    let(:record) do
+      {
+        'db_preview_datasets' => db_preview_datasets,
+        'db_preview_enabled' => db_preview_enabled,
+        'db_premium' => db_premium,
+        'code' => code
+      }
+    end
+
+    context 'premium' do
+      context 'code is in preview datasets list' do
+        context 'previews not enabled' do
+          let(:db_preview_enabled) { 'f' }
+
+          it 'is not previewable' do
+            expect(subject[:sample]).to eq(false)
+          end
+        end
+
+        context 'previews enabled' do
+          it 'is previewable' do
+            expect(subject[:sample]).to eq(true)
+          end
+        end
+      end
+
+      context 'code is not in preview datasets list' do
+        let(:db_preview_datasets) { 'BBB,AA,CCC' }
+
+        it 'is not previewable' do
+          expect(subject[:sample]).to eq(false)
+        end
+      end
+
+      context 'preview datasets list is empty' do
+        let(:db_preview_datasets) { '' }
+
+        it 'is previewable since empty means all datasets are whitelisted' do
+          expect(subject[:sample]).to eq(true)
+        end
+      end
+    end
+
+    context 'free' do
+      let(:db_premium) { 'f' }
+      it 'is not previewable' do
+        expect(subject[:sample]).to eq(false)
+      end
+    end
+  end
+end