quandl-elasticsearch 2.1.0.rc5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +20 -0
- data/.rubocop.yml +34 -0
- data/COMMANDS.md +29 -0
- data/Gemfile +10 -0
- data/Gemfile.lock +155 -0
- data/LICENSE.txt +22 -0
- data/README.md +50 -0
- data/Rakefile +1 -0
- data/config/elasticsearch.yml +32 -0
- data/elasticsearch/elasticsearch.yml +386 -0
- data/elasticsearch/logging.yml +56 -0
- data/elasticsearch/stopwords/english.txt +38 -0
- data/elasticsearch/synonyms/synonyms_english.txt +318 -0
- data/fixtures/vcr_cassettes/search_spec_database_1.yml +38 -0
- data/fixtures/vcr_cassettes/search_spec_database_2.yml +38 -0
- data/fixtures/vcr_cassettes/search_spec_dataset_1.yml +48 -0
- data/fixtures/vcr_cassettes/search_spec_dataset_2.yml +41 -0
- data/fixtures/vcr_cassettes/setup.yml +139 -0
- data/lib/quandl/elasticsearch.rb +61 -0
- data/lib/quandl/elasticsearch/base.rb +20 -0
- data/lib/quandl/elasticsearch/database.rb +22 -0
- data/lib/quandl/elasticsearch/dataset.rb +51 -0
- data/lib/quandl/elasticsearch/indice.rb +96 -0
- data/lib/quandl/elasticsearch/query.rb +282 -0
- data/lib/quandl/elasticsearch/search.rb +150 -0
- data/lib/quandl/elasticsearch/tag.rb +21 -0
- data/lib/quandl/elasticsearch/template.rb +189 -0
- data/lib/quandl/elasticsearch/utility.rb +6 -0
- data/lib/quandl/elasticsearch/version.rb +6 -0
- data/quandl +77 -0
- data/quandl-elasticsearch.gemspec +34 -0
- data/solano.yml +20 -0
- data/spec/lib/quandl/elasticsearch/database_spec.rb +98 -0
- data/spec/lib/quandl/elasticsearch/dataset_spec.rb +124 -0
- data/spec/lib/quandl/elasticsearch/indice_spec.rb +10 -0
- data/spec/lib/quandl/elasticsearch/query_spec.rb +239 -0
- data/spec/lib/quandl/elasticsearch/search_spec.rb +83 -0
- data/spec/lib/quandl/elasticsearch/template_spec.rb +182 -0
- data/spec/lib/quandl/elasticsearch/utility_spec.rb +10 -0
- data/spec/lib/quandl/elasticsearch_spec.rb +99 -0
- data/spec/spec_helper.rb +27 -0
- data/templates/database_mapping.json +11 -0
- data/templates/dataset_mapping.json +9 -0
- data/templates/quandl_delimiter.json +0 -0
- data/templates/search_term_mapping.json +13 -0
- data/tests/Database-Ratings.csv +405 -0
- data/tests/Database-Tags.csv +341 -0
- data/tests/compare.csv +1431 -0
- data/tests/compare.rb +33 -0
- data/tests/console.rb +4 -0
- data/tests/generated_db_tags.csv +341 -0
- data/tests/search.rb +14 -0
- data/tests/search_db_mapping.txt +402 -0
- data/tests/status.rb +2 -0
- data/tests/test_search.csv +87 -0
- data/tests/test_search.rb +113 -0
- data/tests/testing-list.txt +183 -0
- data/tests/top500searches.csv +477 -0
- metadata +300 -0
@@ -0,0 +1,38 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://staging-search.quandl.com:9200//quandl_index/database/_search
|
6
|
+
body:
|
7
|
+
encoding: UTF-8
|
8
|
+
string: ! '{"explain":false,"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1","description","documentation^0.8"],"query":"stocks","type":"phrase","operator":"and","zero_terms_query":"all","slop":2}},{"has_child":{"type":"dataset","score_mode":"max","query":{"bool":{"should":[{"multi_match":{"fields":["name^1.1"],"query":"stocks","type":"phrase","slop":10}},{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}}}},{"prefix":{"code":"stocks"}}]}},"filter":{"bool":{"must":[{"term":{"premium":true}}]}}}},"from":0,"size":10,"rescore":{"window_size":10,"query":{"rescore_query":{"match":{"name":{"query":"stocks","type":"phrase","slop":10}}},"query_weight":0.7,"rescore_query_weight":1.2}}}'
|
9
|
+
headers:
|
10
|
+
User-Agent:
|
11
|
+
- Faraday v0.9.1
|
12
|
+
Accept:
|
13
|
+
- ! '*/*'
|
14
|
+
Date:
|
15
|
+
- Fri, 22 May 2015 16:05:37 GMT
|
16
|
+
Content-Type:
|
17
|
+
- application/x-www-form-urlencoded
|
18
|
+
response:
|
19
|
+
status:
|
20
|
+
code: 200
|
21
|
+
message: !binary |-
|
22
|
+
T0s=
|
23
|
+
headers:
|
24
|
+
!binary "Q29udGVudC1UeXBl":
|
25
|
+
- !binary |-
|
26
|
+
YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
|
27
|
+
!binary "Q29udGVudC1MZW5ndGg=":
|
28
|
+
- !binary |-
|
29
|
+
MTE5Mw==
|
30
|
+
!binary "Q29ubmVjdGlvbg==":
|
31
|
+
- !binary |-
|
32
|
+
a2VlcC1hbGl2ZQ==
|
33
|
+
body:
|
34
|
+
encoding: US-ASCII
|
35
|
+
string: ! '{"took":769,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":18,"max_score":2.0353467,"hits":[{"_index":"quandl_main_v6","_type":"database","_id":"12910","_score":2.7696624,"fields":{"code":["EOD"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13091","_score":1.0613412,"fields":{"code":["ZFA"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13122","_score":1.0613412,"fields":{"code":["ZFB"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13065","_score":0.5353001,"fields":{"code":["ZSEE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12266","_score":0.504785,"fields":{"code":["ZEE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13000","_score":0.49280426,"fields":{"code":["ZET"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12985","_score":0.461574,"fields":{"code":["ZAR"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12908","_score":0.46129575,"fields":{"code":["SF1"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12973","_score":0.43754363,"fields":{"code":["ZDIV"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12991","_score":0.37978202,"fields":{"code":["ZSS"]}}]}}'
|
36
|
+
http_version:
|
37
|
+
recorded_at: Fri, 22 May 2015 16:05:38 GMT
|
38
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1,48 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://staging-search.quandl.com:9200//quandl_index/dataset/_search
|
6
|
+
body:
|
7
|
+
encoding: UTF-8
|
8
|
+
string: ! '{"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","minimum_should_match":1,"type":"best_fields","operator":"and"}},{"prefix":{"code":"stocks"}}]}},"filter":{"bool":{"must":[{"term":{"_parent":10471}}],"should":[{"term":{"frequency":"daily"}}]}}}},"sort":[{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"from":0,"size":10}'
|
9
|
+
headers:
|
10
|
+
User-Agent:
|
11
|
+
- Faraday v0.9.1
|
12
|
+
Accept:
|
13
|
+
- ! '*/*'
|
14
|
+
Date:
|
15
|
+
- Fri, 22 May 2015 15:57:50 GMT
|
16
|
+
Content-Type:
|
17
|
+
- application/x-www-form-urlencoded
|
18
|
+
response:
|
19
|
+
status:
|
20
|
+
code: 200
|
21
|
+
message: !binary |-
|
22
|
+
T0s=
|
23
|
+
headers:
|
24
|
+
!binary "Q29udGVudC1UeXBl":
|
25
|
+
- !binary |-
|
26
|
+
YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
|
27
|
+
!binary "Q29udGVudC1MZW5ndGg=":
|
28
|
+
- !binary |-
|
29
|
+
MjM5Mw==
|
30
|
+
!binary "Q29ubmVjdGlvbg==":
|
31
|
+
- !binary |-
|
32
|
+
a2VlcC1hbGl2ZQ==
|
33
|
+
body:
|
34
|
+
encoding: US-ASCII
|
35
|
+
string: ! '{"took":6,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":30,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"13147174","_score":null,"fields":{"_parent":"10471","code":["OMXIGI"]},"highlight":{"name":["OMX
|
36
|
+
Iceland All-<em>Share</em> GI (OMXIGI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147176","_score":null,"fields":{"_parent":"10471","code":["OMXIPI"]},"highlight":{"name":["OMX
|
37
|
+
Iceland All-<em>Share</em> PI (OMXIPI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147219","_score":null,"fields":{"_parent":"10471","code":["OMXSCAPGI"]},"highlight":{"name":["OMX
|
38
|
+
Stockholm All-<em>Share</em> Cap_GI (OMXSCAPGI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147220","_score":null,"fields":{"_parent":"10471","code":["OMXSCAPPI"]},"highlight":{"name":["OMX
|
39
|
+
Stockholm All-<em>Share</em> Cap_PI (OMXSCAPPI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147404","_score":null,"fields":{"_parent":"10471","code":["OMXIFOPI"]},"highlight":{"name":["OMXI-FO
|
40
|
+
All <em>Share</em> PI (OMXIFOPI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147405","_score":null,"fields":{"_parent":"10471","code":["OMXIFOGI"]},"highlight":{"name":["OMXI-FO
|
41
|
+
All <em>Share</em> GI (OMXIFOGI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13127717","_score":null,"fields":{"_parent":"10471","code":["NQCAPFDT"]},"highlight":{"name":["NASDAQ
|
42
|
+
Select Canadian Preferred <em>Share</em> Index TR (NQCAPFDT)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13127718","_score":null,"fields":{"_parent":"10471","code":["NQCAPFD"]},"highlight":{"name":["NASDAQ
|
43
|
+
Select Canadian Preferred <em>Share</em> Index (NQCAPFD)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13961773","_score":null,"fields":{"_parent":"10471","code":["BSCP03"]},"highlight":{"name":["NASDAQ
|
44
|
+
Bullet<em>Shares</em> Corporate Debt 0-3 Ladder (BSCP03)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13961774","_score":null,"fields":{"_parent":"10471","code":["BSHY03T"]},"highlight":{"name":["NASDAQ
|
45
|
+
Bullet<em>Shares</em> High Yield Debt 0-3 Ladder TR (BSHY03T)"]},"sort":[1431648000000]}]}}'
|
46
|
+
http_version:
|
47
|
+
recorded_at: Fri, 22 May 2015 15:57:50 GMT
|
48
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1,41 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://staging-search.quandl.com:9200//quandl_index/dataset/_search
|
6
|
+
body:
|
7
|
+
encoding: UTF-8
|
8
|
+
string: ! '{"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"mexico","minimum_should_match":1,"type":"best_fields","operator":"and"}},{"prefix":{"code":"mexico"}}]}},"filter":{"bool":{"must":[{"term":{"_parent":10471}}]}}}},"sort":[{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"from":0,"size":3}'
|
9
|
+
headers:
|
10
|
+
User-Agent:
|
11
|
+
- Faraday v0.9.1
|
12
|
+
Accept:
|
13
|
+
- ! '*/*'
|
14
|
+
Date:
|
15
|
+
- Fri, 22 May 2015 16:05:38 GMT
|
16
|
+
Content-Type:
|
17
|
+
- application/x-www-form-urlencoded
|
18
|
+
response:
|
19
|
+
status:
|
20
|
+
code: 200
|
21
|
+
message: !binary |-
|
22
|
+
T0s=
|
23
|
+
headers:
|
24
|
+
!binary "Q29udGVudC1UeXBl":
|
25
|
+
- !binary |-
|
26
|
+
YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
|
27
|
+
!binary "Q29udGVudC1MZW5ndGg=":
|
28
|
+
- !binary |-
|
29
|
+
Nzk4
|
30
|
+
!binary "Q29ubmVjdGlvbg==":
|
31
|
+
- !binary |-
|
32
|
+
a2VlcC1hbGl2ZQ==
|
33
|
+
body:
|
34
|
+
encoding: US-ASCII
|
35
|
+
string: ! '{"took":38,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":315,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"13124809","_score":null,"fields":{"_parent":"10471","code":["NQMX1000AUD"]},"highlight":{"name":["NASDAQ
|
36
|
+
<em>Mexico</em> Basic Matls AUD Index (NQMX1000AUD)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13124810","_score":null,"fields":{"_parent":"10471","code":["NQMXAUD"]},"highlight":{"name":["NASDAQ
|
37
|
+
<em>Mexico</em> AUD Index (NQMXAUD)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13124813","_score":null,"fields":{"_parent":"10471","code":["NQMXAUDN"]},"highlight":{"name":["NASDAQ
|
38
|
+
<em>Mexico</em> AUD NTR Index (NQMXAUDN)"]},"sort":[1431648000000]}]}}'
|
39
|
+
http_version:
|
40
|
+
recorded_at: Fri, 22 May 2015 16:05:38 GMT
|
41
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1,139 @@
|
|
1
|
+
---
|
2
|
+
http_interactions:
|
3
|
+
- request:
|
4
|
+
method: get
|
5
|
+
uri: http://staging-search.quandl.com:9200//quandl_index/database/_search
|
6
|
+
body:
|
7
|
+
encoding: UTF-8
|
8
|
+
string: ! '{"explain":false,"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1","description","documentation^0.8"],"query":"stocks","type":"phrase","operator":"and","zero_terms_query":"all","slop":2}},{"has_child":{"type":"dataset","score_mode":"max","query":{"bool":{"should":[{"multi_match":{"fields":["name^1.1"],"query":"stocks","type":"phrase","slop":10}},{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}}}},{"prefix":{"code":"stocks"}}]}}}},"from":0,"size":10,"rescore":{"window_size":10,"query":{"rescore_query":{"match":{"name":{"query":"stocks","type":"phrase","slop":10}}},"query_weight":0.7,"rescore_query_weight":1.2}}}'
|
9
|
+
headers:
|
10
|
+
User-Agent:
|
11
|
+
- Faraday v0.9.1
|
12
|
+
Accept:
|
13
|
+
- ! '*/*'
|
14
|
+
Date:
|
15
|
+
- Fri, 22 May 2015 15:57:50 GMT
|
16
|
+
Content-Type:
|
17
|
+
- application/x-www-form-urlencoded
|
18
|
+
response:
|
19
|
+
status:
|
20
|
+
code: 200
|
21
|
+
message: !binary |-
|
22
|
+
T0s=
|
23
|
+
headers:
|
24
|
+
!binary "Q29udGVudC1UeXBl":
|
25
|
+
- !binary |-
|
26
|
+
YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
|
27
|
+
!binary "Q29udGVudC1MZW5ndGg=":
|
28
|
+
- !binary |-
|
29
|
+
MTIwNg==
|
30
|
+
!binary "Q29ubmVjdGlvbg==":
|
31
|
+
- !binary |-
|
32
|
+
a2VlcC1hbGl2ZQ==
|
33
|
+
body:
|
34
|
+
encoding: US-ASCII
|
35
|
+
string: ! '{"took":765,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":148,"max_score":2.7896662,"hits":[{"_index":"quandl_main_v6","_type":"database","_id":"33","_score":3.7459924,"fields":{"code":["NSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"460","_score":3.4402916,"fields":{"code":["LUXSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"442","_score":3.391498,"fields":{"code":["ZAGREBSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"462","_score":3.381187,"fields":{"code":["BUCHARESTSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"5628","_score":3.2165365,"fields":{"code":["EURONEXT"]}},{"_index":"quandl_main_v6","_type":"database","_id":"450","_score":3.0062313,"fields":{"code":["PHILSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12910","_score":2.7696624,"fields":{"code":["EOD"]}},{"_index":"quandl_main_v6","_type":"database","_id":"1383","_score":1.5982721,"fields":{"code":["SIX"]}},{"_index":"quandl_main_v6","_type":"database","_id":"464","_score":1.5879608,"fields":{"code":["NIKKEI"]}},{"_index":"quandl_main_v6","_type":"database","_id":"8401","_score":1.2901201,"fields":{"code":["CRYPTOCHART"]}}]}}'
|
36
|
+
http_version:
|
37
|
+
recorded_at: Fri, 22 May 2015 15:57:51 GMT
|
38
|
+
- request:
|
39
|
+
method: get
|
40
|
+
uri: http://staging-search.quandl.com:9200//_msearch
|
41
|
+
body:
|
42
|
+
encoding: UTF-8
|
43
|
+
string: ! '{"index":"quandl_index","type":"dataset"}
|
44
|
+
|
45
|
+
{"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"33"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
|
46
|
+
|
47
|
+
{"index":"quandl_index","type":"dataset"}
|
48
|
+
|
49
|
+
{"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"460"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
|
50
|
+
|
51
|
+
{"index":"quandl_index","type":"dataset"}
|
52
|
+
|
53
|
+
{"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"442"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
|
54
|
+
|
55
|
+
{"index":"quandl_index","type":"dataset"}
|
56
|
+
|
57
|
+
{"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"462"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
|
58
|
+
|
59
|
+
{"index":"quandl_index","type":"dataset"}
|
60
|
+
|
61
|
+
{"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"5628"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
|
62
|
+
|
63
|
+
{"index":"quandl_index","type":"dataset"}
|
64
|
+
|
65
|
+
{"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"450"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
|
66
|
+
|
67
|
+
{"index":"quandl_index","type":"dataset"}
|
68
|
+
|
69
|
+
{"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"12910"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
|
70
|
+
|
71
|
+
{"index":"quandl_index","type":"dataset"}
|
72
|
+
|
73
|
+
{"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"1383"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
|
74
|
+
|
75
|
+
{"index":"quandl_index","type":"dataset"}
|
76
|
+
|
77
|
+
{"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"464"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
|
78
|
+
|
79
|
+
{"index":"quandl_index","type":"dataset"}
|
80
|
+
|
81
|
+
{"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"8401"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
|
82
|
+
|
83
|
+
'
|
84
|
+
headers:
|
85
|
+
User-Agent:
|
86
|
+
- Faraday v0.9.1
|
87
|
+
Accept:
|
88
|
+
- ! '*/*'
|
89
|
+
Date:
|
90
|
+
- Fri, 22 May 2015 15:57:51 GMT
|
91
|
+
Content-Type:
|
92
|
+
- application/x-www-form-urlencoded
|
93
|
+
response:
|
94
|
+
status:
|
95
|
+
code: 200
|
96
|
+
message: !binary |-
|
97
|
+
T0s=
|
98
|
+
headers:
|
99
|
+
!binary "Q29udGVudC1UeXBl":
|
100
|
+
- !binary |-
|
101
|
+
YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
|
102
|
+
!binary "Q29udGVudC1MZW5ndGg=":
|
103
|
+
- !binary |-
|
104
|
+
NzQzMA==
|
105
|
+
!binary "Q29ubmVjdGlvbg==":
|
106
|
+
- !binary |-
|
107
|
+
a2VlcC1hbGl2ZQ==
|
108
|
+
body:
|
109
|
+
encoding: US-ASCII
|
110
|
+
string: ! '{"responses":[{"took":20,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":82,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2449180","_score":0.15413277,"fields":{"_parent":"33","code":["CNXIT"]},"highlight":{"name":["National
|
111
|
+
<em>Stock</em> Exchange of India <em>Stock</em> Index - CNX IT"]},"sort":[0.15413277,1410480000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"12849054","_score":0.12844397,"fields":{"_parent":"33","code":["LIX_15"]},"highlight":{"name":["National
|
112
|
+
<em>Stock</em> Exchange of India (NSE) <em>Stock</em> Index - LIX 15"]},"sort":[0.12844397,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"12849055","_score":0.12844397,"fields":{"_parent":"33","code":["LIX15_MIDCAP"]},"highlight":{"name":["National
|
113
|
+
<em>Stock</em> Exchange of India (NSE) <em>Stock</em> Index - LIX15 Midcap"]},"sort":[0.12844397,1431648000000]}]}},{"took":4,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":4,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2424381","_score":0.10898833,"fields":{"_parent":"460","code":["LUXXR"]},"highlight":{"name":["Luxembourg
|
114
|
+
<em>Stock</em> Index: LuxX Returns"]},"sort":[0.10898833,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2424382","_score":0.10898833,"fields":{"_parent":"460","code":["LUXXC"]},"highlight":{"name":["Luxembourg
|
115
|
+
<em>Stock</em> Index: LuxX Cours"]},"sort":[0.10898833,1417478400000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2424383","_score":0.10898833,"fields":{"_parent":"460","code":["LUXGC"]},"highlight":{"name":["Luxembourg
|
116
|
+
<em>Stock</em> Index: Lux General Cours"]},"sort":[0.10898833,1410220800000]}]}},{"took":20,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":4,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2422954","_score":0.15413277,"fields":{"_parent":"442","code":["CROBEX"]},"highlight":{"name":["Zagreb
|
117
|
+
<em>Stock</em> Exchange <em>Stock</em> Index: CROBEX"]},"sort":[0.15413277,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422955","_score":0.15413277,"fields":{"_parent":"442","code":["CROBEX10"]},"highlight":{"name":["Zagreb
|
118
|
+
<em>Stock</em> Exchange <em>Stock</em> Index: CROBEX10"]},"sort":[0.15413277,1361232000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422956","_score":0.10898833,"fields":{"_parent":"442","code":["CROBIS"]},"highlight":{"name":["Zagreb
|
119
|
+
<em>Stock</em> Exchange Bond Index: CROBIS"]},"sort":[0.10898833,1361145600000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2424385","_score":0.14531778,"fields":{"_parent":"462","code":["INDICES"]},"highlight":{"name":["Bucharest
|
120
|
+
<em>Stock</em> Exchange Indices"]},"sort":[0.14531778,1431648000000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"9820158","_score":0.14531778,"fields":{"_parent":"5628","code":["LSEL"]},"highlight":{"name":["London
|
121
|
+
<em>Stock</em> Exch (LSEL)"]},"sort":[0.14531778,1377734400000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":8,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2422989","_score":0.17982157,"fields":{"_parent":"450","code":["ALL"]},"highlight":{"name":["Philippine
|
122
|
+
<em>Stock</em> Index: All <em>Shares</em>"]},"sort":[0.17982157,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422988","_score":0.14531778,"fields":{"_parent":"450","code":["PSE"]},"highlight":{"name":["Philippine
|
123
|
+
<em>Stock</em> Index: PSEi"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422990","_score":0.14531778,"fields":{"_parent":"450","code":["FIN"]},"highlight":{"name":["Philippine
|
124
|
+
<em>Stock</em> Index: Financials"]},"sort":[0.14531778,1431648000000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":8641,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"14031653","_score":0.15731111,"fields":{"_parent":"12910","code":["IPFF"]},"highlight":{"name":["i<em>Shares</em>
|
125
|
+
International Preferred <em>Stock</em> (IPFF) <em>Stock</em> Prices, Dividends
|
126
|
+
and Splits"]},"sort":[0.15731111,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"14028742","_score":0.15731111,"fields":{"_parent":"12910","code":["CHNA"]},"highlight":{"name":["Power<em>Shares</em>
|
127
|
+
China A-<em>Share</em> ETF (CHNA) <em>Stock</em> Prices, Dividends and Splits"]},"sort":[0.15731111,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"14026179","_score":0.15413277,"fields":{"_parent":"12910","code":["EQC"]},"highlight":{"name":["<em>Equity</em>
|
128
|
+
Commonwealth (EQC) <em>Stock</em> Prices, Dividends and Splits"]},"sort":[0.15413277,1431648000000]}]}},{"took":16,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":292,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"8536116","_score":0.15413277,"fields":{"_parent":"1383","code":["CH0006089921CHF4"]},"highlight":{"name":["PRIVATE
|
129
|
+
<em>EQUITY</em> N - <em>Share</em> Price CHF"]},"sort":[0.15413277,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8535935","_score":0.12715305,"fields":{"_parent":"1383","code":["CH0014424524USD4"]},"highlight":{"name":["ALTIN
|
130
|
+
N - <em>Share</em> Price USD"]},"sort":[0.12715305,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8535984","_score":0.12715305,"fields":{"_parent":"1383","code":["US1491231015CHF4"]},"highlight":{"name":["CATERPILLAR
|
131
|
+
- <em>Share</em> Price CHF"]},"sort":[0.12715305,1431648000000]}]}},{"took":16,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":7,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"8473284","_score":0.14531778,"fields":{"_parent":"464","code":["ALL_STOCK"]},"highlight":{"name":["Nikkei
|
132
|
+
All <em>Stock</em> Index"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8471661","_score":0.14531778,"fields":{"_parent":"464","code":["SI300"]},"highlight":{"name":["Nikkei
|
133
|
+
<em>Stock</em> Index 300"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8472417","_score":0.12715305,"fields":{"_parent":"464","code":["500SA"]},"highlight":{"name":["Nikkei
|
134
|
+
500 <em>Stock</em> Average Index"]},"sort":[0.12715305,1431648000000]}]}},{"took":19,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":28,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"19686015","_score":8.422265,"fields":{"_parent":"8401","code":["STOCK"]},"sort":[8.422265,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"11320655","_score":0.14531778,"fields":{"_parent":"8401","code":["NRS"]},"highlight":{"name":["Noir<em>Shares</em>/BITCOIN
|
135
|
+
Price"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"11320692","_score":0.14531778,"fields":{"_parent":"8401","code":["PTS"]},"highlight":{"name":["Proto<em>Shares</em>/BITCOIN
|
136
|
+
Price"]},"sort":[0.14531778,1431648000000]}]}}]}'
|
137
|
+
http_version:
|
138
|
+
recorded_at: Fri, 22 May 2015 15:57:51 GMT
|
139
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'elasticsearch'
|
2
|
+
require 'yaml'
|
3
|
+
require 'quandl/config'
|
4
|
+
require 'quandl/elasticsearch/version'
|
5
|
+
require 'quandl/elasticsearch/search'
|
6
|
+
require 'quandl/elasticsearch/indice'
|
7
|
+
require 'quandl/elasticsearch/utility'
|
8
|
+
|
9
|
+
module Quandl
|
10
|
+
module Elasticsearch
|
11
|
+
extend Quandl::Configurable
|
12
|
+
|
13
|
+
class << self
|
14
|
+
def file_name
|
15
|
+
'elasticsearch'
|
16
|
+
end
|
17
|
+
|
18
|
+
def configure
|
19
|
+
yield(configuration)
|
20
|
+
end
|
21
|
+
|
22
|
+
def client(host = nil)
|
23
|
+
client_for_host(host || configuration.host)
|
24
|
+
end
|
25
|
+
|
26
|
+
def disconnect!
|
27
|
+
@clients = {}
|
28
|
+
end
|
29
|
+
|
30
|
+
# Helper method for testing to reset block configuration and reload file
|
31
|
+
def reset_configuration!
|
32
|
+
Quandl::Config.clear_internal_cache
|
33
|
+
@configuration = Quandl::Config.new(file_name)
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def client_for_host(host)
|
39
|
+
return clients[host] if clients.key?(host)
|
40
|
+
clients[host] = ::Elasticsearch::Client.new(host: host,
|
41
|
+
transport_options: {
|
42
|
+
request: {
|
43
|
+
timeout: configuration.timeout,
|
44
|
+
open_timeout: configuration.open_timeout
|
45
|
+
}
|
46
|
+
})
|
47
|
+
end
|
48
|
+
|
49
|
+
def clients
|
50
|
+
@clients ||= {}
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
# Backwards compatability
|
55
|
+
class Configuration
|
56
|
+
def self.client
|
57
|
+
Quandl::Elasticsearch.client
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Elasticsearch
|
3
|
+
class Base
|
4
|
+
def initialize(record)
|
5
|
+
@record = record
|
6
|
+
end
|
7
|
+
|
8
|
+
def id
|
9
|
+
@id ||= @record['id'].to_i
|
10
|
+
end
|
11
|
+
|
12
|
+
protected
|
13
|
+
|
14
|
+
# this pertains to the postgres db's values!
|
15
|
+
def to_bool(value)
|
16
|
+
value == 't' || value == true
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Elasticsearch
|
3
|
+
## Database
|
4
|
+
class Database < Base
|
5
|
+
def data
|
6
|
+
@data ||= { name: @record['name'],
|
7
|
+
code: @record['code'],
|
8
|
+
updated_at: @record['updated_at'],
|
9
|
+
description: @record['description'],
|
10
|
+
documentation: @record['documentation'],
|
11
|
+
premium: to_bool(@record['premium']),
|
12
|
+
hidden: to_bool(@record['hidden']),
|
13
|
+
exclusive: to_bool(@record['exclusive']),
|
14
|
+
tags: @record['elasticsearch_flags'] ? @record['elasticsearch_flags'].split(',') : [] }
|
15
|
+
end
|
16
|
+
|
17
|
+
def remove?
|
18
|
+
!@record['deleted_at'].nil? || !@record['type'].nil?
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'set'
|
2
|
+
|
3
|
+
module Quandl
|
4
|
+
module Elasticsearch
|
5
|
+
## Database
|
6
|
+
class Dataset < Base
|
7
|
+
def database_id
|
8
|
+
@database_id ||= @record['database_id']
|
9
|
+
end
|
10
|
+
|
11
|
+
def data
|
12
|
+
@data ||= { code: @record['code'],
|
13
|
+
name: @record['name'],
|
14
|
+
to_date: @record['to_date'],
|
15
|
+
is_private: to_bool(@record['is_private']),
|
16
|
+
frequency: @record['frequency'],
|
17
|
+
db_hidden: to_bool(@record['db_hidden']),
|
18
|
+
db_exclusive: to_bool(@record['db_exclusive']),
|
19
|
+
sample: previewable? }
|
20
|
+
end
|
21
|
+
|
22
|
+
def remove?
|
23
|
+
(@record['type'] && @record['type'] == 'Superset') || to_bool(@record['is_private']) || !@record['deleted_at'].nil?
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def previewable_datasets
|
29
|
+
@_preview_datasets ||= @record['db_preview_datasets'].split(',').to_set
|
30
|
+
end
|
31
|
+
|
32
|
+
def preview_enabled?
|
33
|
+
to_bool(@record['db_preview_enabled'])
|
34
|
+
end
|
35
|
+
|
36
|
+
def premium?
|
37
|
+
to_bool(@record['db_premium'])
|
38
|
+
end
|
39
|
+
|
40
|
+
def previewable?
|
41
|
+
if !premium? || !preview_enabled?
|
42
|
+
false
|
43
|
+
elsif previewable_datasets.empty?
|
44
|
+
true
|
45
|
+
else
|
46
|
+
previewable_datasets.include?(@record['code'])
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
require 'quandl/elasticsearch/base'
|
2
|
+
require 'quandl/elasticsearch/template'
|
3
|
+
require 'quandl/elasticsearch/database'
|
4
|
+
require 'quandl/elasticsearch/dataset'
|
5
|
+
require 'elasticsearch'
|
6
|
+
require 'json'
|
7
|
+
|
8
|
+
module Quandl
|
9
|
+
module Elasticsearch
|
10
|
+
## Indice
|
11
|
+
class Indice
|
12
|
+
def initialize
|
13
|
+
@client = Quandl::Elasticsearch.client
|
14
|
+
end
|
15
|
+
|
16
|
+
def create_main(index)
|
17
|
+
@client.indices.create index: index.to_s, body: Quandl::Elasticsearch::Template.main_index
|
18
|
+
end
|
19
|
+
|
20
|
+
def build_database(index, databases)
|
21
|
+
dbs = []
|
22
|
+
databases.each { |d| dbs << Database.new(d) }
|
23
|
+
database_bulk(index, dbs)
|
24
|
+
end
|
25
|
+
|
26
|
+
def build_dataset(index, datasets)
|
27
|
+
ds = []
|
28
|
+
datasets.each { |d| ds << Dataset.new(d) }
|
29
|
+
dataset_bulk(index, ds)
|
30
|
+
end
|
31
|
+
|
32
|
+
def databases_empty?(index)
|
33
|
+
result = @client.count index: index.to_s, type: 'database'
|
34
|
+
result['count'].to_i == 0
|
35
|
+
end
|
36
|
+
|
37
|
+
def database_exist?(database_id, index)
|
38
|
+
@client.exists? index: index.to_s, type: 'database', id: database_id.to_s
|
39
|
+
end
|
40
|
+
|
41
|
+
def get_database(database_id, index)
|
42
|
+
@client.get index: index.to_s, type: 'database', id: database_id.to_s
|
43
|
+
end
|
44
|
+
|
45
|
+
def aliases(from, _to)
|
46
|
+
@client.indices.update_aliases body: { actions: [{ add: { index: from.to_s, alias: '_to' } }] }
|
47
|
+
end
|
48
|
+
|
49
|
+
def swap(old_index, new_index, index)
|
50
|
+
@client.indices.update_aliases body: { actions: [{ remove: { index: old_index.to_s, alias: index.to_s } },
|
51
|
+
{ add: { index: new_index.to_s, alias: index.to_s } }] }
|
52
|
+
end
|
53
|
+
|
54
|
+
def delete(index)
|
55
|
+
@client.indices.delete index: [index.to_s]
|
56
|
+
end
|
57
|
+
|
58
|
+
def show_settings
|
59
|
+
@client.indices.get_settings
|
60
|
+
end
|
61
|
+
|
62
|
+
def status(index)
|
63
|
+
@client.indices.stats index: index.to_s
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
def database_bulk(index, dbs)
|
69
|
+
body = []
|
70
|
+
dbs.each do |db|
|
71
|
+
action = if db.remove?
|
72
|
+
{ delete: { _index: index.to_s, _type: 'database', _id: db.id } }
|
73
|
+
else
|
74
|
+
{ index: { _index: index.to_s, _type: 'database', _id: db.id, data: db.data } }
|
75
|
+
end
|
76
|
+
|
77
|
+
body << action
|
78
|
+
end
|
79
|
+
@client.bulk body: body
|
80
|
+
end
|
81
|
+
|
82
|
+
def dataset_bulk(index, datasets)
|
83
|
+
body = []
|
84
|
+
datasets.each do |d|
|
85
|
+
action = if d.remove?
|
86
|
+
{ delete: { _index: index.to_s, _type: 'dataset', _id: d.id, parent: d.database_id } }
|
87
|
+
else
|
88
|
+
{ index: { _index: index.to_s, _type: 'dataset', _id: d.id, parent: d.database_id, data: d.data } }
|
89
|
+
end
|
90
|
+
body << action
|
91
|
+
end
|
92
|
+
@client.bulk body: body
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|