quandl-elasticsearch 2.1.0.rc5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +20 -0
  3. data/.rubocop.yml +34 -0
  4. data/COMMANDS.md +29 -0
  5. data/Gemfile +10 -0
  6. data/Gemfile.lock +155 -0
  7. data/LICENSE.txt +22 -0
  8. data/README.md +50 -0
  9. data/Rakefile +1 -0
  10. data/config/elasticsearch.yml +32 -0
  11. data/elasticsearch/elasticsearch.yml +386 -0
  12. data/elasticsearch/logging.yml +56 -0
  13. data/elasticsearch/stopwords/english.txt +38 -0
  14. data/elasticsearch/synonyms/synonyms_english.txt +318 -0
  15. data/fixtures/vcr_cassettes/search_spec_database_1.yml +38 -0
  16. data/fixtures/vcr_cassettes/search_spec_database_2.yml +38 -0
  17. data/fixtures/vcr_cassettes/search_spec_dataset_1.yml +48 -0
  18. data/fixtures/vcr_cassettes/search_spec_dataset_2.yml +41 -0
  19. data/fixtures/vcr_cassettes/setup.yml +139 -0
  20. data/lib/quandl/elasticsearch.rb +61 -0
  21. data/lib/quandl/elasticsearch/base.rb +20 -0
  22. data/lib/quandl/elasticsearch/database.rb +22 -0
  23. data/lib/quandl/elasticsearch/dataset.rb +51 -0
  24. data/lib/quandl/elasticsearch/indice.rb +96 -0
  25. data/lib/quandl/elasticsearch/query.rb +282 -0
  26. data/lib/quandl/elasticsearch/search.rb +150 -0
  27. data/lib/quandl/elasticsearch/tag.rb +21 -0
  28. data/lib/quandl/elasticsearch/template.rb +189 -0
  29. data/lib/quandl/elasticsearch/utility.rb +6 -0
  30. data/lib/quandl/elasticsearch/version.rb +6 -0
  31. data/quandl +77 -0
  32. data/quandl-elasticsearch.gemspec +34 -0
  33. data/solano.yml +20 -0
  34. data/spec/lib/quandl/elasticsearch/database_spec.rb +98 -0
  35. data/spec/lib/quandl/elasticsearch/dataset_spec.rb +124 -0
  36. data/spec/lib/quandl/elasticsearch/indice_spec.rb +10 -0
  37. data/spec/lib/quandl/elasticsearch/query_spec.rb +239 -0
  38. data/spec/lib/quandl/elasticsearch/search_spec.rb +83 -0
  39. data/spec/lib/quandl/elasticsearch/template_spec.rb +182 -0
  40. data/spec/lib/quandl/elasticsearch/utility_spec.rb +10 -0
  41. data/spec/lib/quandl/elasticsearch_spec.rb +99 -0
  42. data/spec/spec_helper.rb +27 -0
  43. data/templates/database_mapping.json +11 -0
  44. data/templates/dataset_mapping.json +9 -0
  45. data/templates/quandl_delimiter.json +0 -0
  46. data/templates/search_term_mapping.json +13 -0
  47. data/tests/Database-Ratings.csv +405 -0
  48. data/tests/Database-Tags.csv +341 -0
  49. data/tests/compare.csv +1431 -0
  50. data/tests/compare.rb +33 -0
  51. data/tests/console.rb +4 -0
  52. data/tests/generated_db_tags.csv +341 -0
  53. data/tests/search.rb +14 -0
  54. data/tests/search_db_mapping.txt +402 -0
  55. data/tests/status.rb +2 -0
  56. data/tests/test_search.csv +87 -0
  57. data/tests/test_search.rb +113 -0
  58. data/tests/testing-list.txt +183 -0
  59. data/tests/top500searches.csv +477 -0
  60. metadata +300 -0
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://staging-search.quandl.com:9200//quandl_index/database/_search
6
+ body:
7
+ encoding: UTF-8
8
+ string: ! '{"explain":false,"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1","description","documentation^0.8"],"query":"stocks","type":"phrase","operator":"and","zero_terms_query":"all","slop":2}},{"has_child":{"type":"dataset","score_mode":"max","query":{"bool":{"should":[{"multi_match":{"fields":["name^1.1"],"query":"stocks","type":"phrase","slop":10}},{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}}}},{"prefix":{"code":"stocks"}}]}},"filter":{"bool":{"must":[{"term":{"premium":true}}]}}}},"from":0,"size":10,"rescore":{"window_size":10,"query":{"rescore_query":{"match":{"name":{"query":"stocks","type":"phrase","slop":10}}},"query_weight":0.7,"rescore_query_weight":1.2}}}'
9
+ headers:
10
+ User-Agent:
11
+ - Faraday v0.9.1
12
+ Accept:
13
+ - ! '*/*'
14
+ Date:
15
+ - Fri, 22 May 2015 16:05:37 GMT
16
+ Content-Type:
17
+ - application/x-www-form-urlencoded
18
+ response:
19
+ status:
20
+ code: 200
21
+ message: !binary |-
22
+ T0s=
23
+ headers:
24
+ !binary "Q29udGVudC1UeXBl":
25
+ - !binary |-
26
+ YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
27
+ !binary "Q29udGVudC1MZW5ndGg=":
28
+ - !binary |-
29
+ MTE5Mw==
30
+ !binary "Q29ubmVjdGlvbg==":
31
+ - !binary |-
32
+ a2VlcC1hbGl2ZQ==
33
+ body:
34
+ encoding: US-ASCII
35
+ string: ! '{"took":769,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":18,"max_score":2.0353467,"hits":[{"_index":"quandl_main_v6","_type":"database","_id":"12910","_score":2.7696624,"fields":{"code":["EOD"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13091","_score":1.0613412,"fields":{"code":["ZFA"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13122","_score":1.0613412,"fields":{"code":["ZFB"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13065","_score":0.5353001,"fields":{"code":["ZSEE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12266","_score":0.504785,"fields":{"code":["ZEE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13000","_score":0.49280426,"fields":{"code":["ZET"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12985","_score":0.461574,"fields":{"code":["ZAR"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12908","_score":0.46129575,"fields":{"code":["SF1"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12973","_score":0.43754363,"fields":{"code":["ZDIV"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12991","_score":0.37978202,"fields":{"code":["ZSS"]}}]}}'
36
+ http_version:
37
+ recorded_at: Fri, 22 May 2015 16:05:38 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,48 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://staging-search.quandl.com:9200//quandl_index/dataset/_search
6
+ body:
7
+ encoding: UTF-8
8
+ string: ! '{"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","minimum_should_match":1,"type":"best_fields","operator":"and"}},{"prefix":{"code":"stocks"}}]}},"filter":{"bool":{"must":[{"term":{"_parent":10471}}],"should":[{"term":{"frequency":"daily"}}]}}}},"sort":[{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"from":0,"size":10}'
9
+ headers:
10
+ User-Agent:
11
+ - Faraday v0.9.1
12
+ Accept:
13
+ - ! '*/*'
14
+ Date:
15
+ - Fri, 22 May 2015 15:57:50 GMT
16
+ Content-Type:
17
+ - application/x-www-form-urlencoded
18
+ response:
19
+ status:
20
+ code: 200
21
+ message: !binary |-
22
+ T0s=
23
+ headers:
24
+ !binary "Q29udGVudC1UeXBl":
25
+ - !binary |-
26
+ YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
27
+ !binary "Q29udGVudC1MZW5ndGg=":
28
+ - !binary |-
29
+ MjM5Mw==
30
+ !binary "Q29ubmVjdGlvbg==":
31
+ - !binary |-
32
+ a2VlcC1hbGl2ZQ==
33
+ body:
34
+ encoding: US-ASCII
35
+ string: ! '{"took":6,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":30,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"13147174","_score":null,"fields":{"_parent":"10471","code":["OMXIGI"]},"highlight":{"name":["OMX
36
+ Iceland All-<em>Share</em> GI (OMXIGI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147176","_score":null,"fields":{"_parent":"10471","code":["OMXIPI"]},"highlight":{"name":["OMX
37
+ Iceland All-<em>Share</em> PI (OMXIPI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147219","_score":null,"fields":{"_parent":"10471","code":["OMXSCAPGI"]},"highlight":{"name":["OMX
38
+ Stockholm All-<em>Share</em> Cap_GI (OMXSCAPGI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147220","_score":null,"fields":{"_parent":"10471","code":["OMXSCAPPI"]},"highlight":{"name":["OMX
39
+ Stockholm All-<em>Share</em> Cap_PI (OMXSCAPPI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147404","_score":null,"fields":{"_parent":"10471","code":["OMXIFOPI"]},"highlight":{"name":["OMXI-FO
40
+ All <em>Share</em> PI (OMXIFOPI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147405","_score":null,"fields":{"_parent":"10471","code":["OMXIFOGI"]},"highlight":{"name":["OMXI-FO
41
+ All <em>Share</em> GI (OMXIFOGI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13127717","_score":null,"fields":{"_parent":"10471","code":["NQCAPFDT"]},"highlight":{"name":["NASDAQ
42
+ Select Canadian Preferred <em>Share</em> Index TR (NQCAPFDT)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13127718","_score":null,"fields":{"_parent":"10471","code":["NQCAPFD"]},"highlight":{"name":["NASDAQ
43
+ Select Canadian Preferred <em>Share</em> Index (NQCAPFD)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13961773","_score":null,"fields":{"_parent":"10471","code":["BSCP03"]},"highlight":{"name":["NASDAQ
44
+ Bullet<em>Shares</em> Corporate Debt 0-3 Ladder (BSCP03)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13961774","_score":null,"fields":{"_parent":"10471","code":["BSHY03T"]},"highlight":{"name":["NASDAQ
45
+ Bullet<em>Shares</em> High Yield Debt 0-3 Ladder TR (BSHY03T)"]},"sort":[1431648000000]}]}}'
46
+ http_version:
47
+ recorded_at: Fri, 22 May 2015 15:57:50 GMT
48
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,41 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://staging-search.quandl.com:9200//quandl_index/dataset/_search
6
+ body:
7
+ encoding: UTF-8
8
+ string: ! '{"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"mexico","minimum_should_match":1,"type":"best_fields","operator":"and"}},{"prefix":{"code":"mexico"}}]}},"filter":{"bool":{"must":[{"term":{"_parent":10471}}]}}}},"sort":[{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"from":0,"size":3}'
9
+ headers:
10
+ User-Agent:
11
+ - Faraday v0.9.1
12
+ Accept:
13
+ - ! '*/*'
14
+ Date:
15
+ - Fri, 22 May 2015 16:05:38 GMT
16
+ Content-Type:
17
+ - application/x-www-form-urlencoded
18
+ response:
19
+ status:
20
+ code: 200
21
+ message: !binary |-
22
+ T0s=
23
+ headers:
24
+ !binary "Q29udGVudC1UeXBl":
25
+ - !binary |-
26
+ YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
27
+ !binary "Q29udGVudC1MZW5ndGg=":
28
+ - !binary |-
29
+ Nzk4
30
+ !binary "Q29ubmVjdGlvbg==":
31
+ - !binary |-
32
+ a2VlcC1hbGl2ZQ==
33
+ body:
34
+ encoding: US-ASCII
35
+ string: ! '{"took":38,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":315,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"13124809","_score":null,"fields":{"_parent":"10471","code":["NQMX1000AUD"]},"highlight":{"name":["NASDAQ
36
+ <em>Mexico</em> Basic Matls AUD Index (NQMX1000AUD)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13124810","_score":null,"fields":{"_parent":"10471","code":["NQMXAUD"]},"highlight":{"name":["NASDAQ
37
+ <em>Mexico</em> AUD Index (NQMXAUD)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13124813","_score":null,"fields":{"_parent":"10471","code":["NQMXAUDN"]},"highlight":{"name":["NASDAQ
38
+ <em>Mexico</em> AUD NTR Index (NQMXAUDN)"]},"sort":[1431648000000]}]}}'
39
+ http_version:
40
+ recorded_at: Fri, 22 May 2015 16:05:38 GMT
41
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,139 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://staging-search.quandl.com:9200//quandl_index/database/_search
6
+ body:
7
+ encoding: UTF-8
8
+ string: ! '{"explain":false,"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1","description","documentation^0.8"],"query":"stocks","type":"phrase","operator":"and","zero_terms_query":"all","slop":2}},{"has_child":{"type":"dataset","score_mode":"max","query":{"bool":{"should":[{"multi_match":{"fields":["name^1.1"],"query":"stocks","type":"phrase","slop":10}},{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}}}},{"prefix":{"code":"stocks"}}]}}}},"from":0,"size":10,"rescore":{"window_size":10,"query":{"rescore_query":{"match":{"name":{"query":"stocks","type":"phrase","slop":10}}},"query_weight":0.7,"rescore_query_weight":1.2}}}'
9
+ headers:
10
+ User-Agent:
11
+ - Faraday v0.9.1
12
+ Accept:
13
+ - ! '*/*'
14
+ Date:
15
+ - Fri, 22 May 2015 15:57:50 GMT
16
+ Content-Type:
17
+ - application/x-www-form-urlencoded
18
+ response:
19
+ status:
20
+ code: 200
21
+ message: !binary |-
22
+ T0s=
23
+ headers:
24
+ !binary "Q29udGVudC1UeXBl":
25
+ - !binary |-
26
+ YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
27
+ !binary "Q29udGVudC1MZW5ndGg=":
28
+ - !binary |-
29
+ MTIwNg==
30
+ !binary "Q29ubmVjdGlvbg==":
31
+ - !binary |-
32
+ a2VlcC1hbGl2ZQ==
33
+ body:
34
+ encoding: US-ASCII
35
+ string: ! '{"took":765,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":148,"max_score":2.7896662,"hits":[{"_index":"quandl_main_v6","_type":"database","_id":"33","_score":3.7459924,"fields":{"code":["NSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"460","_score":3.4402916,"fields":{"code":["LUXSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"442","_score":3.391498,"fields":{"code":["ZAGREBSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"462","_score":3.381187,"fields":{"code":["BUCHARESTSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"5628","_score":3.2165365,"fields":{"code":["EURONEXT"]}},{"_index":"quandl_main_v6","_type":"database","_id":"450","_score":3.0062313,"fields":{"code":["PHILSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12910","_score":2.7696624,"fields":{"code":["EOD"]}},{"_index":"quandl_main_v6","_type":"database","_id":"1383","_score":1.5982721,"fields":{"code":["SIX"]}},{"_index":"quandl_main_v6","_type":"database","_id":"464","_score":1.5879608,"fields":{"code":["NIKKEI"]}},{"_index":"quandl_main_v6","_type":"database","_id":"8401","_score":1.2901201,"fields":{"code":["CRYPTOCHART"]}}]}}'
36
+ http_version:
37
+ recorded_at: Fri, 22 May 2015 15:57:51 GMT
38
+ - request:
39
+ method: get
40
+ uri: http://staging-search.quandl.com:9200//_msearch
41
+ body:
42
+ encoding: UTF-8
43
+ string: ! '{"index":"quandl_index","type":"dataset"}
44
+
45
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"33"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
46
+
47
+ {"index":"quandl_index","type":"dataset"}
48
+
49
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"460"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
50
+
51
+ {"index":"quandl_index","type":"dataset"}
52
+
53
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"442"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
54
+
55
+ {"index":"quandl_index","type":"dataset"}
56
+
57
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"462"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
58
+
59
+ {"index":"quandl_index","type":"dataset"}
60
+
61
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"5628"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
62
+
63
+ {"index":"quandl_index","type":"dataset"}
64
+
65
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"450"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
66
+
67
+ {"index":"quandl_index","type":"dataset"}
68
+
69
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"12910"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
70
+
71
+ {"index":"quandl_index","type":"dataset"}
72
+
73
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"1383"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
74
+
75
+ {"index":"quandl_index","type":"dataset"}
76
+
77
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"464"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
78
+
79
+ {"index":"quandl_index","type":"dataset"}
80
+
81
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"8401"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
82
+
83
+ '
84
+ headers:
85
+ User-Agent:
86
+ - Faraday v0.9.1
87
+ Accept:
88
+ - ! '*/*'
89
+ Date:
90
+ - Fri, 22 May 2015 15:57:51 GMT
91
+ Content-Type:
92
+ - application/x-www-form-urlencoded
93
+ response:
94
+ status:
95
+ code: 200
96
+ message: !binary |-
97
+ T0s=
98
+ headers:
99
+ !binary "Q29udGVudC1UeXBl":
100
+ - !binary |-
101
+ YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
102
+ !binary "Q29udGVudC1MZW5ndGg=":
103
+ - !binary |-
104
+ NzQzMA==
105
+ !binary "Q29ubmVjdGlvbg==":
106
+ - !binary |-
107
+ a2VlcC1hbGl2ZQ==
108
+ body:
109
+ encoding: US-ASCII
110
+ string: ! '{"responses":[{"took":20,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":82,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2449180","_score":0.15413277,"fields":{"_parent":"33","code":["CNXIT"]},"highlight":{"name":["National
111
+ <em>Stock</em> Exchange of India <em>Stock</em> Index - CNX IT"]},"sort":[0.15413277,1410480000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"12849054","_score":0.12844397,"fields":{"_parent":"33","code":["LIX_15"]},"highlight":{"name":["National
112
+ <em>Stock</em> Exchange of India (NSE) <em>Stock</em> Index - LIX 15"]},"sort":[0.12844397,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"12849055","_score":0.12844397,"fields":{"_parent":"33","code":["LIX15_MIDCAP"]},"highlight":{"name":["National
113
+ <em>Stock</em> Exchange of India (NSE) <em>Stock</em> Index - LIX15 Midcap"]},"sort":[0.12844397,1431648000000]}]}},{"took":4,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":4,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2424381","_score":0.10898833,"fields":{"_parent":"460","code":["LUXXR"]},"highlight":{"name":["Luxembourg
114
+ <em>Stock</em> Index: LuxX Returns"]},"sort":[0.10898833,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2424382","_score":0.10898833,"fields":{"_parent":"460","code":["LUXXC"]},"highlight":{"name":["Luxembourg
115
+ <em>Stock</em> Index: LuxX Cours"]},"sort":[0.10898833,1417478400000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2424383","_score":0.10898833,"fields":{"_parent":"460","code":["LUXGC"]},"highlight":{"name":["Luxembourg
116
+ <em>Stock</em> Index: Lux General Cours"]},"sort":[0.10898833,1410220800000]}]}},{"took":20,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":4,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2422954","_score":0.15413277,"fields":{"_parent":"442","code":["CROBEX"]},"highlight":{"name":["Zagreb
117
+ <em>Stock</em> Exchange <em>Stock</em> Index: CROBEX"]},"sort":[0.15413277,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422955","_score":0.15413277,"fields":{"_parent":"442","code":["CROBEX10"]},"highlight":{"name":["Zagreb
118
+ <em>Stock</em> Exchange <em>Stock</em> Index: CROBEX10"]},"sort":[0.15413277,1361232000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422956","_score":0.10898833,"fields":{"_parent":"442","code":["CROBIS"]},"highlight":{"name":["Zagreb
119
+ <em>Stock</em> Exchange Bond Index: CROBIS"]},"sort":[0.10898833,1361145600000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2424385","_score":0.14531778,"fields":{"_parent":"462","code":["INDICES"]},"highlight":{"name":["Bucharest
120
+ <em>Stock</em> Exchange Indices"]},"sort":[0.14531778,1431648000000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"9820158","_score":0.14531778,"fields":{"_parent":"5628","code":["LSEL"]},"highlight":{"name":["London
121
+ <em>Stock</em> Exch (LSEL)"]},"sort":[0.14531778,1377734400000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":8,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2422989","_score":0.17982157,"fields":{"_parent":"450","code":["ALL"]},"highlight":{"name":["Philippine
122
+ <em>Stock</em> Index: All <em>Shares</em>"]},"sort":[0.17982157,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422988","_score":0.14531778,"fields":{"_parent":"450","code":["PSE"]},"highlight":{"name":["Philippine
123
+ <em>Stock</em> Index: PSEi"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422990","_score":0.14531778,"fields":{"_parent":"450","code":["FIN"]},"highlight":{"name":["Philippine
124
+ <em>Stock</em> Index: Financials"]},"sort":[0.14531778,1431648000000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":8641,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"14031653","_score":0.15731111,"fields":{"_parent":"12910","code":["IPFF"]},"highlight":{"name":["i<em>Shares</em>
125
+ International Preferred <em>Stock</em> (IPFF) <em>Stock</em> Prices, Dividends
126
+ and Splits"]},"sort":[0.15731111,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"14028742","_score":0.15731111,"fields":{"_parent":"12910","code":["CHNA"]},"highlight":{"name":["Power<em>Shares</em>
127
+ China A-<em>Share</em> ETF (CHNA) <em>Stock</em> Prices, Dividends and Splits"]},"sort":[0.15731111,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"14026179","_score":0.15413277,"fields":{"_parent":"12910","code":["EQC"]},"highlight":{"name":["<em>Equity</em>
128
+ Commonwealth (EQC) <em>Stock</em> Prices, Dividends and Splits"]},"sort":[0.15413277,1431648000000]}]}},{"took":16,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":292,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"8536116","_score":0.15413277,"fields":{"_parent":"1383","code":["CH0006089921CHF4"]},"highlight":{"name":["PRIVATE
129
+ <em>EQUITY</em> N - <em>Share</em> Price CHF"]},"sort":[0.15413277,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8535935","_score":0.12715305,"fields":{"_parent":"1383","code":["CH0014424524USD4"]},"highlight":{"name":["ALTIN
130
+ N - <em>Share</em> Price USD"]},"sort":[0.12715305,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8535984","_score":0.12715305,"fields":{"_parent":"1383","code":["US1491231015CHF4"]},"highlight":{"name":["CATERPILLAR
131
+ - <em>Share</em> Price CHF"]},"sort":[0.12715305,1431648000000]}]}},{"took":16,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":7,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"8473284","_score":0.14531778,"fields":{"_parent":"464","code":["ALL_STOCK"]},"highlight":{"name":["Nikkei
132
+ All <em>Stock</em> Index"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8471661","_score":0.14531778,"fields":{"_parent":"464","code":["SI300"]},"highlight":{"name":["Nikkei
133
+ <em>Stock</em> Index 300"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8472417","_score":0.12715305,"fields":{"_parent":"464","code":["500SA"]},"highlight":{"name":["Nikkei
134
+ 500 <em>Stock</em> Average Index"]},"sort":[0.12715305,1431648000000]}]}},{"took":19,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":28,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"19686015","_score":8.422265,"fields":{"_parent":"8401","code":["STOCK"]},"sort":[8.422265,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"11320655","_score":0.14531778,"fields":{"_parent":"8401","code":["NRS"]},"highlight":{"name":["Noir<em>Shares</em>/BITCOIN
135
+ Price"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"11320692","_score":0.14531778,"fields":{"_parent":"8401","code":["PTS"]},"highlight":{"name":["Proto<em>Shares</em>/BITCOIN
136
+ Price"]},"sort":[0.14531778,1431648000000]}]}}]}'
137
+ http_version:
138
+ recorded_at: Fri, 22 May 2015 15:57:51 GMT
139
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,61 @@
1
+ require 'elasticsearch'
2
+ require 'yaml'
3
+ require 'quandl/config'
4
+ require 'quandl/elasticsearch/version'
5
+ require 'quandl/elasticsearch/search'
6
+ require 'quandl/elasticsearch/indice'
7
+ require 'quandl/elasticsearch/utility'
8
+
9
+ module Quandl
10
+ module Elasticsearch
11
+ extend Quandl::Configurable
12
+
13
+ class << self
14
+ def file_name
15
+ 'elasticsearch'
16
+ end
17
+
18
+ def configure
19
+ yield(configuration)
20
+ end
21
+
22
+ def client(host = nil)
23
+ client_for_host(host || configuration.host)
24
+ end
25
+
26
+ def disconnect!
27
+ @clients = {}
28
+ end
29
+
30
+ # Helper method for testing to reset block configuration and reload file
31
+ def reset_configuration!
32
+ Quandl::Config.clear_internal_cache
33
+ @configuration = Quandl::Config.new(file_name)
34
+ end
35
+
36
+ private
37
+
38
+ def client_for_host(host)
39
+ return clients[host] if clients.key?(host)
40
+ clients[host] = ::Elasticsearch::Client.new(host: host,
41
+ transport_options: {
42
+ request: {
43
+ timeout: configuration.timeout,
44
+ open_timeout: configuration.open_timeout
45
+ }
46
+ })
47
+ end
48
+
49
+ def clients
50
+ @clients ||= {}
51
+ end
52
+ end
53
+
54
+ # Backwards compatability
55
+ class Configuration
56
+ def self.client
57
+ Quandl::Elasticsearch.client
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,20 @@
1
+ module Quandl
2
+ module Elasticsearch
3
+ class Base
4
+ def initialize(record)
5
+ @record = record
6
+ end
7
+
8
+ def id
9
+ @id ||= @record['id'].to_i
10
+ end
11
+
12
+ protected
13
+
14
+ # this pertains to the postgres db's values!
15
+ def to_bool(value)
16
+ value == 't' || value == true
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,22 @@
1
+ module Quandl
2
+ module Elasticsearch
3
+ ## Database
4
+ class Database < Base
5
+ def data
6
+ @data ||= { name: @record['name'],
7
+ code: @record['code'],
8
+ updated_at: @record['updated_at'],
9
+ description: @record['description'],
10
+ documentation: @record['documentation'],
11
+ premium: to_bool(@record['premium']),
12
+ hidden: to_bool(@record['hidden']),
13
+ exclusive: to_bool(@record['exclusive']),
14
+ tags: @record['elasticsearch_flags'] ? @record['elasticsearch_flags'].split(',') : [] }
15
+ end
16
+
17
+ def remove?
18
+ !@record['deleted_at'].nil? || !@record['type'].nil?
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,51 @@
1
+ require 'set'
2
+
3
+ module Quandl
4
+ module Elasticsearch
5
+ ## Database
6
+ class Dataset < Base
7
+ def database_id
8
+ @database_id ||= @record['database_id']
9
+ end
10
+
11
+ def data
12
+ @data ||= { code: @record['code'],
13
+ name: @record['name'],
14
+ to_date: @record['to_date'],
15
+ is_private: to_bool(@record['is_private']),
16
+ frequency: @record['frequency'],
17
+ db_hidden: to_bool(@record['db_hidden']),
18
+ db_exclusive: to_bool(@record['db_exclusive']),
19
+ sample: previewable? }
20
+ end
21
+
22
+ def remove?
23
+ (@record['type'] && @record['type'] == 'Superset') || to_bool(@record['is_private']) || !@record['deleted_at'].nil?
24
+ end
25
+
26
+ private
27
+
28
+ def previewable_datasets
29
+ @_preview_datasets ||= @record['db_preview_datasets'].split(',').to_set
30
+ end
31
+
32
+ def preview_enabled?
33
+ to_bool(@record['db_preview_enabled'])
34
+ end
35
+
36
+ def premium?
37
+ to_bool(@record['db_premium'])
38
+ end
39
+
40
+ def previewable?
41
+ if !premium? || !preview_enabled?
42
+ false
43
+ elsif previewable_datasets.empty?
44
+ true
45
+ else
46
+ previewable_datasets.include?(@record['code'])
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,96 @@
1
+ require 'quandl/elasticsearch/base'
2
+ require 'quandl/elasticsearch/template'
3
+ require 'quandl/elasticsearch/database'
4
+ require 'quandl/elasticsearch/dataset'
5
+ require 'elasticsearch'
6
+ require 'json'
7
+
8
+ module Quandl
9
+ module Elasticsearch
10
+ ## Indice
11
+ class Indice
12
+ def initialize
13
+ @client = Quandl::Elasticsearch.client
14
+ end
15
+
16
+ def create_main(index)
17
+ @client.indices.create index: index.to_s, body: Quandl::Elasticsearch::Template.main_index
18
+ end
19
+
20
+ def build_database(index, databases)
21
+ dbs = []
22
+ databases.each { |d| dbs << Database.new(d) }
23
+ database_bulk(index, dbs)
24
+ end
25
+
26
+ def build_dataset(index, datasets)
27
+ ds = []
28
+ datasets.each { |d| ds << Dataset.new(d) }
29
+ dataset_bulk(index, ds)
30
+ end
31
+
32
+ def databases_empty?(index)
33
+ result = @client.count index: index.to_s, type: 'database'
34
+ result['count'].to_i == 0
35
+ end
36
+
37
+ def database_exist?(database_id, index)
38
+ @client.exists? index: index.to_s, type: 'database', id: database_id.to_s
39
+ end
40
+
41
+ def get_database(database_id, index)
42
+ @client.get index: index.to_s, type: 'database', id: database_id.to_s
43
+ end
44
+
45
+ def aliases(from, _to)
46
+ @client.indices.update_aliases body: { actions: [{ add: { index: from.to_s, alias: '_to' } }] }
47
+ end
48
+
49
+ def swap(old_index, new_index, index)
50
+ @client.indices.update_aliases body: { actions: [{ remove: { index: old_index.to_s, alias: index.to_s } },
51
+ { add: { index: new_index.to_s, alias: index.to_s } }] }
52
+ end
53
+
54
+ def delete(index)
55
+ @client.indices.delete index: [index.to_s]
56
+ end
57
+
58
+ def show_settings
59
+ @client.indices.get_settings
60
+ end
61
+
62
+ def status(index)
63
+ @client.indices.stats index: index.to_s
64
+ end
65
+
66
+ private
67
+
68
+ def database_bulk(index, dbs)
69
+ body = []
70
+ dbs.each do |db|
71
+ action = if db.remove?
72
+ { delete: { _index: index.to_s, _type: 'database', _id: db.id } }
73
+ else
74
+ { index: { _index: index.to_s, _type: 'database', _id: db.id, data: db.data } }
75
+ end
76
+
77
+ body << action
78
+ end
79
+ @client.bulk body: body
80
+ end
81
+
82
+ def dataset_bulk(index, datasets)
83
+ body = []
84
+ datasets.each do |d|
85
+ action = if d.remove?
86
+ { delete: { _index: index.to_s, _type: 'dataset', _id: d.id, parent: d.database_id } }
87
+ else
88
+ { index: { _index: index.to_s, _type: 'dataset', _id: d.id, parent: d.database_id, data: d.data } }
89
+ end
90
+ body << action
91
+ end
92
+ @client.bulk body: body
93
+ end
94
+ end
95
+ end
96
+ end