quandl-elasticsearch 2.1.0.rc5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +20 -0
  3. data/.rubocop.yml +34 -0
  4. data/COMMANDS.md +29 -0
  5. data/Gemfile +10 -0
  6. data/Gemfile.lock +155 -0
  7. data/LICENSE.txt +22 -0
  8. data/README.md +50 -0
  9. data/Rakefile +1 -0
  10. data/config/elasticsearch.yml +32 -0
  11. data/elasticsearch/elasticsearch.yml +386 -0
  12. data/elasticsearch/logging.yml +56 -0
  13. data/elasticsearch/stopwords/english.txt +38 -0
  14. data/elasticsearch/synonyms/synonyms_english.txt +318 -0
  15. data/fixtures/vcr_cassettes/search_spec_database_1.yml +38 -0
  16. data/fixtures/vcr_cassettes/search_spec_database_2.yml +38 -0
  17. data/fixtures/vcr_cassettes/search_spec_dataset_1.yml +48 -0
  18. data/fixtures/vcr_cassettes/search_spec_dataset_2.yml +41 -0
  19. data/fixtures/vcr_cassettes/setup.yml +139 -0
  20. data/lib/quandl/elasticsearch.rb +61 -0
  21. data/lib/quandl/elasticsearch/base.rb +20 -0
  22. data/lib/quandl/elasticsearch/database.rb +22 -0
  23. data/lib/quandl/elasticsearch/dataset.rb +51 -0
  24. data/lib/quandl/elasticsearch/indice.rb +96 -0
  25. data/lib/quandl/elasticsearch/query.rb +282 -0
  26. data/lib/quandl/elasticsearch/search.rb +150 -0
  27. data/lib/quandl/elasticsearch/tag.rb +21 -0
  28. data/lib/quandl/elasticsearch/template.rb +189 -0
  29. data/lib/quandl/elasticsearch/utility.rb +6 -0
  30. data/lib/quandl/elasticsearch/version.rb +6 -0
  31. data/quandl +77 -0
  32. data/quandl-elasticsearch.gemspec +34 -0
  33. data/solano.yml +20 -0
  34. data/spec/lib/quandl/elasticsearch/database_spec.rb +98 -0
  35. data/spec/lib/quandl/elasticsearch/dataset_spec.rb +124 -0
  36. data/spec/lib/quandl/elasticsearch/indice_spec.rb +10 -0
  37. data/spec/lib/quandl/elasticsearch/query_spec.rb +239 -0
  38. data/spec/lib/quandl/elasticsearch/search_spec.rb +83 -0
  39. data/spec/lib/quandl/elasticsearch/template_spec.rb +182 -0
  40. data/spec/lib/quandl/elasticsearch/utility_spec.rb +10 -0
  41. data/spec/lib/quandl/elasticsearch_spec.rb +99 -0
  42. data/spec/spec_helper.rb +27 -0
  43. data/templates/database_mapping.json +11 -0
  44. data/templates/dataset_mapping.json +9 -0
  45. data/templates/quandl_delimiter.json +0 -0
  46. data/templates/search_term_mapping.json +13 -0
  47. data/tests/Database-Ratings.csv +405 -0
  48. data/tests/Database-Tags.csv +341 -0
  49. data/tests/compare.csv +1431 -0
  50. data/tests/compare.rb +33 -0
  51. data/tests/console.rb +4 -0
  52. data/tests/generated_db_tags.csv +341 -0
  53. data/tests/search.rb +14 -0
  54. data/tests/search_db_mapping.txt +402 -0
  55. data/tests/status.rb +2 -0
  56. data/tests/test_search.csv +87 -0
  57. data/tests/test_search.rb +113 -0
  58. data/tests/testing-list.txt +183 -0
  59. data/tests/top500searches.csv +477 -0
  60. metadata +300 -0
@@ -0,0 +1,38 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://staging-search.quandl.com:9200//quandl_index/database/_search
6
+ body:
7
+ encoding: UTF-8
8
+ string: ! '{"explain":false,"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1","description","documentation^0.8"],"query":"stocks","type":"phrase","operator":"and","zero_terms_query":"all","slop":2}},{"has_child":{"type":"dataset","score_mode":"max","query":{"bool":{"should":[{"multi_match":{"fields":["name^1.1"],"query":"stocks","type":"phrase","slop":10}},{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}}}},{"prefix":{"code":"stocks"}}]}},"filter":{"bool":{"must":[{"term":{"premium":true}}]}}}},"from":0,"size":10,"rescore":{"window_size":10,"query":{"rescore_query":{"match":{"name":{"query":"stocks","type":"phrase","slop":10}}},"query_weight":0.7,"rescore_query_weight":1.2}}}'
9
+ headers:
10
+ User-Agent:
11
+ - Faraday v0.9.1
12
+ Accept:
13
+ - ! '*/*'
14
+ Date:
15
+ - Fri, 22 May 2015 16:05:37 GMT
16
+ Content-Type:
17
+ - application/x-www-form-urlencoded
18
+ response:
19
+ status:
20
+ code: 200
21
+ message: !binary |-
22
+ T0s=
23
+ headers:
24
+ !binary "Q29udGVudC1UeXBl":
25
+ - !binary |-
26
+ YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
27
+ !binary "Q29udGVudC1MZW5ndGg=":
28
+ - !binary |-
29
+ MTE5Mw==
30
+ !binary "Q29ubmVjdGlvbg==":
31
+ - !binary |-
32
+ a2VlcC1hbGl2ZQ==
33
+ body:
34
+ encoding: US-ASCII
35
+ string: ! '{"took":769,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":18,"max_score":2.0353467,"hits":[{"_index":"quandl_main_v6","_type":"database","_id":"12910","_score":2.7696624,"fields":{"code":["EOD"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13091","_score":1.0613412,"fields":{"code":["ZFA"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13122","_score":1.0613412,"fields":{"code":["ZFB"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13065","_score":0.5353001,"fields":{"code":["ZSEE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12266","_score":0.504785,"fields":{"code":["ZEE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"13000","_score":0.49280426,"fields":{"code":["ZET"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12985","_score":0.461574,"fields":{"code":["ZAR"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12908","_score":0.46129575,"fields":{"code":["SF1"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12973","_score":0.43754363,"fields":{"code":["ZDIV"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12991","_score":0.37978202,"fields":{"code":["ZSS"]}}]}}'
36
+ http_version:
37
+ recorded_at: Fri, 22 May 2015 16:05:38 GMT
38
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,48 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://staging-search.quandl.com:9200//quandl_index/dataset/_search
6
+ body:
7
+ encoding: UTF-8
8
+ string: ! '{"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","minimum_should_match":1,"type":"best_fields","operator":"and"}},{"prefix":{"code":"stocks"}}]}},"filter":{"bool":{"must":[{"term":{"_parent":10471}}],"should":[{"term":{"frequency":"daily"}}]}}}},"sort":[{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"from":0,"size":10}'
9
+ headers:
10
+ User-Agent:
11
+ - Faraday v0.9.1
12
+ Accept:
13
+ - ! '*/*'
14
+ Date:
15
+ - Fri, 22 May 2015 15:57:50 GMT
16
+ Content-Type:
17
+ - application/x-www-form-urlencoded
18
+ response:
19
+ status:
20
+ code: 200
21
+ message: !binary |-
22
+ T0s=
23
+ headers:
24
+ !binary "Q29udGVudC1UeXBl":
25
+ - !binary |-
26
+ YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
27
+ !binary "Q29udGVudC1MZW5ndGg=":
28
+ - !binary |-
29
+ MjM5Mw==
30
+ !binary "Q29ubmVjdGlvbg==":
31
+ - !binary |-
32
+ a2VlcC1hbGl2ZQ==
33
+ body:
34
+ encoding: US-ASCII
35
+ string: ! '{"took":6,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":30,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"13147174","_score":null,"fields":{"_parent":"10471","code":["OMXIGI"]},"highlight":{"name":["OMX
36
+ Iceland All-<em>Share</em> GI (OMXIGI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147176","_score":null,"fields":{"_parent":"10471","code":["OMXIPI"]},"highlight":{"name":["OMX
37
+ Iceland All-<em>Share</em> PI (OMXIPI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147219","_score":null,"fields":{"_parent":"10471","code":["OMXSCAPGI"]},"highlight":{"name":["OMX
38
+ Stockholm All-<em>Share</em> Cap_GI (OMXSCAPGI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147220","_score":null,"fields":{"_parent":"10471","code":["OMXSCAPPI"]},"highlight":{"name":["OMX
39
+ Stockholm All-<em>Share</em> Cap_PI (OMXSCAPPI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147404","_score":null,"fields":{"_parent":"10471","code":["OMXIFOPI"]},"highlight":{"name":["OMXI-FO
40
+ All <em>Share</em> PI (OMXIFOPI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13147405","_score":null,"fields":{"_parent":"10471","code":["OMXIFOGI"]},"highlight":{"name":["OMXI-FO
41
+ All <em>Share</em> GI (OMXIFOGI)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13127717","_score":null,"fields":{"_parent":"10471","code":["NQCAPFDT"]},"highlight":{"name":["NASDAQ
42
+ Select Canadian Preferred <em>Share</em> Index TR (NQCAPFDT)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13127718","_score":null,"fields":{"_parent":"10471","code":["NQCAPFD"]},"highlight":{"name":["NASDAQ
43
+ Select Canadian Preferred <em>Share</em> Index (NQCAPFD)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13961773","_score":null,"fields":{"_parent":"10471","code":["BSCP03"]},"highlight":{"name":["NASDAQ
44
+ Bullet<em>Shares</em> Corporate Debt 0-3 Ladder (BSCP03)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13961774","_score":null,"fields":{"_parent":"10471","code":["BSHY03T"]},"highlight":{"name":["NASDAQ
45
+ Bullet<em>Shares</em> High Yield Debt 0-3 Ladder TR (BSHY03T)"]},"sort":[1431648000000]}]}}'
46
+ http_version:
47
+ recorded_at: Fri, 22 May 2015 15:57:50 GMT
48
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,41 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://staging-search.quandl.com:9200//quandl_index/dataset/_search
6
+ body:
7
+ encoding: UTF-8
8
+ string: ! '{"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"mexico","minimum_should_match":1,"type":"best_fields","operator":"and"}},{"prefix":{"code":"mexico"}}]}},"filter":{"bool":{"must":[{"term":{"_parent":10471}}]}}}},"sort":[{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"from":0,"size":3}'
9
+ headers:
10
+ User-Agent:
11
+ - Faraday v0.9.1
12
+ Accept:
13
+ - ! '*/*'
14
+ Date:
15
+ - Fri, 22 May 2015 16:05:38 GMT
16
+ Content-Type:
17
+ - application/x-www-form-urlencoded
18
+ response:
19
+ status:
20
+ code: 200
21
+ message: !binary |-
22
+ T0s=
23
+ headers:
24
+ !binary "Q29udGVudC1UeXBl":
25
+ - !binary |-
26
+ YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
27
+ !binary "Q29udGVudC1MZW5ndGg=":
28
+ - !binary |-
29
+ Nzk4
30
+ !binary "Q29ubmVjdGlvbg==":
31
+ - !binary |-
32
+ a2VlcC1hbGl2ZQ==
33
+ body:
34
+ encoding: US-ASCII
35
+ string: ! '{"took":38,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":315,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"13124809","_score":null,"fields":{"_parent":"10471","code":["NQMX1000AUD"]},"highlight":{"name":["NASDAQ
36
+ <em>Mexico</em> Basic Matls AUD Index (NQMX1000AUD)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13124810","_score":null,"fields":{"_parent":"10471","code":["NQMXAUD"]},"highlight":{"name":["NASDAQ
37
+ <em>Mexico</em> AUD Index (NQMXAUD)"]},"sort":[1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"13124813","_score":null,"fields":{"_parent":"10471","code":["NQMXAUDN"]},"highlight":{"name":["NASDAQ
38
+ <em>Mexico</em> AUD NTR Index (NQMXAUDN)"]},"sort":[1431648000000]}]}}'
39
+ http_version:
40
+ recorded_at: Fri, 22 May 2015 16:05:38 GMT
41
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,139 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://staging-search.quandl.com:9200//quandl_index/database/_search
6
+ body:
7
+ encoding: UTF-8
8
+ string: ! '{"explain":false,"fields":["_parent","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1","description","documentation^0.8"],"query":"stocks","type":"phrase","operator":"and","zero_terms_query":"all","slop":2}},{"has_child":{"type":"dataset","score_mode":"max","query":{"bool":{"should":[{"multi_match":{"fields":["name^1.1"],"query":"stocks","type":"phrase","slop":10}},{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}}}},{"prefix":{"code":"stocks"}}]}}}},"from":0,"size":10,"rescore":{"window_size":10,"query":{"rescore_query":{"match":{"name":{"query":"stocks","type":"phrase","slop":10}}},"query_weight":0.7,"rescore_query_weight":1.2}}}'
9
+ headers:
10
+ User-Agent:
11
+ - Faraday v0.9.1
12
+ Accept:
13
+ - ! '*/*'
14
+ Date:
15
+ - Fri, 22 May 2015 15:57:50 GMT
16
+ Content-Type:
17
+ - application/x-www-form-urlencoded
18
+ response:
19
+ status:
20
+ code: 200
21
+ message: !binary |-
22
+ T0s=
23
+ headers:
24
+ !binary "Q29udGVudC1UeXBl":
25
+ - !binary |-
26
+ YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
27
+ !binary "Q29udGVudC1MZW5ndGg=":
28
+ - !binary |-
29
+ MTIwNg==
30
+ !binary "Q29ubmVjdGlvbg==":
31
+ - !binary |-
32
+ a2VlcC1hbGl2ZQ==
33
+ body:
34
+ encoding: US-ASCII
35
+ string: ! '{"took":765,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":148,"max_score":2.7896662,"hits":[{"_index":"quandl_main_v6","_type":"database","_id":"33","_score":3.7459924,"fields":{"code":["NSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"460","_score":3.4402916,"fields":{"code":["LUXSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"442","_score":3.391498,"fields":{"code":["ZAGREBSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"462","_score":3.381187,"fields":{"code":["BUCHARESTSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"5628","_score":3.2165365,"fields":{"code":["EURONEXT"]}},{"_index":"quandl_main_v6","_type":"database","_id":"450","_score":3.0062313,"fields":{"code":["PHILSE"]}},{"_index":"quandl_main_v6","_type":"database","_id":"12910","_score":2.7696624,"fields":{"code":["EOD"]}},{"_index":"quandl_main_v6","_type":"database","_id":"1383","_score":1.5982721,"fields":{"code":["SIX"]}},{"_index":"quandl_main_v6","_type":"database","_id":"464","_score":1.5879608,"fields":{"code":["NIKKEI"]}},{"_index":"quandl_main_v6","_type":"database","_id":"8401","_score":1.2901201,"fields":{"code":["CRYPTOCHART"]}}]}}'
36
+ http_version:
37
+ recorded_at: Fri, 22 May 2015 15:57:51 GMT
38
+ - request:
39
+ method: get
40
+ uri: http://staging-search.quandl.com:9200//_msearch
41
+ body:
42
+ encoding: UTF-8
43
+ string: ! '{"index":"quandl_index","type":"dataset"}
44
+
45
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"33"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
46
+
47
+ {"index":"quandl_index","type":"dataset"}
48
+
49
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"460"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
50
+
51
+ {"index":"quandl_index","type":"dataset"}
52
+
53
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"442"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
54
+
55
+ {"index":"quandl_index","type":"dataset"}
56
+
57
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"462"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
58
+
59
+ {"index":"quandl_index","type":"dataset"}
60
+
61
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"5628"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
62
+
63
+ {"index":"quandl_index","type":"dataset"}
64
+
65
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"450"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
66
+
67
+ {"index":"quandl_index","type":"dataset"}
68
+
69
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"12910"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
70
+
71
+ {"index":"quandl_index","type":"dataset"}
72
+
73
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"1383"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
74
+
75
+ {"index":"quandl_index","type":"dataset"}
76
+
77
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"464"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
78
+
79
+ {"index":"quandl_index","type":"dataset"}
80
+
81
+ {"fields":["_parent","_id","code"],"query":{"filtered":{"query":{"bool":{"should":[{"multi_match":{"fields":["code","name^1.1"],"query":"stocks","type":"best_fields"}},{"prefix":{"code":"stocks"}}]}},"filter":{"term":{"_parent":"8401"}}}},"sort":["_score",{"to_date":{"order":"desc"}}],"highlight":{"fields":{"name":{}}},"size":3}
82
+
83
+ '
84
+ headers:
85
+ User-Agent:
86
+ - Faraday v0.9.1
87
+ Accept:
88
+ - ! '*/*'
89
+ Date:
90
+ - Fri, 22 May 2015 15:57:51 GMT
91
+ Content-Type:
92
+ - application/x-www-form-urlencoded
93
+ response:
94
+ status:
95
+ code: 200
96
+ message: !binary |-
97
+ T0s=
98
+ headers:
99
+ !binary "Q29udGVudC1UeXBl":
100
+ - !binary |-
101
+ YXBwbGljYXRpb24vanNvbjsgY2hhcnNldD1VVEYtOA==
102
+ !binary "Q29udGVudC1MZW5ndGg=":
103
+ - !binary |-
104
+ NzQzMA==
105
+ !binary "Q29ubmVjdGlvbg==":
106
+ - !binary |-
107
+ a2VlcC1hbGl2ZQ==
108
+ body:
109
+ encoding: US-ASCII
110
+ string: ! '{"responses":[{"took":20,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":82,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2449180","_score":0.15413277,"fields":{"_parent":"33","code":["CNXIT"]},"highlight":{"name":["National
111
+ <em>Stock</em> Exchange of India <em>Stock</em> Index - CNX IT"]},"sort":[0.15413277,1410480000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"12849054","_score":0.12844397,"fields":{"_parent":"33","code":["LIX_15"]},"highlight":{"name":["National
112
+ <em>Stock</em> Exchange of India (NSE) <em>Stock</em> Index - LIX 15"]},"sort":[0.12844397,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"12849055","_score":0.12844397,"fields":{"_parent":"33","code":["LIX15_MIDCAP"]},"highlight":{"name":["National
113
+ <em>Stock</em> Exchange of India (NSE) <em>Stock</em> Index - LIX15 Midcap"]},"sort":[0.12844397,1431648000000]}]}},{"took":4,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":4,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2424381","_score":0.10898833,"fields":{"_parent":"460","code":["LUXXR"]},"highlight":{"name":["Luxembourg
114
+ <em>Stock</em> Index: LuxX Returns"]},"sort":[0.10898833,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2424382","_score":0.10898833,"fields":{"_parent":"460","code":["LUXXC"]},"highlight":{"name":["Luxembourg
115
+ <em>Stock</em> Index: LuxX Cours"]},"sort":[0.10898833,1417478400000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2424383","_score":0.10898833,"fields":{"_parent":"460","code":["LUXGC"]},"highlight":{"name":["Luxembourg
116
+ <em>Stock</em> Index: Lux General Cours"]},"sort":[0.10898833,1410220800000]}]}},{"took":20,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":4,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2422954","_score":0.15413277,"fields":{"_parent":"442","code":["CROBEX"]},"highlight":{"name":["Zagreb
117
+ <em>Stock</em> Exchange <em>Stock</em> Index: CROBEX"]},"sort":[0.15413277,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422955","_score":0.15413277,"fields":{"_parent":"442","code":["CROBEX10"]},"highlight":{"name":["Zagreb
118
+ <em>Stock</em> Exchange <em>Stock</em> Index: CROBEX10"]},"sort":[0.15413277,1361232000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422956","_score":0.10898833,"fields":{"_parent":"442","code":["CROBIS"]},"highlight":{"name":["Zagreb
119
+ <em>Stock</em> Exchange Bond Index: CROBIS"]},"sort":[0.10898833,1361145600000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2424385","_score":0.14531778,"fields":{"_parent":"462","code":["INDICES"]},"highlight":{"name":["Bucharest
120
+ <em>Stock</em> Exchange Indices"]},"sort":[0.14531778,1431648000000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"9820158","_score":0.14531778,"fields":{"_parent":"5628","code":["LSEL"]},"highlight":{"name":["London
121
+ <em>Stock</em> Exch (LSEL)"]},"sort":[0.14531778,1377734400000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":8,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"2422989","_score":0.17982157,"fields":{"_parent":"450","code":["ALL"]},"highlight":{"name":["Philippine
122
+ <em>Stock</em> Index: All <em>Shares</em>"]},"sort":[0.17982157,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422988","_score":0.14531778,"fields":{"_parent":"450","code":["PSE"]},"highlight":{"name":["Philippine
123
+ <em>Stock</em> Index: PSEi"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"2422990","_score":0.14531778,"fields":{"_parent":"450","code":["FIN"]},"highlight":{"name":["Philippine
124
+ <em>Stock</em> Index: Financials"]},"sort":[0.14531778,1431648000000]}]}},{"took":17,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":8641,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"14031653","_score":0.15731111,"fields":{"_parent":"12910","code":["IPFF"]},"highlight":{"name":["i<em>Shares</em>
125
+ International Preferred <em>Stock</em> (IPFF) <em>Stock</em> Prices, Dividends
126
+ and Splits"]},"sort":[0.15731111,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"14028742","_score":0.15731111,"fields":{"_parent":"12910","code":["CHNA"]},"highlight":{"name":["Power<em>Shares</em>
127
+ China A-<em>Share</em> ETF (CHNA) <em>Stock</em> Prices, Dividends and Splits"]},"sort":[0.15731111,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"14026179","_score":0.15413277,"fields":{"_parent":"12910","code":["EQC"]},"highlight":{"name":["<em>Equity</em>
128
+ Commonwealth (EQC) <em>Stock</em> Prices, Dividends and Splits"]},"sort":[0.15413277,1431648000000]}]}},{"took":16,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":292,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"8536116","_score":0.15413277,"fields":{"_parent":"1383","code":["CH0006089921CHF4"]},"highlight":{"name":["PRIVATE
129
+ <em>EQUITY</em> N - <em>Share</em> Price CHF"]},"sort":[0.15413277,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8535935","_score":0.12715305,"fields":{"_parent":"1383","code":["CH0014424524USD4"]},"highlight":{"name":["ALTIN
130
+ N - <em>Share</em> Price USD"]},"sort":[0.12715305,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8535984","_score":0.12715305,"fields":{"_parent":"1383","code":["US1491231015CHF4"]},"highlight":{"name":["CATERPILLAR
131
+ - <em>Share</em> Price CHF"]},"sort":[0.12715305,1431648000000]}]}},{"took":16,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":7,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"8473284","_score":0.14531778,"fields":{"_parent":"464","code":["ALL_STOCK"]},"highlight":{"name":["Nikkei
132
+ All <em>Stock</em> Index"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8471661","_score":0.14531778,"fields":{"_parent":"464","code":["SI300"]},"highlight":{"name":["Nikkei
133
+ <em>Stock</em> Index 300"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"8472417","_score":0.12715305,"fields":{"_parent":"464","code":["500SA"]},"highlight":{"name":["Nikkei
134
+ 500 <em>Stock</em> Average Index"]},"sort":[0.12715305,1431648000000]}]}},{"took":19,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":28,"max_score":null,"hits":[{"_index":"quandl_main_v6","_type":"dataset","_id":"19686015","_score":8.422265,"fields":{"_parent":"8401","code":["STOCK"]},"sort":[8.422265,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"11320655","_score":0.14531778,"fields":{"_parent":"8401","code":["NRS"]},"highlight":{"name":["Noir<em>Shares</em>/BITCOIN
135
+ Price"]},"sort":[0.14531778,1431648000000]},{"_index":"quandl_main_v6","_type":"dataset","_id":"11320692","_score":0.14531778,"fields":{"_parent":"8401","code":["PTS"]},"highlight":{"name":["Proto<em>Shares</em>/BITCOIN
136
+ Price"]},"sort":[0.14531778,1431648000000]}]}}]}'
137
+ http_version:
138
+ recorded_at: Fri, 22 May 2015 15:57:51 GMT
139
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,61 @@
1
+ require 'elasticsearch'
2
+ require 'yaml'
3
+ require 'quandl/config'
4
+ require 'quandl/elasticsearch/version'
5
+ require 'quandl/elasticsearch/search'
6
+ require 'quandl/elasticsearch/indice'
7
+ require 'quandl/elasticsearch/utility'
8
+
9
+ module Quandl
10
+ module Elasticsearch
11
+ extend Quandl::Configurable
12
+
13
+ class << self
14
+ def file_name
15
+ 'elasticsearch'
16
+ end
17
+
18
+ def configure
19
+ yield(configuration)
20
+ end
21
+
22
+ def client(host = nil)
23
+ client_for_host(host || configuration.host)
24
+ end
25
+
26
+ def disconnect!
27
+ @clients = {}
28
+ end
29
+
30
+ # Helper method for testing to reset block configuration and reload file
31
+ def reset_configuration!
32
+ Quandl::Config.clear_internal_cache
33
+ @configuration = Quandl::Config.new(file_name)
34
+ end
35
+
36
+ private
37
+
38
+ def client_for_host(host)
39
+ return clients[host] if clients.key?(host)
40
+ clients[host] = ::Elasticsearch::Client.new(host: host,
41
+ transport_options: {
42
+ request: {
43
+ timeout: configuration.timeout,
44
+ open_timeout: configuration.open_timeout
45
+ }
46
+ })
47
+ end
48
+
49
+ def clients
50
+ @clients ||= {}
51
+ end
52
+ end
53
+
54
+ # Backwards compatability
55
+ class Configuration
56
+ def self.client
57
+ Quandl::Elasticsearch.client
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,20 @@
1
+ module Quandl
2
+ module Elasticsearch
3
+ class Base
4
+ def initialize(record)
5
+ @record = record
6
+ end
7
+
8
+ def id
9
+ @id ||= @record['id'].to_i
10
+ end
11
+
12
+ protected
13
+
14
+ # this pertains to the postgres db's values!
15
+ def to_bool(value)
16
+ value == 't' || value == true
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,22 @@
1
+ module Quandl
2
+ module Elasticsearch
3
+ ## Database
4
+ class Database < Base
5
+ def data
6
+ @data ||= { name: @record['name'],
7
+ code: @record['code'],
8
+ updated_at: @record['updated_at'],
9
+ description: @record['description'],
10
+ documentation: @record['documentation'],
11
+ premium: to_bool(@record['premium']),
12
+ hidden: to_bool(@record['hidden']),
13
+ exclusive: to_bool(@record['exclusive']),
14
+ tags: @record['elasticsearch_flags'] ? @record['elasticsearch_flags'].split(',') : [] }
15
+ end
16
+
17
+ def remove?
18
+ !@record['deleted_at'].nil? || !@record['type'].nil?
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,51 @@
1
+ require 'set'
2
+
3
+ module Quandl
4
+ module Elasticsearch
5
+ ## Database
6
+ class Dataset < Base
7
+ def database_id
8
+ @database_id ||= @record['database_id']
9
+ end
10
+
11
+ def data
12
+ @data ||= { code: @record['code'],
13
+ name: @record['name'],
14
+ to_date: @record['to_date'],
15
+ is_private: to_bool(@record['is_private']),
16
+ frequency: @record['frequency'],
17
+ db_hidden: to_bool(@record['db_hidden']),
18
+ db_exclusive: to_bool(@record['db_exclusive']),
19
+ sample: previewable? }
20
+ end
21
+
22
+ def remove?
23
+ (@record['type'] && @record['type'] == 'Superset') || to_bool(@record['is_private']) || !@record['deleted_at'].nil?
24
+ end
25
+
26
+ private
27
+
28
+ def previewable_datasets
29
+ @_preview_datasets ||= @record['db_preview_datasets'].split(',').to_set
30
+ end
31
+
32
+ def preview_enabled?
33
+ to_bool(@record['db_preview_enabled'])
34
+ end
35
+
36
+ def premium?
37
+ to_bool(@record['db_premium'])
38
+ end
39
+
40
+ def previewable?
41
+ if !premium? || !preview_enabled?
42
+ false
43
+ elsif previewable_datasets.empty?
44
+ true
45
+ else
46
+ previewable_datasets.include?(@record['code'])
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,96 @@
1
+ require 'quandl/elasticsearch/base'
2
+ require 'quandl/elasticsearch/template'
3
+ require 'quandl/elasticsearch/database'
4
+ require 'quandl/elasticsearch/dataset'
5
+ require 'elasticsearch'
6
+ require 'json'
7
+
8
+ module Quandl
9
+ module Elasticsearch
10
+ ## Indice
11
+ class Indice
12
+ def initialize
13
+ @client = Quandl::Elasticsearch.client
14
+ end
15
+
16
+ def create_main(index)
17
+ @client.indices.create index: index.to_s, body: Quandl::Elasticsearch::Template.main_index
18
+ end
19
+
20
+ def build_database(index, databases)
21
+ dbs = []
22
+ databases.each { |d| dbs << Database.new(d) }
23
+ database_bulk(index, dbs)
24
+ end
25
+
26
+ def build_dataset(index, datasets)
27
+ ds = []
28
+ datasets.each { |d| ds << Dataset.new(d) }
29
+ dataset_bulk(index, ds)
30
+ end
31
+
32
+ def databases_empty?(index)
33
+ result = @client.count index: index.to_s, type: 'database'
34
+ result['count'].to_i == 0
35
+ end
36
+
37
+ def database_exist?(database_id, index)
38
+ @client.exists? index: index.to_s, type: 'database', id: database_id.to_s
39
+ end
40
+
41
+ def get_database(database_id, index)
42
+ @client.get index: index.to_s, type: 'database', id: database_id.to_s
43
+ end
44
+
45
+ def aliases(from, _to)
46
+ @client.indices.update_aliases body: { actions: [{ add: { index: from.to_s, alias: '_to' } }] }
47
+ end
48
+
49
+ def swap(old_index, new_index, index)
50
+ @client.indices.update_aliases body: { actions: [{ remove: { index: old_index.to_s, alias: index.to_s } },
51
+ { add: { index: new_index.to_s, alias: index.to_s } }] }
52
+ end
53
+
54
+ def delete(index)
55
+ @client.indices.delete index: [index.to_s]
56
+ end
57
+
58
+ def show_settings
59
+ @client.indices.get_settings
60
+ end
61
+
62
+ def status(index)
63
+ @client.indices.stats index: index.to_s
64
+ end
65
+
66
+ private
67
+
68
+ def database_bulk(index, dbs)
69
+ body = []
70
+ dbs.each do |db|
71
+ action = if db.remove?
72
+ { delete: { _index: index.to_s, _type: 'database', _id: db.id } }
73
+ else
74
+ { index: { _index: index.to_s, _type: 'database', _id: db.id, data: db.data } }
75
+ end
76
+
77
+ body << action
78
+ end
79
+ @client.bulk body: body
80
+ end
81
+
82
+ def dataset_bulk(index, datasets)
83
+ body = []
84
+ datasets.each do |d|
85
+ action = if d.remove?
86
+ { delete: { _index: index.to_s, _type: 'dataset', _id: d.id, parent: d.database_id } }
87
+ else
88
+ { index: { _index: index.to_s, _type: 'dataset', _id: d.id, parent: d.database_id, data: d.data } }
89
+ end
90
+ body << action
91
+ end
92
+ @client.bulk body: body
93
+ end
94
+ end
95
+ end
96
+ end