quandl 0.2.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +15 -0
  2. data/.gitignore +16 -0
  3. data/.travis.yml +20 -0
  4. data/Gemfile +14 -0
  5. data/Guardfile +8 -0
  6. data/LICENSE +7 -0
  7. data/README.md +316 -0
  8. data/Rakefile +39 -0
  9. data/UPGRADE.md +143 -0
  10. data/bin/quandl +26 -0
  11. data/dist/resources/pkg/Distribution.erb +15 -0
  12. data/dist/resources/pkg/PackageInfo.erb +6 -0
  13. data/dist/resources/pkg/postinstall +45 -0
  14. data/dist/resources/pkg/quandl +25 -0
  15. data/dist/resources/ruby/PackageInfo +3 -0
  16. data/lib/commander/command/quandl_ext.rb +21 -0
  17. data/lib/quandl/command.rb +45 -0
  18. data/lib/quandl/command/client_ext.rb +1 -0
  19. data/lib/quandl/command/client_ext/user.rb +11 -0
  20. data/lib/quandl/command/compatibility_check.rb +11 -0
  21. data/lib/quandl/command/qconfig.rb +76 -0
  22. data/lib/quandl/command/tasks.rb +15 -0
  23. data/lib/quandl/command/tasks/base.rb +263 -0
  24. data/lib/quandl/command/tasks/delete.rb +58 -0
  25. data/lib/quandl/command/tasks/download.rb +111 -0
  26. data/lib/quandl/command/tasks/info.rb +46 -0
  27. data/lib/quandl/command/tasks/list.rb +40 -0
  28. data/lib/quandl/command/tasks/login.rb +46 -0
  29. data/lib/quandl/command/tasks/update.rb +205 -0
  30. data/lib/quandl/command/tasks/upload.rb +69 -0
  31. data/lib/quandl/command/version.rb +5 -0
  32. data/lib/quandl/utility.rb +2 -0
  33. data/lib/quandl/utility/config.rb +43 -0
  34. data/lib/quandl/utility/ruby_version.rb +143 -0
  35. data/quandl.gemspec +39 -0
  36. data/scripts/compile_ruby_pkg.sh +34 -0
  37. data/scripts/install.sh +51 -0
  38. data/scripts/win/quandl_toolbelt.iss +82 -0
  39. data/spec/lib/quandl/command/delete_spec.rb +31 -0
  40. data/spec/lib/quandl/command/download_spec.rb +42 -0
  41. data/spec/lib/quandl/command/upload_spec.rb +39 -0
  42. data/spec/lib/quandl/command_spec.rb +46 -0
  43. data/spec/spec_helper.rb +20 -0
  44. data/tasks/toolbelt.rake +133 -0
  45. data/tasks/toolbelt.rb +92 -0
  46. data/tasks/toolbelt/build.rb +2 -0
  47. data/tasks/toolbelt/build/darwin.rb +71 -0
  48. data/tasks/toolbelt/build/ruby.rb +105 -0
  49. data/tasks/toolbelt/build/tarball.rb +73 -0
  50. data/tasks/toolbelt/build/windows.rb +25 -0
  51. data/tasks/toolbelt/push.rb +15 -0
  52. data/tasks/toolbelt/storage.rb +28 -0
  53. data/tasks/toolbelt/tar.rb +21 -0
  54. metadata +354 -0
@@ -0,0 +1,15 @@
1
+ module Quandl
2
+ module Command
3
+ module Tasks
4
+
5
+ def self.root
6
+ @root ||= File.expand_path(File.join(File.dirname(__FILE__), '../../../'))
7
+ end
8
+
9
+ # require base task
10
+ require 'quandl/command/tasks/base'
11
+ # require tasks
12
+ Dir.glob(File.join(root, 'lib/quandl/command/tasks/*.rb')){|t| require(t) }
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,263 @@
1
+ autoload :OpenURI, 'open-uri'
2
+
3
+ module Quandl
4
+ module Command
5
+ module Tasks
6
+
7
+ class Base
8
+ class << self
9
+
10
+ def configure(app)
11
+ return if disabled?
12
+ app.command(command_name) do |c|
13
+ c.syntax = syntax
14
+ c.description = description
15
+ c.action{|a,o| call(a,o) }
16
+ configure_options(c)
17
+ end
18
+ end
19
+
20
+ def disabled?
21
+ @disabled == true
22
+ end
23
+
24
+ def disable!
25
+ @disabled = true
26
+ end
27
+
28
+ def syntax(value=nil)
29
+ @syntax = value if value.present?
30
+ @syntax ||= "quandl #{command_name}"
31
+ end
32
+
33
+ def command_name(value=nil)
34
+ @command_name = value if value.present?
35
+ @command_name ||= name.to_s.split("::").last.downcase
36
+ end
37
+
38
+ def description(value=nil)
39
+ @description = value if value.present?
40
+ @description ||= "No description."
41
+ end
42
+
43
+ def options(value=nil)
44
+ @options = value if value.present?
45
+ @options ||= {}
46
+ end
47
+
48
+ def authenticated_users_only!
49
+ before_execute :authenticated_users_only!
50
+ end
51
+
52
+ def warn_unauthenticated_users
53
+ before_execute :warn_unauthenticated_users
54
+ end
55
+
56
+ def call(args=[], options={})
57
+ args = Array(args)
58
+ options = ensure_options_are_command_options!(options)
59
+ self.new( args, options ).call
60
+ end
61
+
62
+ protected
63
+
64
+ def ensure_options_are_command_options!(options)
65
+ return options if options.class == Commander::Command::Options
66
+ OpenStruct.new(options)
67
+ end
68
+
69
+ def configure_options(c)
70
+ options.each do |class_type, options|
71
+ options.each do |name, desc|
72
+ c.option "--#{name} #{class_type.to_s.upcase}", class_type, desc
73
+ end
74
+ end
75
+ end
76
+
77
+ end
78
+
79
+ attr_accessor :args, :options, :request_timer
80
+
81
+ include ActiveModel::Validations
82
+
83
+ extend ActiveModel::Callbacks
84
+ define_model_callbacks :execute
85
+
86
+ before_execute :configure_client, :raise_error_unless_valid!, :check_for_update, :start_request_timer
87
+ after_execute :log_request_time
88
+
89
+ def call
90
+ run_callbacks(:execute) do
91
+ execute
92
+ end
93
+ end
94
+
95
+ def initialize(args, options)
96
+ self.args = args
97
+ self.options = options
98
+ end
99
+
100
+ def verbose?
101
+ options.verbose == true
102
+ end
103
+
104
+ def force_yes?
105
+ options.force_yes == true
106
+ end
107
+
108
+ def ask_yes_or_no
109
+ ['y','yes'].include?( ask("Are you sure? (y/n)") )
110
+ end
111
+
112
+ def summarize(item)
113
+ return summarize_hash(item) if item.kind_of?(Hash)
114
+ item
115
+ end
116
+
117
+ def summarize_hash(item)
118
+ item.collect do |k,v|
119
+ next "#{k}: '#{v}'" if v.kind_of?(String)
120
+ "#{k}: #{v}"
121
+ end.join(', ')
122
+ end
123
+
124
+ def table(*args)
125
+ Array(args).flatten.join(" | ")
126
+ end
127
+
128
+ def info(*args)
129
+ logger.info(*args)
130
+ end
131
+
132
+ def debug(*args)
133
+ logger.debug(*args) if verbose?
134
+ end
135
+
136
+ def error(*args)
137
+ logger.error(*args)
138
+ end
139
+
140
+ def fatal(*args)
141
+ logger.fatal("FATAL: #{args.join(" ")}")
142
+ end
143
+
144
+ def logger
145
+ Quandl::Logger
146
+ end
147
+
148
+ def current_user
149
+ @current_user ||= Quandl::Client::User.info
150
+ end
151
+
152
+ protected
153
+
154
+ # THREAD POOL
155
+
156
+ def mutex
157
+ @mutex ||= Mutex.new
158
+ end
159
+
160
+ def pool
161
+ @pool ||= Thread.pool( threads )
162
+ end
163
+
164
+ def threads
165
+ options.threads || 4
166
+ end
167
+
168
+
169
+ def start_request_timer
170
+ self.request_timer = Time.now
171
+ end
172
+
173
+ def log_request_time
174
+ debug("# Started: #{request_timer}. Finished: #{Time.now}. Elapsed: #{request_timer.elapsed_ms}")
175
+ end
176
+
177
+ def reload_session!
178
+ @auth_token = nil
179
+ @current_user = nil
180
+ configure_client
181
+ end
182
+
183
+ def authenticated_users_only!
184
+ if auth_token.blank?
185
+ fatal("You must authenticate to use #{self.class.command_name}! 'quandl login' OR --token xyz923")
186
+ false
187
+ end
188
+ end
189
+
190
+ def warn_unauthenticated_users
191
+ error("WARN: Authenticate your requests! 'quandl login' OR --token xyz923") if auth_token.blank?
192
+ end
193
+
194
+ def raise_error_unless_valid!
195
+ unless valid?
196
+ error( table(errors.full_messages) )
197
+ false
198
+ end
199
+ end
200
+
201
+ def configure_client
202
+ Quandl::Client.use( quandl_url )
203
+ Quandl::Client.token = auth_token
204
+ end
205
+
206
+ def check_for_update
207
+ check_time = QConfig.configuration.last_checked_for_update
208
+ # check time present?
209
+ if check_time.present? && check_time.is_a?(Time)
210
+ # has it been more than one day?
211
+ run_update_check if Time.now - 1.day > check_time || check_time > Time.now
212
+ else
213
+ run_update_check
214
+ end
215
+ end
216
+
217
+ def run_update_check
218
+ print("# Checking for updates ... ")
219
+
220
+ uri = URI.parse("https://raw.github.com/quandl/quandl_command/master/lib/quandl/command/version.rb")
221
+ http = Net::HTTP.new(uri.host, uri.port)
222
+ http.use_ssl = true
223
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
224
+
225
+ request = Net::HTTP::Get.new(uri.request_uri)
226
+
227
+ response = http.request(request)
228
+
229
+ master = response.body
230
+ master = master.split("\n").detect{|r| r =~ /VERSION/ }.split("'").last
231
+ if Quandl::Command::VERSION != master
232
+ info(" A new version of quandl toolbelt has been released. #{master}. Please run 'quandl update'")
233
+ else
234
+ info(" you are up to date! #{master}")
235
+ end
236
+ rescue => err
237
+ error("An unexpected error occured while checking for updates ... #{err}")
238
+ ensure
239
+ QConfig.configuration.last_checked_for_update = Time.now
240
+ end
241
+
242
+ def quandl_url
243
+ return @quandl_url if @quandl_url.present?
244
+ @quandl_url = options.url if options.try(:url).present?
245
+ @quandl_url = ENV['QUANDL_URL'] if @quandl_url.blank?
246
+ @quandl_url = QConfig.configuration.quandl_url if @quandl_url.blank?
247
+ @quandl_url = 'http://quandl.com/api/' if @quandl_url.blank?
248
+ @quandl_url
249
+ end
250
+
251
+ def auth_token
252
+ return @auth_token if @auth_token.present?
253
+ @auth_token = options.token if options.try(:token).present?
254
+ @auth_token = ENV['QUANDL_TOKEN'] if @auth_token.blank?
255
+ @auth_token = QConfig.configuration.token if @auth_token.blank?
256
+ @auth_token
257
+ end
258
+
259
+ end
260
+
261
+ end
262
+ end
263
+ end
@@ -0,0 +1,58 @@
1
+ class Quandl::Command::Tasks::Delete < Quandl::Command::Tasks::Base
2
+
3
+ syntax "quandl #{command_name} (SOURCE_CODE/)CODE"
4
+ description "Delete a dataset by its quandl code."
5
+
6
+ authenticated_users_only!
7
+
8
+ def execute
9
+ # download using arguments when present
10
+ return delete_each_argument if args.first.present?
11
+ # otherwise delete using stdin
12
+ delete_each_stdin
13
+ end
14
+
15
+ def delete_each_argument
16
+ args.each do |code|
17
+ pool.process{ delete( code ) }
18
+ end
19
+ pool.shutdown
20
+ end
21
+
22
+ def delete_each_stdin
23
+ return error("Cannot delete datasets from STDIN unless --force-yes is set!") unless force_yes?
24
+ $stdin.each_line do |code|
25
+ pool.process{ delete( code.strip.rstrip ) }
26
+ end
27
+ pool.shutdown
28
+ end
29
+
30
+ def delete(code)
31
+ return error("You must provide a code!") if code.blank?
32
+ # find by code
33
+ dataset = Quandl::Client::Dataset.find(code)
34
+ # fail fast if dataset does not exist
35
+ return error(table(Quandl::Client::HTTP_STATUS_CODES[404], code)) if dataset.nil?
36
+ return report(dataset) unless dataset.exists?
37
+ # confirm deletion
38
+ return error("You must confirm deletion!") unless confirmed?(dataset)
39
+ # delete if exists
40
+ dataset.destroy if dataset.exists?
41
+ # output status
42
+ report(dataset)
43
+ end
44
+
45
+ def report(dataset)
46
+ # message
47
+ m = table dataset.elapsed_request_time_ms, dataset.full_code
48
+ # report
49
+ dataset.status == 200 ? info(table("Deleted", m)) : error(table(dataset.human_status, m))
50
+ end
51
+
52
+ def confirmed?(dataset)
53
+ return true if force_yes?
54
+ info summarize dataset.attributes.slice('source_code', 'code', 'name', 'from_date', 'to_date', 'column_names', 'created_at')
55
+ ask_yes_or_no
56
+ end
57
+
58
+ end
@@ -0,0 +1,111 @@
1
+ require 'thread/pool'
2
+
3
+ class Quandl::Command::Tasks::Download < Quandl::Command::Tasks::Base
4
+
5
+ syntax "quandl download (SOURCE_CODE/)CODE"
6
+ description "Download a dataset using its quandl code."
7
+ options({
8
+ String => {
9
+ order: "Return rows in either ASC or DESC order",
10
+ transform: "Transform data using one of: #{Quandl::Operation::Transform.valid_transformations.join(', ')}",
11
+ collapse: "Collapse data to one of: #{Quandl::Operation::Collapse.valid_collapses.join(', ')}",
12
+ trim_start: "Exclude rows that are older than trim_start",
13
+ trim_end: "Exclude rows that are newer than trim_end",
14
+ },
15
+ Integer => {
16
+ threads: "How many workers to use during download.",
17
+ limit: "Limit the number of rows returned",
18
+ column: "Pluck a specific column by index",
19
+ row: "Pluck a specific row by index",
20
+ offset: "Offset the start of the rows",
21
+ }
22
+ })
23
+
24
+ warn_unauthenticated_users
25
+
26
+ validates :trim_start, :trim_end, allow_nil: true, format: { with: Quandl::Pattern.dataset_date, message: "is invalid. Expected format: #{Quandl::Pattern.dataset_date.to_example}" }
27
+ validates :order, allow_nil: true, inclusion: { in: ['asc', 'desc'], message: "must be one of: asc, desc" }
28
+ validates :collapse, allow_nil: true, inclusion: { in: Quandl::Operation::Collapse.valid_collapses.collect(&:to_s), message: "is not included in the list: #{Quandl::Operation::Collapse.valid_collapses.join(", ")}" }
29
+ validates :transform, allow_nil: true, inclusion: { in: Quandl::Operation::Transform.valid_transformations.collect(&:to_s), message: "is not included in the list: #{Quandl::Operation::Transform.valid_transformations.join(", ")}" }
30
+
31
+ def execute
32
+ # download using arguments when present
33
+ return download_each_argument if args.first.present?
34
+ # otherwise download using stdin
35
+ download_each_stdin
36
+ end
37
+
38
+ def download_each_argument
39
+ args.each do |code|
40
+ pool.process{ download(code) }
41
+ end
42
+ pool.shutdown
43
+ end
44
+
45
+ def download_each_stdin
46
+ $stdin.each_line do |code|
47
+ pool.process{ download(code.strip.rstrip) }
48
+ end
49
+ pool.shutdown
50
+ end
51
+
52
+ def download(code)
53
+ timer = Time.now
54
+ # find dataset
55
+ dataset = Quandl::Client::Dataset.find( code )
56
+ # fail fast
57
+ return error(table(Quandl::Client::HTTP_STATUS_CODES[404], code)) if dataset.nil?
58
+ # set data operations
59
+ dataset.data.assign_attributes(data_params) unless dataset.blank?
60
+ # send request & check for errors.
61
+ if !dataset.exists? && !dataset.valid?
62
+ # raise detailed error
63
+ return error( table( dataset.human_status, code, dataset.elapsed_request_time_ms ) )
64
+ end
65
+ # generate qdf
66
+ elapsed = timer.elapsed_ms
67
+ qdf = dataset.to_qdf
68
+ # write to STDOUT
69
+ mutex.synchronize{
70
+ debug("# #{dataset.try(:full_url)} downloaded in #{elapsed}")
71
+ info(qdf)
72
+ }
73
+ true
74
+ end
75
+
76
+ def trim_start
77
+ options.trim_start
78
+ end
79
+
80
+ def trim_end
81
+ options.trim_end
82
+ end
83
+
84
+ def order
85
+ options.order
86
+ end
87
+
88
+ def collapse
89
+ options.collapse
90
+ end
91
+
92
+ def transform
93
+ options.transform
94
+ end
95
+
96
+ def data_params
97
+ params = {}
98
+ self.class.options.each do |class_type, opts|
99
+ opts.each do |name, desc|
100
+ if options.is_a?(OpenStruct)
101
+ params[name] = self.options.send(name)
102
+ else
103
+ params[name] = self.options[name] if self.options[name].present?
104
+ end
105
+
106
+ end
107
+ end
108
+ params
109
+ end
110
+
111
+ end