quandl 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/.gitignore +16 -0
- data/.travis.yml +20 -0
- data/Gemfile +14 -0
- data/Guardfile +8 -0
- data/LICENSE +7 -0
- data/README.md +316 -0
- data/Rakefile +39 -0
- data/UPGRADE.md +143 -0
- data/bin/quandl +26 -0
- data/dist/resources/pkg/Distribution.erb +15 -0
- data/dist/resources/pkg/PackageInfo.erb +6 -0
- data/dist/resources/pkg/postinstall +45 -0
- data/dist/resources/pkg/quandl +25 -0
- data/dist/resources/ruby/PackageInfo +3 -0
- data/lib/commander/command/quandl_ext.rb +21 -0
- data/lib/quandl/command.rb +45 -0
- data/lib/quandl/command/client_ext.rb +1 -0
- data/lib/quandl/command/client_ext/user.rb +11 -0
- data/lib/quandl/command/compatibility_check.rb +11 -0
- data/lib/quandl/command/qconfig.rb +76 -0
- data/lib/quandl/command/tasks.rb +15 -0
- data/lib/quandl/command/tasks/base.rb +263 -0
- data/lib/quandl/command/tasks/delete.rb +58 -0
- data/lib/quandl/command/tasks/download.rb +111 -0
- data/lib/quandl/command/tasks/info.rb +46 -0
- data/lib/quandl/command/tasks/list.rb +40 -0
- data/lib/quandl/command/tasks/login.rb +46 -0
- data/lib/quandl/command/tasks/update.rb +205 -0
- data/lib/quandl/command/tasks/upload.rb +69 -0
- data/lib/quandl/command/version.rb +5 -0
- data/lib/quandl/utility.rb +2 -0
- data/lib/quandl/utility/config.rb +43 -0
- data/lib/quandl/utility/ruby_version.rb +143 -0
- data/quandl.gemspec +39 -0
- data/scripts/compile_ruby_pkg.sh +34 -0
- data/scripts/install.sh +51 -0
- data/scripts/win/quandl_toolbelt.iss +82 -0
- data/spec/lib/quandl/command/delete_spec.rb +31 -0
- data/spec/lib/quandl/command/download_spec.rb +42 -0
- data/spec/lib/quandl/command/upload_spec.rb +39 -0
- data/spec/lib/quandl/command_spec.rb +46 -0
- data/spec/spec_helper.rb +20 -0
- data/tasks/toolbelt.rake +133 -0
- data/tasks/toolbelt.rb +92 -0
- data/tasks/toolbelt/build.rb +2 -0
- data/tasks/toolbelt/build/darwin.rb +71 -0
- data/tasks/toolbelt/build/ruby.rb +105 -0
- data/tasks/toolbelt/build/tarball.rb +73 -0
- data/tasks/toolbelt/build/windows.rb +25 -0
- data/tasks/toolbelt/push.rb +15 -0
- data/tasks/toolbelt/storage.rb +28 -0
- data/tasks/toolbelt/tar.rb +21 -0
- metadata +354 -0
@@ -0,0 +1,15 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
module Tasks
|
4
|
+
|
5
|
+
def self.root
|
6
|
+
@root ||= File.expand_path(File.join(File.dirname(__FILE__), '../../../'))
|
7
|
+
end
|
8
|
+
|
9
|
+
# require base task
|
10
|
+
require 'quandl/command/tasks/base'
|
11
|
+
# require tasks
|
12
|
+
Dir.glob(File.join(root, 'lib/quandl/command/tasks/*.rb')){|t| require(t) }
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,263 @@
|
|
1
|
+
autoload :OpenURI, 'open-uri'
|
2
|
+
|
3
|
+
module Quandl
|
4
|
+
module Command
|
5
|
+
module Tasks
|
6
|
+
|
7
|
+
class Base
|
8
|
+
class << self
|
9
|
+
|
10
|
+
def configure(app)
|
11
|
+
return if disabled?
|
12
|
+
app.command(command_name) do |c|
|
13
|
+
c.syntax = syntax
|
14
|
+
c.description = description
|
15
|
+
c.action{|a,o| call(a,o) }
|
16
|
+
configure_options(c)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def disabled?
|
21
|
+
@disabled == true
|
22
|
+
end
|
23
|
+
|
24
|
+
def disable!
|
25
|
+
@disabled = true
|
26
|
+
end
|
27
|
+
|
28
|
+
def syntax(value=nil)
|
29
|
+
@syntax = value if value.present?
|
30
|
+
@syntax ||= "quandl #{command_name}"
|
31
|
+
end
|
32
|
+
|
33
|
+
def command_name(value=nil)
|
34
|
+
@command_name = value if value.present?
|
35
|
+
@command_name ||= name.to_s.split("::").last.downcase
|
36
|
+
end
|
37
|
+
|
38
|
+
def description(value=nil)
|
39
|
+
@description = value if value.present?
|
40
|
+
@description ||= "No description."
|
41
|
+
end
|
42
|
+
|
43
|
+
def options(value=nil)
|
44
|
+
@options = value if value.present?
|
45
|
+
@options ||= {}
|
46
|
+
end
|
47
|
+
|
48
|
+
def authenticated_users_only!
|
49
|
+
before_execute :authenticated_users_only!
|
50
|
+
end
|
51
|
+
|
52
|
+
def warn_unauthenticated_users
|
53
|
+
before_execute :warn_unauthenticated_users
|
54
|
+
end
|
55
|
+
|
56
|
+
def call(args=[], options={})
|
57
|
+
args = Array(args)
|
58
|
+
options = ensure_options_are_command_options!(options)
|
59
|
+
self.new( args, options ).call
|
60
|
+
end
|
61
|
+
|
62
|
+
protected
|
63
|
+
|
64
|
+
def ensure_options_are_command_options!(options)
|
65
|
+
return options if options.class == Commander::Command::Options
|
66
|
+
OpenStruct.new(options)
|
67
|
+
end
|
68
|
+
|
69
|
+
def configure_options(c)
|
70
|
+
options.each do |class_type, options|
|
71
|
+
options.each do |name, desc|
|
72
|
+
c.option "--#{name} #{class_type.to_s.upcase}", class_type, desc
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
|
79
|
+
attr_accessor :args, :options, :request_timer
|
80
|
+
|
81
|
+
include ActiveModel::Validations
|
82
|
+
|
83
|
+
extend ActiveModel::Callbacks
|
84
|
+
define_model_callbacks :execute
|
85
|
+
|
86
|
+
before_execute :configure_client, :raise_error_unless_valid!, :check_for_update, :start_request_timer
|
87
|
+
after_execute :log_request_time
|
88
|
+
|
89
|
+
def call
|
90
|
+
run_callbacks(:execute) do
|
91
|
+
execute
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def initialize(args, options)
|
96
|
+
self.args = args
|
97
|
+
self.options = options
|
98
|
+
end
|
99
|
+
|
100
|
+
def verbose?
|
101
|
+
options.verbose == true
|
102
|
+
end
|
103
|
+
|
104
|
+
def force_yes?
|
105
|
+
options.force_yes == true
|
106
|
+
end
|
107
|
+
|
108
|
+
def ask_yes_or_no
|
109
|
+
['y','yes'].include?( ask("Are you sure? (y/n)") )
|
110
|
+
end
|
111
|
+
|
112
|
+
def summarize(item)
|
113
|
+
return summarize_hash(item) if item.kind_of?(Hash)
|
114
|
+
item
|
115
|
+
end
|
116
|
+
|
117
|
+
def summarize_hash(item)
|
118
|
+
item.collect do |k,v|
|
119
|
+
next "#{k}: '#{v}'" if v.kind_of?(String)
|
120
|
+
"#{k}: #{v}"
|
121
|
+
end.join(', ')
|
122
|
+
end
|
123
|
+
|
124
|
+
def table(*args)
|
125
|
+
Array(args).flatten.join(" | ")
|
126
|
+
end
|
127
|
+
|
128
|
+
def info(*args)
|
129
|
+
logger.info(*args)
|
130
|
+
end
|
131
|
+
|
132
|
+
def debug(*args)
|
133
|
+
logger.debug(*args) if verbose?
|
134
|
+
end
|
135
|
+
|
136
|
+
def error(*args)
|
137
|
+
logger.error(*args)
|
138
|
+
end
|
139
|
+
|
140
|
+
def fatal(*args)
|
141
|
+
logger.fatal("FATAL: #{args.join(" ")}")
|
142
|
+
end
|
143
|
+
|
144
|
+
def logger
|
145
|
+
Quandl::Logger
|
146
|
+
end
|
147
|
+
|
148
|
+
def current_user
|
149
|
+
@current_user ||= Quandl::Client::User.info
|
150
|
+
end
|
151
|
+
|
152
|
+
protected
|
153
|
+
|
154
|
+
# THREAD POOL
|
155
|
+
|
156
|
+
def mutex
|
157
|
+
@mutex ||= Mutex.new
|
158
|
+
end
|
159
|
+
|
160
|
+
def pool
|
161
|
+
@pool ||= Thread.pool( threads )
|
162
|
+
end
|
163
|
+
|
164
|
+
def threads
|
165
|
+
options.threads || 4
|
166
|
+
end
|
167
|
+
|
168
|
+
|
169
|
+
def start_request_timer
|
170
|
+
self.request_timer = Time.now
|
171
|
+
end
|
172
|
+
|
173
|
+
def log_request_time
|
174
|
+
debug("# Started: #{request_timer}. Finished: #{Time.now}. Elapsed: #{request_timer.elapsed_ms}")
|
175
|
+
end
|
176
|
+
|
177
|
+
def reload_session!
|
178
|
+
@auth_token = nil
|
179
|
+
@current_user = nil
|
180
|
+
configure_client
|
181
|
+
end
|
182
|
+
|
183
|
+
def authenticated_users_only!
|
184
|
+
if auth_token.blank?
|
185
|
+
fatal("You must authenticate to use #{self.class.command_name}! 'quandl login' OR --token xyz923")
|
186
|
+
false
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
def warn_unauthenticated_users
|
191
|
+
error("WARN: Authenticate your requests! 'quandl login' OR --token xyz923") if auth_token.blank?
|
192
|
+
end
|
193
|
+
|
194
|
+
def raise_error_unless_valid!
|
195
|
+
unless valid?
|
196
|
+
error( table(errors.full_messages) )
|
197
|
+
false
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
def configure_client
|
202
|
+
Quandl::Client.use( quandl_url )
|
203
|
+
Quandl::Client.token = auth_token
|
204
|
+
end
|
205
|
+
|
206
|
+
def check_for_update
|
207
|
+
check_time = QConfig.configuration.last_checked_for_update
|
208
|
+
# check time present?
|
209
|
+
if check_time.present? && check_time.is_a?(Time)
|
210
|
+
# has it been more than one day?
|
211
|
+
run_update_check if Time.now - 1.day > check_time || check_time > Time.now
|
212
|
+
else
|
213
|
+
run_update_check
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def run_update_check
|
218
|
+
print("# Checking for updates ... ")
|
219
|
+
|
220
|
+
uri = URI.parse("https://raw.github.com/quandl/quandl_command/master/lib/quandl/command/version.rb")
|
221
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
222
|
+
http.use_ssl = true
|
223
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
224
|
+
|
225
|
+
request = Net::HTTP::Get.new(uri.request_uri)
|
226
|
+
|
227
|
+
response = http.request(request)
|
228
|
+
|
229
|
+
master = response.body
|
230
|
+
master = master.split("\n").detect{|r| r =~ /VERSION/ }.split("'").last
|
231
|
+
if Quandl::Command::VERSION != master
|
232
|
+
info(" A new version of quandl toolbelt has been released. #{master}. Please run 'quandl update'")
|
233
|
+
else
|
234
|
+
info(" you are up to date! #{master}")
|
235
|
+
end
|
236
|
+
rescue => err
|
237
|
+
error("An unexpected error occured while checking for updates ... #{err}")
|
238
|
+
ensure
|
239
|
+
QConfig.configuration.last_checked_for_update = Time.now
|
240
|
+
end
|
241
|
+
|
242
|
+
def quandl_url
|
243
|
+
return @quandl_url if @quandl_url.present?
|
244
|
+
@quandl_url = options.url if options.try(:url).present?
|
245
|
+
@quandl_url = ENV['QUANDL_URL'] if @quandl_url.blank?
|
246
|
+
@quandl_url = QConfig.configuration.quandl_url if @quandl_url.blank?
|
247
|
+
@quandl_url = 'http://quandl.com/api/' if @quandl_url.blank?
|
248
|
+
@quandl_url
|
249
|
+
end
|
250
|
+
|
251
|
+
def auth_token
|
252
|
+
return @auth_token if @auth_token.present?
|
253
|
+
@auth_token = options.token if options.try(:token).present?
|
254
|
+
@auth_token = ENV['QUANDL_TOKEN'] if @auth_token.blank?
|
255
|
+
@auth_token = QConfig.configuration.token if @auth_token.blank?
|
256
|
+
@auth_token
|
257
|
+
end
|
258
|
+
|
259
|
+
end
|
260
|
+
|
261
|
+
end
|
262
|
+
end
|
263
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
class Quandl::Command::Tasks::Delete < Quandl::Command::Tasks::Base
|
2
|
+
|
3
|
+
syntax "quandl #{command_name} (SOURCE_CODE/)CODE"
|
4
|
+
description "Delete a dataset by its quandl code."
|
5
|
+
|
6
|
+
authenticated_users_only!
|
7
|
+
|
8
|
+
def execute
|
9
|
+
# download using arguments when present
|
10
|
+
return delete_each_argument if args.first.present?
|
11
|
+
# otherwise delete using stdin
|
12
|
+
delete_each_stdin
|
13
|
+
end
|
14
|
+
|
15
|
+
def delete_each_argument
|
16
|
+
args.each do |code|
|
17
|
+
pool.process{ delete( code ) }
|
18
|
+
end
|
19
|
+
pool.shutdown
|
20
|
+
end
|
21
|
+
|
22
|
+
def delete_each_stdin
|
23
|
+
return error("Cannot delete datasets from STDIN unless --force-yes is set!") unless force_yes?
|
24
|
+
$stdin.each_line do |code|
|
25
|
+
pool.process{ delete( code.strip.rstrip ) }
|
26
|
+
end
|
27
|
+
pool.shutdown
|
28
|
+
end
|
29
|
+
|
30
|
+
def delete(code)
|
31
|
+
return error("You must provide a code!") if code.blank?
|
32
|
+
# find by code
|
33
|
+
dataset = Quandl::Client::Dataset.find(code)
|
34
|
+
# fail fast if dataset does not exist
|
35
|
+
return error(table(Quandl::Client::HTTP_STATUS_CODES[404], code)) if dataset.nil?
|
36
|
+
return report(dataset) unless dataset.exists?
|
37
|
+
# confirm deletion
|
38
|
+
return error("You must confirm deletion!") unless confirmed?(dataset)
|
39
|
+
# delete if exists
|
40
|
+
dataset.destroy if dataset.exists?
|
41
|
+
# output status
|
42
|
+
report(dataset)
|
43
|
+
end
|
44
|
+
|
45
|
+
def report(dataset)
|
46
|
+
# message
|
47
|
+
m = table dataset.elapsed_request_time_ms, dataset.full_code
|
48
|
+
# report
|
49
|
+
dataset.status == 200 ? info(table("Deleted", m)) : error(table(dataset.human_status, m))
|
50
|
+
end
|
51
|
+
|
52
|
+
def confirmed?(dataset)
|
53
|
+
return true if force_yes?
|
54
|
+
info summarize dataset.attributes.slice('source_code', 'code', 'name', 'from_date', 'to_date', 'column_names', 'created_at')
|
55
|
+
ask_yes_or_no
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require 'thread/pool'
|
2
|
+
|
3
|
+
class Quandl::Command::Tasks::Download < Quandl::Command::Tasks::Base
|
4
|
+
|
5
|
+
syntax "quandl download (SOURCE_CODE/)CODE"
|
6
|
+
description "Download a dataset using its quandl code."
|
7
|
+
options({
|
8
|
+
String => {
|
9
|
+
order: "Return rows in either ASC or DESC order",
|
10
|
+
transform: "Transform data using one of: #{Quandl::Operation::Transform.valid_transformations.join(', ')}",
|
11
|
+
collapse: "Collapse data to one of: #{Quandl::Operation::Collapse.valid_collapses.join(', ')}",
|
12
|
+
trim_start: "Exclude rows that are older than trim_start",
|
13
|
+
trim_end: "Exclude rows that are newer than trim_end",
|
14
|
+
},
|
15
|
+
Integer => {
|
16
|
+
threads: "How many workers to use during download.",
|
17
|
+
limit: "Limit the number of rows returned",
|
18
|
+
column: "Pluck a specific column by index",
|
19
|
+
row: "Pluck a specific row by index",
|
20
|
+
offset: "Offset the start of the rows",
|
21
|
+
}
|
22
|
+
})
|
23
|
+
|
24
|
+
warn_unauthenticated_users
|
25
|
+
|
26
|
+
validates :trim_start, :trim_end, allow_nil: true, format: { with: Quandl::Pattern.dataset_date, message: "is invalid. Expected format: #{Quandl::Pattern.dataset_date.to_example}" }
|
27
|
+
validates :order, allow_nil: true, inclusion: { in: ['asc', 'desc'], message: "must be one of: asc, desc" }
|
28
|
+
validates :collapse, allow_nil: true, inclusion: { in: Quandl::Operation::Collapse.valid_collapses.collect(&:to_s), message: "is not included in the list: #{Quandl::Operation::Collapse.valid_collapses.join(", ")}" }
|
29
|
+
validates :transform, allow_nil: true, inclusion: { in: Quandl::Operation::Transform.valid_transformations.collect(&:to_s), message: "is not included in the list: #{Quandl::Operation::Transform.valid_transformations.join(", ")}" }
|
30
|
+
|
31
|
+
def execute
|
32
|
+
# download using arguments when present
|
33
|
+
return download_each_argument if args.first.present?
|
34
|
+
# otherwise download using stdin
|
35
|
+
download_each_stdin
|
36
|
+
end
|
37
|
+
|
38
|
+
def download_each_argument
|
39
|
+
args.each do |code|
|
40
|
+
pool.process{ download(code) }
|
41
|
+
end
|
42
|
+
pool.shutdown
|
43
|
+
end
|
44
|
+
|
45
|
+
def download_each_stdin
|
46
|
+
$stdin.each_line do |code|
|
47
|
+
pool.process{ download(code.strip.rstrip) }
|
48
|
+
end
|
49
|
+
pool.shutdown
|
50
|
+
end
|
51
|
+
|
52
|
+
def download(code)
|
53
|
+
timer = Time.now
|
54
|
+
# find dataset
|
55
|
+
dataset = Quandl::Client::Dataset.find( code )
|
56
|
+
# fail fast
|
57
|
+
return error(table(Quandl::Client::HTTP_STATUS_CODES[404], code)) if dataset.nil?
|
58
|
+
# set data operations
|
59
|
+
dataset.data.assign_attributes(data_params) unless dataset.blank?
|
60
|
+
# send request & check for errors.
|
61
|
+
if !dataset.exists? && !dataset.valid?
|
62
|
+
# raise detailed error
|
63
|
+
return error( table( dataset.human_status, code, dataset.elapsed_request_time_ms ) )
|
64
|
+
end
|
65
|
+
# generate qdf
|
66
|
+
elapsed = timer.elapsed_ms
|
67
|
+
qdf = dataset.to_qdf
|
68
|
+
# write to STDOUT
|
69
|
+
mutex.synchronize{
|
70
|
+
debug("# #{dataset.try(:full_url)} downloaded in #{elapsed}")
|
71
|
+
info(qdf)
|
72
|
+
}
|
73
|
+
true
|
74
|
+
end
|
75
|
+
|
76
|
+
def trim_start
|
77
|
+
options.trim_start
|
78
|
+
end
|
79
|
+
|
80
|
+
def trim_end
|
81
|
+
options.trim_end
|
82
|
+
end
|
83
|
+
|
84
|
+
def order
|
85
|
+
options.order
|
86
|
+
end
|
87
|
+
|
88
|
+
def collapse
|
89
|
+
options.collapse
|
90
|
+
end
|
91
|
+
|
92
|
+
def transform
|
93
|
+
options.transform
|
94
|
+
end
|
95
|
+
|
96
|
+
def data_params
|
97
|
+
params = {}
|
98
|
+
self.class.options.each do |class_type, opts|
|
99
|
+
opts.each do |name, desc|
|
100
|
+
if options.is_a?(OpenStruct)
|
101
|
+
params[name] = self.options.send(name)
|
102
|
+
else
|
103
|
+
params[name] = self.options[name] if self.options[name].present?
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
end
|
108
|
+
params
|
109
|
+
end
|
110
|
+
|
111
|
+
end
|