quandl 0.2.27 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/README.md +70 -17
- data/Rakefile +7 -86
- data/UPGRADE.md +23 -0
- data/VERSION +1 -0
- data/lib/quandl/command.rb +11 -3
- data/lib/quandl/command/config.rb +88 -0
- data/lib/quandl/command/presenter.rb +74 -0
- data/lib/quandl/command/presenter/record.rb +121 -0
- data/lib/quandl/command/presenters/dataset_presenter.rb +19 -0
- data/lib/quandl/command/presenters/error_presenter.rb +31 -0
- data/lib/quandl/command/presenters/nil_class_presenter.rb +10 -0
- data/lib/quandl/command/presenters/scraper_presenter.rb +25 -0
- data/lib/quandl/command/presenters/superset_presenter.rb +10 -0
- data/lib/quandl/command/task.rb +40 -0
- data/lib/quandl/command/task/callbacks.rb +47 -0
- data/lib/quandl/command/task/clientable.rb +65 -0
- data/lib/quandl/command/task/commandable.rb +104 -0
- data/lib/quandl/command/task/configurable.rb +79 -0
- data/lib/quandl/command/task/inputable.rb +37 -0
- data/lib/quandl/command/task/logging.rb +83 -0
- data/lib/quandl/command/task/presentation.rb +37 -0
- data/lib/quandl/command/task/reportable.rb +35 -0
- data/lib/quandl/command/task/threading.rb +117 -0
- data/lib/quandl/command/task/translations.rb +37 -0
- data/lib/quandl/command/task/updatable.rb +77 -0
- data/lib/quandl/command/tasks.rb +6 -5
- data/lib/quandl/command/tasks/delete.rb +10 -67
- data/lib/quandl/command/tasks/download.rb +11 -78
- data/lib/quandl/command/tasks/info.rb +2 -2
- data/lib/quandl/command/tasks/list.rb +12 -24
- data/lib/quandl/command/tasks/login.rb +4 -4
- data/lib/quandl/command/tasks/replace.rb +58 -0
- data/lib/quandl/command/tasks/schedule.rb +106 -0
- data/lib/quandl/command/tasks/search.rb +39 -0
- data/lib/quandl/command/tasks/superset.rb +59 -0
- data/lib/quandl/command/tasks/uninstall.rb +1 -1
- data/lib/quandl/command/tasks/update.rb +2 -2
- data/lib/quandl/command/tasks/upload.rb +13 -26
- data/lib/quandl/command/version.rb +1 -1
- data/quandl.gemspec +5 -3
- data/spec/fixtures/scraper.rb +6 -0
- data/spec/lib/quandl/command/delete_spec.rb +19 -11
- data/spec/lib/quandl/command/download_spec.rb +11 -4
- data/spec/lib/quandl/command/replace_spec.rb +23 -0
- data/spec/lib/quandl/command/schedule_spec.rb +53 -0
- data/spec/lib/quandl/command/superset_spec.rb +28 -0
- data/spec/lib/quandl/command/upload_spec.rb +71 -24
- data/spec/lib/quandl/command_spec.rb +1 -9
- data/spec/spec_helper.rb +36 -1
- data/tasks/toolbelt/build/tarball.rb +2 -2
- metadata +55 -10
- data/lib/quandl/command/qconfig.rb +0 -86
- data/lib/quandl/command/tasks/base.rb +0 -314
@@ -1,314 +0,0 @@
|
|
1
|
-
require 'active_model'
|
2
|
-
require 'quandl/pattern'
|
3
|
-
require 'quandl/pattern/client'
|
4
|
-
require 'quandl/operation'
|
5
|
-
require 'quandl/operation'
|
6
|
-
|
7
|
-
module Quandl
|
8
|
-
module Command
|
9
|
-
module Tasks
|
10
|
-
|
11
|
-
class Base
|
12
|
-
class << self
|
13
|
-
|
14
|
-
def inherited(klass)
|
15
|
-
Tasks.tasks << klass unless Tasks.tasks.include?(klass)
|
16
|
-
end
|
17
|
-
|
18
|
-
def configure(app)
|
19
|
-
return if disabled?
|
20
|
-
app.command(command_name) do |c|
|
21
|
-
c.syntax = syntax
|
22
|
-
c.description = description
|
23
|
-
c.action{|a,o| call(a,o) }
|
24
|
-
configure_options(c)
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
def disabled?
|
29
|
-
@disabled == true
|
30
|
-
end
|
31
|
-
|
32
|
-
def disable!
|
33
|
-
@disabled = true
|
34
|
-
end
|
35
|
-
|
36
|
-
def syntax(value=nil)
|
37
|
-
@syntax = value if value.present?
|
38
|
-
@syntax ||= "quandl #{command_name}"
|
39
|
-
end
|
40
|
-
|
41
|
-
def command_name(value=nil)
|
42
|
-
@command_name = value if value.present?
|
43
|
-
@command_name ||= name.to_s.split("::").last.downcase
|
44
|
-
end
|
45
|
-
|
46
|
-
def description(value=nil)
|
47
|
-
@description = value if value.present?
|
48
|
-
@description ||= "No description."
|
49
|
-
end
|
50
|
-
|
51
|
-
def options(value=nil)
|
52
|
-
@options = value if value.present?
|
53
|
-
@options ||= {}
|
54
|
-
end
|
55
|
-
|
56
|
-
def autoload_client_library
|
57
|
-
before_execute :configure_client
|
58
|
-
end
|
59
|
-
|
60
|
-
def authenticated_users_only!
|
61
|
-
before_execute :authenticated_users_only!
|
62
|
-
end
|
63
|
-
|
64
|
-
def disable_in_gem!
|
65
|
-
before_execute :disable_in_gem!
|
66
|
-
end
|
67
|
-
|
68
|
-
def warn_unauthenticated_users
|
69
|
-
before_execute :warn_unauthenticated_users
|
70
|
-
end
|
71
|
-
|
72
|
-
def call(args=[], options={})
|
73
|
-
args = Array(args)
|
74
|
-
options = ensure_options_are_command_options!(options)
|
75
|
-
self.new( args, options ).call
|
76
|
-
end
|
77
|
-
|
78
|
-
def t(key)
|
79
|
-
key = key.to_s
|
80
|
-
translation = lang
|
81
|
-
key.split('.').each{|m| translation = translation.respond_to?(m) ? translation.send(m) : nil }
|
82
|
-
translation
|
83
|
-
end
|
84
|
-
|
85
|
-
def lang
|
86
|
-
@lang ||= Quandl::Lang.send(language).quandl.command.tasks.send(command_name)
|
87
|
-
end
|
88
|
-
|
89
|
-
def language
|
90
|
-
# stub
|
91
|
-
'en'
|
92
|
-
end
|
93
|
-
|
94
|
-
protected
|
95
|
-
|
96
|
-
def ensure_options_are_command_options!(options)
|
97
|
-
return options if options.class == Commander::Command::Options
|
98
|
-
OpenStruct.new(options)
|
99
|
-
end
|
100
|
-
|
101
|
-
def configure_options(c)
|
102
|
-
options.each do |class_type, options|
|
103
|
-
options.each do |name, desc|
|
104
|
-
c.option "--#{name} #{class_type.to_s.upcase}", class_type, desc
|
105
|
-
end
|
106
|
-
end
|
107
|
-
end
|
108
|
-
|
109
|
-
end
|
110
|
-
|
111
|
-
attr_accessor :args, :options, :request_timer
|
112
|
-
|
113
|
-
include ActiveModel::Validations
|
114
|
-
|
115
|
-
extend ActiveModel::Callbacks
|
116
|
-
define_model_callbacks :execute
|
117
|
-
|
118
|
-
before_execute :raise_error_unless_valid!, :check_for_update, :start_request_timer
|
119
|
-
after_execute :log_request_time
|
120
|
-
|
121
|
-
def call
|
122
|
-
run_callbacks(:execute) do
|
123
|
-
execute
|
124
|
-
end
|
125
|
-
end
|
126
|
-
|
127
|
-
def initialize(args, options)
|
128
|
-
self.args = args
|
129
|
-
self.options = options
|
130
|
-
end
|
131
|
-
|
132
|
-
def verbose?
|
133
|
-
options.verbose == true
|
134
|
-
end
|
135
|
-
|
136
|
-
def force_yes?
|
137
|
-
options.force_yes == true
|
138
|
-
end
|
139
|
-
|
140
|
-
def ask_yes_or_no
|
141
|
-
['y','yes'].include?( ask("Are you sure? (y/n)") )
|
142
|
-
end
|
143
|
-
|
144
|
-
def summarize(item)
|
145
|
-
return summarize_hash(item) if item.kind_of?(Hash)
|
146
|
-
item
|
147
|
-
end
|
148
|
-
|
149
|
-
def summarize_hash(item)
|
150
|
-
item.collect do |k,v|
|
151
|
-
next "#{k}: '#{v}'" if v.kind_of?(String)
|
152
|
-
"#{k}: #{v}"
|
153
|
-
end.join(', ')
|
154
|
-
end
|
155
|
-
|
156
|
-
def table(*args)
|
157
|
-
Array(args).flatten.join(" | ")
|
158
|
-
end
|
159
|
-
|
160
|
-
def info(*args)
|
161
|
-
logger.info(*args)
|
162
|
-
end
|
163
|
-
|
164
|
-
def debug(*args)
|
165
|
-
logger.debug(*args) if verbose?
|
166
|
-
end
|
167
|
-
|
168
|
-
def error(*args)
|
169
|
-
logger.error(*args)
|
170
|
-
end
|
171
|
-
|
172
|
-
def fatal(*args)
|
173
|
-
logger.fatal("FATAL: #{args.join(" ")}")
|
174
|
-
end
|
175
|
-
|
176
|
-
def logger
|
177
|
-
Quandl::Logger
|
178
|
-
end
|
179
|
-
|
180
|
-
def current_user
|
181
|
-
@current_user ||= Quandl::Client::User.info
|
182
|
-
end
|
183
|
-
|
184
|
-
protected
|
185
|
-
|
186
|
-
# THREAD POOL
|
187
|
-
|
188
|
-
def mutex
|
189
|
-
@mutex ||= Mutex.new
|
190
|
-
end
|
191
|
-
|
192
|
-
def pool
|
193
|
-
@pool ||= Thread.pool( threads )
|
194
|
-
end
|
195
|
-
|
196
|
-
def threads
|
197
|
-
options.threads || 4
|
198
|
-
end
|
199
|
-
|
200
|
-
|
201
|
-
def start_request_timer
|
202
|
-
self.request_timer = Time.now
|
203
|
-
end
|
204
|
-
|
205
|
-
def log_request_time
|
206
|
-
debug("# Started: #{request_timer}. Finished: #{Time.now}. Elapsed: #{request_timer.elapsed_ms}")
|
207
|
-
end
|
208
|
-
|
209
|
-
def reload_session!
|
210
|
-
@auth_token = nil
|
211
|
-
@current_user = nil
|
212
|
-
configure_client
|
213
|
-
end
|
214
|
-
|
215
|
-
def authenticated_users_only!
|
216
|
-
if auth_token.blank?
|
217
|
-
fatal("You must authenticate to use #{self.class.command_name}! 'quandl login' OR --token xyz923")
|
218
|
-
false
|
219
|
-
end
|
220
|
-
end
|
221
|
-
|
222
|
-
def disable_in_gem!
|
223
|
-
if force_yes?
|
224
|
-
info("You have forced update!")
|
225
|
-
true
|
226
|
-
elsif Dir.exists?( File.join( Tasks.root, ".git") ) || File.exists?( File.join( Tasks.root, "Gemfile") )
|
227
|
-
fatal("#{self.class.command_name} is only permitted when installed as a package! http://quandl.com/help/toolbelt")
|
228
|
-
false
|
229
|
-
end
|
230
|
-
end
|
231
|
-
|
232
|
-
def warn_unauthenticated_users
|
233
|
-
error("WARN: Authenticate your requests! 'quandl login' OR --token xyz923") if auth_token.blank?
|
234
|
-
end
|
235
|
-
|
236
|
-
def raise_error_unless_valid!
|
237
|
-
unless valid?
|
238
|
-
error( table(errors.full_messages) )
|
239
|
-
false
|
240
|
-
end
|
241
|
-
end
|
242
|
-
|
243
|
-
def configure_client
|
244
|
-
require 'thread/pool'
|
245
|
-
require 'quandl/format'
|
246
|
-
require 'quandl/command/client_ext'
|
247
|
-
Quandl::Client.use( quandl_url )
|
248
|
-
Quandl::Client.token = auth_token
|
249
|
-
Quandl::Client.request_source = 'quandl_command'
|
250
|
-
Quandl::Client.request_version = Quandl::Command::VERSION
|
251
|
-
end
|
252
|
-
|
253
|
-
def check_for_update
|
254
|
-
check_time = QConfig.configuration.last_checked_for_update
|
255
|
-
# check time present?
|
256
|
-
if check_time.present? && check_time.is_a?(Time)
|
257
|
-
# has it been more than one day?
|
258
|
-
run_update_check if Time.now - 1.day > check_time || check_time > Time.now
|
259
|
-
else
|
260
|
-
run_update_check
|
261
|
-
end
|
262
|
-
end
|
263
|
-
|
264
|
-
def run_update_check
|
265
|
-
require 'uri'
|
266
|
-
require 'net/http'
|
267
|
-
require 'open-uri'
|
268
|
-
|
269
|
-
print("# Checking for updates ... ")
|
270
|
-
|
271
|
-
uri = URI.parse("https://raw.github.com/quandl/quandl_command/master/lib/quandl/command/version.rb")
|
272
|
-
http = Net::HTTP.new(uri.host, uri.port)
|
273
|
-
http.use_ssl = true
|
274
|
-
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
275
|
-
|
276
|
-
request = Net::HTTP::Get.new(uri.request_uri)
|
277
|
-
|
278
|
-
response = http.request(request)
|
279
|
-
|
280
|
-
master = response.body
|
281
|
-
master = master.split("\n").detect{|r| r =~ /VERSION/ }.split("'").last
|
282
|
-
if Quandl::Command::VERSION != master
|
283
|
-
info(" A new version of quandl toolbelt has been released. #{master}. Please run 'quandl update'")
|
284
|
-
else
|
285
|
-
info(" you are up to date! #{master}")
|
286
|
-
end
|
287
|
-
rescue => err
|
288
|
-
error("An unexpected error occured while checking for updates ... #{err}")
|
289
|
-
ensure
|
290
|
-
QConfig.configuration.last_checked_for_update = Time.now
|
291
|
-
end
|
292
|
-
|
293
|
-
def quandl_url
|
294
|
-
return @quandl_url if @quandl_url.present?
|
295
|
-
@quandl_url = options.url if options.try(:url).present?
|
296
|
-
@quandl_url = ENV['QUANDL_URL'] if @quandl_url.blank?
|
297
|
-
@quandl_url = QConfig.configuration.quandl_url if @quandl_url.blank?
|
298
|
-
@quandl_url = 'http://quandl.com/api/' if @quandl_url.blank?
|
299
|
-
@quandl_url
|
300
|
-
end
|
301
|
-
|
302
|
-
def auth_token
|
303
|
-
return @auth_token if @auth_token.present?
|
304
|
-
@auth_token = options.token if options.try(:token).present?
|
305
|
-
@auth_token = ENV['QUANDL_TOKEN'] if @auth_token.blank?
|
306
|
-
@auth_token = QConfig.configuration.token if @auth_token.blank?
|
307
|
-
@auth_token
|
308
|
-
end
|
309
|
-
|
310
|
-
end
|
311
|
-
|
312
|
-
end
|
313
|
-
end
|
314
|
-
end
|