quandl 0.2.27 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/README.md +70 -17
- data/Rakefile +7 -86
- data/UPGRADE.md +23 -0
- data/VERSION +1 -0
- data/lib/quandl/command.rb +11 -3
- data/lib/quandl/command/config.rb +88 -0
- data/lib/quandl/command/presenter.rb +74 -0
- data/lib/quandl/command/presenter/record.rb +121 -0
- data/lib/quandl/command/presenters/dataset_presenter.rb +19 -0
- data/lib/quandl/command/presenters/error_presenter.rb +31 -0
- data/lib/quandl/command/presenters/nil_class_presenter.rb +10 -0
- data/lib/quandl/command/presenters/scraper_presenter.rb +25 -0
- data/lib/quandl/command/presenters/superset_presenter.rb +10 -0
- data/lib/quandl/command/task.rb +40 -0
- data/lib/quandl/command/task/callbacks.rb +47 -0
- data/lib/quandl/command/task/clientable.rb +65 -0
- data/lib/quandl/command/task/commandable.rb +104 -0
- data/lib/quandl/command/task/configurable.rb +79 -0
- data/lib/quandl/command/task/inputable.rb +37 -0
- data/lib/quandl/command/task/logging.rb +83 -0
- data/lib/quandl/command/task/presentation.rb +37 -0
- data/lib/quandl/command/task/reportable.rb +35 -0
- data/lib/quandl/command/task/threading.rb +117 -0
- data/lib/quandl/command/task/translations.rb +37 -0
- data/lib/quandl/command/task/updatable.rb +77 -0
- data/lib/quandl/command/tasks.rb +6 -5
- data/lib/quandl/command/tasks/delete.rb +10 -67
- data/lib/quandl/command/tasks/download.rb +11 -78
- data/lib/quandl/command/tasks/info.rb +2 -2
- data/lib/quandl/command/tasks/list.rb +12 -24
- data/lib/quandl/command/tasks/login.rb +4 -4
- data/lib/quandl/command/tasks/replace.rb +58 -0
- data/lib/quandl/command/tasks/schedule.rb +106 -0
- data/lib/quandl/command/tasks/search.rb +39 -0
- data/lib/quandl/command/tasks/superset.rb +59 -0
- data/lib/quandl/command/tasks/uninstall.rb +1 -1
- data/lib/quandl/command/tasks/update.rb +2 -2
- data/lib/quandl/command/tasks/upload.rb +13 -26
- data/lib/quandl/command/version.rb +1 -1
- data/quandl.gemspec +5 -3
- data/spec/fixtures/scraper.rb +6 -0
- data/spec/lib/quandl/command/delete_spec.rb +19 -11
- data/spec/lib/quandl/command/download_spec.rb +11 -4
- data/spec/lib/quandl/command/replace_spec.rb +23 -0
- data/spec/lib/quandl/command/schedule_spec.rb +53 -0
- data/spec/lib/quandl/command/superset_spec.rb +28 -0
- data/spec/lib/quandl/command/upload_spec.rb +71 -24
- data/spec/lib/quandl/command_spec.rb +1 -9
- data/spec/spec_helper.rb +36 -1
- data/tasks/toolbelt/build/tarball.rb +2 -2
- metadata +55 -10
- data/lib/quandl/command/qconfig.rb +0 -86
- data/lib/quandl/command/tasks/base.rb +0 -314
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'quandl/command/presenter/record'
|
2
|
+
|
3
|
+
require 'quandl/command/presenters/dataset_presenter'
|
4
|
+
require 'quandl/command/presenters/nil_class_presenter'
|
5
|
+
require 'quandl/command/presenters/error_presenter'
|
6
|
+
require 'quandl/command/presenters/scraper_presenter'
|
7
|
+
require 'quandl/command/presenters/superset_presenter'
|
8
|
+
|
9
|
+
module Quandl
|
10
|
+
module Command
|
11
|
+
class Presenter
|
12
|
+
|
13
|
+
class << self
|
14
|
+
|
15
|
+
def wrap_record_in_presenter(record, options)
|
16
|
+
klass = find_presenter_for_record(record, options)
|
17
|
+
klass.new(record, options)
|
18
|
+
end
|
19
|
+
|
20
|
+
def find_presenter_for_record(record, options={})
|
21
|
+
name = 'Error' if record.kind_of?(Exception)
|
22
|
+
name = options[:as] || record.class.name.to_s.split("::").last unless name.present?
|
23
|
+
"Quandl::Command::Presenters::#{name}Presenter".constantize rescue Quandl::Command::Presenter::Record
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
attr_accessor :options, :type
|
29
|
+
|
30
|
+
def initialize( object, opts={} )
|
31
|
+
self.type = ( object.try(:class) == Her::Collection ) ? :docs : :doc
|
32
|
+
self.options = { type: type }.merge( opts.symbolize_keys! )
|
33
|
+
self.collection = object
|
34
|
+
end
|
35
|
+
|
36
|
+
def each(&block)
|
37
|
+
collection.each do |presenter|
|
38
|
+
block.call( presenter )
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def each_to_format(type, &block)
|
43
|
+
collection.each do |record|
|
44
|
+
output = record.send("to_#{type}") if record.respond_to?("to_#{type}")
|
45
|
+
block.call( output, record.object )
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def to_format(type)
|
50
|
+
output = []
|
51
|
+
collection.each do |record|
|
52
|
+
output << record.send("to_#{type}") if record.respond_to?("to_#{type}")
|
53
|
+
end
|
54
|
+
output.join("\n")
|
55
|
+
end
|
56
|
+
|
57
|
+
def collection=(records)
|
58
|
+
@collection = []
|
59
|
+
records = Array(records).flatten
|
60
|
+
records.each do |record|
|
61
|
+
@collection << self.class.wrap_record_in_presenter(record, options)
|
62
|
+
end
|
63
|
+
# present nil if the collection is empty
|
64
|
+
@collection = [self.class.wrap_record_in_presenter( nil, options )] if @collection.blank?
|
65
|
+
@collection
|
66
|
+
end
|
67
|
+
|
68
|
+
def collection
|
69
|
+
@collection ||= []
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Presenter
|
4
|
+
|
5
|
+
class Record
|
6
|
+
|
7
|
+
attr_accessor :object, :options
|
8
|
+
|
9
|
+
def initialize(*args)
|
10
|
+
self.options = args.extract_options!
|
11
|
+
self.object = args.first
|
12
|
+
end
|
13
|
+
|
14
|
+
# supported formats
|
15
|
+
|
16
|
+
def to_format(f)
|
17
|
+
self.send("to_#{f}") if self.respond_to?("to_#{f}")
|
18
|
+
end
|
19
|
+
|
20
|
+
|
21
|
+
def to_pipes
|
22
|
+
column = options[:output_column] if options.present? && options.is_a?(Hash)
|
23
|
+
pipes = as_pipes.dup.compact
|
24
|
+
pipes = [ pipes[ column.to_i ] ] if column.present? && column.to_s.numeric?
|
25
|
+
pipes.join(" | ")
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_stderr
|
29
|
+
as_stderr
|
30
|
+
end
|
31
|
+
|
32
|
+
def as_stderr
|
33
|
+
output = ''
|
34
|
+
output += "#{id}\n" if id.present?
|
35
|
+
output += "#{detail}\n" if detail.present?
|
36
|
+
output += errors if errors.present?
|
37
|
+
output += object.to_s if object.kind_of?(Exception)
|
38
|
+
output += object.backtrace.join("\n") if object.respond_to?(:backtrace) && options[:trace] == true
|
39
|
+
output += "\n---\n"
|
40
|
+
end
|
41
|
+
|
42
|
+
def to_qdf
|
43
|
+
object.try(:to_qdf)
|
44
|
+
end
|
45
|
+
|
46
|
+
def to_csv
|
47
|
+
object.try(:to_csv)
|
48
|
+
end
|
49
|
+
|
50
|
+
def to_json
|
51
|
+
as_json.try(:to_json)
|
52
|
+
end
|
53
|
+
|
54
|
+
def as_pipes
|
55
|
+
[ status, id, detail, message ]
|
56
|
+
end
|
57
|
+
|
58
|
+
def as_json
|
59
|
+
json = { status: status, id: id, detail: detail, message: message }
|
60
|
+
json[:attributes] = object.attributes if object.respond_to?(:attributes)
|
61
|
+
json[:metadata] = object.metadata if object.respond_to?(:metadata)
|
62
|
+
json[:errors] = object.error_messages if object.respond_to?(:error_messages)
|
63
|
+
json
|
64
|
+
end
|
65
|
+
|
66
|
+
# default attribute mappings
|
67
|
+
|
68
|
+
def valid?
|
69
|
+
return true if object.nil?
|
70
|
+
return true if [200,201,404].include?(object.try(:status))
|
71
|
+
false
|
72
|
+
end
|
73
|
+
|
74
|
+
def status
|
75
|
+
return Quandl::Client::HTTP_STATUS_CODES[422] if object.try(:errors).try(:present?)
|
76
|
+
return object.try(:human_status) if object.respond_to?(:human_status) && object.human_status.present?
|
77
|
+
Quandl::Client::HTTP_STATUS_CODES[404]
|
78
|
+
end
|
79
|
+
|
80
|
+
def id
|
81
|
+
object.try(:id)
|
82
|
+
end
|
83
|
+
|
84
|
+
def message
|
85
|
+
errors_without_spaces
|
86
|
+
end
|
87
|
+
|
88
|
+
def detail
|
89
|
+
end
|
90
|
+
|
91
|
+
def errors_without_spaces
|
92
|
+
errors.to_s.gsub("\n", ' ')
|
93
|
+
end
|
94
|
+
|
95
|
+
def errors
|
96
|
+
result = []
|
97
|
+
messages = object.try(:error_messages) || []
|
98
|
+
# extract repsonse errors
|
99
|
+
re = messages.delete(:response_errors) if messages.is_a?(Hash) && messages.has_key?(:response_errors)
|
100
|
+
re.each{|k,v| messages[k] = v } if re.present? && re.is_a?(Hash)
|
101
|
+
# format
|
102
|
+
messages.each do |k,v|
|
103
|
+
v = v.join(" & ") if v.respond_to?(:join)
|
104
|
+
result << "#{k}: #{v}"
|
105
|
+
end
|
106
|
+
result.join(", ")
|
107
|
+
end
|
108
|
+
|
109
|
+
def docs?
|
110
|
+
options[:type] == :docs
|
111
|
+
end
|
112
|
+
|
113
|
+
def doc?
|
114
|
+
options[:type] == :doc
|
115
|
+
end
|
116
|
+
|
117
|
+
end
|
118
|
+
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Presenters
|
4
|
+
|
5
|
+
class ErrorPresenter < Quandl::Command::Presenter::Record
|
6
|
+
|
7
|
+
def as_pipes
|
8
|
+
[ status, id, detail, message.gsub(/\n+/, '. ') ]
|
9
|
+
end
|
10
|
+
|
11
|
+
def status
|
12
|
+
Quandl::Client::HTTP_STATUS_CODES[422]
|
13
|
+
end
|
14
|
+
|
15
|
+
def id
|
16
|
+
'Error'
|
17
|
+
end
|
18
|
+
|
19
|
+
def detail
|
20
|
+
object.class.to_s
|
21
|
+
end
|
22
|
+
|
23
|
+
def message
|
24
|
+
object.to_s
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Presenters
|
4
|
+
|
5
|
+
class ScraperPresenter < Quandl::Command::Presenter::Record
|
6
|
+
|
7
|
+
def as_pipes
|
8
|
+
pipes = [object.name]
|
9
|
+
#pipes += [object.scraper_url, object.git_url, object.git_reference].compact if doc?
|
10
|
+
pipes
|
11
|
+
end
|
12
|
+
|
13
|
+
def id
|
14
|
+
object.try(:name)
|
15
|
+
end
|
16
|
+
|
17
|
+
def detail
|
18
|
+
object.try(:type)
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'active_model'
|
2
|
+
require 'quandl/pattern'
|
3
|
+
require 'quandl/pattern/client'
|
4
|
+
require 'quandl/operation'
|
5
|
+
|
6
|
+
require 'quandl/command/task/configurable'
|
7
|
+
require 'quandl/command/task/callbacks'
|
8
|
+
require 'quandl/command/task/reportable'
|
9
|
+
require 'quandl/command/task/commandable'
|
10
|
+
require 'quandl/command/task/logging'
|
11
|
+
require 'quandl/command/task/presentation'
|
12
|
+
require 'quandl/command/task/threading'
|
13
|
+
require 'quandl/command/task/translations'
|
14
|
+
require 'quandl/command/task/updatable'
|
15
|
+
require 'quandl/command/task/clientable'
|
16
|
+
require 'quandl/command/task/inputable'
|
17
|
+
|
18
|
+
module Quandl
|
19
|
+
module Command
|
20
|
+
|
21
|
+
class Task
|
22
|
+
|
23
|
+
include ActiveModel::Validations
|
24
|
+
|
25
|
+
include Quandl::Command::Task::Configurable
|
26
|
+
include Quandl::Command::Task::Callbacks
|
27
|
+
include Quandl::Command::Task::Commandable
|
28
|
+
include Quandl::Command::Task::Logging
|
29
|
+
include Quandl::Command::Task::Presentation
|
30
|
+
include Quandl::Command::Task::Threading
|
31
|
+
include Quandl::Command::Task::Translations
|
32
|
+
include Quandl::Command::Task::Updatable
|
33
|
+
include Quandl::Command::Task::Clientable
|
34
|
+
include Quandl::Command::Task::Reportable
|
35
|
+
include Quandl::Command::Task::Inputable
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Task
|
4
|
+
|
5
|
+
module Callbacks
|
6
|
+
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
|
11
|
+
extend ActiveModel::Callbacks
|
12
|
+
define_model_callbacks :execute
|
13
|
+
|
14
|
+
before_execute :run_task_validations!
|
15
|
+
|
16
|
+
end
|
17
|
+
|
18
|
+
module ClassMethods
|
19
|
+
|
20
|
+
def disable_in_gem!
|
21
|
+
before_execute :disable_in_gem!
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
|
26
|
+
def run_task_validations!
|
27
|
+
unless valid?
|
28
|
+
error( table(errors.full_messages) )
|
29
|
+
false
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def disable_in_gem!
|
34
|
+
if force_yes?
|
35
|
+
info("You have forced update!")
|
36
|
+
true
|
37
|
+
elsif Dir.exists?( File.join( Tasks.root, ".git") ) || File.exists?( File.join( Tasks.root, "Gemfile") )
|
38
|
+
fatal("#{self.class.command_name} is only permitted when installed as a package! http://quandl.com/help/toolbelt")
|
39
|
+
false
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Task
|
4
|
+
|
5
|
+
module Clientable
|
6
|
+
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
end
|
11
|
+
|
12
|
+
module ClassMethods
|
13
|
+
|
14
|
+
def authenticated_users_only!
|
15
|
+
before_execute :authenticated_users_only!
|
16
|
+
end
|
17
|
+
|
18
|
+
def warn_unauthenticated_users
|
19
|
+
before_execute :warn_unauthenticated_users
|
20
|
+
end
|
21
|
+
|
22
|
+
def autoload_quandl_client
|
23
|
+
before_execute :autoload_quandl_client
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
def current_user
|
29
|
+
@current_user ||= Quandl::Client::User.info
|
30
|
+
end
|
31
|
+
|
32
|
+
protected
|
33
|
+
|
34
|
+
def clear_session!
|
35
|
+
@auth_token = nil
|
36
|
+
@current_user = nil
|
37
|
+
autoload_quandl_client
|
38
|
+
end
|
39
|
+
|
40
|
+
def authenticated_users_only!
|
41
|
+
if auth_token.blank?
|
42
|
+
fatal("You must authenticate to use #{self.class.command_name}! 'quandl login' OR --token xyz923")
|
43
|
+
false
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def warn_unauthenticated_users
|
48
|
+
error("WARN: Authenticate your requests! 'quandl login' OR --token xyz923") if auth_token.blank?
|
49
|
+
end
|
50
|
+
|
51
|
+
def autoload_quandl_client
|
52
|
+
require 'thread/pool'
|
53
|
+
require 'quandl/format'
|
54
|
+
require 'quandl/command/client_ext'
|
55
|
+
Quandl::Client.use( quandl_url )
|
56
|
+
Quandl::Client.token = auth_token
|
57
|
+
Quandl::Client.request_source = 'quandl_command'
|
58
|
+
Quandl::Client.request_version = Quandl::Command::VERSION
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|