quandl 0.2.27 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/README.md +70 -17
- data/Rakefile +7 -86
- data/UPGRADE.md +23 -0
- data/VERSION +1 -0
- data/lib/quandl/command.rb +11 -3
- data/lib/quandl/command/config.rb +88 -0
- data/lib/quandl/command/presenter.rb +74 -0
- data/lib/quandl/command/presenter/record.rb +121 -0
- data/lib/quandl/command/presenters/dataset_presenter.rb +19 -0
- data/lib/quandl/command/presenters/error_presenter.rb +31 -0
- data/lib/quandl/command/presenters/nil_class_presenter.rb +10 -0
- data/lib/quandl/command/presenters/scraper_presenter.rb +25 -0
- data/lib/quandl/command/presenters/superset_presenter.rb +10 -0
- data/lib/quandl/command/task.rb +40 -0
- data/lib/quandl/command/task/callbacks.rb +47 -0
- data/lib/quandl/command/task/clientable.rb +65 -0
- data/lib/quandl/command/task/commandable.rb +104 -0
- data/lib/quandl/command/task/configurable.rb +79 -0
- data/lib/quandl/command/task/inputable.rb +37 -0
- data/lib/quandl/command/task/logging.rb +83 -0
- data/lib/quandl/command/task/presentation.rb +37 -0
- data/lib/quandl/command/task/reportable.rb +35 -0
- data/lib/quandl/command/task/threading.rb +117 -0
- data/lib/quandl/command/task/translations.rb +37 -0
- data/lib/quandl/command/task/updatable.rb +77 -0
- data/lib/quandl/command/tasks.rb +6 -5
- data/lib/quandl/command/tasks/delete.rb +10 -67
- data/lib/quandl/command/tasks/download.rb +11 -78
- data/lib/quandl/command/tasks/info.rb +2 -2
- data/lib/quandl/command/tasks/list.rb +12 -24
- data/lib/quandl/command/tasks/login.rb +4 -4
- data/lib/quandl/command/tasks/replace.rb +58 -0
- data/lib/quandl/command/tasks/schedule.rb +106 -0
- data/lib/quandl/command/tasks/search.rb +39 -0
- data/lib/quandl/command/tasks/superset.rb +59 -0
- data/lib/quandl/command/tasks/uninstall.rb +1 -1
- data/lib/quandl/command/tasks/update.rb +2 -2
- data/lib/quandl/command/tasks/upload.rb +13 -26
- data/lib/quandl/command/version.rb +1 -1
- data/quandl.gemspec +5 -3
- data/spec/fixtures/scraper.rb +6 -0
- data/spec/lib/quandl/command/delete_spec.rb +19 -11
- data/spec/lib/quandl/command/download_spec.rb +11 -4
- data/spec/lib/quandl/command/replace_spec.rb +23 -0
- data/spec/lib/quandl/command/schedule_spec.rb +53 -0
- data/spec/lib/quandl/command/superset_spec.rb +28 -0
- data/spec/lib/quandl/command/upload_spec.rb +71 -24
- data/spec/lib/quandl/command_spec.rb +1 -9
- data/spec/spec_helper.rb +36 -1
- data/tasks/toolbelt/build/tarball.rb +2 -2
- metadata +55 -10
- data/lib/quandl/command/qconfig.rb +0 -86
- data/lib/quandl/command/tasks/base.rb +0 -314
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'quandl/command/presenter/record'
|
2
|
+
|
3
|
+
require 'quandl/command/presenters/dataset_presenter'
|
4
|
+
require 'quandl/command/presenters/nil_class_presenter'
|
5
|
+
require 'quandl/command/presenters/error_presenter'
|
6
|
+
require 'quandl/command/presenters/scraper_presenter'
|
7
|
+
require 'quandl/command/presenters/superset_presenter'
|
8
|
+
|
9
|
+
module Quandl
|
10
|
+
module Command
|
11
|
+
class Presenter
|
12
|
+
|
13
|
+
class << self
|
14
|
+
|
15
|
+
def wrap_record_in_presenter(record, options)
|
16
|
+
klass = find_presenter_for_record(record, options)
|
17
|
+
klass.new(record, options)
|
18
|
+
end
|
19
|
+
|
20
|
+
def find_presenter_for_record(record, options={})
|
21
|
+
name = 'Error' if record.kind_of?(Exception)
|
22
|
+
name = options[:as] || record.class.name.to_s.split("::").last unless name.present?
|
23
|
+
"Quandl::Command::Presenters::#{name}Presenter".constantize rescue Quandl::Command::Presenter::Record
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
attr_accessor :options, :type
|
29
|
+
|
30
|
+
def initialize( object, opts={} )
|
31
|
+
self.type = ( object.try(:class) == Her::Collection ) ? :docs : :doc
|
32
|
+
self.options = { type: type }.merge( opts.symbolize_keys! )
|
33
|
+
self.collection = object
|
34
|
+
end
|
35
|
+
|
36
|
+
def each(&block)
|
37
|
+
collection.each do |presenter|
|
38
|
+
block.call( presenter )
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def each_to_format(type, &block)
|
43
|
+
collection.each do |record|
|
44
|
+
output = record.send("to_#{type}") if record.respond_to?("to_#{type}")
|
45
|
+
block.call( output, record.object )
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def to_format(type)
|
50
|
+
output = []
|
51
|
+
collection.each do |record|
|
52
|
+
output << record.send("to_#{type}") if record.respond_to?("to_#{type}")
|
53
|
+
end
|
54
|
+
output.join("\n")
|
55
|
+
end
|
56
|
+
|
57
|
+
def collection=(records)
|
58
|
+
@collection = []
|
59
|
+
records = Array(records).flatten
|
60
|
+
records.each do |record|
|
61
|
+
@collection << self.class.wrap_record_in_presenter(record, options)
|
62
|
+
end
|
63
|
+
# present nil if the collection is empty
|
64
|
+
@collection = [self.class.wrap_record_in_presenter( nil, options )] if @collection.blank?
|
65
|
+
@collection
|
66
|
+
end
|
67
|
+
|
68
|
+
def collection
|
69
|
+
@collection ||= []
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Presenter
|
4
|
+
|
5
|
+
class Record
|
6
|
+
|
7
|
+
attr_accessor :object, :options
|
8
|
+
|
9
|
+
def initialize(*args)
|
10
|
+
self.options = args.extract_options!
|
11
|
+
self.object = args.first
|
12
|
+
end
|
13
|
+
|
14
|
+
# supported formats
|
15
|
+
|
16
|
+
def to_format(f)
|
17
|
+
self.send("to_#{f}") if self.respond_to?("to_#{f}")
|
18
|
+
end
|
19
|
+
|
20
|
+
|
21
|
+
def to_pipes
|
22
|
+
column = options[:output_column] if options.present? && options.is_a?(Hash)
|
23
|
+
pipes = as_pipes.dup.compact
|
24
|
+
pipes = [ pipes[ column.to_i ] ] if column.present? && column.to_s.numeric?
|
25
|
+
pipes.join(" | ")
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_stderr
|
29
|
+
as_stderr
|
30
|
+
end
|
31
|
+
|
32
|
+
def as_stderr
|
33
|
+
output = ''
|
34
|
+
output += "#{id}\n" if id.present?
|
35
|
+
output += "#{detail}\n" if detail.present?
|
36
|
+
output += errors if errors.present?
|
37
|
+
output += object.to_s if object.kind_of?(Exception)
|
38
|
+
output += object.backtrace.join("\n") if object.respond_to?(:backtrace) && options[:trace] == true
|
39
|
+
output += "\n---\n"
|
40
|
+
end
|
41
|
+
|
42
|
+
def to_qdf
|
43
|
+
object.try(:to_qdf)
|
44
|
+
end
|
45
|
+
|
46
|
+
def to_csv
|
47
|
+
object.try(:to_csv)
|
48
|
+
end
|
49
|
+
|
50
|
+
def to_json
|
51
|
+
as_json.try(:to_json)
|
52
|
+
end
|
53
|
+
|
54
|
+
def as_pipes
|
55
|
+
[ status, id, detail, message ]
|
56
|
+
end
|
57
|
+
|
58
|
+
def as_json
|
59
|
+
json = { status: status, id: id, detail: detail, message: message }
|
60
|
+
json[:attributes] = object.attributes if object.respond_to?(:attributes)
|
61
|
+
json[:metadata] = object.metadata if object.respond_to?(:metadata)
|
62
|
+
json[:errors] = object.error_messages if object.respond_to?(:error_messages)
|
63
|
+
json
|
64
|
+
end
|
65
|
+
|
66
|
+
# default attribute mappings
|
67
|
+
|
68
|
+
def valid?
|
69
|
+
return true if object.nil?
|
70
|
+
return true if [200,201,404].include?(object.try(:status))
|
71
|
+
false
|
72
|
+
end
|
73
|
+
|
74
|
+
def status
|
75
|
+
return Quandl::Client::HTTP_STATUS_CODES[422] if object.try(:errors).try(:present?)
|
76
|
+
return object.try(:human_status) if object.respond_to?(:human_status) && object.human_status.present?
|
77
|
+
Quandl::Client::HTTP_STATUS_CODES[404]
|
78
|
+
end
|
79
|
+
|
80
|
+
def id
|
81
|
+
object.try(:id)
|
82
|
+
end
|
83
|
+
|
84
|
+
def message
|
85
|
+
errors_without_spaces
|
86
|
+
end
|
87
|
+
|
88
|
+
def detail
|
89
|
+
end
|
90
|
+
|
91
|
+
def errors_without_spaces
|
92
|
+
errors.to_s.gsub("\n", ' ')
|
93
|
+
end
|
94
|
+
|
95
|
+
def errors
|
96
|
+
result = []
|
97
|
+
messages = object.try(:error_messages) || []
|
98
|
+
# extract repsonse errors
|
99
|
+
re = messages.delete(:response_errors) if messages.is_a?(Hash) && messages.has_key?(:response_errors)
|
100
|
+
re.each{|k,v| messages[k] = v } if re.present? && re.is_a?(Hash)
|
101
|
+
# format
|
102
|
+
messages.each do |k,v|
|
103
|
+
v = v.join(" & ") if v.respond_to?(:join)
|
104
|
+
result << "#{k}: #{v}"
|
105
|
+
end
|
106
|
+
result.join(", ")
|
107
|
+
end
|
108
|
+
|
109
|
+
def docs?
|
110
|
+
options[:type] == :docs
|
111
|
+
end
|
112
|
+
|
113
|
+
def doc?
|
114
|
+
options[:type] == :doc
|
115
|
+
end
|
116
|
+
|
117
|
+
end
|
118
|
+
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Presenters
|
4
|
+
|
5
|
+
class ErrorPresenter < Quandl::Command::Presenter::Record
|
6
|
+
|
7
|
+
def as_pipes
|
8
|
+
[ status, id, detail, message.gsub(/\n+/, '. ') ]
|
9
|
+
end
|
10
|
+
|
11
|
+
def status
|
12
|
+
Quandl::Client::HTTP_STATUS_CODES[422]
|
13
|
+
end
|
14
|
+
|
15
|
+
def id
|
16
|
+
'Error'
|
17
|
+
end
|
18
|
+
|
19
|
+
def detail
|
20
|
+
object.class.to_s
|
21
|
+
end
|
22
|
+
|
23
|
+
def message
|
24
|
+
object.to_s
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Presenters
|
4
|
+
|
5
|
+
class ScraperPresenter < Quandl::Command::Presenter::Record
|
6
|
+
|
7
|
+
def as_pipes
|
8
|
+
pipes = [object.name]
|
9
|
+
#pipes += [object.scraper_url, object.git_url, object.git_reference].compact if doc?
|
10
|
+
pipes
|
11
|
+
end
|
12
|
+
|
13
|
+
def id
|
14
|
+
object.try(:name)
|
15
|
+
end
|
16
|
+
|
17
|
+
def detail
|
18
|
+
object.try(:type)
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'active_model'
|
2
|
+
require 'quandl/pattern'
|
3
|
+
require 'quandl/pattern/client'
|
4
|
+
require 'quandl/operation'
|
5
|
+
|
6
|
+
require 'quandl/command/task/configurable'
|
7
|
+
require 'quandl/command/task/callbacks'
|
8
|
+
require 'quandl/command/task/reportable'
|
9
|
+
require 'quandl/command/task/commandable'
|
10
|
+
require 'quandl/command/task/logging'
|
11
|
+
require 'quandl/command/task/presentation'
|
12
|
+
require 'quandl/command/task/threading'
|
13
|
+
require 'quandl/command/task/translations'
|
14
|
+
require 'quandl/command/task/updatable'
|
15
|
+
require 'quandl/command/task/clientable'
|
16
|
+
require 'quandl/command/task/inputable'
|
17
|
+
|
18
|
+
module Quandl
|
19
|
+
module Command
|
20
|
+
|
21
|
+
class Task
|
22
|
+
|
23
|
+
include ActiveModel::Validations
|
24
|
+
|
25
|
+
include Quandl::Command::Task::Configurable
|
26
|
+
include Quandl::Command::Task::Callbacks
|
27
|
+
include Quandl::Command::Task::Commandable
|
28
|
+
include Quandl::Command::Task::Logging
|
29
|
+
include Quandl::Command::Task::Presentation
|
30
|
+
include Quandl::Command::Task::Threading
|
31
|
+
include Quandl::Command::Task::Translations
|
32
|
+
include Quandl::Command::Task::Updatable
|
33
|
+
include Quandl::Command::Task::Clientable
|
34
|
+
include Quandl::Command::Task::Reportable
|
35
|
+
include Quandl::Command::Task::Inputable
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Task
|
4
|
+
|
5
|
+
module Callbacks
|
6
|
+
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
|
11
|
+
extend ActiveModel::Callbacks
|
12
|
+
define_model_callbacks :execute
|
13
|
+
|
14
|
+
before_execute :run_task_validations!
|
15
|
+
|
16
|
+
end
|
17
|
+
|
18
|
+
module ClassMethods
|
19
|
+
|
20
|
+
def disable_in_gem!
|
21
|
+
before_execute :disable_in_gem!
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
|
26
|
+
def run_task_validations!
|
27
|
+
unless valid?
|
28
|
+
error( table(errors.full_messages) )
|
29
|
+
false
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def disable_in_gem!
|
34
|
+
if force_yes?
|
35
|
+
info("You have forced update!")
|
36
|
+
true
|
37
|
+
elsif Dir.exists?( File.join( Tasks.root, ".git") ) || File.exists?( File.join( Tasks.root, "Gemfile") )
|
38
|
+
fatal("#{self.class.command_name} is only permitted when installed as a package! http://quandl.com/help/toolbelt")
|
39
|
+
false
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Task
|
4
|
+
|
5
|
+
module Clientable
|
6
|
+
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
end
|
11
|
+
|
12
|
+
module ClassMethods
|
13
|
+
|
14
|
+
def authenticated_users_only!
|
15
|
+
before_execute :authenticated_users_only!
|
16
|
+
end
|
17
|
+
|
18
|
+
def warn_unauthenticated_users
|
19
|
+
before_execute :warn_unauthenticated_users
|
20
|
+
end
|
21
|
+
|
22
|
+
def autoload_quandl_client
|
23
|
+
before_execute :autoload_quandl_client
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
def current_user
|
29
|
+
@current_user ||= Quandl::Client::User.info
|
30
|
+
end
|
31
|
+
|
32
|
+
protected
|
33
|
+
|
34
|
+
def clear_session!
|
35
|
+
@auth_token = nil
|
36
|
+
@current_user = nil
|
37
|
+
autoload_quandl_client
|
38
|
+
end
|
39
|
+
|
40
|
+
def authenticated_users_only!
|
41
|
+
if auth_token.blank?
|
42
|
+
fatal("You must authenticate to use #{self.class.command_name}! 'quandl login' OR --token xyz923")
|
43
|
+
false
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def warn_unauthenticated_users
|
48
|
+
error("WARN: Authenticate your requests! 'quandl login' OR --token xyz923") if auth_token.blank?
|
49
|
+
end
|
50
|
+
|
51
|
+
def autoload_quandl_client
|
52
|
+
require 'thread/pool'
|
53
|
+
require 'quandl/format'
|
54
|
+
require 'quandl/command/client_ext'
|
55
|
+
Quandl::Client.use( quandl_url )
|
56
|
+
Quandl::Client.token = auth_token
|
57
|
+
Quandl::Client.request_source = 'quandl_command'
|
58
|
+
Quandl::Client.request_version = Quandl::Command::VERSION
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|