quandl 0.2.27 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/README.md +70 -17
- data/Rakefile +7 -86
- data/UPGRADE.md +23 -0
- data/VERSION +1 -0
- data/lib/quandl/command.rb +11 -3
- data/lib/quandl/command/config.rb +88 -0
- data/lib/quandl/command/presenter.rb +74 -0
- data/lib/quandl/command/presenter/record.rb +121 -0
- data/lib/quandl/command/presenters/dataset_presenter.rb +19 -0
- data/lib/quandl/command/presenters/error_presenter.rb +31 -0
- data/lib/quandl/command/presenters/nil_class_presenter.rb +10 -0
- data/lib/quandl/command/presenters/scraper_presenter.rb +25 -0
- data/lib/quandl/command/presenters/superset_presenter.rb +10 -0
- data/lib/quandl/command/task.rb +40 -0
- data/lib/quandl/command/task/callbacks.rb +47 -0
- data/lib/quandl/command/task/clientable.rb +65 -0
- data/lib/quandl/command/task/commandable.rb +104 -0
- data/lib/quandl/command/task/configurable.rb +79 -0
- data/lib/quandl/command/task/inputable.rb +37 -0
- data/lib/quandl/command/task/logging.rb +83 -0
- data/lib/quandl/command/task/presentation.rb +37 -0
- data/lib/quandl/command/task/reportable.rb +35 -0
- data/lib/quandl/command/task/threading.rb +117 -0
- data/lib/quandl/command/task/translations.rb +37 -0
- data/lib/quandl/command/task/updatable.rb +77 -0
- data/lib/quandl/command/tasks.rb +6 -5
- data/lib/quandl/command/tasks/delete.rb +10 -67
- data/lib/quandl/command/tasks/download.rb +11 -78
- data/lib/quandl/command/tasks/info.rb +2 -2
- data/lib/quandl/command/tasks/list.rb +12 -24
- data/lib/quandl/command/tasks/login.rb +4 -4
- data/lib/quandl/command/tasks/replace.rb +58 -0
- data/lib/quandl/command/tasks/schedule.rb +106 -0
- data/lib/quandl/command/tasks/search.rb +39 -0
- data/lib/quandl/command/tasks/superset.rb +59 -0
- data/lib/quandl/command/tasks/uninstall.rb +1 -1
- data/lib/quandl/command/tasks/update.rb +2 -2
- data/lib/quandl/command/tasks/upload.rb +13 -26
- data/lib/quandl/command/version.rb +1 -1
- data/quandl.gemspec +5 -3
- data/spec/fixtures/scraper.rb +6 -0
- data/spec/lib/quandl/command/delete_spec.rb +19 -11
- data/spec/lib/quandl/command/download_spec.rb +11 -4
- data/spec/lib/quandl/command/replace_spec.rb +23 -0
- data/spec/lib/quandl/command/schedule_spec.rb +53 -0
- data/spec/lib/quandl/command/superset_spec.rb +28 -0
- data/spec/lib/quandl/command/upload_spec.rb +71 -24
- data/spec/lib/quandl/command_spec.rb +1 -9
- data/spec/spec_helper.rb +36 -1
- data/tasks/toolbelt/build/tarball.rb +2 -2
- metadata +55 -10
- data/lib/quandl/command/qconfig.rb +0 -86
- data/lib/quandl/command/tasks/base.rb +0 -314
@@ -0,0 +1,104 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Task
|
4
|
+
|
5
|
+
module Commandable
|
6
|
+
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
attr_accessor :args, :options
|
11
|
+
end
|
12
|
+
|
13
|
+
module ClassMethods
|
14
|
+
|
15
|
+
def inherited(klass)
|
16
|
+
Quandl::Command::Tasks.tasks << klass unless Quandl::Command::Tasks.tasks.include?(klass)
|
17
|
+
klass.options(@options)
|
18
|
+
end
|
19
|
+
|
20
|
+
def configure(app)
|
21
|
+
return if disabled?
|
22
|
+
app.command(command_name) do |c|
|
23
|
+
c.syntax = syntax
|
24
|
+
c.description = description
|
25
|
+
c.action{|a,o| call(a,o) }
|
26
|
+
configure_options(c)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def disabled?
|
31
|
+
@disabled == true
|
32
|
+
end
|
33
|
+
|
34
|
+
def disable!
|
35
|
+
@disabled = true
|
36
|
+
end
|
37
|
+
|
38
|
+
def syntax(value=nil)
|
39
|
+
@syntax = value if value.present?
|
40
|
+
@syntax ||= "quandl #{command_name}"
|
41
|
+
end
|
42
|
+
|
43
|
+
def command_name(value=nil)
|
44
|
+
@command_name = value if value.present?
|
45
|
+
@command_name ||= name.to_s.split("::").last.downcase
|
46
|
+
end
|
47
|
+
|
48
|
+
def description(value=nil)
|
49
|
+
@description = value if value.present?
|
50
|
+
@description ||= "No description."
|
51
|
+
end
|
52
|
+
|
53
|
+
def options(value=nil)
|
54
|
+
@options = value if value.present?
|
55
|
+
@options ||= {}
|
56
|
+
end
|
57
|
+
|
58
|
+
def call(args=[], options={})
|
59
|
+
args = Array(args)
|
60
|
+
options = ensure_options_are_command_options!(options)
|
61
|
+
self.new( args, options ).call
|
62
|
+
end
|
63
|
+
|
64
|
+
protected
|
65
|
+
|
66
|
+
def ensure_options_are_command_options!(options)
|
67
|
+
return options if options.class == Commander::Command::Options
|
68
|
+
OpenStruct.new(options)
|
69
|
+
end
|
70
|
+
|
71
|
+
def configure_options(c)
|
72
|
+
options.each do |class_type, options|
|
73
|
+
options.each do |name, desc|
|
74
|
+
c.option "--#{name} #{class_type.to_s.upcase}", class_type, desc
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
80
|
+
|
81
|
+
def call
|
82
|
+
run_callbacks(:execute) do
|
83
|
+
execute
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def execute
|
88
|
+
# fire a subcommand if specified
|
89
|
+
return self.send(args.shift) if args.first.present? && self.respond_to?(args.first)
|
90
|
+
# otherwise show syntax
|
91
|
+
puts " SYNTAX\n\n #{self.class.syntax}"
|
92
|
+
end
|
93
|
+
|
94
|
+
def initialize(args, options)
|
95
|
+
self.args = args
|
96
|
+
self.options = options
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Task
|
4
|
+
|
5
|
+
module Configurable
|
6
|
+
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
delegate :auth_token, :quandl_url, to: :config
|
11
|
+
end
|
12
|
+
|
13
|
+
module ClassMethods
|
14
|
+
end
|
15
|
+
|
16
|
+
def declared_params
|
17
|
+
params = {}
|
18
|
+
self.class.options.each do |class_type, opts|
|
19
|
+
opts.each do |name, desc|
|
20
|
+
if options.is_a?(OpenStruct)
|
21
|
+
params[name] = self.options.send(name)
|
22
|
+
else
|
23
|
+
params[name] = self.options[name] if self.options[name].present?
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
params
|
28
|
+
end
|
29
|
+
|
30
|
+
def verbose?
|
31
|
+
options.verbose == true
|
32
|
+
end
|
33
|
+
|
34
|
+
def trace?
|
35
|
+
options.trace == true
|
36
|
+
end
|
37
|
+
|
38
|
+
def confirmed?
|
39
|
+
return true if force_yes?
|
40
|
+
ask_for_confirmation!
|
41
|
+
end
|
42
|
+
|
43
|
+
def force_yes?
|
44
|
+
options.force_yes == true
|
45
|
+
end
|
46
|
+
|
47
|
+
def ask_for_confirmation!
|
48
|
+
['y','yes'].include?( ask("Are you sure? (y/n)") )
|
49
|
+
end
|
50
|
+
|
51
|
+
def page
|
52
|
+
@page ||= options.page.to_i || 1
|
53
|
+
end
|
54
|
+
|
55
|
+
def config
|
56
|
+
@config ||= Quandl::Command::Config.new( config_options )
|
57
|
+
end
|
58
|
+
|
59
|
+
protected
|
60
|
+
|
61
|
+
def config_options
|
62
|
+
opts = { file_path: config_file_path }
|
63
|
+
opts.merge!(options.to_h) if options.to_h.present?
|
64
|
+
opts
|
65
|
+
end
|
66
|
+
|
67
|
+
def config_file_path
|
68
|
+
File.join( ENV['HOME'], '.quandl', config_profile )
|
69
|
+
end
|
70
|
+
|
71
|
+
def config_profile
|
72
|
+
options.environment || 'config'
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Task
|
4
|
+
|
5
|
+
module Inputable
|
6
|
+
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
end
|
11
|
+
|
12
|
+
module ClassMethods
|
13
|
+
end
|
14
|
+
|
15
|
+
def each_line_in_background(args, &block)
|
16
|
+
args.each_line do |arg|
|
17
|
+
background_job do
|
18
|
+
block.call( arg )
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def args_or_stdin
|
24
|
+
return args.join("\n") if args.first.present?
|
25
|
+
$stdin
|
26
|
+
end
|
27
|
+
|
28
|
+
def file_or_stdin
|
29
|
+
return File.open( args.first, 'r' ) if args.first.present?
|
30
|
+
$stdin
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Task
|
4
|
+
|
5
|
+
module Logging
|
6
|
+
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
before_execute :start_request_timer
|
11
|
+
before_execute :configure_logger
|
12
|
+
after_execute :log_request_time
|
13
|
+
|
14
|
+
attr_accessor :request_timer
|
15
|
+
|
16
|
+
end
|
17
|
+
|
18
|
+
def start_request_timer
|
19
|
+
self.request_timer = Time.now
|
20
|
+
end
|
21
|
+
|
22
|
+
def configure_logger
|
23
|
+
@stderr_logger = initialize_logger(config.stderr) if config.stderr.present?
|
24
|
+
@stdout_logger = initialize_logger(config.stdout) if config.stdout.present?
|
25
|
+
end
|
26
|
+
|
27
|
+
def initialize_logger(path)
|
28
|
+
l = ::Logger.new( path, 2, 52428800 )
|
29
|
+
l.formatter = proc do |severity, datetime, progname, msg|
|
30
|
+
"# [#{datetime.strftime("%Y-%m-%d %H:%M:%S")}]\n#{msg}"
|
31
|
+
end
|
32
|
+
l
|
33
|
+
end
|
34
|
+
|
35
|
+
def log_request_time
|
36
|
+
debug("# Started: #{request_timer}. Finished: #{Time.now}. Elapsed: #{request_timer.elapsed_ms}")
|
37
|
+
end
|
38
|
+
|
39
|
+
def summarize(item)
|
40
|
+
return summarize_hash(item) if item.kind_of?(Hash)
|
41
|
+
item
|
42
|
+
end
|
43
|
+
|
44
|
+
def summarize_hash(item)
|
45
|
+
item.collect do |k,v|
|
46
|
+
next "#{k}: '#{v}'" if v.kind_of?(String)
|
47
|
+
"#{k}: #{v}"
|
48
|
+
end.join(', ')
|
49
|
+
end
|
50
|
+
|
51
|
+
def table(*args)
|
52
|
+
Array(args).flatten.join(" | ")
|
53
|
+
end
|
54
|
+
|
55
|
+
def info(*args)
|
56
|
+
logger.info(*args)
|
57
|
+
end
|
58
|
+
|
59
|
+
def debug(*args)
|
60
|
+
args.each{|a| logger.debug("# #{a}") } if verbose?
|
61
|
+
end
|
62
|
+
|
63
|
+
def error(*args)
|
64
|
+
stderr_logger.error(*args)
|
65
|
+
end
|
66
|
+
|
67
|
+
def fatal(*args)
|
68
|
+
stderr_logger.fatal("FATAL: #{args.join(" ")}")
|
69
|
+
end
|
70
|
+
|
71
|
+
def logger
|
72
|
+
@stdout_logger ||= Quandl::Logger
|
73
|
+
end
|
74
|
+
|
75
|
+
def stderr_logger
|
76
|
+
@stderr_logger ||= Quandl::Logger
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Task
|
4
|
+
|
5
|
+
module Presentation
|
6
|
+
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
def present(object, opts={})
|
10
|
+
opts = opts.symbolize_keys!
|
11
|
+
opts.reverse_merge!(options.to_h) if options.to_h.present?
|
12
|
+
presenters = Quandl::Command::Presenter.new( object, opts )
|
13
|
+
# format order of importance
|
14
|
+
format = get_output_format( opts )
|
15
|
+
# convert each object to_format and write to stdout
|
16
|
+
presenters.each do |presenter|
|
17
|
+
# write requested format to stdout
|
18
|
+
info( presenter.to_format(format) )
|
19
|
+
# write errors to stderr
|
20
|
+
error( presenter.to_stderr ) unless presenter.valid?
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def get_output_format(override={})
|
27
|
+
format = options.output_format || config.output_format || override[:output_format] || 'pipes'
|
28
|
+
format = 'pipes' unless %w{ pipes json qdf }.include?(format.to_s)
|
29
|
+
debug("output_format: #{format}")
|
30
|
+
format
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Task
|
4
|
+
|
5
|
+
module Reportable
|
6
|
+
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
before_execute :ask_to_send_crash_reports
|
11
|
+
end
|
12
|
+
|
13
|
+
def ask_to_send_crash_reports
|
14
|
+
if config.send_crash_reports.nil?
|
15
|
+
status = ['y','yes'].include? ask("Do you want to send crash reports if toolbelt fails? (y/n)").to_s.strip.rstrip.downcase
|
16
|
+
config.send_crash_reports = status
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def call
|
21
|
+
begin
|
22
|
+
super
|
23
|
+
rescue Exception => err
|
24
|
+
if config.send_crash_reports == true
|
25
|
+
Quandl::Client::Report.create( message: err.to_s, full_message: err.backtrace ) rescue nil
|
26
|
+
end
|
27
|
+
raise $!, "#{$!}", $!.backtrace
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Command
|
3
|
+
class Task
|
4
|
+
|
5
|
+
module Threading
|
6
|
+
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
included do
|
10
|
+
|
11
|
+
before_execute :shutdown_thread_pool_on_sigint
|
12
|
+
|
13
|
+
after_execute :shutdown_thread_pool_uninterruptedly!
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
module ClassMethods
|
18
|
+
end
|
19
|
+
|
20
|
+
def shutdown_thread_pool_on_sigint
|
21
|
+
return unless threads?
|
22
|
+
trap('SIGINT') do
|
23
|
+
debug "# exit signal received"
|
24
|
+
unless thread_pool.shutdown?
|
25
|
+
debug "# waiting for executing jobs to finish"
|
26
|
+
thread_pool.shutdown
|
27
|
+
end
|
28
|
+
debug "# exiting now"
|
29
|
+
exit
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def background_job(*args, &block)
|
34
|
+
# dont execute jobs when shutdown signaled
|
35
|
+
return false if exiting?
|
36
|
+
# options
|
37
|
+
opts = args.extract_options!.symbolize_keys!
|
38
|
+
key = opts[:lock]
|
39
|
+
# without threads process in foreground
|
40
|
+
return block.call unless threads?
|
41
|
+
# process with pool
|
42
|
+
job = thread_pool.process do
|
43
|
+
# wait for lock
|
44
|
+
await_thread_pool_lock!(key) if key.present?
|
45
|
+
# if this key is locked
|
46
|
+
block.call
|
47
|
+
# unlock
|
48
|
+
release_thread_pool_lock(key) if key.present?
|
49
|
+
end
|
50
|
+
# onwards
|
51
|
+
job
|
52
|
+
end
|
53
|
+
|
54
|
+
def mutex
|
55
|
+
@mutex ||= Mutex.new
|
56
|
+
end
|
57
|
+
|
58
|
+
def await_thread_pool_lock!(key)
|
59
|
+
got_lock = false
|
60
|
+
# try to get a lock
|
61
|
+
mutex.synchronize do
|
62
|
+
unless thread_pool_locked?(key)
|
63
|
+
obtain_thread_pool_lock(key)
|
64
|
+
got_lock = true
|
65
|
+
end
|
66
|
+
end
|
67
|
+
# try again unless lock was obtained
|
68
|
+
if !got_lock
|
69
|
+
sleep(0.05)
|
70
|
+
await_thread_pool_lock!(key)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def thread_pool
|
75
|
+
@thread_pool ||= Thread.pool( threads )
|
76
|
+
end
|
77
|
+
|
78
|
+
def obtain_thread_pool_lock(key)
|
79
|
+
thread_pool_locks[key] = true
|
80
|
+
end
|
81
|
+
|
82
|
+
def release_thread_pool_lock(key)
|
83
|
+
thread_pool_locks.delete(key)
|
84
|
+
end
|
85
|
+
|
86
|
+
def thread_pool_locked?(key)
|
87
|
+
thread_pool_locks[key] == true
|
88
|
+
end
|
89
|
+
|
90
|
+
def thread_pool_locks
|
91
|
+
@thread_pool_locks ||= {}
|
92
|
+
end
|
93
|
+
|
94
|
+
def threads?
|
95
|
+
threads > 1
|
96
|
+
end
|
97
|
+
|
98
|
+
def threads
|
99
|
+
options.threads.to_i || 1
|
100
|
+
end
|
101
|
+
|
102
|
+
def exiting?
|
103
|
+
threads? && thread_pool.shutdown?
|
104
|
+
end
|
105
|
+
|
106
|
+
protected
|
107
|
+
|
108
|
+
def shutdown_thread_pool_uninterruptedly!
|
109
|
+
return false unless Thread.respond_to?(:pool)
|
110
|
+
thread_pool.shutdown unless exiting?
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|