quandl 0.2.27 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +8 -8
  2. data/README.md +70 -17
  3. data/Rakefile +7 -86
  4. data/UPGRADE.md +23 -0
  5. data/VERSION +1 -0
  6. data/lib/quandl/command.rb +11 -3
  7. data/lib/quandl/command/config.rb +88 -0
  8. data/lib/quandl/command/presenter.rb +74 -0
  9. data/lib/quandl/command/presenter/record.rb +121 -0
  10. data/lib/quandl/command/presenters/dataset_presenter.rb +19 -0
  11. data/lib/quandl/command/presenters/error_presenter.rb +31 -0
  12. data/lib/quandl/command/presenters/nil_class_presenter.rb +10 -0
  13. data/lib/quandl/command/presenters/scraper_presenter.rb +25 -0
  14. data/lib/quandl/command/presenters/superset_presenter.rb +10 -0
  15. data/lib/quandl/command/task.rb +40 -0
  16. data/lib/quandl/command/task/callbacks.rb +47 -0
  17. data/lib/quandl/command/task/clientable.rb +65 -0
  18. data/lib/quandl/command/task/commandable.rb +104 -0
  19. data/lib/quandl/command/task/configurable.rb +79 -0
  20. data/lib/quandl/command/task/inputable.rb +37 -0
  21. data/lib/quandl/command/task/logging.rb +83 -0
  22. data/lib/quandl/command/task/presentation.rb +37 -0
  23. data/lib/quandl/command/task/reportable.rb +35 -0
  24. data/lib/quandl/command/task/threading.rb +117 -0
  25. data/lib/quandl/command/task/translations.rb +37 -0
  26. data/lib/quandl/command/task/updatable.rb +77 -0
  27. data/lib/quandl/command/tasks.rb +6 -5
  28. data/lib/quandl/command/tasks/delete.rb +10 -67
  29. data/lib/quandl/command/tasks/download.rb +11 -78
  30. data/lib/quandl/command/tasks/info.rb +2 -2
  31. data/lib/quandl/command/tasks/list.rb +12 -24
  32. data/lib/quandl/command/tasks/login.rb +4 -4
  33. data/lib/quandl/command/tasks/replace.rb +58 -0
  34. data/lib/quandl/command/tasks/schedule.rb +106 -0
  35. data/lib/quandl/command/tasks/search.rb +39 -0
  36. data/lib/quandl/command/tasks/superset.rb +59 -0
  37. data/lib/quandl/command/tasks/uninstall.rb +1 -1
  38. data/lib/quandl/command/tasks/update.rb +2 -2
  39. data/lib/quandl/command/tasks/upload.rb +13 -26
  40. data/lib/quandl/command/version.rb +1 -1
  41. data/quandl.gemspec +5 -3
  42. data/spec/fixtures/scraper.rb +6 -0
  43. data/spec/lib/quandl/command/delete_spec.rb +19 -11
  44. data/spec/lib/quandl/command/download_spec.rb +11 -4
  45. data/spec/lib/quandl/command/replace_spec.rb +23 -0
  46. data/spec/lib/quandl/command/schedule_spec.rb +53 -0
  47. data/spec/lib/quandl/command/superset_spec.rb +28 -0
  48. data/spec/lib/quandl/command/upload_spec.rb +71 -24
  49. data/spec/lib/quandl/command_spec.rb +1 -9
  50. data/spec/spec_helper.rb +36 -1
  51. data/tasks/toolbelt/build/tarball.rb +2 -2
  52. metadata +55 -10
  53. data/lib/quandl/command/qconfig.rb +0 -86
  54. data/lib/quandl/command/tasks/base.rb +0 -314
@@ -0,0 +1,104 @@
1
+ module Quandl
2
+ module Command
3
+ class Task
4
+
5
+ module Commandable
6
+
7
+ extend ActiveSupport::Concern
8
+
9
+ included do
10
+ attr_accessor :args, :options
11
+ end
12
+
13
+ module ClassMethods
14
+
15
+ def inherited(klass)
16
+ Quandl::Command::Tasks.tasks << klass unless Quandl::Command::Tasks.tasks.include?(klass)
17
+ klass.options(@options)
18
+ end
19
+
20
+ def configure(app)
21
+ return if disabled?
22
+ app.command(command_name) do |c|
23
+ c.syntax = syntax
24
+ c.description = description
25
+ c.action{|a,o| call(a,o) }
26
+ configure_options(c)
27
+ end
28
+ end
29
+
30
+ def disabled?
31
+ @disabled == true
32
+ end
33
+
34
+ def disable!
35
+ @disabled = true
36
+ end
37
+
38
+ def syntax(value=nil)
39
+ @syntax = value if value.present?
40
+ @syntax ||= "quandl #{command_name}"
41
+ end
42
+
43
+ def command_name(value=nil)
44
+ @command_name = value if value.present?
45
+ @command_name ||= name.to_s.split("::").last.downcase
46
+ end
47
+
48
+ def description(value=nil)
49
+ @description = value if value.present?
50
+ @description ||= "No description."
51
+ end
52
+
53
+ def options(value=nil)
54
+ @options = value if value.present?
55
+ @options ||= {}
56
+ end
57
+
58
+ def call(args=[], options={})
59
+ args = Array(args)
60
+ options = ensure_options_are_command_options!(options)
61
+ self.new( args, options ).call
62
+ end
63
+
64
+ protected
65
+
66
+ def ensure_options_are_command_options!(options)
67
+ return options if options.class == Commander::Command::Options
68
+ OpenStruct.new(options)
69
+ end
70
+
71
+ def configure_options(c)
72
+ options.each do |class_type, options|
73
+ options.each do |name, desc|
74
+ c.option "--#{name} #{class_type.to_s.upcase}", class_type, desc
75
+ end
76
+ end
77
+ end
78
+
79
+ end
80
+
81
+ def call
82
+ run_callbacks(:execute) do
83
+ execute
84
+ end
85
+ end
86
+
87
+ def execute
88
+ # fire a subcommand if specified
89
+ return self.send(args.shift) if args.first.present? && self.respond_to?(args.first)
90
+ # otherwise show syntax
91
+ puts " SYNTAX\n\n #{self.class.syntax}"
92
+ end
93
+
94
+ def initialize(args, options)
95
+ self.args = args
96
+ self.options = options
97
+ end
98
+
99
+
100
+ end
101
+
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,79 @@
1
+ module Quandl
2
+ module Command
3
+ class Task
4
+
5
+ module Configurable
6
+
7
+ extend ActiveSupport::Concern
8
+
9
+ included do
10
+ delegate :auth_token, :quandl_url, to: :config
11
+ end
12
+
13
+ module ClassMethods
14
+ end
15
+
16
+ def declared_params
17
+ params = {}
18
+ self.class.options.each do |class_type, opts|
19
+ opts.each do |name, desc|
20
+ if options.is_a?(OpenStruct)
21
+ params[name] = self.options.send(name)
22
+ else
23
+ params[name] = self.options[name] if self.options[name].present?
24
+ end
25
+ end
26
+ end
27
+ params
28
+ end
29
+
30
+ def verbose?
31
+ options.verbose == true
32
+ end
33
+
34
+ def trace?
35
+ options.trace == true
36
+ end
37
+
38
+ def confirmed?
39
+ return true if force_yes?
40
+ ask_for_confirmation!
41
+ end
42
+
43
+ def force_yes?
44
+ options.force_yes == true
45
+ end
46
+
47
+ def ask_for_confirmation!
48
+ ['y','yes'].include?( ask("Are you sure? (y/n)") )
49
+ end
50
+
51
+ def page
52
+ @page ||= options.page.to_i || 1
53
+ end
54
+
55
+ def config
56
+ @config ||= Quandl::Command::Config.new( config_options )
57
+ end
58
+
59
+ protected
60
+
61
+ def config_options
62
+ opts = { file_path: config_file_path }
63
+ opts.merge!(options.to_h) if options.to_h.present?
64
+ opts
65
+ end
66
+
67
+ def config_file_path
68
+ File.join( ENV['HOME'], '.quandl', config_profile )
69
+ end
70
+
71
+ def config_profile
72
+ options.environment || 'config'
73
+ end
74
+
75
+ end
76
+
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,37 @@
1
+ module Quandl
2
+ module Command
3
+ class Task
4
+
5
+ module Inputable
6
+
7
+ extend ActiveSupport::Concern
8
+
9
+ included do
10
+ end
11
+
12
+ module ClassMethods
13
+ end
14
+
15
+ def each_line_in_background(args, &block)
16
+ args.each_line do |arg|
17
+ background_job do
18
+ block.call( arg )
19
+ end
20
+ end
21
+ end
22
+
23
+ def args_or_stdin
24
+ return args.join("\n") if args.first.present?
25
+ $stdin
26
+ end
27
+
28
+ def file_or_stdin
29
+ return File.open( args.first, 'r' ) if args.first.present?
30
+ $stdin
31
+ end
32
+
33
+ end
34
+
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,83 @@
1
+ module Quandl
2
+ module Command
3
+ class Task
4
+
5
+ module Logging
6
+
7
+ extend ActiveSupport::Concern
8
+
9
+ included do
10
+ before_execute :start_request_timer
11
+ before_execute :configure_logger
12
+ after_execute :log_request_time
13
+
14
+ attr_accessor :request_timer
15
+
16
+ end
17
+
18
+ def start_request_timer
19
+ self.request_timer = Time.now
20
+ end
21
+
22
+ def configure_logger
23
+ @stderr_logger = initialize_logger(config.stderr) if config.stderr.present?
24
+ @stdout_logger = initialize_logger(config.stdout) if config.stdout.present?
25
+ end
26
+
27
+ def initialize_logger(path)
28
+ l = ::Logger.new( path, 2, 52428800 )
29
+ l.formatter = proc do |severity, datetime, progname, msg|
30
+ "# [#{datetime.strftime("%Y-%m-%d %H:%M:%S")}]\n#{msg}"
31
+ end
32
+ l
33
+ end
34
+
35
+ def log_request_time
36
+ debug("# Started: #{request_timer}. Finished: #{Time.now}. Elapsed: #{request_timer.elapsed_ms}")
37
+ end
38
+
39
+ def summarize(item)
40
+ return summarize_hash(item) if item.kind_of?(Hash)
41
+ item
42
+ end
43
+
44
+ def summarize_hash(item)
45
+ item.collect do |k,v|
46
+ next "#{k}: '#{v}'" if v.kind_of?(String)
47
+ "#{k}: #{v}"
48
+ end.join(', ')
49
+ end
50
+
51
+ def table(*args)
52
+ Array(args).flatten.join(" | ")
53
+ end
54
+
55
+ def info(*args)
56
+ logger.info(*args)
57
+ end
58
+
59
+ def debug(*args)
60
+ args.each{|a| logger.debug("# #{a}") } if verbose?
61
+ end
62
+
63
+ def error(*args)
64
+ stderr_logger.error(*args)
65
+ end
66
+
67
+ def fatal(*args)
68
+ stderr_logger.fatal("FATAL: #{args.join(" ")}")
69
+ end
70
+
71
+ def logger
72
+ @stdout_logger ||= Quandl::Logger
73
+ end
74
+
75
+ def stderr_logger
76
+ @stderr_logger ||= Quandl::Logger
77
+ end
78
+
79
+ end
80
+
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,37 @@
1
+ module Quandl
2
+ module Command
3
+ class Task
4
+
5
+ module Presentation
6
+
7
+ extend ActiveSupport::Concern
8
+
9
+ def present(object, opts={})
10
+ opts = opts.symbolize_keys!
11
+ opts.reverse_merge!(options.to_h) if options.to_h.present?
12
+ presenters = Quandl::Command::Presenter.new( object, opts )
13
+ # format order of importance
14
+ format = get_output_format( opts )
15
+ # convert each object to_format and write to stdout
16
+ presenters.each do |presenter|
17
+ # write requested format to stdout
18
+ info( presenter.to_format(format) )
19
+ # write errors to stderr
20
+ error( presenter.to_stderr ) unless presenter.valid?
21
+ end
22
+ end
23
+
24
+ private
25
+
26
+ def get_output_format(override={})
27
+ format = options.output_format || config.output_format || override[:output_format] || 'pipes'
28
+ format = 'pipes' unless %w{ pipes json qdf }.include?(format.to_s)
29
+ debug("output_format: #{format}")
30
+ format
31
+ end
32
+
33
+ end
34
+
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,35 @@
1
+ module Quandl
2
+ module Command
3
+ class Task
4
+
5
+ module Reportable
6
+
7
+ extend ActiveSupport::Concern
8
+
9
+ included do
10
+ before_execute :ask_to_send_crash_reports
11
+ end
12
+
13
+ def ask_to_send_crash_reports
14
+ if config.send_crash_reports.nil?
15
+ status = ['y','yes'].include? ask("Do you want to send crash reports if toolbelt fails? (y/n)").to_s.strip.rstrip.downcase
16
+ config.send_crash_reports = status
17
+ end
18
+ end
19
+
20
+ def call
21
+ begin
22
+ super
23
+ rescue Exception => err
24
+ if config.send_crash_reports == true
25
+ Quandl::Client::Report.create( message: err.to_s, full_message: err.backtrace ) rescue nil
26
+ end
27
+ raise $!, "#{$!}", $!.backtrace
28
+ end
29
+ end
30
+
31
+ end
32
+
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,117 @@
1
+ module Quandl
2
+ module Command
3
+ class Task
4
+
5
+ module Threading
6
+
7
+ extend ActiveSupport::Concern
8
+
9
+ included do
10
+
11
+ before_execute :shutdown_thread_pool_on_sigint
12
+
13
+ after_execute :shutdown_thread_pool_uninterruptedly!
14
+
15
+ end
16
+
17
+ module ClassMethods
18
+ end
19
+
20
+ def shutdown_thread_pool_on_sigint
21
+ return unless threads?
22
+ trap('SIGINT') do
23
+ debug "# exit signal received"
24
+ unless thread_pool.shutdown?
25
+ debug "# waiting for executing jobs to finish"
26
+ thread_pool.shutdown
27
+ end
28
+ debug "# exiting now"
29
+ exit
30
+ end
31
+ end
32
+
33
+ def background_job(*args, &block)
34
+ # dont execute jobs when shutdown signaled
35
+ return false if exiting?
36
+ # options
37
+ opts = args.extract_options!.symbolize_keys!
38
+ key = opts[:lock]
39
+ # without threads process in foreground
40
+ return block.call unless threads?
41
+ # process with pool
42
+ job = thread_pool.process do
43
+ # wait for lock
44
+ await_thread_pool_lock!(key) if key.present?
45
+ # if this key is locked
46
+ block.call
47
+ # unlock
48
+ release_thread_pool_lock(key) if key.present?
49
+ end
50
+ # onwards
51
+ job
52
+ end
53
+
54
+ def mutex
55
+ @mutex ||= Mutex.new
56
+ end
57
+
58
+ def await_thread_pool_lock!(key)
59
+ got_lock = false
60
+ # try to get a lock
61
+ mutex.synchronize do
62
+ unless thread_pool_locked?(key)
63
+ obtain_thread_pool_lock(key)
64
+ got_lock = true
65
+ end
66
+ end
67
+ # try again unless lock was obtained
68
+ if !got_lock
69
+ sleep(0.05)
70
+ await_thread_pool_lock!(key)
71
+ end
72
+ end
73
+
74
+ def thread_pool
75
+ @thread_pool ||= Thread.pool( threads )
76
+ end
77
+
78
+ def obtain_thread_pool_lock(key)
79
+ thread_pool_locks[key] = true
80
+ end
81
+
82
+ def release_thread_pool_lock(key)
83
+ thread_pool_locks.delete(key)
84
+ end
85
+
86
+ def thread_pool_locked?(key)
87
+ thread_pool_locks[key] == true
88
+ end
89
+
90
+ def thread_pool_locks
91
+ @thread_pool_locks ||= {}
92
+ end
93
+
94
+ def threads?
95
+ threads > 1
96
+ end
97
+
98
+ def threads
99
+ options.threads.to_i || 1
100
+ end
101
+
102
+ def exiting?
103
+ threads? && thread_pool.shutdown?
104
+ end
105
+
106
+ protected
107
+
108
+ def shutdown_thread_pool_uninterruptedly!
109
+ return false unless Thread.respond_to?(:pool)
110
+ thread_pool.shutdown unless exiting?
111
+ end
112
+
113
+ end
114
+
115
+ end
116
+ end
117
+ end