quandl 0.2.27 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +8 -8
  2. data/README.md +70 -17
  3. data/Rakefile +7 -86
  4. data/UPGRADE.md +23 -0
  5. data/VERSION +1 -0
  6. data/lib/quandl/command.rb +11 -3
  7. data/lib/quandl/command/config.rb +88 -0
  8. data/lib/quandl/command/presenter.rb +74 -0
  9. data/lib/quandl/command/presenter/record.rb +121 -0
  10. data/lib/quandl/command/presenters/dataset_presenter.rb +19 -0
  11. data/lib/quandl/command/presenters/error_presenter.rb +31 -0
  12. data/lib/quandl/command/presenters/nil_class_presenter.rb +10 -0
  13. data/lib/quandl/command/presenters/scraper_presenter.rb +25 -0
  14. data/lib/quandl/command/presenters/superset_presenter.rb +10 -0
  15. data/lib/quandl/command/task.rb +40 -0
  16. data/lib/quandl/command/task/callbacks.rb +47 -0
  17. data/lib/quandl/command/task/clientable.rb +65 -0
  18. data/lib/quandl/command/task/commandable.rb +104 -0
  19. data/lib/quandl/command/task/configurable.rb +79 -0
  20. data/lib/quandl/command/task/inputable.rb +37 -0
  21. data/lib/quandl/command/task/logging.rb +83 -0
  22. data/lib/quandl/command/task/presentation.rb +37 -0
  23. data/lib/quandl/command/task/reportable.rb +35 -0
  24. data/lib/quandl/command/task/threading.rb +117 -0
  25. data/lib/quandl/command/task/translations.rb +37 -0
  26. data/lib/quandl/command/task/updatable.rb +77 -0
  27. data/lib/quandl/command/tasks.rb +6 -5
  28. data/lib/quandl/command/tasks/delete.rb +10 -67
  29. data/lib/quandl/command/tasks/download.rb +11 -78
  30. data/lib/quandl/command/tasks/info.rb +2 -2
  31. data/lib/quandl/command/tasks/list.rb +12 -24
  32. data/lib/quandl/command/tasks/login.rb +4 -4
  33. data/lib/quandl/command/tasks/replace.rb +58 -0
  34. data/lib/quandl/command/tasks/schedule.rb +106 -0
  35. data/lib/quandl/command/tasks/search.rb +39 -0
  36. data/lib/quandl/command/tasks/superset.rb +59 -0
  37. data/lib/quandl/command/tasks/uninstall.rb +1 -1
  38. data/lib/quandl/command/tasks/update.rb +2 -2
  39. data/lib/quandl/command/tasks/upload.rb +13 -26
  40. data/lib/quandl/command/version.rb +1 -1
  41. data/quandl.gemspec +5 -3
  42. data/spec/fixtures/scraper.rb +6 -0
  43. data/spec/lib/quandl/command/delete_spec.rb +19 -11
  44. data/spec/lib/quandl/command/download_spec.rb +11 -4
  45. data/spec/lib/quandl/command/replace_spec.rb +23 -0
  46. data/spec/lib/quandl/command/schedule_spec.rb +53 -0
  47. data/spec/lib/quandl/command/superset_spec.rb +28 -0
  48. data/spec/lib/quandl/command/upload_spec.rb +71 -24
  49. data/spec/lib/quandl/command_spec.rb +1 -9
  50. data/spec/spec_helper.rb +36 -1
  51. data/tasks/toolbelt/build/tarball.rb +2 -2
  52. metadata +55 -10
  53. data/lib/quandl/command/qconfig.rb +0 -86
  54. data/lib/quandl/command/tasks/base.rb +0 -314
@@ -0,0 +1,106 @@
1
+ class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
2
+
3
+ autoload_quandl_client
4
+ authenticated_users_only!
5
+
6
+ description "Schedule a script to be run in the quandl cloud."
7
+ syntax %{quandl schedule command [file]
8
+
9
+ COMMANDS:
10
+
11
+ schedule list
12
+ schedule add file
13
+ schedule show file
14
+ schedule delete file
15
+ schedule replace file
16
+
17
+ EXAMPLES:
18
+
19
+ $ quandl schedule add scraper.rb
20
+ You have successfully scheduled scraper.rb to be run daily.
21
+
22
+ $ quandl schedule add scraper.rb --name weekly_scraper.rb --frequency weekly
23
+ You have successfully scheduled weekly-scraper.rb to be run weekly.
24
+
25
+ $ quandl schedule replace scraper.rb
26
+ You have successfully replace scraper.rb}
27
+
28
+ options({
29
+ String => {
30
+ name: "The name used to reference your scraper in the quandl cloud.",
31
+ frequency: "How often the scraper should be run. [ daily, weekly, monthly, quaterly, annually ]",
32
+ }
33
+ })
34
+
35
+ def list
36
+ present(scrapers)
37
+ end
38
+
39
+ def show
40
+ present(scraper)
41
+ end
42
+
43
+ def add
44
+ Quandl::Client::Scraper.create( name: name, scraper: args.first )
45
+ info("'#{scraper.name}' was added.")
46
+ info("#{schedule_message}")
47
+ end
48
+
49
+ def download
50
+ begin
51
+ require 'open-uri'
52
+ open( scraper.name, 'wb') do |file|
53
+ $stdout << open(scraper.scraper_url).read
54
+ end
55
+ rescue => err
56
+ present err
57
+ end
58
+ end
59
+
60
+ def delete
61
+ (info("'#{name}' does not exist "); return;) if scraper.blank?
62
+ info("You are about to delete '#{scraper.name}'")
63
+ return unless confirmed?
64
+ scraper.destroy if scraper.respond_to?(:destroy)
65
+ info("'#{scraper.name}' was deleted")
66
+ end
67
+
68
+
69
+ def replace
70
+ (info("'#{name}' does not exist "); return;) if scraper.blank?
71
+ info("You are about to replace '#{scraper.name}'")
72
+ return unless confirmed?
73
+ scraper.destroy if scraper.respond_to?(:destroy)
74
+ Quandl::Client::Scraper.create( name: name, scraper: args.first )
75
+ info("'#{scraper.name}' was replaced.")
76
+ info("#{schedule_message}")
77
+ end
78
+
79
+ private
80
+
81
+ def schedule_message
82
+ "It will run 3 times a day starting immediately. Check your scrapers run status at #{quandl_url.gsub(/\/api\/?/,'')}/scrapers"
83
+ end
84
+
85
+ def scrapers
86
+ @scrapers ||= Quandl::Client::Scraper.where( page: page ).fetch
87
+ end
88
+
89
+ def scraper
90
+ defined?(@scraper) ? @scraper : ( @scraper = find_scraper )
91
+ end
92
+
93
+ def find_scraper
94
+ return nil unless args.first.present?
95
+ # if the arg is numeric lookup the id. otherwise search for a scraper by name and grab the id
96
+ id = args.first.numeric? ? args.first : Quandl::Client::Scraper.where( name: name ).try(:first).try(:id)
97
+ # find the scraper with the id
98
+ Quandl::Client::Scraper.find(id) unless id.nil?
99
+ end
100
+
101
+ def name
102
+ return options.name if options.name.present?
103
+ return File.basename(args.first) if args.first.present?
104
+ end
105
+
106
+ end
@@ -0,0 +1,39 @@
1
+ class Quandl::Command::Tasks::Search < Quandl::Command::Task
2
+ autoload_quandl_client
3
+
4
+ syntax "quandl search STRING"
5
+ description "Search for datasets matching STRING"
6
+
7
+ options({
8
+ String => {
9
+ source_code: "Return datasets matching: [SOURCE_CODE]",
10
+ code: "Return datasets where code matches [code]",
11
+ },
12
+ Integer => {
13
+ limit: "Limit the number of datasets returned by the search. Defaults to unlimited.",
14
+ page: "Return datasets starting from: [page]",
15
+ }
16
+ })
17
+
18
+ before_execute :upcase_source_code
19
+
20
+ validates :query, presence: true
21
+
22
+ def execute
23
+ Quandl::Client::Dataset.query( query ).where( declared_params ).each_in_page do |dataset|
24
+ present(dataset)
25
+ end
26
+ end
27
+
28
+
29
+ protected
30
+
31
+ def query
32
+ args.first
33
+ end
34
+
35
+ def upcase_source_code
36
+ options.source_code.to_s.upcase!
37
+ end
38
+
39
+ end
@@ -0,0 +1,59 @@
1
+ class Quandl::Command::Tasks::Superset < Quandl::Command::Task
2
+
3
+ autoload_quandl_client
4
+ authenticated_users_only!
5
+
6
+ description "Superset a script to be run in the quandl cloud."
7
+ syntax %{quandl superset command [file]
8
+
9
+ COMMANDS:
10
+
11
+ superset new
12
+ superset list
13
+ superset upload [file]
14
+ superset download [code]
15
+ superset delete [code] }
16
+
17
+ def new
18
+ info Quandl::Client::Superset.example.to_qdf
19
+ end
20
+
21
+ def list
22
+ Quandl::Client::Superset.owner('myself').each_in_page do |superset|
23
+ present(superset)
24
+ end
25
+ end
26
+
27
+ def upload
28
+ Quandl::Format::Superset.foreach( file_or_stdin ) do |superset|
29
+ # execute in parallel
30
+ background_job do
31
+ # upload the superset
32
+ superset.save if superset.respond_to?(:valid?) && superset.valid?
33
+ # present the result
34
+ present(superset)
35
+ end
36
+ end
37
+ end
38
+
39
+ def download
40
+ each_line_in_background( args_or_stdin ) do |code|
41
+ # find superset
42
+ dataset = Quandl::Client::Superset.find( code )
43
+ # present
44
+ present dataset, output_format: :qdf
45
+ end
46
+ end
47
+
48
+ def delete
49
+ each_line_in_background( args_or_stdin ) do |code|
50
+ # find
51
+ dataset = Quandl::Client::Dataset.find( code )
52
+ # destroy
53
+ dataset.destroy
54
+ # present
55
+ present dataset
56
+ end
57
+ end
58
+
59
+ end
@@ -1,4 +1,4 @@
1
- class Quandl::Command::Tasks::Uninstall < Quandl::Command::Tasks::Base
1
+ class Quandl::Command::Tasks::Uninstall < Quandl::Command::Task
2
2
 
3
3
  # curl -s https://s3.amazonaws.com/quandl-command/install.sh | bash
4
4
 
@@ -1,11 +1,11 @@
1
- class Quandl::Command::Tasks::Update < Quandl::Command::Tasks::Base
1
+ class Quandl::Command::Tasks::Update < Quandl::Command::Task
2
2
 
3
3
  # curl -s https://s3.amazonaws.com/quandl-command/install.sh | bash
4
4
 
5
5
  description "Update Quandl Toolbelt to the latest version."
6
6
  syntax %Q{quandl update [revision]
7
7
 
8
- Examples:
8
+ EXAMPLES:
9
9
 
10
10
  $ quandl update
11
11
  Updating from 0.2.18 ...
@@ -1,10 +1,10 @@
1
- class Quandl::Command::Tasks::Upload < Quandl::Command::Tasks::Base
2
- autoload_client_library
1
+ class Quandl::Command::Tasks::Upload < Quandl::Command::Task
2
+ autoload_quandl_client
3
3
 
4
4
  description "Upload a dataset using its quandl code."
5
5
  syntax %{quandl upload file.qdf
6
6
 
7
- Examples:
7
+ EXAMPLES:
8
8
 
9
9
  $ quandl upload file.qcsv
10
10
  OK | 98ms | http://quandl.com/USERNAME/CODE_1
@@ -25,11 +25,6 @@ class Quandl::Command::Tasks::Upload < Quandl::Command::Tasks::Base
25
25
  2013-11-22,1252.0,454.95,448.2
26
26
  2013-11-21,452.25,457.75,449.1
27
27
  }
28
- options({
29
- Integer => {
30
- threads: "How many workers to use during download.",
31
- }
32
- })
33
28
 
34
29
  authenticated_users_only!
35
30
 
@@ -37,30 +32,22 @@ class Quandl::Command::Tasks::Upload < Quandl::Command::Tasks::Base
37
32
  # datasets from file_path if given
38
33
  interface = file_path.present? ? File.open(file_path, "r") : $stdin
39
34
  # for each dataset streamed from interface
40
- Quandl::Format::Dataset.each_line(interface) do |dataset|
41
- pool.process{ upload( dataset ) }
35
+ Quandl::Format::Dataset.each_line(interface) do |dataset, error|
36
+ # present error if given
37
+ next present( error ) unless error.nil?
38
+ # process in background using thread key
39
+ background_job( lock: dataset.client.full_code ) do
40
+ # upload
41
+ upload( dataset )
42
+ end
42
43
  end
43
- pool.shutdown
44
44
  end
45
45
 
46
46
  def upload(dataset)
47
- # display debug info when verbose
48
- debug dataset.attributes.to_s
49
- # upload
47
+ # upload unless errors were raised
50
48
  dataset.upload if dataset.valid?
51
49
  # output report to $stdout or $stderr
52
- report(dataset)
53
- end
54
-
55
- def report(dataset)
56
- if [200,201].include?( dataset.client.status )
57
- info table dataset.client.human_status,
58
- dataset.client.elapsed_request_time_ms,
59
- dataset.client.full_url
60
- else
61
- error(dataset.human_errors)
62
- end
63
- debug "---"
50
+ present(dataset.client)
64
51
  end
65
52
 
66
53
  def file_path
@@ -1,5 +1,5 @@
1
1
  module Quandl
2
2
  module Command
3
- VERSION = '0.2.27'
3
+ VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), '../../../VERSION'))).strip.rstrip
4
4
  end
5
5
  end
data/quandl.gemspec CHANGED
@@ -18,9 +18,10 @@ Gem::Specification.new do |s|
18
18
  s.require_paths = ["lib"]
19
19
 
20
20
  s.add_runtime_dependency "commander", '4.1.5'
21
- s.add_runtime_dependency "quandl_format", "0.2.8"
22
- s.add_runtime_dependency "quandl_client", '2.5.1'
23
- s.add_runtime_dependency "quandl_data", '1.3.9'
21
+
22
+ s.add_runtime_dependency "quandl_format", "0.4.2"
23
+ s.add_runtime_dependency "quandl_client", '2.7.4'
24
+ s.add_runtime_dependency "quandl_data", '1.4.1'
24
25
  s.add_runtime_dependency "quandl_operation", '0.3.2'
25
26
  s.add_runtime_dependency "quandl_babelfish", '0.0.9'
26
27
  s.add_runtime_dependency "quandl_logger", '0.2.5'
@@ -36,4 +37,5 @@ Gem::Specification.new do |s|
36
37
  s.add_development_dependency "pry", "~> 0.9"
37
38
  s.add_development_dependency "guard", "~> 2.3"
38
39
  s.add_development_dependency "guard-rspec", "~> 4.2"
40
+ s.add_development_dependency "quandl_utility"
39
41
  end
@@ -0,0 +1,6 @@
1
+ puts %Q{
2
+ code: SCRAPE_SOME_DATA
3
+ -
4
+ Date,Value
5
+ #{Time.now.strftime("%Y-%m-%d")},10
6
+ }
@@ -1,6 +1,17 @@
1
1
  # encoding: utf-8
2
2
  require 'spec_helper'
3
3
 
4
+ describe "./bin/quandl delete" do
5
+
6
+ let(:command){ self.class.superclass.description }
7
+ subject{ quandl("delete #{command}") }
8
+
9
+ context "DOES_NOT_EXIST" do
10
+ its(:stdout){ should match 'Not Found' }
11
+ end
12
+
13
+ end
14
+
4
15
  describe Quandl::Command::Tasks::Delete do
5
16
 
6
17
  let(:subject_args){ [] }
@@ -11,21 +22,18 @@ describe Quandl::Command::Tasks::Delete do
11
22
  before(:each){ Quandl::Command::Tasks::Upload.call( 'spec/fixtures/data/datasets.qdf' ) }
12
23
 
13
24
  it "should delete spec/fixtures/data/datasets.qdf" do
14
- Quandl::Logger.should_receive(:info).with(/Deleted/).exactly(4).times
15
- # load each dataset from file
25
+ TestOutput.should_receive(:info).with(/OK/).at_least(4).times
16
26
  datasets = Quandl::Format::Dataset.load_from_file('spec/fixtures/data/datasets.qdf')
17
- # send command to delete each dataset
18
27
  datasets.each{|d| Quandl::Command::Tasks::Delete.call( d.code, force_yes: true ) }
19
-
20
- Quandl::Logger.should_receive(:error).with(/Not Found/).exactly(4).times
28
+ end
29
+
30
+ it "should not download" do
31
+ datasets = Quandl::Format::Dataset.load_from_file('spec/fixtures/data/datasets.qdf')
32
+ datasets.each{|d| Quandl::Command::Tasks::Delete.call( d.code, force_yes: true ) }
33
+ TestOutput.should_receive(:info).with(nil).at_least(4).times
21
34
  datasets.each{|d| Quandl::Command::Tasks::Download.call( d.code ) }
22
35
  end
23
36
 
24
37
  end
25
38
 
26
- it "should fail to delete missing dataset" do
27
- Quandl::Logger.should_receive(:error).with(/Not Found/).exactly(1).times
28
- Quandl::Command::Tasks::Delete.call("DOES_NOT_EXIST", force_yes: true )
29
- end
30
-
31
- end
39
+ end
@@ -34,9 +34,16 @@ describe Quandl::Command::Tasks::Download do
34
34
 
35
35
  end
36
36
 
37
- it "should not download TEST_DOES_NOT_EXIST." do
38
- Quandl::Logger.should_receive(:error).with(/Not Found/)
39
- Quandl::Command::Tasks::Download.call( 'TEST_DOES_NOT_EXIST' )
40
- end
37
+ end
38
+
39
+
40
+ describe "./bin/quandl download" do
41
41
 
42
+ let(:command){ self.class.superclass.description }
43
+ subject{ quandl("download #{command}") }
44
+
45
+ context "TEST_DOES_NOT_EXIST" do
46
+ its(:stdout){ should be_blank }
47
+ end
48
+
42
49
  end
@@ -0,0 +1,23 @@
1
+ # encoding: utf-8
2
+ require 'spec_helper'
3
+
4
+ replace_string = "source_code: BLAKEHIL\ncode: SCRAPER_TEST_1394485290783983\nname: Untitled Dataset 2014-04-04 13:08:49\ndescription: This dataset has no description.\nprivate: false\nreference_url: \nfrequency: daily\n-\nDate,High,Low,Settle\n2014-03-11,9.958,11.0,13.92\n\n"
5
+
6
+ describe "./bin/quandl replace" do
7
+
8
+ let(:prefix){ "replace" }
9
+ let(:command){ self.class.superclass.description }
10
+ subject{ quandl( [prefix, command].flatten.join(" ") ) }
11
+
12
+ context data_path("replace.qdf") do
13
+ its(:stdout){ should match /OK|Created/ }
14
+ its(:stderr){ should be_blank }
15
+ end
16
+
17
+ context "SCRAPER_TEST_1394485290783983" do
18
+ let(:prefix){ "download" }
19
+
20
+ its(:stdout){ should eq replace_string }
21
+ end
22
+
23
+ end
@@ -0,0 +1,53 @@
1
+ # encoding: utf-8
2
+ require 'spec_helper'
3
+ require 'open3'
4
+
5
+ describe "./bin/quandl schedule" do
6
+
7
+ let(:command){ self.class.superclass.description }
8
+ subject{ quandl("schedule #{command}") }
9
+
10
+ context "delete spec/fixtures/scraper.rb" do
11
+ its(:stdout){ should match 'Not Found' }
12
+ end
13
+
14
+ context "add spec/fixtures/scraper.rb" do
15
+ # first upload
16
+ its(:stdout){ should match 'Created' }
17
+ # second upload
18
+ its(:stdout){ should match 'name: has already been taken' }
19
+ end
20
+
21
+ context "list" do
22
+ its(:stdout){ should match 'scraper.rb' }
23
+ end
24
+
25
+ context "show scraper.rb --output-format json" do
26
+ its(:stdout){ should match 'scraper_url' }
27
+ end
28
+
29
+ context "show scraper.rb -C 0" do
30
+ its(:stdout){ should eq "OK\n" }
31
+ end
32
+
33
+ context "show scraper.rb -C 1" do
34
+ its(:stdout){ should eq "scraper.rb\n" }
35
+ end
36
+
37
+ context "show scraper.rb -C 2" do
38
+ its(:stdout){ should eq "Scraper::Script\n" }
39
+ end
40
+
41
+ context "show scraper.rb --output-column 2" do
42
+ its(:stdout){ should eq "Scraper::Script\n" }
43
+ end
44
+
45
+ context "show scraper.rb --output-column 4" do
46
+ its(:stdout){ should match 's3.amazonaws' }
47
+ end
48
+
49
+ context "delete spec/fixtures/scraper.rb" do
50
+ its(:stdout){ should match 'OK' }
51
+ end
52
+
53
+ end