quandl 0.2.27 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/README.md +70 -17
- data/Rakefile +7 -86
- data/UPGRADE.md +23 -0
- data/VERSION +1 -0
- data/lib/quandl/command.rb +11 -3
- data/lib/quandl/command/config.rb +88 -0
- data/lib/quandl/command/presenter.rb +74 -0
- data/lib/quandl/command/presenter/record.rb +121 -0
- data/lib/quandl/command/presenters/dataset_presenter.rb +19 -0
- data/lib/quandl/command/presenters/error_presenter.rb +31 -0
- data/lib/quandl/command/presenters/nil_class_presenter.rb +10 -0
- data/lib/quandl/command/presenters/scraper_presenter.rb +25 -0
- data/lib/quandl/command/presenters/superset_presenter.rb +10 -0
- data/lib/quandl/command/task.rb +40 -0
- data/lib/quandl/command/task/callbacks.rb +47 -0
- data/lib/quandl/command/task/clientable.rb +65 -0
- data/lib/quandl/command/task/commandable.rb +104 -0
- data/lib/quandl/command/task/configurable.rb +79 -0
- data/lib/quandl/command/task/inputable.rb +37 -0
- data/lib/quandl/command/task/logging.rb +83 -0
- data/lib/quandl/command/task/presentation.rb +37 -0
- data/lib/quandl/command/task/reportable.rb +35 -0
- data/lib/quandl/command/task/threading.rb +117 -0
- data/lib/quandl/command/task/translations.rb +37 -0
- data/lib/quandl/command/task/updatable.rb +77 -0
- data/lib/quandl/command/tasks.rb +6 -5
- data/lib/quandl/command/tasks/delete.rb +10 -67
- data/lib/quandl/command/tasks/download.rb +11 -78
- data/lib/quandl/command/tasks/info.rb +2 -2
- data/lib/quandl/command/tasks/list.rb +12 -24
- data/lib/quandl/command/tasks/login.rb +4 -4
- data/lib/quandl/command/tasks/replace.rb +58 -0
- data/lib/quandl/command/tasks/schedule.rb +106 -0
- data/lib/quandl/command/tasks/search.rb +39 -0
- data/lib/quandl/command/tasks/superset.rb +59 -0
- data/lib/quandl/command/tasks/uninstall.rb +1 -1
- data/lib/quandl/command/tasks/update.rb +2 -2
- data/lib/quandl/command/tasks/upload.rb +13 -26
- data/lib/quandl/command/version.rb +1 -1
- data/quandl.gemspec +5 -3
- data/spec/fixtures/scraper.rb +6 -0
- data/spec/lib/quandl/command/delete_spec.rb +19 -11
- data/spec/lib/quandl/command/download_spec.rb +11 -4
- data/spec/lib/quandl/command/replace_spec.rb +23 -0
- data/spec/lib/quandl/command/schedule_spec.rb +53 -0
- data/spec/lib/quandl/command/superset_spec.rb +28 -0
- data/spec/lib/quandl/command/upload_spec.rb +71 -24
- data/spec/lib/quandl/command_spec.rb +1 -9
- data/spec/spec_helper.rb +36 -1
- data/tasks/toolbelt/build/tarball.rb +2 -2
- metadata +55 -10
- data/lib/quandl/command/qconfig.rb +0 -86
- data/lib/quandl/command/tasks/base.rb +0 -314
@@ -0,0 +1,106 @@
|
|
1
|
+
class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
|
2
|
+
|
3
|
+
autoload_quandl_client
|
4
|
+
authenticated_users_only!
|
5
|
+
|
6
|
+
description "Schedule a script to be run in the quandl cloud."
|
7
|
+
syntax %{quandl schedule command [file]
|
8
|
+
|
9
|
+
COMMANDS:
|
10
|
+
|
11
|
+
schedule list
|
12
|
+
schedule add file
|
13
|
+
schedule show file
|
14
|
+
schedule delete file
|
15
|
+
schedule replace file
|
16
|
+
|
17
|
+
EXAMPLES:
|
18
|
+
|
19
|
+
$ quandl schedule add scraper.rb
|
20
|
+
You have successfully scheduled scraper.rb to be run daily.
|
21
|
+
|
22
|
+
$ quandl schedule add scraper.rb --name weekly_scraper.rb --frequency weekly
|
23
|
+
You have successfully scheduled weekly-scraper.rb to be run weekly.
|
24
|
+
|
25
|
+
$ quandl schedule replace scraper.rb
|
26
|
+
You have successfully replace scraper.rb}
|
27
|
+
|
28
|
+
options({
|
29
|
+
String => {
|
30
|
+
name: "The name used to reference your scraper in the quandl cloud.",
|
31
|
+
frequency: "How often the scraper should be run. [ daily, weekly, monthly, quaterly, annually ]",
|
32
|
+
}
|
33
|
+
})
|
34
|
+
|
35
|
+
def list
|
36
|
+
present(scrapers)
|
37
|
+
end
|
38
|
+
|
39
|
+
def show
|
40
|
+
present(scraper)
|
41
|
+
end
|
42
|
+
|
43
|
+
def add
|
44
|
+
Quandl::Client::Scraper.create( name: name, scraper: args.first )
|
45
|
+
info("'#{scraper.name}' was added.")
|
46
|
+
info("#{schedule_message}")
|
47
|
+
end
|
48
|
+
|
49
|
+
def download
|
50
|
+
begin
|
51
|
+
require 'open-uri'
|
52
|
+
open( scraper.name, 'wb') do |file|
|
53
|
+
$stdout << open(scraper.scraper_url).read
|
54
|
+
end
|
55
|
+
rescue => err
|
56
|
+
present err
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def delete
|
61
|
+
(info("'#{name}' does not exist "); return;) if scraper.blank?
|
62
|
+
info("You are about to delete '#{scraper.name}'")
|
63
|
+
return unless confirmed?
|
64
|
+
scraper.destroy if scraper.respond_to?(:destroy)
|
65
|
+
info("'#{scraper.name}' was deleted")
|
66
|
+
end
|
67
|
+
|
68
|
+
|
69
|
+
def replace
|
70
|
+
(info("'#{name}' does not exist "); return;) if scraper.blank?
|
71
|
+
info("You are about to replace '#{scraper.name}'")
|
72
|
+
return unless confirmed?
|
73
|
+
scraper.destroy if scraper.respond_to?(:destroy)
|
74
|
+
Quandl::Client::Scraper.create( name: name, scraper: args.first )
|
75
|
+
info("'#{scraper.name}' was replaced.")
|
76
|
+
info("#{schedule_message}")
|
77
|
+
end
|
78
|
+
|
79
|
+
private
|
80
|
+
|
81
|
+
def schedule_message
|
82
|
+
"It will run 3 times a day starting immediately. Check your scrapers run status at #{quandl_url.gsub(/\/api\/?/,'')}/scrapers"
|
83
|
+
end
|
84
|
+
|
85
|
+
def scrapers
|
86
|
+
@scrapers ||= Quandl::Client::Scraper.where( page: page ).fetch
|
87
|
+
end
|
88
|
+
|
89
|
+
def scraper
|
90
|
+
defined?(@scraper) ? @scraper : ( @scraper = find_scraper )
|
91
|
+
end
|
92
|
+
|
93
|
+
def find_scraper
|
94
|
+
return nil unless args.first.present?
|
95
|
+
# if the arg is numeric lookup the id. otherwise search for a scraper by name and grab the id
|
96
|
+
id = args.first.numeric? ? args.first : Quandl::Client::Scraper.where( name: name ).try(:first).try(:id)
|
97
|
+
# find the scraper with the id
|
98
|
+
Quandl::Client::Scraper.find(id) unless id.nil?
|
99
|
+
end
|
100
|
+
|
101
|
+
def name
|
102
|
+
return options.name if options.name.present?
|
103
|
+
return File.basename(args.first) if args.first.present?
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
class Quandl::Command::Tasks::Search < Quandl::Command::Task
|
2
|
+
autoload_quandl_client
|
3
|
+
|
4
|
+
syntax "quandl search STRING"
|
5
|
+
description "Search for datasets matching STRING"
|
6
|
+
|
7
|
+
options({
|
8
|
+
String => {
|
9
|
+
source_code: "Return datasets matching: [SOURCE_CODE]",
|
10
|
+
code: "Return datasets where code matches [code]",
|
11
|
+
},
|
12
|
+
Integer => {
|
13
|
+
limit: "Limit the number of datasets returned by the search. Defaults to unlimited.",
|
14
|
+
page: "Return datasets starting from: [page]",
|
15
|
+
}
|
16
|
+
})
|
17
|
+
|
18
|
+
before_execute :upcase_source_code
|
19
|
+
|
20
|
+
validates :query, presence: true
|
21
|
+
|
22
|
+
def execute
|
23
|
+
Quandl::Client::Dataset.query( query ).where( declared_params ).each_in_page do |dataset|
|
24
|
+
present(dataset)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
protected
|
30
|
+
|
31
|
+
def query
|
32
|
+
args.first
|
33
|
+
end
|
34
|
+
|
35
|
+
def upcase_source_code
|
36
|
+
options.source_code.to_s.upcase!
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
class Quandl::Command::Tasks::Superset < Quandl::Command::Task
|
2
|
+
|
3
|
+
autoload_quandl_client
|
4
|
+
authenticated_users_only!
|
5
|
+
|
6
|
+
description "Superset a script to be run in the quandl cloud."
|
7
|
+
syntax %{quandl superset command [file]
|
8
|
+
|
9
|
+
COMMANDS:
|
10
|
+
|
11
|
+
superset new
|
12
|
+
superset list
|
13
|
+
superset upload [file]
|
14
|
+
superset download [code]
|
15
|
+
superset delete [code] }
|
16
|
+
|
17
|
+
def new
|
18
|
+
info Quandl::Client::Superset.example.to_qdf
|
19
|
+
end
|
20
|
+
|
21
|
+
def list
|
22
|
+
Quandl::Client::Superset.owner('myself').each_in_page do |superset|
|
23
|
+
present(superset)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def upload
|
28
|
+
Quandl::Format::Superset.foreach( file_or_stdin ) do |superset|
|
29
|
+
# execute in parallel
|
30
|
+
background_job do
|
31
|
+
# upload the superset
|
32
|
+
superset.save if superset.respond_to?(:valid?) && superset.valid?
|
33
|
+
# present the result
|
34
|
+
present(superset)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def download
|
40
|
+
each_line_in_background( args_or_stdin ) do |code|
|
41
|
+
# find superset
|
42
|
+
dataset = Quandl::Client::Superset.find( code )
|
43
|
+
# present
|
44
|
+
present dataset, output_format: :qdf
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def delete
|
49
|
+
each_line_in_background( args_or_stdin ) do |code|
|
50
|
+
# find
|
51
|
+
dataset = Quandl::Client::Dataset.find( code )
|
52
|
+
# destroy
|
53
|
+
dataset.destroy
|
54
|
+
# present
|
55
|
+
present dataset
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
@@ -1,11 +1,11 @@
|
|
1
|
-
class Quandl::Command::Tasks::Update < Quandl::Command::
|
1
|
+
class Quandl::Command::Tasks::Update < Quandl::Command::Task
|
2
2
|
|
3
3
|
# curl -s https://s3.amazonaws.com/quandl-command/install.sh | bash
|
4
4
|
|
5
5
|
description "Update Quandl Toolbelt to the latest version."
|
6
6
|
syntax %Q{quandl update [revision]
|
7
7
|
|
8
|
-
|
8
|
+
EXAMPLES:
|
9
9
|
|
10
10
|
$ quandl update
|
11
11
|
Updating from 0.2.18 ...
|
@@ -1,10 +1,10 @@
|
|
1
|
-
class Quandl::Command::Tasks::Upload < Quandl::Command::
|
2
|
-
|
1
|
+
class Quandl::Command::Tasks::Upload < Quandl::Command::Task
|
2
|
+
autoload_quandl_client
|
3
3
|
|
4
4
|
description "Upload a dataset using its quandl code."
|
5
5
|
syntax %{quandl upload file.qdf
|
6
6
|
|
7
|
-
|
7
|
+
EXAMPLES:
|
8
8
|
|
9
9
|
$ quandl upload file.qcsv
|
10
10
|
OK | 98ms | http://quandl.com/USERNAME/CODE_1
|
@@ -25,11 +25,6 @@ class Quandl::Command::Tasks::Upload < Quandl::Command::Tasks::Base
|
|
25
25
|
2013-11-22,1252.0,454.95,448.2
|
26
26
|
2013-11-21,452.25,457.75,449.1
|
27
27
|
}
|
28
|
-
options({
|
29
|
-
Integer => {
|
30
|
-
threads: "How many workers to use during download.",
|
31
|
-
}
|
32
|
-
})
|
33
28
|
|
34
29
|
authenticated_users_only!
|
35
30
|
|
@@ -37,30 +32,22 @@ class Quandl::Command::Tasks::Upload < Quandl::Command::Tasks::Base
|
|
37
32
|
# datasets from file_path if given
|
38
33
|
interface = file_path.present? ? File.open(file_path, "r") : $stdin
|
39
34
|
# for each dataset streamed from interface
|
40
|
-
Quandl::Format::Dataset.each_line(interface) do |dataset|
|
41
|
-
|
35
|
+
Quandl::Format::Dataset.each_line(interface) do |dataset, error|
|
36
|
+
# present error if given
|
37
|
+
next present( error ) unless error.nil?
|
38
|
+
# process in background using thread key
|
39
|
+
background_job( lock: dataset.client.full_code ) do
|
40
|
+
# upload
|
41
|
+
upload( dataset )
|
42
|
+
end
|
42
43
|
end
|
43
|
-
pool.shutdown
|
44
44
|
end
|
45
45
|
|
46
46
|
def upload(dataset)
|
47
|
-
#
|
48
|
-
debug dataset.attributes.to_s
|
49
|
-
# upload
|
47
|
+
# upload unless errors were raised
|
50
48
|
dataset.upload if dataset.valid?
|
51
49
|
# output report to $stdout or $stderr
|
52
|
-
|
53
|
-
end
|
54
|
-
|
55
|
-
def report(dataset)
|
56
|
-
if [200,201].include?( dataset.client.status )
|
57
|
-
info table dataset.client.human_status,
|
58
|
-
dataset.client.elapsed_request_time_ms,
|
59
|
-
dataset.client.full_url
|
60
|
-
else
|
61
|
-
error(dataset.human_errors)
|
62
|
-
end
|
63
|
-
debug "---"
|
50
|
+
present(dataset.client)
|
64
51
|
end
|
65
52
|
|
66
53
|
def file_path
|
data/quandl.gemspec
CHANGED
@@ -18,9 +18,10 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.require_paths = ["lib"]
|
19
19
|
|
20
20
|
s.add_runtime_dependency "commander", '4.1.5'
|
21
|
-
|
22
|
-
s.add_runtime_dependency "
|
23
|
-
s.add_runtime_dependency "
|
21
|
+
|
22
|
+
s.add_runtime_dependency "quandl_format", "0.4.2"
|
23
|
+
s.add_runtime_dependency "quandl_client", '2.7.4'
|
24
|
+
s.add_runtime_dependency "quandl_data", '1.4.1'
|
24
25
|
s.add_runtime_dependency "quandl_operation", '0.3.2'
|
25
26
|
s.add_runtime_dependency "quandl_babelfish", '0.0.9'
|
26
27
|
s.add_runtime_dependency "quandl_logger", '0.2.5'
|
@@ -36,4 +37,5 @@ Gem::Specification.new do |s|
|
|
36
37
|
s.add_development_dependency "pry", "~> 0.9"
|
37
38
|
s.add_development_dependency "guard", "~> 2.3"
|
38
39
|
s.add_development_dependency "guard-rspec", "~> 4.2"
|
40
|
+
s.add_development_dependency "quandl_utility"
|
39
41
|
end
|
@@ -1,6 +1,17 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'spec_helper'
|
3
3
|
|
4
|
+
describe "./bin/quandl delete" do
|
5
|
+
|
6
|
+
let(:command){ self.class.superclass.description }
|
7
|
+
subject{ quandl("delete #{command}") }
|
8
|
+
|
9
|
+
context "DOES_NOT_EXIST" do
|
10
|
+
its(:stdout){ should match 'Not Found' }
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
14
|
+
|
4
15
|
describe Quandl::Command::Tasks::Delete do
|
5
16
|
|
6
17
|
let(:subject_args){ [] }
|
@@ -11,21 +22,18 @@ describe Quandl::Command::Tasks::Delete do
|
|
11
22
|
before(:each){ Quandl::Command::Tasks::Upload.call( 'spec/fixtures/data/datasets.qdf' ) }
|
12
23
|
|
13
24
|
it "should delete spec/fixtures/data/datasets.qdf" do
|
14
|
-
|
15
|
-
# load each dataset from file
|
25
|
+
TestOutput.should_receive(:info).with(/OK/).at_least(4).times
|
16
26
|
datasets = Quandl::Format::Dataset.load_from_file('spec/fixtures/data/datasets.qdf')
|
17
|
-
# send command to delete each dataset
|
18
27
|
datasets.each{|d| Quandl::Command::Tasks::Delete.call( d.code, force_yes: true ) }
|
19
|
-
|
20
|
-
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should not download" do
|
31
|
+
datasets = Quandl::Format::Dataset.load_from_file('spec/fixtures/data/datasets.qdf')
|
32
|
+
datasets.each{|d| Quandl::Command::Tasks::Delete.call( d.code, force_yes: true ) }
|
33
|
+
TestOutput.should_receive(:info).with(nil).at_least(4).times
|
21
34
|
datasets.each{|d| Quandl::Command::Tasks::Download.call( d.code ) }
|
22
35
|
end
|
23
36
|
|
24
37
|
end
|
25
38
|
|
26
|
-
|
27
|
-
Quandl::Logger.should_receive(:error).with(/Not Found/).exactly(1).times
|
28
|
-
Quandl::Command::Tasks::Delete.call("DOES_NOT_EXIST", force_yes: true )
|
29
|
-
end
|
30
|
-
|
31
|
-
end
|
39
|
+
end
|
@@ -34,9 +34,16 @@ describe Quandl::Command::Tasks::Download do
|
|
34
34
|
|
35
35
|
end
|
36
36
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
37
|
+
end
|
38
|
+
|
39
|
+
|
40
|
+
describe "./bin/quandl download" do
|
41
41
|
|
42
|
+
let(:command){ self.class.superclass.description }
|
43
|
+
subject{ quandl("download #{command}") }
|
44
|
+
|
45
|
+
context "TEST_DOES_NOT_EXIST" do
|
46
|
+
its(:stdout){ should be_blank }
|
47
|
+
end
|
48
|
+
|
42
49
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'spec_helper'
|
3
|
+
|
4
|
+
replace_string = "source_code: BLAKEHIL\ncode: SCRAPER_TEST_1394485290783983\nname: Untitled Dataset 2014-04-04 13:08:49\ndescription: This dataset has no description.\nprivate: false\nreference_url: \nfrequency: daily\n-\nDate,High,Low,Settle\n2014-03-11,9.958,11.0,13.92\n\n"
|
5
|
+
|
6
|
+
describe "./bin/quandl replace" do
|
7
|
+
|
8
|
+
let(:prefix){ "replace" }
|
9
|
+
let(:command){ self.class.superclass.description }
|
10
|
+
subject{ quandl( [prefix, command].flatten.join(" ") ) }
|
11
|
+
|
12
|
+
context data_path("replace.qdf") do
|
13
|
+
its(:stdout){ should match /OK|Created/ }
|
14
|
+
its(:stderr){ should be_blank }
|
15
|
+
end
|
16
|
+
|
17
|
+
context "SCRAPER_TEST_1394485290783983" do
|
18
|
+
let(:prefix){ "download" }
|
19
|
+
|
20
|
+
its(:stdout){ should eq replace_string }
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'spec_helper'
|
3
|
+
require 'open3'
|
4
|
+
|
5
|
+
describe "./bin/quandl schedule" do
|
6
|
+
|
7
|
+
let(:command){ self.class.superclass.description }
|
8
|
+
subject{ quandl("schedule #{command}") }
|
9
|
+
|
10
|
+
context "delete spec/fixtures/scraper.rb" do
|
11
|
+
its(:stdout){ should match 'Not Found' }
|
12
|
+
end
|
13
|
+
|
14
|
+
context "add spec/fixtures/scraper.rb" do
|
15
|
+
# first upload
|
16
|
+
its(:stdout){ should match 'Created' }
|
17
|
+
# second upload
|
18
|
+
its(:stdout){ should match 'name: has already been taken' }
|
19
|
+
end
|
20
|
+
|
21
|
+
context "list" do
|
22
|
+
its(:stdout){ should match 'scraper.rb' }
|
23
|
+
end
|
24
|
+
|
25
|
+
context "show scraper.rb --output-format json" do
|
26
|
+
its(:stdout){ should match 'scraper_url' }
|
27
|
+
end
|
28
|
+
|
29
|
+
context "show scraper.rb -C 0" do
|
30
|
+
its(:stdout){ should eq "OK\n" }
|
31
|
+
end
|
32
|
+
|
33
|
+
context "show scraper.rb -C 1" do
|
34
|
+
its(:stdout){ should eq "scraper.rb\n" }
|
35
|
+
end
|
36
|
+
|
37
|
+
context "show scraper.rb -C 2" do
|
38
|
+
its(:stdout){ should eq "Scraper::Script\n" }
|
39
|
+
end
|
40
|
+
|
41
|
+
context "show scraper.rb --output-column 2" do
|
42
|
+
its(:stdout){ should eq "Scraper::Script\n" }
|
43
|
+
end
|
44
|
+
|
45
|
+
context "show scraper.rb --output-column 4" do
|
46
|
+
its(:stdout){ should match 's3.amazonaws' }
|
47
|
+
end
|
48
|
+
|
49
|
+
context "delete spec/fixtures/scraper.rb" do
|
50
|
+
its(:stdout){ should match 'OK' }
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|