quandl 0.2.27 → 0.3.0

data/lib/quandl/command/tasks/schedule.rb

@@ -0,0 +1,106 @@
+class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
+  
+  autoload_quandl_client
+  authenticated_users_only!
+  
+  description "Schedule a script to be run in the quandl cloud."
+  syntax %{quandl schedule command [file]
+
+  COMMANDS:
+  
+    schedule list
+    schedule add       file
+    schedule show      file
+    schedule delete    file
+    schedule replace   file
+    
+  EXAMPLES:
+  
+    $ quandl schedule add scraper.rb
+    You have successfully scheduled scraper.rb to be run daily.
+
+    $ quandl schedule add scraper.rb --name weekly_scraper.rb --frequency weekly
+    You have successfully scheduled weekly-scraper.rb to be run weekly.
+  
+    $ quandl schedule replace scraper.rb
+    You have successfully replace scraper.rb}
+    
+  options({
+    String => {
+      name:            "The name used to reference your scraper in the quandl cloud.",
+      frequency:       "How often the scraper should be run. [ daily, weekly, monthly, quaterly, annually ]",
+    }
+  })
+  
+  def list
+    present(scrapers)
+  end
+  
+  def show
+    present(scraper)
+  end
+  
+  def add
+    Quandl::Client::Scraper.create( name: name, scraper: args.first )
+    info("'#{scraper.name}' was added.")
+    info("#{schedule_message}")
+  end
+  
+  def download
+    begin
+      require 'open-uri'
+      open( scraper.name, 'wb') do |file|
+        $stdout << open(scraper.scraper_url).read
+      end
+    rescue => err
+      present err
+    end
+  end
+  
+  def delete
+    (info("'#{name}' does not exist "); return;) if scraper.blank?
+    info("You are about to delete '#{scraper.name}'")
+    return unless confirmed?
+    scraper.destroy if scraper.respond_to?(:destroy)
+    info("'#{scraper.name}' was deleted")
+  end
+
+
+  def replace
+    (info("'#{name}' does not exist "); return;) if scraper.blank?
+    info("You are about to replace '#{scraper.name}'")
+    return unless confirmed?
+    scraper.destroy if scraper.respond_to?(:destroy)
+    Quandl::Client::Scraper.create( name: name, scraper: args.first )
+    info("'#{scraper.name}' was replaced.")
+    info("#{schedule_message}")
+  end
+  
+  private
+
+  def schedule_message
+    "It will run 3 times a day starting immediately. Check your scrapers run status at #{quandl_url.gsub(/\/api\/?/,'')}/scrapers"
+  end
+
+  def scrapers
+    @scrapers ||= Quandl::Client::Scraper.where( page: page ).fetch
+  end
+  
+  def scraper
+    defined?(@scraper) ? @scraper : ( @scraper = find_scraper )
+  end
+  
+  def find_scraper
+    return nil unless args.first.present?
+    # if the arg is numeric lookup the id. otherwise search for a scraper by name and grab the id
+    id = args.first.numeric? ? args.first : Quandl::Client::Scraper.where( name: name ).try(:first).try(:id)
+    # find the scraper with the id
+    Quandl::Client::Scraper.find(id) unless id.nil?
+  end
+  
+  def name
+    return options.name if options.name.present?
+    return File.basename(args.first) if args.first.present?
+  end
+  
+end

data/lib/quandl/command/tasks/search.rb

@@ -0,0 +1,39 @@
+class Quandl::Command::Tasks::Search < Quandl::Command::Task
+  autoload_quandl_client
+  
+  syntax "quandl search STRING"
+  description "Search for datasets matching STRING"
+  
+  options({
+    String => {
+      source_code:  "Return datasets matching: [SOURCE_CODE]",
+      code:         "Return datasets where code matches [code]",
+    },
+    Integer => {
+      limit:        "Limit the number of datasets returned by the search. Defaults to unlimited.",
+      page:         "Return datasets starting from: [page]",
+    }
+  })
+  
+  before_execute :upcase_source_code
+  
+  validates :query, presence: true
+  
+  def execute
+    Quandl::Client::Dataset.query( query ).where( declared_params ).each_in_page do |dataset|
+      present(dataset)
+    end
+  end
+  
+  
+  protected
+  
+  def query
+    args.first
+  end
+  
+  def upcase_source_code
+    options.source_code.to_s.upcase!
+  end
+  
+end

data/lib/quandl/command/tasks/superset.rb

@@ -0,0 +1,59 @@
+class Quandl::Command::Tasks::Superset < Quandl::Command::Task
+  
+  autoload_quandl_client
+  authenticated_users_only!
+  
+  description "Superset a script to be run in the quandl cloud."
+  syntax %{quandl superset command [file]
+
+  COMMANDS:
+    
+    superset new
+    superset list
+    superset upload [file]
+    superset download [code]
+    superset delete [code] }
+
+  def new
+    info Quandl::Client::Superset.example.to_qdf
+  end
+
+  def list
+    Quandl::Client::Superset.owner('myself').each_in_page do |superset|
+      present(superset)
+    end
+  end
+  
+  def upload
+    Quandl::Format::Superset.foreach( file_or_stdin ) do |superset|
+      # execute in parallel
+      background_job do
+        # upload the superset
+        superset.save if superset.respond_to?(:valid?) && superset.valid?
+        # present the result
+        present(superset)
+      end
+    end
+  end
+  
+  def download
+    each_line_in_background( args_or_stdin ) do |code|
+      # find superset
+      dataset = Quandl::Client::Superset.find( code )
+      # present
+      present dataset, output_format: :qdf
+    end
+  end
+  
+  def delete
+    each_line_in_background( args_or_stdin ) do |code|
+      # find
+      dataset = Quandl::Client::Dataset.find( code )
+      # destroy
+      dataset.destroy
+      # present
+      present dataset
+    end
+  end
+ 
+end

data/lib/quandl/command/tasks/uninstall.rb

@@ -1,4 +1,4 @@
-class Quandl::Command::Tasks::Uninstall < Quandl::Command::Tasks::Base
+class Quandl::Command::Tasks::Uninstall < Quandl::Command::Task
   
   # curl -s https://s3.amazonaws.com/quandl-command/install.sh | bash
   

data/lib/quandl/command/tasks/update.rb

@@ -1,11 +1,11 @@
-class Quandl::Command::Tasks::Update < Quandl::Command::Tasks::Base
+class Quandl::Command::Tasks::Update < Quandl::Command::Task
   
   # curl -s https://s3.amazonaws.com/quandl-command/install.sh | bash
   
   description "Update Quandl Toolbelt to the latest version."
   syntax %Q{quandl update [revision]
 
-    Examples:
+    EXAMPLES:
 
       $ quandl update
       Updating from 0.2.18 ...

data/lib/quandl/command/tasks/upload.rb

@@ -1,10 +1,10 @@
-class Quandl::Command::Tasks::Upload < Quandl::Command::Tasks::Base
-  autoload_client_library
+class Quandl::Command::Tasks::Upload < Quandl::Command::Task
+  autoload_quandl_client
   
   description "Upload a dataset using its quandl code."
   syntax %{quandl upload file.qdf
 
-  Examples:
+  EXAMPLES:
   
     $ quandl upload file.qcsv
     OK | 98ms | http://quandl.com/USERNAME/CODE_1
@@ -25,11 +25,6 @@ class Quandl::Command::Tasks::Upload < Quandl::Command::Tasks::Base
     2013-11-22,1252.0,454.95,448.2
     2013-11-21,452.25,457.75,449.1
   }
-  options({
-    Integer => {
-      threads:         "How many workers to use during download.",
-    }
-  })
   
   authenticated_users_only!
   
@@ -37,30 +32,22 @@ class Quandl::Command::Tasks::Upload < Quandl::Command::Tasks::Base
     # datasets from file_path if given
     interface = file_path.present? ? File.open(file_path, "r") : $stdin
     # for each dataset streamed from interface
-    Quandl::Format::Dataset.each_line(interface) do |dataset|
-      pool.process{ upload( dataset ) }
+    Quandl::Format::Dataset.each_line(interface) do |dataset, error|
+      # present error if given
+      next present( error ) unless error.nil?
+      # process in background using thread key
+      background_job( lock: dataset.client.full_code ) do
+        # upload
+        upload( dataset )
+      end
     end
-    pool.shutdown
   end
   
   def upload(dataset)
-    # display debug info when verbose
-    debug dataset.attributes.to_s
-    # upload
+    # upload unless errors were raised
     dataset.upload if dataset.valid?
     # output report to $stdout or $stderr
-    report(dataset)
-  end
-  
-  def report(dataset)
-    if [200,201].include?( dataset.client.status )
-      info table dataset.client.human_status,
-        dataset.client.elapsed_request_time_ms,
-        dataset.client.full_url
-    else
-      error(dataset.human_errors)
-    end
-    debug "---"
+    present(dataset.client)
   end
   
   def file_path

data/lib/quandl/command/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Command
-  VERSION = '0.2.27'
+  VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), '../../../VERSION'))).strip.rstrip
 end
 end

data/quandl.gemspec

@@ -18,9 +18,10 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_runtime_dependency "commander", '4.1.5'
-  s.add_runtime_dependency "quandl_format", "0.2.8"
-  s.add_runtime_dependency "quandl_client", '2.5.1'
-  s.add_runtime_dependency "quandl_data", '1.3.9'
+  
+  s.add_runtime_dependency "quandl_format", "0.4.2"
+  s.add_runtime_dependency "quandl_client", '2.7.4'
+  s.add_runtime_dependency "quandl_data", '1.4.1'
   s.add_runtime_dependency "quandl_operation", '0.3.2'
   s.add_runtime_dependency "quandl_babelfish", '0.0.9'
   s.add_runtime_dependency "quandl_logger", '0.2.5'
@@ -36,4 +37,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency "pry", "~> 0.9"
   s.add_development_dependency "guard", "~> 2.3"
   s.add_development_dependency "guard-rspec", "~> 4.2"
+  s.add_development_dependency "quandl_utility"
 end

data/spec/fixtures/scraper.rb

@@ -0,0 +1,6 @@
+puts %Q{
+code: SCRAPE_SOME_DATA
+-
+Date,Value
+#{Time.now.strftime("%Y-%m-%d")},10
+}

data/spec/lib/quandl/command/delete_spec.rb

@@ -1,6 +1,17 @@
 # encoding: utf-8
 require 'spec_helper'
 
+describe "./bin/quandl delete" do
+
+  let(:command){ self.class.superclass.description }
+  subject{ quandl("delete #{command}") }
+  
+  context "DOES_NOT_EXIST" do
+    its(:stdout){ should match 'Not Found' }
+  end
+  
+end
+
 describe Quandl::Command::Tasks::Delete do
   
   let(:subject_args){ [] }
@@ -11,21 +22,18 @@ describe Quandl::Command::Tasks::Delete do
     before(:each){ Quandl::Command::Tasks::Upload.call( 'spec/fixtures/data/datasets.qdf' ) }
 
     it "should delete spec/fixtures/data/datasets.qdf" do
-      Quandl::Logger.should_receive(:info).with(/Deleted/).exactly(4).times
-      # load each dataset from file
+      TestOutput.should_receive(:info).with(/OK/).at_least(4).times
       datasets = Quandl::Format::Dataset.load_from_file('spec/fixtures/data/datasets.qdf')
-      # send command to delete each dataset
       datasets.each{|d| Quandl::Command::Tasks::Delete.call( d.code, force_yes: true ) }
-      
-      Quandl::Logger.should_receive(:error).with(/Not Found/).exactly(4).times
+    end
+    
+    it "should not download" do
+      datasets = Quandl::Format::Dataset.load_from_file('spec/fixtures/data/datasets.qdf')
+      datasets.each{|d| Quandl::Command::Tasks::Delete.call( d.code, force_yes: true ) }
+      TestOutput.should_receive(:info).with(nil).at_least(4).times
       datasets.each{|d| Quandl::Command::Tasks::Download.call( d.code ) }
     end
     
   end
   
-  it "should fail to delete missing dataset" do
-    Quandl::Logger.should_receive(:error).with(/Not Found/).exactly(1).times
-    Quandl::Command::Tasks::Delete.call("DOES_NOT_EXIST", force_yes: true )
-  end
-  
-end
+end

data/spec/lib/quandl/command/download_spec.rb

@@ -34,9 +34,16 @@ describe Quandl::Command::Tasks::Download do
     
   end
   
-  it "should not download TEST_DOES_NOT_EXIST." do
-    Quandl::Logger.should_receive(:error).with(/Not Found/)
-    Quandl::Command::Tasks::Download.call( 'TEST_DOES_NOT_EXIST' )
-  end
+end
+
+
+describe "./bin/quandl download" do
 
+  let(:command){ self.class.superclass.description }
+  subject{ quandl("download #{command}") }
+  
+  context "TEST_DOES_NOT_EXIST" do
+    its(:stdout){ should be_blank }
+  end
+  
 end

data/spec/lib/quandl/command/replace_spec.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+require 'spec_helper'
+
+replace_string = "source_code: BLAKEHIL\ncode: SCRAPER_TEST_1394485290783983\nname: Untitled Dataset 2014-04-04 13:08:49\ndescription: This dataset has no description.\nprivate: false\nreference_url: \nfrequency: daily\n-\nDate,High,Low,Settle\n2014-03-11,9.958,11.0,13.92\n\n"
+
+describe "./bin/quandl replace" do
+  
+  let(:prefix){ "replace" }
+  let(:command){ self.class.superclass.description }
+  subject{ quandl( [prefix, command].flatten.join(" ") ) }
+  
+  context data_path("replace.qdf") do
+    its(:stdout){ should match /OK|Created/ }
+    its(:stderr){ should be_blank }
+  end
+  
+  context "SCRAPER_TEST_1394485290783983" do
+    let(:prefix){ "download" }
+    
+    its(:stdout){ should eq replace_string }
+  end
+  
+end

data/spec/lib/quandl/command/schedule_spec.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+require 'spec_helper'
+require 'open3'
+
+describe "./bin/quandl schedule" do
+
+  let(:command){ self.class.superclass.description }
+  subject{ quandl("schedule #{command}") }
+  
+  context "delete spec/fixtures/scraper.rb" do
+    its(:stdout){ should match 'Not Found' }
+  end
+  
+  context "add spec/fixtures/scraper.rb" do
+    # first upload
+    its(:stdout){ should match 'Created' }
+    # second upload
+    its(:stdout){ should match 'name: has already been taken' }
+  end
+  
+  context "list" do
+    its(:stdout){ should match 'scraper.rb' }
+  end
+  
+  context "show scraper.rb --output-format json" do
+    its(:stdout){ should match 'scraper_url' }
+  end
+  
+  context "show scraper.rb -C 0" do
+    its(:stdout){ should eq "OK\n" }
+  end
+  
+  context "show scraper.rb -C 1" do
+    its(:stdout){ should eq "scraper.rb\n" }
+  end
+  
+  context "show scraper.rb -C 2" do
+    its(:stdout){ should eq "Scraper::Script\n" }
+  end
+  
+  context "show scraper.rb --output-column 2" do
+    its(:stdout){ should eq "Scraper::Script\n" }
+  end
+  
+  context "show scraper.rb --output-column 4" do
+    its(:stdout){ should match 's3.amazonaws' }
+  end
+  
+  context "delete spec/fixtures/scraper.rb" do
+    its(:stdout){ should match 'OK' }
+  end
+  
+end