quandl 0.3.0 → 0.3.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MmI3MGNiMmJjYzNkNzU3ZWQzMjE3NTI0ZTlhNmZlMjMxMGNiMTk5NQ==
+    MDYxNjY3NjA1NTU3NmFiZGFiM2Q3YWU4YWNjZjM3NGRhZjJhZjIwOQ==
   data.tar.gz: !binary |-
-    NzUwMGIyYjkxZGRkYTk5ZTc0ZTk3YTFjNzY5YzMzZDQ5OTY3ODNlYg==
+    MjdiZTc0M2UzOTczNjBiOTQxMDdkNTgwY2JmYWQ2OTAzYjc4YjFiNQ==
 SHA512:
   metadata.gz: !binary |-
-    YmJjMTA3OGI1ODJhZWRjY2VkOWFkNzgxMTM4MTA5NmE0NzA1MDExZDFmNTBh
-    MmUzNTgyMzM0OGIxYWY5MmJlNjk0ODIxZDhmYTNjMTRlYzE4NzA2NzAzNWIx
-    NjU4ODMyOTgwZWYyYmU0OWQ4ZTg1ZWE3Y2NmZjk0YmFjZmM1NWQ=
+    ZDIwYTk5YWY2ZGZmYWY2MjBmMTNiNWI5ZjI2MTEwMzUwNmY2YzY3ZGEzMjYy
+    ZDhkNWJlMGRmZDA2MDJhZjA5NjNlN2I1NmIzNzg4MjE2MmEzMDI5MTMyZDEy
+    MzZmOGQyMWZjMjE4NmJjMDViZjBjNjAxNzg0ODFmZGIyMTQyMzI=
   data.tar.gz: !binary |-
-    NWJiNzYyYWE5MmVkOGQxNGUwYjE5MTMzZGRiZTAxMzBlMTIxNmJkNDI4NGQz
-    MjkxMjFkYzlmNzljNTJhZjlkOWQyMmVkN2JhMTE4YzdiZTYyNGRjOGRjOGM4
-    ZTg0M2Q3YTM0MmQxY2U3ODE4NWYxYzNkN2MwY2RhM2ZkY2NhNTg=
+    MDg4MjRmYTY0NDI5NmRlYzA4OGQxNTMzMWVmNWFhMGQ4M2QzODU1YThjMmVi
+    ODYyMDBjNDViYjgwZDg2ZmEwZDUzNjViMWY1ZDZjZjk1ZDIyOTU3YjFlN2Nj
+    OWQ1MjZlMmE0NzE4NDgwNTI5NWQyN2JlMzRiZjA5MWQzZTE5ZGM=

data/UPGRADE.md

@@ -1,4 +1,21 @@
-## 0.3.0 
+## 0.3.3 
+
+
+
+
+
+## 0.3.2 
+
+* QUGC-151 quandl superset upload --> quandl superset
+* QUGC-140 if a source_code is given but does not exist, it should warn the user
+
+
+
+
+## 0.3.1
+* Added support for timed scheduling
+
+## 0.3.0
 
 * QUGC-132 Add thread_pool locking so that requests for the same dataset are not made in parallel
 * QUGC-120 update Task::Threading to catch SIGINT and shutdown thread_pool. Second occurence of SIGINT will exit immediately.

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.3.3

data/lib/quandl/command/presenter.rb

@@ -1,3 +1,4 @@
+require 'quandl/command/presenter/helper'
 require 'quandl/command/presenter/record'
 
 require 'quandl/command/presenters/dataset_presenter'

data/lib/quandl/command/presenter/helper.rb

@@ -0,0 +1,19 @@
+module Quandl
+  module Command
+    class Presenter
+      class << self
+        def pretty_errors(messages)
+          result = []
+          re = messages.delete(:response_errors) if messages.is_a?(Hash) && messages.has_key?(:response_errors)
+          re.each{|k,v| messages[k] = v } if re.present? && re.is_a?(Hash)
+          # format
+          messages.each do |k,v|
+            v = v.join(" & ") if v.respond_to?(:join)
+            result << "#{k}: #{v}"
+          end
+          result.join(", ")
+        end
+      end
+    end
+  end
+end

data/lib/quandl/command/presenter/record.rb

@@ -93,17 +93,9 @@ class Record
   end
   
   def errors
-    result = []
     messages = object.try(:error_messages) || []
-    # extract repsonse errors
-    re = messages.delete(:response_errors) if messages.is_a?(Hash) && messages.has_key?(:response_errors)
-    re.each{|k,v| messages[k] = v } if re.present? && re.is_a?(Hash)
-    # format
-    messages.each do |k,v|
-      v = v.join(" & ") if v.respond_to?(:join)
-      result << "#{k}: #{v}"
-    end
-    result.join(", ")
+    # extract response errors
+    Presenter.pretty_errors(messages)
   end
   
   def docs?

data/lib/quandl/command/tasks/schedule.rb

@@ -1,5 +1,6 @@
 class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
-  
+
+  before_execute :require_dependencies
   autoload_quandl_client
   authenticated_users_only!
   
@@ -17,40 +18,46 @@ class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
   EXAMPLES:
   
     $ quandl schedule add scraper.rb
-    You have successfully scheduled scraper.rb to be run daily.
+    You have successfully scheduled scraper.rb.
 
-    $ quandl schedule add scraper.rb --name weekly_scraper.rb --frequency weekly
-    You have successfully scheduled weekly-scraper.rb to be run weekly.
+    $ quandl schedule add scraper.rb --name weekly_scraper.rb --at "7pm Monday"
+    You have successfully scheduled weekly-scraper.rb.
   
-    $ quandl schedule replace scraper.rb
-    You have successfully replace scraper.rb}
+    $ quandl schedule replace scraper.rb --at "13:00"
+    You have successfully replaced scraper.rb}
     
   options({
     String => {
       name:            "The name used to reference your scraper in the quandl cloud.",
-      frequency:       "How often the scraper should be run. [ daily, weekly, monthly, quaterly, annually ]",
+      at:              "Time to run your script in UTC timezone. e.g. '14:30','7pm Monday', 'friday 13:00'"
     }
   })
-  
+
+  DAYS_OF_THE_WEEK = %w(Sunday Monday Tuesday Wednesday Thursday Friday Saturday)
+
   def list
-    present(scrapers)
+    scrapers.each { |x| script_info x}
   end
   
   def show
-    present(scraper)
+    script_info scraper
   end
   
   def add
-    Quandl::Client::Scraper.create( name: name, scraper: args.first )
-    info("'#{scraper.name}' was added.")
-    info("#{schedule_message}")
+    result=Quandl::Client::Scraper.create( name: name, scraper: args.first, schedule_at: cron_at)
+    if result.valid?
+      info("You have successfully scheduled '#{scraper.name}'.")
+      info("#{schedule_message}")
+    else
+      raise "#{Quandl::Command::Presenter.pretty_errors(result.errors.messages).to_s.gsub("\n", ' ')}"
+    end
   end
   
   def download
     begin
       require 'open-uri'
       open( scraper.name, 'wb') do |file|
-        $stdout << open(scraper.scraper_url).read
+        $stdout << open(scraper.scraper_url, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
       end
     rescue => err
       present err
@@ -61,25 +68,65 @@ class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
     (info("'#{name}' does not exist "); return;) if scraper.blank?
     info("You are about to delete '#{scraper.name}'")
     return unless confirmed?
-    scraper.destroy if scraper.respond_to?(:destroy)
-    info("'#{scraper.name}' was deleted")
+    result=scraper.destroy if scraper.respond_to?(:destroy)
+    if result.valid?
+      info("You have successfully deleted '#{scraper.name}'")
+    else
+      raise "#{Quandl::Command::Presenter.pretty_errors(result.errors.messages).to_s.gsub("\n", ' ')}"
+    end
+
   end
 
 
   def replace
     (info("'#{name}' does not exist "); return;) if scraper.blank?
+    cron_at   #validate if cron format is valid before deleting scraper
     info("You are about to replace '#{scraper.name}'")
     return unless confirmed?
-    scraper.destroy if scraper.respond_to?(:destroy)
-    Quandl::Client::Scraper.create( name: name, scraper: args.first )
-    info("'#{scraper.name}' was replaced.")
-    info("#{schedule_message}")
+    scraper.scraper = args.first
+    scraper.schedule_at = cron_at
+    scraper.save
+    if scraper.valid?
+      info("You have successfully replaced '#{scraper.name}'.")
+      info("#{schedule_message}")
+    else
+      raise "#{Quandl::Command::Presenter.pretty_errors(scraper.errors.messages).to_s.gsub("\n", ' ')}"
+      
+    end
   end
   
   private
 
+  def require_dependencies
+    require 'chronic'
+    require 'whedon'
+  end
+
+  def script_info(script)
+    unless script.schedule_at.empty?
+      info("#{(script.name+'.').ljust(20)} Scheduled: #{script_run_time(script.schedule_at)}")
+    else
+      info("#{(script.name+'.').ljust(20)} Scheduled: 3 times a day")
+    end
+  end
+
+  
+  def script_run_time(cron_time)
+    "every #{cron_time.ends_with?('*') ? 'day' : DAYS_OF_THE_WEEK[cron_time[-1].to_i]} at #{script_next_run(cron_time).strftime("%H:%M (UTC)")}"
+  end
+
+  def script_next_run(cron_time)
+    Whedon::Schedule.new(cron_time).next
+  end
+
+
   def schedule_message
-    "It will run 3 times a day starting immediately. Check your scrapers run status at #{quandl_url.gsub(/\/api\/?/,'')}/scrapers"
+    if options.at
+      "Your script first run will happen #{script_next_run(cron_at).strftime("on %d %B at %H:%M (UTC)")}. It will continue to run #{script_run_time(cron_at)}
+Check your scrapers run status at #{quandl_url.gsub(/\/api\/?/,'')}/scrapers"
+    else
+      "It will run 3 times a day starting immediately.\n Check your scrapers run status at #{quandl_url.gsub(/\/api\/?/,'')}/scrapers"
+    end
   end
 
   def scrapers
@@ -97,7 +144,15 @@ class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
     # find the scraper with the id
     Quandl::Client::Scraper.find(id) unless id.nil?
   end
-  
+
+  def cron_at
+    return nil unless options.at
+    time_value=Chronic.parse(options.at)
+    raise "#{options.at} is invalid time" if time_value.nil?
+    day_of_the_week = options.at =~ (/Mon|Tue|Wed|Thu|Fri|Sat|Sun/i) ? time_value.wday : '*'
+    "#{time_value.min} #{time_value.hour} * * #{day_of_the_week}"
+  end
+
   def name
     return options.name if options.name.present?
     return File.basename(args.first) if args.first.present?

data/lib/quandl/command/tasks/superset.rb

@@ -14,6 +14,13 @@ class Quandl::Command::Tasks::Superset < Quandl::Command::Task
     superset download [code]
     superset delete [code] }
 
+  def execute
+    # fire a subcommand if specified
+    return self.send(args.shift) if args.first.present? && self.respond_to?(args.first)
+    # otherwise default to upload
+    upload
+  end
+
   def new
     info Quandl::Client::Superset.example.to_qdf
   end

data/quandl.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "commander", '4.1.5'
   
   s.add_runtime_dependency "quandl_format", "0.4.2"
-  s.add_runtime_dependency "quandl_client", '2.7.4'
+  s.add_runtime_dependency "quandl_client", '2.7.7'
   s.add_runtime_dependency "quandl_data", '1.4.1'
   s.add_runtime_dependency "quandl_operation", '0.3.2'
   s.add_runtime_dependency "quandl_babelfish", '0.0.9'
@@ -28,7 +28,9 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "json", '1.7.7'
   s.add_runtime_dependency "minitar", '0.5.4'
   s.add_runtime_dependency "thread", "0.1.3"
-  
+  s.add_runtime_dependency "whedon", "0.0.2"
+  s.add_runtime_dependency "chronic", "0.10.2"
+
   s.add_development_dependency "rake", "~> 10.1"
   s.add_development_dependency "fog", "~> 1.19"
   s.add_development_dependency "factory_girl", "~> 4.3"
@@ -37,5 +39,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency "pry", "~> 0.9"
   s.add_development_dependency "guard", "~> 2.3"
   s.add_development_dependency "guard-rspec", "~> 4.2"
-  s.add_development_dependency "quandl_utility"
+  s.add_development_dependency "quandl_utility", '0.1.1'
 end

data/scripts/win/quandl_toolbelt.iss

@@ -37,6 +37,7 @@ Source: "..\..\build\tarball\quandl-command\lib\*.*"; DestDir: "{app}\lib"; Flag
 Source: "..\..\build\tarball\quandl-command\config\*.*"; DestDir: "{app}\config"; Flags: recursesubdirs;
 Source: "..\..\build\tarball\quandl-command\vendor\*.*"; DestDir: "{app}\vendor"; Flags: recursesubdirs;
 Source: "..\..\build\tarball\quandl\quandl.bat"; DestDir: "{app}\bin"; Flags: recursesubdirs;
+Source: "..\..\VERSION"; DestDir: "{app}";
 
 [Registry]
 Root: HKLM; Subkey: "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"; ValueType: "expandsz"; ValueName: "QuandlPath"; \

data/spec/lib/quandl/command/schedule_spec.rb

@@ -1,53 +1,18 @@
 # encoding: utf-8
 require 'spec_helper'
 require 'open3'
+require 'chronic'
 
-describe "./bin/quandl schedule" do
+describe "Quandl::Command::Tasks::Schedule" do
+  let(:schedule) { Quandl::Command::Tasks::Schedule.new(nil, nil)}
 
-  let(:command){ self.class.superclass.description }
-  subject{ quandl("schedule #{command}") }
-  
-  context "delete spec/fixtures/scraper.rb" do
-    its(:stdout){ should match 'Not Found' }
+  its "should parse 'at' time to cron correctly " do
+    schedule.stub_chain(:options, :at){'13:00'}
+    schedule.instance_eval{cron_at}.should == "0 13 * * *"
   end
-  
-  context "add spec/fixtures/scraper.rb" do
-    # first upload
-    its(:stdout){ should match 'Created' }
-    # second upload
-    its(:stdout){ should match 'name: has already been taken' }
-  end
-  
-  context "list" do
-    its(:stdout){ should match 'scraper.rb' }
-  end
-  
-  context "show scraper.rb --output-format json" do
-    its(:stdout){ should match 'scraper_url' }
-  end
-  
-  context "show scraper.rb -C 0" do
-    its(:stdout){ should eq "OK\n" }
-  end
-  
-  context "show scraper.rb -C 1" do
-    its(:stdout){ should eq "scraper.rb\n" }
-  end
-  
-  context "show scraper.rb -C 2" do
-    its(:stdout){ should eq "Scraper::Script\n" }
-  end
-  
-  context "show scraper.rb --output-column 2" do
-    its(:stdout){ should eq "Scraper::Script\n" }
-  end
-  
-  context "show scraper.rb --output-column 4" do
-    its(:stdout){ should match 's3.amazonaws' }
-  end
-  
-  context "delete spec/fixtures/scraper.rb" do
-    its(:stdout){ should match 'OK' }
+
+  its "should parse 'at' time to cron correctly " do
+    schedule.stub_chain(:options, :at){'7pm monday'}
+    schedule.instance_eval{cron_at}.should == "0 19 * * 1"
   end
-  
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quandl
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.3
 platform: ruby
 authors:
 - Blake Hilscher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-15 00:00:00.000000000 Z
+date: 2014-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: commander
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.7.4
+        version: 2.7.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.7.4
+        version: 2.7.7
 - !ruby/object:Gem::Dependency
   name: quandl_data
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +150,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.1.3
+- !ruby/object:Gem::Dependency
+  name: whedon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
+  name: chronic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.10.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.10.2
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -266,16 +294,16 @@ dependencies:
   name: quandl_utility
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.1
 description: http://quandl.com/help/toolbelt
 email:
 - blake@hilscher.ca
@@ -307,6 +335,7 @@ files:
 - lib/quandl/command/compatibility_check.rb
 - lib/quandl/command/config.rb
 - lib/quandl/command/presenter.rb
+- lib/quandl/command/presenter/helper.rb
 - lib/quandl/command/presenter/record.rb
 - lib/quandl/command/presenters/dataset_presenter.rb
 - lib/quandl/command/presenters/error_presenter.rb