quandl 0.3.4 → 0.3.6

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MmU5MDQ4ZjEwNDk3MzY5ZTlmZjExNDdkMjczY2I0MmE0NzNiYmU3Ng==
+    N2NjNmZlMWQwNGM3ZDFiM2UwM2YwOGYxYTg4Mzk4ZjJkODQ0NzAzZA==
   data.tar.gz: !binary |-
-    NDAxOGUzOTQ4NmIyNmIxZjZiYzIyNTk5N2JhYTlkMTNmOGYzYzhhZA==
+    MGM4MTU5Y2Q1ZWQyZWY4NDc4YTZkMTY0ZjMzMGUyZDAxZjM3ZDJjYw==
 SHA512:
   metadata.gz: !binary |-
-    ZWFjNTM1YmQ0NzU5OTc2MjcxODlkM2E0YjFhN2QxNjMxZmNmODkxOTQwZGU3
-    MGI5ZDdmMmM0YTZhYmZkMDA5MGQwMjM4YTAwNzM1ZjA4NGM5NjA3YWFmZGMy
-    MTc3ZWNmM2M1ZDcxNTJmYjc5YmJlMTc3YTM5MDgxNGM5YjRjMzU=
+    MzM4NjBiMWRiMThmNGQxNmUxMDE2YTM4Njg3OTFjYmExMWRjODFkMDBhMTIy
+    ODQ3OWExM2I2YzkwYzFiMDQ1NThmNmMyNDYzN2Q0Y2RhZWU3ZmRhMTVlN2I5
+    ZmI3MjJjNTUzNzc5ZTgzM2U1ZDQwN2Q4OWIwYjgwNDUyNGNmODE=
   data.tar.gz: !binary |-
-    YzA3Zjg5NmI5NTdiOTljYWI5ZDFjZDY4N2Q5ZDA1YTQ0ZDNmODdmOThkYWM5
-    MWZiODgxZDZhZTBhNmFjMDRiZDgzNGFkOGQwZjBhMDNlM2Y2M2I4ZjRjY2Y2
-    MThmYjM2N2Y2ZGZiYjEwZmI4NzhjNzZiY2E3OGJiOTU5ZDVlNjI=
+    MTZlZjQyMTAzMzFkYmI5ZDQyNWY0MWQ4M2M2YjE2MWRlYTk4MTA3YTIzZDc1
+    ZWQ1NTk4YzllMDU1Y2I4M2EzODY2NzVhMzk3YmFhMDRmMGU0NTRmNjdhZTQz
+    MTA4NDU3NzQwODRjYWMyNzIyMWYxMzkxYjY3OGIzNjRiYzliZTA=

data/UPGRADE.md

@@ -1,3 +1,9 @@
+## 0.3.6 
+
+
+
+
+
 ## 0.3.4 
 
 * QUGC-154 disable

data/VERSION

@@ -1 +1 @@
-0.3.4
+0.3.6

data/lib/quandl/command/task.rb

@@ -13,6 +13,8 @@ require 'quandl/command/task/threading'
 require 'quandl/command/task/translations'
 require 'quandl/command/task/updatable'
 require 'quandl/command/task/clientable'
+require 'quandl/command/task/validations'
+require 'quandl/command/task/dependable'
 require 'quandl/command/task/inputable'
 
 module Quandl
@@ -24,6 +26,8 @@ class Task
 
   include Quandl::Command::Task::Configurable
   include Quandl::Command::Task::Callbacks
+  include Quandl::Command::Task::Dependable
+  include Quandl::Command::Task::Validations
   include Quandl::Command::Task::Commandable
   include Quandl::Command::Task::Logging
   include Quandl::Command::Task::Presentation

data/lib/quandl/command/task/callbacks.rb

@@ -9,34 +9,15 @@ module Callbacks
   included do
   
     extend ActiveModel::Callbacks
-    define_model_callbacks :execute
-    
-    before_execute :run_task_validations!
+    define_model_callbacks :execute, :call
     
   end
 
-  module ClassMethods
-    
-    def disable_in_gem!
-      before_execute :disable_in_gem!
-    end
-  
-  end
-  
-  def run_task_validations!
-    unless valid?
-      error( table(errors.full_messages) )
-      false
-    end
-  end
-  
-  def disable_in_gem!
-    if force_yes?
-      info("You have forced update!")
-      true
-    elsif Dir.exists?( File.join( Tasks.root, ".git") ) || File.exists?( File.join( Tasks.root, "Gemfile") )
-      fatal("#{self.class.command_name} is only permitted when installed as a package! http://quandl.com/help/toolbelt") 
-      false
+  def call
+    run_callbacks(:call) do
+      run_callbacks(:execute) do
+        execute
+      end
     end
   end
   

data/lib/quandl/command/task/clientable.rb

@@ -19,10 +19,6 @@ module Clientable
       before_execute :warn_unauthenticated_users
     end
     
-    def autoload_quandl_client
-      before_execute :autoload_quandl_client
-    end
-    
   end
   
   def current_user
@@ -48,16 +44,6 @@ module Clientable
     error("WARN: Authenticate your requests! 'quandl login' OR --token xyz923") if auth_token.blank?
   end
   
-  def autoload_quandl_client
-    require 'thread/pool'
-    require 'quandl/format'
-    require 'quandl/command/client_ext'
-    Quandl::Client.use( quandl_url )
-    Quandl::Client.token = auth_token
-    Quandl::Client.request_source = 'quandl_command'
-    Quandl::Client.request_version = Quandl::Command::VERSION
-  end
-  
 end
 
 end

data/lib/quandl/command/task/commandable.rb

@@ -78,17 +78,11 @@ module Commandable
     
   end
   
-  def call
-    run_callbacks(:execute) do
-      execute
-    end
-  end
-
   def execute
     # fire a subcommand if specified
     return self.send(args.shift) if args.first.present? && self.respond_to?(args.first)
     # otherwise show syntax
-    puts "  SYNTAX\n\n    #{self.class.syntax}"
+    info(syntax)
   end
   
   def initialize(args, options)
@@ -96,6 +90,11 @@ module Commandable
     self.options = options
   end
   
+  protected
+  
+  def syntax
+    "  SYNTAX\n\n    #{self.class.syntax}"
+  end
   
 end
 

data/lib/quandl/command/task/dependable.rb

@@ -0,0 +1,70 @@
+module Quandl
+module Command
+class Task
+
+module Dependable
+  
+  extend ActiveSupport::Concern
+  
+  included do
+    before_call :require_dependencies
+  end
+  
+  module ClassMethods
+    
+    def autoload_quandl_client
+      before_call :autoload_quandl_client
+    end
+    
+    def disable_in_gem!
+      before_call :disable_in_gem!
+    end
+    
+    def depends(*args)
+      args.each do |arg|
+        arg = arg.to_s
+        self.dependencies << arg unless dependencies.include?(arg)
+      end
+    end
+    
+    def dependencies
+      @dependencies ||= []
+    end
+
+  end
+  
+  private
+  
+  def require_dependencies
+    self.class.dependencies.each do |d|
+      debug("require #{d}")
+      require(d)
+    end
+    true
+  end
+  
+  def disable_in_gem!
+    if force_yes?
+      info("You have forced update!")
+      true
+    elsif Dir.exists?( File.join( Tasks.root, ".git") ) || File.exists?( File.join( Tasks.root, "Gemfile") )
+      fatal("#{self.class.command_name} is only permitted when installed as a package! http://quandl.com/help/toolbelt") 
+      false
+    end
+  end
+  
+  def autoload_quandl_client
+    require 'thread/pool'
+    require 'quandl/format'
+    require 'quandl/command/client_ext'
+    Quandl::Client.use( quandl_url )
+    Quandl::Client.token = auth_token
+    Quandl::Client.request_source = 'quandl_command'
+    Quandl::Client.request_version = Quandl::Command::VERSION
+  end
+  
+end
+
+end
+end
+end

data/lib/quandl/command/task/logging.rb

@@ -7,9 +7,9 @@ module Logging
   extend ActiveSupport::Concern
 
   included do
-    before_execute :start_request_timer
+    before_call :start_request_timer
     before_execute :configure_logger
-    after_execute :log_request_time
+    after_call :log_request_time
     
     attr_accessor :request_timer
     
@@ -33,7 +33,7 @@ module Logging
   end
   
   def log_request_time
-    debug("# Started: #{request_timer}. Finished: #{Time.now}. Elapsed: #{request_timer.elapsed_ms}")
+    debug("Started: #{request_timer}. Finished: #{Time.now}. Elapsed: #{request_timer.elapsed_ms}")
   end
   
   def summarize(item)

data/lib/quandl/command/task/presentation.rb

@@ -26,7 +26,6 @@ module Presentation
   def get_output_format(override={})
     format = options.output_format || config.output_format || override[:output_format] || 'pipes'
     format = 'pipes' unless %w{ pipes json qdf }.include?(format.to_s)
-    debug("output_format: #{format}")
     format
   end
   

data/lib/quandl/command/task/threading.rb

@@ -18,14 +18,15 @@ module Threading
   end
   
   def shutdown_thread_pool_on_sigint
+    debug("starting up with #{threads} threads")
     return unless threads?
     trap('SIGINT') do
-      debug "# exit signal received"
+      debug "exit signal received"
       unless thread_pool.shutdown?
-        debug "# waiting for executing jobs to finish"
+        debug "waiting for executing jobs to finish"
         thread_pool.shutdown
       end
-      debug "# exiting now"
+      debug "exiting now"
       exit
     end
   end
@@ -96,7 +97,7 @@ module Threading
   end
   
   def threads
-    options.threads.to_i || 10
+    @threads ||= options.threads.present? ? options.threads.to_i : 10
   end
   
   def exiting?

data/lib/quandl/command/task/validations.rb

@@ -0,0 +1,25 @@
+module Quandl
+module Command
+class Task
+
+module Validations
+  
+  extend ActiveSupport::Concern
+
+  included do
+    before_execute :run_task_validations!
+  end
+
+  def run_task_validations!
+    unless valid?
+      debug("run_task_validations! valid? false")
+      error( table(errors.full_messages) )
+      false
+    end
+  end
+  
+end
+
+end
+end
+end

data/lib/quandl/command/tasks/list.rb

@@ -1,6 +1,7 @@
 require 'quandl/command/tasks/search'
 
 class Quandl::Command::Tasks::List < Quandl::Command::Tasks::Search
+  
   autoload_quandl_client
   
   description "List datasets matching conditions."

data/lib/quandl/command/tasks/schedule.rb

@@ -1,6 +1,7 @@
 class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
 
-  before_execute :require_dependencies
+  depends 'chronic', 'whedon', 'open-uri'
+  
   autoload_quandl_client
   authenticated_users_only!
   
@@ -31,20 +32,31 @@ class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
       at:              "Time to run your script in UTC timezone. e.g. '14:30','7pm Monday', 'friday 13:00'"
     }
   })
-
+  
   DAYS_OF_THE_WEEK = %w(Sunday Monday Tuesday Wednesday Thursday Friday Saturday)
-
+  
+  validate :cron_at_should_be_valid!
+  
+  def execute
+    # fire a subcommand if specified
+    return self.send(args.shift) if args.first.present? && self.respond_to?(args.first)
+    # otherwise fire add
+    return add if args.first.present?
+    # otherwise display syntax
+    info(syntax)
+  end
+  
   def list
     if args.first
+      (info("'#{name}' does not exist "); return;) if scraper.blank?
       script_info scraper
     else
       scrapers.each { |x| script_info x}
     end
   end
-  
 
   def add
-    result=Quandl::Client::Scraper.create( name: name, scraper: args.first, schedule_at: cron_at)
+    result=Quandl::Client::Scraper.create( name: name, scraper: args.first, schedule_at: cron_at_utc)
     if result.valid?
       info("You have successfully scheduled '#{scraper.name}'.")
       info("#{schedule_message}")
@@ -55,7 +67,6 @@ class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
   
   def download
     begin
-      require 'open-uri'
       $stdout << open(scraper.scraper_url, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
     rescue => err
       present err
@@ -78,15 +89,14 @@ class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
 
   def replace
     (info("'#{name}' does not exist "); return;) if scraper.blank?
-    cron_at   #validate if cron format is valid before deleting scraper
     info("You are about to replace '#{scraper.name}'")
     return unless confirmed?
     scraper.scraper = args.first
-    scraper.schedule_at = cron_at
+    scraper.schedule_at = cron_at_utc
     scraper.save
     if scraper.valid?
       info("You have successfully replaced '#{scraper.name}'.")
-      info("#{schedule_message}")
+      info("#{schedule_message}") if cron_at
     else
       error "#{Quandl::Command::Presenter.pretty_errors(scraper.errors.messages).to_s.gsub("\n", ' ')}"
     end
@@ -94,11 +104,6 @@ class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
   
   private
 
-  def require_dependencies
-    require 'chronic'
-    require 'whedon'
-  end
-
   def script_info(script)
     unless script.schedule_at.empty?
       info("#{(script.name+'.').ljust(20)} Scheduled: #{script_run_time(script.schedule_at)}")
@@ -119,10 +124,10 @@ class Quandl::Command::Tasks::Schedule < Quandl::Command::Task
 
   def schedule_message
     if options.at
-      "Your script first run will happen #{script_next_run(cron_at).strftime("on %d %B at %H:%M (UTC)")}. It will continue to run #{script_run_time(cron_at)}
+      "Your script first run will happen #{script_next_run(cron_at).strftime("on %d %B at %H:%M")}. It will continue to run #{script_run_time(cron_at_utc)}
 Check your scrapers run status at #{quandl_url.gsub(/\/api\/?/,'')}/scrapers"
     else
-      "It will run 3 times a day starting immediately.\n Check your scrapers run status at #{quandl_url.gsub(/\/api\/?/,'')}/scrapers"
+      "It will run 3 times a day starting immediately.\nCheck your scrapers run status at #{quandl_url.gsub(/\/api\/?/,'')}/scrapers"
     end
   end
 
@@ -144,8 +149,15 @@ Check your scrapers run status at #{quandl_url.gsub(/\/api\/?/,'')}/scrapers"
 
   def cron_at
     return nil unless options.at
-    time_value=Chronic.parse(options.at)
-    error "#{options.at} is invalid time" if time_value.nil?
+    time_to_cron Chronic.parse(options.at)
+  end
+
+  def cron_at_utc
+    return nil unless options.at
+    time_to_cron Chronic.parse(options.at).utc
+  end
+
+  def time_to_cron(time_value)
     day_of_the_week = options.at =~ (/Mon|Tue|Wed|Thu|Fri|Sat|Sun/i) ? time_value.wday : '*'
     "#{time_value.min} #{time_value.hour} * * #{day_of_the_week}"
   end
@@ -154,5 +166,14 @@ Check your scrapers run status at #{quandl_url.gsub(/\/api\/?/,'')}/scrapers"
     return options.remote_name if options.remote_name.present?
     return File.basename(args.first) if args.first.present?
   end
-  
+
+  def cron_at_should_be_valid!
+    if options.at.present? && !Chronic.parse(options.at)
+      self.errors.add(:base,"#{options.at} is invalid time format" )
+      return false
+    end
+    true
+  end
+
+
 end

data/lib/quandl/command/tasks/superset.rb

@@ -3,7 +3,7 @@ class Quandl::Command::Tasks::Superset < Quandl::Command::Task
   autoload_quandl_client
   authenticated_users_only!
   
-  description "Superset a script to be run in the quandl cloud."
+  description "Define a superset"
   syntax %{quandl superset command [file]
 
   COMMANDS:

data/lib/quandl/command/tasks/update.rb

@@ -1,5 +1,7 @@
 class Quandl::Command::Tasks::Update < Quandl::Command::Task
-  
+    
+  depends 'uri', 'net/http', 'quandl/utility/config', 'zlib', 'archive/tar/minitar'
+    
   # curl -s https://s3.amazonaws.com/quandl-command/install.sh | bash
   
   description "Update Quandl Toolbelt to the latest version."
@@ -31,8 +33,6 @@ class Quandl::Command::Tasks::Update < Quandl::Command::Task
   end
   
   def execute
-    # load libraries needed by this action
-    require_dependencies
     info "Updating from #{Quandl::Command::VERSION} ... "
     # wipe update/ and backup/ folders
     prepare_for_update
@@ -51,7 +51,7 @@ class Quandl::Command::Tasks::Update < Quandl::Command::Task
     
   rescue => err
     # log error
-    error(err)
+    info(err)
     debug(err.backtrace.join("\n"))
     # report failure and rollback
     info("----\nAn error has occured! Rolling back to previous version ... ")
@@ -90,15 +90,7 @@ class Quandl::Command::Tasks::Update < Quandl::Command::Task
   
   
   private
-  
-  def require_dependencies
-    require 'uri'
-    require 'net/http'
-    require 'quandl/utility/config'
-    require 'zlib'
-    require 'archive/tar/minitar'
-  end
-  
+
   def prepare_for_update
     [backup_path, update_path].each do |path|
       # remove previous directory if present

data/lib/quandl/command/tasks/upload.rb

@@ -1,7 +1,7 @@
 class Quandl::Command::Tasks::Upload < Quandl::Command::Task
   autoload_quandl_client
   
-  description "Upload a dataset using its quandl code."
+  description "Upload one or more datasets."
   syntax %{quandl upload file.qdf
 
   EXAMPLES:
@@ -33,6 +33,7 @@ class Quandl::Command::Tasks::Upload < Quandl::Command::Task
     interface = file_path.present? ? File.open(file_path, "r") : $stdin
     # for each dataset streamed from interface
     Quandl::Format::Dataset.each_line(interface) do |dataset, error|
+      debug("started processing #{dataset.full_code}") if dataset.respond_to?(:full_code)
       # present error if given
       next present( error ) unless error.nil?
       # process in background using thread key

data/quandl.gemspec

@@ -20,10 +20,10 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "commander", '4.1.5'
   
   s.add_runtime_dependency "quandl_format", "0.4.2"
-  s.add_runtime_dependency "quandl_client", '2.7.7'
+  s.add_runtime_dependency "quandl_client", '2.7.8'
   s.add_runtime_dependency "quandl_data", '1.4.1'
   s.add_runtime_dependency "quandl_operation", '0.3.2'
-  s.add_runtime_dependency "quandl_babelfish", '0.0.9'
+  s.add_runtime_dependency "quandl_babelfish", '0.0.10'
   s.add_runtime_dependency "quandl_logger", '0.2.5'
   s.add_runtime_dependency "json", '1.7.7'
   s.add_runtime_dependency "minitar", '0.5.4'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quandl
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.6
 platform: ruby
 authors:
 - Blake Hilscher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-22 00:00:00.000000000 Z
+date: 2014-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: commander
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.7.7
+        version: 2.7.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.7.7
+        version: 2.7.8
 - !ruby/object:Gem::Dependency
   name: quandl_data
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.9
+        version: 0.0.10
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.9
+        version: 0.0.10
 - !ruby/object:Gem::Dependency
   name: quandl_logger
   requirement: !ruby/object:Gem::Requirement
@@ -330,6 +330,7 @@ files:
 - dist/resources/ruby/PackageInfo
 - lib/commander/command/quandl_ext.rb
 - lib/quandl/command.rb
+- lib/quandl/command/client.rb
 - lib/quandl/command/client_ext.rb
 - lib/quandl/command/client_ext/user.rb
 - lib/quandl/command/compatibility_check.rb
@@ -347,6 +348,7 @@ files:
 - lib/quandl/command/task/clientable.rb
 - lib/quandl/command/task/commandable.rb
 - lib/quandl/command/task/configurable.rb
+- lib/quandl/command/task/dependable.rb
 - lib/quandl/command/task/inputable.rb
 - lib/quandl/command/task/logging.rb
 - lib/quandl/command/task/presentation.rb
@@ -354,6 +356,7 @@ files:
 - lib/quandl/command/task/threading.rb
 - lib/quandl/command/task/translations.rb
 - lib/quandl/command/task/updatable.rb
+- lib/quandl/command/task/validations.rb
 - lib/quandl/command/tasks.rb
 - lib/quandl/command/tasks/delete.rb
 - lib/quandl/command/tasks/download.rb