quandl 0.2.22

data/lib/quandl/command/tasks.rb

@@ -0,0 +1,15 @@
+module Quandl
+module Command
+module Tasks
+  
+  def self.root
+    @root ||= File.expand_path(File.join(File.dirname(__FILE__), '../../../'))
+  end
+  
+  # require base task
+  require 'quandl/command/tasks/base'
+  # require tasks
+  Dir.glob(File.join(root, 'lib/quandl/command/tasks/*.rb')){|t| require(t) }
+end
+end
+end

data/lib/quandl/command/tasks/base.rb

@@ -0,0 +1,263 @@
+autoload :OpenURI, 'open-uri'
+
+module Quandl
+module Command
+module Tasks
+  
+class Base
+  class << self
+    
+    def configure(app)
+      return if disabled?
+      app.command(command_name) do |c|
+        c.syntax = syntax
+        c.description = description
+        c.action{|a,o| call(a,o) }
+        configure_options(c)
+      end
+    end
+    
+    def disabled?
+      @disabled == true
+    end
+    
+    def disable!
+      @disabled = true
+    end
+    
+    def syntax(value=nil)
+      @syntax = value if value.present?
+      @syntax ||= "quandl #{command_name}"
+    end
+    
+    def command_name(value=nil)
+      @command_name = value if value.present?
+      @command_name ||= name.to_s.split("::").last.downcase
+    end
+    
+    def description(value=nil)
+      @description = value if value.present?
+      @description ||= "No description."
+    end
+    
+    def options(value=nil)
+      @options = value if value.present?
+      @options ||= {}
+    end
+  
+    def authenticated_users_only!
+      before_execute :authenticated_users_only!
+    end
+  
+    def warn_unauthenticated_users
+      before_execute :warn_unauthenticated_users
+    end
+    
+    def call(args=[], options={})
+      args = Array(args)
+      options = ensure_options_are_command_options!(options)
+      self.new( args, options ).call
+    end
+    
+    protected
+    
+    def ensure_options_are_command_options!(options)
+      return options if options.class == Commander::Command::Options
+      OpenStruct.new(options)
+    end
+    
+    def configure_options(c)
+      options.each do |class_type, options|
+        options.each do |name, desc|
+          c.option "--#{name} #{class_type.to_s.upcase}", class_type, desc
+        end
+      end
+    end
+    
+  end
+  
+  attr_accessor :args, :options, :request_timer
+  
+  include ActiveModel::Validations
+  
+  extend ActiveModel::Callbacks
+  define_model_callbacks :execute
+  
+  before_execute :configure_client, :raise_error_unless_valid!, :check_for_update, :start_request_timer
+  after_execute :log_request_time
+  
+  def call
+    run_callbacks(:execute) do
+      execute
+    end
+  end
+  
+  def initialize(args, options)
+    self.args = args
+    self.options = options
+  end
+  
+  def verbose?
+    options.verbose == true
+  end
+  
+  def force_yes?
+    options.force_yes == true
+  end
+  
+  def ask_yes_or_no
+    ['y','yes'].include?( ask("Are you sure? (y/n)") )
+  end
+  
+  def summarize(item)
+    return summarize_hash(item) if item.kind_of?(Hash)
+    item
+  end
+  
+  def summarize_hash(item)
+    item.collect do |k,v|
+      next "#{k}: '#{v}'" if v.kind_of?(String)
+      "#{k}: #{v}"
+    end.join(', ')
+  end
+  
+  def table(*args)
+    Array(args).flatten.join(" | ")
+  end
+  
+  def info(*args)
+    logger.info(*args)
+  end
+  
+  def debug(*args)
+    logger.debug(*args) if verbose?
+  end
+  
+  def error(*args)
+    logger.error(*args)
+  end
+  
+  def fatal(*args)
+    logger.fatal("FATAL: #{args.join(" ")}")
+  end
+  
+  def logger
+    Quandl::Logger
+  end
+  
+  def current_user
+    @current_user ||= Quandl::Client::User.info
+  end
+  
+  protected
+  
+  # THREAD POOL
+  
+  def mutex
+    @mutex ||= Mutex.new
+  end
+  
+  def pool
+    @pool ||= Thread.pool( threads )
+  end
+  
+  def threads
+    options.threads || 4
+  end
+  
+  
+  def start_request_timer
+    self.request_timer = Time.now
+  end
+  
+  def log_request_time
+    debug("# Started: #{request_timer}. Finished: #{Time.now}. Elapsed: #{request_timer.elapsed_ms}")
+  end
+  
+  def reload_session!
+    @auth_token = nil
+    @current_user = nil
+    configure_client
+  end
+  
+  def authenticated_users_only!
+    if auth_token.blank?
+      fatal("You must authenticate to use #{self.class.command_name}! 'quandl login' OR --token xyz923")
+      false
+    end
+  end
+  
+  def warn_unauthenticated_users
+    error("WARN: Authenticate your requests! 'quandl login' OR --token xyz923") if auth_token.blank?
+  end
+  
+  def raise_error_unless_valid!
+    unless valid?
+      error( table(errors.full_messages) )
+      false
+    end
+  end
+  
+  def configure_client
+    Quandl::Client.use( quandl_url )
+    Quandl::Client.token = auth_token
+  end
+  
+  def check_for_update
+    check_time = QConfig.configuration.last_checked_for_update
+    # check time present?
+    if check_time.present? && check_time.is_a?(Time)
+      # has it been more than one day?  
+      run_update_check if Time.now - 1.day > check_time || check_time > Time.now
+    else
+      run_update_check
+    end
+  end
+  
+  def run_update_check
+    print("# Checking for updates ... ")
+    
+    uri = URI.parse("https://raw.github.com/quandl/quandl_command/master/lib/quandl/command/version.rb")
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+    request = Net::HTTP::Get.new(uri.request_uri)
+    
+    response = http.request(request)
+    
+    master = response.body
+    master = master.split("\n").detect{|r| r =~ /VERSION/ }.split("'").last
+    if Quandl::Command::VERSION != master
+      info(" A new version of quandl toolbelt has been released. #{master}. Please run 'quandl update'")
+    else
+      info(" you are up to date! #{master}")
+    end
+  rescue => err
+    error("An unexpected error occured while checking for updates ... #{err}")
+  ensure
+    QConfig.configuration.last_checked_for_update = Time.now
+  end
+  
+  def quandl_url
+    return @quandl_url if @quandl_url.present?
+    @quandl_url = options.url if options.try(:url).present?
+    @quandl_url = ENV['QUANDL_URL'] if @quandl_url.blank?
+    @quandl_url = QConfig.configuration.quandl_url if @quandl_url.blank?
+    @quandl_url = 'http://quandl.com/api/' if @quandl_url.blank?
+    @quandl_url
+  end
+  
+  def auth_token
+    return @auth_token if @auth_token.present?
+    @auth_token = options.token if options.try(:token).present?
+    @auth_token = ENV['QUANDL_TOKEN'] if @auth_token.blank?
+    @auth_token = QConfig.configuration.token if @auth_token.blank?
+    @auth_token
+  end
+  
+end
+
+end
+end
+end

data/lib/quandl/command/tasks/delete.rb

@@ -0,0 +1,58 @@
+class Quandl::Command::Tasks::Delete < Quandl::Command::Tasks::Base
+  
+  syntax "quandl #{command_name} (SOURCE_CODE/)CODE"
+  description "Delete a dataset by its quandl code."
+  
+  authenticated_users_only!
+    
+  def execute
+    # download using arguments when present
+    return delete_each_argument if args.first.present?
+    # otherwise delete using stdin
+    delete_each_stdin
+  end
+  
+  def delete_each_argument
+    args.each do |code| 
+      pool.process{ delete( code ) }
+    end
+    pool.shutdown
+  end
+  
+  def delete_each_stdin
+    return error("Cannot delete datasets from STDIN unless --force-yes is set!") unless force_yes?
+    $stdin.each_line do |code|
+      pool.process{ delete( code.strip.rstrip ) }
+    end
+    pool.shutdown
+  end
+  
+  def delete(code)
+    return error("You must provide a code!") if code.blank?
+    # find by code
+    dataset = Quandl::Client::Dataset.find(code)
+    # fail fast if dataset does not exist
+    return error(table(Quandl::Client::HTTP_STATUS_CODES[404], code)) if dataset.nil?
+    return report(dataset) unless dataset.exists?
+    # confirm deletion
+    return error("You must confirm deletion!") unless confirmed?(dataset)
+    # delete if exists
+    dataset.destroy if dataset.exists?
+    # output status
+    report(dataset)
+  end
+  
+  def report(dataset)
+    # message
+    m = table dataset.elapsed_request_time_ms, dataset.full_code
+    # report
+    dataset.status == 200 ? info(table("Deleted", m)) : error(table(dataset.human_status, m))
+  end
+  
+  def confirmed?(dataset)
+    return true if force_yes?
+    info summarize dataset.attributes.slice('source_code', 'code', 'name', 'from_date', 'to_date', 'column_names', 'created_at')
+    ask_yes_or_no
+  end
+  
+end

data/lib/quandl/command/tasks/download.rb

@@ -0,0 +1,111 @@
+require 'thread/pool'
+
+class Quandl::Command::Tasks::Download < Quandl::Command::Tasks::Base
+  
+  syntax "quandl download (SOURCE_CODE/)CODE"
+  description "Download a dataset using its quandl code."
+  options({
+    String => {
+      order:        "Return rows in either ASC or DESC order",
+      transform:    "Transform data using one of: #{Quandl::Operation::Transform.valid_transformations.join(', ')}",
+      collapse:     "Collapse data to one of: #{Quandl::Operation::Collapse.valid_collapses.join(', ')}",
+      trim_start:   "Exclude rows that are older than trim_start",
+      trim_end:     "Exclude rows that are newer than trim_end",
+    },
+    Integer => {
+      threads:      "How many workers to use during download.",
+      limit:        "Limit the number of rows returned",
+      column:       "Pluck a specific column by index",
+      row:          "Pluck a specific row by index",
+      offset:       "Offset the start of the rows",
+    }
+  })
+  
+  warn_unauthenticated_users
+  
+  validates :trim_start, :trim_end, allow_nil: true, format: { with: Quandl::Pattern.dataset_date, message: "is invalid. Expected format: #{Quandl::Pattern.dataset_date.to_example}" }
+  validates :order, allow_nil: true, inclusion: { in: ['asc', 'desc'], message: "must be one of: asc, desc" }
+  validates :collapse, allow_nil: true, inclusion: { in: Quandl::Operation::Collapse.valid_collapses.collect(&:to_s), message: "is not included in the list: #{Quandl::Operation::Collapse.valid_collapses.join(", ")}" }
+  validates :transform, allow_nil: true, inclusion: { in: Quandl::Operation::Transform.valid_transformations.collect(&:to_s), message: "is not included in the list: #{Quandl::Operation::Transform.valid_transformations.join(", ")}" }
+  
+  def execute
+    # download using arguments when present
+    return download_each_argument if args.first.present?
+    # otherwise download using stdin
+    download_each_stdin
+  end
+  
+  def download_each_argument
+    args.each do |code|
+      pool.process{ download(code) }
+    end
+    pool.shutdown
+  end
+  
+  def download_each_stdin
+    $stdin.each_line do |code|
+      pool.process{ download(code.strip.rstrip) }
+    end
+    pool.shutdown
+  end
+  
+  def download(code)
+    timer = Time.now
+    # find dataset
+    dataset = Quandl::Client::Dataset.find( code )
+    # fail fast
+    return error(table(Quandl::Client::HTTP_STATUS_CODES[404], code)) if dataset.nil?
+    # set data operations
+    dataset.data.assign_attributes(data_params) unless dataset.blank?
+    # send request & check for errors.
+    if !dataset.exists? && !dataset.valid?
+      # raise detailed error
+      return error( table( dataset.human_status, code, dataset.elapsed_request_time_ms ) )
+    end
+    # generate qdf
+    elapsed = timer.elapsed_ms
+    qdf = dataset.to_qdf
+    # write to STDOUT
+    mutex.synchronize{
+      debug("# #{dataset.try(:full_url)} downloaded in #{elapsed}")
+      info(qdf)
+    }
+    true
+  end
+  
+  def trim_start
+    options.trim_start
+  end
+  
+  def trim_end
+    options.trim_end
+  end
+  
+  def order
+    options.order
+  end
+  
+  def collapse
+    options.collapse
+  end
+  
+  def transform
+    options.transform
+  end
+  
+  def data_params
+    params = {}
+    self.class.options.each do |class_type, opts|
+      opts.each do |name, desc|
+        if options.is_a?(OpenStruct)
+          params[name] = self.options.send(name)
+        else
+          params[name] = self.options[name] if self.options[name].present?
+        end
+        
+      end
+    end
+    params
+  end
+  
+end