quandl 0.2.27 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/README.md +70 -17
- data/Rakefile +7 -86
- data/UPGRADE.md +23 -0
- data/VERSION +1 -0
- data/lib/quandl/command.rb +11 -3
- data/lib/quandl/command/config.rb +88 -0
- data/lib/quandl/command/presenter.rb +74 -0
- data/lib/quandl/command/presenter/record.rb +121 -0
- data/lib/quandl/command/presenters/dataset_presenter.rb +19 -0
- data/lib/quandl/command/presenters/error_presenter.rb +31 -0
- data/lib/quandl/command/presenters/nil_class_presenter.rb +10 -0
- data/lib/quandl/command/presenters/scraper_presenter.rb +25 -0
- data/lib/quandl/command/presenters/superset_presenter.rb +10 -0
- data/lib/quandl/command/task.rb +40 -0
- data/lib/quandl/command/task/callbacks.rb +47 -0
- data/lib/quandl/command/task/clientable.rb +65 -0
- data/lib/quandl/command/task/commandable.rb +104 -0
- data/lib/quandl/command/task/configurable.rb +79 -0
- data/lib/quandl/command/task/inputable.rb +37 -0
- data/lib/quandl/command/task/logging.rb +83 -0
- data/lib/quandl/command/task/presentation.rb +37 -0
- data/lib/quandl/command/task/reportable.rb +35 -0
- data/lib/quandl/command/task/threading.rb +117 -0
- data/lib/quandl/command/task/translations.rb +37 -0
- data/lib/quandl/command/task/updatable.rb +77 -0
- data/lib/quandl/command/tasks.rb +6 -5
- data/lib/quandl/command/tasks/delete.rb +10 -67
- data/lib/quandl/command/tasks/download.rb +11 -78
- data/lib/quandl/command/tasks/info.rb +2 -2
- data/lib/quandl/command/tasks/list.rb +12 -24
- data/lib/quandl/command/tasks/login.rb +4 -4
- data/lib/quandl/command/tasks/replace.rb +58 -0
- data/lib/quandl/command/tasks/schedule.rb +106 -0
- data/lib/quandl/command/tasks/search.rb +39 -0
- data/lib/quandl/command/tasks/superset.rb +59 -0
- data/lib/quandl/command/tasks/uninstall.rb +1 -1
- data/lib/quandl/command/tasks/update.rb +2 -2
- data/lib/quandl/command/tasks/upload.rb +13 -26
- data/lib/quandl/command/version.rb +1 -1
- data/quandl.gemspec +5 -3
- data/spec/fixtures/scraper.rb +6 -0
- data/spec/lib/quandl/command/delete_spec.rb +19 -11
- data/spec/lib/quandl/command/download_spec.rb +11 -4
- data/spec/lib/quandl/command/replace_spec.rb +23 -0
- data/spec/lib/quandl/command/schedule_spec.rb +53 -0
- data/spec/lib/quandl/command/superset_spec.rb +28 -0
- data/spec/lib/quandl/command/upload_spec.rb +71 -24
- data/spec/lib/quandl/command_spec.rb +1 -9
- data/spec/spec_helper.rb +36 -1
- data/tasks/toolbelt/build/tarball.rb +2 -2
- metadata +55 -10
- data/lib/quandl/command/qconfig.rb +0 -86
- data/lib/quandl/command/tasks/base.rb +0 -314
| @@ -1,314 +0,0 @@ | |
| 1 | 
            -
            require 'active_model'
         | 
| 2 | 
            -
            require 'quandl/pattern'
         | 
| 3 | 
            -
            require 'quandl/pattern/client'
         | 
| 4 | 
            -
            require 'quandl/operation'
         | 
| 5 | 
            -
            require 'quandl/operation'
         | 
| 6 | 
            -
             | 
| 7 | 
            -
            module Quandl
         | 
| 8 | 
            -
            module Command
         | 
| 9 | 
            -
            module Tasks
         | 
| 10 | 
            -
              
         | 
| 11 | 
            -
            class Base
         | 
| 12 | 
            -
              class << self
         | 
| 13 | 
            -
                
         | 
| 14 | 
            -
                def inherited(klass)
         | 
| 15 | 
            -
                  Tasks.tasks << klass unless Tasks.tasks.include?(klass)
         | 
| 16 | 
            -
                end
         | 
| 17 | 
            -
                
         | 
| 18 | 
            -
                def configure(app)
         | 
| 19 | 
            -
                  return if disabled?
         | 
| 20 | 
            -
                  app.command(command_name) do |c|
         | 
| 21 | 
            -
                    c.syntax = syntax
         | 
| 22 | 
            -
                    c.description = description
         | 
| 23 | 
            -
                    c.action{|a,o| call(a,o) }
         | 
| 24 | 
            -
                    configure_options(c)
         | 
| 25 | 
            -
                  end
         | 
| 26 | 
            -
                end
         | 
| 27 | 
            -
                
         | 
| 28 | 
            -
                def disabled?
         | 
| 29 | 
            -
                  @disabled == true
         | 
| 30 | 
            -
                end
         | 
| 31 | 
            -
                
         | 
| 32 | 
            -
                def disable!
         | 
| 33 | 
            -
                  @disabled = true
         | 
| 34 | 
            -
                end
         | 
| 35 | 
            -
                
         | 
| 36 | 
            -
                def syntax(value=nil)
         | 
| 37 | 
            -
                  @syntax = value if value.present?
         | 
| 38 | 
            -
                  @syntax ||= "quandl #{command_name}"
         | 
| 39 | 
            -
                end
         | 
| 40 | 
            -
                
         | 
| 41 | 
            -
                def command_name(value=nil)
         | 
| 42 | 
            -
                  @command_name = value if value.present?
         | 
| 43 | 
            -
                  @command_name ||= name.to_s.split("::").last.downcase
         | 
| 44 | 
            -
                end
         | 
| 45 | 
            -
                
         | 
| 46 | 
            -
                def description(value=nil)
         | 
| 47 | 
            -
                  @description = value if value.present?
         | 
| 48 | 
            -
                  @description ||= "No description."
         | 
| 49 | 
            -
                end
         | 
| 50 | 
            -
                
         | 
| 51 | 
            -
                def options(value=nil)
         | 
| 52 | 
            -
                  @options = value if value.present?
         | 
| 53 | 
            -
                  @options ||= {}
         | 
| 54 | 
            -
                end
         | 
| 55 | 
            -
                
         | 
| 56 | 
            -
                def autoload_client_library
         | 
| 57 | 
            -
                  before_execute :configure_client
         | 
| 58 | 
            -
                end
         | 
| 59 | 
            -
              
         | 
| 60 | 
            -
                def authenticated_users_only!
         | 
| 61 | 
            -
                  before_execute :authenticated_users_only!
         | 
| 62 | 
            -
                end
         | 
| 63 | 
            -
              
         | 
| 64 | 
            -
                def disable_in_gem!
         | 
| 65 | 
            -
                  before_execute :disable_in_gem!
         | 
| 66 | 
            -
                end
         | 
| 67 | 
            -
              
         | 
| 68 | 
            -
                def warn_unauthenticated_users
         | 
| 69 | 
            -
                  before_execute :warn_unauthenticated_users
         | 
| 70 | 
            -
                end
         | 
| 71 | 
            -
                
         | 
| 72 | 
            -
                def call(args=[], options={})
         | 
| 73 | 
            -
                  args = Array(args)
         | 
| 74 | 
            -
                  options = ensure_options_are_command_options!(options)
         | 
| 75 | 
            -
                  self.new( args, options ).call
         | 
| 76 | 
            -
                end
         | 
| 77 | 
            -
                
         | 
| 78 | 
            -
                def t(key)
         | 
| 79 | 
            -
                  key = key.to_s
         | 
| 80 | 
            -
                  translation = lang
         | 
| 81 | 
            -
                  key.split('.').each{|m| translation = translation.respond_to?(m) ? translation.send(m) : nil }
         | 
| 82 | 
            -
                  translation
         | 
| 83 | 
            -
                end
         | 
| 84 | 
            -
                
         | 
| 85 | 
            -
                def lang
         | 
| 86 | 
            -
                  @lang ||= Quandl::Lang.send(language).quandl.command.tasks.send(command_name)
         | 
| 87 | 
            -
                end
         | 
| 88 | 
            -
                
         | 
| 89 | 
            -
                def language
         | 
| 90 | 
            -
                  # stub
         | 
| 91 | 
            -
                  'en'
         | 
| 92 | 
            -
                end
         | 
| 93 | 
            -
                
         | 
| 94 | 
            -
                protected
         | 
| 95 | 
            -
                
         | 
| 96 | 
            -
                def ensure_options_are_command_options!(options)
         | 
| 97 | 
            -
                  return options if options.class == Commander::Command::Options
         | 
| 98 | 
            -
                  OpenStruct.new(options)
         | 
| 99 | 
            -
                end
         | 
| 100 | 
            -
                
         | 
| 101 | 
            -
                def configure_options(c)
         | 
| 102 | 
            -
                  options.each do |class_type, options|
         | 
| 103 | 
            -
                    options.each do |name, desc|
         | 
| 104 | 
            -
                      c.option "--#{name} #{class_type.to_s.upcase}", class_type, desc
         | 
| 105 | 
            -
                    end
         | 
| 106 | 
            -
                  end
         | 
| 107 | 
            -
                end
         | 
| 108 | 
            -
                
         | 
| 109 | 
            -
              end
         | 
| 110 | 
            -
              
         | 
| 111 | 
            -
              attr_accessor :args, :options, :request_timer
         | 
| 112 | 
            -
              
         | 
| 113 | 
            -
              include ActiveModel::Validations
         | 
| 114 | 
            -
              
         | 
| 115 | 
            -
              extend ActiveModel::Callbacks
         | 
| 116 | 
            -
              define_model_callbacks :execute
         | 
| 117 | 
            -
              
         | 
| 118 | 
            -
              before_execute :raise_error_unless_valid!, :check_for_update, :start_request_timer
         | 
| 119 | 
            -
              after_execute :log_request_time
         | 
| 120 | 
            -
              
         | 
| 121 | 
            -
              def call
         | 
| 122 | 
            -
                run_callbacks(:execute) do
         | 
| 123 | 
            -
                  execute
         | 
| 124 | 
            -
                end
         | 
| 125 | 
            -
              end
         | 
| 126 | 
            -
              
         | 
| 127 | 
            -
              def initialize(args, options)
         | 
| 128 | 
            -
                self.args = args
         | 
| 129 | 
            -
                self.options = options
         | 
| 130 | 
            -
              end
         | 
| 131 | 
            -
              
         | 
| 132 | 
            -
              def verbose?
         | 
| 133 | 
            -
                options.verbose == true
         | 
| 134 | 
            -
              end
         | 
| 135 | 
            -
              
         | 
| 136 | 
            -
              def force_yes?
         | 
| 137 | 
            -
                options.force_yes == true
         | 
| 138 | 
            -
              end
         | 
| 139 | 
            -
              
         | 
| 140 | 
            -
              def ask_yes_or_no
         | 
| 141 | 
            -
                ['y','yes'].include?( ask("Are you sure? (y/n)") )
         | 
| 142 | 
            -
              end
         | 
| 143 | 
            -
              
         | 
| 144 | 
            -
              def summarize(item)
         | 
| 145 | 
            -
                return summarize_hash(item) if item.kind_of?(Hash)
         | 
| 146 | 
            -
                item
         | 
| 147 | 
            -
              end
         | 
| 148 | 
            -
              
         | 
| 149 | 
            -
              def summarize_hash(item)
         | 
| 150 | 
            -
                item.collect do |k,v|
         | 
| 151 | 
            -
                  next "#{k}: '#{v}'" if v.kind_of?(String)
         | 
| 152 | 
            -
                  "#{k}: #{v}"
         | 
| 153 | 
            -
                end.join(', ')
         | 
| 154 | 
            -
              end
         | 
| 155 | 
            -
              
         | 
| 156 | 
            -
              def table(*args)
         | 
| 157 | 
            -
                Array(args).flatten.join(" | ")
         | 
| 158 | 
            -
              end
         | 
| 159 | 
            -
              
         | 
| 160 | 
            -
              def info(*args)
         | 
| 161 | 
            -
                logger.info(*args)
         | 
| 162 | 
            -
              end
         | 
| 163 | 
            -
              
         | 
| 164 | 
            -
              def debug(*args)
         | 
| 165 | 
            -
                logger.debug(*args) if verbose?
         | 
| 166 | 
            -
              end
         | 
| 167 | 
            -
              
         | 
| 168 | 
            -
              def error(*args)
         | 
| 169 | 
            -
                logger.error(*args)
         | 
| 170 | 
            -
              end
         | 
| 171 | 
            -
              
         | 
| 172 | 
            -
              def fatal(*args)
         | 
| 173 | 
            -
                logger.fatal("FATAL: #{args.join(" ")}")
         | 
| 174 | 
            -
              end
         | 
| 175 | 
            -
              
         | 
| 176 | 
            -
              def logger
         | 
| 177 | 
            -
                Quandl::Logger
         | 
| 178 | 
            -
              end
         | 
| 179 | 
            -
              
         | 
| 180 | 
            -
              def current_user
         | 
| 181 | 
            -
                @current_user ||= Quandl::Client::User.info
         | 
| 182 | 
            -
              end
         | 
| 183 | 
            -
              
         | 
| 184 | 
            -
              protected
         | 
| 185 | 
            -
              
         | 
| 186 | 
            -
              # THREAD POOL
         | 
| 187 | 
            -
              
         | 
| 188 | 
            -
              def mutex
         | 
| 189 | 
            -
                @mutex ||= Mutex.new
         | 
| 190 | 
            -
              end
         | 
| 191 | 
            -
              
         | 
| 192 | 
            -
              def pool
         | 
| 193 | 
            -
                @pool ||= Thread.pool( threads )
         | 
| 194 | 
            -
              end
         | 
| 195 | 
            -
              
         | 
| 196 | 
            -
              def threads
         | 
| 197 | 
            -
                options.threads || 4
         | 
| 198 | 
            -
              end
         | 
| 199 | 
            -
              
         | 
| 200 | 
            -
              
         | 
| 201 | 
            -
              def start_request_timer
         | 
| 202 | 
            -
                self.request_timer = Time.now
         | 
| 203 | 
            -
              end
         | 
| 204 | 
            -
              
         | 
| 205 | 
            -
              def log_request_time
         | 
| 206 | 
            -
                debug("# Started: #{request_timer}. Finished: #{Time.now}. Elapsed: #{request_timer.elapsed_ms}")
         | 
| 207 | 
            -
              end
         | 
| 208 | 
            -
              
         | 
| 209 | 
            -
              def reload_session!
         | 
| 210 | 
            -
                @auth_token = nil
         | 
| 211 | 
            -
                @current_user = nil
         | 
| 212 | 
            -
                configure_client
         | 
| 213 | 
            -
              end
         | 
| 214 | 
            -
              
         | 
| 215 | 
            -
              def authenticated_users_only!
         | 
| 216 | 
            -
                if auth_token.blank?
         | 
| 217 | 
            -
                  fatal("You must authenticate to use #{self.class.command_name}! 'quandl login' OR --token xyz923")
         | 
| 218 | 
            -
                  false
         | 
| 219 | 
            -
                end
         | 
| 220 | 
            -
              end
         | 
| 221 | 
            -
              
         | 
| 222 | 
            -
              def disable_in_gem!
         | 
| 223 | 
            -
                if force_yes?
         | 
| 224 | 
            -
                  info("You have forced update!")
         | 
| 225 | 
            -
                  true
         | 
| 226 | 
            -
                elsif Dir.exists?( File.join( Tasks.root, ".git") ) || File.exists?( File.join( Tasks.root, "Gemfile") )
         | 
| 227 | 
            -
                  fatal("#{self.class.command_name} is only permitted when installed as a package! http://quandl.com/help/toolbelt") 
         | 
| 228 | 
            -
                  false
         | 
| 229 | 
            -
                end
         | 
| 230 | 
            -
              end
         | 
| 231 | 
            -
              
         | 
| 232 | 
            -
              def warn_unauthenticated_users
         | 
| 233 | 
            -
                error("WARN: Authenticate your requests! 'quandl login' OR --token xyz923") if auth_token.blank?
         | 
| 234 | 
            -
              end
         | 
| 235 | 
            -
              
         | 
| 236 | 
            -
              def raise_error_unless_valid!
         | 
| 237 | 
            -
                unless valid?
         | 
| 238 | 
            -
                  error( table(errors.full_messages) )
         | 
| 239 | 
            -
                  false
         | 
| 240 | 
            -
                end
         | 
| 241 | 
            -
              end
         | 
| 242 | 
            -
              
         | 
| 243 | 
            -
              def configure_client
         | 
| 244 | 
            -
                require 'thread/pool'
         | 
| 245 | 
            -
                require 'quandl/format'
         | 
| 246 | 
            -
                require 'quandl/command/client_ext'
         | 
| 247 | 
            -
                Quandl::Client.use( quandl_url )
         | 
| 248 | 
            -
                Quandl::Client.token = auth_token
         | 
| 249 | 
            -
                Quandl::Client.request_source = 'quandl_command'
         | 
| 250 | 
            -
                Quandl::Client.request_version = Quandl::Command::VERSION
         | 
| 251 | 
            -
              end
         | 
| 252 | 
            -
              
         | 
| 253 | 
            -
              def check_for_update
         | 
| 254 | 
            -
                check_time = QConfig.configuration.last_checked_for_update
         | 
| 255 | 
            -
                # check time present?
         | 
| 256 | 
            -
                if check_time.present? && check_time.is_a?(Time)
         | 
| 257 | 
            -
                  # has it been more than one day?  
         | 
| 258 | 
            -
                  run_update_check if Time.now - 1.day > check_time || check_time > Time.now
         | 
| 259 | 
            -
                else
         | 
| 260 | 
            -
                  run_update_check
         | 
| 261 | 
            -
                end
         | 
| 262 | 
            -
              end
         | 
| 263 | 
            -
              
         | 
| 264 | 
            -
              def run_update_check
         | 
| 265 | 
            -
                require 'uri'
         | 
| 266 | 
            -
                require 'net/http'
         | 
| 267 | 
            -
                require 'open-uri'
         | 
| 268 | 
            -
                
         | 
| 269 | 
            -
                print("# Checking for updates ... ")
         | 
| 270 | 
            -
                
         | 
| 271 | 
            -
                uri = URI.parse("https://raw.github.com/quandl/quandl_command/master/lib/quandl/command/version.rb")
         | 
| 272 | 
            -
                http = Net::HTTP.new(uri.host, uri.port)
         | 
| 273 | 
            -
                http.use_ssl = true
         | 
| 274 | 
            -
                http.verify_mode = OpenSSL::SSL::VERIFY_NONE
         | 
| 275 | 
            -
             | 
| 276 | 
            -
                request = Net::HTTP::Get.new(uri.request_uri)
         | 
| 277 | 
            -
                
         | 
| 278 | 
            -
                response = http.request(request)
         | 
| 279 | 
            -
                
         | 
| 280 | 
            -
                master = response.body
         | 
| 281 | 
            -
                master = master.split("\n").detect{|r| r =~ /VERSION/ }.split("'").last
         | 
| 282 | 
            -
                if Quandl::Command::VERSION != master
         | 
| 283 | 
            -
                  info(" A new version of quandl toolbelt has been released. #{master}. Please run 'quandl update'")
         | 
| 284 | 
            -
                else
         | 
| 285 | 
            -
                  info(" you are up to date! #{master}")
         | 
| 286 | 
            -
                end
         | 
| 287 | 
            -
              rescue => err
         | 
| 288 | 
            -
                error("An unexpected error occured while checking for updates ... #{err}")
         | 
| 289 | 
            -
              ensure
         | 
| 290 | 
            -
                QConfig.configuration.last_checked_for_update = Time.now
         | 
| 291 | 
            -
              end
         | 
| 292 | 
            -
              
         | 
| 293 | 
            -
              def quandl_url
         | 
| 294 | 
            -
                return @quandl_url if @quandl_url.present?
         | 
| 295 | 
            -
                @quandl_url = options.url if options.try(:url).present?
         | 
| 296 | 
            -
                @quandl_url = ENV['QUANDL_URL'] if @quandl_url.blank?
         | 
| 297 | 
            -
                @quandl_url = QConfig.configuration.quandl_url if @quandl_url.blank?
         | 
| 298 | 
            -
                @quandl_url = 'http://quandl.com/api/' if @quandl_url.blank?
         | 
| 299 | 
            -
                @quandl_url
         | 
| 300 | 
            -
              end
         | 
| 301 | 
            -
              
         | 
| 302 | 
            -
              def auth_token
         | 
| 303 | 
            -
                return @auth_token if @auth_token.present?
         | 
| 304 | 
            -
                @auth_token = options.token if options.try(:token).present?
         | 
| 305 | 
            -
                @auth_token = ENV['QUANDL_TOKEN'] if @auth_token.blank?
         | 
| 306 | 
            -
                @auth_token = QConfig.configuration.token if @auth_token.blank?
         | 
| 307 | 
            -
                @auth_token
         | 
| 308 | 
            -
              end
         | 
| 309 | 
            -
              
         | 
| 310 | 
            -
            end
         | 
| 311 | 
            -
             | 
| 312 | 
            -
            end
         | 
| 313 | 
            -
            end
         | 
| 314 | 
            -
            end
         |