quandl 0.3.7 → 0.4.0

data/quandl.gemspec

@@ -19,12 +19,12 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "commander", '4.1.5'
   
-  s.add_runtime_dependency "quandl_format", "0.4.2"
-  s.add_runtime_dependency "quandl_client", '2.7.8'
+  s.add_runtime_dependency "quandl_format", "0.4.3"
+  s.add_runtime_dependency "quandl_client", '2.8.0'
   s.add_runtime_dependency "quandl_data", '1.4.1'
   s.add_runtime_dependency "quandl_operation", '0.3.2'
-  s.add_runtime_dependency "quandl_babelfish", '0.0.10'
-  s.add_runtime_dependency "quandl_logger", '0.2.5'
+  s.add_runtime_dependency "quandl_babelfish", '0.0.11'
+  s.add_runtime_dependency "quandl_logger", '0.2.6'
   s.add_runtime_dependency "json", '1.7.7'
   s.add_runtime_dependency "minitar", '0.5.4'
   s.add_runtime_dependency "thread", "0.1.3"

data/scripts/win/quandl_toolbelt.iss

@@ -32,12 +32,15 @@ Name: "english"; MessagesFile: "compiler:Default.isl"
 
 [Files]
 Source: "..\..\build\tarball\installer\rubyinstaller.exe"; DestDir: "{tmp}";
-Source: "..\..\build\tarball\quandl-command\bin\*.*"; DestDir: "{app}\bin"; Flags: recursesubdirs;
-Source: "..\..\build\tarball\quandl-command\lib\*.*"; DestDir: "{app}\lib"; Flags: recursesubdirs;
-Source: "..\..\build\tarball\quandl-command\config\*.*"; DestDir: "{app}\config"; Flags: recursesubdirs;
-Source: "..\..\build\tarball\quandl-command\vendor\*.*"; DestDir: "{app}\vendor"; Flags: recursesubdirs;
+Source: "..\..\dist\resources\windows\*.*"; DestDir: "{app}\bin"; Flags: recursesubdirs;
+Source: "..\..\build\tarball\quandl-command\lib\*.*"; DestDir: "{app}\current\lib"; Flags: recursesubdirs;
+Source: "..\..\build\tarball\quandl-command\config\*.*"; DestDir: "{app}\current\config"; Flags: recursesubdirs;
+Source: "..\..\build\tarball\quandl-command\vendor\*.*"; DestDir: "{app}\current\vendor"; Flags: recursesubdirs;
 Source: "..\..\build\tarball\quandl\quandl.bat"; DestDir: "{app}\bin"; Flags: recursesubdirs;
-Source: "..\..\VERSION"; DestDir: "{app}";
+Source: "..\..\VERSION"; DestDir: "{app}\current";
+
+[Dirs]
+Name: "{app}\releases"; Permissions: everyone-modify
 
 [Registry]
 Root: HKLM; Subkey: "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"; ValueType: "expandsz"; ValueName: "QuandlPath"; \
@@ -80,5 +83,7 @@ Type: filesandordirs; Name: "{app}/ruby-1.9.3"
 Type: filesandordirs; Name: "{app}/vendor"
 Type: filesandordirs; Name: "{app}/backup"
 Type: filesandordirs; Name: "{app}/update"
+Type: filesandordirs; Name: "{app}/current"
+Type: filesandordirs; Name: "{app}/releases"
 Type: files; Name: "{app}/*"
 Type: dirifempty; Name: "{app}"

data/spec/factories/dataset.rb

@@ -0,0 +1,10 @@
+FactoryGirl.define do
+
+  factory :dataset do
+    association :source, factory: :source
+    sequence(:code) { |n| "QUANDL_CLIENT_#{(Time.now.to_f * 1000).to_i}_#{n}" }
+    name "Quandl Client Dataset"
+    description "Quandl Client Dataset Spec"
+  end
+
+end

data/spec/factories/source.rb

@@ -0,0 +1,10 @@
+FactoryGirl.define do
+
+  factory :source do
+    sequence(:code) { |n| "QUANDL_CLIENT_#{(Time.now.to_f * 1000).to_i}_#{n}" }
+    name "Quandl Client Source"
+    description "Quandl Client Source Spec"
+    sequence(:host) { |n| "http://quandl.com/host/#{(Time.now.to_f * 1000).to_i}_#{n}" }
+  end
+
+end

data/spec/lib/quandl/command/delete_spec.rb

@@ -5,11 +5,17 @@ describe "./bin/quandl delete" do
 
   let(:command){ self.class.superclass.description }
   subject{ quandl("delete #{command}") }
+
   
   context "DOES_NOT_EXIST" do
     its(:stdout){ should match 'Not Found' }
   end
   
+  context "DATASET_1" do
+    before(:each){ create(:dataset, code: 'DATASET_1') }
+    its(:stdout){ should match 'Deleted' }
+  end
+  
 end
 
 describe Quandl::Command::Tasks::Delete do
@@ -22,7 +28,7 @@ describe Quandl::Command::Tasks::Delete do
     before(:each){ Quandl::Command::Tasks::Upload.call( 'spec/fixtures/data/datasets.qdf' ) }
 
     it "should delete spec/fixtures/data/datasets.qdf" do
-      TestOutput.should_receive(:info).with(/OK/).at_least(4).times
+      TestOutput.should_receive(:info).with(/Deleted/).at_least(4).times
       datasets = Quandl::Format::Dataset.load_from_file('spec/fixtures/data/datasets.qdf')
       datasets.each{|d| Quandl::Command::Tasks::Delete.call( d.code, force_yes: true ) }
     end

data/spec/lib/quandl/command/replace_spec.rb

@@ -10,7 +10,7 @@ describe "./bin/quandl replace" do
   subject{ quandl( [prefix, command].flatten.join(" ") ) }
   
   context data_path("replace.qdf") do
-    its(:stdout){ should match /OK|Created/ }
+    its(:stdout){ should match /Updated|Created/ }
     its(:stderr){ should be_blank }
   end
   

data/spec/lib/quandl/command/schedule_spec.rb

@@ -8,11 +8,42 @@ describe "Quandl::Command::Tasks::Schedule" do
 
   its "should parse 'at' time to cron correctly " do
     schedule.stub_chain(:options, :at){'13:00'}
-    schedule.instance_eval{cron_at}.should == "0 13 * * *"
+    schedule.instance_eval{cron_at}.should == "* 0 13 * * *"
   end
 
   its "should parse 'at' time to cron correctly " do
     schedule.stub_chain(:options, :at){'7pm monday'}
-    schedule.instance_eval{cron_at}.should == "0 19 * * 1"
+    schedule.instance_eval{cron_at}.should == "* 0 19 * * 1"
   end
+end
+
+describe "./bin/quandl schedule" do
+
+  let(:command){ self.class.superclass.description }
+  subject{ quandl("schedule #{command}") }
+  
+  context "list" do
+    its(:stdout){ should_not match "scraper-test-file.rb" }
+  end
+  
+  context "add spec/fixtures/scraper-test-file.rb" do
+    its(:stdout){ should match "You have successfully scheduled 'scraper-test-file.rb'" }
+  end
+  
+  context "delete scraper-test-file.rb --force-yes" do
+    its(:stdout){ should match "You have successfully deleted 'scraper-test-file.rb'" }
+  end
+  
+  context "add spec/fixtures/scraper-test-file.rb --at 7pm" do
+    its(:stdout){ should match "You have successfully scheduled 'scraper-test-file.rb'" }
+  end
+  
+  context "list" do
+    its(:stdout){ should match "scraper-test-file.rb Scheduled: every day at 23:00"}
+  end
+  
+  context "delete scraper-test-file.rb --force-yes" do
+    its(:stdout){ should match "You have successfully deleted 'scraper-test-file.rb'" }
+  end
+  
 end

data/spec/lib/quandl/command/superset_spec.rb

@@ -14,7 +14,7 @@ describe "./bin/quandl superset" do
   end
   
   context "upload spec/fixtures/superset.qdf" do
-    its(:stdout){ should match /Created|OK/ }
+    its(:stdout){ should match /Created|Updated/ }
   end
   
   context "download SUPERSET_EXAMPLE" do
@@ -22,7 +22,7 @@ describe "./bin/quandl superset" do
   end
   
   context "delete SUPERSET_EXAMPLE" do
-    its(:stdout){ should match /OK/ }
+    its(:stdout){ should match /Deleted/ }
   end
   
 end

data/spec/lib/quandl/command/upload_spec.rb

@@ -8,17 +8,17 @@ describe "./bin/quandl upload" do
   subject{ quandl( [prefix, command].flatten.join(" ") ) }
   
   context data_path("datasets.qdf") do
-    its(:stdout){ should match /OK|Created/ }
+    its(:stdout){ should match /Updated|Created/ }
     its(:stderr){ should be_blank }
   end
   
   context data_path("metadata.qdf") do
-    its(:stdout){ should match /OK|Created/ }
+    its(:stdout){ should match /Updated|Created/ }
     its(:stderr){ should be_blank }
   end
   
   context data_path("QUGC-42.qdf") do
-    its(:stdout){ should match /OK|Created/ }
+    its(:stdout){ should match /Updated|Created/ }
     its(:stderr){ should be_blank }
   end
   

data/spec/lib/quandl/command_spec.rb

@@ -13,7 +13,7 @@ describe "./bin/quandl" do
   end
   
   context "upload spec/fixtures/data/metadata.qdf" do
-    its(:stdout){ should match /Created|OK/ }
+    its(:stdout){ should match /Created|Updated/ }
     its(:stderr){ should be_blank }
   end
   
@@ -23,11 +23,11 @@ describe "./bin/quandl" do
   end
   
   context "delete TEST_1 TEST_5 TEST_15 --force-yes" do
-    its(:stdout){ should match 'OK' }
+    its(:stdout){ should match 'Deleted' }
   end
   
   context "info" do
-    its(:stdout){ should match ENV['QUANDL_TEST_URL'] }
+    its(:stdout){ should match QuandlTestConfig.quandl_url }
     its(:stderr){ should be_blank }
   end
   

data/spec/spec_helper.rb

@@ -1,15 +1,26 @@
 $:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+
+require 'yaml'
+require 'ostruct'
+
+QuandlTestConfig = OpenStruct.new(YAML.load(File.read(File.join(ENV['HOME'], '.quandl/test'))))
  
-ENV['QUANDL_TOKEN'] = ENV['QUANDL_TEST_TOKEN']
-ENV['QUANDL_URL'] = ENV['QUANDL_TEST_URL']
+ENV['QUANDL_TOKEN'] = QuandlTestConfig.token
+ENV['QUANDL_URL'] = QuandlTestConfig.quandl_url
 
 require "rspec"
 require 'factory_girl'
 require 'pry'
 require 'commander'
 require 'quandl/command'
+require 'quandl/client'
 require 'open3'
 
+include Quandl::Client
+
+Quandl::Client.token = QuandlTestConfig.token
+Quandl::Client.use QuandlTestConfig.quandl_url
+ 
 # disable log output
 class TestOutput
   
@@ -41,7 +52,7 @@ end
 def quandl(statement)
   version = "#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL}"
   path = File.join(Quandl::Command::Tasks.root, 'bin/quandl')
-  command = "bundle exec #{path} #{statement} --token #{ENV['QUANDL_TEST_TOKEN']} --url #{ENV['QUANDL_TEST_URL']}  --force-yes --trace"
+  command = "bundle exec #{path} #{statement} --token #{QuandlTestConfig.token} --url #{QuandlTestConfig.quandl_url}  --threads 1 --force-yes --trace --sandbox"
   stdin, stdout, stderr = Open3.popen3(command)
   OpenStruct.new( stderr: stderr.read, stdout: stdout.read )
 end

data/tasks/toolbelt.rake

@@ -27,42 +27,48 @@ def windows_root
   end
 end
 
-namespace :toolbelt do
+namespace :quandl do
   
   namespace :release do
     
     desc "Build tarball, windows, and darwin and push to storage"
-    task :all, :revision do |t, args|
+    task :all do |t, args|
       # build tarball
-      Rake::Task["toolbelt:release:tarball"].execute args.to_hash
+      Rake::Task["quandl:release:tarball"].execute args.to_hash
       # build windows exe
-      Rake::Task["toolbelt:release:windows"].execute args.to_hash
+      Rake::Task["quandl:release:windows"].execute args.to_hash
       # build darwin pkg
-      Rake::Task["toolbelt:release:darwin"].execute args.to_hash
+      Rake::Task["quandl:release:darwin"].execute args.to_hash
     end
     
     desc "Build tarball and push to storage"
-    task :tarball, :revision do |t, args|
+    task :tarball do
+      revision = `git branch | grep '*'`.gsub("*", '').strip.rstrip
+      args = {}
+      args[:revision] = revision unless revision == 'master'
       # build package
-      Rake::Task["toolbelt:build:tarball"].execute args.to_hash
+      Rake::Task["quandl:build:tarball"].execute args
       # push package to s3
-      Rake::Task["toolbelt:push:tarball"].execute args.to_hash
+      Rake::Task["quandl:push:tarball"].execute args
     end
   
     desc "Build darwin and push to storage"
-    task :darwin, :revision do |t, args|
+    task :darwin do |t, args|
       # build package
-      Rake::Task["toolbelt:build:darwin"].execute args.to_hash
+      Rake::Task["quandl:build:darwin"].execute args.to_hash
       # push package to s3
-      Rake::Task["toolbelt:push:darwin"].execute args.to_hash
+      Rake::Task["quandl:push:darwin"].execute args.to_hash
     end
     
     desc "Build windows executable and push to storage"
-    task :windows, :revision do |t, args|
+    task :windows do |t, args|
+      revision = `git branch | grep '*'`.gsub("*", '').strip.rstrip
+      args = {}
+      args[:revision] = revision unless revision == 'master'
       # build  tarball
-      Rake::Task["toolbelt:build:tarball"].execute args.to_hash
+      Rake::Task["quandl:build:tarball"].execute args.to_hash.merge({ platform: 'windows' })
       # push tarball to s3
-      Rake::Task["toolbelt:push:tarball"].execute args.to_hash.merge( platform: 'windows' )
+      Rake::Task["quandl:push:tarball"].execute args.to_hash.merge( platform: 'windows' )
       # prereleases should not build Quandl Setup
       unless args[:revision].present?
         # inno is required
@@ -71,7 +77,7 @@ namespace :toolbelt do
           puts "Cannot build 'Quandl Setup.exe'"
         else
           # create windows exe setup
-          Rake::Task["toolbelt:build:windows"].execute
+          Rake::Task["quandl:build:windows"].execute
           # push windows-package to s3
           input_file =  File.join( Toolbelt.pkg_path, 'Quandl Setup.exe')
           # upload to s3
@@ -92,7 +98,7 @@ namespace :toolbelt do
       Toolbelt::Build::Windows.execute args.to_hash
     end
     
-    task :tarball, :revision do |t, args|
+    task :tarball, :revision, :platform do |t, args|
       Toolbelt::Build::Tarball.execute args.to_hash
     end
     
@@ -120,10 +126,13 @@ namespace :toolbelt do
     end
     
     task :darwin, :revision, :platform do |t,args|
-      Toolbelt::Storage.create( 'quandl-toolbelt.pkg', File.open( File.join(Toolbelt.build_path, 'quandl-toolbelt.pkg') ) )
+      name = "quandl-toolbelt"
+      name += "-#{args[:revision]}" if args[:revision].present?
+      Toolbelt::Storage.create( "#{name}.pkg", File.open( File.join(Toolbelt.build_path, 'quandl-toolbelt.pkg') ) )
+      Toolbelt::Storage.create( "VERSION", File.open( File.join(Toolbelt.root_path, 'VERSION') ) )
+
     end
     
   end
-
   
 end

data/tasks/toolbelt.rb

@@ -15,6 +15,10 @@ class Toolbelt
       @resource_path ||= File.join(root_path, "dist/resources/")
     end
   
+    def pkg_releases_path
+      @pkg_releases_path ||= File.join( build_path, 'releases' )
+    end
+  
     def release_path
       @release_path ||= File.join(tarball_path, "quandl-command" )
     end
@@ -46,12 +50,12 @@ class Toolbelt
     def trace?
       Rake.application.options.trace == true
     end
-    
+  
   end
   
   attr_accessor :args, :options, :block
   
-  delegate :root_path, :ruby_pkg_path, :pkg_path, :tmp_path, :build_path, :tarball_path, :release_path, :resource_path, :trace?, to: :class
+  delegate :root_path, :ruby_pkg_path, :pkg_path, :tmp_path, :build_path, :tarball_path, :pkg_releases_path, :release_path, :resource_path, :trace?, to: :class
   
   def initialize(*args, &block)
     self.options = OpenStruct.new(args.extract_options!)
@@ -72,6 +76,26 @@ class Toolbelt
       run_callbacks(:ensure)
     end
   end
+
+  def timestamp
+    @timestamp ||= (Time.now.getutc.to_f * 10000).to_i.to_s.gsub('.','')
+  end
+
+  def git_short_reference
+    @git_short_reference ||= git_reference[0..7]
+  end
+
+  def git_reference
+    @git_reference ||= `git rev-parse HEAD`.strip.rstrip
+  end
+  
+  def git_branch
+    @git_branch ||= `git rev-parse --abbrev-ref HEAD`.strip.rstrip
+  end
+  
+  def prerelease_information
+    "(#{git_branch.split('/').first}/#{git_short_reference})" unless git_branch == 'master'
+  end
   
   def revision
     @revision ||= options.revision.to_s.gsub("/","-") || "master"

data/tasks/toolbelt/build/darwin.rb

@@ -12,32 +12,77 @@ class Tasks::Toolbelt::Build::Darwin < Tasks::Toolbelt
     # compile build
     Tasks::Toolbelt::Build::Tarball.execute(options)
     
-    rm "#{release_path}/bin/quandl"
-    cp resource("pkg/quandl"), "#{release_path}/bin/quandl"
+    create_release
+    add_quandl_executable
+    
+    generate_distribution
+    generate_package_info
+    generate_postinstall
+    generate_bom
     
-    kbytes = %x{ du -ks #{release_path} | cut -f 1 }.strip.rstrip
-    num_files = %x{ find #{release_path} | wc -l }.strip.rstrip
+    add_payload
+    add_ruby
     
+    package_and_sign
+
+  end
+  
+  def kbytes
+    @kbytes ||= %x{ du -ks #{release_path} | cut -f 1 }.strip.rstrip
+  end
+  
+  def num_files
+    @num_files ||= %x{ find #{release_path} | wc -l }.strip.rstrip
+  end
+  
+  private
+  
+  def create_release
+    # create releases dir and move quandl into it
+    mv release_path, "#{pkg_releases_path}/#{timestamp}"
+    mkdir_p release_path
+    mv pkg_releases_path, "#{release_path}/releases"
+  end
+  
+  def add_quandl_executable
+    mkdir_p "#{release_path}/bin"
+    cp resource("pkg/quandl"), "#{release_path}/bin/quandl"
+  end
+  
+  def generate_distribution
     mkdir_p "#{pkg_path}/Resources"
     mkdir_p "#{pkg_path}/quandl-toolbelt.pkg"
     
     dist = File.read(resource("pkg/Distribution.erb"))
     dist = ERB.new(dist).result(binding)
     File.open("#{pkg_path}/Distribution", "w") { |f| f.puts dist }
-
+  end
+  
+  def generate_package_info
     dist = File.read(resource("pkg/PackageInfo.erb"))
     dist = ERB.new(dist).result(binding)
     File.open("#{pkg_path}/quandl-toolbelt.pkg/PackageInfo", "w") { |f| f.puts dist }
-
+  end
+  
+  def generate_postinstall
+    file = File.read(resource("pkg/postinstall.erb"))
+    file = ERB.new(file).result(binding)
     mkdir_p "#{pkg_path}/quandl-toolbelt.pkg/Scripts"
-    cp resource("pkg/postinstall"), "#{pkg_path}/quandl-toolbelt.pkg/Scripts/postinstall"
+    File.open("#{pkg_path}/quandl-toolbelt.pkg/Scripts/postinstall", "w") { |f| f.puts file }
     chmod 0755, "#{pkg_path}/quandl-toolbelt.pkg/Scripts/postinstall"
-
+  end
+  
+  def generate_bom
     sh %{ mkbom -s #{release_path} #{pkg_path}/quandl-toolbelt.pkg/Bom }
-
+  end
+  
+  def add_payload
     Dir.chdir(release_path) do
       sh %{ pax -wz -x cpio . > #{pkg_path}/quandl-toolbelt.pkg/Payload }
     end
+  end
+  
+  def add_ruby
     
     # download, compile, and build ruby pkg unless already done
     Tasks::Toolbelt::Build::Ruby.execute unless File.exists?(File.join(build_path, 'ruby.pkg'))
@@ -49,20 +94,22 @@ class Tasks::Toolbelt::Build::Darwin < Tasks::Toolbelt
       mv "ruby.pkg", "#{pkg_path}/ruby.pkg"  
     end
     
+  end
+  
+  def package_and_sign
     sh %{ pkgutil --flatten #{pkg_path} #{build_path}/quandl-toolbelt-unsigned.pkg }
     sh %{ productsign --sign #{CERTIFICATE_ID} #{build_path}/quandl-toolbelt-unsigned.pkg #{build_path}/quandl-toolbelt.pkg }
     
   end
   
-  protected
-  
-  
   def clean
     rm "#{build_path}/quandl-toolbelt.pkg" if File.exists?("#{build_path}/quandl-toolbelt.pkg")
     rm_rf tmp_path
     mkdir_p tmp_path
     rm_rf pkg_path
     mkdir_p pkg_path
+    rm_rf pkg_releases_path
+    mkdir_p pkg_releases_path
   end
   
   def cleanup