quandl_cassandra_models 0.3.12 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19c6f57810b76a53543512153c5b92509b37067e
-  data.tar.gz: ba82d7c28e4e048f1abd221765d779a5d981fe4a
+  metadata.gz: 4259f1381c30f2b0931bd4972551d33c70286c71
+  data.tar.gz: 6bb2d31d9a58d24f89ee3379185e2c0af713d420
 SHA512:
-  metadata.gz: 9d39a799aeb081836cd8debf0e0c033fc2f06ee9b9433f0c7e9ced7307a06ec67a30d3490e7e27172486e8048a15effe8119f37a4b505eff7e7755afb3c17d84
-  data.tar.gz: b7417e67ee39584d3d34fc8b14e6524929827ef28358950f0d0692c44ae136871399299faed4168bda9ff08378977524360cb7c7141080d7166b4d25cdd5b72b
+  metadata.gz: f789664acaf5f8afae282a7fcc42147735af37bedcb1661b2aa76f6c13081393d2cdbd441a9b76deb3ab5f65485a44a956877df03bd9e3cd25b50caa5b020cb6
+  data.tar.gz: 773b3492342adc6372329848736fbfb0c376b7724407416ebbf64e20ba02d814e23fde5592c1f9681fcc304282de3e6ff0f2360a2a79c0e9580262e74d7c602a

data/UPGRADE.md

@@ -1,3 +1,11 @@
+## 0.3.13
+
+* dataset.data.delete_rows_by_time(*times) passes
+* add write validation that asserts required keys
+* dataset.data.delete_collapses pass specs
+* remove travis.yml
+
+
 ## 0.3.12
 
 * refactor read to use define_attributes for trim_start, trim_end, collapse WIKI-28 passes

data/lib/quandl/cassandra/models/column/write/insert_column_attributes.rb

@@ -13,7 +13,7 @@ class Quandl::Cassandra::Models::Column::Write::InsertColumnAttributes < Quandl:
     # insert dataset_attribute
     Quandl::Cassandra::Base.execute("INSERT INTO dataset_attributes (id, updated_at, frequency) VALUES (#{id}, #{(Time.now.to_f * 1000).to_i}, '#{frequency}')")
     # insert data
-    Quandl::Cassandra::Batch.insert(rows_values) do |id, column_id, position, frequency|
+    Quandl::Cassandra::Batch.execute(rows_values) do |id, column_id, position, frequency|
       %Q{INSERT INTO datasets (id, column_id, position) VALUES (#{id}, #{column_id}, #{position})
       INSERT INTO column_attributes ( id, frequency ) VALUES ( #{column_id}, '#{frequency}' )}
     end

data/lib/quandl/cassandra/models/column/write/insert_columns.rb

@@ -1,7 +1,7 @@
 class Quandl::Cassandra::Models::Column::Write::InsertColumns < Quandl::Cassandra::Models::Column::Write
   
   def perform
-    Quandl::Cassandra::Batch.insert(statement_values) do |id, type, time, value|
+    Quandl::Cassandra::Batch.execute(statement_values) do |id, type, time, value|
       "INSERT INTO columns (id, type, time, value) VALUES (#{id}, '#{type}', #{time}, #{value})"
     end
   end

data/lib/quandl/cassandra/models/column/write/validate.rb

@@ -0,0 +1,10 @@
+class Quandl::Cassandra::Models::Column::Write::Validate < Quandl::Cassandra::Models::Column::Write
+  
+  def perform
+    # ensure args are present
+    if attributes[:id].blank? || attributes[:data].blank?
+      raise ArgumentError, "attributes require keys [:id, :data], you passed #{attributes.keys} "
+    end
+  end
+  
+end

data/lib/quandl/cassandra/models/column/write.rb

@@ -3,6 +3,7 @@ class Quandl::Cassandra::Models::Column::Write < Quandl::Strategy::Strategize
   # strategy attributes
   define_attributes :id, :data, :frequency, :column_ids, :frequency_data, :frequency_column_data, :statement_values
   
+  require_relative 'write/validate'
   require_relative 'write/insert_columns'
   require_relative 'write/insert_column_attributes'
   require_relative 'write/group_data_by_column'
@@ -10,7 +11,9 @@ class Quandl::Cassandra::Models::Column::Write < Quandl::Strategy::Strategize
   
   # execute strategy
   def self.perform(attributes)
+    # perform strategy
     strategy = Quandl::Strategy.new( attributes ) do |c|
+      c.use Quandl::Cassandra::Models::Column::Write::Validate
       c.use Quandl::Cassandra::Models::Column::Write::GroupDataByFrequency
       c.use Quandl::Cassandra::Models::Column::Write::GroupDataByColumn
       c.use Quandl::Cassandra::Models::Column::Write::InsertColumns
@@ -18,7 +21,7 @@ class Quandl::Cassandra::Models::Column::Write < Quandl::Strategy::Strategize
     end
     strategy.perform
   rescue => e
-    Quandl::Logger.error("#{e} #{strategy.attributes}")
+    Quandl::Logger.error("#{e} #{attributes}")
     raise
   end
   

data/lib/quandl/cassandra/models/data/search.rb

@@ -44,6 +44,7 @@ module Quandl::Cassandra::Models::Data::Search
         date = Date.jd(value) if value.is_a?(Integer)
         date = Date.parse(value) if value.is_a?(String) && value =~ /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/
         date = value if value.is_a?(Date)
+        date = value.to_date if value.respond_to?(:to_date)
         date.jd
       rescue
         nil
@@ -55,13 +56,34 @@ module Quandl::Cassandra::Models::Data::Search
       delegate :to_a, :==, :inspect, :first, :flatten, :[], :collect, :<=>, :each, :each_with_index, :to_date, :to_h, to: :all, allow_nil: true
       
       def delete_all(*args)
-        if attributes[:column_ids].present?
-          result = Quandl::Cassandra::Models::Column.where( id: attributes[:column_ids] ).delete_all
+        if dataset_column_ids.present?
+          result = Quandl::Cassandra::Models::Column.where( id: dataset_column_ids ).delete_all
           return result.nil? ? true : result
         end
         false
       end
-      
+  
+      def delete_rows_by_time(*times)
+        times = Array(times).flatten
+        # dataset_column_ids are required
+        return false unless dataset_column_ids
+        statement_values = []
+        # for each column_id
+        dataset_column_ids.each do |column_id|
+          # for each time
+          times.each do |time|
+            # delete the requested column_id_time
+            statement_values << [column_id, time]
+          end
+        end
+        # execute deletion of rquested rows
+        Quandl::Cassandra::Batch.execute(statement_values) do |id, time|
+          "DELETE FROM columns WHERE id = #{id} AND type = 'source' AND time = #{time}"
+        end
+        # delete collapse data and regenerate
+        regenerate_collapses_from_source
+      end
+    
       def count(*args)
         attributes[:count] = true
         result = dataset? ? fetch.to_i : 0
@@ -74,7 +96,7 @@ module Quandl::Cassandra::Models::Data::Search
       end
       
       def dataset?
-        self.id.present? || attributes[:column_ids].present?
+        self.id.present? || dataset_column_ids.present?
       end
       
       def fetched?
@@ -84,11 +106,15 @@ module Quandl::Cassandra::Models::Data::Search
       def scoped
         s = self.class.new
         s.id( self.id ) if id.present?
-        s.column_ids( attributes[:column_ids] ) if attributes[:column_ids].present?
+        s.column_ids( dataset_column_ids ) if dataset_column_ids.present?
         s.column_frequencies(attributes[:column_frequencies]) if attributes[:column_frequencies].present?
         s
       end
       
+      def dataset_column_ids
+        attributes[:column_ids]
+      end
+      
       protected
       
       def fetch
@@ -98,6 +124,31 @@ module Quandl::Cassandra::Models::Data::Search
         Quandl::Cassandra::Models::Column.read( attributes.merge(scope_attributes) )
       end
       
+      private
+    
+      def regenerate_collapses_from_source
+        # delete collapse data
+        delete_collapses
+        # get existing data
+        data = Quandl::Cassandra::Models::Column.read( column_ids: dataset_column_ids )
+        # rewrite data
+        Quandl::Cassandra::Models::Column.write( id: id, data: data, column_ids: dataset_column_ids )
+      end
+    
+      def delete_collapses
+        statement_values = []
+        # for each column
+        dataset_column_ids.each do |column_id|
+          # for each collapse
+          Quandl::Operation::Collapse.collapses_greater_than(:daily).each do |collapse|
+            statement_values << [ column_id, collapse ]
+          end
+        end
+        # execute deletion
+        Quandl::Cassandra::Batch.execute(statement_values) do |id, type|
+          "DELETE FROM columns WHERE id = #{id} AND type = '#{type}'"
+        end
+      end
       
     end
     

data/lib/quandl/cassandra/models/version.rb

@@ -1,7 +1,7 @@
 module Quandl
 module Cassandra
 module Models
-  VERSION = '0.3.12'
+  VERSION = '0.3.13'
 end
 end
 end

data/spec/lib/quandl/cassandra/models/data/search_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe Quandl::Cassandra::Models::Data::Search do
+  
+  let(:dataset){ create(:dataset) }
+  
+  describe "#delete_collapses" do
+    before(:each){ dataset.data.send(:delete_collapses) }
+    
+    it "dataset.data.to_a should be_present" do
+      dataset.data.scoped.to_a.should be_present
+    end
+    
+    [:weekly, :monthly, :quarterly, :annual].each do |collapse|
+      it "data.collapse(#{collapse}).to_a should be_blank" do
+        dataset.data.scoped.collapse(collapse).to_a.should be_blank
+      end
+      
+      describe "#regenerate_collapses_from_source" do
+        before(:each){ dataset.data.send(:regenerate_collapses_from_source) }
+        
+        it "data.collapse(#{collapse}).to_a should be_present" do
+          dataset.data.scoped.collapse(collapse).to_a.should be_present
+        end
+        
+      end
+    end
+  end
+  
+  describe "#delete_rows_by_time" do
+    
+    let(:dates){ dataset.data.scoped.to_table.to_h.keys }
+    
+    let(:times){ [ dates[1], dates[5], dates[7], dates[10] ] }
+    
+    before(:each){ dataset.data.delete_rows_by_time(times) }
+    
+    it "times should be removed from data" do
+      data = dataset.data.scoped.to_table.to_h
+      times.each do |time|
+        data[time].should be_nil
+      end
+    end
+    
+  end
+  
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: quandl_cassandra_models
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.13
 platform: ruby
 authors:
 - Blake Hilscher
@@ -130,7 +130,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
-- .travis.yml
 - Gemfile
 - Guardfile
 - LICENSE
@@ -157,6 +156,7 @@ files:
 - lib/quandl/cassandra/models/column/write/group_data_by_frequency.rb
 - lib/quandl/cassandra/models/column/write/insert_column_attributes.rb
 - lib/quandl/cassandra/models/column/write/insert_columns.rb
+- lib/quandl/cassandra/models/column/write/validate.rb
 - lib/quandl/cassandra/models/column_attribute.rb
 - lib/quandl/cassandra/models/data.rb
 - lib/quandl/cassandra/models/data/search.rb
@@ -181,6 +181,7 @@ files:
 - spec/lib/quandl/cassandra/models/column/write_spec.rb
 - spec/lib/quandl/cassandra/models/column_attribute_spec.rb
 - spec/lib/quandl/cassandra/models/column_spec.rb
+- spec/lib/quandl/cassandra/models/data/search_spec.rb
 - spec/lib/quandl/cassandra/models/data_spec.rb
 - spec/lib/quandl/cassandra/models/dataset/collapse_spec.rb
 - spec/lib/quandl/cassandra/models/dataset/columns_spec.rb
@@ -233,6 +234,7 @@ test_files:
 - spec/lib/quandl/cassandra/models/column/write_spec.rb
 - spec/lib/quandl/cassandra/models/column_attribute_spec.rb
 - spec/lib/quandl/cassandra/models/column_spec.rb
+- spec/lib/quandl/cassandra/models/data/search_spec.rb
 - spec/lib/quandl/cassandra/models/data_spec.rb
 - spec/lib/quandl/cassandra/models/dataset/collapse_spec.rb
 - spec/lib/quandl/cassandra/models/dataset/columns_spec.rb

data/.travis.yml

@@ -1,5 +0,0 @@
-language: ruby
-rvm:
-  - "1.9.3"
-script: bundle exec rspec
-source_key: "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBNGFHWmVzQk00V0V2TWtyVzNJODNwS2VmN3B0c3k5ZGRaNW1mT01zYm5PQnZBZ1NvClpPdHprdmw2ellYYU9Tazc4UmlYamozQVJhTG9VMU5ORWNEbVcxbHpEQlFWdjFLbmVEUGduZkoyNlNmZGJoYjAKV1ZXdWlSRldLV0I0c0Rabk0yWm42TTQ3MUpodVhFVGU5a3BiSjFIOEFSbll5LzBtSGJZQjVhcm9qNzBTZ1ZLVQpxcFNGZkZjT24vUjRxUTlrWGZrL0VmblgyYWhaUTdZYjV6Z2kxa2xRZVpHRG5ReThyZWZEM1BPYjlxeXlxV1lJCmtsT3N1SVowTVcyY244Z280dWFqNE5ZVXdvU0ltbmJyMXNldWNzRTZLTkVXV2pNMnJrNFlIQ1gvM2RKa1hpWmYKSHJMSlhRNmpReFlOaHZtUzJtOFZpRHEzcWVZVi9DQmt1MnExZXdJREFRQUJBb0gvT1VuT3Jsd2xLUlNENTBFWgpUT2NRT2I1VjNUK1pZQ2hPanY5eWJma2tDYkR5ZnBDd01PK1hhTG52ZEQrSUtQdFBQT2dCa3h3YXRaT25hWUM5CkVNVHB1RWY5N2hNcmNtaUtIUHNQRFF1REphLzVpUHBMRUdPdzdSeDV4QWU2blFNek1OblBpRXpVNUNYUDM1MmMKL3V2aU9SbVg0cSt0WEIzcWo2Q20ycW1LWXZRa3NCRWFZeVFEWWJqN1JQcUltTHlKSWI0S3JNbmlQb1p0NHdiMwprNElpdEZ3RWdicldoNFNHOFJLN1ZTTllkY1FZQ1BiaHJ1aWpXUTJxcWJ6QmFtNS9OTXl1dG52cFRKaEZGMlpGClBVSXVleW5BYU95aUFSOWdaQVRPQVNRUWxYNkJHNjUrbHJmMnd6dGduR2RLREhkMFM2ZXAxU2pxbXhMUWRqWTkKSkFweEFvR0JBUE9ncGJEeUVHaVhIR1c1MHBIV1lXOG54bzhMQzZ4SjdTQ0p5RUNYT1F6bkZVSDhNbVY0eG9PdgpaTmFXNzM2V0pQSHQ2TVhoOVRvSjVBN2FpbldPSHZWOWFlSmh6cEdXUkVjenA4OEdwQUExd2FuT2NKME82ZlhBClI2Mlk5bWc4Nmw3SE4xTmJqbVMyMzRxRTM3MUxXZGJzSUE3UHR5ZmhuaDlTUVlMMzNDRjVBb0dCQU8wVy9IdUkKUzJXck1Wb3BQajhrVXRvcFNhS1JjZUNaNzdVaWVFYzJma0JyU05mZzArbnJ2Q2R1WE5RTGM2ak9YaS9MQTM4bgpXVDU0dGdmdTVYRk1BRERpV2I5NmIyU0UxNTA0MllDeWZKdVNTWHBuZCsrNmhzVFE1aU9IVnJSSUR3OElhclFOCklqZHVSaTBOMkM3eHdUTUZnbU5TejJGeHJMV0tRb1hLVlNXVEFvR0FZZWJNcnE2WHFYQVFHVkR6TFdmWS83SWMKT0o5WHNZU0RwZSs0YVR4YjBMT0w5MWhCYUVXQWdLcWEreURzWFJJNUJuOTBheERDV3ZCZWxSbDlTb0F5TUxTVgpPRVBWbnV1anhIa2RsalEwODJDbTBuWURDeHBUS2s1NmYyaFluTXJkNC9ocVl6VUhvN0ZaWTNEQitHK3ZTbzdZCjB4cmxuMEVyS2FzRWFIenlHUUVDZ1lFQW8rMHR1emY4Y1pwUk1EYXQvRGhnTk1reFNYN296dE9BZ3F3SG42MVUKMDdPYUZQRTFkRnB5cUl6TkVDdXUxM3VBY2VrYlROSm5DU09yTUljRUN6YnlOdHA2RlVOUk81NEIrb254RU9QWAo5alRLS3lEWk0xNnppZWVxQjh5dXJVM1pZNXA5TkdxVVN1RXJFa0d0UktBUTVmY2NOUkp2b3I5WEc5b3hycEc4CkY1TUNnWUVBM1ZzeFNvNWpPQzBTcnVHaUZyZjFFZDByZVkyODBNMERwRUYvNHMwM0RFL00xdGRDWFFoM1VKZGwKTjVtUC9kUUl0QzNUTzNaMDNTK3dGckp2MGk0TGJkZHhZUDlXZVZQU2RYeUN3OFBCYjRwV3JSRGptbFh6K2F3eApyczE0WWk1WnlQUDNzWHpqTE43TWhYcGJMSUE0S3BCZ0lSVTd1SXR1SzJRMm1RSEplcW89Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg=="