quandl_cassandra 0.2.1 → 0.3.0

data/UPGRADE.md

@@ -1,3 +1,11 @@
+## 0.3.0
+
+* Base::Connection methods use #with_connection
+* InsertColumnAttributes uses Batch.insert
+* InsertColumns uses Batch.insert
+* add Quandl::Cassandra::Batch
+* insert_columns_in_batches_with_threads does not care how many rows are sent
+
 ## 0.0.1
 
 * Begin

data/lib/quandl/cassandra.rb

@@ -7,6 +7,7 @@ require "active_support/core_ext/object"
 
 require 'cql'
 require 'scope_composer'
+require 'facter'
 
 require 'quandl/logger'
 require "quandl/data"
@@ -16,6 +17,7 @@ require 'quandl/cassandra/error'
 require 'quandl/cassandra/types'
 require 'quandl/cassandra/base'
 require 'quandl/cassandra/configuration'
+require 'quandl/cassandra/batch'
 
 require 'quandl/cassandra_models/column'
 require 'quandl/cassandra_models/column_attribute'

data/lib/quandl/cassandra/base/connection.rb

@@ -12,17 +12,40 @@ module Quandl::Cassandra::Base::Connection
     end
     
     def prepare(statement)
-      connection.prepare(statement)
+      with_connection do |c|
+        c.prepare(statement)
+      end
     end
     
     def execute_async(statement, query_consistency = nil)
       query_consistency = consistency unless query_consistency.present?
-      connection.async.execute( statement, query_consistency )
+      with_connection do |c|
+        c.async.execute( statement, query_consistency )
+      end
     end
     
-    def execute(statement, query_consistency = nil)
-      query_consistency = consistency unless query_consistency.present?
-      connection.execute( statement, query_consistency )
+    def execute(statement, qconsistency = nil)
+      qconsistency = consistency unless qconsistency.present?
+      with_connection do |c|
+        c.execute( statement, qconsistency )
+      end
+    end
+    
+    def with_connection(&block)
+      begin
+        yield(connection)
+        
+      rescue Cql::Io::ConnectionError => e
+        Quandl::Logger.error(e)
+        reset_connection
+        raise Cql::Io::ConnectionError
+        
+      rescue Cql::NotConnectedError => e
+        Quandl::Logger.error(e)
+        reset_connection
+        raise Cql::NotConnectedError
+        
+      end
     end
     
     def connection
@@ -33,6 +56,12 @@ module Quandl::Cassandra::Base::Connection
       Quandl::Cassandra.configuration.consistency
     end
     
+    def reset_connection
+      connection.close
+      @@connection = establish_connection
+      true
+    end
+    
     def establish_connection
       c = Cql::Client.connect(
         hosts:        Quandl::Cassandra.configuration.hosts, 

data/lib/quandl/cassandra/base/logging.rb

@@ -10,19 +10,7 @@ module Quandl::Cassandra::Base::Logging
       Quandl::Logger.debug(statement)
       super if defined?(super)
     end
-    
-    def execute_async(*args, &block)
-      statement = args.first.to_s
-      statement = "#{statement[0..200]} ... #{statement.length} chars" if statement.length > 200
-      t1 = Time.now
-      begin
-        r = super if defined?(super)
-      ensure
-        Quandl::Logger.debug("(#{t1.elapsed_ms}) #{statement}")
-      end
-      r
-    end
-    
+
     def execute(*args, &block)
       statement = args.first.to_s
       statement = "#{statement[0..200]} ... #{statement.length} chars" if statement.length > 200

data/lib/quandl/cassandra/batch.rb

@@ -0,0 +1,9 @@
+require 'quandl/cassandra/batch/logging'
+require 'quandl/cassandra/batch/insert'
+
+class Quandl::Cassandra::Batch
+  
+  include Quandl::Cassandra::Batch::Insert
+  include Quandl::Cassandra::Batch::Logging if defined?(QUANDL_LOGGER) && QUANDL_LOGGER == true
+  
+end

data/lib/quandl/cassandra/batch/insert.rb

@@ -0,0 +1,53 @@
+module Quandl::Cassandra::Batch::Insert
+
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    
+    # Quandl::Cassandra::Batch.insert(rows) do |id, type, time, value|
+    #   "INSERT INTO columns (id, type, time, value) VALUES (#{id}, '#{type}', #{time}, #{value})"
+    # end
+    def insert(rows, &block)
+      insert_in_batches(rows, &block)
+    end
+  
+    def batch_size
+      Quandl::Cassandra.configuration.batch_size
+    end
+  
+  
+    protected
+  
+    def insert_in_batches_with_threads(rows, &block)
+      # split rows into groups by rows_per_thread
+      threads = rows.each_slice( rows_per_thread(rows) ).map do |rows_slice|
+        Thread.start{ insert_in_batches(rows_slice, &block) }
+      end
+      threads.each(&:join)
+    end
+
+    def insert_in_batches(rows, &block)
+      futures = []
+      rows.each_slice( batch_size ).each do |rows_slice|
+        statements = rows_slice.collect{|row| block.call( *row ) }
+        futures << execute_async_batch(statements)
+      end
+      futures.collect(&:value)
+    end
+
+    def rows_per_thread(rows)
+      r = rows.count / Facter.processorcount.to_i
+      r = 1 if r <= 0
+      r
+    end
+  
+  
+    private
+
+    def execute_async_batch(statements)
+      batch = %Q{BEGIN UNLOGGED BATCH\n#{statements.join("\n")}\nAPPLY BATCH;}
+      future = Quandl::Cassandra::Base.execute_async( batch )
+    end
+  
+  end
+end

data/lib/quandl/cassandra/batch/logging.rb

@@ -0,0 +1,22 @@
+class Quandl::Cassandra::Batch
+module Logging
+
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+  
+    def insert(rows, &block)
+      # log init
+      statement = block.call(rows[0].collect{'?'})
+      t1 = Time.now
+      # call method
+      r = super if defined?(super)
+      # log write
+      Quandl::Logger.debug("(#{t1.elapsed_ms}) [#{rows.count} rows] BATCH #{statement}")
+      r
+    end
+  
+  end  
+
+end
+end

data/lib/quandl/cassandra/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Cassandra
-  VERSION = '0.2.1'
+  VERSION = '0.3.0'
 end
 end

data/lib/quandl/cassandra_models/column/read/select_columns.rb

@@ -18,39 +18,27 @@ class Quandl::Cassandra::Column::Read::SelectColumns < Quandl::Cassandra::Column
   end
   
   def select_data
+    t1 = Time.now
+    # fire off the queries
     prepared = Quandl::Cassandra::Base.prepare( statement )
     data = {}
+    futures = []
     attributes[:column_ids].each_with_index do | id, index |
       # pluck column type from collapses
       type = attributes[:column_collapses][index].to_s
       # bind and execute query
-      rows = prepared.execute( id, type, :one )
-      # collect result
-      rows.each do |row|
+      futures << prepared.async.execute( id, type, Quandl::Cassandra::Base.consistency )
+    end
+    # collect the results
+    futures.each_with_index do |future, index|
+      # collect result 
+      future.value.each do |row|
         data[row['time']] ||= Array.new( attributes[:column_ids].count )
         data[row['time']][index] ||= row['value']
       end
     end
-    data.collect(&:flatten)
-    # # fire off the queries
-    # futures = []
-    # attrs[:column_ids].each_with_index do | id, index |
-    #   # pluck column type from collapses
-    #   type = attrs[:column_collapses][index].to_s
-    #   # bind and execute query
-    #   futures << Quandl::Cassandra::Base.connection.execute_async( statement.bind( id, type ) )
-    # end
-    # # collect the results
-    # futures.each_with_index do |future, column_index|
-    #   t1 = Time.now
-    #   rows = JavaDriver::ResultSet.new( future.get_uninterruptibly ).to_a
-    #   rows.each do |row|
-    #     data[row[0]] ||= Array.new(attrs[:column_ids].count)
-    #     data[row[0]][column_index] ||= row[1]
-    #   end
-    #   JCQL::CommonLogger.info "#{cql} (#{attrs[:column_ids][column_index]}) (#{t1.elapsed.microseconds}ms)"
-    # end
-    # data
+    Quandl::Logger.debug("(#{t1.elapsed_ms}) #{self.class.name}.select_data")
+    data
   end
   
   def statement

data/lib/quandl/cassandra_models/column/write/insert_column_attributes.rb

@@ -5,18 +5,16 @@ class Quandl::Cassandra::Column::Write::InsertColumnAttributes < Quandl::Cassand
   
   def perform
     return if column_ids.blank?
-    column_ids.each_with_index{|column_id, position| 
-      Quandl::Cassandra::Base.execute( datasets_statement( column_id, position ) ) 
-      Quandl::Cassandra::Base.execute( column_attributes_statement( column_id ) ) 
-    }
-  end
-  
-  def datasets_statement( column_id, position )
-    "INSERT INTO datasets (id, column_id, position) VALUES (#{id}, #{column_id}, #{position})"
-  end
-  
-  def column_attributes_statement( column_id )
-    "INSERT INTO column_attributes ( id, frequency ) VALUES ( #{column_id}, '#{frequency}' )"
+    # format data for batch insertion
+    rows_values = []
+    column_ids.each_with_index do |column_id, position|
+      rows_values << [id, column_id, position, frequency]
+    end
+    # insert data
+    Quandl::Cassandra::Batch.insert(rows_values) do |id, column_id, position, frequency|
+      %Q{INSERT INTO datasets (id, column_id, position) VALUES (#{id}, #{column_id}, #{position})
+      INSERT INTO column_attributes ( id, frequency ) VALUES ( #{column_id}, '#{frequency}' )}
+    end
   end
   
 end

data/lib/quandl/cassandra_models/column/write/insert_columns.rb

@@ -1,30 +1,9 @@
 class Quandl::Cassandra::Column::Write::InsertColumns < Quandl::Cassandra::Column::Write
   
   def perform
-    insert_columns_in_batches
-  end
-  
-  def insert_columns_in_batches
-    threads = statement_values.each_slice( statement_values.size / 8 ).map do |threads_slice|
-      Thread.start do
-        futures = []
-        threads_slice.each_slice( Quandl::Cassandra.configuration.batch_size ).each do |batch_slice|
-          statements = batch_slice.collect{|row| statement( *row ) }
-          futures << execute_async_batch(statements)
-        end
-        futures.collect(&:value)
-      end
+    Quandl::Cassandra::Batch.insert(statement_values) do |id, type, time, value|
+      "INSERT INTO columns (id, type, time, value) VALUES (#{id}, '#{type}', #{time}, #{value})"
     end
-    threads.each(&:join)
-  end
-  
-  def execute_async_batch(statements)
-    batch = %Q{BEGIN UNLOGGED BATCH\n#{statements.join("\n")}\nAPPLY BATCH;}
-    future = Quandl::Cassandra::Base.execute_async( batch )
-  end
-  
-  def statement( id, type, time, value )
-    "INSERT INTO columns (id, type, time, value) VALUES (#{id}, '#{type}', #{time}, #{value})"
   end
   
 end

data/lib/quandl/cassandra_models/dataset.rb

@@ -20,7 +20,7 @@ class Quandl::Cassandra::Dataset < Quandl::Cassandra::Base
       self.columns[index].assign_attributes(attrs)
     end
   end
-   
+  
   def columns
     @columns ||= column_ids.collect{|cid| Quandl::Cassandra::ColumnAttribute.find(cid) }
   end

data/quandl_cassandra.gemspec

@@ -25,7 +25,8 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "activesupport", ">= 3.0.0"
   s.add_runtime_dependency "activemodel", ">= 3.0.0"
-  
+
+  s.add_runtime_dependency "facter", "~> 1.7.3"
   s.add_runtime_dependency "scope_composer", "~> 0.4"
   s.add_runtime_dependency "quandl_data", "~> 1.0"
   s.add_runtime_dependency "quandl_logger", "~> 0.1"

data/spec/lib/quandl/cassandra_models/column/write_spec.rb

@@ -12,4 +12,12 @@ describe Quandl::Cassandra::Column::Write do
     data.should eq source_data
   end
   
+  context "given tiny data array" do
+    let(:data){ Quandl::Fabricate::Data.rand( columns: 1, rows: 2, nils: false ) }
+    before(:each){ Quandl::Cassandra::Column.write( id: id, data: data ) }
+    it "should have written the data" do
+      Quandl::Cassandra::Column.read( id: id ).should eq data
+    end
+  end
+  
 end

data/spec/lib/quandl/cassandra_models/dataset_spec.rb

@@ -41,62 +41,66 @@ describe Quandl::Cassandra::Dataset do
     its(:table_name){ should eq 'datasets' }
   end
   
-  describe "#save" do
-    
-    before(:each){ 
-      dataset.data = Quandl::Fabricate::Data.rand(rows: 10, columns: 2, nils: false)
-      dataset.save
-    }
+  context "given data" do
+    before(:each){ dataset.data = Quandl::Fabricate::Data.rand(rows: 10, columns: 2, nils: false) }
+  
+    describe "#save" do
+      before(:each){ dataset.save }
     
-    subject{ dataset }
+      subject{ dataset }
     
-    its(:changes){ should be_blank }
+      its(:changes){ should be_blank }
+      its(:frequency){ should eq 'daily' }
     
-    describe ".find" do
-      subject{ Quandl::Cassandra::Dataset.find(id) }
-      its(:data){ should eq dataset.data.to_table }
-      it "data should count and return data" do
-        subject.data.count.should eq 10
-        subject.data.to_table.should be_a Quandl::Cassandra::Data
-      end
-      it "columns should eq dataset.columns" do
-        subject.columns.collect{|c| c.id.to_s }.should eq dataset.columns.collect{|c| c.id.to_s }
-      end
-      it "column_ids should eq dataset.column_ids" do
-        subject.column_ids.collect(&:to_s).should eq dataset.column_ids.collect(&:to_s)
+      describe ".find" do
+        subject{ Quandl::Cassandra::Dataset.find(id) }
+        its(:data){ should eq dataset.data.to_table }
+        its(:frequency){ should eq 'daily' }
+
+        it "data should count and return data" do
+          subject.data.count.should eq 10
+          subject.data.to_table.should be_a Quandl::Cassandra::Data
+        end
+        it "columns should eq dataset.columns" do
+          subject.columns.collect{|c| c.id.to_s }.should eq dataset.columns.collect{|c| c.id.to_s }
+        end
+        it "column_ids should eq dataset.column_ids" do
+          subject.column_ids.collect(&:to_s).should eq dataset.column_ids.collect(&:to_s)
+        end
       end
-    end
     
-    describe "#reload" do
-      before(:each){ 
-        dataset.data.limit(5).to_a
-        dataset.reload
-      }
+      describe "#reload" do
+        before(:each){ 
+          dataset.data.limit(5).to_a
+          dataset.reload
+        }
       
-      describe "#attributes" do
-        subject{ dataset.attributes }
-        its([:data]){ should eq nil }
-      end
+        describe "#attributes" do
+          subject{ dataset.attributes }
+          its([:data]){ should eq nil }
+        end
       
-    end
+      end
     
-    describe "#data" do
-      subject{ dataset.data }
-      its(:count){ should eq 10 }
-    end
+      describe "#data" do
+        subject{ dataset.data }
+        its(:count){ should eq 10 }
+      end
 
-    describe "#column_ids" do
-      subject{ dataset.column_ids }
-      its(:count){ should eq 2 }
-      its(:first){ should be_a Cql::Uuid }
-    end
+      describe "#column_ids" do
+        subject{ dataset.column_ids }
+        its(:count){ should eq 2 }
+        its(:first){ should be_a Cql::Uuid }
+      end
     
-    describe "#columns" do
-      subject{ dataset.columns }
-      its(:count){ should eq 2 }
-      its(:first){ should be_a Quandl::Cassandra::ColumnAttribute }
-    end
+      describe "#columns" do
+        subject{ dataset.columns }
+        its(:count){ should eq 2 }
+        its(:first){ should be_a Quandl::Cassandra::ColumnAttribute }
+      end
     
+    end
+  
   end
   
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: quandl_cassandra
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-03 00:00:00.000000000 Z
+date: 2013-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -123,6 +123,22 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: facter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.3
 - !ruby/object:Gem::Dependency
   name: scope_composer
   requirement: !ruby/object:Gem::Requirement
@@ -227,6 +243,9 @@ files:
 - lib/quandl/cassandra/base/sanitization.rb
 - lib/quandl/cassandra/base/schema.rb
 - lib/quandl/cassandra/base/scoping.rb
+- lib/quandl/cassandra/batch.rb
+- lib/quandl/cassandra/batch/insert.rb
+- lib/quandl/cassandra/batch/logging.rb
 - lib/quandl/cassandra/configuration.rb
 - lib/quandl/cassandra/error.rb
 - lib/quandl/cassandra/types.rb