quandl_cassandra 0.2.0 → 0.2.1

data/.gitignore

@@ -4,4 +4,5 @@
 quandl_cassandra-*
 .rvmrc
 *.gem
-*.log
+*.log
+log/*

data/lib/quandl/cassandra/configuration.rb

@@ -21,7 +21,7 @@ module Cassandra
     def initialize
       @hosts = ['localhost']
       @consistency = :all
-      @batch_size = 35
+      @batch_size = 35 * 4
     end
     
     def to_h

data/lib/quandl/cassandra/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Cassandra
-  VERSION = '0.2.0'
+  VERSION = '0.2.1'
 end
 end

data/lib/quandl/cassandra_models/column/write.rb

@@ -1,7 +1,7 @@
 class Quandl::Cassandra::Column::Write < Quandl::Strategy::Strategize
   
   # strategy attributes
-  define_attributes :id, :data, :frequency, :column_ids, :frequency_data, :frequency_column_data
+  define_attributes :id, :data, :frequency, :column_ids, :frequency_data, :frequency_column_data, :statement_values
   
   require_relative 'write/insert_columns'
   require_relative 'write/insert_column_attributes'

data/lib/quandl/cassandra_models/column/write/group_data_by_column.rb

@@ -7,36 +7,31 @@ class Quandl::Cassandra::Column::Write::GroupDataByColumn < Quandl::Cassandra::C
   # { source: { UUID: [[1,2], [2,4]], UUID: [[1,3],[2,8]] }}
   
   def perform
-    self.frequency_column_data = {}
-    # for each { frequency: [ [12,3,4], ... ] }
-    frequency_data.each do |frequency, data|
-      # assign grouped data to frequency_column_data
-      self.frequency_column_data[frequency] = group_data_by_column(data)
-    end
+    group_by_statement_values
   end
   
-  def group_data_by_column(data)
-    column_data = {}
+  def group_by_statement_values
     # for each [ [date, val, val], ... ]
-    data.each do |row|
-      # extract date
-      date = row[0]
-      # for each [ val, val, ... ]
-      row[1..-1].each_with_index do |value, index|
-        # ensure array
-        column_data[ column_id(index) ] ||= []
-        # group each each [date, value] by column_id, excluding nil
-        column_data[ column_id(index) ] << [date, value] unless value.blank?
+    self.statement_values = []
+    frequency_data.each do |frequency, rows|
+      frequency = frequency.to_s
+      rows.each do |row|
+        # extract date
+        date = row[0]
+        # for each [ val, val, ... ]
+        row[1..-1].each_with_index do |value, index|
+          # ensure array
+          self.statement_values << [ column_id(index), frequency, date, value] unless value.blank?
+        end
       end
     end
-    column_data
   end
   
   def column_id(index)
     # ensure column_ids is defined
     self.column_ids ||= Quandl::Cassandra::Dataset.find_column_ids_by_id(id)
     # ensure column_ids[index] is present
-    self.column_ids[index] ||= SecureRandom.uuid
+    self.column_ids[index] ||= Cql::Uuid.new(SecureRandom.uuid)
   end
   
 end

data/lib/quandl/cassandra_models/column/write/insert_columns.rb

@@ -1,30 +1,21 @@
 class Quandl::Cassandra::Column::Write::InsertColumns < Quandl::Cassandra::Column::Write
   
   def perform
-    insert_data_in_batches.collect(&:value)
+    insert_columns_in_batches
   end
   
-  def insert_data_in_batches
-    futures = []
-    statements = []
-    frequency_column_data.each do |frequency, column_data|
-      column_data.each do |column_id, rows|
-        rows.each do |time_value|
-          # collect statements
-          statements << statement( column_id, frequency, time_value[0], time_value[1] )
-          # after 30 statements are collected, execute a batch insert
-          if statements.count >= Quandl::Cassandra.configuration.batch_size
-            # collect the futures
-            futures << execute_async_batch(statements)
-            # clear statements
-            statements = []
-          end
+  def insert_columns_in_batches
+    threads = statement_values.each_slice( statement_values.size / 8 ).map do |threads_slice|
+      Thread.start do
+        futures = []
+        threads_slice.each_slice( Quandl::Cassandra.configuration.batch_size ).each do |batch_slice|
+          statements = batch_slice.collect{|row| statement( *row ) }
+          futures << execute_async_batch(statements)
         end
+        futures.collect(&:value)
       end
     end
-    # execute any remaining statements
-    futures << execute_async_batch(statements) if statements.count > 0
-    futures
+    threads.each(&:join)
   end
   
   def execute_async_batch(statements)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: quandl_cassandra
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-02 00:00:00.000000000 Z
+date: 2013-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake