active_data_frame 0.1.1 → 0.1.2

data/lib/active_data_frame/point.rb

@@ -0,0 +1,4 @@
+module ActiveDataFrame
+  class Point  < Struct.new(:index, :offset, :position)
+  end
+end

data/lib/active_data_frame/row.rb

@@ -3,8 +3,8 @@ module ActiveDataFrame
 
     attr_accessor :instance
 
-    def initialize(block_type, data_frame_type, instance)
-      super(block_type, data_frame_type)
+    def initialize(block_type, data_frame_type, instance, value_map: nil, singular_df_name: '', plural_df_name: '')
+      super(block_type, data_frame_type, value_map: value_map, singular_df_name: singular_df_name, plural_df_name: plural_df_name)
       self.instance = instance
     end
 
@@ -16,25 +16,30 @@ module ActiveDataFrame
       to     = (from + values.length) - 1
       bounds = get_bounds(from, to)
 
-      self.class.suppress_logs do
-        new_blocks = Hash.new do |h, k|
-          h[k] = [[0] * block_type::BLOCK_SIZE]
-        end
+      new_blocks = Hash.new do |h, k|
+        h[k] = [[0] * block_type::BLOCK_SIZE]
+      end
 
-        existing = blocks_between([bounds]).pluck(:id, :period_index, *block_type::COLUMNS).map do |id, period_index, *block_values|
-          [period_index, [block_values, id]]
-        end.to_h
+      deleted_indices = []
 
-        iterate_bounds([bounds]) do |index, left, right, cursor, size|
-          chunk = values[cursor...cursor + size]
+      existing = blocks_between([bounds]).pluck(:data_frame_id, :period_index, *block_type::COLUMNS).map do |id, period_index, *block_values|
+        [period_index, [block_values, id]]
+      end.to_h
+
+      iterate_bounds([bounds]) do |index, left, right, cursor, size|
+        chunk = values[cursor...cursor + size]
+        if size == block_type::BLOCK_SIZE && chunk.all?(&:zero?)
+          deleted_indices << index
+        else
           block = existing[index] || new_blocks[index]
           block.first[left..right] = chunk.to_a
         end
-
-        bulk_update(existing) unless existing.size.zero?
-        bulk_insert(new_blocks) unless new_blocks.size.zero?
-        values
       end
+
+      database.bulk_delete(self.id, deleted_indices) unless deleted_indices.size.zero?
+      database.bulk_update(existing)                 unless existing.size.zero?
+      database.bulk_insert(new_blocks, instance)     unless new_blocks.size.zero?
+      values
     end
 
     def get(ranges)
@@ -60,9 +65,9 @@ module ActiveDataFrame
           [range.first, range.size, last_total]
         end
         index_of = ->(column){
-          selected = range_sizes.find{|start, size, total| start <= column && start + size >= column}
+          selected = range_sizes.find{|start, size| start <= column && start + size >= column}
           if selected
-            start, size, total = selected
+            start, _, total = selected
             (column - start) + total
           else
             nil
@@ -76,57 +81,6 @@ module ActiveDataFrame
     end
 
     private
-      ##
-      # Update block data for all blocks in a single call
-      ##
-      def bulk_update(existing)
-        case ActiveRecord::Base.connection_config[:adapter]
-        when 'postgresql'
-          # Fast bulk update
-          updates = ''
-          existing.each do |period_index, (values, id)|
-            updates <<  "(#{id}, #{values.map{|v| v.inspect.gsub('"',"'") }.join(',')}),"
-          end
-          perform_update(updates)
-        else
-          ids = existing.map {|_, (_, id)| id}
-          updates = block_type::COLUMNS.map.with_index do |column, column_idx|
-            [column, "CASE period_index\n#{existing.map{|period_index, (values, id)| "WHEN #{period_index} then #{values[column_idx]}"}.join("\n")} \nEND\n"]
-          end.to_h
-          update_statement = updates.map{|cl, up| "#{cl} = #{up}" }.join(', ')
-          block_type.connection.execute("UPDATE #{block_type.table_name} SET #{update_statement} WHERE #{block_type.table_name}.id IN (#{ids.join(',')});")
-        end
-      end
-
-      ##
-      # Insert block data for all blocks in a single call
-      ##
-      def bulk_insert(new_blocks)
-        inserts = ''
-        new_blocks.each do |period_index, (values)|
-          inserts << \
-          case ActiveRecord::Base.connection_config[:adapter]
-          when 'postgresql', 'mysql2' then "(#{values.map{|v| v.inspect.gsub('"',"'") }.join(',')}, #{instance.id}, #{period_index}, '#{data_frame_type.name}', now(), now()),"
-          else "(#{values.map{|v| v.inspect.gsub('"',"'") }.join(',')}, #{instance.id}, #{period_index}, '#{data_frame_type.name}', datetime(), datetime()),"
-          end
-        end
-        perform_insert(inserts)
-      end
-
-      def perform_update(updates)
-        block_type.transaction do
-          block_type.connection.execute(
-            "UPDATE #{block_type.table_name} SET #{block_type::COLUMNS.map{|col| "#{col} = t.#{col}" }.join(", ")} FROM(VALUES #{updates[0..-2]}) as t(id, #{block_type::COLUMNS.join(',')}) WHERE #{block_type.table_name}.id = t.id"
-          )
-        end
-        true
-      end
-
-      def perform_insert(inserts)
-        sql = "INSERT INTO #{block_type.table_name} (#{block_type::COLUMNS.join(',')}, data_frame_id, period_index, data_frame_type, created_at, updated_at) VALUES #{inserts[0..-2]}"
-        block_type.connection.execute sql
-      end
-
       def scope
         @scope ||= block_type.where(data_frame_type: data_frame_type.name, data_frame_id: instance.id)
       end

data/lib/active_data_frame/table.rb

@@ -2,8 +2,10 @@ module ActiveDataFrame
   class Table < DataFrameProxy
 
     def set(from, values)
-      data_frame_type.find_each do |instance|
-        Row.new(self.block_type, self.data_frame_type, instance).set(from, values)
+      ActiveDataFrame::Database.batch do
+        data_frame_type.each do |instance|
+          Row.new(self.block_type, self.data_frame_type, instance).set(from, values)
+        end
       end
     end
 
@@ -35,7 +37,7 @@ module ActiveDataFrame
       map
     end
 
-    def column_cases(cases, agg=nil)
+    def column_cases(cases, aggregation_function=nil)
       block_type::COLUMNS.map do |col|
         col_cases = cases[col].sort_by(&:begin).reduce([]) do |agg, col_case|
           if agg.empty?
@@ -51,9 +53,9 @@ module ActiveDataFrame
           end
         end
 
-        if agg
+        if aggregation_function
           case col_cases.length
-          when 0 then "NULL as #{col}"
+          when 0 then "NULL::float as #{col}"
           else
             case_str = col_cases.map do |match|
               case
@@ -61,7 +63,7 @@ module ActiveDataFrame
               else "period_index BETWEEN #{match.begin} AND #{match.end}"
               end
             end.join(" OR ")
-            "CASE WHEN #{case_str} THEN #{agg}(#{col}) ELSE NULL END"
+            "CASE WHEN #{case_str} THEN #{aggregation_function}(#{col}) ELSE NULL END"
           end
         else
           case col_cases.length
@@ -92,7 +94,6 @@ module ActiveDataFrame
       index_map = {}
       res = ActiveRecord::Base.transaction do
         ids = data_frame_type.pluck(:id)
-
         as_sql = blocks_between(
           all_bounds,
           block_scope: data_frame_type.unscoped
@@ -130,9 +131,9 @@ module ActiveDataFrame
           [range.first, range.size, last_total]
         end
         index_of = ->(column){
-          selected = range_sizes.find{|start, size, total| start <= column && start + size >= column}
+          selected = range_sizes.find{|start, size| start <= column && start + size >= column}
           if selected
-            start, size, total = selected
+            start, _, total = selected
             (column - start) + total
           else
             nil
@@ -156,11 +157,11 @@ module ActiveDataFrame
     end
 
     def idx_where_sum_gte(*ranges, max)
-      select_agg_indices(extract_ranges(ranges), 'SUM', ->(x, y){ x <= y } , 'SUM(%) > :max', max: max)
+      select_agg_indices(extract_ranges(ranges), 'SUM', ->(x, y){ x <= y } , 'SUM(%) >= :max', max: max)
     end
 
     def idx_where_sum_lte(*ranges, min)
-      select_agg_indices(extract_ranges(ranges), 'SUM', ->(x, y){ x >= y } , 'SUM(%) < :min', min: min)
+      select_agg_indices(extract_ranges(ranges), 'SUM', ->(x, y){ x >= y } , 'SUM(%) <= :min', min: min)
     end
 
     def AggregateProxy(agg)
@@ -234,9 +235,7 @@ module ActiveDataFrame
         end
 
         case_map  = build_case_map(all_bounds)
-        existing  = blocks_between(all_bounds)
-                    .group(:period_index)
-                    .pluck(:period_index, *column_cases(case_map, agg))
+        existing  = blocks_between(all_bounds).group(:period_index).pluck(:period_index, *column_cases(case_map, agg))
                     .map{|pi, *values| [pi, values]}.to_h
         result = M.blank(columns: all_bounds.map(&:length).sum, typecode: block_type::TYPECODE)
 

data/lib/active_data_frame/version.rb

@@ -1,3 +1,3 @@
 module ActiveDataFrame
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/generators/active_data_frame/install_generator.rb

@@ -4,7 +4,7 @@ module ActiveDataFrame
   class InstallGenerator < ActiveRecord::Generators::Base
     desc "Generates a new data_frame type"
 
-    STREAM_TYPES = %w(bit byte int long float double)
+    STREAM_TYPES = %w(bit byte integer long float double)
     # Commandline options can be defined here using Thor-like options:
     argument :type,    :type => :string, :default => 'float', :desc => "DataFrame type. One of(#{STREAM_TYPES*" ,"})"
     argument :columns, :type => :numeric, :default => 512, :desc => "Number of columns"
@@ -46,12 +46,20 @@ module ActiveDataFrame
       end
     end
 
+    def get_typecode
+      case type
+      when "float", "double" then M::Typecode::FLOAT
+      when "integer", "long" then M::Typecode::INT
+      when "bit", "byte"     then M::Typecode::BYTE
+      end
+    end
+
     def inject_data_frame_helpers
       content = \
 <<RUBY
   BLOCK_SIZE = #{columns}
   COLUMNS = %w(#{columns.times.map{|i| "t#{i+1}" }.join(" ")})
-  TYPECODE = M::Typecode::FLOAT
+  TYPECODE = #{get_typecode}
   self.table_name = '#{block_table_name}'
 RUBY
       class_name = "Blocks::#{singular_block_table_name.camelize}"
@@ -76,9 +84,9 @@ RUBY
 
     def migration_data
 <<RUBY
-      t.integer :data_frame_id, index: true
-      t.string  :data_frame_type, index: true
-      t.integer :period_index, index: true
+      t.integer :data_frame_id
+      t.string  :data_frame_type
+      t.integer :period_index
 #{
     columns.times.map do |i|
 "      t.#{type} :t#{i+1}"

data/lib/generators/active_data_frame/templates/has_concern.rb

@@ -1,6 +1,3 @@
-require 'active_support/concern'
-
 module <%= concern_name %>
-  extend ActiveSupport::Concern
-  include ActiveDataFrame::HasDataFrame('<%= singular_table_name %>', '<%= table_name %>',Blocks::<%= block_type %>)
+  include ActiveDataFrame::HasDataFrame('<%= singular_table_name %>', Blocks::<%= block_type %>)
 end

data/lib/generators/active_data_frame/templates/migration.rb

@@ -2,10 +2,8 @@ class ActiveDataFrameCreate<%= table_name.camelize %> < ActiveRecord::Migration<
   def change
     create_table :<%= block_table_name %> do |t|
 <%= migration_data -%>
-      t.timestamps null: false
     end
 
-
-    add_index :<%= block_table_name %>, [:data_frame_type, :data_frame_id , :period_index], :unique => true, name: 'index_<%= block_table_name %>_on_type_id_and_index'
+    add_index :<%= block_table_name %>, [:data_frame_id , :period_index, :data_frame_type], :unique => true, name: 'index_<%= block_table_name %>_id_index_and_type'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: active_data_frame
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Wouter Coppieters
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-01 00:00:00.000000000 Z
+date: 2018-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -78,20 +78,82 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.11'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: minitest-around
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.4.1
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: rmatrix
   requirement: !ruby/object:Gem::Requirement
@@ -124,13 +186,19 @@ files:
 - Gemfile
 - README.md
 - Rakefile
+- active_data_frame-0.1.1.gem
 - active_data_frame.gemspec
 - active_data_frame.todo
 - bin/console
 - bin/setup
+- examples.rb
 - lib/active_data_frame.rb
+- lib/active_data_frame/bounds.rb
 - lib/active_data_frame/data_frame_proxy.rb
+- lib/active_data_frame/database.rb
+- lib/active_data_frame/group_proxy.rb
 - lib/active_data_frame/has_data_frame.rb
+- lib/active_data_frame/point.rb
 - lib/active_data_frame/row.rb
 - lib/active_data_frame/table.rb
 - lib/active_data_frame/version.rb