active_data_frame 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5c1042e5d6a9e65c386a0dc5353e6fcc3e065a84
+  data.tar.gz: 13ee5f0520a97c563dc5bfedb4408010464b2e5f
+SHA512:
+  metadata.gz: 7e9f1118a5c18a0aed0bc933ec2e9bfc7d443412e762da9cee9707bdf4084922e9d2904f3668088ff2d3866bcebf9a90b0b8c9e21c90e023c9fb2ca19d1c57c3
+  data.tar.gz: f957ad5532cfcd4a5d635d278a0a5ee6de2c4b6179e5e82f785f6aa9961ae5ea64846d3c138816c1d2e5b2443dc766bcfe7576e2403c7d21fcf77b8e37315b6a

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at TODO: Write your email address. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in active_data_frame.gemspec
+gemspec

data/README.md

@@ -0,0 +1,36 @@
+# ActiveDataFrame
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/active_data_frame`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'active_data_frame'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install active_data_frame
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/active_data_frame. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/active_data_frame.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'active_data_frame/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "active_data_frame"
+  spec.version       = ActiveDataFrame::VERSION
+  spec.authors       = ["Wouter Coppieters"]
+  spec.email         = ["wc@pico.net.nz"]
+
+  spec.summary       = 'An active data frame helper'
+  spec.description   = 'An active data frame helper'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "pry-byebug", "~> 3.4.0", '>= 3.4.0'
+  spec.add_development_dependency 'pry', '~> 0.10.2', '>= 0.10.0'
+  spec.add_runtime_dependency     'activerecord', '~> 5.0.0'
+  spec.add_runtime_dependency     'rmatrix', '~> 0.1.10', '>=0.1.10'
+end

data/active_data_frame.todo

@@ -0,0 +1,83 @@
+Refactor:
+ ✔ Is Engine neccessary? @done (17-03-31 08:20)
+ ✔ Add Typecode expectations @done (17-04-06 08:16)
+ ☐ Add enum capabilities
+ ☐ Better errors when using bad indices in RMatrix
+ ☐ Better printing in RMatrix
+ ☐ Refactor + Tidy
+ ☐ Tests
+ ☐ Experiment with MonetDB speed
+ ☐ Check support for different numeric/string/bool.etc types
+ ✔ Experiment with single precision @done (17-03-31 08:18)
+ActiveRecordMonetDBAdapter:
+   ☐ Work on support for MonetDB
+
+ActiveDataFrame:
+   ✔ Refactor grouping/summing code @done (17-03-31 08:20)
+   ✔ Allow includes to combine frames @done (17-03-27 10:36)
+   ✔ Performance test on ICP data @done (17-03-27 08:41)
+   ✔ Alternate RDBMS support (SQLLite, MySQL) @done (17-03-27 09:58)
+
+
+ Utilities:
+   ☐ KMeans clustering and DBScan built in to multi-d array
+
+ Later:
+   ☐ Build generic Merge/Cache structure which will either cache infinite columns or rows
+   - class Unit
+       - df_cache :all_loads, ::loads,  direction: :row
+   - end
+
+Ruby dataframe library inspiration:
+   - Integration with Nyaplot
+   - Integration with Statsample
+
+ ✔ Generator creates A migration and data_frame and block classes. Block/DataFrame classes have a type, a period unit and a period length @done (17-01-12 10:29)
+ ✔ Type is: @done (17-01-12 10:29)
+   ✔ Bit @done (17-01-12 10:29)
+   ✔ Short @done (17-01-12 10:29)
+   ✔ Int @done (17-01-12 10:29)
+   ✔ Long @done (17-01-12 10:29)
+   ✔ Float @done (17-01-12 10:29)
+   ✔ Double @done (17-01-12 10:29)
+
+ ✔ Insert useful metadata into block type class. @done (17-01-12 19:43)
+   ✔ Number of columns @done (17-01-12 19:43)
+   ✔ Column getters @done (17-01-12 19:43)
+   ✔ Column setters @done (17-01-12 19:43)
+   ✔ Select SQL @done (17-01-12 19:43)
+
+ ✔ DataBlock and DataDataFrame provides: @done (17-01-12 19:43)
+   ✔ #[] @done (17-01-12 19:43)
+   ✔ #[]= @done (17-01-12 19:43)
+   ✔ #self.matrix(columns:, *time, period_unit: period_length: default derive from first) @done (17-01-12 19:43)
+   ✔ #self.avg @done (17-01-12 19:43)
+   ✔ #self.sum @done (17-01-12 19:43)
+   ✔ #self.count_zero @done (17-01-12 19:43)
+   ✔ #self.where @done (17-01-12 19:43)
+   ✔ #self.max @done (17-01-12 19:43)
+   ✔ #self.min @done (17-01-12 19:43)
+ ✔ Bulk service can bulk insert and update (Implement for PostgreSQL first) @done (17-01-12 19:43)
+ ✔ Time helper service @done (17-01-12 19:43)
+
+ Thoughts:
+   ✔ Can extract non-contiguous columns at once (Optimise) @done (17-01-14 09:40)
+   ✔ Extract doesn't use from, count. Use either: @done (17-01-14 09:40)
+         ✔ [from..to] @done (17-01-14 09:40)
+         ✔ [from1, from2, from3] @done (17-01-14 09:40)
+ ✔ Class can define column mapper array (create reverse object to index hash from this) @done (17-01-14 21:46)
+ ✔ Column mapper function is used in column_mapper for results @done (17-01-14 21:46)
+ ✔ Row mapper will translate ActiveRecord items to indices @done (17-01-14 21:46)
+ ✔ Add column and row maps to results @done (17-01-14 21:46)
+
+✔ Add where queries based on column names. @done (17-01-16 09:21)
+   ✔ E.g @done (17-01-16 09:21)
+      ✔ Iris.where(Iris.columns.sepal_length == 3).or(Iris.columns.petal_length < 3) @done (17-01-16 09:21)
+
+✔ Rename to active data frame @done (17-01-24 18:28)
+✔ Add option to name columns and rows RMatrix (For printing) @done (17-02-27 09:20)
+✔ Finish RMatrix @done (17-03-02 09:01)
+
+RMatrix:
+   ✔ Ensure assignment works @done (17-03-21 09:56)
+   ✘ Raw is simply a copy of self without mappings @cancelled (17-03-21 09:56)

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "active_data_frame"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/active_data_frame.rb

@@ -0,0 +1,5 @@
+require 'active_data_frame/data_frame_proxy'
+require 'active_data_frame/table'
+require 'active_data_frame/row'
+require 'active_data_frame/has_data_frame'
+require 'rmatrix'

data/lib/active_data_frame/data_frame_proxy.rb

@@ -0,0 +1,112 @@
+module ActiveDataFrame
+  class DataFrameProxy
+    attr_accessor :block_type, :data_frame_type, :block_type_name
+    def initialize(block_type, data_frame_type)
+      self.block_type      = block_type
+      self.data_frame_type = data_frame_type
+      self.block_type_name = block_type.table_name.gsub(/_blocks$/,'').gsub(/^blocks_/,'')
+    end
+
+    def [](*ranges)
+      get(extract_ranges(ranges))
+    end
+
+    def []=(from, values)
+      from = column_map[from] if column_map && column_map[from]
+      set(from, M[values, typecode: block_type::TYPECODE].to_a.flatten)
+    end
+
+    def column_map
+      data_frame_type.column_map(self.block_type_name)
+    end
+
+    def column_name_map
+      data_frame_type.column_name_map(self.block_type_name)
+    end
+
+    def reverse_column_map
+      data_frame_type.reverse_column_map(self.block_type_name)
+    end
+
+    def method_missing(name, *args, &block)
+      if column_name_map && column_map[name]
+        self[name]
+      else
+        super
+      end
+    end
+
+    def extract_ranges(ranges)
+      ranges = unmap_ranges(ranges, column_map) if column_map
+      ranges.map do |range|
+        case range
+        when Range then range
+        when Fixnum then range..range
+        else raise "Unexpected index #{range}"
+        end
+      end
+    end
+
+    def range_size
+      0
+    end
+
+    def flatten_ranges(ranges)
+    end
+
+    def unmap_ranges(ranges, map)
+      ranges.map do |range|
+        case range
+        when Range
+          first       = (map[range.first] rescue nil) || range.first
+          ends        = (map[range.end] rescue nil) || range.end
+          range.exclude_end? ? first...ends : first..ends
+        else map[range] || range
+        end
+      end
+    end
+
+    def get_bounds(from, to, index=0)
+      from_block_index  = from / block_type::BLOCK_SIZE
+      from_block_offset = from % block_type::BLOCK_SIZE
+      to_block_index    = to / block_type::BLOCK_SIZE
+      to_block_offset   = to % block_type::BLOCK_SIZE
+      return Struct.new(:from, :to, :length, :index).new(
+        Struct.new(:index, :offset, :position).new(from_block_index, from_block_offset, from),
+        Struct.new(:index, :offset, :position).new(to_block_index,   to_block_offset, to),
+        (to - from) + 1,
+        index
+      )
+    end
+
+    def self.suppress_logs
+      ActiveRecord::Base.logger, old_logger = nil,  ActiveRecord::Base.logger
+      yield.tap do
+        ActiveRecord::Base.logger = old_logger
+      end
+    end
+
+    def iterate_bounds(all_bounds)
+      cursor = 0
+      all_bounds.each do |bounds|
+        index = bounds.from.index
+        while index <= bounds.to.index
+          left  = index == bounds.from.index ? bounds.from.offset : 0
+          right = index == bounds.to.index   ? bounds.to.offset   : block_type::BLOCK_SIZE - 1
+          size  = (right - left)+1
+          yield index, left, right, cursor, size
+          cursor += size
+          index += 1
+        end
+      end
+    end
+
+    def blocks_between(bounds, block_scope: scope)
+      bounds[1..-1].reduce(
+        block_scope.where( block_type.table_name => { period_index: (bounds[0].from.index..bounds[0].to.index)})
+      ) do | or_chain, bound|
+        or_chain.or(block_scope.where( block_type.table_name => { period_index: (bound.from.index..bound.to.index)}))
+      end
+    end
+  end
+end

data/lib/active_data_frame/has_data_frame.rb

@@ -0,0 +1,152 @@
+require 'active_support/concern'
+
+
+module ActiveDataFrame
+  class GroupProxy
+    attr_accessor :groups
+    def initialize(groups)
+      self.groups = groups
+    end
+
+    def min(column_name)
+      aggregate('minimum', column_name)
+    end
+
+    def max(column_name)
+      aggregate('maximum', column_name)
+    end
+
+    def sum(column_name)
+      aggregate('sum', column_name)
+    end
+
+    def average(column_name)
+      aggregate('average', column_name)
+    end
+
+    def count
+      aggregate('count')
+    end
+
+    private
+      def aggregate *agg
+        counts = self.groups.send(*agg)
+        grouped = {}
+        counts.each do |keys, value|
+          keys = Array(keys)
+          child = keys[0..-2].reduce(grouped){|parent, key| parent[key] ||= {}}
+          child[keys[-1]] = value
+        end
+        grouped
+      end
+  end
+
+  def self.HasDataFrame(singular_table_name, table_name, block_type)
+    to_inject = Module.new
+    to_inject.extend ActiveSupport::Concern
+    to_inject.included do
+      define_method(singular_table_name){
+        @data_frame_proxies ||= {}
+        @data_frame_proxies[singular_table_name] ||= Row.new(block_type, self.class, self)
+      }
+
+      define_method(:inspect){
+        inspection = "not initialized"
+        if defined?(@attributes) && @attributes
+           inspection = @attributes.keys.collect { |name|
+             if has_attribute?(name)
+               "#{name}: #{attribute_for_inspect(name)}"
+             end
+           }.compact.join(", ")
+        end
+        "<#{self.class} #{inspection}>"
+      }
+    end
+
+    to_inject.class_methods do
+      define_method(:df_column_names){
+        @@column_names ||= {}
+      }
+
+      define_method(:df_column_maps){
+        @@column_maps ||= {}
+      }
+
+      define_method(:df_reverse_column_maps){
+        @@reverse_column_maps ||= {}
+      }
+
+      define_method(:with_groups){|*groups|
+        GroupProxy.new(group(*groups))
+      }
+
+      define_method(table_name){
+        Table.new(block_type, all)
+      }
+
+      define_method("include_#{table_name}"){|*dimensions, unmap: true|
+        scope = self.all
+        blocks_for_tables = scope.instance_eval{ @blocks_for_tables ||= {} }
+        included_blocks = blocks_for_tables[singular_table_name] ||= {}
+        dimensions.flatten.each do |key|
+          if unmap && column_map(singular_table_name)
+            idx = column_map(singular_table_name)[key]
+          else
+            idx = key
+            key = "t#{key}"
+          end
+          block_index  = idx / block_type::BLOCK_SIZE
+          block_offset = (idx % block_type::BLOCK_SIZE).succ
+          included_blocks[block_index] ||= []
+          included_blocks[block_index] << {name: key, idx: block_offset}
+        end
+        query = "(SELECT * FROM #{self.table_name} " + blocks_for_tables.reduce('') do |aggregate, (table_name, included_blocks)|
+        aggregate +
+          included_blocks.reduce('') do |aggregate, (block_idx, blocks)|
+            blocks_table_name = "#{table_name}_blocks"
+            aggregate + " LEFT JOIN(SELECT #{blocks_table_name}.data_frame_type, #{blocks_table_name}.data_frame_id, " + blocks.map{|block| "#{blocks_table_name}.t#{block[:idx]} as \"#{block[:name]}\""}.join(', ') + " FROM #{table_name}_blocks "+
+            " WHERE #{blocks_table_name}.period_index = #{block_idx}"+") b#{table_name}#{block_idx} ON b#{table_name}#{block_idx}.data_frame_type = '#{self.name}' AND b#{table_name}#{block_idx}.data_frame_id = #{self.table_name}.id"
+          end
+        end + ") as #{self.table_name}"
+        scope.from(query)
+      }
+
+      define_method("#{singular_table_name}_column_names") do |names|
+        df_column_names[singular_table_name] ||= {}
+        df_column_maps[singular_table_name] ||= {}
+        df_column_names[singular_table_name][self] = names
+        df_column_maps[singular_table_name][self] = names.map.with_index.to_h
+      end
+
+      define_method("#{singular_table_name}_column_map") do |column_map|
+        df_column_names[singular_table_name] = nil
+        df_column_maps[singular_table_name] ||= {}
+        df_column_maps[singular_table_name][self] = column_map
+      end
+
+      define_method("#{singular_table_name}_reverse_column_map"){|reverse_column_map|
+        df_reverse_column_maps[singular_table_name] ||= {}
+        df_reverse_column_maps[singular_table_name][self] = reverse_column_map
+      }
+
+      define_method(:include_data_blocks){|table_name, *args|
+        send("include_#{table_name}", *args)
+      }
+
+      define_method(:column_map){|table_name|
+        df_column_maps[table_name][self] if defined? df_column_maps[table_name] rescue nil
+      }
+
+      define_method(:column_name_map){|table_name|
+        df_column_names[table_name][self] if defined? df_column_names[table_name]
+      }
+
+      define_method(:reverse_column_map){|table_name|
+        df_reverse_column_maps[table_name] ||= {}
+        df_reverse_column_maps[table_name][self] ||= column_map(table_name).invert if column_map(table_name)
+      }
+    end
+
+    return to_inject
+  end
+end

data/lib/active_data_frame/row.rb

@@ -0,0 +1,134 @@
+module ActiveDataFrame
+  class Row < DataFrameProxy
+
+    attr_accessor :instance
+
+    def initialize(block_type, data_frame_type, instance)
+      super(block_type, data_frame_type)
+      self.instance = instance
+    end
+
+    def inspect
+      "#{data_frame_type.name} Row(#{instance.id})"
+    end
+
+    def set(from, values)
+      to     = (from + values.length) - 1
+      bounds = get_bounds(from, to)
+
+      self.class.suppress_logs do
+        new_blocks = Hash.new do |h, k|
+          h[k] = [[0] * block_type::BLOCK_SIZE]
+        end
+
+        existing = blocks_between([bounds]).pluck(:id, :period_index, *block_type::COLUMNS).map do |id, period_index, *block_values|
+          [period_index, [block_values, id]]
+        end.to_h
+
+        iterate_bounds([bounds]) do |index, left, right, cursor, size|
+          chunk = values[cursor...cursor + size]
+          block = existing[index] || new_blocks[index]
+          block.first[left..right] = chunk.to_a
+        end
+
+        bulk_update(existing) unless existing.size.zero?
+        bulk_insert(new_blocks) unless new_blocks.size.zero?
+        values
+      end
+    end
+
+    def get(ranges)
+      all_bounds = ranges.map.with_index do |range, index|
+        get_bounds(range.first, range.exclude_end? ? range.end - 1 : range.end, index)
+      end
+
+      existing = blocks_between(all_bounds).pluck(:period_index, *block_type::COLUMNS).map{|pi, *values| [pi, values]}.to_h
+      result   = M.blank(typecode: block_type::TYPECODE, columns: all_bounds.map(&:length).sum)
+
+      iterate_bounds(all_bounds) do |index, left, right, cursor, size|
+        if block = existing[index]
+          chunk = block[left..right]
+          result.narray[cursor...cursor + size] = chunk.length == 1 ? chunk.first : chunk
+        end
+      end
+
+      if column_map && !column_map.default_proc
+        total = 0
+        range_sizes = ranges.map do |range, memo|
+          last_total = total
+          total += range.size
+          [range.first, range.size, last_total]
+        end
+        index_of = ->(column){
+          selected = range_sizes.find{|start, size, total| start <= column && start + size >= column}
+          if selected
+            start, size, total = selected
+            (column - start) + total
+          else
+            nil
+          end
+        }
+        result.column_map = column_map.map do |name, column|
+          [name, index_of[column_map[name]]]
+        end.to_h
+      end
+      result
+    end
+
+    private
+      ##
+      # Update block data for all blocks in a single call
+      ##
+      def bulk_update(existing)
+        case ActiveRecord::Base.connection_config[:adapter]
+        when 'postgresql'
+          # Fast bulk update
+          updates = ''
+          existing.each do |period_index, (values, id)|
+            updates <<  "(#{id}, #{values.map{|v| v.inspect.gsub('"',"'") }.join(',')}),"
+          end
+          perform_update(updates)
+        else
+          ids = existing.map {|_, (_, id)| id}
+          updates = block_type::COLUMNS.map.with_index do |column, column_idx|
+            [column, "CASE period_index\n#{existing.map{|period_index, (values, id)| "WHEN #{period_index} then #{values[column_idx]}"}.join("\n")} \nEND\n"]
+          end.to_h
+          update_statement = updates.map{|cl, up| "#{cl} = #{up}" }.join(', ')
+          block_type.connection.execute("UPDATE #{block_type.table_name} SET #{update_statement} WHERE #{block_type.table_name}.id IN (#{ids.join(',')});")
+        end
+      end
+
+      ##
+      # Insert block data for all blocks in a single call
+      ##
+      def bulk_insert(new_blocks)
+        inserts = ''
+        new_blocks.each do |period_index, (values)|
+          inserts << \
+          case ActiveRecord::Base.connection_config[:adapter]
+          when 'postgresql', 'mysql2' then "(#{values.map{|v| v.inspect.gsub('"',"'") }.join(',')}, #{instance.id}, #{period_index}, '#{data_frame_type.name}', now(), now()),"
+          else "(#{values.map{|v| v.inspect.gsub('"',"'") }.join(',')}, #{instance.id}, #{period_index}, '#{data_frame_type.name}', datetime(), datetime()),"
+          end
+        end
+        perform_insert(inserts)
+      end
+
+      def perform_update(updates)
+        block_type.transaction do
+          block_type.connection.execute(
+            "UPDATE #{block_type.table_name} SET #{block_type::COLUMNS.map{|col| "#{col} = t.#{col}" }.join(", ")} FROM(VALUES #{updates[0..-2]}) as t(id, #{block_type::COLUMNS.join(',')}) WHERE #{block_type.table_name}.id = t.id"
+          )
+        end
+        true
+      end
+
+      def perform_insert(inserts)
+        sql = "INSERT INTO #{block_type.table_name} (#{block_type::COLUMNS.join(',')}, data_frame_id, period_index, data_frame_type, created_at, updated_at) VALUES #{inserts[0..-2]}"
+        block_type.connection.execute sql
+      end
+
+      def scope
+        @scope ||= block_type.where(data_frame_type: data_frame_type.name, data_frame_id: instance.id)
+      end
+  end
+end

data/lib/active_data_frame/table.rb

@@ -0,0 +1,253 @@
+module ActiveDataFrame
+  class Table < DataFrameProxy
+
+    def set(from, values)
+      data_frame_type.find_each do |instance|
+        Row.new(self.block_type, self.data_frame_type, instance).set(from, values)
+      end
+    end
+
+    def inspect
+      "#{data_frame_type.name} Table"
+    end
+
+    def build_case_map(all_bounds)
+      map = block_type::COLUMNS.map{|col| [col, []]}.to_h
+
+      all_bounds.each do |bound|
+        case bound.from.index
+        when bound.to.index
+          (bound.from.offset+1..bound.to.offset+1).each do |col_idx|
+            map["t#{col_idx}"] << (bound.from.index..bound.from.index)
+          end
+        else
+          (bound.from.offset+1..block_type::COLUMNS.size).each do |col_idx|
+            map["t#{col_idx}"] << (bound.from.index..bound.from.index)
+          end
+          (1..block_type::COLUMNS.size).each do |col_idx|
+            map["t#{col_idx}"] << (bound.from.index.succ..bound.to.index-1)
+          end if bound.from.index.succ != bound.to.index
+          (1..bound.to.offset+1).each do |col_idx|
+            map["t#{col_idx}"] << (bound.to.index..bound.to.index)
+          end
+        end
+      end
+      map
+    end
+
+    def column_cases(cases, agg=nil)
+      block_type::COLUMNS.map do |col|
+        col_cases = cases[col].sort_by(&:begin).reduce([]) do |agg, col_case|
+          if agg.empty?
+            agg << col_case
+            agg
+          else
+            if agg[-1].end.succ == col_case.begin
+              agg[-1] = (agg[-1].begin..col_case.end)
+            else
+              agg << col_case
+            end
+            agg
+          end
+        end
+
+        if agg
+          case col_cases.length
+          when 0 then "NULL as #{col}"
+          else
+            case_str = col_cases.map do |match|
+              case
+              when match.begin == match.end then "period_index = #{match.begin}"
+              else "period_index BETWEEN #{match.begin} AND #{match.end}"
+              end
+            end.join(" OR ")
+            "CASE WHEN #{case_str} THEN #{agg}(#{col}) ELSE NULL END"
+          end
+        else
+          case col_cases.length
+          when 0 then "NULL as #{col}"
+          else
+            case_str = col_cases.map do |match|
+              case
+              when match.begin == match.end then "period_index = #{match.begin}"
+              else "period_index BETWEEN #{match.begin} AND #{match.end}"
+              end
+            end.join(" OR ")
+            "CASE WHEN #{case_str} THEN #{col} ELSE NULL END"
+          end
+        end
+      end
+    end
+
+    def get(ranges)
+      ranges = extract_ranges(ranges)
+      all_bounds = ranges.map.with_index do |range, index|
+        get_bounds(range.first, range.exclude_end? ? range.end - 1 : range.end, index)
+      end
+
+      case_map  = build_case_map(all_bounds)
+
+      existing_blocks = Hash.new{|h, index| h[index] = {}}
+
+      index_map = {}
+      res = ActiveRecord::Base.transaction do
+        ids = data_frame_type.pluck(:id)
+
+        as_sql = blocks_between(
+          all_bounds,
+          block_scope: data_frame_type.unscoped
+                                    .joins("LEFT JOIN #{block_type.table_name} ON #{data_frame_type.table_name}.id = #{block_type.table_name}.data_frame_id")
+                                    .joins("RIGHT JOIN (#{data_frame_type.select(:id).to_sql}) as ref ON ref.id = #{block_type.table_name}.data_frame_id")
+
+        ).where(
+          block_type.table_name => {data_frame_type: data_frame_type.name }
+        ).select(:period_index, :data_frame_id, *column_cases(case_map)).to_sql
+
+        index_map = ids.each_with_index.to_h
+        ActiveRecord::Base.connection.execute(as_sql)
+      end
+
+      res.each_row do |pi, data_frame_id, *values|
+        existing_blocks[pi][data_frame_id] = values
+      end
+
+      result = M.blank(typecode: block_type::TYPECODE, columns: all_bounds.map(&:length).sum, rows: index_map.size)
+      iterate_bounds(all_bounds) do |index, left, right, cursor, size|
+        if blocks = existing_blocks[index]
+          blocks.each do |data_frame_id, block|
+            row = index_map[data_frame_id]
+            next unless row
+            chunk = block[left..right]
+            result.narray[cursor...cursor + size, row] = chunk
+          end
+        end
+      end
+      if column_map && !column_map.default_proc
+        total = 0
+        range_sizes = ranges.map do |range, memo|
+          last_total = total
+          total += range.size
+          [range.first, range.size, last_total]
+        end
+        index_of = ->(column){
+          selected = range_sizes.find{|start, size, total| start <= column && start + size >= column}
+          if selected
+            start, size, total = selected
+            (column - start) + total
+          else
+            nil
+          end
+        }
+        result.column_map = column_map.map do |name, column|
+          [name, index_of[column_map[name]]]
+        end.to_h
+      end
+      result.row_map = Hash.new do |h ,k|
+        h[k] = begin
+          case k
+          when ActiveRecord::Base then index_map[k.id]
+          when ActiveRecord::Relation then k.pluck(:id).map{|i| index_map[i] }
+          when ->(list){ list.kind_of?(Array) && list.all?{|v| v.kind_of?(ActiveRecord::Base)}} then k.map{|i| index_map[i.id] }
+          when Numeric then index_map[k]
+          end
+        end
+      end
+      result
+    end
+
+    def idx_where_sum_gte(*ranges, max)
+      select_agg_indices(extract_ranges(ranges), 'SUM', ->(x, y){ x <= y } , 'SUM(%) > :max', max: max)
+    end
+
+    def idx_where_sum_lte(*ranges, min)
+      select_agg_indices(extract_ranges(ranges), 'SUM', ->(x, y){ x >= y } , 'SUM(%) < :min', min: min)
+    end
+
+    def AggregateProxy(agg)
+      proxy = Object.new
+      aggregate, extract_ranges = method(:aggregate), method(:extract_ranges)
+      proxy.define_singleton_method(:[]) do |*ranges|
+        aggregate[extract_ranges[ranges], agg]
+      end
+      proxy
+    end
+
+    def avg
+      @avg ||= AggregateProxy('AVG')
+    end
+
+    def sum
+      @sum ||= AggregateProxy('SUM')
+    end
+
+    def max
+      @max ||= AggregateProxy('MAX')
+    end
+
+    def min
+      @min ||= AggregateProxy('MIN')
+    end
+
+    private
+
+      def scope
+        @scope ||= block_type.where(data_frame_type: data_frame_type.name, data_frame_id: data_frame_type.select(:id))
+      end
+
+      def select_agg_indices(ranges, agg, filter, condition, **args)
+        all_bounds = ranges.map.with_index do |range, index|
+          get_bounds(range.first, range.exclude_end? ? range.end - 1 : range.end, index)
+        end
+        existing = blocks_between(all_bounds)
+          .group(:period_index)
+          .having(
+            block_type::COLUMNS.map do |cl|
+              condition.gsub('%', cl)
+            end.join(" OR "),
+            **args
+          )
+          .pluck(
+            :period_index,
+            *block_type::COLUMNS.map do |cl|
+              "#{agg}(#{cl}) as #{cl}"
+            end
+          )
+          .map{|pi, *values| [pi, values]}.to_h
+        indices = existing.flat_map do |period_index, *values|
+          index = block_type::BLOCK_SIZE * period_index - 1
+          M[values, typecode: block_type::TYPECODE].mask{|x|
+            index += 1
+            !all_bounds.any?{|b| (b.from.position..b.to.position).include?(index) } || filter[x, args.values.first ]
+          }.where.to_a.map{|v| block_type::BLOCK_SIZE * period_index + v}.to_a
+        end
+
+        if column_map
+          indices.map{|i| reverse_column_map[i.to_i] || i.to_i }
+        else
+          indices
+        end
+      end
+
+      def aggregate(ranges, agg)
+        all_bounds = ranges.map.with_index do |range, index|
+          get_bounds(range.first, range.exclude_end? ? range.end - 1 : range.end, index)
+        end
+
+        case_map  = build_case_map(all_bounds)
+        existing  = blocks_between(all_bounds)
+                    .group(:period_index)
+                    .pluck(:period_index, *column_cases(case_map, agg))
+                    .map{|pi, *values| [pi, values]}.to_h
+        result = M.blank(columns: all_bounds.map(&:length).sum, typecode: block_type::TYPECODE)
+
+        iterate_bounds(all_bounds) do |index, left, right, cursor, size|
+          if block = existing[index]
+            chunk = block[left..right]
+            result.narray[cursor...cursor + size] = chunk.length == 1 ? chunk.first : chunk
+          end
+        end
+        result.column_map = column_map if column_map
+        result
+      end
+  end
+end

data/lib/active_data_frame/version.rb

@@ -0,0 +1,3 @@
+module ActiveDataFrame
+  VERSION = "0.1.1"
+end

data/lib/generators/active_data_frame/install_generator.rb

@@ -0,0 +1,101 @@
+require 'rails/generators/active_record'
+
+module ActiveDataFrame
+  class InstallGenerator < ActiveRecord::Generators::Base
+    desc "Generates a new data_frame type"
+
+    STREAM_TYPES = %w(bit byte int long float double)
+    # Commandline options can be defined here using Thor-like options:
+    argument :type,    :type => :string, :default => 'float', :desc => "DataFrame type. One of(#{STREAM_TYPES*" ,"})"
+    argument :columns, :type => :numeric, :default => 512, :desc => "Number of columns"
+    argument :inject,     type: :array, default: []
+
+    def self.source_root
+      @source_root ||= File.join(File.dirname(__FILE__), 'templates')
+    end
+
+    def generate_model
+      invoke "active_record:model", ["blocks/#{singular_block_table_name}"], migration: false
+    end
+
+    def block_type
+      "#{singular_table_name}_block".camelize
+    end
+
+    def block_table_name
+      "#{singular_table_name}_blocks"
+    end
+
+    def singular_block_table_name
+      "#{singular_table_name}_block"
+    end
+
+    def concern_name
+      "Has#{singular_table_name.camelize}"
+    end
+
+    def concern_file_name
+      "has_#{singular_table_name}"
+    end
+
+    def inject_concern_content
+      inject.each do |inject_into|
+        content = "  include #{concern_name}\n"
+        class_name = inject_into.camelize
+        inject_into_class(self.class.path_for_model(inject_into), class_name, content) if self.class.model_exists?(inject_into, destination_root)
+      end
+    end
+
+    def inject_data_frame_helpers
+      content = \
+<<RUBY
+  BLOCK_SIZE = #{columns}
+  COLUMNS = %w(#{columns.times.map{|i| "t#{i+1}" }.join(" ")})
+  TYPECODE = M::Typecode::FLOAT
+  self.table_name = '#{block_table_name}'
+RUBY
+      class_name = "Blocks::#{singular_block_table_name.camelize}"
+      inject_into_class(self.class.path_for_model(singular_block_table_name), class_name, content) if self.class.model_exists?(singular_block_table_name, destination_root)
+    end
+
+    def copy_concern
+      template "has_concern.rb", "app/models/concerns/#{concern_file_name}.rb"
+    end
+
+    def self.path_for_model(model)
+      File.join("app", "models", "blocks", "#{model.underscore}.rb")
+    end
+
+    def self.model_exists?(model, destination_root)
+      File.exist?(File.join(destination_root, self.path_for_model(model)))
+    end
+
+    def copy_migration
+      migration_template "migration.rb", "db/migrate/active_data_frame_create_#{table_name}.rb", migration_version: migration_version
+    end
+
+    def migration_data
+<<RUBY
+      t.integer :data_frame_id, index: true
+      t.string  :data_frame_type, index: true
+      t.integer :period_index, index: true
+#{
+    columns.times.map do |i|
+"      t.#{type} :t#{i+1}"
+    end.join("\n")
+    }
+RUBY
+    end
+
+    def migration_version
+      if rails5?
+       "[#{Rails::VERSION::MAJOR}.#{Rails::VERSION::MINOR}]"
+      end
+    end
+
+    def rails5?
+      Rails.version.start_with? '5'
+    end
+
+  end
+end

data/lib/generators/active_data_frame/templates/has_concern.rb

@@ -0,0 +1,6 @@
+require 'active_support/concern'
+
+module <%= concern_name %>
+  extend ActiveSupport::Concern
+  include ActiveDataFrame::HasDataFrame('<%= singular_table_name %>', '<%= table_name %>',Blocks::<%= block_type %>)
+end

data/lib/generators/active_data_frame/templates/migration.rb

@@ -0,0 +1,11 @@
+class ActiveDataFrameCreate<%= table_name.camelize %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :<%= block_table_name %> do |t|
+<%= migration_data -%>
+      t.timestamps null: false
+    end
+
+
+    add_index :<%= block_table_name %>, [:data_frame_type, :data_frame_id , :period_index], :unique => true, name: 'index_<%= block_table_name %>_on_type_id_and_index'
+  end
+end

metadata

@@ -0,0 +1,163 @@
+--- !ruby/object:Gem::Specification
+name: active_data_frame
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Wouter Coppieters
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-08-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: rmatrix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.10
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.10
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.10
+description: An active data frame helper
+email:
+- wc@pico.net.nz
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- README.md
+- Rakefile
+- active_data_frame.gemspec
+- active_data_frame.todo
+- bin/console
+- bin/setup
+- lib/active_data_frame.rb
+- lib/active_data_frame/data_frame_proxy.rb
+- lib/active_data_frame/has_data_frame.rb
+- lib/active_data_frame/row.rb
+- lib/active_data_frame/table.rb
+- lib/active_data_frame/version.rb
+- lib/generators/active_data_frame/install_generator.rb
+- lib/generators/active_data_frame/templates/has_concern.rb
+- lib/generators/active_data_frame/templates/migration.rb
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: An active data frame helper
+test_files: []