red-arrow-activerecord 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5ac5974438fd59eeab35b60221680f207c80acfc
+  data.tar.gz: 3bac786c5c9058ad3fcac6bbdaff8e6b3f2ff731
+SHA512:
+  metadata.gz: c5c20bb8254c395ea9eafafcf9996e7dac7df8110c49685211fec1edecd82299f2de96c20d67f0f62c339e5e153c255be8e1e09349413891f0fc327422c2bb5d
+  data.tar.gz: e39eb1ea4ec0914dc0360b81c81b0b89ec9c848b2eafd3151574bf442795c24c138387fc91248c3df68ebdff823cd4504bb4165ac98b9ee203dfd2a10701094b

data/README.md

@@ -0,0 +1,33 @@
+# README
+
+## Name
+Red ActiveRecord
+
+## Usage
+
+```rb
+User.all.select(:id, :name).to_arrow
+```
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'red-arrow-activerecord', require: 'arrow-activerecord'
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+```bash
+$ gem install red-arrow-activerecord
+```
+
+## Contributing
+Contribution directions go here.
+
+## License
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,18 @@
+require "rubygems"
+require "bundler/gem_helper"
+
+base_dir = File.join(File.dirname(__FILE__))
+
+helper = Bundler::GemHelper.new(base_dir)
+def helper.version_tag
+  version
+end
+
+helper.install
+
+desc "Run tests"
+task :test do
+  ruby("test/run-test.rb")
+end
+
+task default: :test

data/doc/text/news.md

@@ -0,0 +1,5 @@
+# News
+
+## 0.1.0 - 2018-02-05
+
+Initial release!!!

data/lib/arrow-activerecord/arrowable.rb

@@ -0,0 +1,104 @@
+require 'arrow'
+
+module ArrowActiveRecord
+  module Arrowable
+    def to_arrow(batch_size: 10000)
+      target_column_names = select_values
+      target_column_names = column_names if select_values.empty?
+
+      fields = []
+      data_types = []
+      target_column_names.each do |name|
+        name = name.to_s
+        target_column = columns.find do |column|
+          column.name == name
+        end
+        arrow_data_type = extract_arrow_data_type(target_column)
+        fields << Arrow::Field.new(name, arrow_data_type)
+        data_types << arrow_data_type
+      end
+      schema = Arrow::Schema.new(fields)
+
+      arrow_array_batches = data_types.collect do
+        []
+      end
+      in_batches(of: batch_size).each do |relation|
+        column_values_set = relation.pluck(*target_column_names).transpose
+        data_types.each_with_index do |data_type, i|
+          column_values = column_values_set[i]
+          arrow_array_batches[i] << build_arrow_array(column_values, data_type)
+        end
+      end
+      columns = fields.collect.with_index do |field, i|
+        chunked_array = Arrow::ChunkedArray.new(arrow_array_batches[i])
+        Arrow::Column.new(field, chunked_array)
+      end
+
+      Arrow::Table.new(schema, columns)
+    end
+
+    private
+    def extract_arrow_data_type(column)
+      type = nil
+      if column
+        if column.bigint?
+          type = :bigint
+        else
+          type = column.type
+        end
+      end
+      case type
+      when :bigint
+        Arrow::Int64DataType.new
+      when :binary
+        Arrow::BinaryDataType.new
+      when :boolean
+        Arrow::BooleanDataType.new
+      when :date
+        Arrow::Date32DataType.new
+      when :datetime
+        Arrow::TimestampDataType.new(:nano)
+      # when :decimal
+      when :float
+        Arrow::FloatDataType.new
+      when :integer
+        Arrow::Int32DataType.new
+      # when :json
+      when :string, :text
+        Arrow::StringDataType.new
+      when :time, :timestamp
+        Arrow::TimestampDataType.new(:nano)
+      when :timestamp
+        Arrow::TimestampDataType.new(:nano)
+      else
+        message = "unsupported data type: #{type}: #{column.inspect}"
+        raise NotImplementedError, message
+      end
+    end
+
+    def build_arrow_array(column_values, data_type)
+      case data_type
+      when Arrow::Int64DataType
+        Arrow::Int64Array.new(column_values)
+      when Arrow::BinaryDataType
+        Arrow::BinaryArray.new(column_values)
+      when Arrow::BooleanDataType
+        Arrow::BooleanArray.new(column_values)
+      when Arrow::Date32DataType
+        Arrow::Date32Array.new(column_values)
+      when Arrow::TimestampDataType
+        builder = Arrow::TimestampArrayBuilder.new(data_type)
+        builder.build(column_values)
+      when Arrow::FloatDataType
+        Arrow::FloatArray.new(column_values)
+      when Arrow::Int32DataType
+        Arrow::Int32Array.new(column_values)
+      when Arrow::StringDataType
+        Arrow::StringArray.new(column_values)
+      else
+        message = "unsupported data type: #{data_type.inspect}"
+        raise NotImplementedError, message
+      end
+    end
+  end
+end

data/lib/arrow-activerecord/version.rb

@@ -0,0 +1,3 @@
+module ArrowActiverecord
+  VERSION = '0.1.0'
+end

data/lib/arrow-activerecord.rb

@@ -0,0 +1,4 @@
+require "active_record"
+require "arrow-activerecord/arrowable"
+
+ActiveRecord::Relation.send :include, ArrowActiveRecord::Arrowable

metadata

@@ -0,0 +1,136 @@
+--- !ruby/object:Gem::Specification
+name: red-arrow-activerecord
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- hatappi
+- Kouhei Sutou
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-02-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: red-arrow
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A library that provides conversion method between Apache Arrow and ActiveRecord
+email:
+- hatappi@hatappi.me
+- kou@clear-code.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- doc/text/news.md
+- lib/arrow-activerecord.rb
+- lib/arrow-activerecord/arrowable.rb
+- lib/arrow-activerecord/version.rb
+homepage: https://github.com/red-data-tools/red-arrow-activerecord
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.2
+signing_key: 
+specification_version: 4
+summary: A library that provides conversion method between Apache Arrow and ActiveRecord
+test_files: []