wisconsin-benchmark 0.1.0

data/lib/wisconsin-benchmark/array_generator.rb

@@ -0,0 +1,151 @@
+module WisconsinBenchmark
+  # Array generator
+  class ArrayGenerator
+    def initialize(size)
+      if    size <= 1000      then @generator = 279;   @prime = 1009
+      elsif size <= 10000     then @generator = 2969;  @prime = 10007
+      elsif size <= 100000    then @generator = 21395; @prime = 100003
+      elsif size <= 1000000   then @generator = 2107;  @prime = 1000003
+      elsif size <= 10000000  then @generator = 211;   @prime = 10000019
+      elsif size <= 100000000 then @generator = 21;    @prime = 100000007
+      else
+        raise "too many rows requested #{size}"
+      end
+      @size = size
+    end
+
+    # summary of the object.
+    #
+    # @return [String] return class name, size and value range of each arrays.
+    #
+    def inspect
+      <<~STR
+        <#{self.class} (
+          size=#{@size},
+          unique1=#{inspect_array(@unique1)},
+          unique2=#{inspect_array(@unique2)},
+          stringu1=#{inspect_array(@stringu1)},
+          stringu2=#{inspect_array(@stringu2)},
+          string4=#{inspect_array(@string4)}
+        )>
+      STR
+    end
+
+    # Create a random/unique record array
+    #   0...size in range
+    #
+    # @return [Numo::UInt32] array of attribute :unique1.
+    #
+    def unique1
+      @unique1 ||= begin
+        warn 'Generating unique1'
+
+        seed = @generator
+        ary = @size.times.map do
+          seed = rand(seed)
+          seed - 1
+        end
+        Numo::UInt32.new(@size).store(ary)
+      end
+    end
+
+    # Create a sequential record array as 0...size.
+    #
+    # @return [Numo::UInt32] array of attribute :unique2.
+    #
+    def unique2
+      @unique2 ||= begin
+        warn 'Generating unique2'
+
+        Numo::UInt32.new(@size).seq
+      end
+    end
+
+    # Create a randomly distributed string array, stringu1.
+    #
+    # @return [Array<String>]
+    #   array of generated randomly distributed distinct strings.
+    #   - Each string is 52 bytes.
+    #   - string[0..6] is converted string from numbers in randomly distributed unique1.
+    #     The numbers are converted to n-adic string mapped for 'A-Z'.
+    #     {1 => 'AAAAAAA', 2 => 'AAAAAAB', ... 26 => 'AAAAAAZ', 27 => 'AAAAABA', ...}
+    #   - string[7..] is 'x' * 45
+    # @example
+    #  ['AAAAAFRxxxxx ... ', 'AAAABJVxxxxx ... ', 'AAAABBMxxxxx ... ', ... ]
+    #
+    def stringu1
+      @stringu1 ||= begin
+        warn 'Generating stringu1'
+
+        trailer = 'x' * 45
+        a = unique1.to_a.map do |i|
+          str = i.to_s(26).tr('0-9a-p', 'A-Z')
+          ('A' * (7 - str.size)) << str << trailer
+        end
+        Arrow::StringArray.new(a)
+      end
+    end
+
+    # Create a sequencial string array, stringu2.
+    #
+    # @return [Array<String>]
+    #   sequential array of strings.
+    #   - Each string is 52 bytes.
+    #   - string[0..6] is sequential string consists of 'A'..'Z' started from 'AAAAAAA'.
+    #   - string[7..] is 'x' * 45
+    # @example
+    #  ['AAAAAAAxxxxx ... ', 'AAAAAABxxxxx ... ', 'AAAAAACxxxxx ... ', ... ]
+    #
+    def stringu2
+      @stringu2 ||= begin
+        warn 'Generating stringu2'
+
+        trailer = 'x' * 45
+        a = (('A' * 7)..).take(@size).map { _1 << trailer }
+        Arrow::StringArray.new(a)
+      end
+    end
+
+    # Create a cyclic repeated string array, string4.
+    #
+    # @return [Array<String>]
+    #   cyclic repeating array of strings.
+    #   - Each string is 52 bytes.
+    #   - string[0..3] is one of ['AAAA', 'HHHH', 'OOOO', 'VVVV'].
+    #   - string[7..] is 'x' * 48
+    #   - four strings are repeated.
+    # @example
+    #  ['AAAAxxxxx ... ', 'HHHHxxxxx ... ', 'OOOOxxxxx ... ', 'VVVVxxxxx ...', ... ]
+    #
+    def string4
+      @string4 ||= begin
+        warn 'Generating string4'
+
+        trailer = 'x' * 48
+        array = %w[AAAA HHHH OOOO VVVV].map { _1 << trailer }
+        a = @size.times.map { |i| array[i % 4] }
+        Arrow::StringArray.new(a)
+      end
+    end
+
+    private
+
+    # Get pseudo-random integers by a linear congruential generator.
+    def rand(seed)
+      loop do
+        seed = (@generator * seed) % @prime
+        break if seed <= @size
+      end
+      seed
+    end
+
+    def inspect_array(array)
+      if array.nil?
+        'nil'
+      else
+        s, e = array.minmax
+        "#{s}..#{e}"
+      end
+    end
+  end
+end

data/lib/wisconsin-benchmark/table_generator.rb

@@ -0,0 +1,49 @@
+module WisconsinBenchmark
+  # Arrow::Table generator
+  class TableGenerator
+    # Create Scaled Wisconsin Benchmark dataset object.
+    #
+    # @return size [Integer] number of tuples.
+    #
+    def initialize(size)
+      @size = size
+      @array = WisconsinBenchmark::ArrayGenerator.new(size)
+    end
+
+    attr_reader :table, :size, :array
+
+    # Generate Scaled Wisconsin Benchmark dataset in Arrow::Table.
+    #
+    # @return [Arrow::Table] generated dataset in Arrow::Table.
+    #
+    def generate
+      unique1 = @array.unique1
+      onePercent = unique1 % 100
+
+      @table = Arrow::Table.new(
+        [
+          [:unique1,        unique1],
+          [:unique2,        @array.unique2],
+          [:two,            unique1 % 2],
+          [:four,           unique1 % 4],
+          [:ten,            unique1 % 10],
+          [:twenty,         unique1 % 20],
+          [:onePercent,     onePercent],
+          [:tenPercent,     unique1 % 10],
+          [:twentyPercent,  unique1 % 5],
+          [:fiftyPercent,   unique1 % 2],
+          [:unique3,        unique1],
+          [:evenOnePercent, onePercent * 2],
+          [:oddOnePercent,  (onePercent * 2) + 1],
+          [:stringu1,       @array.stringu1],
+          [:stringu2,       @array.stringu2],
+          [:string4,        @array.string4],
+        ]
+      )
+    end
+
+    def inspect
+      "<#{self.class} (size=#{@size}, table=#{@table ? '#<Arrow::Table>' : 'nil'})>"
+    end
+  end
+end

data/lib/wisconsin-benchmark/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module WisconsinBenchmark
+  VERSION = '0.1.0'
+end

data/lib/wisconsin-benchmark.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'arrow-numo-narray'
+require 'parquet'
+
+require_relative 'wisconsin-benchmark/array_generator'
+require_relative 'wisconsin-benchmark/table_generator'
+require_relative 'wisconsin-benchmark/version'
+
+module WisconsinBenchmark
+  class Error < StandardError; end
+end

data/sig/wisconsin/benchmark.rbs

@@ -0,0 +1,6 @@
+module Wisconsin
+  module Benchmark
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end

data/wisconsin-benchmark.gemspec

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative 'lib/wisconsin-benchmark/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'wisconsin-benchmark'
+  spec.version = WisconsinBenchmark::VERSION
+  spec.authors = ['Hirokazu SUZUKI (heronshoes)']
+  spec.email = ['heronshoes877@gmail.com']
+
+  spec.summary = 'Wisconsin Benchmark dataset generator.'
+  spec.description = 'Scalable Wisconsin Benchmark dataset generator for Arrow/Parquet.'
+  spec.homepage = 'https://github.com/heronshoes/wisconsin-benchmark'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 3.0.0'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/heronshoes/wisconsin-benchmark'
+  spec.metadata['changelog_uri'] = 'https://github.com/heronshoes/wisconsin-benchmark/blob/main/CHANGELOG.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'red-arrow', '~> 10.0.0'
+  spec.add_dependency 'red-arrow-numo-narray'
+  spec.add_dependency 'red-parquet'
+
+  # Development dependency has gone to the Gemfile (rubygems/bundler#7237)
+
+  spec.metadata['rubygems_mfa_required'] = 'true'
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: wisconsin-benchmark
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Hirokazu SUZUKI (heronshoes)
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-12-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: red-arrow
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.0.0
+- !ruby/object:Gem::Dependency
+  name: red-arrow-numo-narray
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: red-parquet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Scalable Wisconsin Benchmark dataset generator for Arrow/Parquet.
+email:
+- heronshoes877@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- datasets/WB_1E2.arrow
+- datasets/WB_1E2.csv
+- datasets/WB_1E2.parquet
+- datasets/WB_1E3.arrow
+- datasets/WB_1E3.csv
+- datasets/WB_1E3.parquet
+- datasets/WB_1E4.arrow
+- datasets/WB_1E4.csv
+- datasets/WB_1E4.parquet
+- lib/wisconsin-benchmark.rb
+- lib/wisconsin-benchmark/array_generator.rb
+- lib/wisconsin-benchmark/table_generator.rb
+- lib/wisconsin-benchmark/version.rb
+- sig/wisconsin/benchmark.rbs
+- wisconsin-benchmark.gemspec
+homepage: https://github.com/heronshoes/wisconsin-benchmark
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/heronshoes/wisconsin-benchmark
+  source_code_uri: https://github.com/heronshoes/wisconsin-benchmark
+  changelog_uri: https://github.com/heronshoes/wisconsin-benchmark/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.1
+signing_key:
+specification_version: 4
+summary: Wisconsin Benchmark dataset generator.
+test_files: []