iso-offline-sort 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4e6bc96d9982a9a958eddb6f6a9ffa2b669c2adfea523cded397e79402488b87
+  data.tar.gz: 5aec83f3929cf85de14dcb28e904f6e6785b17a6c7f221d780fddc9175acc938
+SHA512:
+  metadata.gz: 34d5e7072aabe27a3325cdab2de78528628d595fc17c19317d0d4df9ac499015ca83734369b7e59ed7035a40dc75bde385340196c703f9715422870867c3526b
+  data.tar.gz: 9ee20fd481891bb48a3265ffa7b27fa83c980517b1d34edc04d6623bf7078efdf56bd636405b4e17b05a4a36d7aae24a07db19dd6f09b0a5f9210f63c85de9d1

data/.circleci/config.yml

@@ -0,0 +1,84 @@
+version: 2.1
+jobs:
+  lint:
+    docker:
+      - image: cimg/ruby:2.6.6
+    working_directory: ~/offline-sort
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-gems-ruby-2.6.6-{{ checksum "offline-sort.gemspec" }}-{{ checksum "Gemfile" }}
+            - v1-gems-ruby-2.6.6-
+      - run:
+          name: Install Gems
+          command: |
+            if ! bundle check --path=vendor/bundle; then
+              bundle install --path=vendor/bundle --jobs=4 --retry=3
+              bundle clean
+            fi
+      - save_cache:
+          key: v1-gems-ruby-2.6.6-{{ checksum "offline-sort.gemspec" }}-{{ checksum "Gemfile" }}
+          paths:
+            - "vendor/bundle"
+            - "gemfiles/vendor/bundle"
+      - run:
+          name: Run Rubocop
+          command: bundle exec rubocop
+  test:
+    parameters:
+      gemfile:
+        type: string
+      ruby_version:
+        type: string
+    docker:
+      - image: cimg/ruby:<< parameters.ruby_version >>
+    environment:
+      CIRCLE_TEST_REPORTS: "test-results"
+      BUNDLE_GEMFILE: << parameters.gemfile >>
+    working_directory: ~/offline-sort
+    steps:
+      - checkout
+      - unless:
+          condition:
+            equal: ["gemfiles/rails_edge.gemfile", << parameters.gemfile >>]
+          steps:
+          - restore_cache:
+              keys:
+                - v1-gems-ruby-<< parameters.ruby_version >>-{{ checksum "offline-sort.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
+                - v1-gems-ruby-<< parameters.ruby_version >>-
+      - run:
+          name: Install Gems
+          command: |
+            if ! bundle check --path=vendor/bundle; then
+              bundle install --path=vendor/bundle --jobs=4 --retry=3
+              bundle clean
+            fi
+      - unless:
+          condition:
+            equal: [ "gemfiles/rails_edge.gemfile", << parameters.gemfile >> ]
+          steps:
+          - save_cache:
+              key: v1-gems-ruby-<< parameters.ruby_version >>-{{ checksum "offline-sort.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
+              paths:
+                - "vendor/bundle"
+                - "gemfiles/vendor/bundle"
+      - run:
+          name: Run Tests
+          command: |
+            bundle exec rspec --format RspecJunitFormatter --out $CIRCLE_TEST_REPORTS/rspec/junit.xml --format progress spec
+      - store_test_results:
+          path: "test-results"
+workflows:
+  build:
+    jobs:
+      - lint
+      - test:
+          matrix:
+            parameters:
+              gemfile:
+              - "Gemfile"
+              ruby_version:
+              - "2.6.8"
+              - "2.7.4"
+              - "3.0.2"

data/.github/CODEOWNERS

@@ -0,0 +1 @@
+* @salsify/pim-core-backend

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+.idea/
+*.iml
+Gemfile.lock

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,15 @@
+inherit_gem:
+  salsify_rubocop: conf/rubocop.yml
+
+AllCops:
+  TargetRubyVersion: 2.6
+  Exclude:
+    - 'vendor/**/*'
+    - 'gemfiles/**/*'
+
+# Offense count: 9
+# Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames.
+# AllowedNames: at, by, db, id, in, io, ip, of, on, os, pp, to
+Naming/MethodParameterName:
+  Exclude:
+    - 'lib/offline_sort/fixed_size_min_heap.rb'

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# Changelog
+
+### 0.2.0
+- Add testing for ruby: 2.6, 2.7 & 3.0
+- Add ruby 3 support
+- Require ruby >= 2.6
+- Add rubocop
+
+### 0.1.1
+- Initial implementation

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in offline-sort.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Matthew Cross
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,63 @@
+# offline-sort
+
+Sort arbitrarily large collections of data with limited memory usage. Given an enumerable and a `sort_by` proc, this gem will break the input data into sorted chunks, persist the chunks, and return an `Enumerator`. Data read from this enumerator will be in its final sorted order.
+
+The size of the chunks and the strategy for serializing and deserializing the data are configurable. The gem comes with builtin strategies for `Marshal`, `MessagePack` and `YAML`.
+
+The development of this gem is documented in this [post](http://blog.salsify.com/engineering/ruby-scalable-offline-sort) from the Salsify Engineering Blog.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'offline-sort'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install offline-sort
+
+## Usage
+```ruby
+  arrays = [ [4,5,6], [7,8,9], [1,2,3] ]
+  
+  sorter = OfflineSort.sort_by(arrays, chunk_size: 1) do |array|
+    array.first
+  end
+  
+  # Create a sorted enumerator
+  sorter.sort do |sorted|
+    # Stream results in sorted order
+    sorted.each do |entry|
+      # e.g. write to a file
+    end
+  end
+```
+The example above will create 3 files with 1 array each, then output them in sorted order. You should try different values of `chunk_size` to find the best speed/memory combination for your use case. In general larger chunk sizes will use more memory but run faster.
+
+Sorting is not limited to arrays. You can use anything that can be expressed in a `Enumerable#sort_by` block.
+
+## Using MessagePack
+
+Message pack serialization is faster than the default Ruby `Marshal` strategy. To enable message pack serialization follow these steps.
+
+`gem install msgpack`
+
+`require 'msgpack'`
+
+Pass OfflineSort::Chunk::InputOutput::MessagePack to chunk_input_output_class.
+
+Limitations
+
+The MessagePack serialize/deserialize process stringifies hash keys so it is important to write your sort_by in terms of string keys.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'

data/lib/offline_sort/chunk/input_output/base.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module OfflineSort
+  module Chunk
+    module InputOutput
+
+      class Base
+        MethodNotImplementedError = Class.new(StandardError)
+
+        attr_reader :io
+
+        def initialize(io)
+          @io = io
+        end
+
+        def read_entry
+          raise MethodNotImplementedError.new("#{__method__} must be overridden by #{self.class}")
+        end
+
+        def write_entry(_entry)
+          raise MethodNotImplementedError.new("#{__method__} must be overridden by #{self.class}")
+        end
+
+        def write_entries(entries)
+          entries.each { |entry| write_entry(entry) }
+        end
+
+        def flush
+          io.flush
+        end
+
+        def rewind
+          flush
+          io.rewind
+        end
+
+        def close
+          io.close
+        end
+
+        def each
+          Enumerator.new do |yielder|
+            loop do
+              yielder.yield(read_entry)
+            rescue EOFError
+              break
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/offline_sort/chunk/input_output/marshal.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module OfflineSort
+  module Chunk
+    module InputOutput
+      class Marshal < OfflineSort::Chunk::InputOutput::Base
+        def read_entry
+          ::Marshal.load(io) # rubocop:disable Security/MarshalLoad, this is loading from a trusted source
+        end
+
+        def write_entry(entry)
+          io.write(::Marshal.dump(entry))
+        end
+      end
+    end
+  end
+end

data/lib/offline_sort/chunk/input_output/message_pack.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'msgpack'
+require 'offline_sort/chunk/input_output/base'
+
+module OfflineSort
+  module Chunk
+    module InputOutput
+      class MessagePack < OfflineSort::Chunk::InputOutput::Base
+        attr_reader :packer, :unpacker
+
+        def initialize(io)
+          super
+          @packer = ::MessagePack::Packer.new(io)
+          @unpacker = ::MessagePack::Unpacker.new(io)
+        end
+
+        def read_entry
+          unpacker.read
+        end
+
+        def write_entry(entry)
+          packer.write(entry)
+        end
+
+        def flush
+          packer.flush
+          super
+        end
+      end
+    end
+  end
+end

data/lib/offline_sort/chunk/input_output/yaml.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module OfflineSort
+  module Chunk
+    module InputOutput
+      class Yaml < OfflineSort::Chunk::InputOutput::Base
+        # The yaml parser does not expose a document enumerator that we can call next on without loading the entire file
+        def read_entry
+          YAML.load(next_document) # rubocop:disable Security/YAMLLoad, this is loading from a trusted source
+        end
+
+        def write_entry(entry)
+          io.write(YAML.dump(entry))
+        end
+
+        private
+
+        def next_document
+          sio = StringIO.new
+          document_count = 0
+          line = nil
+
+          loop do
+            line = io.gets
+
+            document_count += 1 if line && line.start_with?('---')
+
+            sio.write(line)
+            break if line.nil? || document_count > 1
+          end
+
+          # reset the io to the beginning of the document
+          io.seek(io.pos - line.length, IO::SEEK_SET) if document_count > 1
+
+          raise EOFError unless sio.size > 0 # rubocop:disable Style/ZeroLengthPredicate
+
+          sio.string
+        end
+      end
+    end
+  end
+end

data/lib/offline_sort/chunk.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require 'offline_sort/chunk/input_output/base'
+require 'offline_sort/chunk/input_output/marshal'
+require 'offline_sort/chunk/input_output/message_pack' if defined?(MessagePack)
+require 'offline_sort/chunk/input_output/yaml'

data/lib/offline_sort/fixed_size_min_heap.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module OfflineSort
+  class FixedSizeMinHeap
+    attr_accessor :array
+    attr_reader :sort_by, :size_limit, :heap_end
+
+    def initialize(array, &sort_by)
+      @array = array
+      @sort_by = sort_by || Proc.new { |item| item }
+      @size_limit = array.size
+      @heap_end = array.size - 1
+      ((array.size * 0.5) - 1).to_i.downto(0) { |i| heapify(i) }
+    end
+
+    def push(item)
+      grow_heap
+      array[heap_end] = item
+      sift_up(heap_end)
+    end
+
+    def pop
+      item = array[0]
+      array[0] = array[heap_end]
+      heapify(0)
+      shrink_heap unless item.nil?
+      item
+    end
+
+    private
+
+    def shrink_heap
+      array[heap_end] = nil
+      @heap_end -= 1
+    end
+
+    def grow_heap
+      raise "Heap Size (#{size_limit}) Exceeded" if heap_end == (size_limit - 1)
+
+      @heap_end += 1
+    end
+
+    # Compare elements at the supplied indices
+    def compare(i, j)
+      (sort_by.call(array[i]) <=> sort_by.call(array[j])) == -1
+    end
+
+    # Swap elements in the array
+    def swap(i, j)
+      temp = array[i]
+      array[i] = array[j]
+      array[j] = temp
+    end
+
+    # Get the parent of the node i > 0.
+    def parent(i)
+      (i - 1) / 2
+    end
+
+    # Get the node left of node i >= 0
+    def left(i)
+      (2 * i) + 1
+    end
+
+    # Get the node right of node i >= 0
+    def right(i)
+      (2 * i) + 2
+    end
+
+    # Keeps an heap sorted with the smallest (largest) element on top
+    def heapify(i)
+      l = left(i)
+      top = (l <= heap_end) && compare(l, i) ? l : i
+
+      r = right(i)
+      top = (r <= heap_end) && compare(r, top) ? r : top
+
+      if top != i
+        swap(i, top)
+        heapify(top)
+      end
+    end
+
+    def sift_up(i)
+      if i > 0
+        p = parent(i)
+
+        if p && compare(i, p)
+          swap(i, p)
+          sift_up(p)
+        end
+      end
+    end
+  end
+end

data/lib/offline_sort/offline_sort.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require 'offline_sort/chunk'
+require 'offline_sort/fixed_size_min_heap'
+
+module OfflineSort
+  def self.sort_by(*args, **kwargs, &sort_by)
+    Sorter.new(*args, **kwargs, &sort_by)
+  end
+
+  class Sorter
+    DEFAULT_CHUNK_IO_CLASS = Chunk::InputOutput::Marshal
+    DEFAULT_CHUNK_SIZE = 1000
+
+    attr_reader :enumerable, :sort_by, :chunk_size, :chunk_input_output_class
+
+    def initialize(enumerable,
+                   chunk_input_output_class: DEFAULT_CHUNK_IO_CLASS,
+                   chunk_size: DEFAULT_CHUNK_SIZE,
+                   &sort_by)
+      @enumerable = enumerable
+      @chunk_input_output_class = chunk_input_output_class
+      @chunk_size = chunk_size
+      @sort_by = sort_by
+      @temp_files = []
+    end
+
+    def sort
+      enumerator = merge(split)
+      yield enumerator
+    ensure
+      @temp_files.each(&:close)
+    end
+
+    private
+
+    # TODO: optimization for when there is less than a single full chunk of data
+    def merge(sorted_chunk_ios)
+      pq = []
+      chunk_enumerators = sorted_chunk_ios.map(&:each)
+
+      chunk_enumerators.each_with_index do |chunk, index|
+        entry = chunk.next
+        pq.push(ChunkEntry.new(index, entry))
+      end
+
+      entry_sort_by = Proc.new { |entry| sort_by.call(entry.data) }
+      pq = FixedSizeMinHeap.new(pq, &entry_sort_by)
+
+      Enumerator.new do |yielder|
+        while (item = pq.pop)
+          yielder.yield(item.data)
+
+          begin
+            entry = chunk_enumerators[item.chunk_number].next
+            pq.push(ChunkEntry.new(item.chunk_number, entry))
+          rescue StopIteration
+            sorted_chunk_ios[item.chunk_number].close
+          end
+        end
+      end
+    end
+
+    def split
+      sorted_chunks = []
+      chunk_entries = []
+
+      enumerable.each do |entry|
+        chunk_entries << entry
+
+        if chunk_entries.size == chunk_size
+          sorted_chunks << write_sorted_chunk(chunk_entries)
+          chunk_entries.clear
+        end
+      end
+
+      sorted_chunks << write_sorted_chunk(chunk_entries) unless chunk_entries.empty?
+
+      sorted_chunks
+    end
+
+    def write_sorted_chunk(entries)
+      file = Tempfile.open('sort-chunk-')
+      @temp_files << file
+      file.unlink
+      file.binmode
+
+      chunk_io = chunk_input_output_class.new(file)
+      entries.sort_by(&sort_by).each { |entry| chunk_io.write_entry(entry) }
+
+      chunk_io.rewind
+      chunk_io
+    end
+
+    class ChunkEntry
+      attr_reader :chunk_number, :data
+
+      def initialize(chunk_number, data)
+        @chunk_number = chunk_number
+        @data = data
+      end
+    end
+  end
+end

data/lib/offline_sort/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module OfflineSort
+  VERSION = '0.3.0'
+end

data/lib/offline_sort.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'offline_sort/version'
+require 'offline_sort/chunk'
+require 'offline_sort/offline_sort'

data/offline-sort.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'offline_sort/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'iso-offline-sort'
+  spec.version       = OfflineSort::VERSION
+  spec.authors       = ['Isometric']
+  spec.email         = ['andy@iso.io']
+  spec.description   = 'Offline sort for any enumerable with pluggable serialization strategies'
+  spec.summary       = 'Offline sort for any enumerable with pluggable serialization strategies'
+  spec.homepage      = 'https://github.com/salsify/offline-sort'
+  spec.license       = 'MIT'
+
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+    spec.metadata['rubygems_mfa_required'] = 'true'
+  else
+    raise 'RubyGems 2.0 or newer is required to set allowed_push_host.'
+  end
+
+  spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+
+  spec.required_ruby_version = '>= 2.6'
+
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'msgpack'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rspec_junit_formatter'
+  spec.add_development_dependency 'salsify_rubocop', '~> 1.0.1'
+end

data/spec/offline_sort/chunk/input_output/base_spec.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+shared_examples "a valid chunk input output" do
+  let(:count) { 1000 }
+
+  let(:arrays) do
+    Array.new(count) do |index|
+      [SecureRandom.hex, index, SecureRandom.hex]
+    end
+  end
+
+  let(:hashes) do
+    Array.new(count) do |index|
+      { 'a' => SecureRandom.hex, 'b' => index, 'c' => SecureRandom.hex }
+    end
+  end
+
+  let(:tempfile) do
+    t = Tempfile.open('chunk-input-output')
+    t.binmode
+    t
+  end
+
+  let(:chunk_class) {}
+  let(:chunk_io) { chunk_class.new(tempfile) }
+
+  describe "#rewind" do
+    before do
+      allow(chunk_io).to receive(:flush)
+      allow(tempfile).to receive(:rewind)
+      chunk_io.rewind
+    end
+
+    it "rewinds the io" do
+      expect(tempfile).to have_received(:rewind)
+    end
+
+    it "flushes the io" do
+      expect(chunk_io).to have_received(:flush)
+    end
+  end
+
+  describe "#flush" do
+    before do
+      allow(tempfile).to receive(:flush)
+      chunk_io.flush
+    end
+
+    it "flushes the io" do
+      expect(tempfile).to have_received(:flush)
+    end
+  end
+
+  shared_examples "a valid integration test" do
+    let(:enumerable) {}
+
+    it "writes the data and reads it back" do
+      expect { chunk_io.write_entries(enumerable) }.not_to raise_error
+
+      chunk_io.rewind
+
+      expect(tempfile.size).not_to eq(0)
+      expect(chunk_io.each.to_a).to match_array(enumerable)
+    end
+  end
+
+  context "arrays" do
+    it_behaves_like "a valid integration test" do
+      let(:enumerable) { arrays }
+    end
+  end
+
+  context "hashes" do
+    it_behaves_like "a valid integration test" do
+      let(:enumerable) { hashes }
+    end
+  end
+end
+
+describe OfflineSort::Chunk::InputOutput::Base do
+  let(:io) { Tempfile.new('chunk') }
+  let(:chunk_io) { OfflineSort::Chunk::InputOutput::Base.new(io) }
+  let(:expected_error_klass) { OfflineSort::Chunk::InputOutput::Base::MethodNotImplementedError }
+
+  describe "#read_entry" do
+    it "raises when read_entry is called" do
+      expect { chunk_io.read_entry }.to raise_error(expected_error_klass)
+    end
+  end
+
+  describe "#write_entry" do
+    it "raises when write_entry is called" do
+      expect { chunk_io.write_entry({}) }.to raise_error(expected_error_klass)
+    end
+  end
+end

data/spec/offline_sort/chunk/input_output/marshal_spec.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe OfflineSort::Chunk::InputOutput::Marshal do
+  it_behaves_like "a valid chunk input output" do
+    let(:chunk_class) { OfflineSort::Chunk::InputOutput::Marshal }
+  end
+end

data/spec/offline_sort/chunk/input_output/message_pack_spec.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe OfflineSort::Chunk::InputOutput::MessagePack do
+  it_behaves_like "a valid chunk input output" do
+    let(:chunk_class) { OfflineSort::Chunk::InputOutput::MessagePack }
+  end
+end

data/spec/offline_sort/chunk/input_output/yaml_spec.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe OfflineSort::Chunk::InputOutput::Yaml do
+  it_behaves_like "a valid chunk input output" do
+    let(:chunk_class) { OfflineSort::Chunk::InputOutput::Yaml }
+  end
+end

data/spec/offline_sort/fixed_size_min_heap_spec.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe OfflineSort::FixedSizeMinHeap do
+  let(:array) { (1..10).to_a.shuffle }
+  let(:heap) { OfflineSort::FixedSizeMinHeap.new(array.dup) }
+
+  describe "#initialize" do
+    it "is a a heap" do
+      expect { assert_min_heap(heap.array) }.not_to raise_error
+    end
+  end
+
+  describe "#push" do
+    context "with a full array" do
+      it "raises an exception" do
+        expect { heap.push(rand(20)) }.to raise_error("Heap Size (#{array.size}) Exceeded")
+      end
+    end
+
+    context "with one space" do
+      before do
+        heap.pop
+      end
+
+      it "adds to the heap" do
+        expect { heap.push(1) }.not_to raise_error
+      end
+    end
+
+    context "with more than one space" do
+      before do
+        5.times { heap.pop }
+      end
+
+      it "adds to the heap" do
+        5.times do
+          expect { heap.push(1) }.not_to raise_error
+        end
+      end
+    end
+  end
+
+  describe "#pop" do
+    context "with empty array" do
+      before do
+        array.size.times { heap.pop }
+      end
+
+      it "is nil" do
+        expect(heap.pop).to be nil
+      end
+    end
+
+    context "until empty" do
+      it "is sorted" do
+        last = -1
+        array.size.times do
+          popped = heap.pop
+          expect(popped).to be > (last)
+          last = popped
+        end
+      end
+    end
+  end
+
+  context "integration test" do
+    it "is always heap ordered" do
+      100.times do
+        heap.pop
+        heap.push(rand(100))
+        expect { assert_min_heap(heap.array) }.not_to raise_error
+      end
+    end
+  end
+
+  def assert_min_heap(array)
+    array.each_with_index do |e, index|
+      left = (2 * index) + 1
+      right = (2 * index) + 2
+
+      if left < array.size && array[left] < e
+        puts "left #{e} #{array}"
+        raise 'not a heap'
+      end
+
+      next unless right < array.size
+
+      unless array[right] >= e
+        puts "right #{e} #{array}"
+        raise 'not a heap'
+      end
+    end
+  end
+end

data/spec/offline_sort/sorter_spec.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe OfflineSort::Sorter do
+
+  shared_examples "a correct offline sort" do
+    let(:count) { 10000 }
+    let(:entries_per_chunk) { 900 }
+    let(:enumerable) {}
+    let(:sort) {}
+
+    before do
+      @unsorted = enumerable.dup
+      r = Benchmark.measure do
+        sorter = OfflineSort.sort_by(enumerable, chunk_size: entries_per_chunk, &sort)
+
+        sorter.sort do |result|
+          @sorted = result.map do |entry|
+            entry
+          end
+        end
+      end
+      puts r
+    end
+
+    it "produces the same sorted result as an in-memory sort" do
+      expect(@unsorted).to match_array(enumerable)
+      expect do
+        last = nil
+        entry_count = 0
+        @sorted.each do |entry|
+          if last.nil?
+            last = entry
+            entry_count += 1
+            next
+          end
+
+          raise "Out of order at line #{entry_count}" unless (sort.call(last) <=> sort.call(entry)) == -1
+
+          last = entry
+          entry_count += 1
+        end
+      end.not_to raise_error
+      expect(@sorted).to match_array(enumerable.sort_by(&sort))
+    end
+  end
+
+  let(:arrays) do
+    Array.new(count) do |index|
+      [SecureRandom.hex, index, SecureRandom.hex]
+    end
+  end
+
+  let(:array_sort_index) { 2 }
+  let(:array_sort) { Proc.new { |arr| arr[array_sort_index] } }
+
+  let(:hashes) do
+    Array.new(count) do |index|
+      { 'a' => SecureRandom.hex, 'b' => index, 'c' => SecureRandom.hex }
+    end
+  end
+
+  let(:hash_sort_key) { 'c' }
+  let(:hash_sort) { Proc.new { |hash| hash[hash_sort_key] } }
+
+
+  context "with arrays" do
+    it_behaves_like "a correct offline sort" do
+      let(:enumerable) { arrays }
+      let(:sort) { array_sort }
+    end
+
+    context "with multiple sort keys" do
+      it_behaves_like "a correct offline sort" do
+        let(:enumerable) do
+          Array.new(count) do |index|
+            [index.round(-1), index, SecureRandom.hex]
+          end.shuffle
+        end
+        let(:sort) { Proc.new { |arr| [arr[0], arr[1]] } }
+      end
+    end
+  end
+
+  context "hashes" do
+    it_behaves_like "a correct offline sort" do
+      let(:enumerable) { hashes }
+      let(:sort) { hash_sort }
+    end
+
+    context "with multiple sort keys" do
+      it_behaves_like "a correct offline sort" do
+        let(:enumerable) do
+          Array.new(count) do |index|
+            { 'a' => index.round(-1), 'b' => index, 'c' => SecureRandom.hex }
+          end.shuffle
+        end
+        let(:sort) { Proc.new { |hash| [hash['a'], hash['c']] } }
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+require 'securerandom'
+require 'benchmark'
+require 'msgpack'
+require 'tempfile'
+
+require 'offline_sort'

metadata

@@ -0,0 +1,163 @@
+--- !ruby/object:Gem::Specification
+name: iso-offline-sort
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Isometric
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-10-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: salsify_rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+description: Offline sort for any enumerable with pluggable serialization strategies
+email:
+- andy@iso.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".github/CODEOWNERS"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/offline_sort.rb
+- lib/offline_sort/chunk.rb
+- lib/offline_sort/chunk/input_output/base.rb
+- lib/offline_sort/chunk/input_output/marshal.rb
+- lib/offline_sort/chunk/input_output/message_pack.rb
+- lib/offline_sort/chunk/input_output/yaml.rb
+- lib/offline_sort/fixed_size_min_heap.rb
+- lib/offline_sort/offline_sort.rb
+- lib/offline_sort/version.rb
+- offline-sort.gemspec
+- spec/offline_sort/chunk/input_output/base_spec.rb
+- spec/offline_sort/chunk/input_output/marshal_spec.rb
+- spec/offline_sort/chunk/input_output/message_pack_spec.rb
+- spec/offline_sort/chunk/input_output/yaml_spec.rb
+- spec/offline_sort/fixed_size_min_heap_spec.rb
+- spec/offline_sort/sorter_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/salsify/offline-sort
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: Offline sort for any enumerable with pluggable serialization strategies
+test_files:
+- spec/offline_sort/chunk/input_output/base_spec.rb
+- spec/offline_sort/chunk/input_output/marshal_spec.rb
+- spec/offline_sort/chunk/input_output/message_pack_spec.rb
+- spec/offline_sort/chunk/input_output/yaml_spec.rb
+- spec/offline_sort/fixed_size_min_heap_spec.rb
+- spec/offline_sort/sorter_spec.rb
+- spec/spec_helper.rb