match_reduce 1.0.0.pre.alpha

data/lib/match_reduce/processor/result_builder.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2019-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'result'
+
+module MatchReduce
+  class Processor
+    # This class understands how to take an aggregate and derive a result for it.
+    class ResultBuilder
+      def initialize(aggregate, resolver)
+        raise ArgumentError, 'aggregate is required'  unless aggregate
+        raise ArgumentError, 'resolver is required'   unless resolver
+
+        @aggregate  = aggregate
+        @resolver   = resolver
+
+        @records    = []
+        @value      = nil
+        @group_ids  = Set.new
+      end
+
+      def add(record, group_id)
+        if aggregate.grouped?
+          return self if group_ids.include?(group_id)
+
+          group_ids << group_id
+        end
+
+        records << record
+
+        @value = aggregate.reduce(value, record, resolver)
+
+        self
+      end
+
+      def result
+        Result.new(aggregate.name, records, value)
+      end
+
+      private
+
+      attr_reader :aggregate,
+                  :group_ids,
+                  :records,
+                  :resolver,
+                  :value
+    end
+  end
+end

data/lib/match_reduce/processor/results_builder.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2019-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'result_builder'
+
+module MatchReduce
+  class Processor
+    # This class knows how to group together aggregates in order to produce results.
+    class ResultsBuilder
+      attr_reader :resolver
+
+      def initialize(aggregates, resolver)
+        raise ArgumentError, 'aggregates are required' unless aggregates
+
+        @result_by_name = aggregates.map { |a| [a.name, ResultBuilder.new(a, resolver)] }.to_h
+        @resolver       = resolver
+
+        freeze
+      end
+
+      def add(aggregate, record, group_id)
+        tap { result_by_name[aggregate.name].add(record, group_id) }
+      end
+
+      def results
+        result_by_name.values.map(&:result)
+      end
+
+      private
+
+      attr_reader :result_by_name
+    end
+  end
+end

data/lib/match_reduce/processor.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2019-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'processor/results_builder'
+
+module MatchReduce
+  # This is the main lifecycle of the algorithm.  You initialize a new instance of this
+  # class using aggregates, then you pump in records into it.  Once done, call #results
+  # to get the results.
+  class Processor
+    extend Forwardable
+
+    def_delegators :results_builder, :results, :resolver
+
+    def_delegators :index, :aggregates
+
+    def initialize(aggregates, resolver)
+      @index            = Index.new(aggregates)
+      @results_builder  = ResultsBuilder.new(index.aggregates, resolver)
+
+      freeze
+    end
+
+    def add_each(records)
+      tap { records.each { |record| add(record) } }
+    end
+
+    def add(record)
+      hit_aggregates = Set.new
+
+      record_patterns(record).each do |hash_pattern|
+        # Each index find hit means the aggregate matched on the record
+        index.find(hash_pattern).each do |aggregate|
+          next if hit_aggregates.include?(aggregate)
+
+          add_to_results_builder(aggregate, record)
+
+          hit_aggregates << aggregate
+        end
+      end
+
+      self
+    end
+
+    private
+
+    attr_reader :index,
+                :results_builder
+
+    def make_group_id(aggregate, record)
+      aggregate.group_keys.map { |group_key| resolver.get(record, group_key) }
+    end
+
+    def record_matrix(record)
+      index.keys.each_with_object(HashMath::Matrix.new) do |key, memo|
+        value = resolver.get(record, key)
+
+        memo.add_each(key, [value, ANY])
+      end
+    end
+
+    def record_patterns(record)
+      [{}] + record_matrix(record).to_a
+    end
+
+    def add_to_results_builder(aggregate, record)
+      group_id = make_group_id(aggregate, record)
+
+      results_builder.add(aggregate, record, group_id)
+    end
+  end
+end

data/lib/match_reduce/version.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2019-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module MatchReduce
+  VERSION = '1.0.0-alpha'
+end

data/lib/match_reduce.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2019-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require 'acts_as_hashable'
+require 'hash_math'
+require 'objectable'
+
+require_relative 'match_reduce/any'
+require_relative 'match_reduce/aggregate'
+require_relative 'match_reduce/index'
+require_relative 'match_reduce/processor'
+
+# Top-level namespace
+module MatchReduce
+  # Define the only instance as a helper constant for the entire library to share.
+  # Technically it is not a singleton, but it does not have to be because it will still
+  # provide equality where we need it: #hash, #eql?, and #==.  We are using this as a
+  # special flag indicating: "match on any value".  So even if we were to instantiate
+  # multiple Any objects, the point is moot.
+  ANY = Any.new
+
+  class << self
+    def process(aggregates, records, resolver = Objectable.resolver)
+      Processor.new(aggregates, resolver)
+               .add_each(records)
+               .results
+    end
+  end
+end

data/match_reduce.gemspec

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require './lib/match_reduce/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'match_reduce'
+  s.version     = MatchReduce::VERSION
+  s.summary     = 'Dataset aggregation and reducer algorithm'
+
+  s.description = <<-DESCRIPTION
+    High speed data aggregator and reducer algorithm based on key-value exact matching and grouping.
+  DESCRIPTION
+
+  s.authors     = ['Matthew Ruggio']
+  s.email       = ['mruggio@bluemarblepayroll.com']
+  s.files       = `git ls-files`.split("\n")
+  s.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  s.homepage    = 'https://github.com/bluemarblepayroll/match_reduce'
+  s.license     = 'MIT'
+
+  s.required_ruby_version = '>= 2.3.8'
+
+  s.add_dependency('acts_as_hashable', '~>1', '>=1.1.0')
+  s.add_dependency('hash_math', '>=1.0.0-alpha')
+  s.add_dependency('objectable', '~>1')
+
+  s.add_development_dependency('guard-rspec', '~>4.7')
+  s.add_development_dependency('pry', '~>0')
+  s.add_development_dependency('rake', '~> 12')
+  s.add_development_dependency('rspec')
+  s.add_development_dependency('rubocop', '~>0.74.0')
+  s.add_development_dependency('simplecov', '~>0.17.0')
+  s.add_development_dependency('simplecov-console', '~>0.5.0')
+end

data/spec/fixtures/snapshots/processor/abstract.yaml

@@ -0,0 +1,68 @@
+records:
+  - { a: a1, b: b1, c: c1, d: d1, e: e1, val: 1 }
+  - { a: a2, b: b2, c: c2, d: d2, e: e2, val: 2 }
+  - { a: a3, b: b3, c: c3, d: d3, e: e3, val: 3 }
+
+aggregates:
+  - name: all (no patterns)
+    sum_reducer_key: val
+    patterns:
+  - name: all (match all patterns)
+    sum_reducer_key: val
+    patterns:
+      - a: a1
+      - b: b2
+      - c: c3
+  - name: match 1
+    sum_reducer_key: val
+    patterns:
+      - a: a1
+        d: d1
+      - z: z2
+        a: a2
+  - name: match 1 and 3
+    sum_reducer_key: val
+    patterns:
+      - a: a1
+        d: d1
+      - z: z2
+        a: a2
+      - a: a3
+        c: c3
+  - name: no matches
+    sum_reducer_key: val
+    patterns:
+      - a: a1
+        b: b2
+      - a: a8
+        b: a8
+      - z: z1
+      - A: a1
+      - a: A1
+      - a: 1
+
+results:
+  - name: all (no patterns)
+    value: 6
+    records:
+      - { a: a1, b: b1, c: c1, d: d1, e: e1, val: 1 }
+      - { a: a2, b: b2, c: c2, d: d2, e: e2, val: 2 }
+      - { a: a3, b: b3, c: c3, d: d3, e: e3, val: 3 }
+  - name: all (match all patterns)
+    value: 6
+    records:
+      - { a: a1, b: b1, c: c1, d: d1, e: e1, val: 1 }
+      - { a: a2, b: b2, c: c2, d: d2, e: e2, val: 2 }
+      - { a: a3, b: b3, c: c3, d: d3, e: e3, val: 3 }
+  - name: match 1
+    value: 1
+    records:
+      - { a: a1, b: b1, c: c1, d: d1, e: e1, val: 1 }
+  - name: match 1 and 3
+    value: 4
+    records:
+      - { a: a1, b: b1, c: c1, d: d1, e: e1, val: 1 }
+      - { a: a3, b: b3, c: c3, d: d3, e: e3, val: 3 }
+  - name: no matches
+    value:
+    records: []

data/spec/fixtures/snapshots/processor/teams_and_players.yaml

@@ -0,0 +1,173 @@
+records:
+  - team: bulls
+    team_points: 1000
+    player: michael_jordan
+    player_points: 750
+
+  - team: bulls
+    team_points: 1000
+    player: scottie_pippen
+    player_points: 150
+
+  - team: wizards
+    team_points: 800
+    player: michael_jordan
+    player_points: 600
+
+  - team: wizards
+    team_points: 800
+    player: rip_hamilton
+    player_points: 200
+
+  - team: pistons
+    team_points: 1200
+    player: rip_hamilton
+    player_points: 300
+
+  - team: bulls
+    team_points: 1000
+    player: rip_hamilton
+    player_points: 100
+
+aggregates:
+  - name: total_points_by_team_player
+    patterns:
+    sum_reducer_key: player_points
+
+  - name: total_points_by_team
+    patterns:
+    sum_reducer_key: team_points
+    group_keys: team
+
+  - name: bulls_points_by_team
+    patterns:
+      team: bulls
+    sum_reducer_key: team_points
+    group_keys: team
+
+  - name: wizards_points_by_team
+    patterns:
+      team: wizards
+    sum_reducer_key: team_points
+    group_keys: team
+
+  - name: jordan_points_for_bulls
+    patterns:
+      team: bulls
+      player: michael_jordan
+    sum_reducer_key: player_points
+
+  - name: all_jordan_points
+    patterns:
+      player: michael_jordan
+    sum_reducer_key: player_points
+
+  - name: rip_points_for_detroit_and_wizards
+    patterns:
+      - player: rip_hamilton
+        team: wizards
+      - player: rip_hamilton
+        team: pistons
+    sum_reducer_key: player_points
+
+results:
+  - name: total_points_by_team_player
+    records:
+      - team: bulls
+        team_points: 1000
+        player: michael_jordan
+        player_points: 750
+
+      - team: bulls
+        team_points: 1000
+        player: scottie_pippen
+        player_points: 150
+
+      - team: wizards
+        team_points: 800
+        player: michael_jordan
+        player_points: 600
+
+      - team: wizards
+        team_points: 800
+        player: rip_hamilton
+        player_points: 200
+
+      - team: pistons
+        team_points: 1200
+        player: rip_hamilton
+        player_points: 300
+
+      - team: bulls
+        team_points: 1000
+        player: rip_hamilton
+        player_points: 100
+    value: 2100
+
+  - name: total_points_by_team
+    records:
+      - team: bulls
+        team_points: 1000
+        player: michael_jordan
+        player_points: 750
+
+      - team: wizards
+        team_points: 800
+        player: michael_jordan
+        player_points: 600
+
+      - team: pistons
+        team_points: 1200
+        player: rip_hamilton
+        player_points: 300
+    value: 3000
+
+  - name: bulls_points_by_team
+    records:
+      - team: bulls
+        team_points: 1000
+        player: michael_jordan
+        player_points: 750
+    value: 1000
+
+  - name: wizards_points_by_team
+    records:
+      - team: wizards
+        team_points: 800
+        player: michael_jordan
+        player_points: 600
+    value: 800
+
+  - name: jordan_points_for_bulls
+    records:
+      - team: bulls
+        team_points: 1000
+        player: michael_jordan
+        player_points: 750
+    value: 750
+
+  - name: all_jordan_points
+    records:
+      - team: bulls
+        team_points: 1000
+        player: michael_jordan
+        player_points: 750
+
+      - team: wizards
+        team_points: 800
+        player: michael_jordan
+        player_points: 600
+    value: 1350
+
+  - name: rip_points_for_detroit_and_wizards
+    records:
+      - team: wizards
+        team_points: 800
+        player: rip_hamilton
+        player_points: 200
+
+      - team: pistons
+        team_points: 1200
+        player: rip_hamilton
+        player_points: 300
+    value: 500

data/spec/match_reduce/aggregate_spec.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2019-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require 'spec_helper'
+
+describe MatchReduce::Aggregate do
+  describe '#initialize' do
+    it 'sets patterns to at least one hash' do
+      subject = described_class.new(name: :sig1)
+
+      expect(subject.patterns).to eq([{}])
+    end
+
+    it 'patterns can be a hash' do
+      subject = described_class.new(name: :sig1, patterns: { a: :b })
+
+      expect(subject.patterns).to eq([{ 'a' => :b }])
+    end
+
+    it 'patterns can be an array of hashes' do
+      subject = described_class.new(name: :sig1, patterns: [{ a: :b }])
+
+      expect(subject.patterns).to eq([{ 'a' => :b }])
+    end
+  end
+end

data/spec/match_reduce/any_spec.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2019-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require 'spec_helper'
+
+describe MatchReduce::Any do
+  describe '#hash' do
+    it 'should be based on an array of the class name and the symbol :any' do
+      expect(described_class.new.hash).to eq(['MatchReduce::Any', :any].hash)
+    end
+  end
+
+  describe 'equality' do
+    specify '#== should always be equal if the classes and hash are the same' do
+      expect(described_class.new).to eq(described_class.new)
+
+      expect(described_class.new).not_to eq(:any)
+    end
+
+    specify '#eql? should always be equal if the classes and hash are the same' do
+      expect(described_class.new).to eql(described_class.new)
+
+      expect(described_class.new).not_to eq(:any)
+    end
+  end
+end

data/spec/match_reduce/index_spec.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2019-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require 'spec_helper'
+
+describe MatchReduce::Index do
+  def lookup_by_name(lookup)
+    lookup.each_with_object({}) do |(pattern, aggregates), memo|
+      memo[pattern] = aggregates.map(&:name)
+    end
+  end
+
+  let(:base_value) { MatchReduce::ANY }
+
+  describe '#initialization' do
+    context 'constructing aggregates' do
+      specify 'when all aggregates when empty' do
+        subject = described_class.new
+
+        expect(subject.aggregates).to eq([])
+      end
+
+      specify 'only each first unique aggregate name is kept' do
+        aggregates = [
+          { name: :sig3 },
+          { name: :sig1 },
+          { name: 'sig2' },
+          { name: :sig2 },
+          { name: :sig3 },
+          { name: 'sig4' },
+          { name: :sig4 },
+          { name: :sig5 },
+          { name: :sig5 }
+        ]
+
+        subject = described_class.new(aggregates)
+
+        expect(subject.aggregates.length).to eq(7)
+      end
+    end
+
+    context 'constructing lookup' do
+      it 'creates lookup with aggregates' do
+        subject = described_class.new
+
+        expected = {}
+
+        expect(subject.lookup).to eq(expected)
+      end
+
+      it 'creates lookup with aggregates that have no patterns' do
+        aggregates = [
+          { name: :sig3 },
+          { name: :sig1 }
+        ]
+
+        subject = described_class.new(aggregates)
+
+        expected = {
+          {} => %i[sig3 sig1]
+        }
+
+        expect(lookup_by_name(subject.lookup)).to eq(expected)
+      end
+
+      it 'creates lookup with aggregates that have patterns and no patterns' do
+        aggregates = [
+          { name: :sig3 },
+          { name: :sig1 },
+          {
+            name: :sig2,
+            patterns: { 'a' => '1', 'b' => [nil], 'c' => :c }
+          }
+        ]
+
+        subject = described_class.new(aggregates)
+
+        expected = {
+          { 'a' => base_value, 'b' => base_value, 'c' => base_value } => %i[sig3 sig1],
+          { 'a' => '1', 'b' => [nil], 'c' => :c } => %i[sig2]
+        }
+
+        expect(lookup_by_name(subject.lookup)).to eq(expected)
+      end
+    end
+  end
+end

data/spec/match_reduce/processor/result_spec.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2019-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require 'spec_helper'
+
+describe MatchReduce::Processor::Result do
+  let(:config) do
+    {
+      name: 'abc',
+      records: [
+        { something: :else }
+      ],
+      value: 123
+    }
+  end
+
+  let(:args) { config.values }
+
+  describe 'equality' do
+    let(:object_a) { described_class.new(*args) }
+
+    let(:object_b) { described_class.new(*args) }
+
+    specify '#== compares class type and attributes' do
+      expect(object_a).to eq(object_b)
+    end
+
+    specify '#== compares class type and attributes' do
+      expect(object_a).to eql(object_b)
+    end
+  end
+
+  specify '#hash is a computed hash of attributes' do
+    subject = described_class.new(*args)
+
+    expected = [
+      config[:name], config[:records], config[:value]
+    ].hash
+
+    expect(subject.hash).to eq(expected)
+  end
+end