oplogjam 0.1.0

data/lib/oplogjam/operators/intermediate_field.rb

@@ -0,0 +1,13 @@
+require 'oplogjam/operators/intermediate'
+
+module Oplogjam
+  module Operators
+    class IntermediateField < Intermediate
+      def update(column)
+        populated_column = column.set(path, Sequel.function(:coalesce, column[path], EMPTY_OBJECT))
+
+        nodes.inject(populated_column, &UPDATE_COLUMN)
+      end
+    end
+  end
+end

data/lib/oplogjam/operators/intermediate_index.rb

@@ -0,0 +1,43 @@
+require 'oplogjam/operators/intermediate'
+
+module Oplogjam
+  module Operators
+    class IntermediateIndex < Intermediate
+      def update(column)
+        # Now for a not-so-fun bit!
+        #
+        # As this is a numeric index, it might either be an index into an existing array or a numeric field name on an
+        # object.
+        #
+        # If it is an index into an array then we need to ensure that all prior indexes down to 0 are either set or null.
+        # If it is anything else, it should be an empty object. In order to figure that out, we need to look at the parent
+        # path and switch based on its type.
+        filled_array_column = (0...index).inject(column) { |subject, i|
+          prior_path = parent_path + [i.to_s]
+
+          subject.set(prior_path, Sequel.function(:coalesce, column[prior_path], NULL))
+        }
+
+        populated_column = Sequel.pg_jsonb_op(
+          Sequel.case(
+            {
+              ARRAY_TYPE => filled_array_column.set(path, Sequel.function(:coalesce, filled_array_column[path], EMPTY_OBJECT))
+            },
+            column.set(path, Sequel.function(:coalesce, column[path], EMPTY_OBJECT)),
+            column[parent_path].typeof
+          )
+        )
+
+        nodes.inject(populated_column, &UPDATE_COLUMN)
+      end
+
+      def parent_path
+        path[0...-1]
+      end
+
+      def index
+        Integer(path.last, 10)
+      end
+    end
+  end
+end

data/lib/oplogjam/operators/set.rb

@@ -0,0 +1,100 @@
+require 'oplogjam/operators/field_assignment'
+require 'oplogjam/operators/index_assignment'
+require 'oplogjam/operators/intermediate_field'
+require 'oplogjam/operators/intermediate_index'
+
+module Oplogjam
+  module Operators
+    class Set
+
+      # Transform a MongoDB $set operation (e.g. $set: { 'a.1.b.0': 'foo' }) into a tree of nodes more amenable to
+      # transforming into SQL.
+      #
+      # Specifically, parse nested field and index assignments into specific node types FieldAssignment, IndexAssignment
+      # (for setting the final value) and IntermediateField, IntermediateIndex (for any intermediate fields and indexes).
+      #
+      # e.g.
+      #
+      #   $set: { a: 1 } will become Set(['a'] => FieldAssignment(['a'], 1))
+      #   $set: { a: 1, b: 2 } will become Set(['a'] => FieldAssignment(['a'], 1), ['b'] => FieldAssignment(['b'], 2))
+      #   $set: { 'a.b': 1 } will become Set(['a'] => IntermediateField(['a'], FieldAssignment(['a', 'b'], 1)))
+      def self.from(operation)
+
+        # Start with an empty Set and iterate over every key in the $set operation, mutating the Set as we go
+        operation.each_with_object(new) do |(dotted_path, value), set|
+
+          # Split the dotted path `a.b.c` into an array `['a', 'b', 'c']`
+          path = dotted_path.split(FIELD_SEPARATOR)
+
+          # Start with an empty path which will incrementally populate
+          current_path = []
+
+          # Starting with the set, go through the successive path segments, building up intermediate paths on the current
+          # node, e.g. 'a.b' will iterate over ['a'], ['a', 'b']
+          #
+          # Note that we exclude the final path segment as that will be used below in a separate set phase
+          populated_node = path[0...-1].inject(set) { |current_node, segment|
+
+            # Extend the current path with the current segment appended.
+            current_path << segment
+
+            # Populate an empty intermediate with a copy of the current path.
+            #
+            # Note that this could either be a numeric index (which might be indexing into an array) or an object field
+            # name.
+            current_node.populate(current_path.dup)
+          }
+
+          # Set the final value on the full path
+          populated_node.set(path, value)
+        end
+      end
+
+      attr_reader :tree
+
+      def initialize(tree = {})
+        @tree = tree
+      end
+
+      def populate(path)
+        if path.last =~ NUMERIC_INDEX
+          populate_index(path)
+        else
+          populate_field(path)
+        end
+      end
+
+      def set(path, value)
+        if path.last =~ NUMERIC_INDEX
+          set_index(path, value)
+        else
+          set_field(path, value)
+        end
+      end
+
+      def populate_field(path)
+        tree[path] ||= IntermediateField.new(path)
+      end
+
+      def populate_index(path)
+        tree[path] ||= IntermediateIndex.new(path)
+      end
+
+      def set_field(path, value)
+        tree[path] = FieldAssignment.new(path, value)
+      end
+
+      def set_index(path, value)
+        tree[path] = IndexAssignment.new(path, value)
+      end
+
+      def update(column)
+        nodes.inject(column, &UPDATE_COLUMN)
+      end
+
+      def nodes
+        tree.values
+      end
+    end
+  end
+end

data/lib/oplogjam/operators/unset.rb

@@ -0,0 +1,40 @@
+require 'oplogjam/operators/unset_field'
+require 'oplogjam/operators/unset_index'
+
+module Oplogjam
+  module Operators
+    class Unset
+      def self.from(operation)
+        operation.each_with_object(new) do |(dotted_path, _), unset|
+          path = dotted_path.split(FIELD_SEPARATOR)
+
+          if path.last =~ NUMERIC_INDEX
+            unset.unset_index(path)
+          else
+            unset.unset_field(path)
+          end
+        end
+      end
+
+      attr_reader :unsets
+
+      def initialize(unsets = [])
+        @unsets = unsets
+      end
+
+      def unset_field(path)
+        unsets << UnsetField.new(path)
+      end
+
+      def unset_index(path)
+        unsets << UnsetIndex.new(path)
+      end
+
+      def delete(column)
+        unsets.inject(column) do |subject, unset|
+          unset.delete(subject)
+        end
+      end
+    end
+  end
+end

data/lib/oplogjam/operators/unset_field.rb

@@ -0,0 +1,15 @@
+module Oplogjam
+  module Operators
+    class UnsetField
+      attr_reader :path
+
+      def initialize(path)
+        @path = path
+      end
+
+      def delete(column)
+        column.delete_path(path)
+      end
+    end
+  end
+end

data/lib/oplogjam/operators/unset_index.rb

@@ -0,0 +1,39 @@
+module Oplogjam
+  module Operators
+    class UnsetIndex
+      attr_reader :path
+
+      def initialize(path)
+        @path = path
+      end
+
+      def delete(column)
+        nullify_or_unset = Sequel.case(
+          [
+            [
+              column[parent_path].array_length > index,
+              column.set(path, NULL)
+            ]
+          ],
+          column.delete_path(path)
+        )
+
+        Sequel.pg_jsonb_op(
+          Sequel.case(
+            { ARRAY_TYPE => nullify_or_unset },
+            column.delete_path(path),
+            column[parent_path].typeof
+          )
+        )
+      end
+
+      def parent_path
+        path[0...-1]
+      end
+
+      def index
+        Integer(path.last, 10)
+      end
+    end
+  end
+end

data/lib/oplogjam/oplog.rb

@@ -0,0 +1,21 @@
+require 'oplogjam/operation'
+
+module Oplogjam
+  class Oplog
+    attr_reader :client
+
+    def initialize(client)
+      @client = client
+    end
+
+    def operations(query = {})
+      Enumerator.new do |yielder|
+        cursor = client.use(LOCAL)[OPLOG].find(query, cursor_type: :tailable_await).no_cursor_timeout
+
+        cursor.each do |document|
+          yielder << Operation.from(document)
+        end
+      end
+    end
+  end
+end

data/lib/oplogjam/sanitizer.rb

@@ -0,0 +1,19 @@
+module Oplogjam
+  module Sanitizer
+    # Strip any null bytes from objects as they will be rejected by PostgreSQL
+    def self.sanitize(obj)
+      case obj
+      when Sequel::Postgres::JSONBHash, Hash
+        obj.each_with_object({}) do |(key, value), acc|
+          acc[sanitize(key)] = sanitize(value)
+        end
+      when Sequel::Postgres::JSONBArray, Array
+        obj.map { |element| sanitize(element) }
+      when String
+        obj.tr("\x00", '')
+      else
+        obj
+      end
+    end
+  end
+end

data/lib/oplogjam/schema.rb

@@ -0,0 +1,44 @@
+module Oplogjam
+  class Schema
+    COLUMNS = %i[id document created_at updated_at].freeze
+
+    attr_reader :db
+
+    def initialize(db)
+      @db = db
+    end
+
+    def import(collection, name, batch_size = 100)
+      collection.find.snapshot(true).each_slice(batch_size) do |documents|
+        values = documents.map { |document|
+          [
+            Sequel.object_to_json(document.fetch(ID)),
+            Sequel.pg_jsonb(document),
+            Time.now.utc,
+            Time.now.utc
+          ]
+        }
+
+        db[name].import(COLUMNS, values)
+      end
+    end
+
+    def create_table(name)
+      db.create_table?(name) do
+        uuid :uuid, default: Sequel.function(:uuid_generate_v1), primary_key: true
+        jsonb :id, null: false
+        jsonb :document, null: false
+        timestamp :created_at, null: false
+        timestamp :updated_at, null: false
+        timestamp :deleted_at
+      end
+    end
+
+    def add_indexes(name)
+      db.alter_table(name) do
+        add_index %i[id deleted_at], unique: true, if_not_exists: true
+        add_index :id, unique: true, where: { deleted_at: nil }, if_not_exists: true
+      end
+    end
+  end
+end

data/lib/oplogjam/update.rb

@@ -0,0 +1,79 @@
+require 'oplogjam/operators'
+
+module Oplogjam
+  InvalidUpdate = Class.new(ArgumentError)
+
+  class Update
+    attr_reader :h, :ts, :ns, :o2, :o
+
+    def self.from(bson)
+      h = bson.fetch(H)
+      ts = bson.fetch(TS)
+      ns = bson.fetch(NS)
+      o2 = bson.fetch(O2)
+      o = bson.fetch(O)
+
+      new(h, ts, ns, o2, o)
+    rescue KeyError => e
+      raise InvalidUpdate, "missing field: #{e}"
+    end
+
+    def initialize(h, ts, ns, o2, o)
+      @h = Integer(h)
+      @ts = Oplogjam::Timestamp(ts)
+      @ns = String(ns)
+      @o2 = Oplogjam::Document(o2)
+      @o = Oplogjam::Document(o)
+    end
+
+    alias id h
+    alias namespace ns
+    alias query o2
+    alias update o
+
+    def timestamp
+      Time.at(ts.seconds, ts.increment)
+    end
+
+    def ==(other)
+      return false unless other.is_a?(Update)
+
+      id == other.id
+    end
+
+    def apply(mapping)
+      table = mapping[namespace]
+      return unless table
+
+      row_id = query.fetch(ID).to_json
+
+      table
+        .where(id: row_id, deleted_at: nil)
+        .update(document: jsonb_update, updated_at: Time.now.utc)
+    end
+
+    private
+
+    def jsonb_update
+      return Sequel.pg_jsonb(query.merge(update)) if replacement?
+
+      unsets_to_jsonb(sets_to_jsonb(Sequel.pg_jsonb_op(:document)))
+    end
+
+    def sets_to_jsonb(column)
+      return column unless update.key?(SET)
+
+      Operators::Set.from(update.fetch(SET)).update(column)
+    end
+
+    def unsets_to_jsonb(column)
+      return column unless update.key?(UNSET)
+
+      Operators::Unset.from(update.fetch(UNSET)).delete(column)
+    end
+
+    def replacement?
+      !update.key?(SET) && !update.key?(UNSET)
+    end
+  end
+end

data/spec/oplogjam/apply_ops_spec.rb

@@ -0,0 +1,174 @@
+require 'bson'
+require 'oplogjam'
+
+module Oplogjam
+  RSpec.describe ApplyOps do
+    describe '.from' do
+      it 'converts a BSON applyOps to an ApplyOps' do
+        bson = BSON::Document.new(
+          ts: BSON::Timestamp.new(1_479_420_028, 1),
+          t: 1,
+          h: -1_789_557_309_812_000_233,
+          v: 2,
+          op: 'c',
+          ns: 'foo.$cmd',
+          o: BSON::Document.new(
+            applyOps: [
+              BSON::Document.new(
+                ts: BSON::Timestamp.new(1_496_414_570, 11),
+                t: 14,
+                h: -3_028_027_288_268_436_781,
+                v: 2,
+                op: 'i',
+                ns: 'foo.bar',
+                o: BSON::Document.new(_id: 1, baz: 'quux')
+              )
+            ]
+          )
+        )
+
+        expect(described_class.from(bson)).to be_a(described_class)
+      end
+
+      it 'raises an error if the operations are missing' do
+        bson = BSON::Document.new(
+          ts: BSON::Timestamp.new(1_479_420_028, 1),
+          t: 1,
+          h: -1_789_557_309_812_000_233,
+          v: 2,
+          op: 'c',
+          ns: 'foo.$cmd'
+        )
+
+        expect { described_class.from(bson) }.to raise_error(InvalidApplyOps)
+      end
+    end
+
+    describe '#timestamp' do
+      it 'returns the timestamp as a Time' do
+        bson = BSON::Document.new(
+          ts: BSON::Timestamp.new(1_479_420_028, 1),
+          t: 1,
+          h: -1_789_557_309_812_000_233,
+          v: 2,
+          op: 'c',
+          ns: 'foo.$cmd',
+          o: BSON::Document.new(
+            applyOps: [
+              BSON::Document.new(
+                ts: BSON::Timestamp.new(1_496_414_570, 11),
+                t: 14,
+                h: -3_028_027_288_268_436_781,
+                v: 2,
+                op: 'i',
+                ns: 'foo.bar',
+                o: BSON::Document.new(_id: 1, baz: 'quux')
+              )
+            ]
+          )
+        )
+        apply_ops = described_class.from(bson)
+
+        expect(apply_ops.timestamp).to eq(Time.at(1_479_420_028, 1))
+      end
+    end
+
+    describe '#namespace' do
+      it 'returns the namespace' do
+        bson = BSON::Document.new(
+          ts: BSON::Timestamp.new(1_479_420_028, 1),
+          t: 1,
+          h: -1_789_557_309_812_000_233,
+          v: 2,
+          op: 'c',
+          ns: 'foo.$cmd',
+          o: BSON::Document.new(
+            applyOps: [
+              BSON::Document.new(
+                ts: BSON::Timestamp.new(1_496_414_570, 11),
+                t: 14,
+                h: -3_028_027_288_268_436_781,
+                v: 2,
+                op: 'i',
+                ns: 'foo.bar',
+                o: BSON::Document.new(_id: 1, baz: 'quux')
+              )
+            ]
+          )
+        )
+        apply_ops = described_class.from(bson)
+
+        expect(apply_ops.namespace).to eq('foo.$cmd')
+      end
+    end
+
+    describe '#id' do
+      it 'returns a unique identifier for the operation' do
+        bson = BSON::Document.new(
+          ts: BSON::Timestamp.new(1_479_420_028, 1),
+          t: 1,
+          h: -1_789_557_309_812_000_233,
+          v: 2,
+          op: 'c',
+          ns: 'foo.$cmd',
+          o: BSON::Document.new(
+            applyOps: [
+              BSON::Document.new(
+                ts: BSON::Timestamp.new(1_496_414_570, 11),
+                t: 14,
+                h: -3_028_027_288_268_436_781,
+                v: 2,
+                op: 'i',
+                ns: 'foo.bar',
+                o: BSON::Document.new(_id: 1, baz: 'quux')
+              )
+            ]
+          )
+        )
+        apply_ops = described_class.from(bson)
+
+        expect(apply_ops.id).to eq(-1_789_557_309_812_000_233)
+      end
+    end
+
+    describe '#operations' do
+      it 'returns the operations' do
+        bson = BSON::Document.new(
+          ts: BSON::Timestamp.new(1_479_420_028, 1),
+          t: 1,
+          h: -1_789_557_309_812_000_233,
+          v: 2,
+          op: 'c',
+          ns: 'foo.$cmd',
+          o: BSON::Document.new(
+            applyOps: [
+              BSON::Document.new(
+                ts: BSON::Timestamp.new(1_496_414_570, 11),
+                t: 14,
+                h: -3_028_027_288_268_436_781,
+                v: 2,
+                op: 'i',
+                ns: 'foo.bar',
+                o: BSON::Document.new(_id: 1, baz: 'quux')
+              )
+            ]
+          )
+        )
+        apply_ops = described_class.from(bson)
+        insert = Insert.from(
+          BSON::Document.new(
+            ts: BSON::Timestamp.new(1_496_414_570, 11),
+            t: 14,
+            h: -3_028_027_288_268_436_781,
+            v: 2,
+            op: 'i',
+            ns: 'foo.bar',
+            o: BSON::Document.new(_id: 1, baz: 'quux')
+          )
+        )
+
+        expect(apply_ops.operations).to contain_exactly(insert)
+      end
+    end
+  end
+end