oplogjam 0.1.0

data/lib/oplogjam.rb

@@ -0,0 +1,69 @@
+require 'bson'
+require 'oj'
+require 'sequel'
+require 'oplogjam/oplog'
+require 'oplogjam/sanitizer'
+require 'oplogjam/schema'
+
+module Oplogjam
+  # Enable Sequel extensions for JSONB
+  Sequel.extension :pg_array, :pg_json, :pg_json_ops
+
+  # Operation types
+  APPLY_OPS = 'applyOps'.freeze
+  C = 'c'.freeze
+  D = 'd'.freeze
+  U = 'u'.freeze
+  I = 'i'.freeze
+  N = 'n'.freeze
+  UNKNOWN = 'unknown'.freeze
+
+  # Operation fields
+  H = 'h'.freeze
+  MSG = 'msg'.freeze
+  NS = 'ns'.freeze
+  O = 'o'.freeze
+  O2 = 'o2'.freeze
+  OP = 'op'.freeze
+  TS = 'ts'.freeze
+
+  # BSON fields
+  ID = '_id'.freeze
+  SET = '$set'.freeze
+  UNSET = '$unset'.freeze
+  FIELD_SEPARATOR = '.'.freeze
+  NUMERIC_INDEX = /\A\d+\z/
+
+  # SQL
+  ARRAY_TYPE = 'array'.freeze
+  EMPTY_OBJECT = Sequel.pg_jsonb({}.freeze)
+  NULL = 'null'.freeze
+  TABLE = 'table'.freeze
+  SCHEMA = 'schema'.freeze
+  PUBLIC = 'public'.freeze
+
+  # Recursive updates
+  UPDATE_COLUMN = proc { |column, node| node.update(column) }
+
+  # Database & collection names
+  LOCAL = 'local'.freeze
+  OPLOG = 'oplog.rs'.freeze
+
+  # Strict type coercion for BSON types
+  def self.Timestamp(ts)
+    raise TypeError, "#{ts} is not a BSON Timestamp" unless ts.is_a?(BSON::Timestamp)
+
+    ts
+  end
+
+  def self.Document(document)
+    raise TypeError, "#{document} is not a BSON Document" unless document.is_a?(BSON::Document)
+
+    document
+  end
+
+  # Override JSONB serialization to sanitize data first
+  def Sequel.object_to_json(obj, *args, &blk)
+    Oj.dump(Sanitizer.sanitize(obj), mode: :rails)
+  end
+end

data/lib/oplogjam/apply_ops.rb

@@ -0,0 +1,49 @@
+module Oplogjam
+  InvalidApplyOps = Class.new(ArgumentError)
+
+  class ApplyOps
+    attr_reader :h, :ts, :ns, :apply_ops
+
+    def self.from(bson)
+      h = bson.fetch(H)
+      ts = bson.fetch(TS)
+      ns = bson.fetch(NS)
+      o = bson.fetch(O)
+      apply_ops = o.fetch(APPLY_OPS)
+
+      new(h, ts, ns, apply_ops)
+    rescue KeyError => e
+      raise InvalidApplyOps, "missing field: #{e}"
+    end
+
+    def initialize(h, ts, ns, apply_ops)
+      @h = Integer(h)
+      @ts = Oplogjam::Timestamp(ts)
+      @ns = String(ns)
+      @apply_ops = Array(apply_ops)
+    end
+
+    alias id h
+    alias namespace ns
+
+    def timestamp
+      Time.at(ts.seconds, ts.increment)
+    end
+
+    def apply(mapping)
+      operations.each do |operation|
+        operation.apply(mapping)
+      end
+    end
+
+    def operations
+      apply_ops.map { |bson| Operation.from(bson) }
+    end
+
+    def ==(other)
+      return unless other.is_a?(ApplyOps)
+
+      id == other.id
+    end
+  end
+end

data/lib/oplogjam/command.rb

@@ -0,0 +1,41 @@
+module Oplogjam
+  InvalidCommand = Class.new(ArgumentError)
+
+  class Command
+    attr_reader :h, :ts, :ns, :o
+
+    def self.from(bson)
+      h = bson.fetch(H)
+      ts = bson.fetch(TS)
+      ns = bson.fetch(NS)
+      o = bson.fetch(O)
+
+      new(h, ts, ns, o)
+    rescue KeyError => e
+      raise InvalidCommand, "missing field: #{e}"
+    end
+
+    def initialize(h, ts, ns, o)
+      @h = Integer(h)
+      @ts = Oplogjam::Timestamp(ts)
+      @ns = String(ns)
+      @o = Oplogjam::Document(o)
+    end
+
+    alias id h
+    alias command o
+    alias namespace ns
+
+    def timestamp
+      Time.at(ts.seconds, ts.increment)
+    end
+
+    def ==(other)
+      return false unless other.is_a?(Command)
+
+      id == other.id
+    end
+
+    def apply(_mapping); end
+  end
+end

data/lib/oplogjam/delete.rb

@@ -0,0 +1,50 @@
+module Oplogjam
+  InvalidDelete = Class.new(ArgumentError)
+
+  class Delete
+    attr_reader :h, :ts, :ns, :o
+
+    def self.from(bson)
+      h = bson.fetch(H)
+      ts = bson.fetch(TS)
+      ns = bson.fetch(NS)
+      o = bson.fetch(O)
+
+      new(h, ts, ns, o)
+    rescue KeyError => e
+      raise InvalidDelete, "missing field: #{e}"
+    end
+
+    def initialize(h, ts, ns, o)
+      @h = Integer(h)
+      @ts = Oplogjam::Timestamp(ts)
+      @ns = String(ns)
+      @o = Oplogjam::Document(o)
+    end
+
+    alias id h
+    alias namespace ns
+    alias query o
+
+    def timestamp
+      Time.at(ts.seconds, ts.increment)
+    end
+
+    def ==(other)
+      return false unless other.is_a?(Delete)
+
+      id == other.id
+    end
+
+    def apply(mapping)
+      table = mapping[namespace]
+      return unless table
+
+      row_id = query.fetch(ID).to_json
+
+      table
+        .where(id: row_id, deleted_at: nil)
+        .update(updated_at: Time.now.utc, deleted_at: Time.now.utc)
+    end
+  end
+end

data/lib/oplogjam/insert.rb

@@ -0,0 +1,62 @@
+module Oplogjam
+  InvalidInsert = Class.new(ArgumentError)
+
+  class Insert
+    attr_reader :h, :ts, :ns, :o
+
+    def self.from(bson)
+      h = bson.fetch(H)
+      ts = bson.fetch(TS)
+      ns = bson.fetch(NS)
+      o = bson.fetch(O)
+
+      new(h, ts, ns, o)
+    rescue KeyError => e
+      raise InvalidInsert, "missing field: #{e}"
+    end
+
+    def initialize(h, ts, ns, o)
+      @h = Integer(h)
+      @ts = Oplogjam::Timestamp(ts)
+      @ns = String(ns)
+      @o = Oplogjam::Document(o)
+    end
+
+    alias namespace ns
+    alias id h
+    alias document o
+
+    def timestamp
+      Time.at(ts.seconds, ts.increment)
+    end
+
+    def ==(other)
+      return false unless other.is_a?(Insert)
+
+      id == other.id
+    end
+
+    def apply(mapping)
+      table = mapping[namespace]
+      return unless table
+
+      row_id = document.fetch(ID).to_json
+
+      table
+        .insert_conflict(
+          target: :id,
+          conflict_where: { deleted_at: nil },
+          update: {
+            document: Sequel[:excluded][:document],
+            updated_at: Time.now.utc
+          }
+        )
+        .insert(
+          id: row_id,
+          document: Sequel.pg_jsonb(document),
+          created_at: Time.now.utc,
+          updated_at: Time.now.utc
+        )
+    end
+  end
+end

data/lib/oplogjam/noop.rb

@@ -0,0 +1,39 @@
+module Oplogjam
+  InvalidNoop = Class.new(ArgumentError)
+
+  class Noop
+    attr_reader :h, :ts, :msg
+
+    def self.from(bson)
+      h = bson.fetch(H)
+      ts = bson.fetch(TS)
+      o = bson.fetch(O)
+      msg = o.fetch(MSG)
+
+      new(h, ts, msg)
+    rescue KeyError => e
+      raise InvalidNoop, "missing field: #{e}"
+    end
+
+    def initialize(h, ts, msg)
+      @h = Integer(h)
+      @ts = Oplogjam::Timestamp(ts)
+      @msg = String(msg)
+    end
+
+    alias message msg
+    alias id h
+
+    def timestamp
+      Time.at(ts.seconds, ts.increment)
+    end
+
+    def ==(other)
+      return false unless other.is_a?(Noop)
+
+      id == other.id
+    end
+
+    def apply(_mapping); end
+  end
+end

data/lib/oplogjam/operation.rb

@@ -0,0 +1,31 @@
+require 'oplogjam/noop'
+require 'oplogjam/insert'
+require 'oplogjam/update'
+require 'oplogjam/delete'
+require 'oplogjam/command'
+require 'oplogjam/apply_ops'
+
+module Oplogjam
+  InvalidOperation = Class.new(ArgumentError)
+
+  class Operation
+    def self.from(bson)
+      op = bson.fetch(OP, UNKNOWN)
+
+      case op
+      when N then Noop.from(bson)
+      when I then Insert.from(bson)
+      when U then Update.from(bson)
+      when D then Delete.from(bson)
+      when C
+        if bson.fetch(O, {}).key?(APPLY_OPS)
+          ApplyOps.from(bson)
+        else
+          Command.from(bson)
+        end
+      else
+        raise InvalidOperation, "invalid operation: #{bson}"
+      end
+    end
+  end
+end

data/lib/oplogjam/operators.rb

@@ -0,0 +1,2 @@
+require 'oplogjam/operators/set'
+require 'oplogjam/operators/unset'

data/lib/oplogjam/operators/assignment.rb

@@ -0,0 +1,12 @@
+module Oplogjam
+  module Operators
+    class Assignment
+      attr_reader :path, :value
+
+      def initialize(path, value)
+        @path = path
+        @value = value
+      end
+    end
+  end
+end

data/lib/oplogjam/operators/field_assignment.rb

@@ -0,0 +1,11 @@
+require 'oplogjam/operators/assignment'
+
+module Oplogjam
+  module Operators
+    class FieldAssignment < Assignment
+      def update(column)
+        column.set(path, value.to_json)
+      end
+    end
+  end
+end

data/lib/oplogjam/operators/index_assignment.rb

@@ -0,0 +1,39 @@
+require 'oplogjam/operators/assignment'
+
+module Oplogjam
+  module Operators
+    class IndexAssignment < Assignment
+      def update(column)
+        # Now for a not-so-fun bit!
+        #
+        # As this is a numeric index, it might either be an index into an existing array or a numeric field name on an
+        # object.
+        #
+        # If it is an index into an array then we need to ensure that all prior indexes down to 0 are either set or null.
+        filled_array_column = (0...index).inject(column) { |subject, i|
+          prior_path = parent_path + [i.to_s]
+
+          subject.set(prior_path, Sequel.function(:coalesce, column[prior_path], NULL))
+        }
+
+        populated_column = Sequel.pg_jsonb_op(
+          Sequel.case(
+            { ARRAY_TYPE => filled_array_column },
+            column,
+            column[parent_path].typeof
+          )
+        )
+
+        populated_column.set(path, value.to_json)
+      end
+
+      def index
+        Integer(path.last, 10)
+      end
+
+      def parent_path
+        path[0...-1]
+      end
+    end
+  end
+end

data/lib/oplogjam/operators/intermediate.rb

@@ -0,0 +1,47 @@
+module Oplogjam
+  module Operators
+    class Intermediate
+      attr_reader :path, :tree
+
+      def initialize(path, tree = {})
+        @path = path
+        @tree = tree
+      end
+
+      def populate(path)
+        if path.last =~ NUMERIC_INDEX
+          populate_index(path)
+        else
+          populate_field(path)
+        end
+      end
+
+      def set(path, value)
+        if path.last =~ NUMERIC_INDEX
+          set_index(path, value)
+        else
+          set_field(path, value)
+        end
+      end
+
+      def populate_field(path)
+        tree[path] ||= IntermediateField.new(path)
+      end
+
+      def populate_index(path)
+        tree[path] ||= IntermediateIndex.new(path)
+      end
+      def set_field(path, value)
+        tree[path] = FieldAssignment.new(path, value)
+      end
+
+      def set_index(path, value)
+        tree[path] = IndexAssignment.new(path, value)
+      end
+
+      def nodes
+        tree.values
+      end
+    end
+  end
+end