mongoriver 0.3.1 → 0.4.0

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - "1.8.7"
+  - "1.9.3"
+  - "2.1.2"
+# command to run tests
+script: "bundle exec rake test"

data/README.md

@@ -1,4 +1,4 @@
-# Mongoriver
+# Mongoriver [![Build Status](https://travis-ci.org/stripe/mongoriver.svg?branch=master)](https://travis-ci.org/stripe/mongoriver)
 
 mongoriver is a library to monitor updates to your Mongo databases in
 near-realtime. It provides a simple interface for you to take actions
@@ -51,3 +51,10 @@ stream.run_forever(starting_timestamp)
 
 `starting_timestamp` here is the time you want the tailer to start at. We use
 this to resume interrupted tailers so that no information is lost.
+
+
+## Version history
+
+### 0.4
+
+Add support for [tokumx](http://www.tokutek.com/products/tokumx-for-mongodb/). Backwards incompatible changes to persistent tailers to accomodate that. See [UPGRADING.md](UPGRADING.md).

data/Rakefile

@@ -8,7 +8,10 @@ Rake::TestTask.new do |t|
 end
 
 Rake::TestTask.new(:'test-unit') do |t|
-  t.test_files = FileList['test/test_mongoriver.rb']
+  t.test_files = FileList[
+    'test/test_mongoriver.rb',
+    'test/test_toku.rb',
+    'test/test_persistent_tailers.rb']
 end
 
 Rake::TestTask.new(:'test-connected') do |t|

data/UPGRADING.md

@@ -0,0 +1,35 @@
+# Version 0.4
+
+## Outline
+
+This version adds support for tailing TokuMX, which uses a different oplog format from Mongo, separating the concepts of time and ordering of oplog entries. This sadly breaks most applications currently using mongoriver and will require invasive changes to persistant tailers.
+
+More specifically - TokuMX uses `BSON::Binary` instead of `BSON::Timestamp` for keeping track of the database oplog position.
+
+## Changes
+
+### Tailers
+
+Tailers gained a few new methods: `database_type` either returns `:toku` or `:mongo` based on which type of database is connected to the tailer.
+
+Tailers also have a `most_recent_position` method, which returns the most recent oplog position in the database. It has an optional `before_time` parameter of type `Time`, which if passed, finds the latest position before (or at) that time. It is useful for converting timestamps to positions to tail from in tailers.
+
+### Persistent tailers
+
+In persistent tailers, the `read_timestamp` and `write_timestamp` methods were replaced with `read_state` and `write_state`, which read/write state hashes from/to the database. `read_state` should return `nil` if not state has been stored in the database yet.
+
+In addition, there are convenience methods `read_timestamp` and `read_position` (already implemented) which extract the appropriate field from the `read_state` results. Both will return `nil` if `read_state` returns `nil`.
+
+## State hash
+
+State hashes have two fields - `'time'` which is a ruby Time object (used for humans, logs) and `'position'` of type `BSON::Binary` or `BSON::Timestamp` which the tailer uses to keep track of its position.
+
+## How to upgrade
+
+Persistent tailers need to define both `read_state` and `write_state` methods, which deserialize and serialize state hashes according to the database type.
+
+Also, calls to `read_timestamp` and other methods mentioned above will have to be updated.
+
+Also, if your application has custom filters for namespaces or [other things tokumx has updated](http://www.tokutek.com/2014/03/comparing-a-tokumx-and-mongodb-oplog-entry/), those need to be updated.
+
+Note that to do seamless upgrades, some more mangling might be needed (e.g. updating database tables, figuring out how to get the position from an existing timestamp). The following file should be a good example of what this might entail: https://github.com/stripe/mosql/blob/karl-mongoriver-support-for-tokumx/lib/mosql/tailer.rb 

data/bin/watch-oplog

@@ -47,7 +47,7 @@ def main
     end
 
     opts.on('-s OPTIME', '--start', 'Starting optime') do |optime|
-      options[:optime] = Integer(optime)
+      options[:optime] = Time.at(optime)
     end
   end
   optparse.parse!

data/lib/mongoriver.rb

@@ -8,6 +8,7 @@ require 'mongoriver/log'
 require 'mongoriver/assertions'
 
 require 'mongoriver/tailer'
+require 'mongoriver/toku'
 require 'mongoriver/abstract_persistent_tailer'
 require 'mongoriver/persistent_tailer'
 require 'mongoriver/abstract_outlet'

data/lib/mongoriver/abstract_persistent_tailer.rb

@@ -1,67 +1,109 @@
 module Mongoriver
 
   # A variant of Tailer that automatically loads and persists the
-  # "last timestamp processes" state. See PersistentTailer for a
+  # "last position processes" state. See PersistentTailer for a
   # concrete subclass that uses the same mongod you are already
   # tailing.
 
   class AbstractPersistentTailer < Tailer
+    attr_reader :last_saved, :last_read
+
     def initialize(upstream, type, opts={})
       raise "You can't instantiate an AbstractPersistentTailer -- did you want PersistentTailer? " if self.class == AbstractPersistentTailer
       super(upstream, type)
 
-      @last_saved       = nil
+      @last_saved       = {}
       @batch            = opts[:batch]
-      @last_read        = nil
+      @last_read        = {}
     end
 
     def tail(opts={})
-      opts[:from] ||= read_timestamp
+      opts[:from] ||= read_position
+      log.debug("Persistent tail options: #{opts}")
       super(opts)
     end
 
     def stream(limit=nil)
-      start_time = BSON::Timestamp.new(connection_config['localTime'].to_i, 0)
+      start_time = Time.at(connection_config['localTime'])
       found_entry = false
 
-      ret = super(limit) do |entry|
+      # Sketchy logic - yield results from Tailer.stream
+      # if nothing is found and nothing in cursor, save the current position
+      entries_left = super(limit) do |entry|
         yield entry
+
         found_entry = true
-        @last_read = entry['ts']
-        maybe_save_timestamp unless @batch
+        @last_read = state_for(entry)
+        maybe_save_state unless @batch
       end
 
-      if !found_entry && !ret
-        @last_read = start_time
-        maybe_save_timestamp unless @batch
+      if !found_entry && !entries_left
+        @last_read['time'] = start_time
+        maybe_save_state unless @batch
       end
 
-      return ret
+      return entries_left
+    end
+
+    # state to save to the database for this record
+    def state_for(record)
+      {
+        'time' => Time.at(record['ts'].seconds),
+        'position' => position(record)
+      }
     end
 
     def batch_done
       raise "You must specify :batch => true to use the batch-processing interface." unless @batch
-      maybe_save_timestamp
+      maybe_save_state
+    end
+
+    # Get the current state from storage. Implement this!
+    # @returns state [Hash, nil]
+    # @option state [BSON::Timestamp, BSON::Binary] 'position'
+    # @option state [Time] 'timestamp'
+    def read_state
+      raise "read_state unimplemented!"
     end
 
+    # Read the most recent timestamp of a read from storage.
+    # Return nil if nothing was found.
     def read_timestamp
-      raise "read_timestamp unimplemented!"
+      state = read_state || {}
+      return state['time']
     end
 
-    def write_timestamp
-      raise "save_timestamp unimplemented!"
+    # Read the most recent position from storage.
+    # Return nil if nothing was found.
+    def read_position
+      state = read_state || {}
+      return state['position']
     end
 
-    def save_timestamp
-      write_timestamp(@last_read)
-      @last_saved = @last_read
-      log.info("Saved timestamp: #{@last_saved} (#{Time.at(@last_saved.seconds)})")
+    # Persist current state. Implement this!
+    # @param state [Hash]
+    # @option state [BSON::Timestamp, BSON::Binary] 'position'
+    # @option state [Time] 'timestamp'
+    def write_state(state)
+      raise "write_state unimplemented!"
     end
 
-    def maybe_save_timestamp
-      # Write timestamps once a minute
-      return unless @last_read
-      save_timestamp if @last_saved.nil? || (@last_read.seconds - @last_saved.seconds) > 60
+    def save_state(state=nil)
+      if state.nil?
+        state = last_read
+      end
+      write_state(state)
+      @last_saved = state
+      log.info("Saved state: #{last_saved}")
+    end
+
+    def maybe_save_state
+      # Write position once a minute
+
+      return unless last_read['time']
+      if last_saved['time'].nil? || last_read['time'] - last_saved['time'] > 60.0
+        save_state
+      end
     end
   end
 end

data/lib/mongoriver/persistent_tailer.rb

@@ -4,7 +4,9 @@ module Mongoriver
   # tailing.
   class PersistentTailer < AbstractPersistentTailer
     def initialize(upstream, type, service, opts={})
-      raise "You can't use PersistentTailer against only a slave. How am I supposed to write state? " if type == :slave
+      if type == :slave
+        raise "You can't use PersistentTailer against only a slave. How am I supposed to write state?"
+      end
       super(upstream, type, opts)
 
       db         = opts[:db] || "_mongoriver"
@@ -13,18 +15,27 @@ module Mongoriver
       @state_collection = @upstream_conn.db(db).collection(collection)
     end
 
-    def read_timestamp
+    def read_state
       row = @state_collection.find_one(:service => @service)
-      row ? row['timestamp'] : BSON::Timestamp.new(0, 0)
-    end
+      return nil unless row
 
-    def write_timestamp(ts)
-      row = @state_collection.find_one(:service => @service)
-      if row
-        @state_collection.update({'_id' => row['_id']}, '$set' => { 'timestamp' => ts })
-      else
-        @state_collection.insert('service' => @service, 'timestamp' => ts)
+      # Try to do seamless upgrades from old mongoriver versions
+      case row['v']
+      when nil
+        log.warn("Old style timestamp found in database. Converting!")
+        ts = Time.at(row['timestamp'].seconds)
+        return {
+          'position' => most_recent_position(ts),
+          'time' => ts
+        }
+      when 1
+        return row['state']
       end
     end
+
+    def write_state(state)
+      @state_collection.update({:service => @service},
+        {:service => @service, :state => state, :v => 1}, :upsert => true)
+    end
   end
 end

data/lib/mongoriver/stream.rb

@@ -21,12 +21,14 @@ module Mongoriver
       @stats
     end
 
-    def run_forever(starting_timestamp=nil)
-      if starting_timestamp
-        @tailer.tail(:from => optime_from_ts(starting_timestamp))
-      else
-        @tailer.tail
+    # @param position [BSON::Timestamp, BSON::Binary, Time] position to start
+    #             following the oplog from. @see Tailer#most_recent_position
+    def run_forever(position=nil)
+      if position.is_a?(Time)
+        position = @tailer.most_recent_position(position)
       end
+      log.debug("Start position: #{position.inspect}")
+      @tailer.tail(:from => position)
 
       until @stop
         @tailer.stream do |op|
@@ -42,19 +44,6 @@ module Mongoriver
 
     private
 
-    def optime_from_ts(timestamp)
-      if timestamp.is_a?(Integer)
-        if timestamp >= 0
-          BSON::Timestamp.new(timestamp, 0)
-        else
-          raise "Invalid optime: #{timestamp}"
-        end
-      else
-        raise "Unrecognized type #{timestamp.class} (#{timestamp.inspect}) " \
-              "for start_timestamp"
-      end
-    end
-
     def trigger(name, *args)
       signature = "#{name}(" + args.map { |arg| arg.inspect }.join(', ') + ")"
       log.debug("triggering #{signature}")
@@ -114,13 +103,13 @@ module Mongoriver
 
     def handle_create_index(spec)
       db_name, collection_name = parse_ns(spec['ns'])
-      index_key = spec['key'].map { |field, dir|
+      index_key = spec['key'].map do |field, dir|
         if dir.is_a?(Numeric)
           [field, dir.round]
         else
           [field, dir]
         end
-      }
+      end
       options = {}
 
       spec.each do |key, value|
@@ -147,7 +136,7 @@ module Mongoriver
         index_name = data['index']
         trigger(:drop_index, db_name, deleted_from_collection, index_name)
       elsif created_collection = data['create']
-        handle_create_collection(db_name, data)
+        handle_create_collection(db_name, created_collection, data)
       elsif dropped_collection = data['drop']
         trigger(:drop_collection, db_name, dropped_collection)
       elsif old_collection_ns = data['renameCollection']
@@ -161,12 +150,11 @@ module Mongoriver
       end
     end
 
-    def handle_create_collection(db_name, data)
-      collection_name = data.delete('create')
-
+    def handle_create_collection(db_name, collection_name, data)
       options = {}
-      data.each do |k, v|
-        options[k.to_sym] = (k == 'size') ? v.round : v
+      data.each do |key, value|
+        next if key == 'create'
+        options[key.to_sym] = (key == 'size') ? value.round : value
       end
 
       trigger(:create_collection, db_name, collection_name, options)

data/lib/mongoriver/tailer.rb

@@ -1,9 +1,11 @@
 module Mongoriver
   class Tailer
     include Mongoriver::Logging
+    include Mongoriver::Assertions
 
     attr_reader :upstream_conn
     attr_reader :oplog
+    attr_reader :database_type
 
     def initialize(upstreams, type, oplog = "oplog.rs")
       @upstreams = upstreams
@@ -14,13 +16,53 @@ module Mongoriver
 
       @cursor = nil
       @stop = false
+      @streaming = false
 
       connect_upstream
+      @database_type = Mongoriver::Toku.conversion_needed?(@upstream_conn) ? :toku : :mongo
     end
 
-    def most_recent_timestamp
-      record = oplog_collection.find_one({}, :sort => [['$natural', -1]])
-      record['ts']
+    # Return a position for a record object
+    #
+    # @return [BSON::Timestamp] if mongo
+    # @return [BSON::Binary] if tokumx
+    def position(record)
+      return nil unless record
+      case database_type
+      when :mongo
+        return record['ts']
+      when :toku
+        return record['_id']
+      end
+    end
+
+    # Find the most recent entry in oplog and return a position for that
+    # position. The position can be passed to the tail function (or run_forever)
+    # and the tailer will start tailing after that.
+    # If before_time is given, it will return the latest position before (or at) time.
+    def most_recent_position(before_time=nil)
+      position(latest_oplog_entry(before_time))
+    end
+
+    def latest_oplog_entry(before_time=nil)
+      query = {}
+      if before_time
+        case database_type
+        when :mongo
+          ts = BSON::Timestamp.new(before_time.to_i + 1, 0)
+        when :toku
+          ts = before_time + 1
+        end
+        query = { 'ts' => { '$lt' => ts } }
+      end
+
+      case database_type
+      when :mongo
+        record = oplog_collection.find_one(query, :sort => [['$natural', -1]])
+      when :toku
+        record = oplog_collection.find_one(query, :sort => [['_id', -1]])
+      end
+      record
     end
 
     def connect_upstream
@@ -71,14 +113,18 @@ module Mongoriver
       @upstream_conn.db('local').collection(oplog)
     end
 
+    # Start tailing the oplog.
+    # 
+    # @param [Hash]
+    # @option opts [BSON::Timestamp, BSON::Binary] :from Placeholder indicating 
+    #           where to start the query from. Binary value is used for tokumx.
+    #           The timestamp is non-inclusive.
+    # @option opts [Hash] :filter Extra filters for the query.
+    # @option opts [Bool] :dont_wait(false) 
     def tail(opts = {})
       raise "Already tailing the oplog!" if @cursor
 
-      query = opts[:filter] || {}
-      if ts = opts[:from]
-        # Maybe if ts is old enough, just start from the beginning?
-        query['ts'] = { '$gte' => ts }
-      end
+      query = build_tail_query(opts)
 
       mongo_opts = {:timeout => false}.merge(opts[:mongo_opts] || {})
 
@@ -87,7 +133,7 @@ module Mongoriver
         oplog.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY) if query['ts']
         oplog.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA) unless opts[:dont_wait]
 
-        log.info("Starting oplog stream from #{ts || 'start'}")
+        log.debug("Starting oplog stream from #{opts[:from] || 'start'}")
         @cursor = oplog
       end
     end
@@ -98,14 +144,28 @@ module Mongoriver
       tail(opts)
     end
 
-    def stream(limit=nil)
+    def tailing
+      !@stop || @streaming
+    end
+
+    def stream(limit=nil, &blk)
       count = 0
+      @streaming = true
       while !@stop && @cursor.has_next?
         count += 1
         break if limit && count >= limit
 
-        yield @cursor.next
+        record = @cursor.next
+
+        case database_type
+        when :mongo
+          blk.call(record)
+        when :toku
+          converted = Toku.convert(record, @upstream_conn)
+          converted.each(&blk)
+        end
       end
+      @streaming = false
 
       return @cursor.has_next?
     end
@@ -119,5 +179,24 @@ module Mongoriver
       @cursor = nil
       @stop = false
     end
+
+    private
+    def build_tail_query(opts = {})
+      query = opts[:filter] || {}
+      return query unless opts[:from]
+
+      case database_type
+      when :mongo
+        assert(opts[:from].is_a?(BSON::Timestamp),
+          "For mongo databases, tail :from must be a BSON::Timestamp")
+        query['ts'] = { '$gt' => opts[:from] }
+      when :toku
+        assert(opts[:from].is_a?(BSON::Binary),
+          "For tokumx databases, tail :from must be a BSON::Binary")
+        query['_id'] = { '$gt' => opts[:from] }
+      end
+
+      query
+    end
   end
 end

data/lib/mongoriver/toku.rb

@@ -0,0 +1,123 @@
+module Mongoriver
+  # This module deals with converting TokuMX oplog records into mongodb oplogs. 
+  class Toku
+    # @returns true if conn is a TokuMX database and the oplog records need to 
+    #               be converted 
+    def self.conversion_needed?(conn)
+      conn.server_info.has_key? "tokumxVersion"
+    end
+
+    def self.operations_for(record, conn=nil)
+      if record["ops"]
+        return record["ops"]
+      end
+      refs_coll = conn.db('local').collection('oplog.refs')
+      mongo_opts = {:sort => [['seq', 1]], :timeout => false}
+
+      refs = refs_coll.find({"_id.oid" => record['ref']}, mongo_opts).to_a
+      refs.map { |r| r["ops"] }.flatten
+    end
+
+    # Convert hash representing a tokumx oplog record to mongodb oplog records.
+    # 
+    # Things to note:
+    #   1) Unlike mongo oplog, the timestamps will not be monotonically
+    #      increasing
+    #   2) h fields (unique ids) will also not be unique on multi-updates
+    #   3) operations marked by 'n' toku are ignored, as these are non-ops
+    # @see http://www.tokutek.com/2014/03/comparing-a-tokumx-and-mongodb-oplog-entry/
+    # @returns Array<Hash> List of mongodb oplog records.
+    def self.convert(record, conn=nil)
+      result = []
+      operations_for(record, conn).each do |operation|
+        case operation["op"]
+        when 'i'
+          result << insert_record(operation, record)
+        when 'ur'
+          result << update_record(operation, record, true)
+        when 'u'
+          result << update_record(operation, record, false)
+        when 'c'
+          result << command_record(operation, record)
+        when 'd'
+          result << remove_record(operation, record)
+        when 'n'
+          # keepOplogAlive requests - safe to ignore
+        else
+          raise "Unrecognized op: #{operation["op"]} (#{record.inspect})"
+        end
+      end
+
+      result
+    end
+
+    def self.timestamp(full_record)
+      # Note that this loses the monotonically increasing property, if not
+      # lost before.
+      BSON::Timestamp.new(full_record["ts"].to_i, 0)
+    end
+
+    def self.insert_record(operation, full_record)
+      {
+        "_id" => full_record["_id"],
+        # Monotonically increasing timestamp in mongodb in oplog.
+        # e.g. <BSON::Timestamp:0x000000100a81c8 @increment=1, @seconds=1408995488>
+        "ts" => timestamp(full_record),
+        # Unique ID for this operation
+        # Note that not so unique anymore
+        "h" => full_record["h"],
+        # Ignoring v ("version") for now
+        # "v" => 1,
+        "op" => "i",
+        # namespace being updated. in form of database-name.collection.name
+        "ns" => operation["ns"],
+        # operation being done. 
+        # e.g. {"_id"=>BSON::ObjectId('53fb8f6b9e126b2106000003')}
+        "o" => operation["o"]
+      }
+    end
+
+    def self.remove_record(operation, full_record)
+      {
+        "_id" => full_record["_id"],
+        "ts" => timestamp(full_record),
+        "h" => full_record["h"],
+        # "v" => 1,
+        "op" => "d",
+        "ns" => operation["ns"],
+        # "b" => true, # TODO: what does this signify?
+        "o" => { "_id" => operation["o"]["_id"] }
+      }
+    end
+
+    def self.command_record(operation, full_record)
+      {
+        "_id" => full_record["_id"],
+        "ts" => timestamp(full_record),
+        "h" => full_record["h"],
+        # "v" => 1,
+        "op" => "c",
+        "ns" => operation["ns"],
+        "o" => operation["o"]
+      }
+    end
+
+
+    def self.update_record(operation, full_record, is_ur_record)
+      ({
+        "_id" => full_record["_id"],
+        "ts" => timestamp(full_record),
+        "h" => full_record["h"],
+        # "v" => 1,
+        "op" => "u",
+        "ns" => operation["ns"],
+        # { _id: BSON::ObjectId } what object was updated
+        "o2" => { "_id" => operation["o"]["_id"] },
+        "o" => operation[is_ur_record ? "m" : "o2"]
+      })
+    end
+
+    private_class_method :timestamp, :insert_record, :update_record
+    private_class_method :remove_record, :command_record
+  end
+end

data/lib/mongoriver/version.rb

@@ -1,3 +1,3 @@
 module Mongoriver
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

data/test/test_mongoriver.rb

@@ -10,7 +10,7 @@ describe 'Mongoriver::Stream' do
   end
 
   before do
-    conn = stub(:db => nil)
+    conn = stub(:db => nil, :server_info => {})
     @tailer = Mongoriver::Tailer.new([conn], :existing)
     @outlet = Mongoriver::AbstractOutlet.new
     @stream = Mongoriver::Stream.new(@tailer, @outlet)

data/test/test_mongoriver_connected.rb

@@ -16,6 +16,12 @@ def connect
   end
 end
 
+def run_stream(stream, start)
+  Timeout::timeout(30) do
+    @stream.run_forever(start)
+  end
+end
+
 describe 'connected tests' do
   before do
     @mongo = connect
@@ -29,13 +35,12 @@ describe 'connected tests' do
 
       @stream = Mongoriver::Stream.new(@tailer, @outlet)
 
-      @tail_from = @tailer.most_recent_timestamp.seconds + 1
-      sleep(1)
+      @tail_from = @tailer.most_recent_position
     end
 
     it 'triggers the correct ops in the correct order' do
-      db = 'test'
-      collection = 'test'
+      db = '_test_mongoriver'
+      collection = '_test_mongoriver'
       doc = {'_id' => 'foo', 'bar' => 'baz'}
       updated_doc = doc.dup.merge('bar' => 'qux')
       index_keys = [['bar', 1]]
@@ -67,17 +72,47 @@ describe 'connected tests' do
       @mongo[db].drop_collection(collection+'_foo')
       @mongo.drop_database(db)
 
-      @stream.run_forever(@tail_from)
+      run_stream(@stream, @tail_from)
     end
 
     it 'passes options to create_collection' do
-      @outlet.expects(:create_collection).once.with('test', 'test', {:capped => true, :size => 10}) { @stream.stop }
+      @outlet.expects(:create_collection).once.with('_test_mongoriver', '_test_mongoriver', {:capped => true, :size => 10}) { @stream.stop }
       @outlet.expects(:update_optime).at_least_once.with(anything) { @stream.stop }
 
-      @mongo['test'].create_collection('test', :capped => true, :size => 10)
-      @mongo.drop_database('test')
+      @mongo['_test_mongoriver'].create_collection('_test_mongoriver', :capped => true, :size => 10)
+      @mongo.drop_database('_test_mongoriver')
+
+      run_stream(@stream, @tail_from)
+    end
+
+    it 'ignores everything before the operation passed in' do
+      name = '_test_mongoriver'
+
+      @mongo[name][name].insert(:a => 5)
+
+      @outlet.expects(:insert).never
+      @outlet.expects(:drop_database).with(anything) { @stream.stop }
+
+      start = @tailer.most_recent_position
+      @mongo.drop_database(name)
+      run_stream(@stream, start)
+    end
+
+    it 'allows passing in a timestamp for the stream following as well' do
+      name = '_test_mongoriver2'
+
+      @outlet.expects(:insert).with do |db_name, col_name, value|
+        db_name != name || value['record'] == 'value'
+      end
+
+      @outlet.expects(:update).with do |db_name, col_name, selector, update|
+        @stream.stop if update['record'] == 'newvalue'
+        db_name != name || update['record'] == 'newvalue'
+      end
 
-      @stream.run_forever(@tail_from)
+      @mongo[name][name].insert('record' => 'value')
+      @mongo[name][name].update({'record' => 'value'}, {'record' => 'newvalue'})
+      run_stream(@stream, Time.now-3)
     end
   end
 end

data/test/test_persistent_tailers.rb

@@ -0,0 +1,74 @@
+require 'mongoriver'
+require 'mongo'
+require 'minitest/autorun'
+require 'mocha/setup'
+
+
+def mocked_mongo()
+  mongo_connection = stub()
+  db = stub()
+  collection = stub()
+
+  mongo_connection.expects(:db).with('_mongoriver').returns(db)
+  db.expects(:collection).with('oplog-tailers').returns(collection)
+
+  # mongodb
+  mongo_connection.expects(:server_info).at_least_once.returns({})
+
+  [mongo_connection, collection]
+end
+
+describe 'Mongoriver::PersistentTailer' do
+  before do
+    @service_name = "_persistenttailer_test"
+
+    db, @state_collection = mocked_mongo
+
+    @tailer = Mongoriver::PersistentTailer.new(
+      [db], :existing, @service_name)
+    @state = {
+      'time' => Time.now,
+      'position' => 'foobar'
+    }
+  end
+
+  describe 'reading and storing state' do
+    it 'should be able to read the written state' do
+
+      @state_collection.expects(:update)
+      @tailer.save_state(@state)
+
+      @state_collection.expects(:find_one).returns({
+        'state' => @state,
+        'v' => 1
+      })
+      assert_equal(@state, @tailer.read_state)
+    end
+
+    it 'should update gracefully' do
+      ts = BSON::Timestamp.new(77, 0)
+
+      @state_collection.expects(:find_one).returns('timestamp' => ts)
+      @tailer.expects(:most_recent_position).with(Time.at(77))
+
+      assert_equal(Time.at(77), @tailer.read_state['time'])
+    end
+  end
+
+  it 'helper methods for timestamps/positions' do
+    @state_collection.expects(:find_one).returns({
+      'state' => @state,
+      'v' => 1
+    }).at_least_once
+
+    assert_equal(@state['time'], @tailer.read_timestamp)
+    assert_equal(@state['position'], @tailer.read_position)
+  end
+
+  it 'should tail from position' do
+    @tailer.expects(:read_position).returns('foo')
+    Mongoriver::Tailer.any_instance.expects(:tail).with(:from => 'foo')
+
+    @tailer.tail
+  end
+end

data/test/test_toku.rb

@@ -0,0 +1,127 @@
+require 'mongoriver'
+require 'mongo'
+require 'minitest/autorun'
+require 'mocha/setup'
+
+describe 'Mongoriver::Toku' do
+  def create_op(ops)
+    ts = Time.now
+    ops = ops.map { |op| op['ns'] ||= 'foo.bar'; op }
+    {
+      '_id' => BSON::Binary.new,
+      'ts' => ts,
+      'h' => 1234,
+      'a' => true,
+      'ops' => ops
+    }
+  end
+
+  def convert(*ops)
+    record = create_op(ops)
+    result = Mongoriver::Toku.convert(record)
+    assert_equal(result.length, 1)
+    result.first
+  end
+
+  describe 'conversions sent to stream' do
+    before do
+      conn = stub(:db => nil, :server_info => {'tokumxVersion' => '1'})
+      @tailer = Mongoriver::Tailer.new([conn], :existing)
+      @outlet = Mongoriver::AbstractOutlet.new
+      @stream = Mongoriver::Stream.new(@tailer, @outlet)
+
+      @outlet.expects(:update_optime).at_least_once
+    end
+
+    it 'triggers insert' do
+      # collection.insert({a: 5})
+      @outlet.expects(:insert).once.with('foo', 'bar', {'_id' => 'baz', 'a' => 5})
+      @stream.send(:handle_op, convert({'op'=>'i', 'o'=>{'_id'=>'baz', 'a' => 5}}))
+    end
+
+    it 'triggers update (ur)' do
+      # collection.update({a:true}, {'$set' => {c: 7}}, multi: true)
+      @outlet.expects(:update).once.with('foo', 'bar', {'_id' => 'baz'}, {'$set' => {'c' => 7}})
+
+      @stream.send(:handle_op, convert({
+        'op' => 'ur',
+        'pk' => {'' => 'baz'},
+        'o' => {'_id' => 'baz', 'a' => true},
+        'm' => {'$set' => {'c' => 7}}
+      }))
+    end
+
+    it 'triggers update (u)' do
+      # collection.update({a:true}, {'b' => 2})
+      @outlet.expects(:update).once.with('foo', 'bar', {'_id' => 'baz'}, {'_id' => 'baz', 'b' => 2})
+
+      @stream.send(:handle_op, convert({
+        'op' => 'u',
+        'pk' => {'' => 'baz'},
+        'o' => {'_id' => 'baz', 'a' => true},
+        'o2' => {'_id' => 'baz', 'b' => 2}
+      }))
+    end
+
+    it 'triggers remove' do
+      # collection.delete({a:5})
+      @outlet.expects(:remove).once.with('foo', 'bar', {'_id' => 'baz'})
+
+      @stream.send(:handle_op, convert({
+        'op' => 'd',
+        'o' => {'_id' => 'baz', 'a' => 5}
+      }))
+    end
+
+    it 'triggers create_index' do
+      # collection.create_index([["baz", 1]])
+      @outlet.expects(:create_index).once.with('foo', 'bar', [['baz', 1]], {:name => 'baz_1'})
+
+      @stream.send(:handle_op, convert({
+        'op' => 'i',
+        'ns' => 'foo.system.indexes',
+        'o' => {'key' => {"baz" => 1}, 'ns' => 'foo.bar', 'name' => 'baz_1'}
+      }))
+    end
+
+    it 'triggers commands (create_collection)' do
+      # db.create_collection('bar', :capped => true, :size => 10)
+      @outlet.expects(:create_collection).once.with('foo', 'bar', {:capped => true, :size => 10})
+
+      @stream.send(:handle_op, convert({
+        'op' => 'c',
+        'ns' => 'foo.$cmd',
+        'o' => {'create' => 'bar', 'capped' => true, 'size' => 10}
+      }))
+    end 
+  end
+
+  describe 'large transactions are joined by convert' do
+    it 'should yield the same result as separate ops' do
+      operations = [
+        {'op'=>'i', 'o'=>{'_id'=>'baz', 'a' => 5}},
+        {'op'=>'i', 'o'=>{'_id'=>'zoo', 'b' => 6}}
+      ]
+
+      refs = [
+        {'_id' => {'_oid' => 'refref'}, 'ops' => [operations[0]]},
+        {'_id' => {'_oid' => 'refref'}, 'ops' => [operations[1]]}
+      ]
+
+      collection = stub()
+      conn = stub(:db => stub(:collection => collection))
+      collection.expects(:find).returns(refs)
+
+      expected = Mongoriver::Toku.convert(create_op(operations))
+      got = Mongoriver::Toku.convert({
+        '_id' => BSON::Binary.new,
+        'ts' => Time.now,
+        'h' => 1234,
+        'a' => true,
+        'ref' => 'refref'
+      }, conn)
+
+      assert_equal(expected, got)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoriver
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-10 00:00:00.000000000 Z
+date: 2014-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
@@ -116,10 +116,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
+- UPGRADING.md
 - bin/watch-oplog
 - lib/mongoriver.rb
 - lib/mongoriver/abstract_outlet.rb
@@ -129,10 +131,13 @@ files:
 - lib/mongoriver/persistent_tailer.rb
 - lib/mongoriver/stream.rb
 - lib/mongoriver/tailer.rb
+- lib/mongoriver/toku.rb
 - lib/mongoriver/version.rb
 - mongoriver.gemspec
 - test/test_mongoriver.rb
 - test/test_mongoriver_connected.rb
+- test/test_persistent_tailers.rb
+- test/test_toku.rb
 homepage: ''
 licenses: []
 post_install_message: 
@@ -160,4 +165,6 @@ summary: monogdb oplog-tailing utilities.
 test_files:
 - test/test_mongoriver.rb
 - test/test_mongoriver_connected.rb
+- test/test_persistent_tailers.rb
+- test/test_toku.rb
 has_rdoc: