mongoriver 0.1.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Greg Brockman
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Mongoriver
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'mongoriver'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mongoriver
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/mongocp

@@ -0,0 +1,250 @@
+#!/usr/bin/env ruby
+require 'logger'
+require 'optparse'
+
+require 'rubygems'
+require 'bundler/setup'
+require 'mongoriver'
+
+module Mongoriver
+  class Mongocp < Streambed
+    include Mongoriver::Logging
+
+    def initialize(upstreams, type, downstream, prefix)
+      super(upstreams, type)
+      @downstream = downstream
+      @prefix = prefix
+      connect_downstream
+    end
+
+    def hook_optime
+      if optime = optime_collection.find_one(:_id => @prefix)
+        optime['ts']
+      else
+        nil
+      end
+    end
+
+    def hook_update_optime(ts, mandatory)
+      optime_collection.update({:_id => @prefix}, {'$set' => {:ts => ts}}, :upsert => true) if mandatory || rand(20) == 0
+    end
+
+    def hook_initial_sync_index(db_name, collection_name, index_key, options)
+      collection = downstream_collection(db_name, collection_name)
+      index_hash = BSON::OrderedHash.new
+      index_key.each {|k,v| index_hash[k] = v}
+      collection.send(:generate_indexes, index_hash, nil, options)
+    end
+
+    def hook_initial_sync_record_batch(db_name, collection_name, records)
+      collection = downstream_collection(db_name, collection_name)
+      bulk_insert(collection, records)
+    end
+
+    # TODO: should probably do the same key checking nonsense as the above
+    def hook_stream_insert(db_name, collection_name, object)
+      collection = downstream_collection(db_name, collection_name)
+      wrap_errors(collection, object['_id']) do
+        # Only needed if safe mode is set in the driver. Note that the
+        # argument here for oplog idempotency in the case of unique
+        # keys is kind of interesting. I believe I can prove
+        # idempotency as long as Mongo has no insert order-dependent
+        # unique indexes (which I believe is true) and that you do all
+        # your object updates as upserts.
+        allow_dupkeys do
+          collection.insert(object)
+        end
+      end
+    end
+
+    def hook_stream_update(db_name, collection_name, selector, update)
+      collection = downstream_collection(db_name, collection_name)
+      wrap_errors(collection, selector['_id']) do
+        collection.update(selector, update, :upsert => true)
+      end
+    end
+
+    def hook_stream_remove(db_name, collection_name, object)
+      collection = downstream_collection(db_name, collection_name)
+      wrap_errors(collection, object['_id']) do
+        collection.remove(object)
+      end
+    end
+
+    def hook_stream_create_collection(db_name, create)
+      db = downstream_db(db_name)
+      wrap_errors(db, create) do
+        db.create_collection(create)
+      end
+    end
+
+    # "Error renaming collection: #<BSON::OrderedHash:0x83869e34 {\"errmsg\"=>\"exception: source namespace does not exist\", \"code\"=>10026, \"ok\"=>0.0}>"
+    #
+    # Possibly need the same thing if the destination already exists
+    def hook_stream_rename_collection(db_name, source, target)
+      db = downstream_db(db_name)
+      wrap_errors(db, "#{source} -> #{target}") do
+        begin
+          db.rename_collection(source, target)
+        rescue Mongo::MongoDBError => e
+          if e.message =~ /Error renaming collection: .*exception: source namespace does not exist"/
+            log.warn("Ignoring rename of non-existent collection #{source} -> #{target}: #{e} (expected when replaying part of the oplog)")
+          elsif e.message =~ /Error renaming collection: .*exception: target namespace exists"/
+            log.warn("Ignoring rename of #{source} to existing collection #{target}: #{e} (expected when replaying part of the oplog)")
+          else
+            raise
+          end
+        end
+      end
+    end
+
+    def hook_stream_drop_index(db_name, collection_name, index_name)
+      collection = downstream_collection(db_name, collection_name)
+      wrap_errors(collection, index_name) do
+        begin
+          collection.drop_index(index_name)
+        rescue Mongo::MongoDBError => e
+          raise
+          if e.message =~ /index not found/
+            log.warn("Ignoring drop of non-existent index #{index_name.inspect}: #{e} (expected when replaying part of the oplog)")
+          else
+            raise
+          end
+        end
+      end
+    end
+
+    def hook_stream_drop_collection(db_name, dropped)
+      db = downstream_db(db_name)
+      wrap_errors(db, dropped) do
+        db.drop_collection(dropped)
+      end
+    end
+
+    def hook_stream_drop_database(db_name)
+      db = downstream_db(db_name)
+      wrap_errors(db, db_name) do
+        db.command(:dropDatabase => 1)
+      end
+    end
+
+    private
+
+    def allow_dupkeys(&blk)
+      begin
+        blk.call
+      rescue Mongo::OperationFailure => e
+        if e.error_code == 11000
+          log.warn("Ignoring unique index violation: #{e} (expected when replaying part of the oplog)")
+        else
+          raise
+        end
+      end
+    end
+
+    def bulk_insert(collection, docs)
+      begin
+        # Use the internal insert_documents method because it lets us
+        # disable key verification
+        collection.send(:insert_documents, docs, collection.name, false)
+      rescue Mongo::MongoRubyError => e
+        log.error("#{ns}: Caught error on batch insert", e)
+        docs.each do |doc|
+          wrap_errors(collection, doc['_id']) do
+            collection.send(:insert_documents, [doc], collection.name, false)
+          end
+        end
+      end
+    end
+
+    def wrap_errors(collection_or_db, object, &blk)
+      begin
+        blk.call
+      rescue Mongo::MongoRubyError => e
+        if collecton_or_db.kind_of?(Mongo::Collection)
+          ns = "#{collection_or_db.db.name}.#{collection_or_db.name}"
+        else
+          ns = collection_or_db.db.name
+        end
+        log.error("#{ns}: Unknown error for #{object}", e)
+      end
+    end
+
+    def downstream_db(db_name)
+      prefixed = "#{@prefix}_#{db_name}"
+      @downstream_conn.db(prefixed)
+    end
+
+    def downstream_collection(db_name, collection_name)
+      downstream_db(db_name).collection(collection_name)
+    end
+
+    def optime_collection
+      @optime_collection ||= @downstream_conn.db('_mongocp').collection('optime')
+    end
+
+    def connect_downstream
+      host, port = @tailer.parse_host_spec(@downstream)
+      @downstream_conn = Mongo::Connection.new(host, port, :safe => true)
+    end
+  end
+end
+
+def main
+  options = {:host => nil, :port => nil, :type => :slave, :verbose => 0}
+  optparse = OptionParser.new do |opts|
+    opts.banner = "Usage: #{$0} [options]"
+
+    opts.on('-v', '--verbosity', 'Verbosity of debugging output') do
+      options[:verbose] += 1
+    end
+
+    opts.on('-h', '--help', 'Display this message') do
+      puts opts
+      exit(1)
+    end
+
+    opts.on('--help', 'Display this message') do
+      puts opts
+      exit(1)
+    end
+
+    opts.on('-h HOST', '--host', 'Upstream host to connect to') do |host|
+      options[:host] = host
+    end
+
+    opts.on('-p PORT', '--port', 'Upstream host to connect to') do |port|
+      options[:port] = Integer(port)
+    end
+
+    opts.on('-a', '--all', 'Allow connections even directly to a primary') do
+      options[:type] = :direct
+    end
+  end
+  optparse.parse!
+
+  if ARGV.length != 0
+    puts optparse
+    return 1
+  end
+
+  log = Log4r::Logger.new('Stripe')
+  log.outputters = Log4r::StdoutOutputter.new(STDERR)
+  if options[:verbose] >= 1
+    log.level = Log4r::DEBUG
+  else
+    log.level = Log4r::INFO
+  end
+  runner = Mongoriver::Mongocp.new(["#{options[:host]}:#{options[:port]}"], options[:type], 'localhost:5001', 'test')
+  runner.run
+  return 0
+end
+
+if $0 == __FILE__
+  ret = main
+  begin
+    exit(ret)
+  rescue TypeError
+    exit(0)
+  end
+end

data/bin/optail

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+require 'logger'
+require 'optparse'
+
+require 'rubygems'
+require 'bundler/setup'
+require 'mongoriver'
+
+module Mongoriver
+  class Mongocp < Streambed
+    include Mongoriver::Logging
+
+    def initialize(upstreams, type, start_optime, pause)
+      super(upstreams, type)
+      @start_optime = start_optime
+      @pause = pause
+    end
+
+    def pause
+      if @pause
+        $stderr.puts("Press enter to continue")
+        $stdin.readline
+      end
+    end
+
+    def hook_optime
+      @start_optime
+    end
+
+    def hook_update_optime(ts, mandatory)
+    end
+
+    all_hooks.each do |name, _, opts|
+      next if name == :optime || name == :update_optime
+      define_method(hook_name(name)) {|*args| pause}
+    end
+  end
+end
+
+def main
+  options = {:host => nil, :port => nil, :type => :slave, :optime => 0, :pause => true, :verbose => 0}
+  optparse = OptionParser.new do |opts|
+    opts.banner = "Usage: #{$0} [options]"
+
+    opts.on('-v', '--verbosity', 'Verbosity of debugging output') do
+      options[:verbose] += 1
+    end
+
+    opts.on('--help', 'Display this message') do
+      puts opts
+      exit(1)
+    end
+
+    opts.on('-h HOST', '--host', 'Upstream host to connect to') do |host|
+      options[:host] = host
+    end
+
+    opts.on('-p PORT', '--port', 'Upstream host to connect to') do |port|
+      options[:port] = Integer(port)
+    end
+
+    opts.on('-a', '--all', 'Allow connections even directly to a primary') do
+      options[:type] = :direct
+    end
+
+    opts.on('-s OPTIME', '--start', 'Starting optime') do |optime|
+      options[:optime] = Integer(optime)
+    end
+
+    opts.on('-f', '--follow-automatically', "Don't prompt between ops") do
+      options[:pause] = false
+    end
+  end
+  optparse.parse!
+
+  if ARGV.length != 0
+    puts optparse
+    return 1
+  end
+
+  log = Log4r::Logger.new('Stripe')
+  log.outputters = Log4r::StdoutOutputter.new(STDERR)
+  if options[:verbose] >= 1
+    log.level = Log4r::DEBUG
+  else
+    log.level = Log4r::INFO
+  end
+
+  runner = Mongoriver::Mongocp.new(["#{options[:host]}:#{options[:port]}"], options[:type], options[:optime], options[:pause])
+  runner.run
+  return 0
+end
+
+if $0 == __FILE__
+  ret = main
+  begin
+    exit(ret)
+  rescue TypeError
+    exit(0)
+  end
+end

data/lib/mongoriver/abstract_persistent_tailer.rb

@@ -0,0 +1,58 @@
+module Mongoriver
+
+  # A variant of Tailer that automatically loads and persists the
+  # "last timestamp processes" state. See PersistentTailer for a
+  # concrete subclass that uses the same mongod you are already
+  # tailing.
+
+  class AbstractPersistentTailer < Tailer
+    def initialize(upstream, type, opts={})
+      raise "You can't instantiate an AbstractPersistentTailer -- did you want PersistentTailer? " if self.class == AbstractPersistentTailer
+      super(upstream, type)
+
+      @last_saved       = nil
+      @batch            = opts[:batch]
+      @last_read        = nil
+    end
+
+    def tail_from(ts, opts={})
+      if ts.nil?
+        ts = read_timestamp
+      end
+      super(ts, opts)
+    end
+
+    def stream(limit=nil)
+      super(limit) do |entry|
+        yield entry
+        @last_read = entry['ts']
+        maybe_save_timestamp unless @batch
+      end
+    end
+
+    def batch_done
+      raise "You must specify :batch => true to use the batch-processing interface." unless @batch
+      maybe_save_timestamp
+    end
+
+    def read_timestamp
+      raise "read_timestamp unimplemented!"
+    end
+
+    def write_timestamp
+      raise "save_timestamp unimplemented!"
+    end
+
+    def save_timestamp
+      write_timestamp(@last_read)
+      @last_saved = @last_read
+      log.info("Saved timestamp: #{@last_saved} (#{Time.at(@last_saved.seconds)})")
+    end
+
+    def maybe_save_timestamp
+      # Write timestamps once a minute
+      return unless @last_read
+      save_timestamp if @last_saved.nil? || (@last_read.seconds - @last_saved.seconds) > 60
+    end
+  end
+end

data/lib/mongoriver/log.rb

@@ -0,0 +1,7 @@
+module Mongoriver
+  module Logging
+    def log
+      @@logger ||= Log4r::Logger.new("Stripe::Mongoriver")
+    end
+  end
+end

data/lib/mongoriver/persistent_tailer.rb

@@ -0,0 +1,30 @@
+module Mongoriver
+  # A variant of AbstractPersistentTailer that automatically persists
+  # the "last timestamp processes" state into the database we are
+  # tailing.
+  class PersistentTailer < AbstractPersistentTailer
+    def initialize(upstream, type, service, opts={})
+      raise "You can't use PersistentTailer against only a slave. How am I supposed to write state? " if type == :slave
+      super(upstream, type, opts)
+
+      db         = opts[:db] || "_mongoriver"
+      collection = opts[:collection] || 'oplog-tailers'
+      @service = service
+      @state_collection = @upstream_conn.db(db).collection(collection)
+    end
+
+    def read_timestamp
+      row = @state_collection.find_one(:service => @service)
+      row ? row['timestamp'] : BSON::Timestamp.new(0, 0)
+    end
+
+    def write_timestamp(ts)
+      row = @state_collection.find_one(:service => @service)
+      if row
+        @state_collection.update({'_id' => row['_id']}, '$set' => { 'timestamp' => ts })
+      else
+        @state_collection.insert('service' => @service, 'timestamp' => ts)
+      end
+    end
+  end
+end

data/lib/mongoriver/streambed.rb

@@ -0,0 +1,299 @@
+module Mongoriver
+  class Streambed
+    include Mongoriver::Logging
+
+    attr_reader :stats
+
+    class AssertionFailure < StandardError; end
+
+    def assert(condition, msg)
+      raise AssertionFailure.new(msg) unless condition
+    end
+
+    def initialize(upstreams, type)
+      @tailer = Mongoriver::Tailer.new(upstreams, type)
+      @record_fetch_batch_size = 1024
+      @record_sync_batch_size = 256
+      @stats = Hash.new(0)
+    end
+
+    def run
+      self.class.validate_hooks!
+
+      unless ts = starting_optime
+        ts = @tailer.most_recent_timestamp
+        initial_sync
+        hook_update_optime(ts, true)
+      end
+
+      tail_from(ts)
+    end
+
+    def self.my_hooks
+      @hooks ||= []
+    end
+
+    def self.all_hooks
+      hooks = my_hooks
+      if superclass <= Streambed
+        hooks + superclass.all_hooks
+      else
+        hooks
+      end
+    end
+
+    def self.validate_hooks!
+      errors = []
+      all_hooks.each do |name, args, opts|
+        method = self.instance_method(hook_name(name))
+        signature = "#{method.name}(#{args.join(', ')})"
+        if method.owner == Streambed && !opts[:default]
+          errors << "Must provide implementation of #{signature}"
+        end
+      end
+
+      raise "You need to fix the following hook errors:
+
+  #{errors.join("\n  ")}" if errors.length > 0
+    end
+
+    def self.hook_name(name)
+      "hook_#{name}"
+    end
+
+    def self.hook(name, args=[], opts={})
+      if default = opts[:default]
+        target = hook_name(default)
+        implementation = Proc.new do |*args, &blk|
+          send(target, *args, &blk)
+        end
+      else
+        implementation = Proc.new do
+          raise NotImplementedError.new("Override in subclass")
+        end
+      end
+
+      define_method(hook_name(name), implementation)
+      my_hooks << [name, args, opts]
+    end
+
+    hook :optime
+    hook :update_optime, [:ts, :mandatory]
+    hook :initial_sync_index, [:db_name, :collection_name, :index_key, :options]
+    hook :initial_sync_record_batch, [:db_name, :collection_name, :records]
+    hook :stream_insert, [:db_name, :collection_name, :object]
+    hook :stream_update, [:db_name, :collection_name, :selector, :update]
+    hook :stream_remove, [:db_name, :collection_name, :object]
+    # Not usually a difference between the initial index creation and
+    # creating it while streaming ops.
+    hook :stream_create_index, [:db_name, :collection_name, :index_key, :options], :default => :initial_sync_index
+    # This seems to be called while doing a mapreduce.
+    hook :stream_create_collection, [:db_name, :create]
+    # This also seems to be called while doing a mapreduce. Note that
+    # I think mongo has a concept of temporary table, which I should
+    # look into, and renameCollection has some temporary table option.
+    hook :stream_rename_collection, [:db_name, :source, :target]
+    hook :stream_drop_index, [:db_name, :collection_name, :index_name]
+    hook :stream_drop_collection, [:db_name, :dropped]
+    hook :stream_drop_database, [:db_name]
+
+    private
+
+    def starting_optime
+      case time = hook_optime
+      when Integer
+        if time >= 0
+          BSON::Timestamp.new(time, 0)
+        elsif time == -1
+          @tailer.most_recent_timestamp
+        else
+          raise "Invalid optime: #{time}"
+        end
+      when BSON::Timestamp, nil
+        time
+      else
+        raise "Unrecognized type #{time.class} (#{time.inspect}) for start time"
+      end
+    end
+
+    def initial_sync
+      initial_sync_all_indexes
+      initial_sync_all_records
+    end
+
+    def initial_sync_all_indexes
+      log.info("Beginning initial sync of indexes")
+      syncable_databases.each {|db| initial_sync_indexes_for_db(db)}
+      log.info("Done initial sync of indexes")
+    end
+
+    def initial_sync_indexes_for_db(db)
+      db.collection('system.indexes').find.each do |index|
+        options = extract_options_from_index_spec(index)
+        index_key = index['key'].to_a
+
+        ns = index['ns']
+        db_name, collection_name = parse_ns(ns)
+        assert(db_name == db.name, "Index db name #{db_name.inspect} differs from current db name #{db.name.inspect}")
+
+        log.info("#{ns}: Initial sync of index #{options[:name]}")
+        hook_initial_sync_index(db_name, collection_name, index_key, options)
+      end
+    end
+
+    def initial_sync_all_records
+      log.info("Beginning initial sync of records")
+      syncable_databases.each {|db| initial_sync_records_for_db(db)}
+      log.info("Done initial sync of records")
+    end
+
+    def initial_sync_records_for_db(db)
+      syncable_collections(db).each do |collection|
+        initial_sync_records_for_collection(collection)
+      end
+    end
+
+    def initial_sync_records_for_collection(collection)
+      db_name = collection.db.name
+      collection_name = collection.name
+      ns = "#{db_name}.#{collection_name}"
+
+      log.info("#{ns}: Starting record initial sync")
+
+      records = []
+      collection.find({}, :batch_size => @record_fetch_batch_size, :timeout => false, :sort => [['$natural', 1]]) do |cursor|
+        while cursor.has_next?
+          records << cursor.next
+          if records.length > @record_sync_batch_size
+            # TODO: add better logging than this
+            log.info("#{ns}: Running sync of batch of #{records.length} records")
+            hook_initial_sync_record_batch(db_name, collection_name, records)
+            records = []
+          end
+        end
+      end
+      log.info("#{ns}: Finishing sync with a batch of #{records.length} records")
+      hook_initial_sync_record_batch(db_name, collection_name, records)
+
+      log.info("#{ns}: Finished record initial sync")
+    end
+
+    # This should be fine to instantiate all at once, since
+    # database_names returns all the dbs as strings anyway
+    def syncable_databases
+      @tailer.upstream_conn.database_names.map do |db_name|
+        next if db_name == 'local'
+        @tailer.upstream_conn.db(db_name)
+      end.compact
+    end
+
+    def syncable_collections(db)
+      db.collection_names.map do |collection_name|
+        next if collection_name.start_with?('system.')
+        db.collection(collection_name)
+      end.compact
+    end
+
+    def extract_options_from_index_spec(index)
+      options = {}
+      index.each do |key, value|
+        case key
+        when 'v'
+          raise NotImplementedError.new("Only v=1 indexes are supported at the moment, not v=#{value.inspect}") unless value == 1
+        when 'ns', 'key'
+        else
+          options[key.to_sym] = value
+        end
+      end
+
+      assert(options.include?(:name), "No name defined for index spec #{index.inspect}")
+      options
+    end
+
+    def stream_op(entry)
+      op = entry['op']
+      data = entry['o']
+      ns = entry['ns']
+
+      if op == 'n'
+        # This happens for initial rs.initiate() op, maybe others.
+        log.info("Skipping no-op #{entry.inspect}")
+        return
+      end
+
+      db_name, collection_name = parse_ns(ns)
+      assert(db_name, "Nil db name #{db_name.inspect} for #{entry.inspect}")
+
+      case op
+      when 'i'
+        if collection_name == 'system.indexes'
+          record(ns, entry, :create_index)
+          index_db_name, index_collection_name = parse_ns(data['ns'])
+          index_key = data['key'].to_a
+          options = extract_options_from_index_spec(data)
+          hook_stream_create_index(index_db_name, index_collection_name, index_key, options)
+        else
+          record(ns, entry, :insert)
+          hook_stream_insert(db_name, collection_name, data)
+        end
+      when 'u'
+        record(ns, entry, :update)
+        hook_stream_update(db_name, collection_name, entry['o2'], data)
+      when 'd'
+        record(ns, entry, :remove)
+        hook_stream_remove(db_name, collection_name, data)
+      when 'c'
+        assert(collection_name == '$cmd', "Command collection name is #{collection_name.inspect} for #{entry.inspect}")
+        if deleted_from = data['deleteIndexes']
+          record(ns, entry, :drop_index)
+          index = data['index']
+          hook_stream_drop_index(db_name, deleted_from, index)
+        elsif dropped = data['drop']
+          record(ns, entry, :drop_collection)
+          hook_stream_drop_collection(db_name, dropped)
+        elsif dropped = data['dropDatabase']
+          record(ns, entry, :drop_database)
+          hook_stream_drop_database(db_name)
+        elsif source = data['renameCollection']
+          record(ns, entry, :rename_collection)
+          target = data['to']
+          hook_stream_rename_collection(db_name, source, target)
+        elsif create = data['create']
+          record(ns, entry, :create)
+          hook_stream_create_collection(db_name, create)
+        else
+          raise "Unrecognized command #{data.inspect}"
+        end
+      else
+        raise "Unrecognized op: #{op} (#{entry.inspect})"
+      end
+
+      optime = entry['ts']
+      hook_update_optime(optime, false)
+    end
+
+    def tail_from(ts)
+      begin
+        @tailer.tail_from(ts)
+        loop do
+          @tailer.stream do |op|
+            stream_op(op)
+          end
+        end
+      ensure
+        @tailer.stop
+      end
+    end
+
+    def record(ns, entry, type)
+      stats[type] += 1
+      log.debug("#{ns}: #{type.inspect} #{entry.inspect}")
+    end
+
+    protected
+
+    def parse_ns(ns)
+      ns.split('.', 2)
+    end
+  end
+end

data/lib/mongoriver/tailer.rb

@@ -0,0 +1,100 @@
+module Mongoriver
+  class Tailer
+    include Mongoriver::Logging
+
+    attr_reader :upstream_conn
+
+    def initialize(upstreams, type)
+      @upstreams = upstreams
+      @type = type
+      # This number seems high
+      @conn_opts = {:op_timeout => 86400}
+
+      @cursor = nil
+
+      connect_upstream
+    end
+
+    def most_recent_timestamp
+      record = oplog_collection.find_one({}, :sort => [['$natural', -1]])
+      record['ts']
+    end
+
+    def connect_upstream
+      case @type
+      when :replset
+        opts = @conn_opts.merge(:read => :secondary)
+        @upstream_conn = Mongo::ReplSetConnection.new(@upstreams, opts)
+      when :slave, :direct
+        opts = @conn_opts.merge(:slave_ok => true)
+        host, port = parse_direct_upstream
+        @upstream_conn = Mongo::Connection.new(host, port, opts)
+        raise "Server at #{@upstream_conn.host}:#{@upstream_conn.port} is the primary -- if you're ok with that, check why your wrapper is passing :direct rather than :slave (HINT: try passing a -a to scripts like optail or mongocp)" if @type == :slave && @upstream_conn.primary?
+        ensure_upstream_replset!
+      when :existing
+        raise "Must pass in a single existing Mongo::Connection with :existing" unless @upstreams.length == 1 && @upstreams[0].respond_to?(:db)
+        @upstream_conn = @upstreams[0]
+      else
+        raise "Invalid connection type: #{@type.inspect}"
+      end
+    end
+
+    def ensure_upstream_replset!
+      # Might be a better way to do this, but not seeing one.
+      config = @upstream_conn['admin'].command(:ismaster => 1)
+      unless config['setName']
+        raise "Server at #{@upstream_conn.host}:#{@upstream_conn.port} is not running as a replica set"
+      end
+    end
+
+    def parse_direct_upstream
+      raise "When connecting directly to a mongo instance, must provide a single upstream" unless @upstreams.length == 1
+      upstream = @upstreams[0]
+      parse_host_spec(upstream)
+    end
+
+    def parse_host_spec(host_spec)
+      host, port = host_spec.split(':')
+      host = '127.0.0.1' if host.to_s.length == 0
+      port = '27017' if port.to_s.length == 0
+      [host, port.to_i]
+    end
+
+    def oplog_collection
+      @upstream_conn.db('local').collection('oplog.rs')
+    end
+
+    def tail_from(ts, opts = {})
+      raise "Already tailing the oplog!" if @cursor
+
+      # Maybe if ts is old enough, just start from the beginning?
+      query = (opts[:filter] || {}).merge({ 'ts' => { '$gte' => ts } })
+
+      oplog_collection.find(query, :timeout => false) do |oplog|
+        oplog.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
+        oplog.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
+
+        oplog.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA) unless opts[:dont_wait]
+
+        log.info("Starting oplog stream from #{ts}")
+        @cursor = oplog
+      end
+    end
+
+    def stop
+      @cursor.close if @cursor
+      @cursor = nil
+    end
+
+    def stream(limit=nil)
+      count = 0
+      while @cursor.has_next?
+        count += 1
+        break if limit && count >= limit
+        yield @cursor.next
+      end
+
+      return @cursor.has_next?
+    end
+  end
+end

data/lib/mongoriver/version.rb

@@ -0,0 +1,3 @@
+module Mongoriver
+  VERSION = "0.1.0"
+end

data/lib/mongoriver.rb

@@ -0,0 +1,12 @@
+require 'mongo'
+require 'log4r'
+
+module Mongoriver; end
+
+require 'mongoriver/log'
+
+require 'mongoriver/streambed'
+require 'mongoriver/tailer'
+require 'mongoriver/abstract_persistent_tailer'
+require 'mongoriver/persistent_tailer'
+require 'mongoriver/version'

data/mongoriver.gemspec

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+$:.unshift(File.expand_path("lib", File.dirname(__FILE__)))
+require 'mongoriver/version'
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Greg Brockman"]
+  gem.email         = ["gdb@gregbrockman.com"]
+  gem.description   = %q{Some tools and libraries to simplify tailing the mongod oplog}
+  gem.summary       = %q{monogdb oplog-tailing utilities.}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "mongoriver"
+  gem.require_paths = ["lib"]
+  gem.version       = Mongoriver::VERSION
+
+  gem.add_runtime_dependency('mongo', '>= 1.7')
+  gem.add_runtime_dependency('bson_ext')
+  gem.add_runtime_dependency('log4r')
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: mongoriver
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Greg Brockman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: log4r
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Some tools and libraries to simplify tailing the mongod oplog
+email:
+- gdb@gregbrockman.com
+executables:
+- mongocp
+- optail
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/mongocp
+- bin/optail
+- lib/mongoriver.rb
+- lib/mongoriver/abstract_persistent_tailer.rb
+- lib/mongoriver/log.rb
+- lib/mongoriver/persistent_tailer.rb
+- lib/mongoriver/streambed.rb
+- lib/mongoriver/tailer.rb
+- lib/mongoriver/version.rb
+- mongoriver.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: monogdb oplog-tailing utilities.
+test_files: []