mongoriver 0.3.0 → 0.3.1

data/README.md

@@ -1,29 +1,53 @@
 # Mongoriver
 
-TODO: Write a gem description
+mongoriver is a library to monitor updates to your Mongo databases in
+near-realtime. It provides a simple interface for you to take actions
+when records are inserted, removed, or updated.
 
-## Installation
 
-Add this line to your application's Gemfile:
+## How it works
 
-    gem 'mongoriver'
+MongoDB has an *oplog*, a log of all write operations. `mongoriver` monitors
+updates to this oplog. See the [Mongo documentation for its oplog](http://docs.mongodb.org/manual/core/replica-set-oplog/) for more.
 
-And then execute:
+## How to use it
 
-    $ bundle
+### Step 1: Create an outlet
 
-Or install it yourself as:
+You'll need to write a class subclassing
+[Mongoriver::AbstractOutlet](https://github.com/stripe/mongoriver/blob/master/lib/mongoriver/abstract_outlet.rb).
 
-    $ gem install mongoriver
+You can override any of these methods:
 
-## Usage
+* `update_optime(timestamp)`
+* `insert(db_name, collection_name, document)`
+* `remove(db_name, collection_name, document)`
+* `update(db_name, collection_name, selector, update)`
+* `create_index(db_name, collection_name, index_key, options)`
+* `drop_index(db_name, collection_name, index_name)`
+* `create_collection(db_name, collection_name,  options)`
+* `drop_collection(db_name, collection_name)`
+* `rename_collection(db_name, old_collection_name, new_collection_name)`
+* `drop_database(db_name)`
 
-TODO: Write usage instructions here
 
-## Contributing
+You should think of these methods like callbacks -- if you want to do something
+every time a document is inserted into the Mongo database, override the
+`insert` method. You don't need to override all the methods -- if you only want
+to take action on insert and update, just override `insert` and `update`.
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Added some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+### Step 2: Create a stream and start the logger
+
+Once you've written your class, you can start tailing the Mongo oplog! Here's
+the code you'll need to use:
+
+```ruby
+mongo = Mongo::MongoClient.from_uri(mongo_uri)
+tailer = Mongoriver::Tailer.new([mongo], :existing)
+outlet = YourOutlet.new(your_params) # Your subclass of Mongoriver::AbstractOutlet here
+stream = Mongoriver::Stream.new(tailer, outlet)
+stream.run_forever(starting_timestamp)
+```
+
+`starting_timestamp` here is the time you want the tailer to start at. We use
+this to resume interrupted tailers so that no information is lost.

data/lib/mongoriver/abstract_persistent_tailer.rb

@@ -21,11 +21,22 @@ module Mongoriver
     end
 
     def stream(limit=nil)
-      super(limit) do |entry|
+      start_time = BSON::Timestamp.new(connection_config['localTime'].to_i, 0)
+      found_entry = false
+
+      ret = super(limit) do |entry|
         yield entry
+        found_entry = true
         @last_read = entry['ts']
         maybe_save_timestamp unless @batch
       end
+
+      if !found_entry && !ret
+        @last_read = start_time
+        maybe_save_timestamp unless @batch
+      end
+
+      return ret
     end
 
     def batch_done

data/lib/mongoriver/stream.rb

@@ -114,7 +114,13 @@ module Mongoriver
 
     def handle_create_index(spec)
       db_name, collection_name = parse_ns(spec['ns'])
-      index_key = spec['key'].map { |field, dir| [field, dir.round] }
+      index_key = spec['key'].map { |field, dir|
+        if dir.is_a?(Numeric)
+          [field, dir.round]
+        else
+          [field, dir]
+        end
+      }
       options = {}
 
       spec.each do |key, value|

data/lib/mongoriver/tailer.rb

@@ -42,9 +42,13 @@ module Mongoriver
       end
     end
 
+    def connection_config
+      @upstream_conn['admin'].command(:ismaster => 1)
+    end
+
     def ensure_upstream_replset!
       # Might be a better way to do this, but not seeing one.
-      config = @upstream_conn['admin'].command(:ismaster => 1)
+      config = connection_config
       unless config['setName']
         raise "Server at #{@upstream_conn.host}:#{@upstream_conn.port} is not running as a replica set"
       end
@@ -76,7 +80,9 @@ module Mongoriver
         query['ts'] = { '$gte' => ts }
       end
 
-      oplog_collection.find(query, :timeout => false) do |oplog|
+      mongo_opts = {:timeout => false}.merge(opts[:mongo_opts] || {})
+
+      oplog_collection.find(query, mongo_opts) do |oplog|
         oplog.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
         oplog.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY) if query['ts']
         oplog.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA) unless opts[:dont_wait]

data/lib/mongoriver/version.rb

@@ -1,3 +1,3 @@
 module Mongoriver
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoriver
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-07 00:00:00.000000000 Z
+date: 2014-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
@@ -153,7 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.23.2
 signing_key: 
 specification_version: 3
 summary: monogdb oplog-tailing utilities.