mongoriver 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +40 -16
- data/lib/mongoriver/abstract_persistent_tailer.rb +12 -1
- data/lib/mongoriver/stream.rb +7 -1
- data/lib/mongoriver/tailer.rb +8 -2
- data/lib/mongoriver/version.rb +1 -1
- metadata +3 -3
data/README.md
CHANGED
@@ -1,29 +1,53 @@
|
|
1
1
|
# Mongoriver
|
2
2
|
|
3
|
-
|
3
|
+
mongoriver is a library to monitor updates to your Mongo databases in
|
4
|
+
near-realtime. It provides a simple interface for you to take actions
|
5
|
+
when records are inserted, removed, or updated.
|
4
6
|
|
5
|
-
## Installation
|
6
7
|
|
7
|
-
|
8
|
+
## How it works
|
8
9
|
|
9
|
-
|
10
|
+
MongoDB has an *oplog*, a log of all write operations. `mongoriver` monitors
|
11
|
+
updates to this oplog. See the [Mongo documentation for its oplog](http://docs.mongodb.org/manual/core/replica-set-oplog/) for more.
|
10
12
|
|
11
|
-
|
13
|
+
## How to use it
|
12
14
|
|
13
|
-
|
15
|
+
### Step 1: Create an outlet
|
14
16
|
|
15
|
-
|
17
|
+
You'll need to write a class subclassing
|
18
|
+
[Mongoriver::AbstractOutlet](https://github.com/stripe/mongoriver/blob/master/lib/mongoriver/abstract_outlet.rb).
|
16
19
|
|
17
|
-
|
20
|
+
You can override any of these methods:
|
18
21
|
|
19
|
-
|
22
|
+
* `update_optime(timestamp)`
|
23
|
+
* `insert(db_name, collection_name, document)`
|
24
|
+
* `remove(db_name, collection_name, document)`
|
25
|
+
* `update(db_name, collection_name, selector, update)`
|
26
|
+
* `create_index(db_name, collection_name, index_key, options)`
|
27
|
+
* `drop_index(db_name, collection_name, index_name)`
|
28
|
+
* `create_collection(db_name, collection_name, options)`
|
29
|
+
* `drop_collection(db_name, collection_name)`
|
30
|
+
* `rename_collection(db_name, old_collection_name, new_collection_name)`
|
31
|
+
* `drop_database(db_name)`
|
20
32
|
|
21
|
-
TODO: Write usage instructions here
|
22
33
|
|
23
|
-
|
34
|
+
You should think of these methods like callbacks -- if you want to do something
|
35
|
+
every time a document is inserted into the Mongo database, override the
|
36
|
+
`insert` method. You don't need to override all the methods -- if you only want
|
37
|
+
to take action on insert and update, just override `insert` and `update`.
|
24
38
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
39
|
+
### Step 2: Create a stream and start the logger
|
40
|
+
|
41
|
+
Once you've written your class, you can start tailing the Mongo oplog! Here's
|
42
|
+
the code you'll need to use:
|
43
|
+
|
44
|
+
```ruby
|
45
|
+
mongo = Mongo::MongoClient.from_uri(mongo_uri)
|
46
|
+
tailer = Mongoriver::Tailer.new([mongo], :existing)
|
47
|
+
outlet = YourOutlet.new(your_params) # Your subclass of Mongoriver::AbstractOutlet here
|
48
|
+
stream = Mongoriver::Stream.new(tailer, outlet)
|
49
|
+
stream.run_forever(starting_timestamp)
|
50
|
+
```
|
51
|
+
|
52
|
+
`starting_timestamp` here is the time you want the tailer to start at. We use
|
53
|
+
this to resume interrupted tailers so that no information is lost.
|
@@ -21,11 +21,22 @@ module Mongoriver
|
|
21
21
|
end
|
22
22
|
|
23
23
|
def stream(limit=nil)
|
24
|
-
|
24
|
+
start_time = BSON::Timestamp.new(connection_config['localTime'].to_i, 0)
|
25
|
+
found_entry = false
|
26
|
+
|
27
|
+
ret = super(limit) do |entry|
|
25
28
|
yield entry
|
29
|
+
found_entry = true
|
26
30
|
@last_read = entry['ts']
|
27
31
|
maybe_save_timestamp unless @batch
|
28
32
|
end
|
33
|
+
|
34
|
+
if !found_entry && !ret
|
35
|
+
@last_read = start_time
|
36
|
+
maybe_save_timestamp unless @batch
|
37
|
+
end
|
38
|
+
|
39
|
+
return ret
|
29
40
|
end
|
30
41
|
|
31
42
|
def batch_done
|
data/lib/mongoriver/stream.rb
CHANGED
@@ -114,7 +114,13 @@ module Mongoriver
|
|
114
114
|
|
115
115
|
def handle_create_index(spec)
|
116
116
|
db_name, collection_name = parse_ns(spec['ns'])
|
117
|
-
index_key = spec['key'].map { |field, dir|
|
117
|
+
index_key = spec['key'].map { |field, dir|
|
118
|
+
if dir.is_a?(Numeric)
|
119
|
+
[field, dir.round]
|
120
|
+
else
|
121
|
+
[field, dir]
|
122
|
+
end
|
123
|
+
}
|
118
124
|
options = {}
|
119
125
|
|
120
126
|
spec.each do |key, value|
|
data/lib/mongoriver/tailer.rb
CHANGED
@@ -42,9 +42,13 @@ module Mongoriver
|
|
42
42
|
end
|
43
43
|
end
|
44
44
|
|
45
|
+
def connection_config
|
46
|
+
@upstream_conn['admin'].command(:ismaster => 1)
|
47
|
+
end
|
48
|
+
|
45
49
|
def ensure_upstream_replset!
|
46
50
|
# Might be a better way to do this, but not seeing one.
|
47
|
-
config =
|
51
|
+
config = connection_config
|
48
52
|
unless config['setName']
|
49
53
|
raise "Server at #{@upstream_conn.host}:#{@upstream_conn.port} is not running as a replica set"
|
50
54
|
end
|
@@ -76,7 +80,9 @@ module Mongoriver
|
|
76
80
|
query['ts'] = { '$gte' => ts }
|
77
81
|
end
|
78
82
|
|
79
|
-
|
83
|
+
mongo_opts = {:timeout => false}.merge(opts[:mongo_opts] || {})
|
84
|
+
|
85
|
+
oplog_collection.find(query, mongo_opts) do |oplog|
|
80
86
|
oplog.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
|
81
87
|
oplog.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY) if query['ts']
|
82
88
|
oplog.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA) unless opts[:dont_wait]
|
data/lib/mongoriver/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mongoriver
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2014-07-10 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: mongo
|
@@ -153,7 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
153
153
|
version: '0'
|
154
154
|
requirements: []
|
155
155
|
rubyforge_project:
|
156
|
-
rubygems_version: 1.8.23
|
156
|
+
rubygems_version: 1.8.23.2
|
157
157
|
signing_key:
|
158
158
|
specification_version: 3
|
159
159
|
summary: monogdb oplog-tailing utilities.
|