mongo_delta 0.1.0

data/.gitignore

@@ -0,0 +1,5 @@
+Gemfile.lock
+.bundle
+vendor/cache
+vendor/ruby
+pkg/

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright 2013, Secret Sauce Partners, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,141 @@
+Mongo Delta
+===========
+
+### Coordinated transfer between MongoDB clusters
+
+Mongo Delta is a command line tool that tails a MongoDB replica set's
+oplog (using [mongoriver](https://github.com/stripe/mongoriver)) and
+based on a configured set of outlets transfers documents to other
+MongoDB instances.
+
+Installation
+------------
+
+Install from Rubygems as:
+
+    $ gem install mongo_delta
+
+Or build from source by:
+
+    $ gem build mongo_delta.gemspec
+
+And then install the built gem.
+
+Configuration
+-------------
+
+Mongo Delta requires a configuration where you set up your source,
+various targets and outlets. This can be stored in a YAML file or in the
+source database.
+
+Here's an example:
+
+``` yaml
+db: mongo_delta
+service: mongo_delta
+
+source: mongodb://mongorsa1:27017,mongorsa2:27017
+
+targets:
+  archive: mongodb://mongoarch:27017
+
+outlets:
+  - outlet: Replicator
+    target: archive
+    db: db_name
+    collection: events
+```
+
+The `db` and `service` options are optional and do the same as their
+command line counterparts. The default for both is `'mongo_delta'`. This
+tells Mongo Delta where to persist the optime which tracks the point of
+time upto which the oplog has been processed. The `service` option
+makes it possible to run multiple Mongo Delta processes using the same
+source.
+
+The `source` is where Mongo Delta is going to tail the oplog.
+Under `targets` several target connections can be listed.
+Use [MongoDB URIs](http://api.mongodb.org/ruby/current/#Environment_variable_MONGODB_URI) for both options.
+
+Finally, list outlets which will handle the incoming data and send them
+out another way. Configure each outlet with the following options:
+
+ * `outlet`: name of one of the outlet implementations (see below)
+ * `target`: name of one of the targets
+ * `db` and `collection`: specify the namespace for which the outlet applies
+ * `target_db` and `target_collection`: optional, send data at target to
+   a different db and collection
+ * some outlets can have further options
+
+### Storing configuration in the source database
+
+You can store this configuration in the source database. Use the
+`--source` command line option and Mongo Delta will assume that the
+configuration is located in the `config` collection of the `mongo_delta`
+database with `_id: 'mongo_delta'`. The database and the service ID can
+be overridden with the `--db` and `--service` options respectively.
+
+Example:
+
+```
+$ mongo mongo_delta
+rs0:PRIMARY> db.config.save({
+... _id: 'mongo_delta',
+... outlets: [{
+...   outlet: 'Replicator',
+...   target: 'live',
+...   db: 'sourcedb',
+...   collection: 'events',
+...   target_db: 'archive'
+... }],
+... targets: {live: 'mongodb://localhost:27017'}
+... })
+$ mongo_delta --source mongodb://localhost:27017
+2013-06-10 21:24:29 - INFO: Registering Replicator outlet for cartman.events
+2013-06-10 21:24:29 - INFO: Starting stream
+```
+
+Usage
+-----
+
+    mongo_delta --config path/to/config.yml [options]
+
+or if the configuration is stored in the source database:
+
+    mongo_delta --source mongodb://mongorsa1:27017,mongorsa2:27017 [options]
+
+Run `mongo_delta --help` for more options.
+
+Outlets
+-------
+
+### Replicator
+
+This outlet simply repeats `insert`, `remove` and `update` operations on
+the configured target. You can use this to keep a remote collection
+in sync with your main MongoDB cluster. Keep in mind that the
+replication is one-way.
+
+Sharded clusters
+----------------
+
+Mongo Delta does not have special support for sharded Mongo clusters at
+this time. It should be possible to run a separate `mongo_delta`
+instance against each of the individual backend shard replica sets,
+but otherwise with the same configuration.
+
+Development
+-----------
+
+Patches and contributions are welcome! Please fork the project and
+open a pull request on [github](https://github.com/sspinc/mongo_delta),
+or just report issues.
+
+Mongo Delta assumes the source MongoDB to be a replica set member. You
+can create a standalone replica set member on your development machine
+by running `mongod` with the `--replSet rs0` option, and then running
+the following command in the mongo shell:
+
+``` javascript
+rs.initiate({_id: 'rs0', members: [{ _id: 0, host: '127.0.0.1:27017'}]})
+```

data/Rakefile

@@ -0,0 +1,4 @@
+#!/usr/bin/env rake
+
+require 'bundler/setup'
+require 'bundler/gem_tasks'

data/bin/mongo_delta

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'mongo_delta/cli'
+
+MongoDelta::CLI.start

data/config/mongo_delta.yml

@@ -0,0 +1,12 @@
+source: mongodb://localhost:27017
+
+targets:
+  archive: mongodb://localhost:27017
+
+outlets:
+  - outlet: Replicator
+    target: archive
+    db: dbname
+    collection: events
+    target_db: archive # optional, defaults to 'db'
+    target_collection: archived_events # optional, defaults to 'collection'

data/lib/mongo_delta/agent.rb

@@ -0,0 +1,27 @@
+module MongoDelta
+  class Agent < Mongoriver::AbstractOutlet
+
+    include MongoDelta::Logging
+
+    attr_reader :outlets
+
+    def initialize
+      @outlets = []
+    end
+
+    %w(insert remove update).each do |method|
+
+      define_method(method) do |db, collection, *args|
+        logger.debug "#{method} for #{db}.#{collection}: #{args.map(&:inspect).join(' ')}"
+
+        outlets.each do |outlet|
+          if outlet.handles?(db, collection, method)
+            outlet.send(method, *args)
+          end
+        end
+      end
+
+    end
+
+  end
+end

data/lib/mongo_delta/cli.rb

@@ -0,0 +1,90 @@
+require 'optparse'
+
+require 'mongo_delta'
+
+module MongoDelta
+  module CLI
+    class << self
+
+      include MongoDelta::Logging
+
+      attr_accessor :config_path, :source_uri, :optime, :db, :service
+
+      def start
+        parse_options
+        validate_options!
+
+        config =
+          if config_path
+            Configuration.load_from_file(config_path)
+          else
+            Configuration.load_from_db(source_uri, db, service)
+          end
+        config.db = db
+        config.service = service
+
+        stream = Stream.new(config)
+        stream.run(optime)
+      rescue Mongo::ConnectionFailure, Configuration::Error
+        logger.fatal($!.message)
+        exit(1)
+      end
+
+      private
+
+      def parse_options
+        logger.level = Logger::INFO
+
+        optparse = OptionParser.new do |opts|
+          opts.banner = "Usage: #{$0} [options]"
+
+          opts.on('-v', '--verbose', 'More verbose output') do
+            logger.level = Logger::DEBUG
+          end
+
+          opts.on('-q', '--quiet', 'Less verbose output') do
+            logger.level = Logger::WARN
+          end
+
+          opts.on('--help', 'Display this message') do
+            puts opts
+            exit(1)
+          end
+
+          opts.on('-c FILE', '--config', 'Configuration file') do |path|
+            self.config_path = path
+          end
+
+          opts.on('-s MONGODB_URI', '--source', 'MongoDB URI for source connection') do |uri|
+            self.source_uri = uri
+          end
+
+          opts.on('-d DB', '--db', "DB for storing optime (default: #{Configuration::DEFAULT_DB})") do |db|
+            self.db = db
+          end
+
+          opts.on('-n NAME', '--service', "Service name (default: #{Configuration::DEFAULT_SERVICE})") do |service|
+            self.service = service
+          end
+
+          opts.on('-o OPTIME', '--optime', 'Starting optime') do |optime|
+            self.optime = Integer(optime)
+          end
+        end
+        optparse.parse!
+      end
+
+      def validate_options!
+        unless config_path or source_uri
+          logger.fatal "Missing configuration. Use --config or --source."
+          exit 1
+        end
+        if config_path and source_uri
+          logger.fatal "Options --config and --source are mutually exclusive."
+          exit 1
+        end
+      end
+
+    end
+  end
+end

data/lib/mongo_delta/configuration.rb

@@ -0,0 +1,81 @@
+require 'erb'
+require 'yaml'
+require 'mongo'
+
+module MongoDelta
+  class Configuration
+
+    class Error < RuntimeError; end
+
+    DEFAULT_DB='mongo_delta'
+    DEFAULT_SERVICE='mongo_delta'
+
+    def self.load_from_file(path)
+      options = YAML.load(ERB.new(File.read(path)).result)
+      new(options)
+    end
+
+    def self.load_from_db(mongodb_uri, db=nil, service=nil)
+      db ||= DEFAULT_DB
+      service ||= DEFAULT_SERVICE
+
+      mongo = connect_to_source(mongodb_uri)
+      collection = mongo.db(db).collection('config')
+
+      unless options = collection.find_one(:_id => service)
+        raise Error, "There was no config in the database at #{mongodb_uri}/#{db} with id '#{service}'"
+      end
+
+      new(options.merge({'source' => mongo, 'db' => db, 'service' => service}))
+    end
+
+    def self.connect_to_source(connection_or_uri)
+      if connection_or_uri.is_a? Mongo::MongoClient
+        connection_or_uri
+      else
+        Mongo::MongoClient.from_uri(connection_or_uri)
+      end
+    end
+
+    attr_reader :source
+
+    def initialize(options={})
+      @options = options
+      @source = self.class.connect_to_source(options['source'])
+      validate!
+    end
+
+    def targets
+      @options['targets']
+    end
+
+    def outlets
+      @options['outlets']
+    end
+
+    def validate!
+      raise Error, "Missing source" unless source
+      raise Error, "Missing outlets" unless outlets and not outlets.empty?
+      outlets.each do |outlet|
+        key = outlet['target']
+        target = (targets || {})[key]
+        raise Error, "Missing target '#{key}'" unless target
+      end
+    end
+
+    def db
+      @options['db'] || DEFAULT_DB
+    end
+    def db=(db)
+      @options['db'] = db
+    end
+
+    def service
+      @options['service'] || DEFAULT_SERVICE
+    end
+    def service=(service)
+      @options['service'] = service
+    end
+
+  end
+end

data/lib/mongo_delta/logging.rb

@@ -0,0 +1,19 @@
+require 'logger'
+
+module MongoDelta
+  module Logging
+
+    def self.setup_logger
+      Logger.new(STDERR).tap do |logger|
+        logger.formatter = proc do |severity, datetime, progname, msg|
+          "#{datetime} - #{severity}: #{msg}\n"
+        end
+      end
+    end
+
+    def logger
+      @@logger ||= MongoDelta::Logging.setup_logger
+    end
+
+  end
+end

data/lib/mongo_delta/outlet/base.rb

@@ -0,0 +1,26 @@
+module MongoDelta
+  module Outlet
+    class Base
+
+      attr_reader :target, :db, :collection, :options, :target_collection
+
+      def initialize(target, db, collection, options={})
+        @target     = target
+        @db         = db
+        @collection = collection
+        @options    = options
+
+        @target_collection = target.db(options['target_db'] || db).collection(options['target_collection'] || collection)
+      end
+
+      def ns
+        "#{db}.#{collection}"
+      end
+
+      def handles?(db, collection, method)
+        @db == db and @collection == collection and respond_to?(method)
+      end
+
+    end
+  end
+end

data/lib/mongo_delta/outlet/replicator.rb

@@ -0,0 +1,21 @@
+require 'mongo_delta/outlet/base'
+
+module MongoDelta
+  module Outlet
+    class Replicator < Base
+
+      def insert(document)
+        target_collection.save(document)
+      end
+
+      def remove(document)
+        target_collection.remove(document)
+      end
+
+      def update(selector, update)
+        target_collection.update(selector, update)
+      end
+
+    end
+  end
+end

data/lib/mongo_delta/outlet.rb

@@ -0,0 +1,26 @@
+require 'mongo'
+
+require 'mongo_delta/outlet/base'
+require 'mongo_delta/outlet/replicator'
+
+module MongoDelta
+  module Outlet
+
+    def self.from_options(config, options)
+      options    = options.dup
+      klass      = const_get(options.delete('outlet'))
+      target     = fetch_target(config, options.delete('target'))
+      db         = options.delete('db')
+      collection = options.delete('collection')
+      klass.new(target, db, collection, options)
+    end
+
+    private
+
+    def self.fetch_target(config, key)
+      @targets ||= {}
+      @targets[key] ||= Mongo::MongoClient.from_uri(config.targets[key])
+    end
+
+  end
+end

data/lib/mongo_delta/stream.rb

@@ -0,0 +1,46 @@
+require 'mongo'
+
+require 'mongo_delta/agent'
+require 'mongo_delta/outlet'
+
+module MongoDelta
+  class Stream
+
+    include MongoDelta::Logging
+
+    def initialize(config)
+      @config = config
+      @tailer = Mongoriver::PersistentTailer.new([@config.source], :existing, @config.service, :db => @config.db)
+      @agent  = MongoDelta::Agent.new
+      @stream = Mongoriver::Stream.new(@tailer, @agent)
+      setup_outlets
+    end
+
+    def run(ts=nil)
+      register_signal_handlers
+      logger.info "Starting stream"
+      @stream.run_forever(ts)
+    end
+
+    private
+
+    def setup_outlets
+      @config.outlets.each do |options|
+        outlet = MongoDelta::Outlet.from_options(@config, options)
+        logger.info "Registering #{options['outlet']} outlet for #{outlet.ns}"
+        @agent.outlets << outlet
+      end
+    end
+
+    def register_signal_handlers
+      logger.debug "Registering signal handlers"
+      %w[TERM INT USR2].each do |sig|
+        Signal.trap(sig) do
+          logger.info "Got SIG#{sig}. Preparing to exit..."
+          @stream.stop
+        end
+      end
+    end
+
+  end
+end

data/lib/mongo_delta/version.rb

@@ -0,0 +1,3 @@
+module MongoDelta
+  VERSION = "0.1.0"
+end

data/lib/mongo_delta.rb

@@ -0,0 +1,9 @@
+require 'mongoriver'
+
+module MongoDelta
+end
+
+require 'mongo_delta/configuration'
+require 'mongo_delta/logging'
+require 'mongo_delta/stream'
+require 'mongo_delta/version'

data/mongo_delta.gemspec

@@ -0,0 +1,23 @@
+$:.unshift(File.expand_path("lib", File.dirname(__FILE__)))
+
+require 'mongo_delta/version'
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Laszlo Bacsi"]
+  gem.email         = ["lackac@lackac.hu"]
+  gem.description   = %q{Streaming documents between MongoDB clusters}
+  gem.summary       = %q{Replicate mongodb documents between clusters}
+  gem.homepage      = "https://github.com/sspinc/mongo_delta"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map {|f| File.basename(f)}
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "mongo_delta"
+  gem.require_paths = ["lib"]
+  gem.version       = MongoDelta::VERSION
+
+  gem.add_runtime_dependency('mongo', '>= 1.7')
+  gem.add_runtime_dependency('mongoriver', '>= 0.3')
+
+  gem.add_development_dependency('rake')
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: mongo_delta
+version: !ruby/object:Gem::Version
+  prerelease: 
+  version: 0.1.0
+platform: ruby
+authors:
+- Laszlo Bacsi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  name: mongo
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.7'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.7'
+    none: false
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  name: mongoriver
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    none: false
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  name: rake
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+description: Streaming documents between MongoDB clusters
+email:
+- lackac@lackac.hu
+executables:
+- mongo_delta
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/mongo_delta
+- config/mongo_delta.yml
+- lib/mongo_delta.rb
+- lib/mongo_delta/agent.rb
+- lib/mongo_delta/cli.rb
+- lib/mongo_delta/configuration.rb
+- lib/mongo_delta/logging.rb
+- lib/mongo_delta/outlet.rb
+- lib/mongo_delta/outlet/base.rb
+- lib/mongo_delta/outlet/replicator.rb
+- lib/mongo_delta/stream.rb
+- lib/mongo_delta/version.rb
+- mongo_delta.gemspec
+homepage: https://github.com/sspinc/mongo_delta
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      segments:
+      - 0
+      hash: -1239396457491943967
+      version: '0'
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      segments:
+      - 0
+      hash: -1239396457491943967
+      version: '0'
+  none: false
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Replicate mongodb documents between clusters
+test_files: []