mongo_delta 0.1.0 → 0.2.0

data/README.md

@@ -40,7 +40,8 @@ targets:
   archive: mongodb://mongoarch:27017
 
 outlets:
-  - outlet: Replicator
+  event_archiver:
+    outlet: Replicator
     target: archive
     db: db_name
     collection: events
@@ -81,17 +82,19 @@ Example:
 $ mongo mongo_delta
 rs0:PRIMARY> db.config.save({
 ... _id: 'mongo_delta',
-... outlets: [{
-...   outlet: 'Replicator',
-...   target: 'live',
-...   db: 'sourcedb',
-...   collection: 'events',
-...   target_db: 'archive'
-... }],
+... outlets: {
+...   event_archiver: {
+...     outlet: 'Replicator',
+...     target: 'live',
+...     db: 'sourcedb',
+...     collection: 'events',
+...     target_db: 'archive'
+...   }
+... },
 ... targets: {live: 'mongodb://localhost:27017'}
 ... })
 $ mongo_delta --source mongodb://localhost:27017
-2013-06-10 21:24:29 - INFO: Registering Replicator outlet for cartman.events
+2013-06-10 21:24:29 - INFO: Registering event_archiver Replicator outlet for cartman.events
 2013-06-10 21:24:29 - INFO: Starting stream
 ```
 

data/config/mongo_delta.yml

@@ -4,7 +4,8 @@ targets:
   archive: mongodb://localhost:27017
 
 outlets:
-  - outlet: Replicator
+  event_archiver:
+    outlet: Replicator
     target: archive
     db: dbname
     collection: events

data/lib/mongo_delta/agent.rb

@@ -5,23 +5,34 @@ module MongoDelta
 
     attr_reader :outlets
 
-    def initialize
-      @outlets = []
+    def initialize(options={})
+      @outlets = {}
+      @options = options
     end
 
-    %w(insert remove update).each do |method|
+    %w(insert remove update create_index drop_index create_collection drop_collection rename_collection).each do |method|
 
       define_method(method) do |db, collection, *args|
         logger.debug "#{method} for #{db}.#{collection}: #{args.map(&:inspect).join(' ')}"
 
-        outlets.each do |outlet|
-          if outlet.handles?(db, collection, method)
-            outlet.send(method, *args)
+        outlets.each do |key, outlet|
+          begin
+            if outlet.handles?(db, collection, method)
+              outlet.send(method, *args)
+            end
+          rescue
+            logger.error "#{key} outlet could not transfer document: #{$!.message}"
+            logger.error "=> #{db}.#{collection} #{args.map(&:inspect).join(' ')}"
+            raise unless ignore_errors?
           end
         end
       end
 
     end
 
+    def ignore_errors?
+      @options[:ignore_errors]
+    end
+
   end
 end

data/lib/mongo_delta/cli.rb

@@ -8,7 +8,7 @@ module MongoDelta
 
       include MongoDelta::Logging
 
-      attr_accessor :config_path, :source_uri, :optime, :db, :service
+      attr_accessor :config_path, :source_uri, :optime, :db, :service, :ignore_errors
 
       def start
         parse_options
@@ -22,6 +22,7 @@ module MongoDelta
           end
         config.db = db
         config.service = service
+        config.ignore_errors = ignore_errors
 
         stream = Stream.new(config)
         stream.run(optime)
@@ -51,6 +52,11 @@ module MongoDelta
             exit(1)
           end
 
+          opts.on('--version', 'Display version and exit') do
+            puts MongoDelta::VERSION
+            exit(1)
+          end
+
           opts.on('-c FILE', '--config', 'Configuration file') do |path|
             self.config_path = path
           end
@@ -70,6 +76,10 @@ module MongoDelta
           opts.on('-o OPTIME', '--optime', 'Starting optime') do |optime|
             self.optime = Integer(optime)
           end
+
+          opts.on('--ignore-errors', 'Ignore transfer errors') do
+            self.ignore_errors = true
+          end
         end
         optparse.parse!
       end

data/lib/mongo_delta/configuration.rb

@@ -56,10 +56,10 @@ module MongoDelta
     def validate!
       raise Error, "Missing source" unless source
       raise Error, "Missing outlets" unless outlets and not outlets.empty?
-      outlets.each do |outlet|
-        key = outlet['target']
-        target = (targets || {})[key]
-        raise Error, "Missing target '#{key}'" unless target
+      outlets.each do |key, outlet|
+        target_key = outlet['target']
+        target = (targets || {})[target_key]
+        raise Error, "Missing target '#{target_key}'" unless target
       end
     end
 
@@ -77,5 +77,12 @@ module MongoDelta
       @options['service'] = service
     end
 
+    def ignore_errors
+      !!@options['ignore_errors']
+    end
+    def ignore_errors=(bool)
+      @options['ignore_errors'] = bool
+    end
+
   end
 end

data/lib/mongo_delta/stream.rb

@@ -11,7 +11,7 @@ module MongoDelta
     def initialize(config)
       @config = config
       @tailer = Mongoriver::PersistentTailer.new([@config.source], :existing, @config.service, :db => @config.db)
-      @agent  = MongoDelta::Agent.new
+      @agent  = MongoDelta::Agent.new(:ignore_errors => config.ignore_errors)
       @stream = Mongoriver::Stream.new(@tailer, @agent)
       setup_outlets
     end
@@ -25,10 +25,10 @@ module MongoDelta
     private
 
     def setup_outlets
-      @config.outlets.each do |options|
+      @config.outlets.each do |key, options|
         outlet = MongoDelta::Outlet.from_options(@config, options)
-        logger.info "Registering #{options['outlet']} outlet for #{outlet.ns}"
-        @agent.outlets << outlet
+        logger.info "Registering #{key} #{options['outlet']} outlet for #{outlet.ns}"
+        @agent.outlets[key] = outlet
       end
     end
 

data/lib/mongo_delta/version.rb

@@ -1,3 +1,3 @@
 module MongoDelta
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -2,14 +2,14 @@
 name: mongo_delta
 version: !ruby/object:Gem::Version
   prerelease: 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Laszlo Bacsi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-10 00:00:00.000000000 Z
+date: 2013-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   prerelease: false
@@ -97,7 +97,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       segments:
       - 0
-      hash: -1239396457491943967
+      hash: -3914604556798742908
       version: '0'
   none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       segments:
       - 0
-      hash: -1239396457491943967
+      hash: -3914604556798742908
       version: '0'
   none: false
 requirements: []