mosql 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 43cb64bb52ef41955332728b063f24470d73c134
-  data.tar.gz: 9ddb444a959cdce77da58255fa64f4704cc4f201
+  metadata.gz: 7ab1024ecdb478cb92b0c2041f7d5f52a67ec0d3
+  data.tar.gz: 2b88ed3d4bb0723f67170d645d61e34b62d29dc6
 SHA512:
-  metadata.gz: 7260354422d7fad24eb8a018f7bf26108e7d3a86e502c0910df72cea011ecb58ee0c26065f8562940b4cca9bff2db06d07016aa4a83bd1dd4c1c0ee04c132f7d
-  data.tar.gz: 7bc1b499a41c0285770f26559c117e864e072490466f9aaabb7bb33093d5202767bc42d98525b2cf5a4405f79d88c5e4704df5aad75df918ad1b97ad1d5f15b8
+  metadata.gz: 5e8cc02a48b261391e0a161e0c35fb92113cc85f9e7436ed14bb9a4e0b54f4389f13093a02f9af6b516bd6d1d1db93202d5e8fd52c76d927564ead0733eb2d1d
+  data.tar.gz: d6a2c35555a5e8d4c99596f8c0cd212dca2546932a023cc1d7686ea6581ef0387813179817ce368cf06dff50753b927cc1ecba8be3bc82b44ff52519c141e31c

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    mosql (0.4.0)
+    mosql (0.4.1)
+      bson (~> 1.10)
       bson_ext
       json
       log4r
@@ -14,24 +15,24 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    bson (1.10.2)
-    bson_ext (1.10.2)
-      bson (~> 1.10.2)
+    bson (1.11.1)
+    bson_ext (1.11.1)
+      bson (~> 1.11.1)
     json (1.8.1)
     log4r (1.1.10)
     metaclass (0.0.4)
-    minitest (3.0.0)
-    mocha (1.0.0)
+    minitest (5.4.2)
+    mocha (1.1.0)
       metaclass (~> 0.0.1)
-    mongo (1.10.2)
-      bson (= 1.10.2)
+    mongo (1.11.1)
+      bson (= 1.11.1)
     mongoriver (0.4.0)
       bson_ext
       log4r
       mongo (>= 1.7)
     pg (0.17.1)
     rake (10.3.2)
-    sequel (4.14.0)
+    sequel (4.16.0)
 
 PLATFORMS
   ruby

data/README.md

@@ -212,7 +212,24 @@ e.g.
 mosql --mongo mongdb://$USER@$PASSWORD:$HOST/admin
 ```
 
-I have not yet tested using MoSQL with 2.4's "roles" support. Drop me
+In order to use MongoDB 2.4's "roles" support (which is different from that in
+2.6), you need to create the user in the admin database, give it explicit read
+access to the databases you want to copy *and* to the `local` database, and
+specify authSource in the URL.  eg, connect to `mydb/admin` with the mongo shell
+and run:
+
+```
+> db.addUser({user: "replicator", pwd: "PASSWORD", roles: [], otherDBRoles: {local: ["read"], sourceDb: ["read"]}})
+```
+
+(Note that `roles: []` ensures that this user has no special access to the
+`admin` database.)  Now specify:
+
+```
+mosql --mongo mongdb://$USER@$PASSWORD:$HOST/sourceDb?authSource=admin
+```
+
+I have not yet tested using MoSQL with 2.6's rewritten "roles" support. Drop me
 a note if you figure out anything I should know.
 
 ## Sharded clusters

data/lib/mosql/cli.rb

@@ -98,6 +98,11 @@ module MoSQL
         opts.on("--unsafe", "Ignore rows that cause errors on insert") do
           @options[:unsafe] = true
         end
+
+        # eg, --oplog-filter '{"ns": {"$regex": "^somedb[0-9]*\\.collection$"}}'
+        opts.on("--oplog-filter [filter]", "An additional JSON filter for the oplog query") do |filter|
+          @options[:oplog_filter] = JSON.parse(filter)
+        end
       end
 
       optparse.parse!(@args)

data/lib/mosql/schema.rb

@@ -159,10 +159,24 @@ module MoSQL
       val
     end
 
-    def fetch_special_source(obj, source)
+    def fetch_exists(obj, dotted)
+      pieces = dotted.split(".")
+      while pieces.length > 1
+        key = pieces.shift
+        obj = obj[key]
+        return false unless obj.is_a?(Hash)
+      end
+      obj.has_key?(pieces.first)
+    end
+
+    def fetch_special_source(obj, source, original)
       case source
       when "$timestamp"
         Sequel.function(:now)
+      when /^\$exists (.+)/
+        # We need to look in the cloned original object, not in the version that
+        # has had some fields deleted.
+        fetch_exists(original, $1)
       else
         raise SchemaError.new("Unknown source: #{source}")
       end
@@ -188,7 +202,12 @@ module MoSQL
     def transform(ns, obj, schema=nil)
       schema ||= find_ns!(ns)
 
-      obj = obj.dup
+      original = obj
+
+      # Do a deep clone, because we're potentially going to be
+      # mutating embedded objects.
+      obj = BSON.deserialize(BSON.serialize(obj))
+
       row = []
       schema[:columns].each do |col|
 
@@ -196,7 +215,7 @@ module MoSQL
         type = col[:type]
 
         if source.start_with?("$")
-          v = fetch_special_source(obj, source)
+          v = fetch_special_source(obj, source, original)
         else
           v = fetch_and_delete_dotted(obj, source)
           case v

data/lib/mosql/streamer.rb

@@ -170,7 +170,7 @@ module MoSQL
       if tail_from.is_a? Time
         tail_from = tailer.most_recent_position(tail_from)
       end
-      tailer.tail(:from => tail_from)
+      tailer.tail(:from => tail_from, :filter => options[:oplog_filter])
       until @done
         tailer.stream(1000) do |op|
           handle_op(op)

data/lib/mosql/version.rb

@@ -1,3 +1,3 @@
 module MoSQL
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

data/mosql.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |gem|
   %w[sequel pg mongo bson_ext rake log4r json
      ].each { |dep| gem.add_runtime_dependency(dep) }
   gem.add_runtime_dependency "mongoriver", "0.4"
+  gem.add_runtime_dependency "bson", "~> 1.10"
 
   gem.add_development_dependency "minitest"
   gem.add_development_dependency "mocha"

data/test/functional/streamer.rb

@@ -262,7 +262,7 @@ EOF
             @streamer.handle_op({ 'ns' => 'mosql_test.collection',
                                   'op' => 'u',
                                   'o2' => { '_id' => 'a' },
-                                  'o'  => { 'var' => 1 << 70 },
+                                  'o'  => { 'var' => 1 << 62 },
                                 })
           end
         end
@@ -272,7 +272,7 @@ EOF
           @streamer.handle_op({ 'ns' => 'mosql_test.collection',
                                 'op' => 'u',
                                 'o2' => { '_id' => 'a' },
-                                'o'  => { 'var' => 1 << 70 },
+                                'o'  => { 'var' => 1 << 62 },
                               })
           assert_equal(0, sequel[:sqltable].where(:_id => 'a').count)
         end

data/test/unit/lib/mosql/schema.rb

@@ -343,6 +343,26 @@ db:
       - mosql_created:
         :source: $timestamp
         :type: timestamp
+  existence:
+    :meta:
+      :table: b_table
+    :columns:
+      - _id: TEXT
+      - has_foo:
+        :source: $exists foo
+        :type: BOOLEAN
+      - has_foo_bar:
+        :source: $exists foo.bar
+        :type: BOOLEAN
+  exists_and_value:
+    :meta:
+      :table: c_table
+    :columns:
+      - _id: TEXT
+      - foo: TEXT
+      - has_foo:
+        :source: $exists foo
+        :type: BOOLEAN
   invalid:
     :meta:
       :table: invalid
@@ -362,6 +382,28 @@ EOF
       assert_equal(['a', Sequel.function(:now)], r)
     end
 
+    it 'translates $exists' do
+      r = @othermap.transform('db.existence', { '_id' => 'a' })
+      assert_equal(['a', false, false], r)
+      r = @othermap.transform('db.existence', { '_id' => 'a', 'foo' => nil })
+      assert_equal(['a', true, false], r)
+      r = @othermap.transform('db.existence', { '_id' => 'a', 'foo' => {} })
+      assert_equal(['a', true, false], r)
+      r = @othermap.transform('db.existence', { '_id' => 'a', 'foo' => {'bar' => nil} })
+      assert_equal(['a', true, true], r)
+      r = @othermap.transform('db.existence', { '_id' => 'a', 'foo' => {'bar' => 42} })
+      assert_equal(['a', true, true], r)
+    end
+
+    it 'can get $exists and value' do
+      r = @othermap.transform('db.exists_and_value', { '_id' => 'a' })
+      assert_equal(['a', nil, false], r)
+      r = @othermap.transform('db.exists_and_value', { '_id' => 'a', 'foo' => nil })
+      assert_equal(['a', nil, true], r)
+      r = @othermap.transform('db.exists_and_value', { '_id' => 'a', 'foo' => 'xxx' })
+      assert_equal(['a', 'xxx', true], r)
+    end
+
     it 'rejects unknown specials' do
       assert_raises(MoSQL::SchemaError) do
         r = @othermap.transform('db.invalid', { '_id' => 'a' })

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mosql
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Nelson Elhage
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-01 00:00:00.000000000 Z
+date: 2014-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -122,6 +122,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: bson
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -213,4 +227,3 @@ test_files:
 - test/functional/streamer.rb
 - test/functional/transform.rb
 - test/unit/lib/mosql/schema.rb
-has_rdoc: