mosql 0.1.0 → 0.1.1

data/Gemfile.lock

@@ -1,42 +1,37 @@
-GIT
-  remote: git@github.com:stripe-internal/mongoriver
-  revision: d5b5ca1471f9efe7c91b3abe2c26f612a2dd4e9c
-  ref: d5b5ca1471f9efe7c91b3abe2c26f612a2dd4e9c
-  specs:
-    mongoriver (0.0.1)
-      bson_ext
-      log4r
-      mongo (>= 1.7)
-
 PATH
   remote: .
   specs:
-    mosql (0.0.1)
+    mosql (0.1.0)
       bson_ext
       json
       log4r
       mongo
+      mongoriver
       pg
       rake
       sequel
 
 GEM
-  remote: https://intgems.stripe.com:446/
+  remote: https://rubygems.org/
   specs:
-    bson (1.7.1)
-    bson_ext (1.7.1)
-      bson (~> 1.7.1)
-    json (1.7.5)
+    bson (1.8.2)
+    bson_ext (1.8.2)
+      bson (~> 1.8.2)
+    json (1.7.6)
     log4r (1.1.10)
     metaclass (0.0.1)
     minitest (3.0.0)
     mocha (0.10.5)
       metaclass (~> 0.0.1)
-    mongo (1.7.1)
-      bson (~> 1.7.1)
+    mongo (1.8.2)
+      bson (~> 1.8.2)
+    mongoriver (0.1.0)
+      bson_ext
+      log4r
+      mongo (>= 1.7)
     pg (0.14.1)
-    rake (10.0.2)
-    sequel (3.41.0)
+    rake (10.0.3)
+    sequel (3.44.0)
 
 PLATFORMS
   ruby
@@ -44,5 +39,4 @@ PLATFORMS
 DEPENDENCIES
   minitest
   mocha
-  mongoriver!
   mosql!

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Nelson Elhage
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -108,10 +108,15 @@ state.
 
 You likely want to run `mosql` against a secondary node, at least for
 the initial import, which will cause large amounts of disk activity on
-the target node. One option is to use read preferences in your
-connection URI:
+the target node. One option is to specify this in your connect URI:
 
-    mosql --mongo mongodb://node1,node2,node3?readPreference=secondary
+    mosql --mongo mongodb://node1,node2,node3?slaveOk=true
+
+(You should be able to specify `?readPreference=secondary`, but the
+Mongo Ruby driver does not appear to support that usage. I've filed a
+[bug with 10gen][bug-read-pref] about this omission).
+
+[bug-read-pref]: https://jira.mongodb.org/browse/RUBY-547
 
 ## Advanced usage
 

data/Rakefile

@@ -10,3 +10,11 @@ Rake::TestTask.new do |t|
     file.end_with?('_lib.rb')
   end
 end
+
+Rake::TestTask.new(:test_unit) do |t|
+  t.libs = ["lib"]
+  t.verbose = true
+  t.test_files = FileList['test/unit/**/*.rb'].reject do |file|
+    file.end_with?('_lib.rb')
+  end
+end

data/lib/mosql/cli.rb

@@ -86,6 +86,10 @@ module MoSQL
         opts.on("--reimport", "Force a data re-import") do
           @options[:reimport] = true
         end
+
+        opts.on("--no-drop-tables", "Don't drop the table if it exists during the initial import") do
+          @options[:no_drop_tables] = true
+        end
       end
 
       optparse.parse!(@args)
@@ -183,7 +187,7 @@ module MoSQL
     end
 
     def initial_import
-      @schemamap.create_schema(@sql.db, true)
+      @schemamap.create_schema(@sql.db, !options[:no_drop_tables])
 
       start_ts = @mongo['local']['oplog.rs'].find_one({}, {:sort => [['$natural', -1]]})['ts']
 
@@ -207,7 +211,7 @@ module MoSQL
       count = 0
       batch = []
       table = @sql.table_for_ns(ns)
-      table.truncate
+      table.truncate unless options[:no_drop_tables]
 
       start    = Time.now
       sql_time = 0
@@ -288,14 +292,20 @@ module MoSQL
           log.debug("resync #{ns}: #{selector['_id']} (update was: #{update.inspect})")
           sync_object(ns, selector['_id'])
         else
-          log.debug("upsert #{ns}: _id=#{update['_id']}")
+          log.debug("upsert #{ns}: _id=#{selector['_id']}")
+
+          # The update operation replaces the existing object, but
+          # preserves its _id field, so grab the _id off of the
+          # 'query' field -- it's not guaranteed to be present on the
+          # update.
+          update = { '_id' => selector['_id'] }.merge(update)
           @sql.upsert_ns(ns, update)
         end
       when 'd'
         if options[:ignore_delete]
           log.debug("Ignoring delete op on #{ns} as instructed.")
         else
-          @sql.table_for_ns(ns).where(:_id => op['o']['_id']).delete
+          @sql.delete_ns(ns, op['o'])
         end
       else
         log.info("Skipping unknown op #{op.inspect}")

data/lib/mosql/sql.rb

@@ -25,6 +25,7 @@ module MoSQL
       @db[@schema.table_for_ns(ns).intern]
     end
 
+
     def upsert_ns(ns, obj)
       h = {}
       cols = @schema.all_columns(@schema.find_ns(ns))
@@ -33,6 +34,15 @@ module MoSQL
       upsert(table_for_ns(ns), h)
     end
 
+    # obj must contain an _id field. All other fields will be ignored.
+    def delete_ns(ns, obj)
+      cols = @schema.all_columns(@schema.find_ns(ns))
+      row  = @schema.transform(ns, obj)
+      sqlid = row[cols.index("_id")]
+      raise "No _id found in transform of #{obj}" if sqlid.nil?
+      table_for_ns(ns).where(:_id => sqlid).delete
+    end
+
     def upsert(table, item)
       begin
         upsert!(table, item)

data/lib/mosql/version.rb

@@ -1,3 +1,3 @@
 module MoSQL
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/test/functional/cli.rb

@@ -0,0 +1,59 @@
+require File.join(File.dirname(__FILE__), '_lib.rb')
+require 'mosql/cli'
+
+class MoSQL::Test::Functional::CLITest < MoSQL::Test::Functional
+  TEST_MAP = <<EOF
+---
+mosql_test:
+  collection:
+    :meta:
+      :table: sqltable
+    :columns:
+      - _id: TEXT
+      - var: INTEGER
+EOF
+
+  def fake_cli
+    # This is a hack. We should refactor cli.rb to be more testable.
+    MoSQL::CLI.any_instance.expects(:setup_signal_handlers)
+    cli = MoSQL::CLI.new([])
+    cli.instance_variable_set(:@mongo, mongo)
+    cli.instance_variable_set(:@schemamap, @map)
+    cli.instance_variable_set(:@sql, @adapter)
+    cli.instance_variable_set(:@options, {})
+    cli
+  end
+
+  before do
+    @map = MoSQL::Schema.new(YAML.load(TEST_MAP))
+    @adapter = MoSQL::SQLAdapter.new(@map, sql_test_uri)
+
+    @sequel.drop_table?(:sqltable)
+    @map.create_schema(@sequel)
+
+    @cli = fake_cli
+  end
+
+  it 'handle "u" ops without _id' do
+    o = { '_id' => BSON::ObjectId.new, 'var' => 17 }
+    @adapter.upsert_ns('mosql_test.collection', o)
+
+    @cli.handle_op({ 'ns' => 'mosql_test.collection',
+                     'op' => 'u',
+                     'o2' => { '_id' => o['_id'] },
+                     'o'  => { 'var' => 27 }
+                   })
+    assert_equal(27, sequel[:sqltable].where(:_id => o['_id'].to_s).select.first[:var])
+  end
+
+  it 'handle "d" ops with BSON::ObjectIds' do
+    o = { '_id' => BSON::ObjectId.new, 'var' => 17 }
+    @adapter.upsert_ns('mosql_test.collection', o)
+
+    @cli.handle_op({ 'ns' => 'mosql_test.collection',
+                     'op' => 'd',
+                     'o' => { '_id' => o['_id'] },
+                   })
+    assert_equal(0, sequel[:sqltable].where(:_id => o['_id'].to_s).count)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mosql
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-05 00:00:00.000000000 Z
+date: 2013-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -182,6 +182,7 @@ files:
 - .gitignore
 - Gemfile
 - Gemfile.lock
+- LICENSE
 - README.md
 - Rakefile
 - bin/mosql
@@ -195,6 +196,7 @@ files:
 - mosql.gemspec
 - test/_lib.rb
 - test/functional/_lib.rb
+- test/functional/cli.rb
 - test/functional/functional.rb
 - test/functional/schema.rb
 - test/functional/sql.rb
@@ -226,6 +228,7 @@ summary: MongoDB -> SQL streaming bridge
 test_files:
 - test/_lib.rb
 - test/functional/_lib.rb
+- test/functional/cli.rb
 - test/functional/functional.rb
 - test/functional/schema.rb
 - test/functional/sql.rb