mosql 0.1.2 → 0.2.0

data/.travis.yml

@@ -0,0 +1,11 @@
+language: ruby
+rvm:
+  - 1.8.7
+  - 1.9.3
+services:
+  - mongodb
+  - postgresql
+before_script:
+    - psql -c 'create database mosql;' -U postgres
+env:
+  - MONGOSQL_TEST_SQL=postgres://localhost/mosql

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mosql (0.1.1)
+    mosql (0.1.2)
       bson_ext
       json
       log4r

data/README.md

@@ -53,8 +53,15 @@ types. An example collection map might be:
     mongodb:
       blog_posts:
         :columns:
-        - _id: TEXT
-        - author: TEXT
+        - id:
+          :source: _id
+          :type: TEXT
+        - author_name:
+          :source: author.name
+          :type: TEXT
+        - author_bio:
+          :source: author.bio
+          :type: TEXT
         - title: TEXT
         - created: DOUBLE PRECISION
         :meta:
@@ -67,9 +74,33 @@ mapping
     <Mongo DB name> -> { <Mongo Collection Name> -> <Collection Definition> }
 
 Where a `<Collection Definition>` is a hash with `:columns` and
-`:meta` fields. `:columns` is a list of one-element hashes, mapping
-field-name to SQL type. It is required to include at least an `_id`
-mapping. `:meta` contains metadata about this collection/table. It is
+`:meta` fields.
+
+`:columns` is a list of hashes mapping SQL column names to an hash
+describing that column. This hash may contain the following fields:
+
+  * `:source`: The name of the attribute inside of MongoDB.
+  * `:type`: (Mandatory) The SQL type.
+
+
+Use of the `:source` attribute allows for renaming attributes, and
+extracting elements of a nested hash using MongoDB's
+[dot notation][dot-notation]. In the above example, the `name` and
+`bio` fields of the `author` sub-document will be expanded, and the
+MongoDB `_id` field will be mapped to an SQL `id` column.
+
+At present, MoSQL does not support using the dot notation to access
+elements inside arrays.
+
+As a shorthand, you can specify a one-elment hash of the form `name:
+TYPE`, in which case `name` will be used for both the source attribute
+and the name of the destination column. You can see this shorthand for
+the `title` and `created` attributes, above.
+
+Every defined collection must include a mapping for the `_id`
+attribute.
+
+`:meta` contains metadata about this collection/table. It is
 required to include at least `:table`, naming the SQL table this
 collection will be mapped to. `extra_props` determines the handling of
 unknown fields in MongoDB objects -- more about that later.
@@ -78,6 +109,8 @@ By default, `mosql` looks for a collection map in a file named
 `collections.yml` in your current working directory, but you can
 specify a different one with `-c` or `--collections`.
 
+[dot-notation]: http://docs.mongodb.org/manual/core/document/#dot-notation
+
 ## Usage
 
 Once you have a collection map. MoSQL usage is easy. The basic form

data/Rakefile

@@ -1,6 +1,6 @@
 require 'rake/testtask'
 
-task :default
+task :default => [:test]
 task :build
 
 Rake::TestTask.new do |t|

data/bin/mosql

@@ -1,7 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'rubygems'
-require 'bundler/setup'
 require 'mosql/cli'
 
 MoSQL::CLI.run(ARGV)

data/lib/mosql/cli.rb

@@ -106,8 +106,11 @@ module MoSQL
     def connect_mongo
       @mongo = Mongo::Connection.from_uri(options[:mongo])
       config = @mongo['admin'].command(:ismaster => 1)
-      if !config['setName']
-        log.warn("`#{options[:mongo]}' is not a replset. Proceeding anyways...")
+      if !config['setName'] && !options[:skip_tail]
+        log.warn("`#{options[:mongo]}' is not a replset.")
+        log.warn("Will run the initial import, then stop.")
+        log.warn("Pass `--skip-tail' to suppress this warning.")
+        options[:skip_tail] = true
       end
       options[:service] ||= config['setName']
     end
@@ -121,7 +124,7 @@ module MoSQL
     end
 
     def load_collections
-      collections = YAML.load(File.read(@options[:collections]))
+      collections = YAML.load_file(@options[:collections])
       @schemamap = MoSQL::Schema.new(collections)
     end
 
@@ -140,7 +143,9 @@ module MoSQL
         initial_import
       end
 
-      optail
+      unless options[:skip_tail]
+        optail
+      end
     end
 
     # Helpers
@@ -159,7 +164,7 @@ module MoSQL
         items.each do |it|
           h = {}
           cols.zip(it).each { |k,v| h[k] = v }
-          @sql.upsert(table, h)
+          @sql.upsert(table, @schemamap.primary_sql_key_for_ns(ns), h)
         end
       end
     end
@@ -189,7 +194,9 @@ module MoSQL
     def initial_import
       @schemamap.create_schema(@sql.db, !options[:no_drop_tables])
 
-      start_ts = @mongo['local']['oplog.rs'].find_one({}, {:sort => [['$natural', -1]]})['ts']
+      unless options[:skip_tail]
+        start_ts = @mongo['local']['oplog.rs'].find_one({}, {:sort => [['$natural', -1]]})['ts']
+      end
 
       want_dbs = @schemamap.all_mongo_dbs & @mongo.database_names
       want_dbs.each do |dbname|
@@ -203,7 +210,7 @@ module MoSQL
         end
       end
 
-      tailer.write_timestamp(start_ts)
+      tailer.write_timestamp(start_ts) unless options[:skip_tail]
     end
 
     def import_collection(ns, collection)
@@ -240,8 +247,6 @@ module MoSQL
     end
 
     def optail
-      return if options[:skip_tail]
-
       tailer.tail_from(options[:tail_from] ?
                        BSON::Timestamp.new(options[:tail_from].to_i, 0) :
                        nil)
@@ -253,12 +258,13 @@ module MoSQL
     end
 
     def sync_object(ns, _id)
-      sqlid = @sql.transform_one_ns(ns, { '_id' => _id })['_id']
-      obj = collection_for_ns(ns).find_one({:_id => _id})
+      primary_sql_key = @schemamap.primary_sql_key_for_ns(ns)
+      sqlid           = @sql.transform_one_ns(ns, { '_id' => _id })[primary_sql_key]
+      obj             = collection_for_ns(ns).find_one({:_id => _id})
       if obj
         @sql.upsert_ns(ns, obj)
       else
-        @sql.table_for_ns(ns).where(:_id => sqlid).delete()
+        @sql.table_for_ns(ns).where(primary_sql_key.to_sym => sqlid).delete()
       end
     end
 

data/lib/mosql/schema.rb

@@ -4,19 +4,44 @@ module MoSQL
   class Schema
     include MoSQL::Logging
 
-    def to_ordered_hash(lst)
-      hash = BSON::OrderedHash.new
+    def to_array(lst)
+      array = []
       lst.each do |ent|
-        raise "Invalid ordered hash entry #{ent.inspect}" unless ent.is_a?(Hash) && ent.keys.length == 1
-        field, type = ent.first
-        hash[field] = type
+        if ent.is_a?(Hash) && ent[:source].is_a?(String) && ent[:type].is_a?(String)
+          # new configuration format
+          array << {
+            :source => ent.delete(:source),
+            :type   => ent.delete(:type),
+            :name   => ent.first.first,
+          }
+        elsif ent.is_a?(Hash) && ent.keys.length == 1 && ent.values.first.is_a?(String)
+          array << {
+            :source => ent.first.first,
+            :name   => ent.first.first,
+            :type   => ent.first.last
+          }
+        else
+          raise "Invalid ordered hash entry #{ent.inspect}"
+        end
+
+      end
+      array
+    end
+
+    def check_columns!(ns, spec)
+      seen = Set.new
+      spec[:columns].each do |col|
+        if seen.include?(col[:source])
+          raise "Duplicate source #{col[:source]} in column definition #{col[:name]} for #{ns}."
+        end
+        seen.add(col[:source])
       end
-      hash
     end
 
-    def parse_spec(spec)
+    def parse_spec(ns, spec)
       out = spec.dup
-      out[:columns] = to_ordered_hash(spec[:columns])
+      out[:columns] = to_array(spec[:columns])
+      check_columns!(ns, out)
       out
     end
 
@@ -25,7 +50,7 @@ module MoSQL
       map.each do |dbname, db|
         @map[dbname] ||= {}
         db.each do |cname, spec|
-          @map[dbname][cname] = parse_spec(spec)
+          @map[dbname][cname] = parse_spec("#{dbname}.#{cname}", spec)
         end
       end
     end
@@ -35,13 +60,16 @@ module MoSQL
         meta = collection[:meta]
         log.info("Creating table '#{meta[:table]}'...")
         db.send(clobber ? :create_table! : :create_table?, meta[:table]) do
-          collection[:columns].each do |field, type|
-            column field, type
+          collection[:columns].each do |col|
+            column col[:name], col[:type]
+
+            if col[:source].to_sym == :_id
+              primary_key [col[:name].to_sym]
+            end
           end
           if meta[:extra_props]
             column '_extra_props', 'TEXT'
           end
-          primary_key [:_id]
         end
       end
     end
@@ -62,13 +90,36 @@ module MoSQL
       schema
     end
 
+    def fetch_and_delete_dotted(obj, dotted)
+      pieces = dotted.split(".")
+      breadcrumbs = []
+      while pieces.length > 1
+        key = pieces.shift
+        breadcrumbs << [obj, key]
+        obj = obj[key]
+        return nil unless obj.is_a?(Hash)
+      end
+
+      val = obj.delete(pieces.first)
+
+      breadcrumbs.reverse.each do |obj, key|
+        obj.delete(key) if obj[key].empty?
+      end
+
+      val
+    end
+
     def transform(ns, obj, schema=nil)
       schema ||= find_ns!(ns)
 
       obj = obj.dup
       row = []
-      schema[:columns].each do |name, type|
-        v = obj.delete(name)
+      schema[:columns].each do |col|
+
+        source = col[:source]
+        type = col[:type]
+
+        v = fetch_and_delete_dotted(obj, source)
         case v
         when BSON::Binary, BSON::ObjectId
           v = v.to_s
@@ -91,7 +142,10 @@ module MoSQL
     end
 
     def all_columns(schema)
-      cols = schema[:columns].keys
+      cols = []
+      schema[:columns].each do |col|
+        cols << col[:name]
+      end
       if schema[:meta][:extra_props]
         cols << "_extra_props"
       end
@@ -100,7 +154,6 @@ module MoSQL
 
     def copy_data(db, ns, objs)
       schema = find_ns!(ns)
-      data = objs.map { |o| transform_to_copy(ns, o, schema) }.join("\n")
       db.synchronize do |pg|
         sql = "COPY \"#{schema[:meta][:table]}\" " +
           "(#{all_columns(schema).map {|c| "\"#{c}\""}.join(",")}) FROM STDIN"
@@ -145,5 +198,9 @@ module MoSQL
     def collections_for_mongo_db(db)
       (@map[db]||{}).keys
     end
+
+    def primary_sql_key_for_ns(ns)
+      find_ns!(ns)[:columns].find {|c| c[:source] == '_id'}[:name]
+    end
   end
 end

data/lib/mosql/sql.rb

@@ -35,35 +35,36 @@ module MoSQL
 
     def upsert_ns(ns, obj)
       h = transform_one_ns(ns, obj)
-      upsert(table_for_ns(ns), h)
+      upsert(table_for_ns(ns), @schema.primary_sql_key_for_ns(ns), h)
     end
 
     # obj must contain an _id field. All other fields will be ignored.
     def delete_ns(ns, obj)
+      primary_sql_key = @schema.primary_sql_key_for_ns(ns)
       h = transform_one_ns(ns, obj)
-      raise "No _id found in transform of #{obj.inspect}" if h['_id'].nil?
-      table_for_ns(ns).where(:_id => h['_id']).delete
+      raise "No #{primary_sql_key} found in transform of #{obj.inspect}" if h[primary_sql_key].nil?
+      table_for_ns(ns).where(primary_sql_key.to_sym => h[primary_sql_key]).delete
     end
 
-    def upsert(table, item)
+    def upsert(table, table_primary_key, item)
       begin
-        upsert!(table, item)
+        upsert!(table, table_primary_key, item)
       rescue Sequel::DatabaseError => e
         wrapped = e.wrapped_exception
         if wrapped.result
-          log.warn("Ignoring row (_id=#{item['_id']}): #{e}")
+          log.warn("Ignoring row (#{table_primary_key}=#{item[table_primary_key]}): #{e}")
         else
           raise e
         end
       end
     end
 
-    def upsert!(table, item)
+    def upsert!(table, table_primary_key, item)
       begin
         table.insert(item)
       rescue Sequel::DatabaseError => e
         raise e unless e.message =~ /duplicate key value violates unique constraint/
-        table.where(:_id => item['_id']).update(item)
+        table.where(table_primary_key.to_sym => item[table_primary_key]).update(item)
       end
     end
   end

data/lib/mosql/version.rb

@@ -1,3 +1,3 @@
 module MoSQL
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/test/_lib.rb

@@ -1,6 +1,3 @@
-require 'rubygems'
-require 'bundler/setup'
-
 require 'minitest/autorun'
 require 'minitest/spec'
 require 'mocha'

data/test/functional/cli.rb

@@ -11,6 +11,14 @@ mosql_test:
     :columns:
       - _id: TEXT
       - var: INTEGER
+  renameid:
+    :meta:
+      :table: sqltable2
+    :columns:
+      - id:
+        :source: _id
+        :type: TEXT
+      - goats: INTEGER
 EOF
 
   def fake_cli
@@ -29,6 +37,7 @@ EOF
     @adapter = MoSQL::SQLAdapter.new(@map, sql_test_uri)
 
     @sequel.drop_table?(:sqltable)
+    @sequel.drop_table?(:sqltable2)
     @map.create_schema(@sequel)
 
     @cli = fake_cli
@@ -73,4 +82,72 @@ EOF
                    })
     assert_equal(100, sequel[:sqltable].where(:_id => o['_id'].to_s).select.first[:var])
   end
+
+  it 'handle "u" ops with $set and a renamed _id' do
+    o = { '_id' => BSON::ObjectId.new, 'goats' => 96 }
+    @adapter.upsert_ns('mosql_test.renameid', o)
+
+    # $set's are currently a bit of a hack where we read the object
+    # from the db, so make sure the new object exists in mongo
+    connect_mongo['mosql_test']['renameid'].insert(o.merge('goats' => 0),
+                                                   :w => 1)
+
+    @cli.handle_op({ 'ns' => 'mosql_test.renameid',
+                     'op' => 'u',
+                     'o2' => { '_id' => o['_id'] },
+                     'o'  => { '$set' => { 'goats' => 0 } },
+                   })
+    assert_equal(0, sequel[:sqltable2].where(:id => o['_id'].to_s).select.first[:goats])
+  end
+
+  it 'handles "d" ops with a renamed id' do
+    o = { '_id' => BSON::ObjectId.new, 'goats' => 1 }
+    @adapter.upsert_ns('mosql_test.renameid', o)
+
+    @cli.handle_op({ 'ns' => 'mosql_test.renameid',
+                     'op' => 'd',
+                     'o' => { '_id' => o['_id'] },
+                   })
+    assert_equal(0, sequel[:sqltable2].where(:id => o['_id'].to_s).count)
+  end
+
+  describe '.bulk_upsert' do
+    it 'inserts multiple rows' do
+      objs = [
+              { '_id' => BSON::ObjectId.new, 'var' => 0 },
+              { '_id' => BSON::ObjectId.new, 'var' => 1 },
+              { '_id' => BSON::ObjectId.new, 'var' => 3 },
+             ].map { |o| @map.transform('mosql_test.collection', o) }
+
+      @cli.bulk_upsert(sequel[:sqltable], 'mosql_test.collection',
+                       objs)
+
+      assert(sequel[:sqltable].where(:_id => objs[0].first, :var => 0).count)
+      assert(sequel[:sqltable].where(:_id => objs[1].first, :var => 1).count)
+      assert(sequel[:sqltable].where(:_id => objs[2].first, :var => 3).count)
+    end
+
+    it 'upserts' do
+      _id = BSON::ObjectId.new
+      objs = [
+              { '_id' => _id, 'var' => 0 },
+              { '_id' => BSON::ObjectId.new, 'var' => 1 },
+              { '_id' => BSON::ObjectId.new, 'var' => 3 },
+             ].map { |o| @map.transform('mosql_test.collection', o) }
+
+      @cli.bulk_upsert(sequel[:sqltable], 'mosql_test.collection',
+                       objs)
+
+      newobjs = [
+                 { '_id' => _id, 'var' => 117 },
+                 { '_id' => BSON::ObjectId.new, 'var' => 32 },
+                ].map { |o| @map.transform('mosql_test.collection', o) }
+      @cli.bulk_upsert(sequel[:sqltable], 'mosql_test.collection',
+                       newobjs)
+
+
+      assert(sequel[:sqltable].where(:_id => newobjs[0].first, :var => 117).count)
+      assert(sequel[:sqltable].where(:_id => newobjs[1].first, :var => 32).count)
+    end
+  end
 end

data/test/functional/schema.rb

@@ -15,7 +15,19 @@ db:
       :table: sqltable2
       :extra_props: true
     :columns:
-      - _id: INTEGER
+      - _id: TEXT
+  with_dotted:
+    :meta:
+      :table: sqltable3
+      :extra_props: true
+    :columns:
+      - _id: TEXT
+      - var_a:
+        :source: vars.a
+        :type: TEXT
+      - var_b:
+        :source: vars.b
+        :type: TEXT
 EOF
 
   before do
@@ -23,11 +35,13 @@ EOF
 
     @sequel.drop_table?(:sqltable)
     @sequel.drop_table?(:sqltable2)
+    @sequel.drop_table?(:sqltable3)
     @map.create_schema(@sequel)
   end
 
   def table; @sequel[:sqltable]; end
   def table2; @sequel[:sqltable2]; end
+  def table3; @sequel[:sqltable3]; end
 
   it 'Creates the tables with the right columns' do
     assert_equal(Set.new([:_id, :var]),
@@ -51,6 +65,30 @@ EOF
     assert_equal(nil, rows[3][:var])
   end
 
+  it 'Can COPY dotted data' do
+    objects = [
+               {'_id' => "a", 'vars' => {'a' => 1, 'b' => 2}},
+               {'_id' => "b", 'vars' => {}},
+               {'_id' => "c", 'vars' => {'a' => 2, 'c' => 6}},
+               {'_id' => "d", 'vars' => {'a' => 1, 'c' => 7}, 'extra' => 'moo'}
+              ]
+    @map.copy_data(@sequel, 'db.with_dotted', objects.map { |o| @map.transform('db.with_dotted', o) } )
+    assert_equal(4, table3.count)
+    o = table3.first(:_id => 'a')
+    assert_equal("1", o[:var_a])
+    assert_equal("2", o[:var_b])
+
+    o = table3.first(:_id => 'b')
+    assert_equal({}, JSON.parse(o[:_extra_props]))
+
+    o = table3.first(:_id => 'c')
+    assert_equal({'vars' => { 'c' => 6} }, JSON.parse(o[:_extra_props]))
+
+    o = table3.first(:_id => 'd')
+    assert_equal({'vars' => { 'c' => 7}, 'extra' => 'moo' }, JSON.parse(o[:_extra_props]))
+    assert_equal(nil, o[:var_b])
+  end
+
   it 'Can COPY BSON::ObjectIDs' do
     o = {'_id' => BSON::ObjectId.new, 'var' => 0}
     @map.copy_data(@sequel, 'db.collection', [ @map.transform('db.collection', o)] )

data/test/functional/sql.rb

@@ -16,8 +16,8 @@ class MoSQL::Test::Functional::SQLTest < MoSQL::Test::Functional
 
   describe 'upsert' do
     it 'inserts new items' do
-      @adapter.upsert!(@table, {'_id' => 0, 'color' => 'red', 'quantity' => 10})
-      @adapter.upsert!(@table, {'_id' => 1, 'color' => 'blue', 'quantity' => 5})
+      @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'red', 'quantity' => 10})
+      @adapter.upsert!(@table, '_id', {'_id' => 1, 'color' => 'blue', 'quantity' => 5})
       assert_equal(2, @table.count)
       assert_equal('red',  @table[:_id => 0][:color])
       assert_equal(10,     @table[:_id => 0][:quantity])
@@ -26,11 +26,11 @@ class MoSQL::Test::Functional::SQLTest < MoSQL::Test::Functional
     end
 
     it 'updates items' do
-      @adapter.upsert!(@table, {'_id' => 0, 'color' => 'red', 'quantity' => 10})
+      @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'red', 'quantity' => 10})
       assert_equal(1, @table.count)
       assert_equal('red',  @table[:_id => 0][:color])
 
-      @adapter.upsert!(@table, {'_id' => 0, 'color' => 'blue', 'quantity' => 5})
+      @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'blue', 'quantity' => 5})
       assert_equal(1, @table.count)
       assert_equal('blue', @table[:_id => 0][:color])
     end

data/test/unit/lib/mosql/schema.rb

@@ -0,0 +1,198 @@
+require File.join(File.dirname(__FILE__), '../../../_lib.rb')
+
+class MoSQL::Test::SchemaTest < MoSQL::Test
+  TEST_MAP = <<EOF
+---
+db:
+  collection:
+    :meta:
+      :table: sqltable
+    :columns:
+      - id:
+        :source: _id
+        :type: TEXT
+      - var: INTEGER
+  with_extra_props:
+    :meta:
+      :table: sqltable2
+      :extra_props: true
+    :columns:
+      - id:
+        :source: _id
+        :type: TEXT
+  old_conf_syntax:
+    :columns:
+      - _id: TEXT
+    :meta:
+      :table: sqltable3
+EOF
+
+  before do
+    @map = MoSQL::Schema.new(YAML.load(TEST_MAP))
+  end
+
+  it 'Loads the schema' do
+    assert(@map)
+  end
+
+  it 'Can find an ns' do
+    assert(@map.find_ns("db.collection"))
+    assert_nil(@map.find_ns("db.other_collection"))
+    assert_raises(MoSQL::SchemaError) do
+      @map.find_ns!("db.other_collection")
+    end
+  end
+
+  it 'Converts columns to an array' do
+    table = @map.find_ns("db.collection")
+    assert(table[:columns].is_a?(Array))
+
+    id_mapping = table[:columns].find{|c| c[:source] == '_id'}
+    assert id_mapping
+    assert_equal '_id', id_mapping[:source]
+    assert_equal 'id', id_mapping[:name]
+    assert_equal 'TEXT', id_mapping[:type]
+
+    var_mapping = table[:columns].find{|c| c[:source] == 'var'}
+    assert var_mapping
+    assert_equal 'var', var_mapping[:source]
+    assert_equal 'var', var_mapping[:name]
+    assert_equal 'INTEGER', var_mapping[:type]
+  end
+
+  it 'Can handle the old configuration format' do
+    table = @map.find_ns('db.old_conf_syntax')
+    assert(table[:columns].is_a?(Array))
+
+    id_mapping = table[:columns].find{|c| c[:source] == '_id'}
+    assert id_mapping
+    assert_equal '_id', id_mapping[:source]
+    assert_equal '_id', id_mapping[:name]
+    assert_equal 'TEXT', id_mapping[:type]
+  end
+
+  it 'Can find the primary key of the SQL table' do
+    assert_equal('id', @map.primary_sql_key_for_ns('db.collection'))
+    assert_equal('_id', @map.primary_sql_key_for_ns('db.old_conf_syntax'))
+  end
+
+  it 'can create a SQL schema' do
+    db = stub()
+    db.expects(:create_table?).with('sqltable')
+    db.expects(:create_table?).with('sqltable2')
+    db.expects(:create_table?).with('sqltable3')
+
+    @map.create_schema(db)
+  end
+
+  it 'creates a SQL schema with the right fields' do
+    db = {}
+    stub_1 = stub()
+    stub_1.expects(:column).with('id', 'TEXT')
+    stub_1.expects(:column).with('var', 'INTEGER')
+    stub_1.expects(:column).with('_extra_props').never
+    stub_1.expects(:primary_key).with([:id])
+    stub_2 = stub()
+    stub_2.expects(:column).with('id', 'TEXT')
+    stub_2.expects(:column).with('_extra_props', 'TEXT')
+    stub_2.expects(:primary_key).with([:id])
+    stub_3 = stub()
+    stub_3.expects(:column).with('_id', 'TEXT')
+    stub_3.expects(:column).with('_extra_props').never
+    stub_3.expects(:primary_key).with([:_id])
+    (class << db; self; end).send(:define_method, :create_table?) do |tbl, &blk|
+      case tbl
+      when "sqltable"
+        o = stub_1
+      when "sqltable2"
+        o = stub_2
+      when "sqltable3"
+        o = stub_3
+      else
+        assert(false, "Tried to create an unexpected table: #{tbl}")
+      end
+      o.instance_eval(&blk)
+    end
+    @map.create_schema(db)
+  end
+
+  describe 'when transforming' do
+    it 'transforms rows' do
+      out = @map.transform('db.collection', {'_id' => "row 1", 'var' => 6})
+      assert_equal(["row 1", 6], out)
+    end
+
+    it 'Includes extra props' do
+      out = @map.transform('db.with_extra_props', {'_id' => 7, 'var' => 6, 'other var' => {'key' => 'value'}})
+      assert_equal(2, out.length)
+      assert_equal(7, out[0])
+      assert_equal({'var' => 6, 'other var' => {'key' => 'value'}}, JSON.parse(out[1]))
+    end
+
+    it 'gets all_columns right' do
+      assert_equal(['id', 'var'], @map.all_columns(@map.find_ns('db.collection')))
+      assert_equal(['id', '_extra_props'], @map.all_columns(@map.find_ns('db.with_extra_props')))
+    end
+  end
+
+  describe 'when copying data' do
+    it 'quotes special characters' do
+      assert_equal(%q{\\\\}, @map.quote_copy(%q{\\}))
+      assert_equal(%Q{\\\t}, @map.quote_copy( %Q{\t}))
+      assert_equal(%Q{\\\n}, @map.quote_copy( %Q{\n}))
+      assert_equal(%Q{some text}, @map.quote_copy(%Q{some text}))
+    end
+  end
+
+  describe 'fetch_and_delete_dotted' do
+    def check(orig, path, expect, result)
+      assert_equal(expect, @map.fetch_and_delete_dotted(orig, path))
+      assert_equal(result, orig)
+    end
+
+    it 'works on things without dots' do
+      check({'a' => 1, 'b' => 2},
+            'a', 1,
+            {'b' => 2})
+    end
+
+    it 'works if the key does not exist' do
+      check({'a' => 1, 'b' => 2},
+            'c', nil,
+            {'a' => 1, 'b' => 2})
+    end
+
+    it 'fetches nested hashes' do
+      check({'a' => 1, 'b' => { 'c' => 1, 'd' => 2 }},
+            'b.d', 2,
+            {'a' => 1, 'b' => { 'c' => 1 }})
+    end
+
+    it 'fetches deeply nested hashes' do
+      check({'a' => 1, 'b' => { 'c' => { 'e' => 8, 'f' => 9 }, 'd' => 2 }},
+            'b.c.e', 8,
+            {'a' => 1, 'b' => { 'c' => { 'f' => 9 }, 'd' => 2 }})
+    end
+
+    it 'cleans up empty hashes' do
+      check({'a' => { 'b' => 4}},
+            'a.b', 4,
+            {})
+      check({'a' => { 'b' => { 'c' => 5 }, 'd' => 9}},
+            'a.b.c', 5,
+            {'a' => { 'd' => 9 }})
+    end
+
+    it 'recursively cleans' do
+      check({'a' => { 'b' => { 'c' => { 'd' => 99 }}}},
+            'a.b.c.d', 99,
+            {})
+    end
+
+    it 'handles missing path components' do
+      check({'a' => { 'c' => 4 }},
+            'a.b.c.d', nil,
+            {'a' => { 'c' => 4 }})
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mosql
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-15 00:00:00.000000000 Z
+date: 2013-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -180,6 +180,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -200,7 +201,7 @@ files:
 - test/functional/functional.rb
 - test/functional/schema.rb
 - test/functional/sql.rb
-- test/unit/lib/mongo-sql/schema.rb
+- test/unit/lib/mosql/schema.rb
 homepage: https://github.com/stripe/mosql
 licenses: []
 post_install_message: 
@@ -232,4 +233,5 @@ test_files:
 - test/functional/functional.rb
 - test/functional/schema.rb
 - test/functional/sql.rb
-- test/unit/lib/mongo-sql/schema.rb
+- test/unit/lib/mosql/schema.rb
+has_rdoc: 

data/test/unit/lib/mongo-sql/schema.rb

@@ -1,102 +0,0 @@
-require File.join(File.dirname(__FILE__), '../../../_lib.rb')
-
-class MoSQL::Test::SchemaTest < MoSQL::Test
-  TEST_MAP = <<EOF
----
-db:
-  collection:
-    :meta:
-      :table: sqltable
-    :columns:
-      - _id: TEXT
-      - var: INTEGER
-  with_extra_props:
-    :meta:
-      :table: sqltable2
-      :extra_props: true
-    :columns:
-      - _id: INTEGER
-EOF
-
-  before do
-    @map = MoSQL::Schema.new(YAML.load(TEST_MAP))
-  end
-
-  it 'Loads the schema' do
-    assert(@map)
-  end
-
-  it 'Can find an ns' do
-    assert(@map.find_ns("db.collection"))
-    assert_nil(@map.find_ns("db.other_collection"))
-    assert_raises(MoSQL::SchemaError) do
-      @map.find_ns!("db.other_collection")
-    end
-  end
-
-  it 'Converts columns to an ordered hash' do
-    table = @map.find_ns("db.collection")
-    assert(table[:columns].is_a?(Hash))
-    assert_equal(['_id', 'var'], table[:columns].keys)
-  end
-
-  it 'can create a SQL schema' do
-    db = stub()
-    db.expects(:create_table?).with('sqltable')
-    db.expects(:create_table?).with('sqltable2')
-
-    @map.create_schema(db)
-  end
-
-  it 'creates a SQL schema with the right fields' do
-    db = {}
-    stub_1 = stub()
-    stub_1.expects(:column).with('_id', 'TEXT')
-    stub_1.expects(:column).with('var', 'INTEGER')
-    stub_1.expects(:column).with('_extra_props').never
-    stub_2 = stub()
-    stub_2.expects(:column).with('_id', 'INTEGER')
-    stub_2.expects(:column).with('_extra_props', 'TEXT')
-    (class << db; self; end).send(:define_method, :create_table?) do |tbl, &blk|
-      case tbl
-      when "sqltable"
-        o = stub_1
-      when "sqltable2"
-        o = stub_2
-      else
-        assert(false, "Tried to create an unexpeced table: #{tbl}")
-      end
-      o.expects(:primary_key).with([:_id])
-      o.instance_eval(&blk)
-    end
-    @map.create_schema(db)
-  end
-
-  describe 'when transforming' do
-    it 'transforms rows' do
-      out = @map.transform('db.collection', {'_id' => "row 1", 'var' => 6})
-      assert_equal(["row 1", 6], out)
-    end
-
-    it 'Includes extra props' do
-      out = @map.transform('db.with_extra_props', {'_id' => 7, 'var' => 6, 'other var' => {'key' => 'value'}})
-      assert_equal(2, out.length)
-      assert_equal(7, out[0])
-      assert_equal({'var' => 6, 'other var' => {'key' => 'value'}}, JSON.parse(out[1]))
-    end
-
-    it 'gets all_columns right' do
-      assert_equal(['_id', 'var'], @map.all_columns(@map.find_ns('db.collection')))
-      assert_equal(['_id', '_extra_props'], @map.all_columns(@map.find_ns('db.with_extra_props')))
-    end
-  end
-
-  describe 'when copying data' do
-    it 'quotes special characters' do
-      assert_equal(%q{\\\\}, @map.quote_copy(%q{\\}))
-      assert_equal(%Q{\\\t}, @map.quote_copy( %Q{\t}))
-      assert_equal(%Q{\\\n}, @map.quote_copy( %Q{\n}))
-      assert_equal(%Q{some text}, @map.quote_copy(%Q{some text}))
-    end
-  end
-end