mosql 0.3.1 → 0.3.2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mosql (0.3.1)
+    mosql (0.3.2)
       bson_ext
       json
       log4r
@@ -14,23 +14,23 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    bson (1.8.5)
-    bson_ext (1.8.5)
-      bson (~> 1.8.5)
-    json (1.8.0)
+    bson (1.9.1)
+    bson_ext (1.9.1)
+      bson (~> 1.9.1)
+    json (1.8.1)
     log4r (1.1.10)
-    metaclass (0.0.1)
+    metaclass (0.0.4)
     minitest (3.0.0)
-    mocha (0.10.5)
+    mocha (1.0.0)
       metaclass (~> 0.0.1)
-    mongo (1.8.5)
-      bson (~> 1.8.5)
+    mongo (1.9.1)
+      bson (~> 1.9.1)
     mongoriver (0.1.0)
       bson_ext
       log4r
       mongo (>= 1.7)
     pg (0.14.1)
-    rake (10.1.0)
+    rake (10.3.1)
     sequel (3.44.0)
 
 PLATFORMS

data/README.md

@@ -154,11 +154,18 @@ For advanced scenarios, you can pass options to control mosql's
 behavior. If you pass `--skip-tail`, mosql will do the initial import,
 but not tail the oplog. This could be used, for example, to do an
 import off of a backup snapshot, and then start the tailer on the live
-cluster.
+cluster. This can also be useful for hosted services where you do not
+have access to the oplog.
 
 If you need to force a fresh reimport, run `--reimport`, which will
 cause `mosql` to drop tables, create them anew, and do another import.
 
+Normaly, MoSQL will scan through a list of the databases on the mongo
+server you connect to. You avoid this behavior by specifiying a specific
+mongo db to connect to with the `--only-db [dbname]` option. This is
+useful for hosted services which do not let you list all databases (via 
+the `listDatabases` command).
+
 ## Schema mismatches and _extra_props
 
 If MoSQL encounters values in the MongoDB database that don't fit
@@ -169,16 +176,24 @@ If it encounters a MongoDB object with fields not listed in the
 collection map, it will discard the extra fields, unless
 `:extra_props` is set in the `:meta` hash. If it is, it will collect
 any missing fields, JSON-encode them in a hash, and store the
-resulting text in `_extra_props` in SQL. It's up to you to do
-something useful with the JSON. One option is to use [plv8][plv8] to
-parse them inside PostgreSQL, or you can just pull the JSON out whole
-and parse it in application code.
-
-This is also currently the only way to handle array or object values
-inside records -- specify `:extra_props`, and they'll get JSON-encoded
-into `_extra_props`. There's no reason we couldn't support
-JSON-encoded values for individual columns/fields, but we haven't
-written that code yet.
+resulting text in `_extra_props` in SQL.
+
+As of PostgreSQL 9.3, you can declare columns as type "JSON" and use
+the [native JSON support][pg-json] to inspect inside of JSON-encoded
+types. In earlier versions, you can write code in an extension
+language, such as [plv8][plv8].
+
+[pg-json]: http://www.postgresql.org/docs/9.3/static/functions-json.html
+
+## Non-scalar types
+
+MoSQL supports array types, using the `INTEGER ARRAY` array type
+syntax. This will cause MoSQL to create the column as an array type in
+PostgreSQL, and insert rows appropriately-formatted.
+
+Fields with hash values, or array values that are not in an
+ARRAY-typed column, will be transformed into JSON TEXT strings before
+being inserted into PostgreSQL.
 
 [plv8]: http://code.google.com/p/plv8js/
 

data/lib/mosql/cli.rb

@@ -71,6 +71,10 @@ module MoSQL
           @options[:ignore_delete] = true
         end
 
+        opts.on("--only-db [dbname]", "Don't scan for mongo dbs, just use the one specified") do |dbname|
+          @options[:dbname] = dbname
+        end
+
         opts.on("--tail-from [timestamp]", "Start tailing from the specified UNIX timestamp") do |ts|
           @options[:tail_from] = ts
         end

data/lib/mosql/schema.rb

@@ -5,17 +5,17 @@ module MoSQL
     include MoSQL::Logging
 
     def to_array(lst)
-      array = []
-      lst.each do |ent|
+      lst.map do |ent|
+        col = nil
         if ent.is_a?(Hash) && ent[:source].is_a?(String) && ent[:type].is_a?(String)
           # new configuration format
-          array << {
+          col = {
             :source => ent.fetch(:source),
             :type   => ent.fetch(:type),
             :name   => (ent.keys - [:source, :type]).first,
           }
         elsif ent.is_a?(Hash) && ent.keys.length == 1 && ent.values.first.is_a?(String)
-          array << {
+          col = {
             :source => ent.first.first,
             :name   => ent.first.first,
             :type   => ent.first.last
@@ -24,8 +24,12 @@ module MoSQL
           raise SchemaError.new("Invalid ordered hash entry #{ent.inspect}")
         end
 
+        if !col.key?(:array_type) && /\A(.+)\s+array\z/i.match(col[:type])
+          col[:array_type] = $1
+        end
+
+        col
       end
-      array
     end
 
     def check_columns!(ns, spec)
@@ -87,7 +91,13 @@ module MoSQL
               end
             end
             if meta[:extra_props]
-              column '_extra_props', 'TEXT'
+              type =
+                if meta[:extra_props] == "JSON"
+                  "JSON"
+                else
+                  "TEXT"
+                end
+              column '_extra_props', type
             end
           end
         end
@@ -104,7 +114,7 @@ module MoSQL
     end
 
     def find_ns(ns)
-      db, collection = ns.split(".")
+      db, collection = ns.split(".", 2)
       unless spec = find_db(db)
         return nil
       end
@@ -166,27 +176,23 @@ module MoSQL
           case v
           when BSON::Binary, BSON::ObjectId, Symbol
             v = v.to_s
-          when Hash, Array
+          when BSON::DBRef
+            v = v.object_id.to_s
+          when Hash
             v = JSON.dump(v)
+          when Array
+            if col[:array_type]
+              v = Sequel.pg_array(v, col[:array_type])
+            else
+              v = JSON.dump(v)
+            end
           end
         end
         row << v
       end
 
       if schema[:meta][:extra_props]
-        # Kludgily delete binary blobs from _extra_props -- they may
-        # contain invalid UTF-8, which to_json will not properly encode.
-        extra = {}
-        obj.each do |k,v|
-          case v
-          when BSON::Binary
-            next
-          when Float
-            # NaN is illegal in JSON. Translate into null.
-            v = nil if v.nan?
-          end
-          extra[k] = v
-        end
+        extra = sanitize(obj)
         row << JSON.dump(extra)
       end
 
@@ -195,6 +201,26 @@ module MoSQL
       row
     end
 
+    def sanitize(value)
+      # Base64-encode binary blobs from _extra_props -- they may
+      # contain invalid UTF-8, which to_json will not properly encode.
+      case value
+      when Hash
+        ret = {}
+        value.each {|k, v| ret[k] = sanitize(v)}
+        ret
+      when Array
+        value.map {|v| sanitize(v)}
+      when BSON::Binary
+        Base64.encode64(value.to_s)
+      when Float
+        # NaN is illegal in JSON. Translate into null.
+        value.nan? ? nil : value
+      else
+        value
+      end
+    end
+
     def copy_column?(col)
       col[:source] != '$timestamp'
     end

data/lib/mosql/sql.rb

@@ -7,6 +7,7 @@ module MoSQL
     def initialize(schema, uri, pgschema=nil)
       @schema = schema
       connect_db(uri, pgschema)
+      @db.extension :pg_array
     end
 
     def connect_db(uri, pgschema)

data/lib/mosql/streamer.rb

@@ -93,23 +93,41 @@ module MoSQL
         start_ts = @mongo['local']['oplog.rs'].find_one({}, {:sort => [['$natural', -1]]})['ts']
       end
 
-      @mongo.database_names.each do |dbname|
-        next unless spec = @schema.find_db(dbname)
+      dbnames = []
+
+      if dbname = options[:dbname]
+        log.info "Skipping DB scan and using db: #{dbname}"
+        dbnames = [ dbname ]
+      else
+        dbnames = @mongo.database_names
+      end
+
+      dbnames.each do |dbname|
+        spec = @schema.find_db(dbname)
+
+        if(spec.nil?)
+          log.info("Mongd DB '#{dbname}' not found in config file. Skipping.")
+          next
+        end
+
         log.info("Importing for Mongo DB #{dbname}...")
         db = @mongo.db(dbname)
-        db.collections.select { |c| spec.key?(c.name) }.each do |collection|
+        collections = db.collections.select { |c| spec.key?(c.name) }
+
+        collections.each do |collection|
           ns = "#{dbname}.#{collection.name}"
-          import_collection(ns, collection)
+          import_collection(ns, collection, spec[collection.name][:meta][:filter])
           exit(0) if @done
         end
       end
 
+
       tailer.write_timestamp(start_ts) unless options[:skip_tail]
     end
 
     def did_truncate; @did_truncate ||= {}; end
 
-    def import_collection(ns, collection)
+    def import_collection(ns, collection, filter)
       log.info("Importing for #{ns}...")
       count = 0
       batch = []
@@ -121,7 +139,7 @@ module MoSQL
 
       start    = Time.now
       sql_time = 0
-      collection.find(nil, :batch_size => BATCH) do |cursor|
+      collection.find(filter, :batch_size => BATCH) do |cursor|
         with_retries do
           cursor.each do |obj|
             batch << @schema.transform(ns, obj)

data/lib/mosql/version.rb

@@ -1,3 +1,3 @@
 module MoSQL
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

data/test/_lib.rb

@@ -5,6 +5,7 @@ require 'mocha'
 $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '../lib')))
 
 require 'mosql'
+require 'mocha/mini_test'
 
 module MoSQL
   class Test < ::MiniTest::Spec

data/test/functional/_lib.rb

@@ -17,6 +17,7 @@ module MoSQL
     def connect_sql
       begin
         conn = Sequel.connect(sql_test_uri)
+        conn.extension :pg_array
         conn.test_connection
         conn
       rescue Sequel::DatabaseConnectionError

data/test/functional/schema.rb

@@ -10,6 +10,7 @@ db:
     :columns:
       - _id: TEXT
       - var: INTEGER
+      - arry: INTEGER ARRAY
   with_extra_props:
     :meta:
       :table: sqltable2
@@ -44,7 +45,7 @@ EOF
   def table3; @sequel[:sqltable3]; end
 
   it 'Creates the tables with the right columns' do
-    assert_equal(Set.new([:_id, :var]),
+    assert_equal(Set.new([:_id, :var, :arry]),
                  Set.new(table.columns))
     assert_equal(Set.new([:_id, :_extra_props]),
                  Set.new(table2.columns))
@@ -53,7 +54,7 @@ EOF
   it 'Can COPY data' do
     objects = [
                {'_id' => "a", 'var' => 0},
-               {'_id' => "b", 'var' => 1},
+               {'_id' => "b", 'var' => 1, 'arry' => "{1, 2, 3}"},
                {'_id' => "c"},
                {'_id' => "d", 'other_var' => "hello"}
               ]
@@ -63,6 +64,7 @@ EOF
     assert_equal(%w[a b c d], rows.map { |r| r[:_id] })
     assert_equal(nil, rows[2][:var])
     assert_equal(nil, rows[3][:var])
+    assert_equal([1 ,2, 3], rows[1][:arry])
   end
 
   it 'Can COPY dotted data' do

data/test/functional/sql.rb

@@ -7,6 +7,7 @@ class MoSQL::Test::Functional::SQLTest < MoSQL::Test::Functional
       column :_id, 'INTEGER'
       column :color, 'TEXT'
       column :quantity, 'INTEGER'
+      column :numbers, 'INTEGER ARRAY'
       primary_key [:_id]
     end
 
@@ -16,13 +17,15 @@ class MoSQL::Test::Functional::SQLTest < MoSQL::Test::Functional
 
   describe 'upsert' do
     it 'inserts new items' do
-      @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'red', 'quantity' => 10})
-      @adapter.upsert!(@table, '_id', {'_id' => 1, 'color' => 'blue', 'quantity' => 5})
+      @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'red', 'quantity' => 10, 'numbers' => Sequel.pg_array([1, 2, 3], :integer)})
+      @adapter.upsert!(@table, '_id', {'_id' => 1, 'color' => 'blue', 'quantity' => 5, 'numbers' => Sequel.pg_array([], :integer)})
       assert_equal(2, @table.count)
       assert_equal('red',  @table[:_id => 0][:color])
       assert_equal(10,     @table[:_id => 0][:quantity])
       assert_equal('blue', @table[:_id => 1][:color])
       assert_equal(5,      @table[:_id => 1][:quantity])
+      assert_equal([1, 2, 3], @table[:_id => 0][:numbers])
+      assert_equal([], @table[:_id => 1][:numbers])
     end
 
     it 'updates items' do

data/test/functional/streamer.rb

@@ -20,6 +20,7 @@ mosql_test:
     :columns:
       - _id: TEXT
       - var: INTEGER
+      - arry: INTEGER ARRAY
   renameid:
     :meta:
       :table: sqltable2
@@ -28,6 +29,30 @@ mosql_test:
         :source: _id
         :type: TEXT
       - goats: INTEGER
+
+filter_test:
+  collection:
+    :meta:
+      :table: filter_sqltable
+      :filter:
+        :_id:
+          '$gte': !ruby/object:BSON::ObjectId
+            data:
+            - 83
+            - 179
+            - 75
+            - 128
+            - 0
+            - 0
+            - 0
+            - 0
+            - 0
+            - 0
+            - 0
+            - 0
+    :columns:
+      - _id: TEXT
+      - var: INTEGER
 EOF
 
     before do
@@ -109,11 +134,28 @@ EOF
       assert_equal(0, sequel[:sqltable2].where(:id => o['_id'].to_s).count)
     end
 
+    it 'filters unwanted records' do
+      data = [{:_id => BSON::ObjectId.from_time(Time.utc(2014, 7, 1)), :var => 2},
+              {:_id => BSON::ObjectId.from_time(Time.utc(2014, 7, 2)), :var => 3}]
+      collection = mongo["filter_test"]["collection"]
+      collection.drop
+      data.map { |rec| collection.insert(rec)}
+
+      @streamer.options[:skip_tail] = true
+      @streamer.initial_import
+
+      inserted_records = @sequel[:filter_sqltable].select
+      assert_equal(1, inserted_records.count)
+      record = inserted_records.first
+      data[1][:_id] = data[1][:_id].to_s
+      assert_equal(data[1], record)
+    end
+
     describe '.bulk_upsert' do
       it 'inserts multiple rows' do
         objs = [
                 { '_id' => BSON::ObjectId.new, 'var' => 0 },
-                { '_id' => BSON::ObjectId.new, 'var' => 1 },
+                { '_id' => BSON::ObjectId.new, 'var' => 1, 'arry' => [1, 2] },
                 { '_id' => BSON::ObjectId.new, 'var' => 3 },
                ].map { |o| @map.transform('mosql_test.collection', o) }
 

data/test/unit/lib/mosql/schema.rb

@@ -13,6 +13,7 @@ db:
         :type: TEXT
       - var: INTEGER
       - str: TEXT
+      - arry: INTEGER ARRAY
   with_extra_props:
     :meta:
       :table: sqltable2
@@ -26,9 +27,22 @@ db:
       - _id: TEXT
     :meta:
       :table: sqltable3
+  with_extra_props_type:
+    :meta:
+      :table: sqltable4
+      :extra_props: JSON
+    :columns:
+      - _id: TEXT
+  treat_array_as_string:
+    :columns:
+      - _id: TEXT
+      - arry: TEXT
+    :meta:
+      :table: sqltable5
 EOF
 
   before do
+    Sequel.extension(:pg_array)
     @map = MoSQL::Schema.new(YAML.load(TEST_MAP))
   end
 
@@ -82,26 +96,37 @@ EOF
     db.expects(:create_table?).with('sqltable')
     db.expects(:create_table?).with('sqltable2')
     db.expects(:create_table?).with('sqltable3')
+    db.expects(:create_table?).with('sqltable4')
+    db.expects(:create_table?).with('sqltable5')
 
     @map.create_schema(db)
   end
 
   it 'creates a SQL schema with the right fields' do
     db = {}
-    stub_1 = stub()
+    stub_1 = stub('table 1')
     stub_1.expects(:column).with('id', 'TEXT', {})
     stub_1.expects(:column).with('var', 'INTEGER', {})
     stub_1.expects(:column).with('str', 'TEXT', {})
+    stub_1.expects(:column).with('arry', 'INTEGER ARRAY', {})
     stub_1.expects(:column).with('_extra_props').never
     stub_1.expects(:primary_key).with([:id])
-    stub_2 = stub()
+    stub_2 = stub('table 2')
     stub_2.expects(:column).with('id', 'TEXT', {})
     stub_2.expects(:column).with('_extra_props', 'TEXT')
     stub_2.expects(:primary_key).with([:id])
-    stub_3 = stub()
+    stub_3 = stub('table 3')
     stub_3.expects(:column).with('_id', 'TEXT', {})
     stub_3.expects(:column).with('_extra_props').never
     stub_3.expects(:primary_key).with([:_id])
+    stub_4 = stub('table 4')
+    stub_4.expects(:column).with('_id', 'TEXT', {})
+    stub_4.expects(:column).with('_extra_props', 'JSON')
+    stub_4.expects(:primary_key).with([:_id])
+    stub_5 = stub('table 5')
+    stub_5.expects(:column).with('_id', 'TEXT', {})
+    stub_5.expects(:column).with('arry', 'TEXT', {})
+    stub_5.expects(:primary_key).with([:_id])
     (class << db; self; end).send(:define_method, :create_table?) do |tbl, &blk|
       case tbl
       when "sqltable"
@@ -110,6 +135,10 @@ EOF
         o = stub_2
       when "sqltable3"
         o = stub_3
+      when "sqltable4"
+        o = stub_4
+      when "sqltable5"
+        o = stub_5
       else
         assert(false, "Tried to create an unexpected table: #{tbl}")
       end
@@ -120,8 +149,8 @@ EOF
 
   describe 'when transforming' do
     it 'transforms rows' do
-      out = @map.transform('db.collection', {'_id' => "row 1", 'var' => 6, 'str' => 'a string'})
-      assert_equal(["row 1", 6, 'a string'], out)
+      out = @map.transform('db.collection', {'_id' => "row 1", 'var' => 6, 'str' => 'a string', 'arry' => [1,2,3]})
+      assert_equal(["row 1", 6, 'a string', [1,2,3]], out)
     end
 
     it 'Includes extra props' do
@@ -132,13 +161,20 @@ EOF
     end
 
     it 'gets all_columns right' do
-      assert_equal(['id', 'var', 'str'], @map.all_columns(@map.find_ns('db.collection')))
+      assert_equal(['id', 'var', 'str', 'arry'], @map.all_columns(@map.find_ns('db.collection')))
       assert_equal(['id', '_extra_props'], @map.all_columns(@map.find_ns('db.with_extra_props')))
     end
 
     it 'stringifies symbols' do
-      out = @map.transform('db.collection', {'_id' => "row 1", 'str' => :stringy})
-      assert_equal(["row 1", nil, 'stringy'], out)
+      out = @map.transform('db.collection', {'_id' => "row 1", 'str' => :stringy, 'arry' => [1,2,3]})
+      assert_equal(["row 1", nil, 'stringy', [1,2,3]], out)
+    end
+
+    it 'extracts object ids from a DBRef' do
+      oid = BSON::ObjectId.new
+      out = @map.transform('db.collection', {'_id' => "row 1",
+          'str' => BSON::DBRef.new('db.otherns', oid)})
+      assert_equal(["row 1", nil, oid.to_s, nil], out)
     end
 
     it 'changes NaN to null in extra_props' do
@@ -147,6 +183,25 @@ EOF
       assert(extra.key?('nancy'))
       assert_equal(nil, extra['nancy'])
     end
+
+    it 'base64-encodes BSON::Binary blobs in extra_props' do
+      out = @map.transform('db.with_extra_props',
+        {'_id' => 7,
+          'blob' => BSON::Binary.new("\x00\x00\x00"),
+          'embedded' => {'thing' => BSON::Binary.new("\x00\x00\x00")}})
+      extra = JSON.parse(out[1])
+      assert(extra.key?('blob'))
+      assert_equal('AAAA', extra['blob'].strip)
+      refute_nil(extra['embedded'])
+      refute_nil(extra['embedded']['thing'])
+      assert_equal('AAAA', extra['embedded']['thing'].strip)
+    end
+
+    it 'will treat arrays as strings when schame says to' do
+      out = @map.transform('db.treat_array_as_string', {'_id' => 1, 'arry' => [1, 2, 3]})
+      assert_equal(out[0], 1)
+      assert_equal(out[1], '[1,2,3]')
+    end
   end
 
   describe 'when copying data' do
@@ -281,4 +336,23 @@ EOF
       end
     end
   end
+
+  describe 'dotted names' do
+    MAP = <<EOF
+db:
+  my.collection:
+    :meta:
+      :table: table
+    :columns:
+      - _id: TEXT
+EOF
+
+    it 'handles dotted names' do
+      @map = MoSQL::Schema.new(YAML.load(MAP))
+      collections = @map.collections_for_mongo_db('db')
+      assert(collections.include?('my.collection'),
+        "#{collections} doesn't include `my.collection`")
+      assert(@map.find_ns('db.my.collection'))
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mosql
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-19 00:00:00.000000000 Z
+date: 2014-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -223,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.23.2
 signing_key: 
 specification_version: 3
 summary: MongoDB -> SQL streaming bridge