mosql 0.2.0 → 0.3.0

data/.travis.yml

@@ -1,6 +1,5 @@
 language: ruby
 rvm:
-  - 1.8.7
   - 1.9.3
 services:
   - mongodb

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mosql (0.1.2)
+    mosql (0.2.0)
       bson_ext
       json
       log4r

data/README.md

@@ -92,7 +92,7 @@ MongoDB `_id` field will be mapped to an SQL `id` column.
 At present, MoSQL does not support using the dot notation to access
 elements inside arrays.
 
-As a shorthand, you can specify a one-elment hash of the form `name:
+As a shorthand, you can specify a one-element hash of the form `name:
 TYPE`, in which case `name` will be used for both the source attribute
 and the name of the destination column. You can see this shorthand for
 the `title` and `created` attributes, above.
@@ -127,7 +127,7 @@ command-line parameters.
 
  1. Create the appropriate SQL tables
  2. Import data from the Mongo database
- 3. Start tailing the mongo oplog, propogating changes from MongoDB to SQL.
+ 3. Start tailing the mongo oplog, propagating changes from MongoDB to SQL.
 
 
 After the first run, `mosql` will store the status of the optailer in
@@ -143,13 +143,10 @@ You likely want to run `mosql` against a secondary node, at least for
 the initial import, which will cause large amounts of disk activity on
 the target node. One option is to specify this in your connect URI:
 
-    mosql --mongo mongodb://node1,node2,node3?slaveOk=true
+    mosql --mongo mongodb://node1,node2,node3?readPreference=secondary
 
-(You should be able to specify `?readPreference=secondary`, but the
-Mongo Ruby driver does not appear to support that usage. I've filed a
-[bug with 10gen][bug-read-pref] about this omission).
-
-[bug-read-pref]: https://jira.mongodb.org/browse/RUBY-547
+(Note that this requires you be using at least version 1.8.3 of
+`mongo-ruby-driver`)
 
 ## Advanced usage
 
@@ -185,6 +182,24 @@ written that code yet.
 
 [plv8]: http://code.google.com/p/plv8js/
 
+## Authentication
+
+At present, in order to use MoSQL with a MongoDB instance requiring
+authentication, you must:
+
+- Have a user with access to the admin database.
+- Specify the `admin` database in the `--mongo` argument
+- Specify the username and password in the `--mongo` argument
+
+e.g.
+
+```
+mosql --mongo mongdb://$USER@$PASSWORD:$HOST/admin
+```
+
+I have not yet tested using MoSQL with 2.4's "roles" support. Drop me
+a note if you figure out anything I should know.
+
 ## Sharded clusters
 
 MoSQL does not have special support for sharded Mongo clusters at this

data/lib/mosql/cli.rb

@@ -27,7 +27,7 @@ module MoSQL
       %w[TERM INT USR2].each do |sig|
         Signal.trap(sig) do
           log.info("Got SIG#{sig}. Preparing to exit...")
-          @done = true
+          @streamer.stop
         end
       end
     end
@@ -90,6 +90,10 @@ module MoSQL
         opts.on("--no-drop-tables", "Don't drop the table if it exists during the initial import") do
           @options[:no_drop_tables] = true
         end
+
+        opts.on("--unsafe", "Ignore rows that cause errors on insert") do
+          @options[:unsafe] = true
+        end
       end
 
       optparse.parse!(@args)
@@ -104,7 +108,7 @@ module MoSQL
     end
 
     def connect_mongo
-      @mongo = Mongo::Connection.from_uri(options[:mongo])
+      @mongo = Mongo::MongoClient.from_uri(options[:mongo])
       config = @mongo['admin'].command(:ismaster => 1)
       if !config['setName'] && !options[:skip_tail]
         log.warn("`#{options[:mongo]}' is not a replset.")
@@ -116,7 +120,7 @@ module MoSQL
     end
 
     def connect_sql
-      @sql = MoSQL::SQLAdapter.new(@schemamap, options[:sql], options[:schema])
+      @sql = MoSQL::SQLAdapter.new(@schema, options[:sql], options[:schema])
       if options[:verbose] >= 2
         @sql.db.sql_log_level = :debug
         @sql.db.loggers << Logger.new($stderr)
@@ -125,7 +129,13 @@ module MoSQL
 
     def load_collections
       collections = YAML.load_file(@options[:collections])
-      @schemamap = MoSQL::Schema.new(collections)
+      begin
+        @schema = MoSQL::Schema.new(collections)
+      rescue MoSQL::SchemaError => e
+        log.error("Error parsing collection map `#{@options[:collections]}':")
+        log.error(e.to_s)
+        exit(1)
+      end
     end
 
     def run
@@ -139,183 +149,16 @@ module MoSQL
       @tailer = MoSQL::Tailer.new([@mongo], :existing, metadata_table,
                                   :service => options[:service])
 
-      if options[:reimport] || tailer.read_timestamp.seconds == 0
-        initial_import
-      end
-
-      unless options[:skip_tail]
-        optail
-      end
-    end
-
-    # Helpers
-
-    def collection_for_ns(ns)
-      dbname, collection = ns.split(".", 2)
-      @mongo.db(dbname).collection(collection)
-    end
-
-    def bulk_upsert(table, ns, items)
-      begin
-        @schemamap.copy_data(table.db, ns, items)
-      rescue Sequel::DatabaseError => e
-        log.debug("Bulk insert error (#{e}), attempting invidual upserts...")
-        cols = @schemamap.all_columns(@schemamap.find_ns(ns))
-        items.each do |it|
-          h = {}
-          cols.zip(it).each { |k,v| h[k] = v }
-          @sql.upsert(table, @schemamap.primary_sql_key_for_ns(ns), h)
-        end
-      end
-    end
-
-    def with_retries(tries=10)
-      tries.times do |try|
-        begin
-          yield
-        rescue Mongo::ConnectionError, Mongo::ConnectionFailure, Mongo::OperationFailure => e
-          # Duplicate key error
-          raise if e.kind_of?(Mongo::OperationFailure) && [11000, 11001].include?(e.error_code)
-          # Cursor timeout
-          raise if e.kind_of?(Mongo::OperationFailure) && e.message =~ /^Query response returned CURSOR_NOT_FOUND/
-          delay = 0.5 * (1.5 ** try)
-          log.warn("Mongo exception: #{e}, sleeping #{delay}s...")
-          sleep(delay)
-        end
-      end
-    end
-
-    def track_time
-      start = Time.now
-      yield
-      Time.now - start
-    end
+      @streamer = Streamer.new(:options => @options,
+                               :tailer  => @tailer,
+                               :mongo   => @mongo,
+                               :sql     => @sql,
+                               :schema  => @schema)
 
-    def initial_import
-      @schemamap.create_schema(@sql.db, !options[:no_drop_tables])
+      @streamer.import
 
       unless options[:skip_tail]
-        start_ts = @mongo['local']['oplog.rs'].find_one({}, {:sort => [['$natural', -1]]})['ts']
-      end
-
-      want_dbs = @schemamap.all_mongo_dbs & @mongo.database_names
-      want_dbs.each do |dbname|
-        log.info("Importing for Mongo DB #{dbname}...")
-        db = @mongo.db(dbname)
-        want = Set.new(@schemamap.collections_for_mongo_db(dbname))
-        db.collections.select { |c| want.include?(c.name) }.each do |collection|
-          ns = "#{dbname}.#{collection.name}"
-          import_collection(ns, collection)
-          exit(0) if @done
-        end
-      end
-
-      tailer.write_timestamp(start_ts) unless options[:skip_tail]
-    end
-
-    def import_collection(ns, collection)
-      log.info("Importing for #{ns}...")
-      count = 0
-      batch = []
-      table = @sql.table_for_ns(ns)
-      table.truncate unless options[:no_drop_tables]
-
-      start    = Time.now
-      sql_time = 0
-      collection.find(nil, :batch_size => BATCH) do |cursor|
-        with_retries do
-          cursor.each do |obj|
-            batch << @schemamap.transform(ns, obj)
-            count += 1
-
-            if batch.length >= BATCH
-              sql_time += track_time do
-                bulk_upsert(table, ns, batch)
-              end
-              elapsed = Time.now - start
-              log.info("Imported #{count} rows (#{elapsed}s, #{sql_time}s SQL)...")
-              batch.clear
-              exit(0) if @done
-            end
-          end
-        end
-      end
-
-      unless batch.empty?
-        bulk_upsert(table, ns, batch)
-      end
-    end
-
-    def optail
-      tailer.tail_from(options[:tail_from] ?
-                       BSON::Timestamp.new(options[:tail_from].to_i, 0) :
-                       nil)
-      until @done
-        tailer.stream(1000) do |op|
-          handle_op(op)
-        end
-      end
-    end
-
-    def sync_object(ns, _id)
-      primary_sql_key = @schemamap.primary_sql_key_for_ns(ns)
-      sqlid           = @sql.transform_one_ns(ns, { '_id' => _id })[primary_sql_key]
-      obj             = collection_for_ns(ns).find_one({:_id => _id})
-      if obj
-        @sql.upsert_ns(ns, obj)
-      else
-        @sql.table_for_ns(ns).where(primary_sql_key.to_sym => sqlid).delete()
-      end
-    end
-
-    def handle_op(op)
-      log.debug("processing op: #{op.inspect}")
-      unless op['ns'] && op['op']
-        log.warn("Weird op: #{op.inspect}")
-        return
-      end
-
-      unless @schemamap.find_ns(op['ns'])
-        log.debug("Skipping op for unknown ns #{op['ns']}...")
-        return
-      end
-
-      ns = op['ns']
-      dbname, collection_name = ns.split(".", 2)
-
-      case op['op']
-      when 'n'
-        log.debug("Skipping no-op #{op.inspect}")
-      when 'i'
-        if collection_name == 'system.indexes'
-          log.info("Skipping index update: #{op.inspect}")
-        else
-          @sql.upsert_ns(ns, op['o'])
-        end
-      when 'u'
-        selector = op['o2']
-        update   = op['o']
-        if update.keys.any? { |k| k.start_with? '$' }
-          log.debug("resync #{ns}: #{selector['_id']} (update was: #{update.inspect})")
-          sync_object(ns, selector['_id'])
-        else
-          log.debug("upsert #{ns}: _id=#{selector['_id']}")
-
-          # The update operation replaces the existing object, but
-          # preserves its _id field, so grab the _id off of the
-          # 'query' field -- it's not guaranteed to be present on the
-          # update.
-          update = { '_id' => selector['_id'] }.merge(update)
-          @sql.upsert_ns(ns, update)
-        end
-      when 'd'
-        if options[:ignore_delete]
-          log.debug("Ignoring delete op on #{ns} as instructed.")
-        else
-          @sql.delete_ns(ns, op['o'])
-        end
-      else
-        log.info("Skipping unknown op #{op.inspect}")
+        @streamer.optail
       end
     end
   end

data/lib/mosql/schema.rb

@@ -10,9 +10,9 @@ module MoSQL
         if ent.is_a?(Hash) && ent[:source].is_a?(String) && ent[:type].is_a?(String)
           # new configuration format
           array << {
-            :source => ent.delete(:source),
-            :type   => ent.delete(:type),
-            :name   => ent.first.first,
+            :source => ent.fetch(:source),
+            :type   => ent.fetch(:type),
+            :name   => (ent.keys - [:source, :type]).first,
           }
         elsif ent.is_a?(Hash) && ent.keys.length == 1 && ent.values.first.is_a?(String)
           array << {
@@ -21,7 +21,7 @@ module MoSQL
             :type   => ent.first.last
           }
         else
-          raise "Invalid ordered hash entry #{ent.inspect}"
+          raise SchemaError.new("Invalid ordered hash entry #{ent.inspect}")
         end
 
       end
@@ -32,7 +32,7 @@ module MoSQL
       seen = Set.new
       spec[:columns].each do |col|
         if seen.include?(col[:source])
-          raise "Duplicate source #{col[:source]} in column definition #{col[:name]} for #{ns}."
+          raise SchemaError.new("Duplicate source #{col[:source]} in column definition #{col[:name]} for #{ns}.")
         end
         seen.add(col[:source])
       end
@@ -40,44 +40,75 @@ module MoSQL
 
     def parse_spec(ns, spec)
       out = spec.dup
-      out[:columns] = to_array(spec[:columns])
+      out[:columns] = to_array(spec.fetch(:columns))
       check_columns!(ns, out)
       out
     end
 
+    def parse_meta(meta)
+      meta = {} if meta.nil?
+      meta[:alias] = [] unless meta.key?(:alias)
+      meta[:alias] = [meta[:alias]] unless meta[:alias].is_a?(Array)
+      meta[:alias] = meta[:alias].map { |r| Regexp.new(r) }
+      meta
+    end
+
     def initialize(map)
       @map = {}
       map.each do |dbname, db|
-        @map[dbname] ||= {}
+        @map[dbname] = { :meta => parse_meta(db[:meta]) }
         db.each do |cname, spec|
-          @map[dbname][cname] = parse_spec("#{dbname}.#{cname}", spec)
+          next unless cname.is_a?(String)
+          begin
+            @map[dbname][cname] = parse_spec("#{dbname}.#{cname}", spec)
+          rescue KeyError => e
+            raise SchemaError.new("In spec for #{dbname}.#{cname}: #{e}")
+          end
         end
       end
     end
 
     def create_schema(db, clobber=false)
-      @map.values.map(&:values).flatten.each do |collection|
-        meta = collection[:meta]
-        log.info("Creating table '#{meta[:table]}'...")
-        db.send(clobber ? :create_table! : :create_table?, meta[:table]) do
-          collection[:columns].each do |col|
-            column col[:name], col[:type]
-
-            if col[:source].to_sym == :_id
-              primary_key [col[:name].to_sym]
+      @map.values.each do |dbspec|
+        dbspec.each do |n, collection|
+          next unless n.is_a?(String)
+          meta = collection[:meta]
+          log.info("Creating table '#{meta[:table]}'...")
+          db.send(clobber ? :create_table! : :create_table?, meta[:table]) do
+            collection[:columns].each do |col|
+              opts = {}
+              if col[:source] == '$timestamp'
+                opts[:default] = Sequel.function(:now)
+              end
+              column col[:name], col[:type], opts
+
+              if col[:source].to_sym == :_id
+                primary_key [col[:name].to_sym]
+              end
+            end
+            if meta[:extra_props]
+              column '_extra_props', 'TEXT'
             end
           end
-          if meta[:extra_props]
-            column '_extra_props', 'TEXT'
-          end
         end
       end
     end
 
+    def find_db(db)
+      unless @map.key?(db)
+        @map[db] = @map.values.find do |spec|
+          spec && spec[:meta][:alias].any? { |a| a.match(db) }
+        end
+      end
+      @map[db]
+    end
+
     def find_ns(ns)
       db, collection = ns.split(".")
-      schema = (@map[db] || {})[collection]
-      if schema.nil?
+      unless spec = find_db(db)
+        return nil
+      end
+      unless schema = spec[collection]
         log.debug("No mapping for ns: #{ns}")
         return nil
       end
@@ -109,6 +140,15 @@ module MoSQL
       val
     end
 
+    def fetch_special_source(obj, source)
+      case source
+      when "$timestamp"
+        Sequel.function(:now)
+      else
+        raise SchemaError.new("Unknown source: #{source}")
+      end
+    end
+
     def transform(ns, obj, schema=nil)
       schema ||= find_ns!(ns)
 
@@ -119,10 +159,16 @@ module MoSQL
         source = col[:source]
         type = col[:type]
 
-        v = fetch_and_delete_dotted(obj, source)
-        case v
-        when BSON::Binary, BSON::ObjectId
-          v = v.to_s
+        if source.start_with?("$")
+          v = fetch_special_source(obj, source)
+        else
+          v = fetch_and_delete_dotted(obj, source)
+          case v
+          when BSON::Binary, BSON::ObjectId, Symbol
+            v = v.to_s
+          when Hash, Array
+            v = JSON.dump(v)
+          end
         end
         row << v
       end
@@ -130,10 +176,18 @@ module MoSQL
       if schema[:meta][:extra_props]
         # Kludgily delete binary blobs from _extra_props -- they may
         # contain invalid UTF-8, which to_json will not properly encode.
+        extra = {}
         obj.each do |k,v|
-          obj.delete(k) if v.is_a?(BSON::Binary)
+          case v
+          when BSON::Binary
+            next
+          when Float
+            # NaN is illegal in JSON. Translate into null.
+            v = nil if v.nan?
+          end
+          extra[k] = v
         end
-        row << obj.to_json
+        row << JSON.dump(extra)
       end
 
       log.debug { "Transformed: #{row.inspect}" }
@@ -141,10 +195,14 @@ module MoSQL
       row
     end
 
-    def all_columns(schema)
+    def copy_column?(col)
+      col[:source] != '$timestamp'
+    end
+
+    def all_columns(schema, copy=false)
       cols = []
       schema[:columns].each do |col|
-        cols << col[:name]
+        cols << col[:name] unless copy && !copy_column?(col)
       end
       if schema[:meta][:extra_props]
         cols << "_extra_props"
@@ -152,11 +210,15 @@ module MoSQL
       cols
     end
 
+    def all_columns_for_copy(schema)
+      all_columns(schema, true)
+    end
+
     def copy_data(db, ns, objs)
       schema = find_ns!(ns)
       db.synchronize do |pg|
         sql = "COPY \"#{schema[:meta][:table]}\" " +
-          "(#{all_columns(schema).map {|c| "\"#{c}\""}.join(",")}) FROM STDIN"
+          "(#{all_columns_for_copy(schema).map {|c| "\"#{c}\""}.join(",")}) FROM STDIN"
         pg.execute(sql)
         objs.each do |o|
           pg.put_copy_data(transform_to_copy(ns, o, schema) + "\n")
@@ -178,13 +240,15 @@ module MoSQL
         't'
       when false
         'f'
+      when Sequel::SQL::Function
+        nil
       else
         val.to_s.gsub(/([\\\t\n\r])/, '\\\\\\1')
       end
     end
 
     def transform_to_copy(ns, row, schema=nil)
-      row.map { |c| quote_copy(c) }.join("\t")
+      row.map { |c| quote_copy(c) }.compact.join("\t")
     end
 
     def table_for_ns(ns)

data/lib/mosql/sql.rb

@@ -35,7 +35,7 @@ module MoSQL
 
     def upsert_ns(ns, obj)
       h = transform_one_ns(ns, obj)
-      upsert(table_for_ns(ns), @schema.primary_sql_key_for_ns(ns), h)
+      upsert!(table_for_ns(ns), @schema.primary_sql_key_for_ns(ns), h)
     end
 
     # obj must contain an _id field. All other fields will be ignored.
@@ -46,26 +46,27 @@ module MoSQL
       table_for_ns(ns).where(primary_sql_key.to_sym => h[primary_sql_key]).delete
     end
 
-    def upsert(table, table_primary_key, item)
-      begin
-        upsert!(table, table_primary_key, item)
-      rescue Sequel::DatabaseError => e
-        wrapped = e.wrapped_exception
-        if wrapped.result
-          log.warn("Ignoring row (#{table_primary_key}=#{item[table_primary_key]}): #{e}")
-        else
-          raise e
+    def upsert!(table, table_primary_key, item)
+      rows = table.where(table_primary_key.to_sym => item[table_primary_key]).update(item)
+      if rows == 0
+        begin
+          table.insert(item)
+        rescue Sequel::DatabaseError => e
+          raise e unless self.class.duplicate_key_error?(e)
+          log.info("RACE during upsert: Upserting #{item} into #{table}: #{e}")
         end
+      elsif rows > 1
+        log.warn("Huh? Updated #{rows} > 1 rows: upsert(#{table}, #{item})")
       end
     end
 
-    def upsert!(table, table_primary_key, item)
-      begin
-        table.insert(item)
-      rescue Sequel::DatabaseError => e
-        raise e unless e.message =~ /duplicate key value violates unique constraint/
-        table.where(table_primary_key.to_sym => item[table_primary_key]).update(item)
-      end
+    def self.duplicate_key_error?(e)
+      # c.f. http://www.postgresql.org/docs/9.2/static/errcodes-appendix.html
+      # for the list of error codes.
+      #
+      # No thanks to Sequel and pg for making it easy to figure out
+      # how to get at this error code....
+      e.wrapped_exception.result.error_field(PG::Result::PG_DIAG_SQLSTATE) == "23505"
     end
   end
 end