wyrm 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c0bb0fe99a301ead2da2ce8a64dc1eb20c925b0
-  data.tar.gz: 031b66ab01f20c5ebad94dbfa3c50338dbd15cba
+  metadata.gz: 7c67007e35f84f5542da888c17fc91f6e1fe03cb
+  data.tar.gz: 7434160ebe9385f72d04546b69480d457a13e315
 SHA512:
-  metadata.gz: 5feadc5c19a9df8417414cb91270ae55b8d114cf94fc9daa33b01b2f23292e858a109b0d703cfc7474952b963c26876d7e732f12c0802146cd5ef7838e803629
-  data.tar.gz: a73a7c30e43a430fb05d22552f5b0f10c81cdcd99ec818ff8f838f6ff1a6f59f3c9d71a9d8eacdd26ff4066f1140e82e0236db0e13c2421fba8b1800b7d35710
+  metadata.gz: 11ccf21e5c471e7c06a2a0c27739875d1197570d3c2049a27d284327bd748a0411f3280ab9c89bb865c2c882aa1019a371ecff5105b86baab81c01ce9be8520c
+  data.tar.gz: 96f35474f2770b85b277b5b1cdb64798b4cba183d9ad6b7753d2ded403f878bd7130c4ca634b69262ee812160147a4c674466ca4347fef42c71c2e99e87f9934

data/.gitignore

@@ -15,3 +15,5 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+wyrm.sublime-project
+wyrm.sublime-workspace

data/.rvmrc

@@ -1 +1 @@
-rvm 2.0.0@wyrm --create
+rvm 2.1.1@wyrm --create

data/Gemfile

@@ -20,8 +20,5 @@ end
 
 preferred_sources.each{|src| source src}
 
-gem 'sequel'
-gem 'fastandand'
-
 # Specify your gem's dependencies in wyrm.gemspec
 gemspec

data/README.md

@@ -1,13 +1,18 @@
 # Wyrm
 
-Transfer data from one database to another. Has been used to dump > 100M dbs,
-and one 850G db. Should theoretically work for any dbs supported by Sequel.
+[![Gem Version](https://badge.fury.io/rb/wyrm.png)](http://badge.fury.io/rb/wyrm)
+
+Transfer a database from one rdbms to another (eg mysql to postgres). Either via
+a set of files, or direct from one db server to another.
+
+Has been used to dump > 100M dbs, and one 850G db.
+Should theoretically work for any rdbms supported by [Sequel](http://sequel.jeremyevans.net/).
 
 Dumps are compressed with bz2, using pbzip2. Fast *and* small :-D For example:
 mysqldump | bzip2 for a certain 850G db comes to 127G. With wyrm it
 comes to 134G.
 
-Currently transfers tables and views only. Does not attempt to transfer
+Transfers tables and views only. Does not attempt to transfer
 stored procs, permissions, triggers etc.
 
 Handles tables with a single numeric key, single non-numeric key, and no
@@ -19,7 +24,7 @@ Will use result set streaming if available.
 Wyrm because:
 
 - I like dragons
-- I can (eventually) have a Wyrm::Hole to transfer data through ;-)
+- I can have a Wyrm::Hole to transfer data ;-)
 
 ## Dependencies
 
@@ -29,7 +34,6 @@ on your path.
 
 ## Installation
 
-
 Add this line to your application's Gemfile:
 
     gem 'wyrm'
@@ -52,6 +56,11 @@ Make sure you install the db gems, typically
 
 Very basic cli at this point.
 
+#### For direct db-to-db transfer
+
+    $ wyrm mysql2://localhost/beeg_data_bays postgres://localhost/betta_dee_bee
+
+#### Via files
 From the source db to the file system
 
     $ wyrm mysql2://localhost/beeg_data_bays /tmp/lots_fs_space
@@ -70,20 +79,18 @@ For restoring. dump will be similar.
 
 ``` ruby
 require 'wyrm/restore_schema'
-rs = RestoreSchema.new 'postgres://postgres@localhost/your_db', '/mnt/disk/wyrm'
-rs.create
-rs.restore_tables
-rs.index
+rs = Restore.new 'postgres://postgres@localhost/your_db', '/mnt/disk/wyrm'
+rs.call
 ```
 
 Or for the lower-level stuff
 
 ``` ruby
 require 'sequel'
-require 'wyrm/db_pump'
+require 'wyrm/pump'
 
 db = Sequel.connect 'postgres://postgres@localhost/other_db'
-dbp = DbPump.new db, :things
+dbp = Wyrm::Pump.new db, :things
 dbp.io = IO.popen 'pbzip2 -d -c /mnt/disk/wyrm/things.dbp.bz2'
 dbp.each_row do |row|
   puts row.inspect

data/bin/wyrm

@@ -1,36 +1,52 @@
 #! /usr/bin/env ruby
 
-require 'pathname'
 require 'uri'
 
-def restore( db, directory )
-  require 'wyrm/restore_schema'
-  rs = RestoreSchema.new db, directory
-  rs.create
-  rs.restore_tables
-  rs.index
-end
+if ARGV.size != 2
+  puts <<EOF
+Usage: #{$0} src_db|dirname dst_db|dirname
+
+dirname contains a set of wyrm files, or will soon.
 
-def dump( db, directory )
-  require 'wyrm/dump_schema'
-  ds = DumpSchema.new db, directory
-  ds.dump_schema
-  ds.dump_tables
+sample db strings:
+  postgres://localhost/lotsa_datsa
+  mysql://root:pwned@localhost/lotsa_datsa
+  mysql2://root:pwned@localhost/lotsa_fastsa_datsa
+EOF
+  exit(1)
 end
 
-if ARGV.empty?
-  puts "Provide source and destination"
-  puts "Either can be a sequel db string or a directory"
+module FsPath
+  def fs_path?
+    scheme == 'file' || scheme.nil?
+  end
 end
 
-src, dst = ARGV.map{|arg| URI.parse arg}
+src, dst = ARGV.map{|arg| URI.parse(arg).extend(FsPath)}
+
+require 'wyrm/cli'
+Wyrm.sanity_check_pbzip2
+
+include Wyrm
+
+case
+when src.fs_path? && dst.fs_path?
+  puts "No point copying one directory to another. Just use filesystem tools. It's faster."
+  exit(1)
+
+when !src.fs_path? && dst.fs_path?
+  # src is a url, and dst is a path, so dump to file system
+  require 'wyrm/dump'
+  Dump.new( src.to_s, dst.path ).call
+
+when src.fs_path? && !dst.fs_path?
+  # src is a path and dst is a url, so restore to db
+  require 'wyrm/restore'
+  Restore.new(src.path, dst.to_s, drop_tables: true).call
 
-if src.scheme && Pathname(dst.to_s).exist?
-  # src is a db path, so dump from it
-  dump( src.to_s, dst.to_s )
-elsif dst.scheme && Pathname(src.to_s).exist?
-  # dst is a path and src is a url, so restore
-  restore( dst.to_s, src.to_s )
 else
-  puts "Don't know how to handle #{src} -> #{dst}"
+  # both db urls, so transfer
+  require 'wyrm/hole.rb'
+  Hole.new( src.to_s, dst.to_s ).call
+
 end

data/bin/wyrm-view

@@ -0,0 +1,34 @@
+#! /usr/bin/env ruby
+
+if ARGV.empty? || ARGV.first =~ /^-+(\?|h(elp)?)$/
+  puts <<EOF
+Usage: #{$0} dbp_file.dbp[.bz2]
+
+Display contents of dbp file, optionally compressed with bz2.
+EOF
+  exit(0)
+end
+
+require 'wyrm/cli'
+Wyrm.sanity_check_pbzip2
+
+require 'pathname'
+require 'yaml'
+
+out_block = lambda do |io|
+  record_count = 1
+  until io.eof?
+    puts "# record #{record_count}"
+    puts Marshal.load(io).to_yaml
+    record_count += 1
+  end
+end
+
+ARGV.map{|a| Pathname(a)}.each do |path|
+  puts "# source: #{path}"
+  if path.extname == '.bz2'
+    IO.popen "pbzip2 -d -c #{path}", &out_block
+  else
+    path.open &out_block
+  end
+end

data/lib/wyrm.rb

@@ -1,8 +1,6 @@
+require 'wyrm/module'
+
 require "wyrm/version"
-require "wyrm/db_pump.rb"
+require "wyrm/pump.rb"
 require "wyrm/restore_schema.rb"
 require "wyrm/dump_schema.rb"
-
-module Wyrm
-  # Your code goes here...
-end

data/lib/wyrm/cli.rb

@@ -0,0 +1,9 @@
+module Wyrm
+  def self.sanity_check_pbzip2
+    rv = `which pbzip2`
+    unless $?.exitstatus == 0
+      puts "\npbzip2 not installed or not in PATH"
+      exit(1)
+    end
+  end
+end

data/lib/wyrm/core_extensions.rb

@@ -0,0 +1,10 @@
+# would be nice to use refinements here, but that breaks 2.0 compatibility
+class Method
+  def kwargs_as_hash( invocation_binding )
+    named_locals = parameters. \
+      select{|type,_| type == :key}. \
+      flat_map{|_,name| [name,invocation_binding.eval(name.to_s)]}
+
+    Hash[ *named_locals ]
+  end
+end

data/lib/wyrm/{dump_schema.rb → dump.rb}

@@ -1,18 +1,25 @@
-require 'logger'
+require 'pathname'
+
+require 'wyrm/module'
 require 'wyrm/pump_maker'
+require 'wyrm/schema_tools'
+require 'wyrm/logger'
 
 # Dump a schema and compressed data from a db to a set of files
 #  src_db = Sequel.connect "postgres://localhost:5454/lots"
 #  ds = DumpSchema.new src_db, Pathname('/var/data/lots')
-#  ds.dump_schema
-#  ds.dump_tables
+#  ds.call
 # TODO possibly use Gem::Package::TarWriter to write tar files
-class DumpSchema
+class Wyrm::Dump
   include PumpMaker
+  include SchemaTools
+  include Wyrm::Logger
 
   def initialize( src_db, container = nil, pump: nil )
+    @container = Pathname.new container || '.'
+    raise "#{@container} does not exist" unless @container.exist?
+
     @src_db = maybe_deebe src_db
-    @container = Pathname.new container
     @pump = make_pump( @src_db, pump )
 
     @src_db.extension :schema_dumper
@@ -20,33 +27,21 @@ class DumpSchema
 
   attr_reader :src_db, :container, :pump
 
-  def schema_migration
-    @schema_migration ||= src_db.dump_schema_migration(:indexes=>false, :same_db => same_db)
-  end
-
-  def index_migration
-    @index_migration ||= src_db.dump_indexes_migration(:same_db => same_db)
-  end
-
-  def fk_migration
-    @fk_migration ||= src_db.dump_foreign_key_migration(:same_db => same_db)
-  end
-
   def same_db
     false
   end
 
-  def logger
-    @logger ||= Logger.new STDERR
+  def numbering
+    @numbering ||= '000'
   end
 
   def dump_schema
-    numbering = '000'
-
     (container + "#{numbering.next!}_schema.rb").open('w') do |io|
       io.write schema_migration
     end
+  end
 
+  def dump_indexes
     (container + "#{numbering.next!}_indexes.rb").open('w') do |io|
       io.write index_migration
     end
@@ -107,4 +102,10 @@ class DumpSchema
       dump_table table_name
     end
   end
+
+  def call
+    dump_schema
+    dump_tables
+    dump_indexes
+  end
 end

data/lib/wyrm/hole.rb

@@ -0,0 +1,164 @@
+require 'thread'
+require 'wyrm/logger'
+
+require 'wyrm/module'
+require 'wyrm/pump'
+require 'wyrm/pump_maker'
+require 'wyrm/schema_tools'
+require 'wyrm/core_extensions'
+
+module Wyrm
+  # This bypasses the need to marshal objects between two pumps.
+  # It uses a queue of the record arrays instead.
+  class Hole
+    include PumpMaker
+    include Logger
+
+    # This is the codec. Named for the mouth of a wormhole. Cos finding a good name for this is hard.
+    #
+    # Connects the two pumps together. Implements Codec, Quacks like IO.
+    class Mouth
+      include Logger
+
+      def initialize
+        @flushed = false
+      end
+
+      # This is a bit weird because io_queue will usually == self
+      def encode( obj, io_queue )
+        io_queue.enq obj
+      end
+
+      # This is a bit weird because io_queue will usually == self
+      def decode( io_queue, &block )
+        obj = io_queue.deq
+        yield obj if block_given?
+        obj
+      end
+
+      def reset
+        # do this first, so any (hopefully not) remaining waiters don't
+        # go into the blocking deq again.
+        @flushed = false
+
+        # clear any poisons, and release any (hopefully not) remaining waiters
+        queue.clear
+      end
+
+      # queue could be empty while producer is generating something,
+      # so only eof after flush has been called.
+      def eof?
+        # queue is not empty if it's been poisoned.
+        @flushed && queue.empty?
+      end
+
+      # use a SizedQueue so we don't run out of memory during a big transfer
+      def queue
+        @queue ||=
+        if RUBY_VERSION == '2.1.0'
+          logger.warn "SizedQueue broken in 2.1.0 (https://bugs.ruby-lang.org/issues/9302). Falling back to Queue, which may run out of memory."
+          Queue.new
+        else
+          SizedQueue.new 5000
+        end
+      end
+
+      def enq( value )
+        queue.enq value
+      end
+
+      def deq( *args )
+        rv = queue.deq( *args )
+        if rv == :poison
+          poison_queue
+          raise StopIteration
+        end
+        rv
+      end
+
+      def poison_queue
+        # poison the queue. waiters will have to re-queue this.
+        queue << :poison if queue.empty? && queue.num_waiting > 0
+      end
+
+      # this gets called after dump is finished, by pump
+      def flush
+        # do this first, so any non-poisoned waiters will eof
+        # synchronisation is not really important because
+        @flushed = true
+        poison_queue
+      end
+    end
+
+    def initialize( src_db, dst_db, drop_tables: true, queue_size: 5000 )
+      # called only once per run, so not really a performance issue
+      @options = method(__method__).kwargs_as_hash( binding )
+
+      @src_db = maybe_deebe src_db
+      @dst_db = maybe_deebe dst_db
+
+      @src_db.extension :schema_dumper
+    end
+
+    attr_reader :src_db, :dst_db, :options
+
+    def mouth
+      @mouth ||= Mouth.new
+    end
+
+    def src_pump
+      @src_pump ||= Pump.new( {db: src_db, io: mouth, codec: mouth, logger: logger}.merge( options[:pump] ||{} ) )
+    end
+
+    def dst_pump
+      @dst_pump ||= Pump.new( {db: dst_db, io: mouth, codec: mouth, logger: logger}.merge( options[:pump] ||{} ) )
+    end
+
+    def transfer_table( table_name )
+      mouth.reset
+      src_pump.table_name = dst_pump.table_name = table_name
+
+      if src_pump.table_dataset.empty?
+        logger.info "No records in #{table_name}"
+        return
+      end
+
+      # Use threads so the db read/writes aren't waiting for one another.
+      recv_thread = Thread.new{ dst_pump.restore }
+      send_thread = Thread.new{ src_pump.dump }
+
+      send_thread.join
+      recv_thread.join
+    end
+
+    include SchemaTools
+
+    def transfer_schema( &transfer_table_block )
+      create_tables
+
+      # transfer tables here
+      yield self if block_given?
+
+      create_indexes
+    end
+
+    def transfer_tables
+      logger.info "transferring tables"
+      src_db.tables.each do |table_name|
+        transfer_table table_name
+      end
+    end
+
+    def call
+      if options[:drop_tables]
+        logger.info "dropping tables"
+        drop_tables src_db.tables
+      end
+
+      transfer_schema do
+        transfer_tables
+      end
+    end
+
+  end
+end