wyrm 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85b0989b28233c55407b8c0277970f7b733235f1
-  data.tar.gz: d12c0b1ccf156bbc808cf302b2c2ad0ded122b52
+  metadata.gz: da207f92151b080d31039a364c1a2c50022f01ff
+  data.tar.gz: ddf38f48b42597ed08671cb67fadd8457e425e07
 SHA512:
-  metadata.gz: 301e296b3108284415d4041773ecb45f92907128fc548886c79b63d7db14683d4fdf79ad4515cee5bf6122b6e5a0a66528e716e29a779c56b1e3a72f01158bf2
-  data.tar.gz: 14084646f239d950c8def8056fc7cc9ea54a458dccbd04384ebeee68bf6103f4f881f84ef1aa44fd56cd24839aeb73e31e8bd617091a3a0b88602fff7bfb898d
+  metadata.gz: 03e699a00d14fa7baacc286b886cf35074766b7b6b3b8e6e10fde08779ded7fda4930f9666bad95274bf773f8fe33f1916f3836414e98c188c81963b3a01459c
+  data.tar.gz: 0052a0b096e62662223f9e4a9da2cfd79e2908f033f83a7a9201463aac3ec9f56407299b68e461b9b850f39ea8c3da5e53c48ad7510561d680b601a38ce739ca

data/History.txt

@@ -1,3 +1,7 @@
+== 0.4.1
+* Improve docs and examples
+* make pbzip2 somewhat configurable.
+
 == 0.4.0
 * Update for released ruby-2.3 Queue#close
 * use &. instead of andand
@@ -9,5 +13,6 @@
 
 == 0.3.3
 * minor fix when src/dst directory not found
+
 == 0.3.2
 * update for >=sequel-4.10.0 internal syntax (Dataset#clause_sql removed)

data/README.md

@@ -4,15 +4,15 @@
 [![Build Status](https://travis-ci.org/djellemah/wyrm.png?branch=master)](https://travis-ci.org/djellemah/wyrm)
 -->
 
-Transfer a database from one rdbms to another (eg mysql to postgres). Either via
-a set of files, or direct from one db server to another.
+Transfer a database (or single tables) from one rdbms to another (eg mysql to
+postgres). Either via a set of files, or direct from one db server to another.
 
 Has been used to dump > 100M dbs, and one 850G db.
 Should theoretically work for any rdbms supported by [Sequel](http://sequel.jeremyevans.net/).
 
-Dumps are compressed with bz2, using pbzip2. Fast *and* small :-D For example:
-mysqldump | bzip2 for a certain 850G db comes to 127G. With wyrm it
-comes to 134G.
+Dumps are (usually) compressed with bz2, using pbzip2. Fast *and* small :-D
+For example: mysqldump | bzip2 for a certain 850G db comes to 127G. With wyrm
+it comes to 134G.
 
 Transfers tables and views only. Does not attempt to transfer
 stored procs, permissions, triggers etc.
@@ -26,13 +26,14 @@ Will use result set streaming if available.
 Wyrm because:
 
 - I like dragons
-- I can have a Wyrm::Hole to transfer data ;-)
+- I can have a Wyrm::Hole to transfer data O-;-)
 
 ## Dependencies
 
 You must have a working
 [pbzip2](http://compression.ca/pbzip2/ "Will use all your cores")
-on your path.
+on your path. If you really have to use something else,
+reassign ```Wyrm::STREAM_DCMP``` and ```Wyrm::STREAM_COMP``` .
 
 ## Installation
 
@@ -75,25 +76,49 @@ On the destination host
 
     $ wyrm /tmp/lots_fs_space postgres://localhost/betta_dee_bee
 
+#### View contents of a dump file as yaml
+
+    $ wyrm-view /tmp/lots_fs_space/some_table.dbp.bz2
+
 ### irb / pry
 
 For restoring. dump will be similar.
 
 ``` ruby
 require 'wyrm/restore_schema'
+
 rs = Restore.new 'postgres://postgres@localhost/your_db', '/mnt/disk/wyrm'
 rs.call
 ```
 
-Or for the lower-level stuff
+Directly transferring one table:
 
 ``` ruby
 require 'sequel'
+require 'wyrm/hole'
+
+hole = Wyrm::Hole.new 'mysql2://localhost/beeg_data_bays', 'postgres://localhost/betta_dee_bee'
+
+# transfer schema (no indexes), using Sequel::SchemaDumper extension, see
+# Sequel::SchemaDumper#dump_schema_migration for options
+table_schema = hole.src_db.dump_table_schema :the_stuff_you_want
+Sequel.migration{ change{ eval table_schema } }.apply hole.dst_db, :up
+
+hole.transfer_table :the_stuff_you_want
+
+# it's just Sequel...
+hold.dst_db[:the_stuff_you_want].where( some_thing: /a pattern/ ).limit(10).all
+```
+
+Get to the dumped rows:
+
+``` ruby
+require 'sequel' # for some demarshaling
 require 'wyrm/pump'
 
-db = Sequel.connect 'postgres://postgres@localhost/other_db'
-dbp = Wyrm::Pump.new db, :things
-dbp.io = IO.popen 'pbzip2 -d -c /mnt/disk/wyrm/things.dbp.bz2'
+dbp = Wyrm::Pump.new io: IO.popen('pbzip2 -d -c /mnt/disk/wyrm/things.dbp.bz2')
+# each_row also returns an Enumerator if no block is given, similar to much
+# ruby core stuff. Although it's not rewindable.
 dbp.each_row do |row|
   puts row.inspect
 end

data/bin/wyrm

@@ -3,7 +3,12 @@
 require 'uri'
 
 if ARGV.size != 2
+  require 'wyrm/version'
+
   puts <<EOF
+
+wyrm-#{Wyrm::VERSION}
+
 Usage: #{$0} src_db|dirname dst_db|dirname
 
 dirname contains a set of wyrm files, or will soon.
@@ -12,6 +17,7 @@ sample db strings:
   postgres://localhost/lotsa_datsa
   mysql://root:pwned@localhost/lotsa_datsa
   mysql2://root:pwned@localhost/lotsa_fastsa_datsa
+
 EOF
   exit(1)
 end
@@ -25,7 +31,7 @@ end
 src, dst = ARGV.map{|arg| URI.parse(arg).extend(FsPath)}
 
 require 'wyrm/cli'
-Wyrm.sanity_check_pbzip2
+Wyrm.sanity_check_dcmp
 
 include Wyrm
 

data/bin/wyrm-view

@@ -4,16 +4,17 @@ if ARGV.empty? || ARGV.first =~ /^-+(\?|h(elp)?)$/
   puts <<EOF
 Usage: #{$0} dbp_file.dbp[.bz2]
 
-Display contents of dbp file, optionally compressed with bz2.
+Display contents of dbp file in yaml, optionally compressed with bz2.
 EOF
   exit(0)
 end
 
 require 'wyrm/cli'
-Wyrm.sanity_check_pbzip2
+Wyrm.sanity_check_dcmp
 
 require 'pathname'
 require 'yaml'
+require 'sequel' # needed for demarshalling some types
 
 out_block = lambda do |io|
   record_count = 1
@@ -27,7 +28,7 @@ end
 ARGV.map{|a| Pathname(a)}.each do |path|
   puts "# source: #{path}"
   if path.extname == '.bz2'
-    IO.popen "pbzip2 -d -c #{path}", &out_block
+    IO.popen "#{Wyrm::STREAM_DCMP} #{path}", &out_block
   else
     path.open &out_block
   end

data/lib/wyrm/cli.rb

@@ -1,8 +1,11 @@
+require 'wyrm/module'
+
 module Wyrm
-  def self.sanity_check_pbzip2
-    rv = `which pbzip2`
+  def self.sanity_check_dcmp
+    bzip_cmd = Wyrm::STREAM_DCMP.split(' ').first
+    rv = `which #{bzip_cmd}`
     unless $?.exitstatus == 0
-      puts "\npbzip2 not installed or not in PATH"
+      puts "\n#{cmd} not installed or not in PATH"
       exit(1)
     end
   end

data/lib/wyrm/dump.rb

@@ -52,7 +52,7 @@ class Wyrm::Dump
   def write_through_bz2( pathname )
     fio = pathname.open('w')
     # open subprocess in read-write mode
-    zio = IO.popen( "pbzip2 -z", 'r+' )
+    zio = IO.popen( STREAM_COMP, 'r+' )
     copier = Thread.new do
       begin
         IO.copy_stream zio, fio

data/lib/wyrm/module.rb

@@ -1 +1,4 @@
-module Wyrm end
+module Wyrm
+  STREAM_COMP = 'pbzip2 -z'
+  STREAM_DCMP = 'pbzip2 -d -c'
+end

data/lib/wyrm/restore.rb

@@ -86,7 +86,7 @@ class Wyrm::Restore
       raise "Don't know what to do with #{table_name.inspect}"
     end
 
-    IO.popen "pbzip2 -d -c #{table_file}", &block
+    IO.popen "#{STREAM_DCMP} #{table_file}", &block
   end
 
   def table_files

data/lib/wyrm/version.rb

@@ -1,3 +1,3 @@
 module Wyrm
-  VERSION = '0.4.0'
+  VERSION = '0.4.1'
 end

data/spec/wyrm_spec.rb

@@ -8,6 +8,6 @@ include Wyrm
 
 describe Wyrm do
   it 'has the right constants' do
-    Wyrm.constants.sort.should == [:Dump, :Hole, :Logger, :Pump, :PumpMaker, :Restore, :SchemaTools, :VERSION].sort
+    Wyrm.constants.sort.should == [:Dump, :Hole, :Logger, :Pump, :PumpMaker, :Restore, :SchemaTools, :VERSION, :STREAM_COMP, :STREAM_DCMP].sort
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wyrm
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - John Anderson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-17 00:00:00.000000000 Z
+date: 2016-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel