wyrm 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c62a5cbc1df1fd2ae5e73af5674a321ae2c04113
-  data.tar.gz: 8e421d58a2cfdb36ab433fccbc5820464318e84b
+  metadata.gz: 85b0989b28233c55407b8c0277970f7b733235f1
+  data.tar.gz: d12c0b1ccf156bbc808cf302b2c2ad0ded122b52
 SHA512:
-  metadata.gz: affd20c4d8423cb9ee3c20d96bf43bfa2de7fc009f2a4fc431828bf2fb08e05fb53b756754887ab6de6acae5cce92955fc59702be131b59f1434e93b409fa73e
-  data.tar.gz: e69e16c69c53ba23d2bacce42b5c000b3271076aef00ffa4f5ca32d708cb9f6fd46894412f369bbf6fb321d84a5d1fc02d1262d3f3f70dc6f2720fb8079f1202
+  metadata.gz: 301e296b3108284415d4041773ecb45f92907128fc548886c79b63d7db14683d4fdf79ad4515cee5bf6122b6e5a0a66528e716e29a779c56b1e3a72f01158bf2
+  data.tar.gz: 14084646f239d950c8def8056fc7cc9ea54a458dccbd04384ebeee68bf6103f4f881f84ef1aa44fd56cd24839aeb73e31e8bd617091a3a0b88602fff7bfb898d

data/.ruby-version

@@ -1 +1 @@
-ruby-2.1
+ruby-2.3

data/.travis.yml

@@ -1,6 +1,4 @@
 language: ruby
-rvm:
-- 2.0.0
-- 2.1.7
-- 2.2.3
+rvm: []
+# - 2.3 not supported as of 16-Mar-2016. srsly, 3 months after release
 script: bundle exec rspec spec

data/Gemfile

@@ -1,3 +1,6 @@
+raise "You need >= ruby-2.3 (or maybe a Queue with close would work)" unless RUBY_VERSION >= '2.3.0'
+
+# TODO this is for older versions of bundler
 def from_gemrc
   # auto-load from ~/.gemrc
   home_gemrc = Pathname('~/.gemrc').expand_path

data/History.txt

@@ -1,5 +1,13 @@
+== 0.4.0
+* Update for released ruby-2.3 Queue#close
+* use &. instead of andand
+* minor tweaks to Pump#inner_dump and Pump#primary_keys
+* pry-debundle for development
+
+== 0.4.0.pre
+* Use ruby-2.3 Queue#close
+
 == 0.3.3
 * minor fix when src/dst directory not found
-
 == 0.3.2
 * update for >=sequel-4.10.0 internal syntax (Dataset#clause_sql removed)

data/README.md

@@ -1,4 +1,8 @@
-# Wyrm [![Gem Version](https://badge.fury.io/rb/wyrm.png)](http://badge.fury.io/rb/wyrm) [![Build Status](https://travis-ci.org/djellemah/wyrm.png?branch=master)](https://travis-ci.org/djellemah/wyrm)
+# Wyrm [![Gem Version](https://badge.fury.io/rb/wyrm.png)](http://badge.fury.io/rb/wyrm)
+
+<!---
+[![Build Status](https://travis-ci.org/djellemah/wyrm.png?branch=master)](https://travis-ci.org/djellemah/wyrm)
+-->
 
 Transfer a database from one rdbms to another (eg mysql to postgres). Either via
 a set of files, or direct from one db server to another.

data/lib/wyrm/dump.rb

@@ -27,9 +27,7 @@ class Wyrm::Dump
 
   attr_reader :src_db, :container, :pump
 
-  def same_db
-    false
-  end
+  def same_db; false end
 
   def numbering
     @numbering ||= '000'

data/lib/wyrm/hole.rb

@@ -14,84 +14,71 @@ module Wyrm
     include PumpMaker
     include Logger
 
-    # This is the codec. Named for the mouth of a wormhole. Cos finding a good name for this is hard.
-    #
-    # Connects the two pumps together. Implements Codec, Quacks like IO.
-    class Mouth
-      include Logger
-
-      def initialize
-        @flushed = false
-      end
-
-      # This is a bit weird because io_queue will usually == self
-      def encode( obj, io_queue )
+    # stateless module methods
+    module QueueCodec
+      def self.encode( obj, io_queue )
         io_queue.enq obj
       end
 
-      # This is a bit weird because io_queue will usually == self
-      def decode( io_queue, &block )
+      def self.decode( io_queue, &block )
         obj = io_queue.deq
         yield obj if block_given?
         obj
       end
+    end
 
-      def reset
-        # do this first, so any (hopefully not) remaining waiters don't
-        # go into the blocking deq again.
-        @flushed = false
+    # This pretends to be just enough of an IO that we can use a queue to
+    # connect a dump to a restore.
+    #
+    # Named for the mouth of a wormhole. Cos finding a good name for it is hard.
+    class Mouth
+      DEFAULT_QUEUE_SIZE = 5000
 
-        # clear any poisons, and release any (hopefully not) remaining waiters
-        queue.clear
+      def initialize( queue_size: DEFAULT_QUEUE_SIZE)
+        raise '>= ruby-2.3.0 needed because we use Queue#close' if RUBY_VERSION < '2.3.0'
+        @queue_size = queue_size
       end
 
-      # queue could be empty while producer is generating something,
-      # so only eof after flush has been called.
-      def eof?
-        # queue is not empty if it's been poisoned.
-        @flushed && queue.empty?
+      #############
+      # interface for Hole
+      def reset
+        if @queue
+          @queue.close.clear
+          @queue = nil
+        end
       end
 
+      # queue could be empty while producer is generating something,
       # use a SizedQueue so we don't run out of memory during a big transfer
       def queue
-        @queue ||=
-        if RUBY_VERSION == '2.1.0'
-          raise "Queue broken in 2.1.0 possibly related to https://bugs.ruby-lang.org/issues/9302"
-        else
-          SizedQueue.new 5000
-        end
+        @queue ||= SizedQueue.new @queue_size
       end
 
-      def enq( value )
-        queue.enq value
-      end
+      ##########
+      # interface for codec
+      def enq( value ); queue.enq value end
 
-      def deq( *args )
-        rv = queue.deq( *args )
-        if rv == :poison
-          poison_queue
-          raise StopIteration
-        end
-        rv
+      def deq
+        queue.deq or raise StopIteration
+      rescue StopIteration
+        raise "nil from deq, but queue not empty" unless queue.empty?
+        raise
       end
 
-      def poison_queue
-        # poison the queue. waiters will have to re-queue this.
-        queue << :poison if queue.empty? && queue.num_waiting > 0
-      end
+      ##############
+      # interface for Pump
 
-      # this gets called after dump is finished, by pump
-      def flush
-        # do this first, so any non-poisoned waiters will eof
-        # synchronisation is not really important because
-        @flushed = true
-        poison_queue
-      end
+      # eof after flush has been called and queue is empty.
+      def eof?; queue.closed? && queue.empty? end
+
+      # this gets called by pump after dump is finished
+      def flush; queue.close end
     end
 
-    def initialize( src_db, dst_db, drop_tables: true, queue_size: 5000 )
+    def initialize( src_db, dst_db, drop_tables: true, queue_size: Mouth::DEFAULT_QUEUE_SIZE )
       # called only once per run, so not really a performance issue
-      @options = method(__method__).kwargs_as_hash( binding )
+      @drop_tables = drop_tables
+      @queue_size = queue_size
 
       @src_db = maybe_deebe src_db
       @dst_db = maybe_deebe dst_db
@@ -99,22 +86,26 @@ module Wyrm
       @src_db.extension :schema_dumper
     end
 
-    attr_reader :src_db, :dst_db, :options
+    attr_reader :src_db, :dst_db, :options, :queue_size
+    def drop_tables?; @drop_tables end
 
     def mouth
-      @mouth ||= Mouth.new
+      @mouth ||= Mouth.new queue_size: queue_size
+    end
+
+    def pump_options
+      {io: mouth, codec: QueueCodec, logger: logger}
     end
 
     def src_pump
-      @src_pump ||= Pump.new( {db: src_db, io: mouth, codec: mouth, logger: logger}.merge( options[:pump] ||{} ) )
+      @src_pump ||= Pump.new db: src_db, **pump_options
     end
 
     def dst_pump
-      @dst_pump ||= Pump.new( {db: dst_db, io: mouth, codec: mouth, logger: logger}.merge( options[:pump] ||{} ) )
+      @dst_pump ||= Pump.new db: dst_db, **pump_options
     end
 
     def transfer_table( table_name )
-      mouth.reset
       src_pump.table_name = dst_pump.table_name = table_name
 
       if src_pump.table_dataset.empty?
@@ -128,36 +119,30 @@ module Wyrm
 
       send_thread.join
       recv_thread.join
+    ensure
+      mouth.reset
+      src_pump.table_name = dst_pump.table_name = nil
     end
 
     include SchemaTools
 
     def transfer_schema( &transfer_table_block )
       create_tables
-
-      # transfer tables here
-      yield self if block_given?
-
+      yield self if block_given? # transfer tables here
       create_indexes
     end
 
     def transfer_tables
       logger.info "transferring tables"
-      src_db.tables.each do |table_name|
-        transfer_table table_name
-      end
+      src_db.tables.each {|table_name| transfer_table table_name }
     end
 
     def call
-      if options[:drop_tables]
+      if drop_tables?
         logger.info "dropping tables"
         drop_tables src_db.tables
       end
-
-      transfer_schema do
-        transfer_tables
-      end
+      transfer_schema { transfer_tables }
     end
-
   end
 end

data/lib/wyrm/pump.rb

@@ -26,7 +26,7 @@ class Wyrm::Pump
   attr_accessor :io, :page_size, :dry_run
   def dry_run?; dry_run; end
 
-  # These affect cached values
+  # These are affected by cached values
   attr_reader :db, :table_name
 
   def invalidate_cached_members
@@ -113,7 +113,8 @@ class Wyrm::Pump
   end
 
   def primary_keys
-    @primary_keys ||= db.schema(table_name).select{|df| df.last[:primary_key]}.map{|df| df.first}
+    # each_with_object([]){...} is only faster for < 3 items in 100000
+    @primary_keys ||= db.schema(table_name).map{|name,column_info| name if column_info[:primary_key]}.compact
   end
 
   def table_dataset
@@ -124,8 +125,8 @@ class Wyrm::Pump
   def paginated_dump( &encode_block )
     records_count = 0
     table_dataset.order(*primary_keys).each_page(page_size) do |page|
-      logger.info{ "#{__method__} #{table_name} #{records_count}" }
-      logger.debug{ page.sql }
+      logger.info "#{__method__} #{table_name} #{records_count}"
+      logger.debug page.sql
       page.each &encode_block
       records_count += page_size
     end
@@ -142,11 +143,12 @@ class Wyrm::Pump
   # http://www.numerati.com/2012/06/26/reading-large-result-sets-with-hibernate-and-mysql/
   def inner_dump( &encode_block )
     # could possibly overrride Dataset#paginate(page_no, page_size, record_count=nil)
-    0.step(table_dataset.count, page_size).each do |offset|
+    on_conditions = primary_keys.map{|f| [f,f]}.to_h
+    (0..table_dataset.count).step(page_size).each do |offset|
       limit_dataset = table_dataset.select( *primary_keys ).limit( page_size, offset ).order( *primary_keys )
-      page = table_dataset.join( limit_dataset, Hash[ primary_keys.map{|f| [f,f]} ] ).order( *primary_keys ).qualify(table_name)
-      logger.info{ "#{__method__} #{table_name} #{offset}" }
-      logger.debug{ page.sql }
+      page = table_dataset.join( limit_dataset, on_conditions ).order( *primary_keys ).qualify(table_name)
+      logger.info "#{__method__} #{table_name} #{offset}"
+      logger.debug page.sql
       page.each &encode_block
     end
   end
@@ -164,14 +166,14 @@ class Wyrm::Pump
     # bigger than max for the last page
     (min..max).step(page_size).each do |offset|
       page = table_dataset.where( id: offset...(offset + page_size) )
-      logger.info{ "#{__method__} #{table_name} #{offset}" }
-      logger.debug{ page.sql }
+      logger.info "#{__method__} #{table_name} #{offset}"
+      logger.debug page.sql
       page.each &encode_block
     end
   end
 
   def stream_dump( &encode_block )
-    logger.debug{ "using result set streaming" }
+    logger.info "using result set streaming"
 
     # I want to output progress every page_size records,
     # without doing a records_count % page_size every iteration.
@@ -186,8 +188,8 @@ class Wyrm::Pump
           records_count += 1
         end
       ensure
-        logger.info{ "#{__method__} #{table_name} #{records_count}" if records_count < page_size }
-        logger.debug{ "  from #{table_dataset.sql}" }
+        logger.info "#{__method__} #{table_name} #{records_count}" if records_count < page_size
+        logger.debug "  #{records_count} from #{table_dataset.sql}"
       end
     end
   end
@@ -218,10 +220,13 @@ class Wyrm::Pump
     case
     when table_dataset.respond_to?( :stream )
       stream_dump &encode_block
+
     when primary_keys.empty?
       paginated_dump &encode_block
+
     when primary_keys.all?{|i| i == :id }
       min_max_dump &encode_block
+
     else
       inner_dump &encode_block
     end
@@ -235,20 +240,21 @@ class Wyrm::Pump
     false
   end
 
+  # start_row is zero-based
+  #
   # TODO don't generate the full insert, ie leave out the fields
   # because we've already checked that the columns and the table
   # match.
   # TODO generate column names in insert, they might still work
   # if columns have been added to the db, but not the dump.
-  # start_row is zero-based
   def restore( start_row: 0, filename: 'io' )
     columns = table_dataset.columns
     row_enum = each_row
 
     return unless dump_matches_columns?( row_enum, columns )
 
-    logger.info{ "#{__method__} inserting to #{table_name} from #{start_row}" }
-    logger.debug{ "  #{columns.inspect}" }
+    logger.info "#{__method__} inserting to #{table_name} from #{start_row}"
+    logger.debug "  #{columns.inspect}"
     rows_restored = 0
 
     if start_row != 0
@@ -283,7 +289,7 @@ class Wyrm::Pump
         end
       end
     end
-    logger.info{ "#{__method__} #{table_name} done. Inserted #{rows_restored}." }
+    logger.info "#{__method__} #{table_name} done. Inserted #{rows_restored}."
     rows_restored
   end
 

data/lib/wyrm/schema_tools.rb

@@ -1,4 +1,3 @@
-require 'fastandand'
 Sequel.extension :migration
 require 'wyrm/module'
 
@@ -8,7 +7,7 @@ require 'wyrm/module'
 module Wyrm::SchemaTools
   # some includers will need to provide a different implementation for this.
   def same_db
-    respond_to?( :dst_db ) && respond_to?( :src_db ) && dst_db.andand.database_type == src_db.andand.database_type
+    respond_to?( :dst_db ) && respond_to?( :src_db ) && dst_db&.database_type == src_db&.database_type
   end
 
   def schema_migration

data/lib/wyrm/version.rb

@@ -1,3 +1,3 @@
 module Wyrm
-  VERSION = "0.3.3"
+  VERSION = '0.4.0'
 end

data/spec/core_extensions_spec.rb

@@ -1,4 +1,5 @@
 require 'rspec_syntax'
+require 'pathname'
 
 require Pathname(__dir__) + '../lib/wyrm/core_extensions.rb'
 

data/spec/hole_mouth_spec.rb

@@ -1,74 +1,26 @@
 require 'rspec_syntax'
+require 'pathname'
 
 require Pathname(__dir__) + '../lib/wyrm/hole.rb'
 
 describe Wyrm::Hole::Mouth do
-if RUBY_VERSION == '2.1.0'
-  it 'Queue broken on 2.1.0'
-else
   describe '#flush' do
-    it 'calls poison_queue' do
-      subject.should_receive(:poison_queue)
+    it 'closes the queue' do
       subject.flush
-    end
-
-    it 'sets flag' do
-      subject.instance_variable_get('@flushed').should_not == true
-      subject.flush
-      subject.instance_variable_get('@flushed').should == true
-    end
-
-    describe 'queue empty with waiters' do
-      THREADS = rand(1..7)
-      def waiters
-        @waiters ||= THREADS.times.map do
-          Thread.new do
-            values = []
-            begin
-              until subject.eof?
-                values << subject.deq
-                sleep( rand * 0.05 )
-              end
-              [:eoffed, values]
-            rescue StopIteration
-              [:poisoned, values]
-            end
-          end
-        end
-      end
-
-      it 'poisons queue' do
-        waiters
-        # wait for thread setup to finish
-        sleep 0.1
-        subject.flush
-        thread_values = waiters.map {|waiter| waiter.join(4).andand.value }
-        thread_values.map(&:first).should == THREADS.times.map{:poisoned}
-      end
-
-      it 'eof queue' do
-        50.times{subject.enq 'hello'}
-        waiters
-        subject.flush
-        thread_values = waiters.map {|waiter| waiter.join(4).andand.value }
-        thread_values.map(&:first).should == THREADS.times.map{:eoffed}
-      end
+      subject.queue.should be_closed
     end
   end
 
   describe '#eof?' do
     describe 'flushed' do
-      before :each do
+      it 'true when queue empty and closed' do
         subject.flush
-      end
-
-      it 'true when queue empty' do
         subject.queue.should be_empty
+        subject.queue.should be_closed
         subject.should be_eof
       end
 
-      it 'false when queue empty' do
-        subject.enq( :arg )
+      it 'false when queue open but empty' do
         subject.should_not be_eof
       end
     end
@@ -87,12 +39,6 @@ else
   end
 
   describe '#reset' do
-    it 'resets flushed' do
-      subject.instance_variable_set '@flushed', true
-      subject.reset
-      subject.instance_variable_get('@flushed').should == false
-    end
-
     it 'clears queue' do
       rand(1..10).times{subject.enq :some_value}
       subject.queue.should_not be_empty
@@ -117,63 +63,10 @@ else
       th.status.should == false
     end
 
-    it 'raise StopIteration on poison' do
-      subject.queue.should be_empty
-      waiter = Thread.new { subject.deq }
-      sleep(0.01) while subject.queue.num_waiting != 1
-      subject.poison_queue
-
-      # this is so cool. Thread#value will re-raise the exception it caught
-      ->{waiter.value}.should raise_error(StopIteration)
-
-      # queue should be empty now
+    it 'raises StopIteration for closed queue' do
       subject.queue.should be_empty
-    end
-
-    it 're-queues poison' do
-      subject.queue << :poison
-      subject.should_receive(:poison_queue)
+      subject.flush
       ->{subject.deq}.should raise_error(StopIteration)
     end
   end
-
-  describe '#poison_queue' do
-    it 'poisons when queue empty with waiters' do
-      subject.queue.should be_empty
-
-      # there has to be a thread waiting for the poison to be added
-      waiter = Thread.new { subject.queue.deq }
-      sleep(0.01) while subject.queue.num_waiting != 1
-      subject.poison_queue
-
-      waiter.value.should == :poison
-
-      # queue should be empty now
-      subject.queue.should be_empty
-    end
-
-    it 'no poison when queue empty' do
-      subject.queue.should be_empty
-      subject.poison_queue
-      subject.queue.should be_empty
-    end
-
-    it 'no poison for no waiters' do
-      subject.queue << :hello
-      subject.queue << :there
-      subject.poison_queue
-      subject.queue.size.should == 2
-    end
-  end
-
-  describe '#logger' do
-    # this is here because the 2.1.0 without SizeQueue branch
-    # has a logger which nothing else uses
-    it 'works' do
-      Wyrm::Hole::Mouth::RUBY_VERSION = '2.1.0'
-      ->{subject.queue}.should raise_error(/broken in 2.1.0/)
-      Wyrm::Hole::Mouth.send :remove_const, :RUBY_VERSION
-    end
-  end
-end #unless RUBY_VERSION == '2.1.0'
 end

data/spec/schema_tools_spec.rb

@@ -22,7 +22,7 @@ describe SchemaTools do
       lgr
     end
     def initialize( src_db, dst_db )
-      @src_db = src_db.andand.extension :schema_dumper
+      @src_db = src_db&.extension :schema_dumper
       @dst_db = dst_db
     end
     attr_reader :src_db, :dst_db

data/wyrm.gemspec

@@ -20,9 +20,9 @@ Gem::Specification.new do |spec|
 
   # need this version because clause_sql was moved to _insert_sql, used by pump
   spec.add_runtime_dependency 'sequel', '>= 4.10.0'
-  spec.add_runtime_dependency "fastandand"
 
   spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-debundle"
   spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "bundler", ">= 1.3"
   spec.add_development_dependency "rake"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wyrm
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - John Anderson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-02 00:00:00.000000000 Z
+date: 2016-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -25,13 +25,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 4.10.0
 - !ruby/object:Gem::Dependency
-  name: fastandand
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: pry-debundle
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -210,7 +210,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Transfer from one SQL rdbms to another