split_pgdump 0.3.5 → 0.3.6

data/lib/split_pgdump.rb

@@ -7,7 +7,7 @@ require 'shellwords'
 $debug = false
 
 module SplitPgDump
-  VERSION = '0.3.5'
+  VERSION = '0.3.6'
 end
 
 class SplitPgDump::Worker
@@ -206,8 +206,8 @@ end
 
 class SplitPgDump::Table
   class NoColumn < StandardError; end
-  ONE_FILE_CACHE_SIZE = 256 * 1024
-  TOTAL_CACHE_SIZE = 5 * 1024 * 1024
+  ONE_FILE_CACHE_SIZE = 3 * 128 * 1024
+  TOTAL_CACHE_SIZE = 4 * 128 * 1024
 
   class OneFile
     attr_reader :file_name, :cache_size
@@ -251,19 +251,38 @@ class SplitPgDump::Table
     end
   end
 
+  module DefaultName
+    def file_name(line)
+      @file_name
+    end
+  end
+  include DefaultName
+
+  module ComputeName
+    def file_name(line)
+      values = line.chomp.split("\t")
+      name = compute_name(values)
+      @file_name[name] ||= begin
+        name_strip = name.gsub(/\.\.|\s|\?|\*|'|"/, '_')
+        "#{table_schema}/#{name_strip}.dat"
+      end
+    end
+  end
+
   attr_reader :table, :columns, :files, :sort_line, :sort_args
   def initialize(dir, schema, name, columns, rule)
     @dir = dir
     @table = name
     @schema = schema
     @columns = columns.map{|c| c.sub(/^"(.+)"$/, '\\1')}
+    @file_name = "#{table_schema}.dat"
     apply_rule rule
     @files = {}
     @total_cache_size = 0
   end
 
-  def _mod(s, len, mod)
-    "%0#{len}d" % (s.to_i / mod * mod)
+  def _mod(s, format, mod)
+    format % (s.to_i / mod * mod)
   end
 
   def apply_rule(rule)
@@ -281,7 +300,7 @@ class SplitPgDump::Table
             if action[:mod]
               mod_s = action[:mod]
               mod = mod_s.to_i
-              field = "_mod(#{field},#{mod_s.size},#{mod})"
+              field = "_mod(#{field}, '%0#{mod_s.size}d', #{mod})"
             elsif action[:range]
               field << "#{action[:range]}"
             end
@@ -290,12 +309,15 @@ class SplitPgDump::Table
         end
       end
 
-      eval <<-"EOF"
-        def self.file_name(values)
-          name = %{#{split_string}}.gsub(/\\.\\.|\\s|\\?|\\*|'|"/, '_')
-          "\#{table_schema}/\#{name}.dat"
-        end
-      EOF
+      if split_string > ''
+        @file_name = {}
+        eval <<-"EOF"
+          def self.compute_name(values)
+            %{#{split_string}}
+          end
+        EOF
+        extend ComputeName
+      end
 
       @sort_args = rule.sort_keys.map do |key|
         i = @columns.find_index(key[:field])
@@ -312,13 +334,12 @@ class SplitPgDump::Table
     @schema == 'public' ? @table : "#@schema/#@table"
   end
 
-  def file_name(values)
-    "#{table_schema}.dat"
+  def file_name(line)
+    @file_name
   end
 
   def add_line(line)
-    values = line.chomp.split("\t")
-    fname = file_name(values)
+    fname = file_name(line)
     one_file = @files[fname] ||= OneFile.new(@dir, fname)
     one_file.add_line(line)
     @total_cache_size += line.size
@@ -336,8 +357,8 @@ class SplitPgDump::Table
 
   def copy_lines
     if block_given?
-      @files.each do |name, one_file|
-        yield "\\copy #{@table} (#{@columns.join(', ')}) from #{one_file.file_name}"
+      @files.map{|n, one_file| one_file.file_name}.sort.each do |file_name|
+        yield "\\copy #{@table} (#{@columns.join(', ')}) from #{file_name}"
       end
     else
       to_enum(:copy_lines)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: split_pgdump
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
   prerelease: 
 platform: ruby
 authors:
@@ -45,7 +45,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.12
+rubygems_version: 1.8.16
 signing_key: 
 specification_version: 3
 summary: split_pgdump is a tool for splitting postgresql dump in a managable set of