file_sort 0.0.1

data/lib/file_sort.rb

@@ -0,0 +1,163 @@
+class FileSort
+
+  DEFAULTS = {
+      sort_column:        0,
+      column_separator:   "\t",
+      num_processes:      3,
+      parse_as:           :int, #other options: :string
+      lines_per_split:    1e6,
+      replace_original:   true,
+      log_output:         true
+    }
+
+  def initialize(filename, options = {})
+    @filename = filename
+    @options = DEFAULTS.merge(options)
+    @options[:lines_per_split] = @options[:lines_per_split].to_i
+
+    @file_id_counter = 0
+    @files_to_sort = []
+    @files_to_merge = []
+    @active_workers = 0
+    @start_time = nil
+    @scheduler_thread = nil
+  end
+
+  def sort!
+    @start_time = Time.now
+    self.log("Sorting #{@filename} with up to #{@options[:num_processes]} processes.")
+    self.make_splits
+    @scheduler_thread = Thread.new do
+      while true
+        break if @active_workers == 0 and @files_to_sort.empty? and @files_to_merge.size == 1
+        if @active_workers < @options[:num_processes]
+          unless @files_to_sort.empty?
+            self.sort_split(@files_to_sort.shift)
+            next
+          end
+          unless @files_to_merge.size < 2
+            self.merge_splits(@files_to_merge.shift, @files_to_merge.shift, self.next_filename)
+            next
+          end
+          sleep
+        else
+          sleep
+        end
+      end
+    end
+    @scheduler_thread.join
+    final_name = "#{@filename}.sorted"
+    File.rename(@files_to_merge.first, final_name)
+    if @options[:replace_original]
+      File.delete(@filename)
+      File.rename(final_name, @filename)
+    end
+    self.log("#{@filename} sort complete.")
+  end
+
+  def make_splits
+    self.log("Splitting #{@filename} every #{@options[:lines_per_split]} lines")
+    self.worker_begin
+    Thread.new do
+      line_counter = 0
+      infile = File.open(@filename)
+      output_filename = self.next_filename
+      outfile = File.open(output_filename, "w")
+      while line = infile.gets
+        if line_counter >= @options[:lines_per_split]
+          outfile.close
+          self.log("Split written: #{fid output_filename}")
+          @files_to_sort << output_filename
+          output_filename = self.next_filename
+          outfile = File.open(output_filename, "w")
+          line_counter = 0
+          @scheduler_thread.wakeup
+        end
+        outfile.print(line)
+        line_counter += 1
+      end
+      infile.close
+      outfile.close
+      self.log("Split written: #{fid output_filename} (final split)")
+      @files_to_sort << output_filename
+      self.worker_done
+    end
+  end
+
+  def sort_split(filename)
+    sorted_filename = self.next_filename
+    self.log("Sorting #{fid filename} as #{fid sorted_filename}")
+    self.worker_begin
+    Thread.new do
+
+      pid = Process.spawn({
+          "input_filename"    => filename,
+          "sorted_filename"   => sorted_filename,
+          "sort_column"       => @options[:sort_column].to_s,
+          "sort_as_int"       => (@options[:parse_as] == :int ? "true" : "false"),
+          "column_separator"  => @options[:column_separator]
+        }, "ruby #{File.join(File.dirname(__FILE__), 'sorter.rb')}")
+      Process.waitpid(pid)
+      File.delete(filename)
+      self.log("Sort complete for #{fid filename} as #{fid sorted_filename}")
+      @files_to_merge << sorted_filename
+      self.worker_done
+    end
+  end
+
+  def merge_splits(filename1, filename2, output_filename)
+    self.log("Merging (#{fid filename1}, #{fid filename2}) => #{fid output_filename}")
+    self.worker_begin
+    Thread.new do
+      pid = Process.spawn({
+          "filename1"         => filename1,
+          "filename2"         => filename2,
+          "output_filename"   => output_filename,
+          "sort_column"       => @options[:sort_column].to_s,
+          "sort_as_int"       => (@options[:parse_as] == :int ? "true" : "false"),
+          "column_separator"  => @options[:column_separator]
+        }, "ruby #{File.join(File.dirname(__FILE__), 'merger.rb')}")
+      Process.waitpid(pid)
+      File.delete(filename1)
+      File.delete(filename2)
+      self.log("Merge complete for (#{fid filename1}, #{fid filename2}) => #{fid output_filename}")
+      @files_to_merge << output_filename
+      self.worker_done
+    end
+  end
+
+  def worker_begin
+    @active_workers += 1
+  end
+
+  def worker_done
+    @active_workers -= 1
+    @scheduler_thread.wakeup
+  end
+
+  def next_filename
+    return "#{@filename}.#{(@file_id_counter += 1)}"
+  end
+
+  def fid(filename)
+    return "F-#{filename.split(".").last}"
+  end
+
+  def seconds_to_pretty_time(num_seconds)
+    hours = (num_seconds / (60**2)).to_i
+    minutes = ((num_seconds % (60**2)) / 60).to_i
+    padded_minutes = minutes < 10 ? "0#{minutes}" : minutes.to_s
+    seconds = num_seconds.round(0).to_i % 60
+    seconds_padded = seconds < 10 ? "0#{seconds}" : seconds.to_s
+    return "#{hours}:#{padded_minutes}:#{seconds_padded}"
+  end
+
+  def log(message)
+    return unless @options[:log_output]
+    puts "#{seconds_to_pretty_time(Time.now - @start_time)} #{message}"
+  end
+
+end
+
+#Run as
+#FileSort.new("large-file-1000000.csv", {parse_as: :string, sort_column: 1}).sort!

data/lib/large_file_generator.rb

@@ -0,0 +1,24 @@
+class LargeFileGenerator
+
+  DEFAULTS = {
+      num_columns:        10,
+      num_rows:           1e6.to_i,
+      column_separator:   "\t"
+  }
+
+  def initialize(filename, options = {})
+    @filename = filename
+    @options = DEFAULTS.merge(options)
+  end
+
+  def generate!
+    File.open(@filename, "w") do |f|
+      @options[:num_rows].times do
+        f.puts (0...@options[:num_columns]).map{rand(@options[:num_rows])}.join(@options[:column_separator])
+      end
+    end
+  end
+end
+
+#Run as
+#LargeFileGenerator.new("large-file-1000000.csv", { num_rows: 1000000 }).generate!

data/lib/merger.rb

@@ -0,0 +1,57 @@
+class Merger
+  def initialize
+    @filename1          = ENV["filename1"]
+    @filename2          = ENV["filename2"]
+    @output_filename    = ENV["output_filename"]
+    @sort_column        = ENV["sort_column"].to_i
+    @sort_as_int        = ENV["sort_as_int"] == "true"
+    @column_separator   = ENV["column_separator"]
+  end
+
+  def merge!
+    outfile = File.open(@output_filename, "w")
+    f1 = File.open(@filename1)
+    f2 = File.open(@filename2)
+
+    f1_line = self.get_line(f1)
+    f2_line = self.get_line(f2)
+    while !f1_line.nil? and !f2_line.nil?
+      if f1_line[@sort_column] < f2_line[@sort_column]
+        self.write_line(outfile, f1_line)
+        f1_line = self.get_line(f1)
+      else
+        self.write_line(outfile, f2_line)
+        f2_line = self.get_line(f2)
+      end
+    end
+
+    while !f1_line.nil?
+      self.write_line(outfile, f1_line)
+      f1_line = self.get_line(f1)
+    end
+
+    while !f2_line.nil?
+      self.write_line(outfile, f2_line)
+      f2_line = self.get_line(f2)
+    end
+
+
+    f1.close
+    f2.close
+    outfile.close
+  end
+
+  def get_line(stream)
+    line = stream.gets
+    return nil if line.nil?
+    line = line.chomp.split(@column_separator)
+    line[@sort_column] = line[@sort_column].to_i if @sort_as_int
+    return line
+  end
+
+  def write_line(stream, line)
+    stream.puts line.join(@column_separator)
+  end
+end
+
+Merger.new.merge!

data/lib/sorter.rb

@@ -0,0 +1,27 @@
+class Sorter
+
+  def initialize
+    @input_filename     = ENV["input_filename"]
+    @sorted_filename    = ENV["sorted_filename"]
+    @sort_column        = ENV["sort_column"].to_i
+    @sort_as_int        = ENV["sort_as_int"] == "true"
+    @column_separator   = ENV["column_separator"]
+  end
+
+  def sort!
+    lines = []
+    infile = File.open(@input_filename)
+    while line = infile.gets
+      line = line.chomp.split(@column_separator)
+      line[@sort_column] = line[@sort_column].to_i if @sort_as_int
+      lines << line
+    end
+    infile.close
+    lines.sort!{ |a, b| a[@sort_column] <=> b[@sort_column] }
+    outfile = File.open(@sorted_filename, "w")
+    lines.each{ |line| outfile.puts(line.join(@column_separator)) }
+    outfile.close
+  end
+end
+
+Sorter.new.sort!

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification
+name: file_sort
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Tom O'Neill
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-01-27 00:00:00.000000000 Z
+dependencies: []
+description: Sorts large files using merge sort on temporary files on the hard drive.
+email: tom.oneill@live.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/file_sort.rb
+- lib/large_file_generator.rb
+- lib/merger.rb
+- lib/sorter.rb
+homepage: https://github.com/mopatches/file_sort_ruby_gem
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: FileSort - Sorts files too large to fit in RAM
+test_files: []