readorder 1.0.0 → 2.0.0

data/HISTORY

@@ -1,4 +1,13 @@
 = Changelog
+
+== Version 2.0.0 - 2009-08-24
+
+=== Enhancements
+
+* complete rewrite of internal file sorting and temporary storage enabling
+  the 'readordering' of large lists of files.
+* switch to amalgalite instead of rbtree for sorting and storage
+
 == Version 1.0.0
 
 * Initial public release

data/gemspec.rb

@@ -21,10 +21,10 @@ Readorder::GEM_SPEC = Gem::Specification.new do |spec|
 
   # add dependencies here
   spec.add_dependency("configuration", "~> 0.0.5")
-  spec.add_dependency("rbtree", "~> 0.2.1")
+  spec.add_dependency("amalgalite", "~> 0.11.0")
   spec.add_dependency("main", "~> 2.8.3")
   spec.add_dependency("logging", "~> 1.1.4")
-  spec.add_dependency("hitimes", "~> 1.0.1")
+  spec.add_dependency("hitimes", "~> 1.0.4")
   
   spec.add_development_dependency( "rake", "~> 0.8.3")
 

data/lib/readorder.rb

@@ -16,6 +16,13 @@ end
 
 require 'rubygems'
 require 'logging'
+
+# require amalgalite explicitly  before hitimes explicitly because of 
+# using flat namespace on OSX
+# http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/344658
+require 'amalgalite'
+require 'hitimes'
+
 require 'readorder/version'
 require 'readorder/paths'
 require 'readorder/cli'

data/lib/readorder/analyzer.rb

@@ -1,6 +1,5 @@
-require 'hitimes'
 require 'readorder/datum'
-require 'rbtree'
+require 'readorder/results'
 
 module Readorder
   #
@@ -8,34 +7,27 @@ module Readorder
   # appropriate Datum instances
   #
   class Analyzer
-    # an Array of Datum instances for files that cannot be processed
-    attr_accessor :bad_data
+    # number of bad_data items encountered
+    attr_accessor :bad_data_count
+  
+    # number of good_data items encountered
+    attr_accessor :good_data_count
 
-    # an Array of Datum instances in the order they were processed
-    attr_accessor :good_data
-
-    # an RBTree of Datum instances of those files that were analyzed
-    # in order by phyiscal disc block number.  This only has items if 
-    # the physical block number was obtained.  It is empty otherwise
-    attr_accessor :physical_order
-
-    # an RBTree of Datum instances of those files that were analyzed
-    # in order by inode
-    attr_accessor :inode_order
+    # The Results handler
+    attr_accessor :results
 
     #
     # Initialize the Analyzer with the Filelist object and whether or
     # not to gather the physical block size.
     #
-    def initialize( filelist, get_physical = true )
+    def initialize( filelist, results, get_physical = true )
       @filelist          = filelist
-      @bad_data          = []
-      @good_data         = []
-      @physical_order    = ::MultiRBTree.new
-      @inode_order       = ::MultiRBTree.new
       @get_physical      = get_physical
       @size_metric       = ::Hitimes::ValueMetric.new( 'size' )
       @time_metric       = ::Hitimes::TimedMetric.new( 'time' )
+      @results           = results
+      @bad_data_count    = 0
+      @good_data_count   = 0
     end
 
     # 
@@ -60,30 +52,35 @@ module Readorder
       logger.info "Begin data collection"
       original_order = 0
       @filelist.each_line do |fname|
-        #logger.debug "  analyzing #{fname.strip}"
+        next if @results.has_datum_for_filename?( fname )
+        logger.debug "  analyzing #{fname.strip}"
         @time_metric.measure do
           d = Datum.new( fname )
-          d.collect( @get_physical )
-          d.original_order = original_order
-          if d.valid? then
-            @good_data << d
-            @size_metric.measure d.stat.size
-            @inode_order[d.inode_number] = d
-            if @get_physical then
-              @physical_order[d.first_physical_block_number] = d
+          begin
+            d.collect( @get_physical )
+            d.original_order = original_order
+
+            @results.add_datum( d )
+
+            if d.valid? then
+              @size_metric.measure d.stat.size
+              @good_data_count += 1
+            else
+              @bad_data_count += 1
             end
-          else
-            @bad_data << d
+          rescue => e
+            logger.error "#{e} : #{d.to_hash.inspect}"
           end
         end
 
         if @time_metric.count % 10_000 == 0 then
-          logger.info "  processed #{@time_metric.count} at #{"%0.3f" % @time_metric.rate} files/sec"
+          logger.info "  processed #{@time_metric.count} at #{"%0.3f" % @time_metric.rate} files/sec ( #{@good_data_count} good, #{@bad_data_count} bad )"
         end
         original_order += 1
       end
+      @results.flush
       logger.info "  processed #{@time_metric.count} at #{"%0.3f" % @time_metric.rate} files/sec"
-      logger.info "  yielded #{@good_data.size} data points"
+      logger.info "  yielded #{@good_data_count} data points"
       logger.info "End data collection" 
       nil
     end
@@ -112,30 +109,30 @@ module Readorder
       s.puts "Files analyzed   : #{"%12d" % @time_metric.count}"
       s.puts "Elapsed time     : #{"%12d" % @time_metric.duration} seconds"
       s.puts "Collection Rate  : #{"%16.3f" % @time_metric.rate} files/sec"
-      s.puts "Good files       : #{"%12d" % @good_data.size}"
+      s.puts "Good files       : #{"%12d" % @good_data_count}"
       s.puts "  average size   : #{"%16.3f" % @size_metric.mean} bytes"
       s.puts "  minimum size   : #{"%16.3f" % @size_metric.min} bytes"
       s.puts "  maximum size   : #{"%16.3f" % @size_metric.max} bytes"
       s.puts "  sum of sizes   : #{"%12d" % @size_metric.sum} bytes"
-      s.puts "Bad files        : #{"%12d" % @bad_data.size}"
+      s.puts "Bad files        : #{"%12d" % @bad_data_count}"
       return s.string
     end
 
     #
     # call-seq:
-    #   analyzer.dump_data_to( IO ) -> nil
+    #   analyzer.dump_errors_to( IO ) -> nil
     #
     # write a csv to the _IO_ object passed in.  The format is:
     #
-    #   error reason,filename
+    #   error_reason,filename
     #
     # If there are no bad Datum instances then do not write anything.
     #
-    def dump_bad_data_to( io )
-      if bad_data.size > 0 then
+    def dump_errors_to( io )
+      if results.error_count > 0 then
         io.puts "error_reason,filename"
-        bad_data.each do |d|
-          io.puts "#{d.error_reason},#{d.filename}"
+        results.each_error do |d|
+          io.puts "#{d['error_reason']},#{d['filename']}"
         end
       end
       nil
@@ -144,7 +141,7 @@ module Readorder
 
     # 
     # call-seq:
-    #   analyzer.dump_good_data_to( IO ) -> nil
+    #   analyzer.dump_valid_to( IO ) -> nil
     #
     # Write a csv fo the _IO_ object passed in.  The format is:
     #
@@ -153,17 +150,18 @@ module Readorder
     # The last two fields *physical_block_count* and *first_physical_block_number* are
     # only written if the analyzer was able to gather physical block information
     #
-    def dump_good_data_to( io )
+    def dump_valid_to( io )
       fields = %w[ filename size inode_number ]
+      by_field = 'inode_number'
       if @get_physical then
         fields << 'physical_block_count'
         fields << 'first_physical_block_number'
+        by_field = 'first_physical_block_number'
       end
-
       io.puts fields.join(",")
-      good_data.each do |d|
-        f = fields.collect { |f| d.send( f ) }
-        io.puts f.join(",")
+      results.each_valid_by_field( by_field ) do |d|
+       f = fields.collect { |f| d[f] }
+       io.puts f.join(",")
       end
     end
   end

data/lib/readorder/cli.rb

@@ -36,6 +36,7 @@ module Readorder
       mixin :argument_filelist
       mixin :option_output
       mixin :option_error_filelist
+      mixin :option_batch_size
 
       run { Cli.run_command_with_params( 'sort', params ) }
     }
@@ -51,6 +52,7 @@ module Readorder
       mixin :argument_filelist
       mixin :option_output
       mixin :option_error_filelist
+      mixin :option_batch_size
 
       option( 'data-csv' ) {
         description "Write the raw data collected to this csv file"
@@ -91,6 +93,7 @@ module Readorder
       mixin :option_output
       mixin :argument_filelist
       mixin :option_error_filelist
+      mixin :option_batch_size
 
       run { Cli.run_command_with_params( 'test', params ) }
     }
@@ -137,6 +140,15 @@ module Readorder
         validate { |f| File.directory?( File.dirname(File.expand_path( f ) ) ) }
       end
     end
+
+    mixin :option_batch_size do
+      option('batch-size' ) do
+        description "The number of files to queue before writing them to the db for storage"
+        argument :required
+        default 10_000
+        cast :integer
+      end
+    end
   } 
 
 

data/lib/readorder/command.rb

@@ -32,6 +32,7 @@ module Readorder
       @filelist = nil
       @analyzer = nil
       @output = nil
+      @delete_results = true
     end
 
     def filelist
@@ -47,7 +48,21 @@ module Readorder
     end
 
     def analyzer
-      @analyzer ||= Analyzer.new( filelist, self.get_physical? )
+      @analyzer ||= Analyzer.new( filelist, self.results, self.get_physical? )
+    end
+
+    def results_dbfile
+      if options['output'] then
+        output_dname = File.dirname( options['output'] )
+        output_bname = File.basename( options['output'], '.*' )
+        return File.join( output_dname, "#{output_bname}.db" )
+      else 
+        return ":memory:"
+      end
+    end
+
+    def results
+      @results ||= Results.new( results_dbfile, options['batch-size'] )
     end
 
     def output
@@ -94,24 +109,32 @@ module Readorder
     end 
 
     # called by the Runner if an error is encountered during the run method
-    def error() nil; end
+    def error() 
+      results.close
+    end
 
     # called by runner if a signal is hit
-    def shutdown() nil; end
+    def shutdown() 
+      results.close
+      @delete_results = false
+    end
 
     # called by runner when all is done
     def after() 
-      if output != $stdout then
-        output.close
-      end
       if options['error-filelist'] then
-        if analyzer.bad_data.size > 0 then
+        if analyzer.bad_data_count > 0 then
           File.open( options['error-filelist'], "w+" ) do |f|
-            analyzer.dump_bad_data_to( f )
+            analyzer.dump_errors_to( f )
           end
           logger.info "wrote error filelist to #{options['error-filelist']}"
         end
       end
+
+      if output != $stdout then
+        output.close
+        results.close 
+        File.unlink( results_dbfile ) if @delete_results
+      end
     end
 
     class << self

data/lib/readorder/commands/sort.rb

@@ -7,18 +7,21 @@ module Readorder
     #
     class Sort < ::Readorder::Command
       def run
+
         analyzer.collect_data
         analyzer.log_summary_report
-        data = nil
+
+        field = nil
         if get_physical? then
-          logger.info "using physical order"
-          data = analyzer.physical_order
+          logger.info "using first physical block number order"
+          field = 'first_physical_block_number'
         else
-          logger.info "using inode order"
-          data = analyzer.inode_order
+          logger.info "using inode number order"
+          field = 'inode_number'
         end
-        data.values.each do |d|
-          output.puts d.filename
+
+        analyzer.results.each_valid_by_field( field ) do |row|
+          output.puts row['filename']
         end
       end
     end

data/lib/readorder/datum.rb

@@ -36,6 +36,10 @@ module Readorder
       @is_linux ||= ::Config::CONFIG['host_os'] =~ /linux/i
     end
 
+    def self.hash_keys
+      %w[ filename inode_number first_physical_block_number original_order size ]
+    end
+
     #
     # call-seq:
     #   Datum.new( filename ) -> Datum
@@ -49,12 +53,39 @@ module Readorder
       @physical_block_count = 0
       @error_reason = nil
       @original_order = 0
+      @size = 0
       
       @stat = nil
       @valid = false
       @collected = false
     end
 
+    #
+    # call-seq:
+    #   datum.to_csv
+    #
+    # return the datum as a CSV in the format:
+    #
+    #   physical_id,inode_id,filename
+    #
+    def to_csv
+      "#{first_physical_block_number},#{inode_number},#{filename}"
+    end
+
+    #
+    # :call-seq;
+    #   datum.to_hash -> Hash
+    #
+    # return all the tiems in the datum as a hash
+    #
+    def to_hash
+      h = {}
+      Datum.hash_keys.each do |k|
+        h[k] = self.send( k )
+      end
+      return h
+    end
+
     #
     # call-seq:
     #   datum.size -> Integer
@@ -62,7 +93,7 @@ module Readorder
     # The number of bytes the file consumes
     #
     def size
-      @stat.size
+      @size ||= @stat.size
     end
 
     #

data/lib/readorder/results.rb

@@ -0,0 +1,224 @@
+require 'amalgalite'
+
+module Readorder
+  # Results persists the results from a readorder run
+  # The results are persisted in an SQlite3 database which allows for ordering
+  # the results by whatever means are wanted.
+  class Results
+    def self.create_table_sql
+      sql = <<-SQL
+        CREATE TABLE readorder_valid ( 
+           original_order              INTEGER PRIMARY KEY NOT NULL,
+           size                        INTEGER NOT NULL,
+           inode_number                INTEGER NOT NULL UNIQUE,
+           first_physical_block_number INTEGER UNIQUE,
+           physical_block_count        INTEGER,
+           filename                    TEXT    NOT NULL UNIQUE
+        );
+
+        CREATE TABLE readorder_errors (
+           original_order              INTEGER PRIMARY KEY NOT NULL,
+           filename                    TEXT NOT NULL UNIQUE,
+           error_reason                TEXT NOT NULL
+        );
+      SQL
+    end
+
+    #
+    # :call-seq:
+    #   Results.new( filename, 10_000 ) -> results
+    #
+    # Create a new Results object with a batch size.  The batch size is how many
+    # items to queue up to run in a single transaction into the sqlite database.
+    #
+    # By default the batch size is 1 which is not very performant.
+    #   
+    def initialize( filename, batch_size = 1 )
+      @db = Amalgalite::Database.new( filename )
+
+      unless @db.schema.tables['readorder_valid'] then
+        logger.info "Creating tables"
+        @db.execute_batch( Results.create_table_sql )
+      end
+      @db.reload_schema!
+      @batch_size = batch_size
+      @valid_queue = []
+      @error_queue = []
+    end
+
+    def close
+      flush
+      @db.close
+    end
+
+    def flush
+      flush_valid
+      flush_error
+    end
+
+    def logger
+      Logging::Logger[ self ]
+    end
+
+    #
+    # :call-seq:
+    #   results.has_datum_for_filename?( filename )
+    #
+    # return true or false if the give filename is alread in the database
+    #
+    def has_datum_for_filename?( filename )
+      @db.first_value_from( "SELECT filename FROM readorder_valid WHERE filename = ?", filename )
+    end
+
+    #
+    # :call-seq:
+    #   results.add_datum( datum )
+    #
+    # add a datum to the database, this will insert the datum into either valid
+    # or errors depending on the state of datum.valid?
+    #
+    def add_datum( datum )
+      if datum.valid?
+        @valid_queue << datum
+      else
+        @error_queue << datum
+      end
+      flush if ((@valid_queue.size + @error_queue.size) >= @batch_size )
+    end
+
+    #
+    # :call-seq:
+    #   results.flush_valid
+    #
+    # Flush all the pending valid items to the sqlite database
+    #
+    def flush_valid
+      if @valid_queue.size > 0 then
+        logger.info "Flushing #{@valid_queue.size} valid items to disk"
+        sql = <<-insert
+        INSERT INTO readorder_valid ( original_order, 
+                                      size,
+                                      inode_number,
+                                      first_physical_block_number,
+                                      physical_block_count,
+                                      filename )
+        VALUES( ?, ?, ?, ?, ?, ? );
+        insert
+        @db.transaction do |trans|
+          trans.prepare( sql ) do |stmt|
+            until @valid_queue.empty? do
+              datum = @valid_queue.shift
+              stmt.execute( datum.original_order,
+                            datum.size,
+                            datum.inode_number,
+                            datum.first_physical_block_number,
+                            datum.physical_block_count,
+                            datum.filename)
+            end
+          end
+        end
+      end
+    end
+
+    #  :call-seq:
+    #    results.valid_count -> Integer
+    #
+    # return the number of valid result rows
+    #
+    def valid_count
+      @db.first_value_from( "SELECT count(original_order) FROM readorder_valid" )
+    end
+
+
+    #
+    # :call-seq:
+    #   results.each_valid { |v| ... }
+    #
+    # Return each valid record without any predefined order
+    #
+    def each_valid( &block )
+      @db.execute( "SELECT * FROM readorder_valid" ) do |row|
+        yield row
+      end
+    end
+
+    #
+    # :call-seq:
+    #   results.each_valid_by_physical_block_number { |v| ... }
+    #
+    # Return each valid record in physical block number order
+    #
+    def each_valid_by_first_physical_block_number( &block )
+      each_valid_by_field( 'first_physical_block_number' ) do |row|
+        block.call( row )
+      end
+    end
+
+    #
+    # :call-seq:
+    #   results.each_valid_by_inode_number { |v| ... }
+    #
+    def each_valid_by_inode_number( &block )
+      each_valid_by_field( 'inode_number' ) do |row|
+        block.call( row )
+      end
+    end
+
+    #
+    # :call-seq:
+    #   results.each_valid_by_field( field ) { |v| ... }
+    #
+    def each_valid_by_field( field, &block )
+      @db.execute( "SELECT * from readorder_valid ORDER BY #{field} ASC" ) do |row|
+        yield row
+      end
+    end
+
+    # :call-seq:
+    #   results.flush_error
+    #
+    # Flush all the error items to disk
+    #
+    def flush_error
+      if @error_queue.size > 0 then
+        logger.info "Flushing #{@error_queue.size} error items to disk"
+        sql = <<-insert
+        INSERT INTO readorder_errors ( original_order, filename, error_reason )
+        VALUES( ?, ?, ? );
+        insert
+        @db.transaction do |trans|
+          trans.prepare( sql ) do |stmt|
+            until @error_queue.empty? do
+              datum = @error_queue.shift
+              stmt.execute( datum.original_order, 
+                            datum.filename,
+                            datum.error_reason  )
+            end
+          end
+        end
+      end
+    end
+
+    #  :call-seq:
+    #    results.error_count -> Integer
+    #
+    # return the number of errors
+    #
+    def error_count
+      @db.first_value_from( "SELECT count(original_order) FROM readorder_errors" )
+    end
+
+    #
+    # :call-seq:
+    #   results.each_error { |e| ... }
+    #
+    # Return each error record without any predefined order
+    #
+    def each_error( &block )
+      @db.execute( "SELECT * FROM readorder_errors" ) do |row|
+        yield row
+      end
+    end
+
+  end
+end

data/lib/readorder/version.rb

@@ -5,7 +5,7 @@
 
 module Readorder
   module Version
-    MAJOR   = 1
+    MAJOR   = 2
     MINOR   = 0
     BUILD   = 0
 

data/spec/analyzer_spec.rb

@@ -5,18 +5,27 @@ require 'readorder/analyzer'
 describe Readorder::Analyzer do
   before( :each ) do
     s = StringIO.new
-    fl = Dir.glob("#{Readorder::Paths.spec_path}/*_spec.rb")
-    s.puts fl.to_a.join("\n")
+    fl = Dir.glob("#{Readorder::Paths.spec_path}*_spec.rb")
+    s.write( fl.join("\n") )
     s.rewind
+
     @filelist = Readorder::Filelist.new( s )
-    @analyzer = Readorder::Analyzer.new( @filelist, false )
+    @r = Readorder::Results.new( ":memory:" )
+
+    @analyzer = Readorder::Analyzer.new( @filelist, @r , false )
+  end
+
+  after( :each ) do
+    @r.close
   end
 
   it "collects data about files" do
     @analyzer.collect_data
-    @analyzer.good_data.size.should > 0
-    @analyzer.inode_order.size.should > 0
-    @analyzer.physical_order.size.should == 0
+    @analyzer.results.valid_count.should > 0
+    check_count = 0
+    @analyzer.results.each_valid { |v| check_count += 1 }
+    check_count.should > 0
+    check_count.should == @analyzer.results.valid_count
   end
 
   it "logs a summary report" do
@@ -29,12 +38,12 @@ describe Readorder::Analyzer do
     s = StringIO.new
     s.puts "/a/nonexistent/file"
     s.rewind
-    analyzer = Readorder::Analyzer.new( Readorder::Filelist.new( s ) )
+    analyzer = Readorder::Analyzer.new( Readorder::Filelist.new( s ), @r, false )
     analyzer.collect_data
-    analyzer.bad_data.size.should > 0
+    analyzer.results.error_count.should > 0
 
     s2 = StringIO.new
-    analyzer.dump_bad_data_to( s2 )
+    analyzer.dump_errors_to( s2 )
     s2.rewind
     s2.gets.should == "error_reason,filename\n"
     s2.gets.should == "No such file or directory - /a/nonexistent/file,/a/nonexistent/file\n"
@@ -43,9 +52,25 @@ describe Readorder::Analyzer do
   it "can dump good data to a csv" do
     @analyzer.collect_data
     s = StringIO.new
-    @analyzer.dump_good_data_to( s )
+    @analyzer.dump_valid_to( s )
     s.rewind
     s.gets.should == "filename,size,inode_number\n"
-    s.read.split("\n").size.should == @analyzer.good_data.size
+    s.read.split("\n").size.should == @analyzer.results.valid_count
+  end
+
+  it "can iterate over inode block numbers" do
+    @analyzer.collect_data
+    by_order = []
+    @analyzer.results.each_valid_by_field( 'original_order' ) do |r|
+      by_order << r['filename']
+    end
+
+    by_inode = []
+    @analyzer.results.each_valid_by_inode_number do |r|
+      by_inode << r['filename']
+    end
+
+    by_order.should_not == by_inode
+    by_order.sort.should == by_inode.sort
   end
 end

data/tasks/distribution.rake

@@ -18,7 +18,7 @@ if pkg_config = Configuration.for_if_exist?("packaging") then
 
     desc "Install as a gem"
     task :install => [:clobber, :package] do
-      sh "sudo gem install pkg/#{Readorder::GEM_SPEC.full_name}.gem"
+      sh "sudo gem install pkg/#{Readorder::GEM_SPEC.full_name}.gem --no-rdoc --no-ri --ignore-dependencies --local"
     end
 
     desc "Uninstall gem"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: readorder
 version: !ruby/object:Gem::Version 
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors: 
 - Jeremy Hinegardner
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-06-27 00:00:00 -06:00
+date: 2009-08-24 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -23,14 +23,14 @@ dependencies:
         version: 0.0.5
     version: 
 - !ruby/object:Gem::Dependency 
-  name: rbtree
+  name: amalgalite
   type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 0.2.1
+        version: 0.11.0
     version: 
 - !ruby/object:Gem::Dependency 
   name: main
@@ -60,7 +60,7 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 1.0.1
+        version: 1.0.4
     version: 
 - !ruby/object:Gem::Dependency 
   name: rake
@@ -92,6 +92,7 @@ extra_rdoc_files:
 - lib/readorder/filelist.rb
 - lib/readorder/log.rb
 - lib/readorder/paths.rb
+- lib/readorder/results.rb
 - lib/readorder/runner.rb
 - lib/readorder/version.rb
 - lib/readorder.rb
@@ -107,6 +108,7 @@ files:
 - lib/readorder/filelist.rb
 - lib/readorder/log.rb
 - lib/readorder/paths.rb
+- lib/readorder/results.rb
 - lib/readorder/runner.rb
 - lib/readorder/version.rb
 - lib/readorder.rb