ruby-ole 1.2.1 → 1.2.2

data/Rakefile

@@ -15,16 +15,31 @@ PKG_VERSION = Ole::Storage::VERSION
 
 task :default => [:test]
 
-Rake::TestTask.new(:test) do |t|
+Rake::TestTask.new do |t|
 	t.test_files = FileList["test/test_*.rb"]
 	t.warning = true
 	t.verbose = true
 end
 
-# RDocTask wasn't working for me
-desc 'Build the rdoc HTML Files'
-task :rdoc do
-	system "rdoc -S -N --main 'Ole::Storage' --tab-width 2 --title '#{PKG_NAME} documentation' lib"
+begin
+	require 'rcov/rcovtask'
+	# NOTE: this will not do anything until you add some tests
+	desc "Create a cross-referenced code coverage report"
+	Rcov::RcovTask.new do |t|
+		t.test_files = FileList['test/test*.rb']
+		t.ruby_opts << "-Ilib" # in order to use this rcov
+		t.rcov_opts << "--xrefs"  # comment to disable cross-references
+		t.verbose = true
+	end
+rescue LoadError
+	# Rcov not available
+end
+
+Rake::RDocTask.new do |t|
+	t.rdoc_dir = 'doc'
+	t.title    = "#{PKG_NAME} documentation"
+	t.options += %w[--line-numbers --inline-source --tab-width 2]
+	t.rdoc_files.include 'lib/**/*.rb'
 end
 
 spec = Gem::Specification.new do |s|
@@ -44,17 +59,148 @@ spec = Gem::Specification.new do |s|
 	s.files += Dir.glob("bin/*")
 
 	s.has_rdoc = true
-	s.rdoc_options += ['--main', 'Ole::Storage',
-					   '--title', "#{PKG_NAME} documentation",
-					   '--tab-width', '2']
+	s.rdoc_options += [
+		'--main', 'Ole::Storage',
+		'--title', "#{PKG_NAME} documentation",
+		'--tab-width', '2'
+	]
 
 	s.autorequire = 'ole/storage'
 end
 
-Rake::GemPackageTask.new(spec) do |p|
-	p.gem_spec = spec
-	p.need_tar = true
-	p.need_zip = false
-	p.package_dir = 'build'
+Rake::GemPackageTask.new(spec) do |t|
+	t.gem_spec = spec
+	t.need_tar = true
+	t.need_zip = false
+	t.package_dir = 'build'
+end
+
+desc 'Run various benchmarks'
+task :benchmark do
+	require 'benchmark'
+	require 'tempfile'
+	require 'ole/file_system'
+
+	# should probably add some read benchmarks too
+	def write_benchmark opts={}
+		files, size = opts[:files], opts[:size]
+		block_size = opts[:block_size] || 100_000
+		block = 0.chr * block_size
+		blocks, remaining = size.divmod block_size
+		remaining = 0.chr * remaining
+		Tempfile.open 'ole_storage_benchmark' do |temp|
+			Ole::Storage.open temp do |ole|
+				files.times do |i|
+					ole.file.open "file_#{i}", 'w' do |f|
+						blocks.times { f.write block }
+						f.write remaining
+					end
+				end
+			end
+		end
+	end
+
+	Benchmark.bm do |bm|
+		bm.report 'write_1mb_1x5' do
+			5.times { write_benchmark :files => 1, :size => 1_000_000 }
+		end
+
+		bm.report 'write_1mb_2x5' do
+			5.times { write_benchmark :files => 1_000, :size => 1_000 }
+		end
+	end
 end
 
+=begin
+
+1.2.1:
+
+      user     system      total        real
+write_1mb_1x5 73.920000   8.400000  82.320000 ( 91.893138)
+
+revision 17 (speed up AllocationTable#free_block by using
+@sparse attribute, and using Array#index otherwise):
+
+      user     system      total        real
+write_1mb_1x5 57.910000   6.190000  64.100000 ( 66.207993)
+write_1mb_2x5266.310000  31.750000 298.060000 (305.877203)
+
+add in extra resize_chain fix (return blocks to avoid calling
+AllocationTable#chain twice):
+
+      user     system      total        real
+write_1mb_1x5 43.140000   5.480000  48.620000 ( 51.835942)
+
+add in RangesIOResizeable fix (cache @blocks, to avoid calling
+AllocationTable#chain at all when resizing now, just pass it
+to AllocationTable#resize_chain):
+
+      user     system      total        real
+write_1mb_1x5 29.770000   5.180000  34.950000 ( 39.916747)
+
+40 seconds is still a really long time to write out 5 megs.
+of course, this is all with a 1_000 byte block size, which is
+a very small wite. upping this to 100_000 bytes:
+
+      user     system      total        real
+write_1mb_1x5  0.540000   0.130000   0.670000 (  1.051862)
+
+so it seems that that makes a massive difference. so i really
+need buffering in RangesIO if I don't want it to really hurt
+for small writes, as all the resize code is kind of expensive.
+
+one of the costly things at the moment, is RangesIO#offset_and_size,
+which is called for each write, and re-finds which range we are in.
+that should obviously be changed, to a fixed one that is invalidated
+on seeks. buffering would hide that problem to some extent, but i
+should fix it anyway.
+
+re-running the original 1.2.1 with 100_000 byte block size:
+
+      user     system      total        real
+write_1mb_1x5 15.590000   2.230000  17.820000 ( 18.704910)
+
+so there the really badly non-linear AllocationTable#resize_chain is
+being felt. 
+
+back to current working copy, running full benchmark:
+
+      user     system      total        real
+write_1mb_1x5  0.530000   0.150000   0.680000 (  0.708919)
+write_1mb_2x5227.940000  31.260000 259.200000 (270.200960)
+
+not surprisingly, the second case hasn't been helped much by the fixes
+so far, as they only really help multiple resizes and writes for a file.
+this could be pain in the new file system code - potentially searching
+through Dirent#children at creation time.
+
+to test, i'll profile creating 1_000 files, without writing anything:
+
+      user     system      total        real
+write_1mb_2x5 16.990000   1.830000  18.820000 ( 19.900568)
+
+hmmm, so thats not all of it. maybe its the initial chain calls, etc?
+writing 1 byte:
+
+      user     system      total        real
+write_1mb_1x5  0.520000   0.120000   0.640000 (  0.660638)
+write_1mb_2x5 19.810000   2.280000  22.090000 ( 22.696214)
+
+weird.
+
+100 bytes:
+
+      user     system      total        real
+write_1mb_1x5  0.560000   0.140000   0.700000 (  1.424974)
+write_1mb_2x5 22.940000   2.840000  25.780000 ( 26.556346)
+
+500 bytes:
+
+      user     system      total        real
+write_1mb_1x5  0.530000   0.150000   0.680000 (  1.139738)
+write_1mb_2x5 77.260000  10.130000  87.390000 ( 91.671086)
+
+what happens there? very strange. 
+
+=end
+

data/lib/ole/file_system.rb

@@ -4,7 +4,7 @@
 # This file intends to provide file system-like api support, a la <tt>zip/zipfilesystem</tt>.
 #
 # Ideally, this will be the recommended interface, allowing Ole::Storage, Dir, and
-# Zip::ZipFile to be used exchangablyk. It should be possible to write recursive copy using
+# Zip::ZipFile to be used exchangably. It should be possible to write recursive copy using
 # the plain api, such that you can copy dirs/files agnostically between any of ole docs, dirs,
 # and zip files.
 #
@@ -19,93 +19,267 @@
 #
 # = Notes
 #
-# *** This file is very incomplete
+# <tt>Ole::Storage</tt> files can have multiple files with the same name,
+# or with / in the name, and other things that are probably invalid anyway.
+# This API is unable to access those files, but of course the core, low-
+# level API can.
+#
+# need to implement some more IO functions on RangesIO, like #puts, #print
+# etc, like AbstractOutputStream from zipfile.
+#
+# TODO
 # 
-# i think its okay to have an api like this on top, but there are certain things that ole
-# does that aren't captured.
-# <tt>Ole::Storage</tt> can have multiple files with the same name, for example, or with
-# / in the name, and other things that are probably invalid anyway.
-# i think this should remain an addon, built on top of my core api.
-# but still the ideas can be reflected in the core, ie, changing the read/write semantics.
+# - check for all new_child calls. eg Dir.mkdir, and File.open, and also
+#   File.rename, to add in filename length checks (max 32 / 31 or something).
+#   do the automatic truncation, and add in any necessary warnings.
+#
+# - File.split('a/') == File.split('a') == ['.', 'a']
+#   the implication of this, is that things that try to force directory
+#   don't work. like, File.rename('a', 'b'), should work if a is a file
+#   or directory, but File.rename('a/', 'b') should only work if a is
+#   a directory. tricky, need to clean things up a bit more.
+#   i think a general path name => dirent method would work, with flags
+#   about what should raise an error. 
+#
+# - Need to look at streamlining things after getting all the tests passing,
+#   as this file's getting pretty long - almost half the real implementation.
+#   and is probably more inefficient than necessary.
+#   too many exceptions in the expected path of certain functions.
 #
-# once the core changes are complete, this will be a pretty straight forward file to complete.
+# - should look at profiles before and after switching ruby-msg to use
+#   the filesystem api.
 #
 
-require 'ole/base'
+require 'ole/storage'
 
 module Ole # :nodoc:
 	class Storage
 		def file
-			@file ||= FileParent.new self
+			@file ||= FileClass.new self
 		end
 
 		def dir
-			@dir ||= DirParent.new self
+			@dir ||= DirClass.new self
 		end
 
-		def dirent_from_path path_str
-			path = path_str.sub(/^\/*/, '').sub(/\/*$/, '')
+		# tries to get a dirent for path. return nil if it doesn't exist
+		# (change it)
+		def dirent_from_path path
 			dirent = @root
-			return dirent if path.empty?
-			path = path.split /\/+/
+			path = file.expand_path path
+			path = path.sub(/^\/*/, '').sub(/\/*$/, '').split(/\/+/)
 			until path.empty?
-				raise "invalid path #{path_str.inspect}" if dirent.file?
-				if tmp = dirent[path.shift]
-					dirent = tmp
-				else
-					# allow write etc later.
-					raise "invalid path #{path_str.inspect}"
-				end
+				return nil if dirent.file?
+				return nil unless dirent = dirent/path.shift
 			end
 			dirent
 		end
 
-		class FileParent
+		class FileClass
+			class Stat
+				attr_reader :ftype, :size, :blocks, :blksize
+				attr_reader :nlink, :uid, :gid, :dev, :rdev, :ino
+				def initialize dirent
+					@dirent = dirent
+					@size = dirent.size
+					if file?
+						@ftype = 'file'
+						bat = dirent.ole.bat_for_size(dirent.size)
+						@blocks = bat.chain(dirent.first_block).length
+						@blksize = bat.block_size
+					else
+						@ftype = 'directory'
+						@blocks = 0
+						@blksize = 0
+					end
+					# a lot of these are bogus. ole file format has no analogs
+					@nlink = 1
+					@uid, @gid = 0, 0
+					@dev, @rdev = 0, 0
+					@ino = 0
+					# need to add times - atime, mtime, ctime. 
+				end
+
+				alias rdev_major :rdev
+				alias rdev_minor :rdev
+
+				def file?
+					@dirent.file?
+				end
+
+				def directory?
+					@dirent.dir?
+				end
+
+				def inspect
+					pairs = (instance_variables - ['@dirent']).map do |n|
+						"#{n[1..-1]}=#{instance_variable_get n}"
+					end
+					"#<#{self.class} #{pairs * ', '}>"
+				end
+			end
+
 			def initialize ole
 				@ole = ole
 			end
 
-			def open path_str, mode='r', &block
-				dirent = @ole.dirent_from_path path_str
-				# like Errno::EISDIR
-				raise "#{path_str.inspect} is a directory" unless dirent.file?
-				dirent.open(&block)
+			def expand_path path
+				# get the raw stored pwd value (its blank for root)
+				pwd = @ole.dir.instance_variable_get :@pwd
+				# its only absolute if it starts with a '/'
+				path = "#{pwd}/#{path}" unless path =~ /^\//
+				# at this point its already absolute. we use File.expand_path
+				# just for the .. and . handling
+				File.expand_path path
+			end
+
+			# +orig_path+ is just so that we can use the requested path
+			# in the error messages even if it has been already modified
+			def dirent_from_path path, orig_path=nil
+				orig_path ||= path
+				dirent = @ole.dirent_from_path path
+				raise Errno::ENOENT,  orig_path unless dirent
+				raise Errno::EISDIR, orig_path if dirent.dir?
+				dirent
+			end
+			private :dirent_from_path
+
+			def exists? path
+				!!@ole.dirent_from_path(path)
+			end
+			alias exist? :exists?
+
+			def file? path
+				dirent = @ole.dirent_from_path path
+				dirent and dirent.file?
+			end
+
+			def directory? path
+				dirent = @ole.dirent_from_path path
+				dirent and dirent.dir?
+			end
+
+			def open path, mode='r', &block
+				# FIXME - mode strings are more complex than this.
+				if mode == 'w'
+					begin
+						dirent = dirent_from_path path
+					rescue Errno::ENOENT
+						# maybe instead of repeating this everywhere, i should have
+						# a get_parent_dirent function.
+						parent_path, basename = File.split expand_path(path)
+						parent = @ole.dir.send :dirent_from_path, parent_path, path
+						dirent = parent.new_child :file
+						dirent.name = basename
+					end
+				else
+					dirent = dirent_from_path path
+				end
+				dirent.open mode, &block
+			end
+
+			# explicit wrapper instead of alias to inhibit block
+			def new path, mode='r'
+				open path, mode
+			end
+
+			def size path
+				dirent_from_path(path).size
+			rescue Errno::EISDIR
+				# kind of arbitrary. I'm getting 4096 from ::File, but
+				# the zip tests want 0.
+				0
 			end
 
-			alias new :open
+			def stat path
+				# we do this to allow dirs.
+				dirent = @ole.dirent_from_path path
+				raise Errno::ENOENT, path unless dirent
+				Stat.new dirent
+			end
 
 			def read path
-				open(path) { |f| f.read }
+				open path, &:read
+			end
+
+			# most of the work this function does is moving the dirent between
+			# 2 parents. the actual name changing is quite simple.
+			# File.rename can move a file into another folder, which is why i've
+			# done it too, though i think its not always possible...
+			#
+			# FIXME File.rename can be used for directories too....
+			def rename from_path, to_path
+				# check what we want to rename from exists. do it this
+				# way to allow directories.
+				dirent = @ole.dirent_from_path from_path
+				raise Errno::ENOENT, from_path unless dirent
+				# delete what we want to rename to if necessary
+				begin
+					unlink to_path
+				rescue Errno::ENOENT
+					# we actually get here, but rcov doesn't think so
+				end
+				# reparent the dirent
+				from_parent_path, from_basename = File.split expand_path(from_path)
+				to_parent_path, to_basename = File.split expand_path(to_path)
+				from_parent = @ole.dir.send :dirent_from_path, from_parent_path, from_path
+				to_parent = @ole.dir.send :dirent_from_path, to_parent_path, to_path
+				from_parent.children.delete dirent
+				# and also change its name
+				dirent.name = to_basename
+				to_parent.children << dirent
+				0
 			end
 
 			# crappy copy from Dir.
-			def unlink path
-				dirent = @ole.dirent_from_path path
-				# EPERM
-				raise "operation not permitted #{path.inspect}" unless dirent.file?
-				# i think we should free all of our blocks. i think the best way to do that would be
-				# like:
-				# open(path) { |f| f.truncate 0 }. which should free all our blocks from the
-				# allocation table. then if we remove ourself from our parent, we won't be part of
-				# the bat at save time.
-				# i think if you run repack, all free blocks should get zeroed.
-				open(path) { |f| f.truncate 0 }
-				parent = @ole.dirent_from_path(('/' + path).sub(/\/[^\/]+$/, ''))
-				parent.children.delete dirent
-				1 # hmmm. as per ::File ?
+			def unlink(*paths)
+				paths.each do |path|
+					dirent = @ole.dirent_from_path path
+					# i think we should free all of our blocks from the
+					# allocation table.
+					# i think if you run repack, all free blocks should get zeroed,
+					# but currently the original data is there unmodified.
+					open(path) { |f| f.truncate 0 }
+					# remove ourself from our parent, so we won't be part of the dir
+					# tree at save time.
+					parent_path, basename = File.split expand_path(path)
+					parent = @ole.dir.send :dirent_from_path, parent_path, path
+					parent.children.delete dirent
+				end
+				paths.length # hmmm. as per ::File ?
 			end
+			alias delete :unlink
 		end
 
-		class DirParent
+		#
+		# an *instance* of this class is supposed to provide similar methods
+		# to the class methods of Dir itself.
+		#
+		# pretty complete. like zip/zipfilesystem's implementation, i provide
+		# everything except chroot and glob. glob could be done with a glob
+		# to regex regex, and then simply match in the entries array... although
+		# recursive glob complicates that somewhat.
+		#
+		# Dir.chroot, Dir.glob, Dir.[], and Dir.tmpdir is the complete list.
+		class DirClass
 			def initialize ole
 				@ole = ole
+				@pwd = ''
+			end
+
+			# +orig_path+ is just so that we can use the requested path
+			# in the error messages even if it has been already modified
+			def dirent_from_path path, orig_path=nil
+				orig_path ||= path
+				dirent = @ole.dirent_from_path path
+				raise Errno::ENOENT,  orig_path unless dirent
+				raise Errno::ENOTDIR, orig_path unless dirent.dir?
+				dirent
 			end
+			private :dirent_from_path
 
-			def open path_str
-				dirent = @ole.dirent_from_path path_str
-				# like Errno::ENOTDIR
-				raise "#{path_str.inspect} is not a directory" unless dirent.dir?
-				dir = Dir.new dirent, path_str
+			def open path
+				dir = Dir.new path, entries(path)
 				if block_given?
 					yield dir
 				else
@@ -113,25 +287,88 @@ module Ole # :nodoc:
 				end
 			end
 
-			# certain Dir class methods proxy in this fashion:
+			# as for file, explicit alias to inhibit block
+			def new path
+				open path
+			end
+
+			# pwd is always stored without the trailing slash. we handle
+			# the root case here
+			def pwd
+				if @pwd.empty?
+					'/'
+				else
+					@pwd
+				end
+			end
+			alias getwd :pwd
+
+			def chdir orig_path
+				# make path absolute, squeeze slashes, and remove trailing slash
+				path = @ole.file.expand_path(orig_path).gsub(/\/+/, '/').sub(/\/$/, '')
+				# this is just for the side effects of the exceptions if invalid
+				dirent_from_path path, orig_path
+				if block_given?
+					old_pwd = @pwd
+					begin
+						@pwd = path
+						yield
+					ensure
+						@pwd = old_pwd
+					end
+				else
+					@pwd = path
+					0
+				end
+			end	
+
 			def entries path
-				open(path) { |dir| dir.entries }
+				dirent = dirent_from_path path
+				# Not sure about adding on the dots...
+				entries = %w[. ..] + dirent.children.map(&:name)
+				# do some checks about un-reachable files
+				seen = {}
+				entries.each do |n|
+					Log.warn "inaccessible file (filename contains slash) - #{n.inspect}" if n['/']
+					Log.warn "inaccessible file (duplicate filename) - #{n.inspect}" if seen[n]
+					seen[n] = true
+				end
+				entries
+			end
+
+			def foreach path, &block
+				entries(path).each(&block)
 			end
 
 			# there are some other important ones, like:
-			# chroot (!), mkdir, chdir, rmdir, glob etc etc. for now, i think
-			# mkdir, and rmdir are the main ones we'd need to support
+			# chroot (!), glob etc etc. for now, i think
+			def mkdir path
+				# as for rmdir below:
+				parent_path, basename = File.split @ole.file.expand_path(path)
+				# note that we will complain about the full path despite accessing
+				# the parent path. this is consistent with ::Dir
+				parent = dirent_from_path parent_path, path
+				# now, we first should ensure that it doesn't already exist
+				# either as a file or a directory.
+				raise Errno::EEXIST, path if parent/basename
+				parent.new_child(:dir) { |child| child.name = basename }
+				0
+			end
+
 			def rmdir path
-				dirent = @ole.dirent_from_path path
-				# repeating myself
-				raise "#{path.inspect} is not a directory" unless dirent.dir?
-				# ENOTEMPTY:
-				raise "directory not empty #{path.inspect}" unless dirent.children.empty?
+				dirent = dirent_from_path path
+				raise Errno::ENOTEMPTY, path unless dirent.children.empty?
+
 				# now delete it, how to do that? the canonical representation that is
 				# maintained is the root tree, and the children array. we must remove it
 				# from the children array.
 				# we need the parent then. this sucks but anyway:
-				parent = @ole.dirent_from_path path.sub(/\/[^\/]+$/, '') || '/'
+				# we need to split the path. but before we can do that, we need
+				# to expand it first. eg. say we need the parent to unlink
+				# a/b/../c. the parent should be a, not a/b/.., or a/b.
+				parent_path, basename = File.split @ole.file.expand_path(path)
+				# this shouldn't be able to fail if the above didn't
+				parent = dirent_from_path parent_path
 				# note that the way this currently works, on save and repack time this will get
 				# reflected. to work properly, ie to make a difference now it would have to re-write
 				# the dirent. i think that Ole::Storage#close will handle that. and maybe include a
@@ -139,33 +376,36 @@ module Ole # :nodoc:
 				parent.children.delete dirent
 				0 # hmmm. as per ::Dir ?
 			end
+			alias delete :rmdir
+			alias unlink :rmdir
 
+			# note that there is nothing remotely ole specific about
+			# this class. it simply provides the dir like sequential access
+			# methods on top of an array.
+			# hmm, doesn't throw the IOError's on use of a closed directory...
 			class Dir
 				include Enumerable
-				attr_reader :dirent, :path, :entries, :pos
 
-				def initialize dirent, path
-					@dirent, @path = dirent, path
-					@pos = 0
-					# FIXME: hack, and probably not really desired
-					@entries = %w[. ..] + @dirent.children.map(&:name)
+				attr_reader :path, :entries, :pos
+				def initialize path, entries
+					@path, @entries, @pos = path, entries, 0
 				end
 
 				def each(&block)
-					@entries.each(&block)
+					entries.each(&block)
 				end
 
 				def close
 				end
 
 				def read
-					@entries[@pos]
+					entries[pos]
 				ensure
-					@pos += 1 if @pos < @entries.length
+					@pos += 1 if pos < entries.length
 				end
 
 				def pos= pos
-					@pos = [[0, pos].max, @entries.length].min
+					@pos = [[0, pos].max, entries.length].min
 				end
 
 				def rewind