mgreenly-s3sync 1.2.4

data/lib/s3sync/s3sync.rb

@@ -0,0 +1,724 @@
+#!/usr/bin/env ruby 
+# This software code is made available "AS IS" without warranties of any        
+# kind.  You may copy, display, modify and redistribute the software            
+# code either by itself or as incorporated into your code; provided that        
+# you do not remove any proprietary notices.  Your use of this software         
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code. 
+# (c) 2007 s3sync.net
+#
+
+module S3sync
+
+	$S3SYNC_MIME_TYPES_FILE = (ENV["S3SYNC_MIME_TYPES_FILE"] or '/etc/mime.types')
+	
+	$S3SYNC_VERSION = '1.2.4'
+
+   # always look "here" for include files (thanks aktxyz)
+   $LOAD_PATH << File.expand_path(File.dirname(__FILE__)) 
+   
+	require 'getoptlong'
+	#require 'generator' # http://www.ruby-doc.org/stdlib/libdoc/generator/rdoc/classes/Generator.html
+	require 'thread_generator' # memory doesn't leak with this one, at least nothing near as bad
+	require 'md5'
+	require 'tempfile'
+	require 's3try'
+   
+   # after other mods, so we don't overwrite yaml vals with defaults
+   require 's3config'
+   include S3Config
+	
+	$S3syncDirString = '{E40327BF-517A-46e8-A6C3-AF51BC263F59}'
+	$S3syncDirTag = 'd66759af42f282e1ba19144df2d405d0'
+	$S3syncDirFile = Tempfile.new("s3sync")
+	$S3syncDirFile.puts $S3syncDirString
+	$S3syncDirFile.close # not final; we need this file again to 'put' directory nodes
+	
+	if $S3SYNC_MIME_TYPES_FILE and FileTest.exist?($S3SYNC_MIME_TYPES_FILE)
+		File.open($S3SYNC_MIME_TYPES_FILE, 'r') do |f|
+			$mimeTypes = {}
+			f.each_line do |l|
+				if l =~ /^(\w\S+)\s+(\S.*)$/
+					type = $1
+					exts = $2.split
+					exts.each do |e|
+						$mimeTypes[e.to_s] = type.to_s
+					end
+				end
+			end
+		end
+	end
+	
+	def S3sync.main 	
+		# ---------- OPTIONS PROCESSING ---------- #	
+	
+		$S3syncOptions = Hash.new
+		optionsParser = GetoptLong.new(
+			  [ '--help',    '-h',	GetoptLong::NO_ARGUMENT ],
+			  [ '--ssl',     '-s',	GetoptLong::NO_ARGUMENT ],
+			  [ '--recursive','-r',	GetoptLong::NO_ARGUMENT ],
+			  [ '--public-read','-p', GetoptLong::NO_ARGUMENT ],
+			  [ '--delete',			GetoptLong::NO_ARGUMENT ],
+			  [ '--verbose', '-v',	GetoptLong::NO_ARGUMENT ],
+			  [ '--dryrun',  '-n',	GetoptLong::NO_ARGUMENT ], 
+			  [ '--debug',   '-d',	GetoptLong::NO_ARGUMENT ],
+			  [ '--memory',   '-m',	GetoptLong::NO_ARGUMENT ],
+			  [ '--progress',	GetoptLong::NO_ARGUMENT ],
+              [ '--expires',        GetoptLong::REQUIRED_ARGUMENT ],
+              [ '--cache-control',  GetoptLong::REQUIRED_ARGUMENT ],
+	      [ '--exclude',        GetoptLong::REQUIRED_ARGUMENT ],
+			  [ '--make-dirs',	GetoptLong::NO_ARGUMENT ]
+			  )
+			  
+		def S3sync.usage(message = nil)
+			$stderr.puts message if message
+			name = $0.split('/').last
+			$stderr.puts <<"ENDUSAGE"
+#{name} [options] <source> <destination>\t\tversion #{$S3SYNC_VERSION}
+  --help    -h          --verbose     -v     --dryrun    -n	
+  --ssl     -s          --recursive   -r     --delete
+  --public-read -p      --expires="<exp>"    --cache-control="<cc>"
+  --exclude="<regexp>"  --progress           --debug   -d
+  --make-dirs
+One of <source> or <destination> must be of S3 format, the other a local path.
+Reminders:
+* An S3 formatted item with bucket 'mybucket' and prefix 'mypre' looks like:
+    mybucket:mypre/some/key/name
+* Local paths should always use forward slashes '/' even on Windows
+* Whether you use a trailing slash on the source path makes a difference.
+* For examples see README.
+ENDUSAGE
+		exit
+		end #usage
+		
+		begin
+			optionsParser.each {|opt, arg| $S3syncOptions[opt] = (arg || true)}
+		rescue StandardError
+			usage # the parser already printed an error message
+		end
+		usage if $S3syncOptions['--help']
+		$S3syncOptions['--verbose'] = true if $S3syncOptions['--dryrun'] or $S3syncOptions['--debug'] or $S3syncOptions['--progress']
+		$S3syncOptions['--ssl'] = true if $S3syncOptions['--ssl'] # change from "" to true to appease s3 port chooser
+
+		
+		# ---------- CONNECT ---------- #
+		S3sync::s3trySetup 
+
+		# ---------- PREFIX PROCESSING ---------- #
+	
+		def S3sync.s3Prefix?(pre)
+			# allow for dos-like things e.g. C:\ to be treated as local even with colon
+			pre.include?(':') and not pre.match('^[A-Za-z]:[\\\\/]')
+		end
+		sourcePrefix, destinationPrefix = ARGV
+		usage("You didn't set up your environment variables; see README.txt") if not($AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY) 
+		usage('Need a source and a destination') if sourcePrefix == nil or destinationPrefix == nil
+		usage('Both arguments can\'t be on S3') if s3Prefix?(sourcePrefix) and s3Prefix?(destinationPrefix)
+		usage('One argument must be on S3') if !s3Prefix?(sourcePrefix) and !s3Prefix?(destinationPrefix)
+
+		# so we can modify them
+		sourcePrefix, destinationPrefix = sourcePrefix.dup, destinationPrefix.dup
+
+		# handle trailing slash for source properly
+		if(sourcePrefix !~ %r{/$})
+			# no slash on end of source means we need to append the last src dir to dst prefix
+			# testing for empty isn't good enough here.. needs to be "empty apart from potentially having 'bucket:'"
+			slash = (destinationPrefix.empty? or destinationPrefix.match(%r{:$}))? "" : "/"
+			# not good enough.. sometimes this coughs up the bucket as a prefix destinationPrefix.replace(destinationPrefix + slash + sourcePrefix.split(/(?:\/|:)/).last)
+			# take everything at the end after a slash or colon
+			destinationPrefix.replace(destinationPrefix + slash + %r{([^/:]*)$}.match(sourcePrefix)[1])
+		end
+		# no trailing slash on dest, ever.
+		destinationPrefix.sub!(%r{/$}, "")
+		
+		# don't repeat slashes
+		sourcePrefix.squeeze!('/')
+		destinationPrefix.squeeze!('/')
+		
+		# here's where we find out what direction we're going
+		sourceIsS3 = s3Prefix?(sourcePrefix)
+		# alias these variables to the other strings (in ruby = does not make copies of strings)
+		s3Prefix = sourceIsS3 ? sourcePrefix : destinationPrefix
+		localPrefix = sourceIsS3 ? destinationPrefix : sourcePrefix
+		
+		# canonicalize the S3 stuff
+		s3Bucket = (/^(.*?):/.match(s3Prefix))[1]
+		s3Prefix.replace((/:(.*)$/.match(s3Prefix))[1])
+		debug("s3Prefix #{s3Prefix}")
+		$S3SyncOriginalS3Prefix = s3Prefix.dup
+		
+		# canonicalize the local stuff
+		# but that can kill a trailing slash, which we need to preserve long enough to know whether we mean "the dir" or "its contents"
+		# it will get re-stripped by the local generator after expressing this knowledge
+		localTrailingSlash = localPrefix.match(%r{/$}) 
+		localPrefix.replace(File.expand_path(localPrefix))
+		localPrefix += '/' if localTrailingSlash
+		debug("localPrefix #{localPrefix}")
+		# used for exclusion parsing
+		$S3SyncOriginalLocalPrefix = localPrefix.dup
+		
+		# exclude preparation
+		# we don't want to build then throw away this regexp for each node in the universe; do it once globally
+		$S3SyncExclude = Regexp.new($S3syncOptions['--exclude']) if $S3syncOptions['--exclude']
+		
+		
+		# ---------- GENERATORS ---------- #
+	
+		
+		# a generator that will return the files/dirs of the local tree one by one
+		# sorted and decorated for easy comparison with the S3 tree
+		localTree = Generator.new do |g|
+			def S3sync.localTreeRecurse(g, prefix, path)
+				debug("localTreeRecurse #{prefix} #{path}")
+				#if $S3syncOptions['--memory']
+				#	$stderr.puts "Starting local recurse"
+				#	stats = ostats stats 
+				#end
+				d = nil
+				begin
+					slash = prefix.empty? ? "" : "/"
+					d = Dir.new(prefix + slash + path)
+				rescue Errno::ENOENT
+					# ok the dir doesn't exist at all (this only really occurs for the root i.e. first dir)
+					return nil
+				rescue Errno::EACCES
+					# vista won't even let us touch some stuff in our own profile
+					return nil
+				end
+				# do some pre-processing
+				# the following sleight of hand is to make the recursion match the way s3 sorts
+				# take for example the directory 'foo' and the file 'foo.bar'
+				# when we encounter the dir we would want to recurse into it
+				# but S3 would just say 'period < slash' and sort 'foo.bar' between the dir node 
+				# and the contents in that 'dir'
+				#
+				# so the solution is to not recurse into the directory until the point where
+				# it would come up "next" in the S3 list
+				# We have to do these hoops on the local side, because we have very little control
+				# over how S3 will return its results
+				toAdd = Array.new
+				d.each do |name|
+					slash = path.empty? ? "" : "/"
+					partialPath = path + slash + name
+					slash = prefix.empty? ? "" : "/"
+					fullPath = prefix + slash + partialPath
+					if name == "." or name == ".."
+						# skip
+					else
+						# add a dir node if appropriate
+						debug("Test #{fullPath}")
+						if ((not FileTest.symlink?(fullPath)) and FileTest.directory?(fullPath)) and $S3syncOptions['--recursive']
+							debug("Adding it as a dir node")
+							toAdd.push(name + '/') # always trail slash here for sorting purposes (removed below with rindex test)
+						end
+					end
+				end
+				dItems = d.collect + toAdd
+				d.close
+				d = toAdd = nil
+				dItems.sort! #aws says we will get alpha sorted results but ruby doesn't
+				dItems.each do |name|
+					isDirNode = false
+					if name.rindex('/') == name.length-1
+						name = name.slice(0...name.length-1)
+						isDirNode = true
+						debug("#{name} is a dir node")
+					end
+					slash = path.empty? ? "" : "/"
+					partialPath = path + slash + name
+					slash = prefix.empty? ? "" : "/"
+					fullPath = prefix + slash + partialPath
+					excludePath = fullPath.slice($S3SyncOriginalLocalPrefix.length...fullPath.length)
+					if name == "." or name == ".."
+						# skip
+					elsif $S3SyncExclude and $S3SyncExclude.match(excludePath)
+						debug("skipping local item #{excludePath} because of --exclude")
+					elsif isDirNode
+						localTreeRecurse(g, prefix, partialPath)
+					else
+						# a normal looking node we should try to process
+						debug("local item #{fullPath}")
+						g.yield(LocalNode.new(prefix, partialPath))
+					end
+				end
+				#if $S3syncOptions['--memory']
+				#	$stderr.puts "Ending local recurse"
+				#	stats = ostats stats 
+				#end
+			end
+			# a bit of a special case for local, since "foo/" and "foo" are essentially treated the same by file systems
+			# so we need to think harder about what the user really meant in the command line.
+			localPrefixTrim = localPrefix
+			if localPrefix !~ %r{/$}
+				# no trailing slash, so yield the root itself first, then recurse if appropriate
+				# gork this is still not quite good enough.. if local is the dest then we don't know whether s3 will have a root dir node yielded a priori, so we can't know whether to do this.  only matters for --erase though
+				g.yield(LocalNode.new(localPrefixTrim, "")) # technically we should check this for exclusion, but excluding the root node is kind of senseless.. and that would be a pain to set up here
+				localTreeRecurse(g, localPrefixTrim, "") if $S3syncOptions['--recursive']
+			else
+				# trailing slash, so ignore the root itself, and just go into the first level
+				localPrefixTrim.sub!(%r{/$}, "") # strip the slash because of how we do local node slash accounting in the recurse above
+				localTreeRecurse(g, localPrefixTrim, "") 
+			end
+		end
+		
+		# a generator that will return the nodes in the S3 tree one by one
+		# sorted and decorated for easy comparison with the local tree
+		s3Tree = Generator.new do |g|
+			def S3sync.s3TreeRecurse(g, bucket, prefix, path)
+				if $S3syncOptions['--memory']
+					$stderr.puts "Starting S3 recurse"
+					GC.start
+					stats = ostats stats 
+				end
+				$stderr.puts "s3TreeRecurse #{bucket} #{prefix} #{path}" if $S3syncOptions['--debug']
+				nextPage = true
+				marker = ''
+				while nextPage do
+					fullPrefix = prefix + path
+					debug("nextPage: #{marker}") if marker != ''
+					options = {}
+					options['prefix'] = fullPrefix # start at the right depth
+					options['delimiter'] = '/' # only one dir at a time please
+					options['max-keys'] = '200' # use manageable chunks
+					options['marker'] = marker unless marker == ''
+					d = S3sync.S3try(:list_bucket, bucket, options)
+					$stderr.puts "S3 ERROR: #{d.http_response}" unless d.http_response.is_a? Net::HTTPSuccess
+					# the 'directories' and leaf nodes are in two separate collections
+					# because a dir will never have the same name as a node, we can just shove them together and sort
+					# it's important to evaluate them alphabetically for efficient comparison to the local tree
+					tItems = d.entries + d.common_prefix_entries
+					tItems.sort! do |a,b|
+						aName = a.respond_to?('key') ? a.key : a.prefix
+						bName = b.respond_to?('key') ? b.key : b.prefix
+						# the full path will be returned, efficient to ignore the part we know will be in common
+						aName.slice(fullPrefix.length..aName.length) <=> bName.slice(fullPrefix.length..bName.length)
+					end
+					# get rid of the big s3 objects asap, just save light-weight nodes and strings
+					items = tItems.collect do |item|
+						if item.respond_to?('key')
+							key = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.key).join
+							Node.new(key, item.size, item.etag)
+						else
+							Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.prefix).join
+						end
+					end
+					nextPage = d.properties.is_truncated
+					marker = (d.properties.next_marker)? d.properties.next_marker : ((d.entries.length > 0)? d.entries.last.key : '')
+					# get this into native char set (because when we feed it back to s3 that's what it will expect)
+					marker = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", marker).join
+					tItems = nil
+					d = nil # get rid of this before recursing; it's big
+					item = nil
+					GC.start # not sure but I think yielding before doing this is causing evil closure bloat
+					items.each do |item|
+						if not (item.kind_of? String)
+							# this is an item
+							excludePath = item.name.slice($S3SyncOriginalS3Prefix.length...item.name.length)
+							if $S3SyncExclude and $S3SyncExclude.match(excludePath)
+								debug("skipping S3 item #{excludePath} due to --exclude")
+							else
+								debug("S3 item #{item.name}")
+								g.yield(S3Node.new(bucket, prefix, item))
+							end
+						else
+							# it's a prefix (i.e. there are sub keys)
+							partialPath = item.slice(prefix.length..item.length) # will have trailing slash
+							excludePath = item.slice($S3SyncOriginalS3Prefix.length...item.length)
+							# recurse
+							if $S3SyncExclude and $S3SyncExclude.match(excludePath)
+								debug("skipping prefix #{excludePath} due to --exclude")
+							else
+								debug("prefix found: #{partialPath}")
+								s3TreeRecurse(g, bucket, prefix, partialPath) if $S3syncOptions['--recursive'] 
+							end
+						end
+					end
+					items = nil
+				end # of while nextPage
+				if $S3syncOptions['--memory']
+					$stderr.puts "Ending S3 recurse"
+					GC.start
+					stats = ostats stats 
+				end
+			end
+			# this will yield the root node first and then recurse
+			s3TreeRecurse(g, s3Bucket, s3Prefix, "")
+			
+		end
+		
+		# alias the tree objects so we don't care below which direction the transfer is going
+		if sourceIsS3
+			sourceTree, destinationTree = s3Tree, localTree
+		else
+			sourceTree, destinationTree = localTree, s3Tree
+		end
+		
+		
+		# ---------- COMPARATOR ---------- #
+		
+		# run the comparison engine and act according to what we find for each check
+		nodesToDelete = Array.new # a stack. have to delete in reverse order of normal create/update processing
+		
+		sourceNode = sourceTree.next? ? sourceTree.next : nil
+		destinationNode = destinationTree.next? ? destinationTree.next : nil
+		while sourceNode or destinationNode do
+			debug("source: #{sourceNode.name}") if sourceNode
+			debug("dest: #{destinationNode.name}") if destinationNode
+			if (!destinationNode) or (sourceNode and (sourceNode.name < destinationNode.name))
+				dNode = 
+				if sourceNode.kind_of? LocalNode
+					S3Node.new(s3Bucket, s3Prefix, sourceNode.name)
+				else
+					LocalNode.new(localPrefix, sourceNode.name)
+				end
+				puts "Create node #{sourceNode.name}" if $S3syncOptions['--verbose']
+				dNode.updateFrom(sourceNode) unless $S3syncOptions['--dryrun']
+				sourceNode = sourceTree.next? ? sourceTree.next : nil
+			elsif (!sourceNode) or (destinationNode and (sourceNode.name > destinationNode.name))
+				$stderr.puts "Source does not have #{destinationNode.name}" if $S3syncOptions['--debug']
+				if $S3syncOptions['--delete']
+					if destinationNode.directory?
+						# have to wait
+						nodesToDelete.push(destinationNode) 
+					else
+						puts "Remove node #{destinationNode.name}" if $S3syncOptions['--verbose']
+						destinationNode.delete unless $S3syncOptions['--dryrun']
+					end
+				end
+				destinationNode = destinationTree.next? ? destinationTree.next : nil
+			elsif sourceNode.name == destinationNode.name
+				if (sourceNode.size != destinationNode.size) or (sourceNode.tag != destinationNode.tag)
+					puts "Update node #{sourceNode.name}" if $S3syncOptions['--verbose']
+					destinationNode.updateFrom(sourceNode) unless $S3syncOptions['--dryrun']
+				elsif $S3syncOptions['--debug']
+					$stderr.puts "Node #{sourceNode.name} unchanged" 
+				end
+				sourceNode = sourceTree.next? ? sourceTree.next : nil
+				destinationNode = destinationTree.next? ? destinationTree.next : nil
+			end					
+		end
+		
+		# get rid of the (now empty, except for other directories) directories
+		nodesToDelete.reverse_each do |node|
+			puts "Remove node #{node.name}" if $S3syncOptions['--verbose']
+			node.delete unless $S3syncOptions['--dryrun']
+		end
+		
+	end #main
+
+	
+		
+	# ---------- NODE ---------- #
+	
+	class Node
+		attr_reader :name
+		attr_reader :size 
+		attr_reader :tag
+		def initialize(name='', size = 0, tag = '')
+			@name = name
+			@size = size
+			@tag = tag
+		end
+		def directory?()
+			@tag == $S3syncDirTag and @size == $S3syncDirString.length
+		end
+	end
+		
+	# ---------- S3Node ---------- #
+	
+	class S3Node < Node
+		@path = nil
+		@bucket = nil
+		@result = nil
+		def initialize(bucket, prefix, itemOrName)
+			@bucket = bucket
+			if itemOrName.kind_of? String
+				@name = itemOrName
+				@name.sub!(%r{/$}, "") # don't create directories with a slash on the end
+				#6/2007. the prefix can be filled but the name empty, in the case of s3sync -r somedir somebucket:
+				if (not prefix.empty? and @name.empty?)
+					@name = prefix
+					itemOrName = prefix
+					prefix = ""
+				end
+				slash = prefix.empty? ? "" : "/"
+				@path = prefix + slash + itemOrName
+			else
+				@name = (itemOrName.name.slice((prefix.length)..itemOrName.name.length) or '')
+				# depending whether the prefix is / tailed, the name might need trimming
+				@name.sub!(%r{^/},"") # get rid of leading slash in name if there (from above simplistic split)
+				@name.sub!(%r{/$}, "") # don't create directories with a slash on the end
+				@path = itemOrName.name
+				@path.sub!(%r{/$}, "") # don't create directories with a slash on the end
+				@size = itemOrName.size
+				@tag = itemOrName.tag.gsub(/"/,'')
+			end
+			debug("s3 node object init. Name:#{@name} Path:#{@path} Size:#{@size} Tag:#{@tag}")
+		end
+		# get this item from s3 into the provided stream
+		# S3 pushes to the local item, due to how http streaming is implemented
+		def to_stream(s) 
+			@result = S3sync.S3try(:get_stream, @bucket, @path, {}, s)
+		end
+		def symlink?() 
+			unless @result
+				@result = S3sync.S3try(:head, @bucket, @path)
+			end
+			debug("symlink value is: #{@result.object.metadata['symlink']}")
+			@result.object.metadata['symlink'] == 'true'
+		end
+		def owner
+			unless @result
+				@result = S3sync.S3try(:head, @bucket, @path)
+			end
+			debug("Owner of this s3 node is #{@result.object.metadata['owner']}")
+			@result.object.metadata['owner'].to_i # if not there, will be nil => 0 which == root so good default
+		end
+		def group
+			unless @result
+				@result = S3sync.S3try(:head, @bucket, @path)
+			end
+			@result.object.metadata['group'].to_i # 0 default ok
+		end
+		def permissions
+			g = @result.object.metadata['permissions']
+			g ? g.to_i : 600 # default to owner only
+		end
+		def updateFrom(fromNode)
+			if fromNode.respond_to?(:stream)
+				meta = Hash.new
+				meta['owner'] = fromNode.owner.to_s
+				meta['group'] = fromNode.group.to_s
+				meta['permissions'] = fromNode.permissions.to_s
+				meta['symlink'] = 'true' if fromNode.symlink?
+				begin
+					theStream = fromNode.stream
+					theStream = ProgressStream.new(theStream, fromNode.size) if $S3syncOptions['--progress']
+
+					s3o = S3::S3Object.new(theStream, meta)
+					debug(@path)
+					headers = {'Content-Length' => (fromNode.size.respond_to?(:nonzero?) ? fromNode.size.to_s : '0')}
+					headers['x-amz-acl'] = 'public-read' if $S3syncOptions['--public-read']
+					headers['Expires'] = $S3syncOptions['--expires'] if $S3syncOptions['--expires']
+					headers['Cache-Control'] = $S3syncOptions['--cache-control'] if $S3syncOptions['--cache-control']
+					fType = @path.split('.').last
+					debug("File extension: #{fType}")
+					if defined?($mimeTypes) and fType != '' and (mType = $mimeTypes[fType]) and mType != ''
+						debug("Mime type: #{mType}")
+						headers['Content-Type'] = mType
+					end
+					@result = S3sync.S3try(:put, @bucket, @path, s3o, headers)
+					theStream.close if (theStream and not theStream.closed?)
+				rescue NoMethodError
+					# when --progress is used and we can't get the stream object, it doesn't report as null
+					# so the above .closed? test will break
+					$stderr.puts "Skipping #{@path}: " + $!
+				rescue SystemCallError
+					theStream.close if (theStream and not theStream.closed?)
+					$stderr.puts "Skipping #{@path}: " + $!
+				end
+			else
+				raise "Node provided as update source doesn't support :stream"
+			end
+		end
+		def delete
+			@result = S3sync.S3try(:delete, @bucket, @path)
+		end
+	end
+		
+	# ---------- LocalNode ---------- #
+	
+	class LocalNode < Node
+		@path = nil
+		def initialize(prefix, partialPath)
+			slash = prefix.empty? ? "" : "/"
+			@path = prefix + slash + partialPath
+			# slash isn't at the front of this any more @name = (partialPath.slice(1..partialPath.length) or '')
+			@name = partialPath or ''
+			if FileTest.symlink?(@path)
+				# this could use the 'file' case below, but why create an extra temp file
+				linkData = File.readlink(@path)
+				$stderr.puts "link to: #{linkData}" if $S3syncOptions['--debug']
+				@size = linkData.length
+				md5 = Digest::MD5.new()
+				md5 << linkData
+				@tag = md5.hexdigest
+			elsif FileTest.file?(@path)
+				@size = FileTest.size(@path)
+				data = nil
+				begin
+					data = self.stream
+					md5 = Digest::MD5.new()
+					while !data.eof?
+						md5 << data.read(2048) # stream so it's not taking all memory
+					end
+					data.close
+					@tag = md5.hexdigest
+				rescue SystemCallError
+					# well we're not going to have an md5 that's for sure
+					@tag = nil
+				end
+			elsif FileTest.directory?(@path)
+				# all s3 directories are dummy nodes contain the same directory string
+				# so for easy comparison, set our size and tag thusly
+				@size = $S3syncDirString.length
+				@tag = $S3syncDirTag
+			end
+			debug("local node object init. Name:#{@name} Path:#{@path} Size:#{@size} Tag:#{@tag}")
+		end
+		# return a stream that will read the contents of the local item
+		# local gets pulled by the S3Node update fn, due to how http streaming is implemented
+		def stream
+			begin
+				# 1.0.8 switch order of these tests because a symlinked file will say yes to 'file?'
+				if FileTest.symlink?(@path) or FileTest.directory?(@path)
+					tf = Tempfile.new('s3sync')
+					if FileTest.symlink?(@path)
+						tf.printf('%s', File.readlink(@path))
+					elsif FileTest.directory?(@path)
+						tf.printf('%s', $S3syncDirString)
+					end
+					tf.close
+					tf.open
+					tf
+				elsif FileTest.file?(@path)
+					File.open(@path, 'rb')
+				end
+			rescue SystemCallError
+				$stderr.puts "Could not read #{@path}: #{$!}"
+				raise
+			end
+		end
+		def stat
+			FileTest.symlink?(@path) ? File.lstat(@path) : File.stat(@path)
+		end
+		def exist?
+			FileTest.exist?(@path) or FileTest.symlink?(@path)
+		end
+		def owner
+			self.exist? ? self.stat().uid : 0
+		end
+		def group
+			self.exist? ? self.stat().gid : 0
+		end
+		def permissions
+			self.exist? ? self.stat().mode : 600
+		end
+		def updateFrom(fromNode)
+			if fromNode.respond_to?(:to_stream)
+				fName = @path + '.s3syncTemp'
+            # handle the case where the user wants us to create dirs that don't exist in S3
+            if $S3syncOptions['--make-dirs']
+               # ensure target's path exists
+               dirs = @path.split('/')
+               # but the last one is a file name
+               dirs.pop()
+               current = ''
+               dirs.each do |dir|
+                  current << dir << '/'
+                  begin
+                     Dir.mkdir(current) unless FileTest.exist?(current)
+                  rescue SystemCallError
+                     $stderr.puts "Could not mkdir #{current}: #{$!}"
+                  end
+               end
+            end
+				unless fromNode.directory?
+					f = File.open(fName, 'wb')
+					f = ProgressStream.new(f, fromNode.size) if $S3syncOptions['--progress']
+
+					fromNode.to_stream(f) 
+					f.close
+				end
+				# get original item out of the way
+				File.unlink(@path) if File.exist?(@path)
+				if fromNode.symlink? 
+					linkTo = ''
+					File.open(fName, 'rb'){|f| linkTo = f.read}
+					debug("#{@path} will be a symlink to #{linkTo}")
+					begin
+						File.symlink(linkTo, @path)
+					rescue NotImplementedError
+						# windows doesn't do symlinks, for example
+						# just bail
+						File.unlink(fName) if File.exist?(fName)
+						return
+					rescue SystemCallError
+						$stderr.puts "Could not write symlink #{@path}: #{$!}"
+					end
+				elsif fromNode.directory?
+					# only get here when the dir doesn't exist.  else they'd compare ==
+					debug(@path)
+					begin
+						Dir.mkdir(@path) unless FileTest.exist?(@path)
+					rescue SystemCallError
+						$stderr.puts "Could not mkdir #{@path}: #{$!}"
+					end
+					
+				else
+					begin
+						File.rename(fName, @path)
+					rescue SystemCallError
+						$stderr.puts "Could not write (rename) #{@path}: #{$!}"
+					end
+						
+				end
+				# clean up if the temp file is still there (as for links)
+				File.unlink(fName) if File.exist?(fName)
+				
+				# update permissions
+				linkCommand = fromNode.symlink? ? 'l' : ''
+				begin
+					File.send(linkCommand + 'chown', fromNode.owner, fromNode.group, @path)
+					File.send(linkCommand + 'chmod', fromNode.permissions, @path)
+				rescue NotImplementedError
+					# no one has lchmod, but who really cares
+				rescue SystemCallError
+					$stderr.puts "Could not change owner/permissions on #{@path}: #{$!}"
+				end
+			else
+				raise "Node provided as update source doesn't support :to_stream"
+			end
+		end
+		def symlink?()
+			FileTest.symlink?(@path)
+		end
+		def delete
+         # don't try to delete the restore root dir
+         # this is a quick fix to deal with the fact that the tree recurse has to visit the root node
+         return unless @name != ''
+			return unless FileTest.exist?(@path)
+			begin
+				if FileTest.directory?(@path)
+					Dir.rmdir(@path)
+				else
+					File.unlink(@path)
+				end
+			rescue SystemCallError
+				$stderr.puts "Could not delete #{@path}: #{$!}"
+			end
+		end
+	end	
+	
+	
+end #module
+
+def debug(str)
+	$stderr.puts str if $S3syncOptions['--debug']
+end
+
+def ostats(last_stat = nil)
+  stats = Hash.new(0)
+  ObjectSpace.each_object {|o| stats[o.class] += 1}
+
+  stats.sort {|(k1,v1),(k2,v2)| v2 <=> v1}.each do |k,v|
+    $stderr.printf "%-30s  %10d", k, v
+    $stderr.printf " delta %10d", (v - last_stat[k]) if last_stat
+    $stderr.puts
+  end
+
+  stats
+end 
+
+# go!
+S3sync::main