tpkg 1.21.1 → 1.22.1

data/Rakefile

@@ -5,7 +5,7 @@ spec = Gem::Specification.new do |s|
   s.add_dependency('facter')
   s.add_dependency('net-ssh')
   s.add_dependency('ddao-kwalify')
-  s.version = '1.21.1'
+  s.version = '1.22.1'
   s.authors = ['Darren Dao', 'Jason Heiss']
   s.email = 'tpkg-users@lists.sourceforge.net'
   s.homepage = 'http://tpkg.sourceforge.net'

data/bin/cpan2tpkg

@@ -256,29 +256,46 @@ MODULE: foreach my $name ($extinst->modules)
 		}
 		PREREQ: foreach my $dep (keys %prereqs)
 		{
-			# Skip dependencies on core modules
-			my $depmod = CPAN::Shell->expand('Module', $dep);
-			# This is a bit of an indirect way to identify core modules
-			# but the only way I can figure out.  Core stuff gets
-			# installed with perl into /home/t/perl-version, CPAN
-			# modules into /home/t/lib/perl5/site_perl.
-			#
-			# What seems like it should be the "right" way is that the "D"
-			# (aka "Development Stage") field in dslip_status has "S" (aka
-			# "Standard, supplied with Perl 5") as one of its possible values,
-			# according to the docs (http://perldoc.perl.org/CPAN.html).
-			# However, that doesn't seem to be set reliably.
-			if ($depmod->inst_file && $depmod->inst_file =~ /$Config{prefix}/)
+			if ($dep eq 'perl')
 			{
-				print "Prereq $dep is a core module, skipping\n";
-				next PREREQ;
+				# We always add a dependency on perl anyway, so the
+				# module's dependency is only relevant if it specifies a
+				# newer version than is running.
+				if ($prereqs{$dep} ne '0')
+				{
+					if ($prereqs{$dep} > $])
+					{
+						die "Module requires perl >= $prereqs{$dep}, this is $]\n";
+					}
+				}
 			}
-			
-			my $deppkgname = module_to_pkg_name($depmod);
-			$deps{$deppkgname} = {};
-			if ($prereqs{$dep} ne '0')
+			else
 			{
-				$deps{$deppkgname}{minimum_version} = $prereqs{$dep};
+				# Skip dependencies on core modules
+				my $depmod = CPAN::Shell->expand('Module', $dep);
+				# This is a bit of an indirect way to identify core
+				# modules but the only way I can figure out.  Core stuff
+				# gets installed with perl into /home/t/perl-version,
+				# CPAN modules into /home/t/lib/perl5/site_perl.
+				#
+				# What seems like it should be the "right" way is that
+				# the "D" (aka "Development Stage") field in
+				# dslip_status has "S" (aka "Standard, supplied with
+				# Perl 5") as one of its possible values, according to
+				# the docs (http://perldoc.perl.org/CPAN.html).
+				# However, that doesn't seem to be set reliably.
+				if ($depmod->inst_file && $depmod->inst_file =~ /$Config{prefix}/)
+				{
+					print "Prereq $dep is a core module, skipping\n";
+					next PREREQ;
+				}
+				
+				my $deppkgname = module_to_pkg_name($depmod);
+				$deps{$deppkgname} = {};
+				if ($prereqs{$dep} ne '0')
+				{
+					$deps{$deppkgname}{minimum_version} = $prereqs{$dep};
+				}
 			}
 		}
 	}

data/bin/gem2tpkg

@@ -164,7 +164,6 @@ end
 # Create the directory we want gem to install into
 @gemdir = tempdir('gem2tpkg')
 ENV['GEM_HOME'] = @gemdir
-ENV['GEM_PATH'] = @gemdir
 
 # Install the gem
 geminst = [@gemcmd, 'install', '--no-rdoc', '--no-ri'] | @gems
@@ -186,6 +185,10 @@ if !r
   abort('gem install failed')
 end
 
+# Now set GEM_PATH so that further operations (particularly `gem list`)
+# only operate against the gems installed in our working directory.
+ENV['GEM_PATH'] = @gemdir
+
 @already_packaged = []
 def package(gem)
   pkgfiles = []
@@ -323,8 +326,6 @@ def package(gem)
         maxver = "#{majorver}.9999.9999"
         deps[depgem][:maximum_version] = maxver
       end
-      # Package the dependency
-      pkgfiles.concat(package(depgem))
     end
   end
   if !$?.success?
@@ -454,28 +455,16 @@ def package(gem)
   pkgfiles
 end
 
-# Count the number of gems installed
-gemcount = 0
+# Package each installed gem
+pkgfiles = []
 IO.popen("#{@gemcmd} list") do |pipe|
   pipe.each_line do |line|
     next if line.include?('***')  # Skip header line
     next if line =~ /^\s*$/       # Skip blank lines
-    gemcount += 1
+    gem, version = line.split
+    pkgfiles |= package(gem)
   end
 end
-if gemcount == 0
-  abort "Zero gems installed according to gem list?"
-end
-
-pkgfiles = []
-@gems.each do | gem |
-  pkgfiles |=  package(gem)
-end
-# Make sure the package method made as many packages as there were gems
-# installed
-if pkgfiles.length != gemcount
-  abort "gem count (#{gemcount}) vs pkg count (#{pkgfiles.length}) mismatch"
-end
 
 # Tell the user what packages were created
 puts 'The following packages were created:'

data/bin/tpkg

@@ -28,8 +28,9 @@ require 'tpkg'
 @servers = nil
 @worker_count = 10
 @rerun_with_sudo = false
-@tpkg_options = {}
-@init_options = {}           
+@tpkg_options = {}	# options for instantiating Tpkg object
+@init_options = {}      # options for how to run init scripts
+@other_options = {}     
 @compress = nil
 
 
@@ -70,8 +71,9 @@ opts.on('--upgrade', '-u', '=PACKAGES', 'Upgrade one or more packages', Array) d
   @action_value = opt
 end
 opts.on('--downgrade', '=PACKAGES', 'Downgrade one or more packages', Array) do |opt|
+  @other_options[:downgrade] = true
   @rerun_with_sudo = true
-  @action = :downgrade
+  @action = :upgrade
   @action_value = opt
 end
 opts.on('--ua', 'Upgrade all packages') do |opt|
@@ -85,12 +87,14 @@ opts.on('--remove', '-r', '=PACKAGES', 'Remove one or more packages', Array) do
 end
 opts.on('--rd', '=PACKAGES', 'Similar to -r but also remove depending packages', Array) do |opt|
   @rerun_with_sudo = true
-  @action = :remove_all_dep
+  @other_options[:remove_all_dep] = true
+  @action = :remove
   @action_value = opt
 end
 opts.on('--rp', '=PACKAGES', 'Similar to -r but also remove prerequisites', Array) do |opt|
   @rerun_with_sudo = true
-  @action = :remove_all_prereq
+  @other_options[:remove_all_prereq] = true
+  @action = :remove
   @action_value = opt
 end
 opts.on('--ra', 'Remove all packages') do |opt|
@@ -229,7 +233,10 @@ end
 opts.on('--lock-force', 'Force the removal of an existing lockfile') do |opt|
   @lockforce = opt
 end
-opts.on('--force', 'Force the execution of a given task.') do |opt|
+opts.on('--force-replace', 'Replace conflicting pkgs with the new one(s)') do |opt|
+  @other_options[:force_replace] = opt
+end
+opts.on('--force', 'Force the execution of a given task') do |opt|
   @force = opt
 end
 opts.on('--use-ssh-key', 'Use ssh key for deploying instead of password') do |opt|
@@ -240,7 +247,7 @@ opts.on('--deploy-as', '=USERNAME', 'What username to use for deploying to remot
   @deploy_options["deploy-as"] = opt
   @deploy_params = @deploy_params - ['--deploy-as']
 end
-opts.on('--compress', '=[TYPE]', 'Compress files when making packages.') do |opt|
+opts.on('--compress', '=[TYPE]', 'Compress files when making packages') do |opt|
   @compress = opt
 end
 opts.on('--debug', 'Print lots of messages about what tpkg is doing') do |opt|
@@ -280,7 +287,7 @@ def instantiate_tpkg(options = {})
   sources = options["sources"] || []
   report_server = nil
   
-  [File.join(Tpkg::CONFIGDIR, 'tpkg.conf'), "#{ENV['HOME']}/.tpkg.conf"].each do |configfile|
+  [File.join(Tpkg::CONFIGDIR, 'tpkg.conf'), File.join(ENV['HOME'], ".tpkg.conf")].each do |configfile|
     if File.exist?(configfile)
       IO.foreach(configfile) do |line|
         line.chomp!
@@ -355,23 +362,13 @@ when :extract
   Tpkg::extract_metadata(@action_value)
 when :install
   tpkg = instantiate_tpkg(@tpkg_options)
-  ret_val = tpkg.install(@action_value, passphrase_callback)
+  ret_val = tpkg.install(@action_value, passphrase_callback, @other_options)
 when :upgrade
   tpkg = instantiate_tpkg(@tpkg_options)
-  ret_val = tpkg.upgrade(@action_value, passphrase_callback)
-when :downgrade
-  downgrade = true
-  tpkg = instantiate_tpkg(@tpkg_options)
-  ret_val = tpkg.upgrade(@action_value, passphrase_callback, downgrade)
+  ret_val = tpkg.upgrade(@action_value, passphrase_callback, @other_options)
 when :remove
   tpkg = instantiate_tpkg(@tpkg_options)
-  ret_val = tpkg.remove(@action_value)
-when :remove_all_dep
-  tpkg = instantiate_tpkg(@tpkg_options)
-  ret_val = tpkg.remove(@action_value, {:remove_all_dep => true})
-when :remove_all_prereq
-  tpkg = instantiate_tpkg(@tpkg_options)
-  ret_val = tpkg.remove(@action_value, {:remove_all_prereq => true})
+  ret_val = tpkg.remove(@action_value, @other_options)
 when :verify
   result = nil
   # Verify a given .tpkg file

data/lib/tpkg.rb

@@ -56,7 +56,7 @@ require 'kwalify'        # for validating yaml
 
 class Tpkg
   
-  VERSION = '1.21.1'
+  VERSION = '1.22.1'
   CONFIGDIR = '/etc'
 
   GENERIC_ERR = 1  
@@ -131,6 +131,8 @@ class Tpkg
   end
   def self.clear_cached_tar
     @@tar = nil
+    @@taroptions = ""
+    @@tarinfo = {:version => 'unknown'}
   end
   
   # Encrypts the given file in-place (the plaintext file is replaced by the
@@ -450,47 +452,6 @@ class Tpkg
     #return FileMetadata.new(YAML::dump(filemetadata),'yml')
     return FileMetadata.new(Marshal::dump(filemetadata),'bin')
   end
-
-  def self.get_xml_filemetadata_from_directory(tpkgdir)
-    filemetadata_xml = REXML::Document.new
-    filemetadata_xml << REXML::Element.new('files')
-
-    # create file_metadata.xml that stores list of files and their checksum
-    # will be used later on to check whether installed files have been changed
-    root_dir = File.join(tpkgdir, "root")
-    reloc_dir = File.join(tpkgdir, "reloc")
-    Find.find(root_dir, reloc_dir) do |f|
-      next if !File.exist?(f)
-      relocatable = "false"
-
-      # check if it's from root dir or reloc dir
-      if f =~ /^#{root_dir}/
-        short_fn = f[root_dir.length ..-1]
-      else
-        short_fn = f[reloc_dir.length + 1..-1]
-        relocatable = "true"
-      end
-
-      next if short_fn.nil? or short_fn.empty?
-
-      file_ele = filemetadata_xml.root.add_element("file", {"relocatable" => relocatable})
-      path_ele = file_ele.add_element("path")
-      path_ele.add_text(short_fn)
-
-      # only do checksum for file
-      if File.file?(f)
-        # this doesn't work for older ruby version
-        #digest = Digest::SHA256.file(f).hexdigest
-        digest = Digest::SHA256.hexdigest(File.read(f))
-        chksum_ele = file_ele.add_element("checksum")
-        alg_ele = chksum_ele.add_element("algorithm")
-        alg_ele.add_text("SHA256")
-        digest_ele = chksum_ele.add_element("digest")
-        digest_ele.add_text(digest)
-      end
-    end
-    return filemetadata_xml
-  end
   
   def self.verify_package_checksum(package_file)
     topleveldir = package_toplevel_directory(package_file)
@@ -552,12 +513,10 @@ class Tpkg
     ['yml','xml'].each do |format|
       file = File.join('tpkg', "tpkg.#{format}")
 
-      extract_tpkg_tar_command = cmd_to_extract_tpkg_tar(package_file, topleveldir)
-
       # use popen3 instead of popen because popen display stderr when there's an error such as
       # tpkg.yml not being there, which is something we want to ignore since old tpkg doesn't 
       # have tpkg.yml file
-      #stdin, stdout, stderr = Open3.popen3("#{find_tar} -xf #{package_file} -O #{File.join(topleveldir, 'tpkg.tar')} | #{find_tar} -xf - -O #{file}") 
+      extract_tpkg_tar_command = cmd_to_extract_tpkg_tar(package_file, topleveldir)
       stdin, stdout, stderr = Open3.popen3("#{extract_tpkg_tar_command} | #{find_tar} -xf - -O #{file}") 
       filecontent = stdout.read
       if filecontent.nil? or filecontent.empty?
@@ -576,60 +535,6 @@ class Tpkg
     return metadata
   end
  
-  # TODO: To be deprecated 
-  # Extracts and returns the metadata from a package file
-  def self.xml_metadata_from_package(package_file)
-    topleveldir = package_toplevel_directory(package_file)
-    # Verify checksum
-    verify_package_checksum(package_file)
-    # Extract and parse tpkg.xml
-    tpkg_xml = nil
-    extract_tpkg_tar_command = cmd_to_extract_tpkg_tar(package_file, topleveldir)
-    IO.popen("#{extract_tpkg_tar_command} | #{find_tar} -xf - -O #{File.join('tpkg', 'tpkg.xml')}") do |pipe|
-      tpkg_xml = REXML::Document.new(pipe.read)
-    end
-    if !$?.success?
-      warn "Warning: Extracting tpkg.xml from #{package_file} failed"
-      return nil
-    end
-
-    # Insert an attribute on the root element with the package filename
-    tpkg_xml.root.attributes['filename'] = File.basename(package_file)
-
-    # Return
-    return tpkg_xml
-  end
-
-  # TODO: To be deprecated
-  # Extracts and returns the metadata from a directory of package files
-  def self.xml_metadata_from_directory(directory)
-    metadata = []
-    # if metadata.xml already exists, then go ahead and
-    # parse it
-    existing_metadata_file = File.join(directory, 'metadata.xml')
-    existing_metadata = {}
-    if File.exists?(existing_metadata_file)
-      tpkg_metadata_xml = REXML::Document.new(File.open(existing_metadata_file))
-
-      tpkg_metadata_xml.root.elements.each do | metadata_xml |
-        existing_metadata[metadata_xml.attributes['filename']] = metadata_xml
-      end
-    end
-
-    # Populate the metadata array with metadata for all of the packages
-    # in the given directory. Reuse existing metadata if possible.
-    Dir.glob(File.join(directory, '*.tpkg')) do |pkg|
-      if existing_metadata[File.basename(pkg)]
-        metadata << existing_metadata[File.basename(pkg)]
-      else
-        xml = xml_metadata_from_package(pkg)
-        metadata << xml.root if xml
-      end
-    end
-
-    return metadata
-  end
-  
   # Extracts and returns the metadata from a directory of package files
   def self.metadata_from_directory(directory)
     metadata = []
@@ -667,26 +572,6 @@ class Tpkg
   # to metadata.xml in that directory
   def self.extract_metadata(directory, dest=nil)
     dest = directory if dest.nil?
-    # we're no longer generating metadata.xml
-    backward_compatible = false
-
-    # If we still want to support metadata.xml
-    if backward_compatible
-      metadata_xml = xml_metadata_from_directory(directory)
-      # Combine all of the individual metadata files into one XML document
-      metadata = REXML::Document.new
-      metadata << REXML::Element.new('tpkg_metadata')
-      metadata_xml.each do |md|
-        metadata.root << md
-      end
-      # And write that out to metadata.xml
-      metadata_tmpfile = Tempfile.new('metadata.xml', dest)
-      metadata.write(metadata_tmpfile)
-      metadata_tmpfile.close
-      File.chmod(0644, metadata_tmpfile.path)
-      File.rename(metadata_tmpfile.path, File.join(dest, 'metadata.xml'))
-    end
-
     metadata = metadata_from_directory(directory)
     # And write that out to metadata.yml
     metadata_tmpfile = Tempfile.new('metadata.yml', dest)
@@ -1093,7 +978,7 @@ class Tpkg
   # deploy_params is an array that holds the list of paramters that is used when invoking tpkg on to the remote
   # servers where we want to deploy to. 
   # 
-  # servers is an array or a callback that list the remote servers where we want to deploy to
+  # servers is an array, a filename or a callback that list the remote servers where we want to deploy to
   def self.deploy(deploy_params, deploy_options, servers)
     servers.uniq!
     deployer = Deployer.new(deploy_options)
@@ -1162,9 +1047,7 @@ class Tpkg
   #  2) the file_defaults settings of the metadata file
   #  3) the explicitly defined settings in the corresponding file section of the metadata file
   def self.predict_file_perms_and_ownership(data)
-    perms = nil
-    uid = nil
-    gid = nil
+    perms = uid = gid = nil
 
     # get current permission and ownership
     if data[:actual_file]
@@ -1193,7 +1076,8 @@ class Tpkg
     return perms, uid, gid
   end 
 
-  # Given a package file, figure out of tpkg.tar was compressed
+  # Given a package file, figure out if tpkg.tar was compressed
+  # Return what type of compression. If tpkg.tar wasn't compressed, then return nil.
   def self.get_compression(package_file)
     compression = nil
     IO.popen("#{find_tar} -tf #{package_file} #{@@taroptions}") do |pipe|
@@ -1239,6 +1123,16 @@ class Tpkg
     end
     return result
   end
+
+  # Used where we wish to capture an exception and modify the message.  This
+  # method returns a new exception with desired message but with the backtrace
+  # from the original exception so that the backtrace info is not lost.  This
+  # is necessary because Exception lacks a set_message method.
+  def self.wrap_exception(e, message)
+    eprime = e.exception(message)
+    eprime.set_backtrace(e.backtrace)
+    eprime
+  end
   
   #
   # Instance methods
@@ -1332,6 +1226,7 @@ class Tpkg
     @lock_pid_file = File.join(@lock_directory, 'pid')
     @locks = 0
     @installed_metadata = {}
+    @available_packages_cache = {}
   end
   
   def source_to_local_directory(source)
@@ -1362,30 +1257,24 @@ class Tpkg
         else
           uri = http = localdate = remotedate = localdir = localpath = nil
 
-          ['metadata.yml', 'metadata.xml'].each do | metadata_file |
-            uri = URI.join(source, metadata_file)
-            http = Tpkg::gethttp(uri)
+          uri = URI.join(source, 'metadata.yml')
+          http = Tpkg::gethttp(uri)
 
-            # Calculate the path to the local copy of the metadata for this URI
-            localdir = source_to_local_directory(source)
-            localpath = File.join(localdir, metadata_file)
-            localdate = nil
-            if File.exist?(localpath)
-              localdate = File.mtime(localpath)
-            end
+          # Calculate the path to the local copy of the metadata for this URI
+          localdir = source_to_local_directory(source)
+          localpath = File.join(localdir, 'metadata.yml')
+          if File.exist?(localpath)
+            localdate = File.mtime(localpath)
+          end
 
-            # For now, we always have to hit the repo once to determine if
-            # it has metadata.yml or metadata.xml. In the future,
-            # we will only support metadata.yml
-            response = http.head(uri.path)
-            case response
-            when Net::HTTPSuccess
-              remotedate = Time.httpdate(response['Date'])
-              break 
-            else
-              puts "Error fetching metadata from #{uri}: #{response.body}"
-              next
-            end
+          # get last modified time of the metadata file from the server
+          response = http.head(uri.path)
+          case response
+          when Net::HTTPSuccess
+            remotedate = Time.httpdate(response['Date'])
+          else
+            puts "Error fetching metadata from #{uri}: #{response.body}"
+            response.error!  # Throws an exception
           end
 
           # Fetch the metadata if necessary
@@ -1416,22 +1305,9 @@ class Tpkg
           else
             metadata_contents = IO.read(localpath)
           end
-
-          if uri.path =~ /yml/
-            Metadata::get_pkgs_metadata_from_yml_doc(metadata_contents, metadata, source)
-          else
-            # At this stage we just break up the metadata.xml document into
-            # per-package chunks and save them for further parsing later.
-            # This allows us to parse the whole metadata.xml just once, and
-            # saves us from having to further parse and convert the
-            # per-package chunks until if/when they are needed.
-            tpkg_metadata = REXML::Document.new(metadata_contents)
-            tpkg_metadata.elements.each('/tpkg_metadata/tpkg') do |metadata_xml|
-              name = metadata_xml.elements['name'].text
-              metadata[name] = [] if !metadata[name]
-              metadata[name] << Metadata.new(metadata_xml.to_s, 'xml', source)
-            end
-          end
+          # This method will parse the yml doc and populate the metadata variable
+          # with list of pkgs' metadata
+          Metadata::get_pkgs_metadata_from_yml_doc(metadata_contents, metadata, source)
         end
       end
       @metadata = metadata
@@ -1754,7 +1630,7 @@ class Tpkg
     end
   end
   
-  # Returns an array of the tpkg.xml metadata for installed packages
+  # Returns an array of metadata for installed packages
   def metadata_for_installed_packages
     metadata = {}
     if File.directory?(@installed_directory)
@@ -1851,44 +1727,54 @@ class Tpkg
 
   # Returns an array of packages which meet the given requirement
   def available_packages_that_meet_requirement(req=nil)
-    pkgs = []
+    pkgs = nil
     puts "avail_pkgs_that_meet_req checking for #{req.inspect}" if @@debug
-    if req
-      if req[:type] == :native
-        load_available_native_packages(req[:name])
-        @available_native_packages[req[:name]].each do |pkg|
-          if Tpkg::package_meets_requirement?(pkg, req)
-            pkgs << pkg
+    if @available_packages_cache[req]
+      puts "avail_pkgs_that_meet_req returning cached result" if @@debug
+      pkgs = @available_packages_cache[req]
+    else
+      pkgs = []
+      if req
+        req = req.clone # we're using req as the key for our cache, so it's important 
+                        # that we clone it here. Otherwise, req can be changed later on from 
+                        # the calling method and modify our cache inadvertently
+        if req[:type] == :native
+          load_available_native_packages(req[:name])
+          @available_native_packages[req[:name]].each do |pkg|
+            if Tpkg::package_meets_requirement?(pkg, req)
+              pkgs << pkg
+            end
           end
-        end
-      else
-        load_available_packages(req[:name])
-        @available_packages[req[:name]].each do |pkg|
-          if Tpkg::package_meets_requirement?(pkg, req)
-            pkgs << pkg
+        else
+          load_available_packages(req[:name])
+          @available_packages[req[:name]].each do |pkg|
+            if Tpkg::package_meets_requirement?(pkg, req)
+              pkgs << pkg
+            end
           end
+          # There's a weird dicotomy here where @available_packages contains
+          # available tpkg and native packages, and _installed_ native
+          # packages, but not installed tpkgs.  That's somewhat intentional,
+          # as we don't want to cache the installed state since that might
+          # change during a run.  We probably should be consistent, and not
+          # cache installed native packages either.  However, we do have
+          # some intelligent caching of the installed tpkg state which would
+          # be hard to replicate for native packages, and this method gets
+          # called a lot so re-running the native package query commands
+          # frequently would not be acceptable.  So maybe we have the right
+          # design, and this just serves as a note that it is not obvious.
+          pkgs.concat(installed_packages_that_meet_requirement(req))
         end
-        # There's a weird dicotomy here where @available_packages contains
-        # available tpkg and native packages, and _installed_ native
-        # packages, but not installed tpkgs.  That's somewhat intentional,
-        # as we don't want to cache the installed state since that might
-        # change during a run.  We probably should be consistent, and not
-        # cache installed native packages either.  However, we do have
-        # some intelligent caching of the installed tpkg state which would
-        # be hard to replicate for native packages, and this method gets
-        # called a lot so re-running the native package query commands
-        # frequently would not be acceptable.  So maybe we have the right
-        # design, and this just serves as a note that it is not obvious.
-        pkgs.concat(installed_packages_that_meet_requirement(req))
+      else
+        # We return everything available if given a nil requirement
+        # We do not include native packages
+        load_available_packages
+        # @available_packages is a hash of pkgname => array of pkgs
+        # Thus m is a 2 element array of [pkgname, array of pkgs]
+        # And thus m[1] is the array of packages
+        pkgs = @available_packages.collect{|m| m[1]}.flatten
       end
-    else
-      # We return everything available if given a nil requirement
-      # We do not include native packages
-      load_available_packages
-      # @available_packages is a hash of pkgname => array of pkgs
-      # Thus m is a 2 element array of [pkgname, array of pkgs]
-      # And thus m[1] is the array of packages
-      pkgs = @available_packages.collect{|m| m[1]}.flatten
+      @available_packages_cache[req] = pkgs
     end
     pkgs
   end
@@ -1949,6 +1835,16 @@ class Tpkg
       files[:normalized] << File.join(@base, relocfile)
     end
   end
+  def normalize_path(path,root=nil,base=nil)
+    root ||= @file_system_root
+    base ||= @base 
+    if path[0,1] == File::SEPARATOR
+      normalized_path = File.join(root, path)
+    else
+      normalized_path = File.join(base, path)
+    end
+    normalized_path
+  end
   def files_for_installed_packages(package_files=nil)
     files = {}
     if !package_files
@@ -2353,12 +2249,7 @@ class Tpkg
     metadata[:files][:files].each do |tpkgfile|
       if tpkgfile[:init]
         tpkg_path = tpkgfile[:path]
-        installed_path = nil
-        if tpkg_path[0,1] == File::SEPARATOR
-          installed_path = File.join(@file_system_root, tpkg_path)
-        else
-          installed_path = File.join(@base, tpkg_path)
-        end
+        installed_path = normalize_path(tpkg_path)
         init_scripts[installed_path] = tpkgfile
       end
     end 
@@ -2431,12 +2322,7 @@ class Tpkg
     metadata[:files][:files].each do |tpkgfile|
       if tpkgfile[:crontab]
         tpkg_path = tpkgfile[:path]
-        installed_path = nil
-        if tpkg_path[0,1] == File::SEPARATOR
-          installed_path = File.join(@file_system_root, tpkg_path)
-        else
-          installed_path = File.join(@base, tpkg_path)
-        end
+        installed_path = normalize_path(tpkg_path)
         destinations[installed_path] = {}
         
         # Decide whether we're going to add the file to a per-user
@@ -2494,18 +2380,24 @@ class Tpkg
         IO.popen("#{externalpath} '#{pkgfile}' install", 'w') do |pipe|
           pipe.write(data)
         end
+        if !$?.success?
+          raise "Exit value #{$?.exitstatus}"
+        end
       rescue => e
         # Tell the user which external and package were involved, otherwise
         # failures in externals are very hard to debug
-        raise e.exception("External #{name} #{operation} for #{File.basename(pkgfile)}: " + e.message)
+        raise Tpkg.wrap_exception(e, "External #{name} #{operation} for #{File.basename(pkgfile)}: " + e.message)
       end
     when :remove
       begin
         IO.popen("#{externalpath} '#{pkgfile}' remove", 'w') do |pipe|
           pipe.write(data)
         end
+        if !$?.success?
+          raise "Exit value #{$?.exitstatus}"
+        end
       rescue => e
-        raise e.exception("External #{name} #{operation} for #{File.basename(pkgfile)}: " + e.message)
+        raise Tpkg.wrap_exception(e, "External #{name} #{operation} for #{File.basename(pkgfile)}: " + e.message)
       end
     else
       raise "Bug, unknown external operation #{operation}"
@@ -2520,6 +2412,14 @@ class Tpkg
     ret_val = 0
     metadata = Tpkg::metadata_from_package(package_file)
 
+    # set env variable to let pre/post install know  whether this unpack 
+    # is part of an install or upgrade
+    if options[:is_doing_upgrade]
+       ENV['TPKG_ACTION'] = "upgrade"
+    else
+       ENV['TPKG_ACTION'] = "install"
+    end
+
     # Unpack files in a temporary directory
     # I'd prefer to unpack on the fly so that the user doesn't need to
     # have disk space to hold three copies of the package (the package
@@ -2533,22 +2433,19 @@ class Tpkg
     system("#{extract_tpkg_tar_cmd} | #{@tar} #{@@taroptions} -C #{workdir} -xpf -")
     files_info = {} # store perms, uid, gid, etc. for files
     checksums_of_decrypted_files = {}
-    root_dir = File.join(workdir, 'tpkg', 'root')
-    reloc_dir = File.join(workdir, 'tpkg', 'reloc')
-    rel_root_dir = File.join('tpkg', 'root')
-    rel_reloc_dir = File.join('tpkg', 'reloc')
- 
+
     # Get list of conflicting files/directories & store their perm/ownership. That way, we can
     # set them to the correct values later on in order to preserve them.
-    # TODO: verify this command works on all platforms
+    rel_root_dir = File.join('tpkg', 'root')
+    rel_reloc_dir = File.join('tpkg', 'reloc')
     files = `#{extract_tpkg_tar_cmd} | #{@tar} #{@@taroptions} -tf -`
     files = files.split("\n")
     conflicting_files = {}
     files.each do | file |
       if file =~ /^#{rel_root_dir}/
-        possible_conflicting_file = "#{@file_system_root}/#{file[rel_root_dir.length ..-1]}"
+        possible_conflicting_file = File.join(@file_system_root, file[rel_root_dir.length ..-1])
       elsif file =~ /^#{rel_reloc_dir}/
-        possible_conflicting_file = "#{@base}/#{file[rel_reloc_dir.length + 1..-1]}"
+        possible_conflicting_file = File.join(@base, file[rel_reloc_dir.length + 1..-1])
       end
       if possible_conflicting_file && (File.exists?(possible_conflicting_file) && !File.symlink?(possible_conflicting_file))
          conflicting_files[File.join(workdir, file)] = File.stat(possible_conflicting_file)
@@ -2568,18 +2465,14 @@ class Tpkg
     default_gid = DEFAULT_OWNERSHIP_UID
     default_perms = nil
 
-    if metadata[:files] && metadata[:files][:file_defaults]
-      if metadata[:files][:file_defaults][:posix]
-        if metadata[:files][:file_defaults][:posix][:owner]
-          default_uid = Tpkg::lookup_uid(metadata[:files][:file_defaults][:posix][:owner])
-        end
-        if metadata[:files][:file_defaults][:posix][:group]
-          default_gid = Tpkg::lookup_gid(metadata[:files][:file_defaults][:posix][:group])
-        end
-        if metadata[:files][:file_defaults][:posix][:perms]
-          default_perms = metadata[:files][:file_defaults][:posix][:perms]
-        end
-      end
+    if (metadata[:files][:file_defaults][:posix][:owner] rescue nil)
+      default_uid = Tpkg::lookup_uid(metadata[:files][:file_defaults][:posix][:owner])
+    end
+    if (metadata[:files][:file_defaults][:posix][:group] rescue nil)
+      default_gid = Tpkg::lookup_gid(metadata[:files][:file_defaults][:posix][:group])
+    end
+    if (metadata[:files][:file_defaults][:posix][:perms] rescue nil)
+      default_perms = metadata[:files][:file_defaults][:posix][:perms]
     end
 
     # Set default dir uid/gid to be same as for file.
@@ -2587,20 +2480,18 @@ class Tpkg
     default_dir_gid = default_gid
     default_dir_perms = 0755
 
-    if metadata[:files] && metadata[:files][:dir_defaults]
-      if metadata[:files][:dir_defaults][:posix]
-        if metadata[:files][:dir_defaults][:posix][:owner]
-          default_dir_uid = Tpkg::lookup_uid(metadata[:files][:dir_defaults][:posix][:owner])
-        end
-        if metadata[:files][:dir_defaults][:posix][:group]
-          default_dir_gid = Tpkg::lookup_gid(metadata[:files][:dir_defaults][:posix][:group])
-        end
-        if metadata[:files][:dir_defaults][:posix][:perms]
-          default_dir_perms = metadata[:files][:dir_defaults][:posix][:perms]
-        end
-      end
+    if (metadata[:files][:dir_defaults][:posix][:owner] rescue nil)
+      default_dir_uid = Tpkg::lookup_uid(metadata[:files][:dir_defaults][:posix][:owner])
+    end
+    if (metadata[:files][:dir_defaults][:posix][:group] rescue nil)
+      default_dir_gid = Tpkg::lookup_gid(metadata[:files][:dir_defaults][:posix][:group])
+    end
+    if (metadata[:files][:dir_defaults][:posix][:perms] rescue nil)
+      default_dir_perms = metadata[:files][:dir_defaults][:posix][:perms]
     end
 
+    root_dir = File.join(workdir, 'tpkg', 'root')
+    reloc_dir = File.join(workdir, 'tpkg', 'reloc')
     Find.find(root_dir, reloc_dir) do |f|
       # If the package doesn't contain either of the top level
       # directories we need to skip them, find will pass them to us
@@ -2636,12 +2527,7 @@ class Tpkg
     # Handle any decryption and ownership/permissions on specific files
     metadata[:files][:files].each do |tpkgfile|
       tpkg_path = tpkgfile[:path]
-      working_path = nil
-      if tpkg_path[0,1] == File::SEPARATOR
-        working_path = File.join(workdir, 'tpkg', 'root', tpkg_path)
-      else
-        working_path = File.join(workdir, 'tpkg', 'reloc', tpkg_path)
-      end
+      working_path = normalize_path(tpkg_path, File.join(workdir, 'tpkg', 'root'), File.join(workdir, 'tpkg', 'reloc'))
       if !File.exist?(working_path) && !File.symlink?(working_path)
         raise "tpkg.xml for #{File.basename(package_file)} references file #{tpkg_path} but that file is not in the package"
       end
@@ -2698,7 +2584,7 @@ class Tpkg
           end
           
           digest = Digest::SHA256.hexdigest(File.read(working_path))
-          # get checksum for the decrypted file. Will be used for creating file_metadata.xml
+          # get checksum for the decrypted file. Will be used for creating file_metadata
           checksums_of_decrypted_files[File.expand_path(tpkg_path)] = digest 
         end
       end
@@ -3048,35 +2934,41 @@ class Tpkg
     r
   end
   
-  def run_externals_for_install(metadata, workdir, externals_to_skip)
+  def run_externals_for_install(metadata, workdir, externals_to_skip=[])
     metadata[:externals].each do |external|
-      # If the external references a datafile or datascript then read/run it
-      # now that we've unpacked the package contents and have the file/script
-      # available.  This will get us the data for the external.
-      if external[:datafile] || external[:datascript]
-        pwd = Dir.pwd
-        # chdir into the working directory so that the user can specify a
-        # relative path to their file/script.
-        Dir.chdir(File.join(workdir, 'tpkg'))
-        if external[:datafile]
-          # Read the file
-          external[:data] = IO.read(external[:datafile])
-          # Drop the datafile key so that we don't waste time re-reading the
-          # datafile again in the future.
-          external.delete(:datafile)
-        elsif external[:datascript]
-          # Run the script
-          IO.popen(external[:datascript]) do |pipe|
-            external[:data] = pipe.read
+      if !externals_to_skip || !externals_to_skip.include?(external)
+        # If the external references a datafile or datascript then read/run it
+        # now that we've unpacked the package contents and have the file/script
+        # available.  This will get us the data for the external.
+        if external[:datafile] || external[:datascript]
+          pwd = Dir.pwd
+          # chdir into the working directory so that the user can specify a
+          # relative path to their file/script.
+          Dir.chdir(File.join(workdir, 'tpkg'))
+          begin
+            if external[:datafile]
+              # Read the file
+              external[:data] = IO.read(external[:datafile])
+              # Drop the datafile key so that we don't waste time re-reading the
+              # datafile again in the future.
+              external.delete(:datafile)
+            elsif external[:datascript]
+              # Run the script
+              IO.popen(external[:datascript]) do |pipe|
+                external[:data] = pipe.read
+              end
+              if !$?.success?
+                raise "Datascript #{external[:datascript]} for package #{File.basename(metadata[:filename])} had exit value #{$?.exitstatus}"
+              end
+              # Drop the datascript key so that we don't waste time re-running the
+              # datascript again in the future.
+              external.delete(:datascript)
+            end
+          ensure
+            # Switch back to our previous directory
+            Dir.chdir(pwd)
           end
-          # Drop the datascript key so that we don't waste time re-running the
-          # datascript again in the future.
-          external.delete(:datascript)
         end
-        # Switch back to our previous directory
-        Dir.chdir(pwd)
-      end
-      if !externals_to_skip || !externals_to_skip.include?(external)
         run_external(metadata[:filename], :install, external[:name], external[:data])
       end
     end if metadata[:externals]
@@ -3091,7 +2983,7 @@ class Tpkg
     metadata.write(metadata_file)
     metadata_file.close
     
-    # Save file_metadata.yml for this pkg
+    # Save file_metadata for this pkg
     if File.exist?(File.join(workdir, 'tpkg', 'file_metadata.bin'))
       file_metadata = FileMetadata.new(File.read(File.join(workdir, 'tpkg', 'file_metadata.bin')), 'bin')
     elsif File.exist?(File.join(workdir, 'tpkg', 'file_metadata.yml'))
@@ -3102,8 +2994,11 @@ class Tpkg
     if file_metadata
       file_metadata[:package_file] = File.basename(package_file)
       file_metadata[:files].each do |file|
+        # update file_metadata with user/group ownership and permission 
         acl = files_info[file[:path]] 
         file.merge!(acl) unless acl.nil?
+
+        # update file_metadata with the checksums of decrypted files
         digest = checksums_of_decrypted_files[File.expand_path(file[:path])]
         if digest
           digests = file[:checksum][:digests]
@@ -3329,22 +3224,62 @@ class Tpkg
     conflicts
   end
 
-  def check_for_conflicting_pkgs(pkgs_to_check)
-    # loop through packages that we're interested in, check for conflict listing,
-    # see if there are any conflicts among each other
-    pkgs_to_check.each do |pkg1|
+  # This method is called by install and upgrade method to make sure there is
+  # no conflicts between the existing pkgs and the pkgs we're about to install
+  def handle_conflicting_pkgs(installed_pkgs, pkgs_to_install, options ={})
+    conflicting_pkgs = []
+
+    # check if existing pkgs have conflicts with pkgs we're about to install
+    installed_pkgs.each do |pkg1|
+      next if pkg1[:metadata][:conflicts].nil?
+      pkg1[:metadata][:conflicts].each do | conflict |
+        pkgs_to_install.each do |pkg2|
+          if Tpkg::package_meets_requirement?(pkg2, conflict)
+            conflicting_pkgs << pkg1
+          end
+        end
+      end
+    end
+
+    # check if pkgs we're about to install conflict with existing pkgs
+    pkgs_to_install.each do |pkg1|
+      next if pkg1[:metadata][:conflicts].nil?
+      pkg1[:metadata][:conflicts].each do | conflict |
+        conflicting_pkgs |= installed_packages_that_meet_requirement(conflict)
+      end
+    end
+
+    # Check if there are conflicts among the pkgs we're about to install
+    # For these type of conflicts, we can't proceed, so raise exception.
+    pkgs_to_install.each do |pkg1|
       # native package might not have conflicts defined so skip
       next if pkg1[:metadata][:conflicts].nil?
       pkg1[:metadata][:conflicts].each do | conflict |
-        pkgs_to_check.each do |pkg2|
+        pkgs_to_install.each do |pkg2|
           if Tpkg::package_meets_requirement?(pkg2, conflict)
-            raise "Package conflicts between #{pkg2.inspect} and #{pkg1.inspect}"
+            raise "Package conflicts between #{pkg2[:metadata][:filename]} and #{pkg1[:metadata][:filename]}"
           end
         end
       end
     end
+
+    # Report to the users if there are conflicts
+    unless conflicting_pkgs.empty?
+      puts "The package(s) you're trying to install conflict with the following package(s):"
+      conflicting_pkgs = conflicting_pkgs.collect{|pkg|pkg[:metadata][:filename]}
+      puts conflicting_pkgs.join("\n")
+      if options[:force_replace] 
+        puts "Attemping to replace the conflicting packages."
+        success = remove(conflicting_pkgs)
+        return success
+      else
+        puts "Try removing the conflicting package(s) first, or rerun tpkg with the --force-replace option." 
+        return false
+      end
+    end
+    return true
   end
-  
+
   def prompt_for_conflicting_files(package_file, mode=CHECK_INSTALL)
     if !@@prompt
       return true
@@ -3404,29 +3339,18 @@ class Tpkg
   end
 
   # See parse_requests for format of requests
-  def install(requests, passphrase=nil)
+  def install(requests, passphrase=nil, options={})
     ret_val = 0
     requirements = []
     packages = {}
     lock
-
     parse_requests(requests, requirements, packages)
     check_requests(packages)
+
     core_packages = []
-    #currently_installed_requirements = []
     requirements.each do |req|
       core_packages << req[:name] if !core_packages.include?(req[:name])
-
-    # This was here to ensure that nothing went backwards.  But I guess in the 
-    # install case (as opposed to upgrade) going backwards can't really happen, 
-    # we may just install an older version alongside a newer version, which is
-    # perfectly fine.
-    #  currently_installed_requirements.concat(
-    #    requirements_for_currently_installed_package(req[:name]))
     end
-    #requirements.concat(currently_installed_requirements).uniq!
-
-
     
     puts "install calling best_solution" if @@debug
     puts "install requirements: #{requirements.inspect}" if @@debug
@@ -3438,7 +3362,8 @@ class Tpkg
       raise "Unable to resolve dependencies"
     end
     
-    check_for_conflicting_pkgs(solution_packages | installed_packages)
+    success = handle_conflicting_pkgs(installed_packages, solution_packages, options)
+    return false if !success
 
     if !prompt_for_install(solution_packages, 'installed')
       unlock
@@ -3564,8 +3489,7 @@ class Tpkg
 
     # send udpate back to reporting server
     unless @report_server.nil?
-      options = {:newly_installed => newly_installed,
-                 :currently_installed => currently_installed}
+      options = {:newly_installed => newly_installed, :currently_installed => currently_installed}
       send_update_to_server(options)
     end
     unlock
@@ -3573,7 +3497,8 @@ class Tpkg
   end
 
   # This method can also be used for doing downgrade
-  def upgrade(requests=nil, passphrase=nil, downgrade=false)
+  def upgrade(requests=nil, passphrase=nil, options={})
+    downgrade = options[:downgrade] || false
     ret_val = 0
     requirements = []
     packages = {}
@@ -3655,7 +3580,8 @@ class Tpkg
       raise "Unable to find solution for upgrading. Please verify that you specified the correct package(s) for upgrade."
     end
 
-    check_for_conflicting_pkgs(solution_packages | installed_packages)
+    success = handle_conflicting_pkgs(installed_packages, solution_packages, options)
+    return false if !success
 
     if downgrade
       prompt_action = 'downgraded'
@@ -3675,7 +3601,6 @@ class Tpkg
     installed_files = files_for_installed_packages
     removed_pkgs = [] # keep track of what we removed so far
     while pkg = solution_packages.shift
-#    solution_packages.each do |pkg|
       if pkg[:source] == :currently_installed ||
          pkg[:source] == :native_installed
         # Nothing to do for packages currently installed
@@ -3798,7 +3723,9 @@ class Tpkg
             end
           end if pkg[:metadata][:dependencies]
           if can_unpack
-            ret_val |= unpack(pkgfile, :passphrase => passphrase, :externals_to_skip => externals_to_skip)
+            is_doing_upgrade = true if removed_pkgs.include?(pkg[:metadata][:name])
+            ret_val |= unpack(pkgfile, :passphrase => passphrase, :externals_to_skip => externals_to_skip,
+                                       :is_doing_upgrade => is_doing_upgrade)
           end
 
           has_updates = true
@@ -3816,8 +3743,7 @@ class Tpkg
     if !has_updates
       puts "No updates available"
     elsif !@report_server.nil? 
-      options = {:newly_installed => newly_installed,
-                 :removed => removed,
+      options = {:newly_installed => newly_installed, :removed => removed,
                  :currently_installed => currently_installed}
       send_update_to_server(options)
     end
@@ -4050,8 +3976,7 @@ class Tpkg
 
     # send update back to reporting server
     unless @report_server.nil? || options[:upgrade]
-      options = {:removed => removed,
-                 :currently_installed => currently_installed}
+      options = {:removed => removed, :currently_installed => currently_installed}
       send_update_to_server(options)
     end
 
@@ -4088,7 +4013,7 @@ class Tpkg
         file_metadata = FileMetadata.new(File.read(file_metadata_xml), 'xml')
       else 
         errors = []
-        errors << "Can't find file_metadata.xml or file_metadata.yml file. Most likely this is because the package was created before the verify feature was added"
+        errors << "Can't find file metadata. Most likely this is because the package was created before the verify feature was added"
         results[package_file] = errors
         return results
       end
@@ -4138,7 +4063,6 @@ class Tpkg
           errors << "File is missing"
         else 
           # get actual values 
-          #chksum_actual = Digest::SHA256.file(fp).hexdigest if File.file?(fp)
           chksum_actual = Digest::SHA256.hexdigest(File.read(fp)) if File.file?(fp)
           uid_actual = File.stat(fp).uid
           gid_actual = File.stat(fp).gid
@@ -4316,7 +4240,7 @@ class Tpkg
 
       # populate the depencency map
       depended_on.each do | req_pkg |
-        dependency_mapping[req_pkg[:metadata][:filename]] = [] if dependency_mapping[req_pkg[:metadata][:filename]].nil?
+        dependency_mapping[req_pkg[:metadata][:filename]] ||= [] 
         dependency_mapping[req_pkg[:metadata][:filename]] << pkg
       end
     end
@@ -4360,7 +4284,7 @@ class Tpkg
   # log changes of pkgs that were installed/removed
   def log_changes(options={})
     msg = ""
-    user = Etc.getlogin
+    user = Etc.getlogin || Etc.getpwuid(Process.uid).name
     newly_installed = removed = [] 
     newly_installed = options[:newly_installed] if options[:newly_installed]
     removed = options[:removed] if options[:removed]
@@ -4379,7 +4303,7 @@ class Tpkg
 
   def send_update_to_server(options={})
     request = {"client"=>Facter['fqdn'].value}
-    request[:user] = Etc.getlogin
+    request[:user] = Etc.getlogin || Etc.getpwuid(Process.uid).name
     request[:tpkg_home] = ENV['TPKG_HOME']
     
     if options[:currently_installed]