imw 0.2.18 → 0.3.0

data/lib/imw/tools/archiver.rb

@@ -1,220 +0,0 @@
-require 'imw/resource'
-
-module IMW
-  module Tools
-    
-    # Packages an Array of input files into a single output archive.
-    # When the archive is extracted, all the input files given will be
-    # in a single directory with a chosen name.  The path to the output
-    # archive determines both the name of the archive and its type (tar,
-    # tar.bz2, zip, &c.).
-    # 
-    # If any of the input files are themselves archives, they will first
-    # be extracted, with only their contents winding up in the final
-    # directory (the file hierarchy of the archive will be preserved).
-    # If any of the input files are compressed, they will first be
-    # uncompressed before being added to the directory.
-    #
-    # Both local and remote files can be archived. An exmaple:
-    # 
-    #   archiver = IMW::Transforms::Archiver.new 'my_archive', '/path/to/my/regular_file.tsv', '/path/to/an/archive.tar.bz2', '/path/to/my_compressed_file.gz', 'http://mywebsite.com/index.html'
-    #   archiver.package! '/path/to/my_archive.zip'
-    #
-    # This will create a ZIP archive at
-    # <tt>/path/to/my_archive.zip</tt>.  When the ZIP archive is
-    # extracted its contents will look like
-    #
-    #   my_archive
-    #   |-- regular_file.tsv
-    #   |-- archive_file1
-    #   |-- archive_dir
-    #   |   |-- archive_file2
-    #   |   `-- archive_file3
-    #   |-- archive_file3
-    #   |-- my_compressed_file
-    #   `-- index.html
-    #
-    # Notice that
-    #
-    # - the name of the extracted directory is given by the first
-    #   argument to the Archiver when it was instantiated.
-    #
-    # - all files wind up in the top-level of this extracted directory
-    #   when possible (<tt>regular_file.tsv</tt>, <tt>index.html</tt>)
-    #
-    # - /path/to/archive.tar.bz2 was not directly included, but its
-    #   contents (<tt>archive_file1</tt>,
-    #   <tt>archive_dir/archive_file2</tt>,
-    #   <tt>archive_dir/archive_file3</tt>) were included instead.
-    #
-    # - /path/to/my_compressed_file.gz was first uncompressed before
-    #   being added to the archive.
-    #
-    # - the remote file <tt>http://mywebsite.com/index.html</tt> was
-    #   downloaded and included
-    #
-    # This process can take a while when the constituent files are
-    # large because there is quite a lot of preparation done to the
-    # files to make this nice output structure in the final archive.
-    # Further calls to <tt>package!</tt> on the same instance of
-    # Archiver will skip the preparation step (the intermediate
-    # results of which are sitting in IMW's temporary directory) and
-    # directly create the package, saving time when attempting to
-    # create multiple package formats from the same input data.
-    class Archiver
-
-      attr_accessor :name, :local_inputs, :remote_inputs
-
-      def initialize name, *raw_inputs
-        @name   = name
-        self.inputs = raw_inputs
-      end
-
-      # Set the inputs for this archiver.
-      #
-      # @param [String, IMW::Resource] new_inputs the inputs to archive, local or remote
-      def inputs= raw_inputs
-        @local_inputs, @remote_inputs = [], []
-        raw_inputs.flatten.each do |raw_input|
-          input = IMW.open(raw_input)
-          if input.is_local?
-            @local_inputs << input
-          else
-            @remote_inputs << input
-          end
-        end
-        @local_inputs.flatten!
-      end
-
-      # Return a list of error messages for this archiver.
-      #
-      # @return [Array] the error messages
-      def errors
-        @errors ||= []      
-      end
-
-      # Was this archiver successful (did it not have any errors)?
-      #
-      # @return [true, false]
-      def success?
-        errors.empty?
-      end
-
-      # A temporary directory to work in.  Its contents will
-      # ultimately consist of a directory named for the package
-      # containing all the input files.
-      #
-      # @return [String]
-      def tmp_dir
-        @tmp_dir ||= File.join(IMW.path_to(:tmp_root, 'packager'), (Time.now.to_i.to_s + "-" + $$.to_s)) # guaranteed unique on a node
-      end
-
-      # A directory which will contain all the content being packaged,
-      # including the contents of any archives that were included in
-      # the list of files to process.
-      #
-      # @return [String]
-      def dir
-        @dir ||= File.join(tmp_dir, name.to_s)
-      end
-
-      # Remove the +tmp_dir+ entirely, getting rid of all temporary
-      # files.
-      def clean!
-        IMW.announce_if_verbose("Cleaning temporary directory #{tmp_dir}...")
-        FileUtils.rm_rf(tmp_dir)
-      end
-
-      # Copy, decompress, or extract the input paths to the temporary
-      # directory, readying them for packaging.
-      def prepare!
-        FileUtils.mkdir_p dir unless File.exist?(dir)
-        
-        local_inputs.each do |existing_file|
-          new_path      = File.join(dir, existing_file.basename)
-          case
-          when existing_file.is_archive?
-            IMW.announce_if_verbose("Extracting #{existing_file}...")
-            FileUtils.cd(dir) do
-              existing_file.extract
-            end
-          when existing_file.is_compressed?
-            IMW.announce_if_verbose("Decompressing #{existing_file}...")
-            existing_file.cp(new_path).decompress!
-          else
-            IMW.announce_if_verbose("Copying #{existing_file}...")
-            existing_file.cp(new_path)
-          end
-        end
-        
-        remote_inputs.each do |remote_input|
-          IMW.announce_if_verbose("Downloading #{remote_input}...")
-          remote_input.cp(File.join(dir, remote_input.effective_basename))
-        end
-      end        
-      
-      # Checks to see if all expected files exist in the temporary
-      # directory for this packager.
-      #
-      # @return [true, false]
-      def prepared?
-        local_inputs.each do |existing_file|
-          case
-          when existing_file.is_archive?
-            existing_file.contents.each do |archived_file_path|
-              return false unless File.exist?(File.join(dir, archived_file_path))
-            end
-          when existing_file.is_compressed?
-            return false unless File.exist?(File.join(dir, existing_file.decompressed_basename))
-          else
-            return false unless File.exist?(File.join(dir, existing_file.basename))
-          end
-        end
-
-        remote_inputs.each do |remote_input|
-          return false unless File.exist?(File.join(dir, remote_input.effective_basename))
-        end
-        
-        true
-      end
-      
-      # Package the contents of the temporary directory to an archive
-      # at +output+ but return exceptions instead of raising them.
-      #
-      # @param [String, IMW::Resource] output the path to the output package
-      # @param [Hash] options
-      # @return [StandardError, IMW::Resource] either the completed package or the error which was raised
-      def package output, options={}
-        begin
-          package! output, options={}
-        rescue StandardError => e
-          return e
-        end
-      end
-
-      # Package the contents of the temporary directory to an archive
-      # at +output+.  The extension of +output+ determines the kind of
-      # archive.
-      #
-      # @param [String, IMW::Resource] output the path to the output package
-      # @param [Hash] options
-      # @return [IMW::Resource] the completed package
-      def package! output, options={}
-        prepare!                          unless prepared?
-        output = IMW.open(output)
-        FileUtils.mkdir_p(output.dirname) unless File.exist?(output.dirname)        
-        output.rm!                        if output.exist?
-        FileUtils.cd(tmp_dir) { IMW.open(output.basename).create(name).mv(output.path) }
-        add_processing_error "Archiver: couldn't create archive #{output.path}" unless output.exists?
-        output
-      end
-
-      protected
-      def add_processing_error error # :nodoc:
-        IMW.logger.warn error      
-        errors << error
-      end
-      
-    end
-  end
-end

data/lib/imw/tools/downloader.rb

@@ -1,63 +0,0 @@
-module IMW
-  module Tools
-
-    # A class to download a collection of resources to a shared
-    # directory.
-    class Downloader
-
-      def initialize dir, *inputs
-        self.dir    = dir
-        self.inputs = inputs unless inputs.blank?
-      end
-
-      def self.dir= new_dir
-        @dir = IMW.open(new_dir)
-        raise IMW::PathError.new("#{@dir} must be a local directory") unless @dir.is_local? && @dir.is_directory?
-        @dir
-      end
-      attr_reader :dir
-
-      def inputs= new_inputs
-        @inputs = new_inputs.flatten.compact.map { |raw_input| IMW.open(raw_input) }
-      end
-      attr_reader :inputs
-
-      def downloaded_path_for input
-        dir.join(input.respond_to?(:effective_basename) ? input.effective_basename : input.basename)
-      end
-
-      def download!
-        before_download
-        inputs.each do |input|
-          downloaded_path = downloaded_path_for(input)
-          IMW.log_if_verbose "Downloading #{input} to #{downloaded_path}"
-          input.cp(downloaded_path)
-        end
-        after_download
-      end
-
-      def downloaded?
-        downloaded_resources.all? { |resource| resource.exist? }
-      end
-
-      def downloaded_resources
-        inputs.map do |input|
-          IMW.open(downloaded_path_for(input))
-        end
-      end
-
-      def clean!
-        IMW.log_if_verbose("Deleting downloader directory #{dir}")
-        dir.rm_rf!
-      end
-
-      def before_download
-      end
-
-      def after_download
-      end
-
-    end
-  end
-end
-

data/lib/imw/tools/extension_analyzer.rb

@@ -1,114 +0,0 @@
-module IMW
-  module Tools
-
-    # Mixin with some heuristic methods for identifying common
-    # extensions and likely data formats for a collection of files.
-    #
-    # Requires the including class to define a method +resources+
-    # which returns an array of IMW::Resource objects as well as a
-    # method +total_size+ which gives the total size of the resources
-    # (for weighting extensions by size).
-    module ExtensionAnalyzer
-
-      # Return the file counts of each extension.
-      #
-      # @return [Hash]
-      def extension_counts
-        @extension_counts ||= {}.tap do |counts|
-          resources.each do |resource|
-            next if resource.is_directory?
-            counts[resource.extension] = 0 unless counts.has_key?(resource.extension)
-            counts[resource.extension] += 1
-          end
-        end
-      end
-
-      # Return the most common extension by count of files.
-      def most_common_extension_by_count
-        return @most_common_extension_by_count if @most_common_extension_by_count
-        current_count, current_extension = 0, ''
-        extension_counts.each_pair do |extension, count|
-          current_extension = extension if count > current_count
-        end
-        if current_extension.strip.blank? then current_extension = 'flat' end
-        @most_common_extension_by_count = current_extension
-      end
-
-      # Return the file counts of each extension, normalized by the
-      # total number of files.
-      #
-      # @return [Hash]
-      def normalized_extension_counts
-        @normalized_extension_counts ||= {}.tap do |weighted|
-          num_files = resources.reject(&:is_directory?).length.to_f
-          extension_counts.each_pair do |extension, count|
-            weighted[extension] = count.to_f / num_files
-          end
-        end
-      end
-
-      # Return the amount of data corresponding to each extension.
-      #
-      # @return [Hash]
-      def extension_sizes
-        @extension_sizes ||= {}.tap do |sizes|
-          resources.each do |resource|
-            next if resource.is_directory?
-            sizes[resource.extension] = 0 unless sizes.has_key?(resource.extension)            
-            sizes[resource.extension] += resource.size
-          end
-        end
-      end
-
-      # Return the most common extension by amount of data.
-      #
-      # @return [String]
-      def most_common_extension_by_size
-        return @most_common_extension_by_size if @most_common_extension_by_size
-        current_size, current_extension = 0, ''
-        extension_sizes.each_pair do |extension, size|
-          if size > current_size
-            current_extension = extension
-            current_size      = size
-          end
-        end
-        current_extension = 'flat' if current_extension.strip.blank?
-        @most_common_extension_by_size = current_extension
-      end
-
-      # Return the fractional share of each extension by file size.
-      #
-      # @return [Hash]
-      def normalized_extension_sizes
-        @normalized_extension_sizes ||= {}.tap do |weighted|
-          extension_sizes.each_pair do |extension, size|
-            weighted[extension] = size.to_f / total_size.to_f
-          end
-        end
-      end
-
-      # Return a guess as to the most common extension format for this
-      # Summarizer's resources.
-      #
-      # @return [String]
-      def most_common_extension
-        return most_common_extension_by_size if most_common_extension_by_size == most_common_extension_by_count # no contest
-        count_fraction = (normalized_extension_counts[most_common_extension_by_count] or 0.0)
-        size_fraction  = (normalized_extension_sizes[most_common_extension_by_size]   or 0.0)
-        return most_common_extension_by_count if count_fraction >= 0.5 and size_fraction < 0.5 # FIXME arbitrary
-        return most_common_extension_by_size  if count_fraction < 0.5  and size_fraction >= 0.5
-        most_common_extension_by_size # default to size
-      end
-
-      # Returns a guess as to the most common data format for this
-      # Summarizer's resources.
-      #
-      # @return [String]
-      def most_common_data_format
-        extension = most_common_extension
-        ['tar', 'tar.bz2', 'tar.gz', 'tgz', 'tbz2', 'zip', 'rar'].include?(extension) ? 'archive' : extension
-      end
-    end
-  end
-end
-

data/lib/imw/tools/summarizer.rb

@@ -1,83 +0,0 @@
-require 'imw/tools/extension_analyzer'
-
-module IMW
-  module Tools
-
-    # A class for producing summary data about a collection of
-    # resources.
-    #
-    # The Summarizer needs recursively IMW.open all files and
-    # directories given so will be very cumbersome if given many
-    # files.  Few large files will not cause a problem.
-    class Summarizer
-
-      # Options for this Summarizer.
-      attr_accessor :options
-
-      # The inputs given to this Summarizer.
-      attr_reader :inputs
-
-      # The resources analyzed, calculated recursively from the
-      # +inputs+.
-      attr_reader :resources
-
-      include IMW::Tools::ExtensionAnalyzer      
-
-      # Initialize a new Summarizer with the given +inputs+.
-      #
-      # A Hash of options can be given as the last parameter.
-      #
-      # @param [Array<String, IMW::Resource>] inputs
-      # @return [IMW::Tools::Summarizer]
-      def initialize *inputs
-        self.options = (inputs.last.is_a?(Hash) && inputs.pop) || {}
-        self.inputs  = inputs.flatten
-      end
-
-      # Return the total size of all resources.
-      #
-      # @return [Integer]
-      def total_size
-        @total_size ||= resources.map(&:size).inject(0) { |e, sum| sum += e }
-      end
-
-      # Return a summary of the +inputs+ to this Summarizer.
-      #
-      # Will swallow errors.
-      #
-      # @return [Array<Hash>]
-      def summary
-        @summary ||= summary! rescue []
-      end
-
-      # Return a summary of the +inputs+ to this summarizer.
-      # 
-      # Delegates to the +summary+ method of each constituent
-      # IMW::Resource in +inputs+.
-      #
-      # @return [Array]
-      def summary!
-        inputs.map do |input|
-          (input.respond_to?(:summary) ? input.summary : {})
-        end
-      end
-      
-      protected
-      # Set new inputs for this summarizer.
-      #
-      # Summarizer statistics are cached as instance variables so be
-      # careful about changing inputs and then using old statistics...
-      #
-      # @param [Array<String, IMW::Resource>] new_inputs
-      def inputs= new_inputs
-        @inputs = new_inputs.map do |path_or_resource|
-          input = IMW.open(path_or_resource)
-        end
-        @resources = inputs.map do |input|
-          input.is_local? && input.is_directory? ? input.all_resources : input
-        end.compact.flatten
-      end
-
-    end
-  end
-end