bagit 0.0.2

data/Rakefile

@@ -0,0 +1,14 @@
+require 'rake'
+require 'rake/rdoctask'
+require 'spec/rake/spectask'
+
+Spec::Rake::SpecTask.new('spec') do |t|
+  t.libs << 'lib'
+  t.libs << 'spec'
+  t.spec_opts << "--color"
+#   t.warning = true
+#   t.rcov = true
+#   t.rcov_opts += ["-x /Library", "-x spec"]
+end
+
+task :default => [:spec]

data/bagit.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |spec|
+  spec.name = "bagit"
+  spec.version = '0.0.2'
+  spec.summary = "BagIt package generation and validation"
+  spec.email = "flazzarino@gmail.com"
+  spec.homepage = 'http://bagit.rubyforge.org'
+  spec.authors = ["Francesco Lazzarino"]
+  
+  spec.files = ["Rakefile", "bagit.gemspec", "lib/bagit/bag.rb",
+                "lib/bagit/fetch.rb", "lib/bagit/file.rb",
+                "lib/bagit/info.rb", "lib/bagit/manifest.rb",
+                "lib/bagit/string.rb", "lib/bagit/valid.rb",
+                "lib/bagit.rb" ]
+  
+  spec.has_rdoc = true
+end

data/lib/bagit.rb

@@ -0,0 +1,11 @@
+# == About bagit.rb
+# Author::    Francesco Lazzarino  (mailto:flazzarino@gmail.com)
+# Functionality conforms to the BagIt Spec v0.95:
+# http://www.cdlib.org/inside/diglib/bagit/bagitspec.html
+
+require 'bagit/bag'
+
+module BagIt
+  # The version of the BagIt specification the code is conforming to.
+  SPEC_VERSION = '0.95'             
+end

data/lib/bagit/bag.rb

@@ -0,0 +1,91 @@
+require 'bagit/fetch'
+require 'bagit/file'
+require 'bagit/info'
+require 'bagit/manifest'
+require 'bagit/string'
+require 'bagit/valid'
+
+module BagIt
+
+  # Represents the state of a bag on a filesystem
+  class Bag
+
+    attr_reader :bag_dir
+
+    include Validity            # Validity functionality
+    include Info                # package & bag info functionality
+    include Manifest            # manifest related functionality
+    include Fetch               # fetch related functionality
+
+    # Make a new Bag based at path
+    def initialize(path)
+      @bag_dir = path
+
+      # make the dir structure if it doesn't exist
+      FileUtils::mkdir bag_dir unless File.directory? bag_dir
+      FileUtils::mkdir data_dir unless File.directory? data_dir
+      
+      # write some tag info if its not there
+      unless File.exist? bagit_txt_file
+        write_bag_info("BagIt-Version" => SPEC_VERSION, "Tag-File-Character-Encoding" => "UTF-8")
+      end
+
+      unless File.exist? package_info_txt_file
+        write_package_info('Bag-Software-Agent' => "BagIt Ruby Gem (http://bagit.rubyforge.org)")
+      end
+      
+    end
+
+    # Return the path to the data directory
+    def data_dir
+      File.join @bag_dir, 'data'
+    end
+
+    # Return the paths to each bag file relative to bag_dir
+    def bag_files
+      Dir[File.join(data_dir, '**', '*')].select { |f| File.file? f }
+    end
+
+    # Return the paths to each tag file relative to bag_dir
+    def tag_files
+      Dir[File.join(@bag_dir, '*')].select { |f| File.file? f }
+    end
+
+    # Add a bag file
+    def add_file(base_path, src_path=nil)
+      path = File.join(data_dir, base_path)
+      raise "Bag file exists: #{base_path}" if File.exist? path
+      FileUtils::mkdir_p File.dirname(path)
+
+      if src_path.nil?
+        open(path, 'w') { |io| yield io }
+      else
+        FileUtils::cp src_path, path
+      end
+      
+    end
+
+    # Remove a bag file
+    def remove_file(base_path)
+      path = File.join(data_dir, base_path)
+      raise "Bag file does not exist: #{base_path}" unless File.exist? path
+      FileUtils::rm path
+    end
+
+    # Remove all empty directory trees from the bag
+    def gc!
+      
+      Dir.entries(data_dir).each do |f|
+        
+        unless %w{.. .}.include? f
+          abs_path = File.join data_dir, f
+          File.clean abs_path
+        end
+        
+      end
+      
+    end
+
+  end
+
+end

data/lib/bagit/fetch.rb

@@ -0,0 +1,50 @@
+require 'open-uri'
+
+module BagIt
+  
+  module Fetch
+    
+    def fetch_txt_file
+      File.join @bag_dir, 'fetch.txt'
+    end
+
+    def add_remote_file(url, path, size, sha1, md5)
+      open(fetch_txt_file, 'a') { |io| io.puts "#{url} #{size || '-'} #{path}" }
+      open(manifest_file('sha1'), 'a') { |io| io.puts "#{sha1} #{File.join 'data', path}" }
+      open(manifest_file('md5'), 'a') { |io| io.puts "#{md5} #{File.join 'data', path}" }
+    end
+
+    # feth all remote files
+    def fetch!
+
+      open(fetch_txt_file) do |io|
+        
+        io.readlines.each do |line|
+          
+          (url, length, path) = line.chomp.split(/\s+/, 3)
+          
+          add_file(path) do |io|
+            io.write open(url)
+          end
+          
+        end
+        
+      end
+
+      # rename the old fetch.txt
+      Dir["#{fetch_txt_file}.?*"].sort.reverse.each do |f|
+        
+        if f =~ /fetch.txt.(\d+)$/
+          new_f = File.join File.dirname(f), "fetch.txt.#{$1.to_i + 1}"
+          FileUtils::mv f, new_f
+        end
+        
+      end
+
+      # move the current fetch_txt
+      FileUtils::mv fetch_txt_file, "#{fetch_txt_file}.0"
+    end
+    
+  end
+  
+end

data/lib/bagit/file.rb

@@ -0,0 +1,19 @@
+class File
+
+  # Clean out all the empty dirs
+  def File.clean(file_name)
+
+    if File.directory? file_name
+      # clean all subdirs
+      subdirs = Dir.entries(file_name).select { |p| File.directory?(File.join(file_name, p)) }
+      subdirs.reject! { |p| %w{. ..}.include? p }
+      subdirs.each { |sd| File.clean File.join(file_name, sd) }
+
+      # if its empty then delete it
+      contents = Dir.entries(file_name).reject { |p| %w{. ..}.include? p }
+      Dir.delete file_name if contents.empty?
+    end
+    
+  end
+
+end

data/lib/bagit/info.rb

@@ -0,0 +1,77 @@
+require 'set'
+
+module BagIt
+
+  module Info
+
+    def package_info_txt_file
+      File.join bag_dir, 'package-info.txt'
+    end
+
+    def package_info
+      read_info_file package_info_txt_file
+    end
+
+    def write_package_info(hash)
+      write_info_file package_info_txt_file, hash
+    end
+
+    def bagit_txt_file
+      File.join bag_dir, 'bagit.txt'
+    end
+
+    def bag_info
+      read_info_file bagit_txt_file
+    end
+    
+    def write_bag_info(hash)
+      write_info_file bagit_txt_file, hash
+    end
+
+    protected
+
+    def read_info_file(file)
+      
+      open(file) do |io|
+        
+        entries = io.read.split /\n(?=[^\s])/
+        
+        entries.inject({}) do |hash, line|
+          name, value = line.chomp.split /\s*:\s*/
+          hash.merge({name => value})
+        end
+        
+      end
+      
+    end
+
+    def write_info_file(file, hash)
+
+      dups = hash.keys.inject(Set.new) do |acc, key|
+        a = hash.keys.grep(/#{key}/i)
+        acc + (a.size > 1 ? a : [])
+      end
+      
+      raise "Multiple labels (#{dups.to_a.join ', '}) in #{file}" unless dups.empty?
+      
+      open(file, 'w') do |io|
+        
+        hash.each do |name, value|
+          simple_entry = "#{name}: #{value.gsub /\s+/, ' '}"
+          
+          entry = if simple_entry.length > 79
+                    simple_entry.wrap(77).indent(2)
+                  else
+                    simple_entry
+                  end
+          
+          io.puts entry
+        end
+        
+      end
+      
+    end
+
+  end
+
+end

data/lib/bagit/manifest.rb

@@ -0,0 +1,111 @@
+require 'pathname'
+require 'digest/sha1'
+require 'digest/md5'
+
+module BagIt
+  
+  # Requires response to bag_dir, tag_files, bag_files
+  module Manifest
+    
+    # All tag files that are bag manifest files (manifest-[algorithm].txt)
+    def manifest_files
+      tag_files.select { |f| File.basename(f) =~ /^manifest-.*.txt/ }
+    end
+
+    # A path to a manifest file of the specified algorithm
+    def manifest_file(algo)
+      File.join bag_dir, "manifest-#{algo}.txt"
+    end
+
+    # Generate manifest files for all the bag files
+    def manifest!
+
+      # nuke all the existing manifest files
+      manifest_files.each { |f| FileUtils::rm f }
+
+      # manifest each tag file for each algorithm
+      bag_files.each do |f|
+        rel_path = Pathname.new(f).relative_path_from(Pathname.new(bag_dir)).to_s
+        data = open(f) { |io| io.read }
+
+        # sha1
+        sha1 = Digest::SHA1.hexdigest data
+        open(manifest_file(:sha1), 'a') { |io| io.puts "#{sha1} #{rel_path}" }
+
+        # md5
+        md5 = Digest::MD5.hexdigest data
+        open(manifest_file(:md5), 'a') { |io| io.puts "#{md5} #{rel_path}" }
+      end
+
+    end
+
+    # All tag files that are bag manifest files (tagmanifest-[algorithm].txt)
+    def tagmanifest_files
+      tag_files.select { |f| File.basename(f) =~ /^tagmanifest-.*.txt/ }
+    end
+
+    # A path to a tagmanifest file of the specified algorithm
+    def tagmanifest_file(algo)
+      File.join bag_dir, "tagmanifest-#{algo}.txt"
+    end
+
+    # Generate manifest files for all the tag files (except the tag
+    # manifest files)
+    def tagmanifest!
+
+      # nuke all the existing tagmanifest files
+      tagmanifest_files.each { |f| FileUtils::rm f }
+
+      # manifest each (non tagmanifest) tag file for each algorithm
+      (tag_files - tagmanifest_files).each do |f|
+        data = open(f) { |io| io.read }
+        rel_path = File.basename f
+
+        # sha1
+        sha1 = Digest::SHA1.hexdigest data
+        open(tagmanifest_file(:sha1), 'a') { |io| io.puts "#{sha1} #{rel_path}" }
+
+        # md5
+        md5 = Digest::MD5.hexdigest data
+        open(tagmanifest_file(:md5), 'a') { |io| io.puts "#{md5} #{rel_path}" }
+      end
+
+    end
+
+    # Returns true if all present manifested files' message digests
+    # match the actual message digest
+    def fixed?
+      (manifest_files + tagmanifest_files).all? do |mf|
+        # extract the algorithm
+        mf =~ /manifest-(.+).txt$/
+
+        algo = case $1
+               when /sha1/i
+                 Digest::SHA1
+               when /md5/i
+                 Digest::MD5
+               else
+                 :unknown
+               end
+
+        # check it, an unknown algorithm is always true
+        unless algo == :unknown
+          lines = open(mf) { |io| io.readlines }
+
+          lines.all? do |line|
+            manifested_digest, path = line.chomp.split /\s+/, 2
+            actual_digest = open(File.join(@bag_dir, path)) { |io| algo.hexdigest io.read }
+            actual_digest == manifested_digest
+          end
+
+        else
+          true
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/bagit/string.rb

@@ -0,0 +1,23 @@
+# Some mixed in functionality for String
+class String
+  
+  # Wrap a string to lines of a specified width. All existing newlines
+  # are not guaranteed to be preserved
+  def wrap(width)
+    s = gsub(/\s+/, ' ').strip
+
+    if s.length > width
+      s[0...width] + '\n' + s[width..-1].wrap(width)
+    else
+      s
+    end
+
+  end
+  
+  # Indent each line of a string by n spaces
+  def indent(n)
+    indent = ' ' * n
+    gsub '\n', "\n#{indent}"
+  end
+
+end

data/lib/bagit/valid.rb

@@ -0,0 +1,104 @@
+require 'validatable'
+
+module BagIt
+
+  class Bag
+    include Validatable
+    validates_true_for :consistency, :logic => lambda { complete? }
+    validates_true_for :completeness, :logic => lambda { consistent? }
+  end
+
+  module Validity
+
+    # Return true if the manifest cover all files and all files are
+    # covered.
+    def complete?
+
+      unmanifested_files.each do |file|
+        errors.add :completeness, "#{file} is present but not manifested"
+      end
+
+      empty_manifests.each do |file|
+        errors.add :completeness, "#{file} is manifested but not present"
+      end
+
+      errors.on(:completeness).nil?
+    end
+
+    # Return true if all manifested files message digests match.
+    def consistent?
+
+      manifest_files.each do |mf|
+
+        # get the algorithm implementation
+        File.basename(mf) =~ /^manifest-(.+).txt$/
+
+        algo = case $1
+               when /sha1/i
+                 Digest::SHA1
+               when /md5/i
+                 Digest::MD5
+               else
+                 :unknown
+               end
+
+        # Check every file in the manifest
+        open(mf) do |io|
+
+          io.each_line do |line|
+            expected, path = line.chomp.split /\s+/, 2
+            file = File.join(bag_dir, path)
+
+            if File.exist? file
+              actual = open(file) { |fio| algo.hexdigest(fio.read) }
+
+              if expected != actual
+                errors.add :consistency, "expected #{file} to have #{algo}: #{expected}, actual is #{actual}"
+              end
+
+            end
+          end
+
+        end
+
+      end
+
+      errors.on(:consistency).nil?
+    end
+
+    protected
+
+    # Returns all files in the instance that are not manifested
+    def unmanifested_files
+      mfs = manifested_files.map { |f| File.join bag_dir, f }
+      bag_files.reject { |f| mfs.member? f }
+    end
+
+    # Returns a list of manifested files that are not present
+    def empty_manifests
+      bfs = bag_files
+      manifested_files.reject { |f| bfs.member? File.join(bag_dir, f) }
+    end
+
+    # Returns a list of all files present in the manifest files
+    def manifested_files
+
+      manifest_files.inject([]) do |acc, mf|
+
+        files = open(mf) do |io|
+
+          io.readlines.map do |line|
+            digest, path = line.chomp.split /\s+/, 2
+            path
+          end
+
+        end
+
+        (acc + files).uniq
+      end
+
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,62 @@
+--- !ruby/object:Gem::Specification 
+name: bagit
+version: !ruby/object:Gem::Version 
+  version: 0.0.2
+platform: ruby
+authors: 
+- Francesco Lazzarino
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-03-02 00:00:00 -05:00
+default_executable: 
+dependencies: []
+
+description: 
+email: flazzarino@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- bagit.gemspec
+- lib/bagit/bag.rb
+- lib/bagit/fetch.rb
+- lib/bagit/file.rb
+- lib/bagit/info.rb
+- lib/bagit/manifest.rb
+- lib/bagit/string.rb
+- lib/bagit/valid.rb
+- lib/bagit.rb
+has_rdoc: true
+homepage: http://bagit.rubyforge.org
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: BagIt package generation and validation
+test_files: []
+