bagit 0.2.0 → 0.3.0.pre

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NjFkNDZjYTUzYmMyOGZlMWZiOGM5YTJiMGU3ODM1ODRkNzYzZTY1OA==
+  data.tar.gz: !binary |-
+    NzU1ODcwNmZhZTI4NmE1YWJmMjNhN2FiNmYwNTFlMzhmMDFjZjY5OA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    ZWJjMjYyYjk5NTMzNDgzMmYyNDE3NjM2YWRjNWUxMzk2NzM4NzAwMWVkY2Qy
+    ODRiODllOGRkZmE5YTYxN2Q5NGZjNjdjZTMwZDdjNGJlMTY5NTFkOWVhMGMy
+    MGJhYjFlNzAwNWQ2NmI5ZDk5ZDUxN2VkNDQxNjY0YTNmYjI1YTc=
+  data.tar.gz: !binary |-
+    YWFhNTcwZWU3OGU4Y2NiNzQ0NjYzM2ZjZmZjMmVlZDRiNTk2YjBlOWE4ZTkx
+    NTM0Y2MxMmI1ZDhkMzYxOWY2OTNkOTQ1NGJmYWIwY2YwNzYwZjNiNmE3YWM4
+    YmYwODE4Yzc5MWVmMjhmZDM4YzY4OWZlZmQ3Y2RiODdhYjA4Mjc=

data/README.md

@@ -1,7 +1,9 @@
 BagIt (for ruby)
 ================
 
-Based on the [BagItspec v0.96](https://confluence.ucop.edu/display/Curation/BagIt).
+[![Build Status](https://secure.travis-ci.org/no-reply/bagit.png)](http://travis-ci.org/no-reply/bagit)
+
+Based on the [BagItspec v0.97](https://confluence.ucop.edu/display/Curation/BagIt).
 
 Supported Features:
 -------------------
@@ -48,12 +50,13 @@ TODO
 * command line tools for common tasks
 * better holy bag (fetch.txt) generation
 * better error reporting.
-* poor mans' checksum
 
 ---
 
 Copyright © 2009, [Francesco Lazzarino](mailto:flazzarino@gmail.com).
 
+Current Maintainer: [Tom Johnson](mailto:thomas.johnson@oregonstate.edu).
+
 Sponsored by [Florida Center for Library Automation](http://www.fcla.edu).
 
 See LICENSE.txt for terms.

data/bagit.gemspec

@@ -1,11 +1,12 @@
+
 BAGIT_SPEC = Gem::Specification.new do |spec|
   spec.name        = "bagit"
-  spec.version     = '0.2.0'
+  spec.version     = '0.3.0.pre'
   spec.summary     = "BagIt package generation and validation"
   spec.description = "Ruby Library and Command Line tools for bagit"
-  spec.email       = "flazzarino@gmail.com"
-  spec.homepage    = 'http://github.com/flazz/bagit'
-  spec.authors     = ["Francesco Lazzarino"]
+  spec.email       = "johnson.tom@gmail.com"
+  spec.homepage    = 'http://github.com/tipr/bagit'
+  spec.authors     = ["Tom Johnson, Francesco Lazzarino"]
 
   spec.add_dependency 'validatable', '~> 1.6'
 

data/lib/bagit.rb

@@ -4,8 +4,10 @@
 # http://www.cdlib.org/inside/diglib/bagit/bagitspec.html
 
 require 'bagit/bag'
+require 'fileutils'
+require 'date'
 
 module BagIt
   # The version of the BagIt specification the code is conforming to.
-  SPEC_VERSION = '0.96'
+  SPEC_VERSION = '0.97'
 end

data/lib/bagit/bag.rb

@@ -17,7 +17,7 @@ module BagIt
     include Fetch               # fetch related functionality
 
     # Make a new Bag based at path
-    def initialize(path)
+    def initialize(path, info={})
       @bag_dir = path
 
       # make the dir structure if it doesn't exist
@@ -30,7 +30,7 @@ module BagIt
       end
 
       unless File.exist? bag_info_txt_file
-        write_bag_info('Bag-Software-Agent' => "BagIt Ruby Gem (http://bagit.rubyforge.org)")
+        write_bag_info(info)
       end
     end
 
@@ -46,7 +46,13 @@ module BagIt
 
     # Return the paths to each tag file relative to bag_dir
     def tag_files
-      Dir[File.join(@bag_dir, '*')].select { |f| File.file? f }
+      files = []
+      if tagmanifest_files != []
+        File.open(tagmanifest_files.first) do |f|
+          f.each_line{|line| files << File.join(@bag_dir, line.split(' ')[1])}
+        end
+      end
+      files
     end
 
     # Add a bag file
@@ -56,10 +62,12 @@ module BagIt
       FileUtils::mkdir_p File.dirname(path)
 
       if src_path.nil?
-        open(path, 'w') { |io| yield io }
+        f = open(path, 'w') { |io| yield io }
       else
-        FileUtils::cp src_path, path
+        f = FileUtils::cp src_path, path
       end
+      write_bag_info
+      return f
     end
     
     # Remove a bag file
@@ -85,6 +93,16 @@ module BagIt
     def paths
       self.bag_files.collect { |f| f.sub(data_dir + '/', '') }
     end
+
+    # Get the Oxum for the payload files
+    def payload_oxum
+      bytes = 0
+      bag_files.each do |f|
+        #TODO: filesystem quirks? Are we getting the stream size or the size on disk?
+        bytes += File.size(f)
+      end
+      return bytes.to_s + '.' + bag_files.count.to_s
+    end
     
     # Remove all empty directory trees from the bag
     def gc!

data/lib/bagit/info.rb

@@ -4,15 +4,41 @@ module BagIt
 
   module Info
 
+    @@bag_info_headers = {  
+      :agent => 'Bag-Software-Agent',
+      :org => 'Source-Organization',
+      :org_addr => 'Organization-Address',
+      :contact_name => 'Contact-Name',
+      :contact_phone => 'Contact-Phone',
+      :contact_email => 'Contact-Email',
+      :ext_desc => 'External-Description',
+      :ext_id => 'External-Identifier',
+      :size => 'Bag-Size',
+      :group_id => 'Bag-Group-Identifier',
+      :group_count => 'Bag-Count',
+      :sender_id => 'Internal-Sender-Identifier',
+      :int_desc => 'Internal-Sender-Description',
+      :date => 'Bagging-Date',
+      :oxum => 'Payload-Oxum'
+    }
+
     def bag_info_txt_file
       File.join bag_dir, 'bag-info.txt'
     end
 
     def bag_info
-      read_info_file bag_info_txt_file
+      begin
+        read_info_file bag_info_txt_file
+      rescue
+        {}
+      end
     end
 
-    def write_bag_info(hash)
+    def write_bag_info(hash={})
+      hash = bag_info.merge(hash) 
+      hash[@@bag_info_headers[:agent]] = "BagIt Ruby Gem (http://bagit.rubyforge.org)" if hash[@@bag_info_headers[:agent]].nil?
+      hash[@@bag_info_headers[:date]] = Date.today.strftime('%Y-%m-%d') if hash[@@bag_info_headers[:date]].nil?
+      hash[@@bag_info_headers[:oxum]] = payload_oxum
       write_info_file bag_info_txt_file, hash
     end
 
@@ -28,6 +54,11 @@ module BagIt
       write_info_file bagit_txt_file, hash
     end
 
+    def update_bag_date
+      hash["Bagging-Date"] = Date.today.strftime('%Y-%m-%d')
+      write_bag_info(hash)
+    end
+
     protected
 
     def read_info_file(file)

data/lib/bagit/manifest.rb

@@ -9,7 +9,10 @@ module BagIt
 
     # All tag files that are bag manifest files (manifest-[algorithm].txt)
     def manifest_files
-      tag_files.select { |f| File.basename(f) =~ /^manifest-.*.txt/ }
+      files = Dir[File.join(@bag_dir, '*')].select { |f|
+        File.file? f and File.basename(f) =~ /^manifest-.*.txt/ 
+      }
+      files
     end
 
     # A path to a manifest file of the specified algorithm
@@ -26,14 +29,13 @@ module BagIt
       # manifest each tag file for each algorithm
       bag_files.each do |f|
         rel_path = Pathname.new(f).relative_path_from(Pathname.new(bag_dir)).to_s
-        data = open(f) { |io| io.read }
 
         # sha1
-        sha1 = Digest::SHA1.hexdigest data
+        sha1 = Digest::SHA1.file f
         open(manifest_file(:sha1), 'a') { |io| io.puts "#{sha1} #{rel_path}" }
 
         # md5
-        md5 = Digest::MD5.hexdigest data
+        md5 = Digest::MD5.file f
         open(manifest_file(:md5), 'a') { |io| io.puts "#{md5} #{rel_path}" }
       end
 
@@ -41,7 +43,10 @@ module BagIt
 
     # All tag files that are bag manifest files (tagmanifest-[algorithm].txt)
     def tagmanifest_files
-      tag_files.select { |f| File.basename(f) =~ /^tagmanifest-.*.txt/ }
+      files = Dir[File.join(@bag_dir, '*')].select { |f|
+        File.file? f and File.basename(f) =~ /^tagmanifest-.*.txt/ 
+      }
+      files
     end
 
     # A path to a tagmanifest file of the specified algorithm
@@ -51,25 +56,77 @@ module BagIt
 
     # Generate manifest files for all the tag files (except the tag
     # manifest files)
-    def tagmanifest!
+    def tagmanifest!(tags=nil)
+      
+      tags = tag_files if tags == nil
 
       # nuke all the existing tagmanifest files
       tagmanifest_files.each { |f| FileUtils::rm f }
+      
+      # ensure presence of manfiest files
+      manifest_files.each do |manifest|
+        tags << manifest unless tags.include?(manifest)
+      end
+
+      # ensure presence of bag info files
+      tags << bag_info_txt_file unless tags.include?(bag_info_txt_file)
+      tags << bagit_txt_file unless tags.include?(bagit_txt_file)
 
       # manifest each (non tagmanifest) tag file for each algorithm
-      (tag_files - tagmanifest_files).each do |f|
-        data = open(f) { |io| io.read }
-        rel_path = File.basename f
+      tags.each do |f|
+        add_tag_file(Pathname.new(f).relative_path_from(Pathname.new(bag_dir)).to_s)
+      end
+      tag_files
+    end
 
-        # sha1
-        sha1 = Digest::SHA1.hexdigest data
-        open(tagmanifest_file(:sha1), 'a') { |io| io.puts "#{sha1} #{rel_path}" }
+    def add_tag_file(path, src_path=nil)
 
-        # md5
-        md5 = Digest::MD5.hexdigest data
-        open(tagmanifest_file(:md5), 'a') { |io| io.puts "#{md5} #{rel_path}" }
+      f = File.join(@bag_dir, path) 
+      raise "Tag file already in manifest: #{path}" if tag_files.include?(f)
+      
+      if not File.exist? f
+        FileUtils::mkdir_p File.dirname(f)
+
+        # write file
+        if src_path.nil?
+          open(f, 'w') { |io| yield io }
+        else
+          FileUtils::cp src_path, f
+        end
+        # this adds the manifest and bag info files on initial creation
+        # it must only run when the manifest doesn't already exist or it will 
+        # infinitely recall add_tag_file. Better way of doing this?
+        tagmanifest! 
+      elsif not src_path.nil? 
+        raise "Tag file already exists, will not overwrite: #{path}\n Use add_tag_file(path) to add an existing tag file."
       end
 
+      data = open(f) { |io| io.read }
+      rel_path = Pathname.new(f).relative_path_from(Pathname.new(bag_dir)).to_s
+
+      # sha1
+      sha1 = Digest::SHA1.hexdigest data
+      open(tagmanifest_file(:sha1), 'a') { |io| io.puts "#{sha1} #{rel_path}" }
+
+      # md5
+      md5 = Digest::MD5.hexdigest data
+      open(tagmanifest_file(:md5), 'a') { |io| io.puts "#{md5} #{rel_path}" }
+      tag_files
+    end
+
+    def remove_tag_file(path)
+      tags = tag_files
+      raise "Tag file is not in manifest: #{path}" unless tags.include?(File.join(@bag_dir, path))
+      tags.delete(File.join(@bag_dir, path))
+      tagmanifest!(tags)
+    end
+
+    def delete_tag_file(path)
+      filepath = File.join(@bag_dir, path)
+      raise "Tag file does not exist: #{path}" unless File.exist? filepath
+      #TODO: delete tags even when they are not in the manifest
+      remove_tag_file(path)
+      FileUtils::rm filepath
     end
 
     # Returns true if all present manifested files' message digests

data/lib/bagit/valid.rb

@@ -21,18 +21,18 @@ module BagIt
       empty_manifests.each do |file|
         errors.add :completeness, "#{file} is manifested but not present"
       end
-
+      tag_empty_manifests.each do |file|
+        errors.add :completeness, "#{file} is a manifested tag but not present"
+      end
+      
       errors.on(:completeness).nil?
     end
 
     # Return true if all manifested files message digests match.
     def consistent?
-
-      manifest_files.each do |mf|
-
+      (manifest_files|tagmanifest_files).each do |mf|
         # get the algorithm implementation
-        File.basename(mf) =~ /^manifest-(.+).txt$/
-
+        File.basename(mf) =~ /manifest-(.+).txt$/
         algo = case $1
                when /sha1/i
                  Digest::SHA1
@@ -41,31 +41,30 @@ module BagIt
                else
                  :unknown
                end
-
         # Check every file in the manifest
         open(mf) do |io|
-
           io.each_line do |line|
             expected, path = line.chomp.split /\s+/, 2
             file = File.join(bag_dir, path)
-
             if File.exist? file
-              actual = open(file) { |fio| algo.hexdigest(fio.read) }
-
+              actual = algo.file(file).hexdigest
               if expected != actual
                 errors.add :consistency, "expected #{file} to have #{algo}: #{expected}, actual is #{actual}"
               end
-
             end
           end
-
         end
-
       end
+      
 
       errors.on(:consistency).nil?
     end
 
+    # Checks for validity against Payload-Oxum
+    def valid_oxum?
+      bag_info["Payload-Oxum"] == payload_oxum
+    end
+
     protected
 
     # Returns all files in the instance that are not manifested
@@ -79,7 +78,16 @@ module BagIt
       bfs = bag_files
       manifested_files.reject { |f| bfs.member? File.join(bag_dir, f) }
     end
-
+    # Returns a list of tag manifested files that are not present
+    def tag_empty_manifests
+      empty = []
+      tag_manifested_files.each do |f|
+        if !File.exists?(File.join(bag_dir,f))
+          empty.push f
+        end
+      end
+      return empty
+    end
     # Returns a list of all files present in the manifest files
     def manifested_files
 
@@ -98,6 +106,18 @@ module BagIt
       end
 
     end
+    # Returns a list of all files in the tag manifest files
+    def tag_manifested_files
+      tagmanifest_files.inject([]) do |acc, mf|
+        files = open(mf) do |io|
+          io.readlines.map do |line|
+            digest, path = line.chomp.split /\s+/, 2
+            path
+          end
+        end
+        (acc+files).uniq
+      end
+    end
 
   end
 

metadata

@@ -1,75 +1,69 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: bagit
-version: !ruby/object:Gem::Version 
-  prerelease: 
-  version: 0.2.0
+version: !ruby/object:Gem::Version
+  version: 0.3.0.pre
 platform: ruby
-authors: 
-- Francesco Lazzarino
+authors:
+- Tom Johnson, Francesco Lazzarino
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-06-08 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2013-04-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: validatable
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: "1.6"
+      - !ruby/object:Gem::Version
+        version: '1.6'
   type: :runtime
-  version_requirements: *id001
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
 description: Ruby Library and Command Line tools for bagit
-email: flazzarino@gmail.com
+email: johnson.tom@gmail.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - Rakefile
 - README.md
 - LICENSE.txt
 - bagit.gemspec
+- lib/bagit.rb
 - lib/bagit/bag.rb
-- lib/bagit/fetch.rb
-- lib/bagit/file.rb
-- lib/bagit/info.rb
-- lib/bagit/manifest.rb
 - lib/bagit/string.rb
+- lib/bagit/manifest.rb
+- lib/bagit/info.rb
+- lib/bagit/file.rb
 - lib/bagit/valid.rb
-- lib/bagit.rb
-homepage: http://github.com/flazz/bagit
+- lib/bagit/fetch.rb
+homepage: http://github.com/tipr/bagit
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.3
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: BagIt package generation and validation
 test_files: []
-
+has_rdoc: