ro-crate 0.4.5 → 0.4.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78897fbce0fd01688c27516dceba600ed7cd984668fd16abde32a1713c0ba40f
-  data.tar.gz: 3194d882e4b7f6cd347d652d5da38cb2fe02cfb18e36d84f50e012193a538a69
+  metadata.gz: cb6b4eba94b7190b7888cf3b1ddc8a8c6d0b464834ff48c45d9b50e6b13844ce
+  data.tar.gz: 70e0079caadfa3f17745e918f8109ec5908a8aa0c87d80aeaebfc55fdc03a9a8
 SHA512:
-  metadata.gz: ddc2f5ffe217717299f2e5d76313b2a124fabfbc7b1e1800db18ca36d2f79c716ef249c6fcb8a6a8bfd4e0c5bba3b8a30d6ae3498ef8fd0068d7df08985c72f5
-  data.tar.gz: 802bf935d2e911ce771e962d6faaf0821f682049424cf5ac893ec780b8fc89d22f844156a83dd38adbe045a31faa7e527b4d8406bcabe937bdf8a2e8afafcd93
+  metadata.gz: 62c9875be216987ceaca7e47f92112ba744109e4df9f8dcbba9023211ef3fc278f74f8ce7421523a0f443000f251627735eb01ce2357e9e526e36c23818bbba0
+  data.tar.gz: 0b5be2eedc7a045ccf25db4e3d800107f30d00b62e1e4b5bdb1deed1eed941897f46621bef9fa9a012891bc058857de82d6afebeb122153f8ffc944b8ca06a7a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ro-crate (0.4.5)
+    ro-crate (0.4.10)
       addressable (~> 2.7.0)
       rubyzip (~> 2.0.0)
 

data/lib/ro_crate/model/contextual_entity.rb

@@ -3,21 +3,9 @@ module ROCrate
   # A class to represent a "Contextual Entity" within an RO-Crate.
   # Contextual Entities are used to describe and provide context to the Data Entities within the crate.
   class ContextualEntity < Entity
-    def self.format_id(id)
+    def self.format_local_id(id)
       i = super
-      begin
-        uri = URI(id)
-      rescue ArgumentError
-        uri = nil
-      end
-
-      if uri&.absolute?
-        i
-      elsif i.start_with?('#')
-        i
-      else
-        "##{i}"
-      end
+      i.start_with?('#') ? i : "##{i}"
     end
 
     ##

data/lib/ro_crate/model/crate.rb

@@ -8,6 +8,11 @@ module ROCrate
     properties(%w[name datePublished author license identifier distribution contactPoint publisher description url hasPart])
 
     def self.format_id(id)
+      i = super(id)
+      i.end_with?('/') ? i : "#{i}/"
+    end
+
+    def self.format_local_id(id)
       return id if id == IDENTIFIER
       super
     end
@@ -217,15 +222,19 @@ module ROCrate
 
     alias_method :own_entries, :entries
     ##
-    # A map of all the files/directories contained in the RO-Crate, where the key is the destination path within the crate
-    # and the value is an Entry where the source data can be read.
+    # # The RO-Crate's "payload" of the crate - a map of all the files/directories contained in the RO-Crate, where the
+    # key is the destination path within the crate and the value is an Entry where the source data can be read.
     #
     # @return [Hash{String => Entry}>]
     def entries
-      entries = {}
-
-      (default_entities | data_entities).each do |entity|
-        (entity == self ? own_entries : entity.entries).each do |path, entry|
+      # Gather a map of entries, starting from the crate itself, then any directory data entities, then finally any
+      # file data entities. This ensures in the case of a conflict, the more "specific" data entities take priority.
+      entries = own_entries
+      non_self_entities = default_entities.reject { |e| e == self }
+      sorted_entities = (non_self_entities | data_entities).sort_by { |e| e.is_a?(ROCrate::Directory) ? 0 : 1 }
+
+      sorted_entities.each do |entity|
+        entity.entries.each do |path, entry|
           entries[path] = entry
         end
       end

data/lib/ro_crate/model/data_entity.rb

@@ -3,7 +3,7 @@ module ROCrate
   # A class to represent a "Data Entity" within an RO-Crate.
   # Data Entities are the actual physical files and directories within the Crate.
   class DataEntity < Entity
-    def self.format_id(id)
+    def self.format_local_id(id)
       super.chomp('/')
     end
 

data/lib/ro_crate/model/directory.rb

@@ -4,7 +4,7 @@ module ROCrate
   class Directory < DataEntity
     properties(%w[name contentSize dateModified encodingFormat identifier sameAs])
 
-    def self.format_id(id)
+    def self.format_local_id(id)
       super + '/'
     end
 
@@ -30,8 +30,8 @@ module ROCrate
     end
 
     ##
-    # A map of all the files/directories under this directory, where the key is the destination path within the crate
-    # and the value is an Entry where the source data can be read.
+    # The "payload" of this directory - a map of all the files/directories, where the key is the destination path
+    # within the crate and the value is an Entry where the source data can be read.
     #
     # @return [Hash{String => Entry}>]
     def entries

data/lib/ro_crate/model/entity.rb

@@ -29,12 +29,34 @@ module ROCrate
     end
 
     ##
-    # Format the given ID with rules appropriate for this type.
+    # Format the given ID with rules appropriate for this type if it is local/relative, leave as-is if absolute.
+    #
+    # @param id [String] The candidate ID to be formatted.
+    # @return [String] The formatted ID.
+    def self.format_id(id)
+      begin
+        uri = URI(id)
+      rescue ArgumentError, URI::InvalidURIError
+        uri = nil
+      end
+
+      if uri&.absolute?
+        id
+      else
+        format_local_id(id)
+      end
+    end
+
+    ##
+    # Format the given local ID with rules appropriate for this type.
     # For example:
     #  * contextual entities MUST be absolute URIs, or begin with: #
     #  * files MUST NOT begin with ./
     #  * directories MUST NOT begin with ./ (except for the crate itself), and MUST end with /
-    def self.format_id(id)
+    #
+    # @param id [String] The candidate local ID to be formatted.
+    # @return [String] The formatted local ID.
+    def self.format_local_id(id)
       Addressable::URI.escape(id.sub(/\A\.\//, '')) # Remove initial ./ if present
     end
 

data/lib/ro_crate/model/file.rb

@@ -52,7 +52,9 @@ module ROCrate
     end
 
     ##
-    # A map of all the files/directories associated with this File. Should only be a single key and value.
+    # The "payload". A map with a single key and value, of the relative filepath within the crate, to the source on disk
+    # where the actual bytes of the file can be read. Blank if remote.
+    #
     # (for compatibility with Directory#entries)
     #
     # @return [Hash{String => Entry}>] The key is the location within the crate, and the value is an Entry.

data/lib/ro_crate/reader.rb

@@ -3,29 +3,67 @@ module ROCrate
   # A class to handle reading of RO-Crates from Zip files or directories.
   class Reader
     ##
-    # Reads an RO-Crate from a directory of zip file.
+    # Reads an RO-Crate from a directory or zip file.
     #
-    # @param source [String, ::File, Pathname] The source location for the crate.
+    # @param source [String, ::File, Pathname, #read] The location of the zip or directory, or an IO-like object containing a zip.
     # @param target_dir [String, ::File, Pathname] The target directory where the crate should be unzipped (if its a Zip file).
     # @return [Crate] The RO-Crate.
     def self.read(source, target_dir: Dir.mktmpdir)
       raise "Not a directory!" unless ::File.directory?(target_dir)
-      if ::File.directory?(source)
+      begin
+        is_dir = ::File.directory?(source)
+      rescue TypeError
+        is_dir = false
+      end
+
+      if is_dir
         read_directory(source)
       else
         read_zip(source, target_dir: target_dir)
       end
     end
 
+    ##
+    # Extract the contents of the given Zip file/data to the given directory.
+    #
+    # @param source [String, ::File, Pathname, #read] The location of the zip file, or an IO-like object.
+    # @param target [String, ::File, Pathname] The target directory where the file should be unzipped.
+    def self.unzip_to(source, target)
+      source = Pathname.new(::File.expand_path(source)) if source.is_a?(String)
+
+      if source.is_a?(Pathname) || source.respond_to?(:path)
+        unzip_file_to(source, target)
+      else
+        unzip_io_to(source, target)
+      end
+    end
+
+    ##
+    # Extract the given Zip file data to the given directory.
+    #
+    # @param source [#read] An IO-like object containing a Zip file.
+    # @param target [String, ::File, Pathname] The target directory where the file should be unzipped.
+    def self.unzip_io_to(io, target)
+      Dir.chdir(target) do
+        Zip::InputStream.open(io) do |input|
+          while (entry = input.get_next_entry)
+            unless ::File.exist?(entry.name) || entry.name_is_directory?
+              FileUtils::mkdir_p(::File.dirname(entry.name))
+              ::File.binwrite(entry.name, input.read)
+            end
+          end
+        end
+      end
+    end
+
     ##
     # Extract the contents of the given Zip file to the given directory.
     #
     # @param source [String, ::File, Pathname] The location of the zip file.
     # @param target [String, ::File, Pathname] The target directory where the file should be unzipped.
-    def self.unzip_to(source, target)
-      source = ::File.expand_path(source)
+    def self.unzip_file_to(file_or_path, target)
       Dir.chdir(target) do
-        Zip::File.open(source) do |zipfile|
+        Zip::File.open(file_or_path) do |zipfile|
           zipfile.each do |entry|
             unless ::File.exist?(entry.name)
               FileUtils::mkdir_p(::File.dirname(entry.name))
@@ -40,13 +78,16 @@ module ROCrate
     # Reads an RO-Crate from a zip file. It first extracts the Zip file to a temporary directory, and then calls
     # #read_directory.
     #
-    # @param source [String, ::File, Pathname] The location of the zip file.
+    # @param source [String, ::File, Pathname, #read] The location of the zip file, or an IO-like object.
     # @param target_dir [String, ::File, Pathname] The target directory where the crate should be unzipped.
     # @return [Crate] The RO-Crate.
     def self.read_zip(source, target_dir: Dir.mktmpdir)
       unzip_to(source, target_dir)
 
-      read_directory(target_dir)
+      # Traverse the unzipped directory to try and find the crate's root
+      root_dir = detect_root_directory(target_dir)
+
+      read_directory(root_dir)
     end
 
     ##
@@ -55,6 +96,8 @@ module ROCrate
     # @param source [String, ::File, Pathname] The location of the directory.
     # @return [Crate] The RO-Crate.
     def self.read_directory(source)
+      raise "Not a directory!" unless ::File.directory?(source)
+
       source = ::File.expand_path(source)
       metadata_file = Dir.entries(source).detect { |entry| entry == ROCrate::Metadata::IDENTIFIER ||
           entry == ROCrate::Metadata::IDENTIFIER_1_0 }
@@ -107,6 +150,7 @@ module ROCrate
         crate.metadata.properties = entity_hash.delete(ROCrate::Metadata::IDENTIFIER)
         preview_properties = entity_hash.delete(ROCrate::Preview::IDENTIFIER)
         crate.preview.properties = preview_properties if preview_properties
+        crate.add_all(source, false)
         extract_data_entities(crate, source, entity_hash).each do |entity|
           crate.add_data_entity(entity)
         end
@@ -211,5 +255,23 @@ module ROCrate
       raise "Metadata entity does not reference any root entity" unless root_id
       entities.delete(root_id)
     end
+
+    ##
+    # Finds an RO-Crate's root directory (where `ro-crate-metdata.json` is located) within a given directory.
+    #
+    # @param source [String, ::File, Pathname] The location of the directory.
+    # @return [Pathname, nil] The path to the root, or nil if not found.
+    def self.detect_root_directory(source)
+      Pathname(source).find do |entry|
+        if entry.file?
+          name = entry.basename.to_s
+          if name == ROCrate::Metadata::IDENTIFIER || name == ROCrate::Metadata::IDENTIFIER_1_0
+            return entry.parent
+          end
+        end
+      end
+
+      nil
+    end
   end
 end

data/ro_crate.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'ro-crate'
-  s.version     = '0.4.5'
+  s.version     = '0.4.10'
   s.summary     = 'Create, manipulate, read RO-Crates.'
   s.authors     = ['Finn Bacall']
   s.email       = 'finn.bacall@manchester.ac.uk'

data/test/entity_test.rb

@@ -70,4 +70,25 @@ class EntityTest < Test::Unit::TestCase
     assert_equal({ '@id' => '#fred' }, person.reference)
     assert_equal(person.canonical_id, crate.author.canonical_id)
   end
+
+  test 'format various IDs' do
+    assert_equal "#Hello%20World/Goodbye%20World", ROCrate::ContextualEntity.format_id('#Hello World/Goodbye World')
+    assert_equal "#Hello%20World/Goodbye%20World", ROCrate::ContextualEntity.format_id('Hello World/Goodbye World')
+    assert_equal "#%F0%9F%98%8A", ROCrate::ContextualEntity.format_id("😊")
+
+    assert_equal "test123/hello.txt", ROCrate::File.format_id('./test123/hello.txt')
+    assert_equal "test123/hello.txt", ROCrate::File.format_id('./test123/hello.txt/')
+    assert_equal "http://www.data.com/my%20data.txt", ROCrate::File.format_id('http://www.data.com/my%20data.txt')
+    assert_equal "http://www.data.com/my%20data.txt/", ROCrate::File.format_id('http://www.data.com/my%20data.txt/'), 'Should not modify absolute URI for DataEntity'
+
+    assert_equal "my%20directory/", ROCrate::Directory.format_id('my directory')
+    assert_equal "my%20directory/", ROCrate::Directory.format_id('my directory/')
+    assert_equal 'http://www.data.com/my%20directory', ROCrate::Directory.format_id('http://www.data.com/my%20directory'), 'Should not modify absolute URI for DataEntity'
+    assert_equal 'http://www.data.com/my%20directory/', ROCrate::Directory.format_id('http://www.data.com/my%20directory/'), 'Should not modify absolute URI for DataEntity'
+
+    assert_equal "./", ROCrate::Crate.format_id('./')
+    assert_equal "cool%20crate/", ROCrate::Crate.format_id('./cool crate')
+    assert_equal "http://www.data.com/my%20crate/", ROCrate::Crate.format_id('http://www.data.com/my%20crate'), 'Crate ID should end with /'
+    assert_equal "http://www.data.com/my%20crate/", ROCrate::Crate.format_id('http://www.data.com/my%20crate/')
+  end
 end

data/test/fixtures/sparse_directory_crate/fish/data/info.txt

@@ -0,0 +1 @@
+5678

data/test/fixtures/sparse_directory_crate/fish/data/nested.txt

@@ -0,0 +1 @@
+I'm nested

data/test/fixtures/sparse_directory_crate/fish/info.txt

@@ -0,0 +1 @@
+1234

data/test/fixtures/sparse_directory_crate/fish/root.txt

@@ -0,0 +1 @@
+I'm at the root

data/test/fixtures/sparse_directory_crate/listed_file.txt

@@ -0,0 +1 @@
+123

data/test/fixtures/sparse_directory_crate/ro-crate-metadata.jsonld

@@ -0,0 +1,32 @@
+{
+  "@context": "https://w3id.org/ro/crate/1.0/context",
+  "@graph": [
+    {
+      "@id": "ro-crate-metadata.jsonld",
+      "@type": "CreativeWork",
+      "about": {
+        "@id": "./"
+      }
+    },
+    {
+      "@id": "ro-crate-preview.html",
+      "@type": "CreativeWork",
+      "about": {
+        "@id": "./"
+      }
+    },
+    {
+      "@id": "./",
+      "@type": "Dataset",
+      "hasPart": [
+        {
+          "@id": "listed_file.txt"
+        }
+      ]
+    },
+    {
+      "@id": "listed_file.txt",
+      "@type": "File"
+    }
+  ]
+}

data/test/fixtures/sparse_directory_crate/ro-crate-preview.html

@@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>New RO-Crate</title>
+  <script type="application/ld+json">{
+  "@context": "https://w3id.org/ro/crate/1.1/context",
+  "@graph": [
+    {
+      "@id": "ro-crate-metadata.jsonld",
+      "@type": "CreativeWork",
+      "about": {
+        "@id": "./"
+      }
+    },
+    {
+      "@id": "ro-crate-preview.html",
+      "@type": "CreativeWork",
+      "about": {
+        "@id": "./"
+      }
+    },
+    {
+      "@id": "./",
+      "@type": "Dataset",
+      "hasPart": [
+        {
+          "@id": "fish/"
+        }
+      ]
+    },
+    {
+      "@id": "fish/",
+      "@type": "Dataset"
+    }
+  ]
+}</script>
+  <meta name="generator" content="https://github.com/fbacall/ro-crate-ruby">
+  <meta name="keywords" content="RO-Crate">
+</head>
+<body>
+  <h1>New RO-Crate</h1>
+  
+  <p>
+    
+  </p>
+  <dl>
+    
+    
+    
+    
+  </dl>
+
+  <h2>Contents</h2>
+  <ul>
+    
+      <li id="__data_entity_fish/">
+        
+          <strong>fish/</strong>
+        
+        
+        
+      </li>
+    
+  </ul>
+</body>
+</html>

data/test/fixtures/sparse_directory_crate/unlisted_file.txt

@@ -0,0 +1 @@
+456

data/test/reader_test.rb

@@ -163,4 +163,68 @@ class ReaderTest < Test::Unit::TestCase
     assert_equal 'http://example.com/external_ref.txt', ext_file.id
     assert_equal 'file contents', ext_file.source.read
   end
+
+  test 'reading from directory with unlisted files' do
+    crate = ROCrate::Reader.read_directory(fixture_file('sparse_directory_crate').path)
+
+    assert_equal 11, crate.entries.count
+    assert crate.entries['listed_file.txt']
+    assert crate.entries['unlisted_file.txt']
+    assert crate.entries['fish']
+    assert_equal '1234', crate.entries['fish/info.txt'].source.read.chomp
+    refute crate.entries['fish/root.txt'].directory?
+    assert crate.entries['fish/data'].directory?
+    assert crate.entries['fish/data/info.txt']
+    refute crate.entries['fish/data/nested.txt'].remote?
+    assert crate.entries['fish/data/binary.jpg']
+    assert_equal ['./', 'listed_file.txt', 'ro-crate-metadata.jsonld', 'ro-crate-preview.html'], crate.entities.map(&:id).sort
+  end
+
+  test 'reading from a zip with unlisted files' do
+    crate = ROCrate::Reader.read_zip(fixture_file('sparse_directory_crate.zip').path)
+
+    assert_equal 11, crate.entries.count
+    assert crate.entries['listed_file.txt']
+    assert crate.entries['unlisted_file.txt']
+    assert crate.entries['fish']
+    assert_equal '1234', crate.entries['fish/info.txt'].source.read.chomp
+    refute crate.entries['fish/root.txt'].directory?
+    assert crate.entries['fish/data'].directory?
+    assert crate.entries['fish/data/info.txt']
+    refute crate.entries['fish/data/nested.txt'].remote?
+    assert crate.entries['fish/data/binary.jpg']
+    assert_equal ['./', 'listed_file.txt', 'ro-crate-metadata.jsonld', 'ro-crate-preview.html'], crate.entities.map(&:id).sort
+  end
+
+  test 'reading a zip from various object types' do
+    string_io = StringIO.new
+    string_io.write(::File.read(fixture_file('sparse_directory_crate.zip').path))
+    string_io.rewind
+    assert string_io.is_a?(StringIO)
+    assert_equal 11, ROCrate::Reader.read_zip(string_io).entries.count
+
+    path = Pathname.new(fixture_file('sparse_directory_crate.zip').path)
+    assert path.is_a?(Pathname)
+    assert_equal 11, ROCrate::Reader.read_zip(path).entries.count
+
+    file = ::File.open(fixture_file('sparse_directory_crate.zip').path)
+    assert file.is_a?(::File)
+    assert_equal 11, ROCrate::Reader.read_zip(file).entries.count
+
+    string = fixture_file('sparse_directory_crate.zip').path
+    assert string.is_a?(String)
+    assert_equal 11, ROCrate::Reader.read_zip(string).entries.count
+  end
+
+  test 'reading from zip where the crate root is nested somewhere within' do
+    crate = ROCrate::Reader.read_zip(fixture_file('nested_directory.zip'))
+
+    assert crate.entries['fish/info.txt']
+    assert_equal '1234', crate.entries['fish/info.txt'].source.read.chomp
+    assert crate.entries['fish/root.txt']
+    assert crate.entries['fish/data/info.txt']
+    assert crate.entries['fish/data/nested.txt']
+    assert crate.entries['fish/data/binary.jpg']
+    assert_equal ['./', 'fish/', 'ro-crate-metadata.json', 'ro-crate-preview.html'], crate.entities.map(&:id).sort
+  end
 end

data/test/writer_test.rb

@@ -112,4 +112,24 @@ class WriterTest < Test::Unit::TestCase
       assert_equal 2529, ::File.size(::File.join(dir, 'data', 'binary.jpg'))
     end
   end
+
+  test 'reading and writing out a directory crate produces an identical crate' do
+    fixture = fixture_file('sparse_directory_crate').path
+    Dir.mktmpdir do |dir|
+      dir = ::File.join(dir, 'new_directory')
+      crate = ROCrate::Reader.read(fixture)
+
+      ROCrate::Writer.new(crate).write(dir)
+      expected_files = Dir.chdir(fixture) { Dir.glob('**/*') }
+      actual_files =  Dir.chdir(dir) { Dir.glob('**/*') }
+      assert_equal expected_files, actual_files
+      expected_files.each do |file|
+        next if file == 'ro-crate-metadata.jsonld' # Expected context gets updated.
+        next if file == 'ro-crate-preview.html' # RO-Crate preview format changed
+        abs_file_path = ::File.join(fixture, file)
+        next if ::File.directory?(abs_file_path)
+        assert_equal ::File.read(abs_file_path), ::File.read(::File.join(dir, file)), "#{file} didn't match"
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ro-crate
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.10
 platform: ruby
 authors:
 - Finn Bacall
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-24 00:00:00.000000000 Z
+date: 2021-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -165,8 +165,19 @@ files:
 - test/fixtures/directory_crate/ro-crate-preview.html
 - test/fixtures/file with spaces.txt
 - test/fixtures/info.txt
+- test/fixtures/nested_directory.zip
 - test/fixtures/spaces/file with spaces.txt
 - test/fixtures/spaces/ro-crate-metadata.jsonld
+- test/fixtures/sparse_directory_crate.zip
+- test/fixtures/sparse_directory_crate/fish/data/binary.jpg
+- test/fixtures/sparse_directory_crate/fish/data/info.txt
+- test/fixtures/sparse_directory_crate/fish/data/nested.txt
+- test/fixtures/sparse_directory_crate/fish/info.txt
+- test/fixtures/sparse_directory_crate/fish/root.txt
+- test/fixtures/sparse_directory_crate/listed_file.txt
+- test/fixtures/sparse_directory_crate/ro-crate-metadata.jsonld
+- test/fixtures/sparse_directory_crate/ro-crate-preview.html
+- test/fixtures/sparse_directory_crate/unlisted_file.txt
 - test/fixtures/workflow-0.2.0.zip
 - test/fixtures/workflow-0.2.0/Dockerfile
 - test/fixtures/workflow-0.2.0/README.md