excavate 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34a72a8ae6eb56395e68cf385d8714d9e6c2aa82245353c2c27bb5cb163d6e2a
-  data.tar.gz: afd30ada69cad61e75e291281f25bed671c3445c9d625db4593a2ee2ea64bf02
+  metadata.gz: d9ea70e2de46706b2324d98253d9012d175a706fc010478a0ccb1b6af00fc02e
+  data.tar.gz: 51c3c8e44013e17201114f4315b4314d9f44dc645c41d3f8627f81353b0269b1
 SHA512:
-  metadata.gz: 345129ddad35de7619aef4b1538ae0fec8b88a64baa45fca4a3e8578067026019614bef3c08ebbd81ef916ef8ba58966cde4cd89cc0afdb3337e2198e09fdafc
-  data.tar.gz: f829b60851340bf8ff5c881a9e0b9fea7daafdfdceac303e499fba08f2be076df5ffb4f4c0a85f0d072d862d3443298eeddc338df0326b3dbf5f582204685fc8
+  metadata.gz: 4f1c1c62794e14c5272cb8c9881dd50a8be3992c446086c0a5f58cecf51100458f962ddaeec1d2d329a0d3e03cff328bb287a9183ab5315534ba203fb1758544
+  data.tar.gz: 79a3aac7b28480c4a1675b68877e21e192d311099f348aaebc84ae157d966e6bb56c2dffe53c25aeea331a08286aa7b34e8e92fd015cb8588514da8340092d85

data/.github/workflows/metanorma.yml

@@ -0,0 +1,35 @@
+name: metanorma
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+
+jobs:
+  test:
+    name: Test on Ruby ${{ matrix.ruby }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        experimental: [ true ]
+    steps:
+      - uses: actions/checkout@master
+        with:
+          repository: metanorma/metanorma
+
+      - uses: actions/checkout@master
+        with:
+          path: "excavate"
+
+      - run: 'echo ''gem "excavate", path: "./excavate"'' > Gemfile.devel'
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - run: bundle exec rake

data/.github/workflows/rspec.yml

@@ -2,7 +2,7 @@ name: rspec
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, main ]
   pull_request:
 
 jobs:

data/Gemfile

@@ -5,9 +5,7 @@ source "https://rubygems.org"
 gemspec
 
 gem "gem-release"
-
 gem "rake", "~> 13.0"
-
 gem "rspec", "~> 3.2"
 gem "rubocop", "0.75.0"
 gem "rubocop-performance"

data/README.adoc

@@ -61,6 +61,13 @@ end
 $ excavate --recursive path/to/archive.cab
 ----
 
+It supports recursive extraction of a directory containing archives:
+
+[source,sh]
+----
+$ excavate --recursive path/to/dir_with_archives
+----
+
 If you'd like to skip extraction of nested archives, just use:
 
 [source,sh]
@@ -68,6 +75,39 @@ If you'd like to skip extraction of nested archives, just use:
 $ excavate path/to/archive.cab
 ----
 
+To extract a particular file or files specify them as last arguments:
+
+[source,sh]
+----
+$ excavate --recursive archive.cab file1 dir/file2
+----
+
+Also `excavate` supports extraction from nested archives:
+
+[source,sh]
+----
+$ excavate --recursive archive.cab dir/nested.zip/file
+----
+
+And filtering:
+
+[source,sh]
+----
+$ excavate archive.cab --filter "**/specialfile*.txt"
+----
+
+== Dependencies
+
+Depends on
+https://github.com/fontist/ffi-libarchive-binary[ffi-libarchive-binary] which
+has the following requirements:
+
+* zlib
+* Expat
+* OpenSSL (for Linux only)
+
+These dependencies are generally present on all systems.
+
 
 == Development
 

data/lib/excavate/archive.rb

@@ -1,5 +1,8 @@
 module Excavate
   class Archive
+    INVALID_MEMORY_MESSAGE =
+      "invalid memory read at address=0x0000000000000000".freeze
+
     TYPES = { "cab" => Extractors::CabExtractor,
               "cpio" => Extractors::CpioExtractor,
               "exe" => Extractors::SevenZipExtractor,
@@ -14,9 +17,10 @@ module Excavate
       @archive = archive
     end
 
-    def files(recursive_packages: false)
+    def files(recursive_packages: false, files: [], filter: nil)
       target = Dir.mktmpdir
-      extract(target, recursive_packages: recursive_packages)
+      extract(target, recursive_packages: recursive_packages,
+                      files: files, filter: filter)
 
       all_files_in(target).map do |file|
         yield file
@@ -25,10 +29,105 @@ module Excavate
       FileUtils.rm_rf(target)
     end
 
-    def extract(target = nil, recursive_packages: false)
+    def extract(target = nil,
+                recursive_packages: false,
+                files: [],
+                filter: nil)
+      if files.size.positive?
+        extract_particular_files(target, files,
+                                 recursive_packages: recursive_packages)
+      elsif filter
+        extract_by_filter(target, filter,
+                          recursive_packages: recursive_packages)
+      else
+        extract_all(target, recursive_packages: recursive_packages)
+      end
+    end
+
+    private
+
+    def extract_particular_files(target, files, recursive_packages: false)
+      tmp = Dir.mktmpdir
+      extract_all(tmp, recursive_packages: recursive_packages)
+      found_files = find_files(tmp, files)
+      copy_files(found_files, target || Dir.pwd)
+    ensure
+      FileUtils.rm_rf(tmp)
+    end
+
+    def copy_files(files, target)
+      files.map do |file|
+        FileUtils.mkdir_p(target)
+        target_path = File.join(target, File.basename(file))
+        ensure_not_exist(target_path)
+
+        FileUtils.cp(file, target_path)
+
+        target_path
+      end
+    end
+
+    def ensure_not_exist(path)
+      if File.exist?(path)
+        type = File.directory?(path) ? "directory" : "file"
+        raise(TargetExistsError,
+              "Target #{type} `#{File.basename(path)}` already exists.")
+      end
+    end
+
+    def find_files(source, files)
+      all_files = all_files_in(source)
+
+      files.map do |target_file|
+        found_file = all_files.find do |source_file|
+          file_matches?(source_file, target_file, source)
+        end
+
+        unless found_file
+          raise(TargetNotFoundError, "File `#{target_file}` not found.")
+        end
+
+        found_file
+      end
+    end
+
+    def file_matches?(source_file, target_file, source_dir)
+      base_path(source_file, source_dir) == target_file
+    end
+
+    def base_path(path, prefix)
+      path.sub(prefix, "").sub(/^\//, "").sub(/^\\/, "")
+    end
+
+    def extract_by_filter(target, filter, recursive_packages: false)
+      tmp = Dir.mktmpdir
+      extract_all(tmp, recursive_packages: recursive_packages)
+      found_files = find_by_filter(tmp, filter)
+      copy_files(found_files, target || Dir.pwd)
+    end
+
+    def find_by_filter(source, filter)
+      all_files = all_files_in(source)
+
+      found_files = all_files.select do |source_file|
+        file_matches_filter?(source_file, filter, source)
+      end
+
+      if found_files.empty?
+        raise(TargetNotFoundError, "Filter `#{filter}` matched no file.")
+      end
+
+      found_files
+    end
+
+    def file_matches_filter?(source_file, filter, source_dir)
+      File.fnmatch?(filter, base_path(source_file, source_dir))
+    end
+
+    def extract_all(target, recursive_packages: false)
       source = File.expand_path(@archive)
       target ||= default_target(source)
-      raise(TargetNotEmptyError, "Target directory `#{File.basename(target)}` is not empty.") unless Dir.empty?(target)
+      ensure_empty(target)
 
       if recursive_packages
         extract_recursively(source, target)
@@ -39,11 +138,16 @@ module Excavate
       target
     end
 
-    private
+    def ensure_empty(path)
+      unless Dir.empty?(path)
+        raise(TargetNotEmptyError,
+              "Target directory `#{File.basename(path)}` is not empty.")
+      end
+    end
 
     def default_target(source)
       target = File.expand_path(File.basename(source, ".*"))
-      raise(TargetExistsError, "Target directory `#{File.basename(target)}` already exists.") if File.exist?(target)
+      ensure_not_exist(target)
 
       FileUtils.mkdir(target)
 
@@ -51,7 +155,11 @@ module Excavate
     end
 
     def extract_recursively(archive, target)
-      extract_once(archive, target)
+      if File.directory?(archive)
+        duplicate_dir(archive, target)
+      else
+        extract_once(archive, target)
+      end
 
       all_files_in(target).each do |file|
         next unless archive?(file)
@@ -60,14 +168,25 @@ module Excavate
       end
     end
 
+    def duplicate_dir(source, target)
+      Dir.chdir(source) do
+        (Dir.entries(".") - [".", ".."]).each do |entry|
+          FileUtils.cp_r(entry, target)
+        end
+      end
+    end
+
     def extract_once(archive, target)
       extension = normalized_extension(archive)
       extractor_class = TYPES[extension]
-      raise(UnknownArchiveError, "Could not unarchive `#{archive}`.") unless extractor_class
+      unless extractor_class
+        raise(UnknownArchiveError, "Could not unarchive `#{archive}`.")
+      end
 
       extractor_class.new(archive).extract(target)
     rescue StandardError => e
-      raise unless extension == "exe" && e.message.start_with?("Invalid file format")
+      raise unless extension == "exe" &&
+        e.message.start_with?("Invalid file format")
 
       Extractors::CabExtractor.new(archive).extract(target)
     end
@@ -81,7 +200,7 @@ module Excavate
     rescue FFI::NullPointerError => e
       FileUtils.rmdir(target)
       raise unless normalized_extension(archive) == "exe" &&
-        e.message.start_with?("invalid memory read at address=0x0000000000000000")
+        e.message.start_with?(INVALID_MEMORY_MESSAGE)
     end
 
     def normalized_extension(file)

data/lib/excavate/cli.rb

@@ -8,6 +8,13 @@ module Excavate
     STATUS_UNKNOWN_ERROR = 1
     STATUS_TARGET_EXISTS = 2
     STATUS_TARGET_NOT_EMPTY = 3
+    STATUS_TARGET_NOT_FOUND = 4
+
+    ERROR_TO_STATUS = {
+      TargetExistsError => STATUS_TARGET_EXISTS,
+      TargetNotEmptyError => STATUS_TARGET_NOT_EMPTY,
+      TargetNotFoundError => STATUS_TARGET_NOT_FOUND,
+    }.freeze
 
     def self.exit_on_failure?
       false
@@ -23,15 +30,22 @@ module Excavate
       super(args, config)
     end
 
-    desc "extract ARCHIVE", "Extract ARCHIVE to a new directory"
-    option :recursive, aliases: :r, type: :boolean, default: false, desc: "Also extract all nested archives."
-    def extract(archive)
-      target = Excavate::Archive.new(archive).extract(recursive_packages: options[:recursive])
-      success("Successfully extracted to #{File.basename(target)}/")
-    rescue TargetExistsError => e
-      error(e.message, STATUS_TARGET_EXISTS)
-    rescue TargetNotEmptyError => e
-      error(e.message, STATUS_TARGET_NOT_EMPTY)
+    desc "extract ARCHIVE [FILE...]",
+         "Extract FILE or all files from ARCHIVE to a new directory"
+    option :recursive, aliases: :r, type: :boolean, default: false,
+                       desc: "Also extract all nested archives."
+    option :filter, type: :string,
+                    desc: "Filter by pattern (supports **, *, ?, etc)"
+    def extract(archive, *files)
+      target = Excavate::Archive.new(archive).extract(
+        recursive_packages: options[:recursive],
+        files: files,
+        filter: options[:filter],
+      )
+
+      success("Successfully extracted to #{format_paths(target)}")
+    rescue Error => e
+      handle_error(e)
     end
     default_task :extract
 
@@ -42,9 +56,24 @@ module Excavate
       STATUS_SUCCESS
     end
 
+    def handle_error(exception)
+      status = ERROR_TO_STATUS[exception.class]
+      raise exception unless status
+
+      error(exception.message, status)
+    end
+
     def error(message, status)
       say(message, :red)
       status
     end
+
+    def format_paths(path_or_paths)
+      paths = Array(path_or_paths).map do |x|
+        File.directory?(x) ? "#{File.basename(x)}/" : File.basename(x)
+      end
+
+      paths.join(", ")
+    end
   end
 end

data/lib/excavate/extractors/ole_extractor.rb

@@ -15,6 +15,8 @@ module Excavate
 
         Ole::Storage.open(@archive) do |ole|
           children(ole).each do |file|
+            next if ole.file.directory?(file)
+
             filename = prepare_filename(file)
             path = File.join(target, filename)
             content = ole.file.read(file)

data/lib/excavate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Excavate
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/excavate.rb

@@ -8,7 +8,12 @@ require_relative "excavate/utils"
 
 module Excavate
   class Error < StandardError; end
-  class UnknownArchiveError < Error; end
+
   class TargetExistsError < Error; end
+
   class TargetNotEmptyError < Error; end
+
+  class TargetNotFoundError < Error; end
+
+  class UnknownArchiveError < Error; end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: excavate
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-03-31 00:00:00.000000000 Z
+date: 2021-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: arr-pm
@@ -116,6 +116,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/metanorma.yml"
 - ".github/workflows/release.yml"
 - ".github/workflows/rspec.yml"
 - ".gitignore"
@@ -168,7 +169,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Extract nested archives with a single command.