rom-distillery 0.1

data/lib/distillery/archiver/archive.rb

@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: EUPL-1.2
+
+module Distillery
+class Archiver
+
+# Allow archive file processing
+#
+# All the operations are forwarded to an {Archiver} instance
+# which is able to process the selected archive file.
+#   
+class Archive
+    include Enumerable
+
+    # Returns a new instance of Archive.
+    #
+    # @param file [String]		file holding the archive
+    #
+    # @raise [ArchiverNotFound]		an archiver able to process this file
+    #					has not been found
+    #
+    def initialize(file)
+        @file     = file
+        @archiver = Archiver.for_file(file)
+
+        if @archiver.nil?
+            raise ArchiverNotFound, "no archiver avalaible for this file"
+        end
+    end
+
+    
+    # Iterate over each archive entry
+    #
+    # @yieldparam entry [String]	entry name
+    # @yieldparam io    [InputStream]	input stream
+    #
+    # @return [self,Enumerator]
+    #
+    def each(&block)
+        @archiver.each(@file, &block)
+        self
+    end
+
+    
+    # List of entries for the archive
+    #
+    # @return [Array<String>]
+    #
+    def entries
+        @archiver.entries(@file)
+    end
+
+
+    # Is the archive emtpy?
+    #
+    # @return [Boolean]
+    #
+    def empty?
+        @archiver.empty?(@file)
+    end
+
+    
+    # Allow to perform read operation on an archive entry
+    #
+    # @param entry [String]		entry name
+    #
+    # @yieldparam io [InputStream]	input stream for reading
+    #
+    # @return 				block value
+    #
+    def reader(entry, &block)
+        @archiver.reader(@file, entry, &block)
+    end
+
+    
+    # Allow to perform write operation on an archive entry
+    #
+    # @param entry [String]		entry name
+    #
+    # @yieldparam io [OutputStream]	output stream for writing
+    #
+    # @return 				block value
+    #
+    def writer(entry, &block)
+        @archiver.writer(@file, entry, &block)
+    end
+
+    
+    # Delete the entry from the archive
+    #
+    # @param entry [String]		entry name
+    #
+    # @return [Boolean]		operation status
+    #
+    def delete!(entry)
+        @archiver.delete!(@file, entry)
+    end
+
+    
+end
+    
+end
+end

data/lib/distillery/archiver/external.rb

@@ -0,0 +1,182 @@
+# SPDX-License-Identifier: EUPL-1.2
+
+require 'open3'
+require 'yaml'
+
+module Distillery
+class Archiver
+
+# Use external program to process archive
+# Selection of external program is described in external.yaml
+#
+class External < Archiver
+    Archiver.add self
+
+    # Perform registration of the various archive format
+    # supported by this archiver provider
+    #
+    # @param list         [Array<String>]	list of format to register
+    # @param default_file [String]		default configuration file
+    #
+    # @return [void]
+    #
+    def self.registering(list         = [ '7z', 'zip' ],
+                         default_file = File.join(__dir__, 'external.yaml'))
+        dflt_config = YAML.load_file(default_file)
+
+        if Array === list
+            list = Hash[list.map {|app| [ app, {} ] }]
+        end
+
+        list.each {|app, cfg|
+            dflt       = dflt_config.dig(app) || {}
+            extensions = Array(cfg.dig('extension')  || dflt.dig('extension'))
+            mimetypes  = Array(cfg.dig('mimetype' )  || dflt.dig('mimetype' ))
+            list       = {
+                :cmd    => cfg .dig('list', 'cmd')   || cfg .dig('cmd') ||
+                           dflt.dig('list', 'cmd')   || dflt.dig('cmd'),
+                :args   => cfg .dig('list', 'args')  ||
+                           dflt.dig('list', 'args'),
+                :parser => if parser = (cfg .dig('list', 'parser')      ||
+                                        dflt.dig('list', 'parser'))
+                               Regexp.new('\A' + parser + '\Z')
+                           end,
+                :validator => cfg .dig('list', 'validator') ||
+                              dflt.dig('list', 'validator')
+            }
+            read       = {
+                :cmd    => cfg .dig('read', 'cmd')   || cfg .dig('cmd') ||
+                           dflt.dig('read', 'cmd')   || dflt.dig('cmd'),
+                :args   => cfg .dig('read', 'args')  ||
+                           dflt.dig('read', 'args'),
+            }
+            write      = {
+                :cmd    => cfg .dig('write', 'cmd')  || cfg .dig('cmd') ||
+                           dflt.dig('write', 'cmd')  || dflt.dig('cmd'),
+                :args   => cfg .dig('write', 'args') ||
+                           dflt.dig('write', 'args'),
+            }
+            delete     = {
+                :cmd    => cfg .dig('delete', 'cmd')  || cfg .dig('cmd') ||
+                           dflt.dig('delete', 'cmd')  || dflt.dig('cmd'),
+                :args   => cfg .dig('delete', 'args') ||
+                           dflt.dig('delete', 'args'),
+            }
+
+            if list[:cmd].nil? || read[:cmd].nil?
+                Archiver.logger&.warn {
+                    "#{self}: command not defined for #{app} program (SKIP)"
+                }
+                next
+            end
+            if write[:cmd].nil?
+                Archiver.logger&.warn {
+                    "#{self}: write mode not supported for #{app} program"
+                }
+            end
+                
+            Archiver.register(External.new(list, read, write, delete,
+                                           extensions: extensions,
+                                            mimetypes: mimetypes))
+        }
+    end
+    
+    def initialize(list, read, write=nil, delete=nil,
+                   extensions:, mimetypes: nil)
+        @list       = list
+        @read       = read
+        @write      = write
+        @delete     = delete
+        @extensions = extensions
+        @mimetypes  = mimetypes
+    end
+    
+                
+    # (see Archiver#extensions)
+    def extensions
+        @extensions
+    end
+
+    
+    # (see Archiver#mimetypes)
+    def mimetypes
+        @mimetypes
+    end
+
+    # (see Archiver#each)
+    def each(file, &block)
+        return to_enum(:each, file) if block.nil?
+        entries(file).each {|entry|
+            reader(file, entry) {|io|
+                block.call(entry, io)
+            }
+        }
+    end
+
+    # (see Archiver#reader)
+    def reader(file, entry, &block)
+        subst = { '$(infile)' => file, '$(entry)' => entry  }
+        cmd   = @read[:cmd ]
+        args  = @read[:args].map{|e| e.gsub(/\$\(\w+\)/, subst) }
+
+        Open3.popen2(cmd, *args) {|stdin, stdout|
+            stdin.close_write
+            block.call(InputStream.new(stdout))
+        }
+    end
+
+    # (see Archiver#writer)
+    def writer(file, entry, &block)
+        subst = { '$(infile)' => file, '$(entry)' => entry  }
+        cmd   = @write[:cmd ]
+        args  = @write[:args].map{|e| e.gsub(/\$\(\w+\)/, subst) }
+
+        Open3.popen2(cmd, *args) {|stdin, stdout|
+            block.call(OutputStream.new(stdin))
+        }
+    end
+
+    # (see Archiver#delete!)
+    def delete!(file, entry)
+        subst = { '$(infile)' => file, '$(entry)' => entry  }
+        cmd   = @delete[:cmd ]
+        args  = @delete[:args].map{|e| e.gsub(/\$\(\w+\)/, subst) }
+
+        Open3.popen2(cmd, *args) {|stdin, stdout|
+            stdin.close_write
+        }
+
+        true
+    end
+
+    # (see Archiver#entries)
+    def entries(file)
+        subst     = { '$(infile)' => file }
+        cmd       = @list[:cmd      ]
+        args      = @list[:args     ].map{|e| e.gsub(/\$\(\w+\)/, subst) }
+        parser    = @list[:parser   ]
+        validator = @list[:validator]
+        
+        stdout, stderr, status = Open3.capture3(cmd, *args)
+
+        if ! status.exitstatus.zero?
+            raise ExecError, "running external command failed (#{stderr})"
+        end
+
+        stdout.force_encoding('BINARY').lines(chomp: true).map {|l|
+            unless m = l.match(parser)
+                raise ProcessingError, "unable to parse entry (#{file}) (#{l})"
+            end
+
+            if validator && validator.find {|k,v| m[k] != v }
+                next
+            end
+
+            m[:entry]
+        }.compact
+    end
+end
+
+end
+end
+

data/lib/distillery/archiver/external.yaml

@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: EUPL-1.2
+
+7z:
+  extension: 7z
+  mimetype: application/x-7z-compressed
+  cmd: 7z
+  list:
+    args: [ l, -ba, $(infile) ]
+    parser: (?:\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})?\s+(?<type>.)....\s+\d+\s+(?:\d+\s+)?(?<entry>.+)
+    validator:
+      :type: '.'
+  read:
+    args: [ e, -so, $(infile), $(entry) ]
+  write:
+    args: [ a, $(infile), -m0=lzma, -mx=9, -si$(entry) ]
+  delete:
+    args: [ d, $(infile), $(entry) ]
+
+zip:
+  extension: zip
+  mimetype: application/zip
+  list:
+    cmd: unzip
+    args: [ -Z1, $(infile) ]
+    parser: (?<entry>.+)
+  read:
+    cmd: unzip
+    args: [ -p, $(infile), $(entry) ]
+  delete:
+    cmd: zip
+    args: [ -d, $(infile), $(entry) ]

data/lib/distillery/archiver/libarchive.rb

@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: EUPL-1.2
+
+begin
+    require 'libarchive'
+rescue LoadError
+    return
+end
+
+
+module Distillery
+class Archiver
+
+# Use binding to libarchive
+#
+class LibArchive < Archiver
+    Archiver.add self
+
+    MODE = { '7z'  => { :extensions =>  '7z',
+                        :mimetypes  => 'application/x-7z-compressed' },
+             'zip' => { :extensions => 'zip',
+                        :mimetypes  => 'application/zip'             },
+           }
+
+
+    class InputStream < Archiver::InputStream
+        def initialize(ar)
+            @read_block = ar.to_enum(:read_data, 16*1024)
+            @buffer     = StringIO.new
+        end
+
+        def read(length=nil)
+            return ''  if length&.zero?		# Zero length request
+            return nil if @buffer.nil?		# End of stream
+
+            # Read data
+            data = @buffer.read(length) || ""
+            while data.size < length do
+                # We are short on data, that means buffer has been exhausted
+                # request new data block from the archive
+                block = @read_block.next
+                # Break if we already read all the archive data
+                if block.nil? || block.empty?
+                    @buffer = nil
+                    break
+                end
+                # Refill buffer from block
+                @buffer.string = block
+                # Continue reading
+                data.concat(@buffer.read(length - data.size))
+            end
+            
+            data.empty? ? nil : data
+        end
+    end
+    
+    
+
+    
+    def self.registering
+        MODE.each_key {|mode|
+            Archiver.register(LibArchive.new(mode))
+        }
+    end
+
+
+    def initialize(mode)
+        raise ArgumentError unless MODE.include?(mode)
+        @mode = mode
+    end
+
+
+    # List of supported extensions
+    #
+    def extensions
+        MODE[@mode][:extensions]
+    end
+
+    
+    # List of supported mimetypes
+    #
+    def mimetypes
+        MODE[@mode][:mimetypes]
+    end
+
+
+    # (see Archiver#each)
+    def each(file, &block)
+        ::Archive.read_open_filename(file) {|ar|
+            while a_entry = ar.next_header
+                next unless a_entry.regular?
+                $stdout.puts a_entry.pathname
+                block.call(a_entry.pathname, InputStream.new(ar))
+            end
+        }
+        self
+    end
+
+end
+
+
+end
+end
+
+
+

data/lib/distillery/archiver/zip.rb

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: EUPL-1.2
+
+begin
+    require 'zip'
+rescue LoadError
+    return
+end
+
+
+module Distillery
+class Archiver
+
+# Use of rubyzip as archiver
+#
+class Zip < Archiver
+    Archiver.add self
+
+
+    
+    # Perform registration of the various archive format
+    # supported by this archiver provider
+    #
+    # @return [void]
+    #
+    def self.registering
+        Archiver.register(Zip.new)
+    end
+   
+    
+    def initialize
+    end
+    
+    # (see Archiver#extensions)
+    def extensions
+        [ 'zip' ]
+    end
+
+    
+    # (see Archiver#mimetypes)
+    def mimetypes
+        [ 'application/zip' ]
+    end
+
+    
+    # (see Archiver#delete!)
+    def delete!(file, entry)
+        ::Zip::File.open(file) {|zip_file|
+             zip_file.remove(entry) ? true : false
+        }
+    rescue Errno::ENOENT
+        false
+    end
+
+    # (see Archiver#reader)
+    def reader(file, entry, &block)
+        ::Zip::File.open(file) {|zip_file|
+            zip_file.get_input_stream(entry) {|is|
+                block.call(InputStream.new(is))
+            }
+        }
+    end
+
+    # (see Archiver#writer)
+    def writer(file, entry, &block)
+        ::Zip::File.open(file, ::Zip::File::CREATE) {|zip_file|
+            zip_file.get_output_stream(entry) {|os|
+                block.call(OutputStream.new(os))
+            }
+        }
+    end 
+
+    # (see Archiver#each) 
+    def each(file, &block)
+        return to_enum(:each, file) if block.nil?
+        ::Zip::File.open(file) {|zip_file|
+            zip_file.each {|zip_entry|
+                next unless zip_entry.ftype == :file
+                block.call(zip_entry.name,
+                           InputStream.new(zip_entry.get_input_stream))
+            }
+        }
+    end
+
+end
+
+end
+end
+