rakpak 1.0.0

data/lib/rakpak/formats.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+module Rakpak
+  # What this machine can actually do. Nothing here is assumed; every codec
+  # is probed against PATH so the UI can grey out what is missing instead of
+  # failing halfway through a job.
+  module Tools
+    module_function
+
+    def which(bin)
+      return @which[bin] if defined?(@which) && @which&.key?(bin)
+
+      @which ||= {}
+      @which[bin] = ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).lazy
+                       .map { |d| File.join(d, bin) }
+                       .find { |p| File.file?(p) && File.executable?(p) }
+    end
+
+    def available?(bin) = !which(bin).nil?
+
+    # Info-ZIP reports its compiled-in methods in `zip -v`.
+    def zip_has_bzip2?
+      return @zip_bz2 if defined?(@zip_bz2)
+
+      @zip_bz2 = available?("zip") && `zip -v 2>/dev/null`.include?("BZIP2_SUPPORT")
+    rescue StandardError
+      @zip_bz2 = false
+    end
+
+    # GNU tar, bsdtar (macOS) and busybox tar disagree about flags. Detect
+    # once so the UI only offers switches this tar actually understands.
+    def tar_flavor
+      return @tar_flavor if defined?(@tar_flavor)
+
+      @tar_flavor =
+        if !available?("tar") then :missing
+        else
+          v = begin
+            `tar --version 2>/dev/null`
+          rescue StandardError
+            ""
+          end
+          case v
+          when /GNU tar/ then :gnu
+          when /bsdtar|libarchive/ then :bsd
+          when /busybox/i then :busybox
+          else v.empty? ? :busybox : :unknown
+          end
+        end
+    end
+
+    # Long form works on GNU tar and bsdtar; -I is GNU-only.
+    def tar_pipes? = %i[gnu bsd unknown].include?(tar_flavor)
+
+  end
+
+  # A tar compression backend.
+  Codec = Struct.new(:id, :label, :ext, :bin, :levels, :default, :threads, :blurb,
+                     :level_opt, keyword_init: true) do
+    def available? = bin.nil? || Tools.available?(bin)
+
+    def why_not = available? ? nil : "#{bin} not installed"
+    def container? = false
+
+    # ".tar.gz" for a tarball, ".gz" when compressing a lone file.
+    def single_ext = ext.delete_prefix(".tar")
+
+    # The compressor reading a named file and writing to stdout.
+    def argv(level)
+      return nil if bin.nil?
+
+      parts = [bin, "-c"]
+      # brotli's short -N form only accepts 0-9; 10 and 11 need -q N.
+      parts.concat(level_opt ? [level_opt, level.to_s] : ["-#{level}"]) if levels && level
+      parts.concat(threads) if threads
+      parts
+    end
+
+    # tar's --use-compress-program string; tar splits it on spaces itself.
+    # The long form is understood by both GNU tar and bsdtar.
+    def filter(level) = argv(level)&.join(" ")
+  end
+
+  TAR_CODECS = [
+    Codec.new(id: :none,  label: "none",  ext: ".tar",     bin: nil,
+              levels: nil,   default: nil, blurb: "no compression, fastest"),
+    Codec.new(id: :gzip,  label: "gzip",  ext: ".tar.gz",  bin: "gzip",
+              levels: 1..9,  default: 6,  blurb: "universal, safe default"),
+    Codec.new(id: :zstd,  label: "zstd",  ext: ".tar.zst", bin: "zstd",
+              levels: 1..19, default: 3,  threads: ["-T0"],
+              blurb: "best speed/ratio balance"),
+    Codec.new(id: :xz,    label: "xz",    ext: ".tar.xz",  bin: "xz",
+              levels: 0..9,  default: 6,  threads: ["-T0"],
+              blurb: "smallest output, slow"),
+    Codec.new(id: :bzip2, label: "bzip2", ext: ".tar.bz2", bin: "bzip2",
+              levels: 1..9,  default: 9,  blurb: "legacy, slow"),
+    Codec.new(id: :lz4,   label: "lz4",   ext: ".tar.lz4", bin: "lz4",
+              levels: 1..12, default: 1,  blurb: "extremely fast, low ratio"),
+    Codec.new(id: :brotli, label: "brotli", ext: ".tar.br", bin: "brotli",
+              levels: 0..11, default: 11, level_opt: "-q", blurb: "great on text")
+  ].freeze
+
+  def self.tar_codec(id) = TAR_CODECS.find { |c| c.id == id }
+
+  # A zip container written by Info-ZIP, with one of its entry methods.
+  ZipMethod = Struct.new(:id, :label, :flag, :levels, :default, :blurb, keyword_init: true) do
+    def available?
+      flag == "bzip2" ? Tools.zip_has_bzip2? : Tools.available?("zip")
+    end
+
+    def why_not
+      return nil if available?
+      return "zip not installed" unless Tools.available?("zip")
+
+      "this zip lacks BZIP2_SUPPORT"
+    end
+
+    def container? = true
+  end
+
+  ZIP_METHODS = [
+    ZipMethod.new(id: :zip, label: "zip", flag: "deflate", levels: 0..9, default: 6,
+                  blurb: ".zip with deflate; reads everywhere"),
+    ZipMethod.new(id: :zip_bzip2, label: "zip (bzip2)", flag: "bzip2", levels: 1..9, default: 9,
+                  blurb: "smaller .zip, needs a modern unzip"),
+    ZipMethod.new(id: :zip_store, label: "zip (store)", flag: "store", levels: nil, default: nil,
+                  blurb: ".zip with no compression")
+  ].freeze
+
+  # Everything the zip target can produce: a .zip of anything, or a lone
+  # file run through one compressor (notes.txt.gz).
+  COMPRESSORS = (ZIP_METHODS + TAR_CODECS.reject { |c| c.id == :none }).freeze
+
+  def self.compressor(id) = COMPRESSORS.find { |c| c.id == id }
+
+  # Toggleable switches, rendered as a checklist.
+  Flag = Struct.new(:id, :label, :args, :on, :blurb, :needs, keyword_init: true) do
+    # `needs` lists the tar flavours that understand this switch.
+    def supported?(flavor) = needs.nil? || needs.include?(flavor)
+  end
+
+  GNU  = %i[gnu unknown].freeze
+  BOTH = %i[gnu bsd unknown].freeze
+  ALL  = %i[gnu bsd busybox unknown].freeze
+
+  # Only flags this machine's tar understands are offered.
+  def self.tar_flags(flavor = Tools.tar_flavor)
+    [
+      Flag.new(id: :verbose, label: "verbose", args: ["-v"], on: true, needs: ALL,
+               blurb: "list each file (drives the progress readout)"),
+      Flag.new(id: :preserve, label: "preserve permissions", args: ["-p"], on: true, needs: BOTH,
+               blurb: "keep modes exactly as on disk"),
+      Flag.new(id: :xattrs, label: "extended attributes", args: ["--xattrs", "--acls"], on: false, needs: BOTH,
+               blurb: "store xattrs and POSIX ACLs"),
+      Flag.new(id: :deref, label: "follow symlinks", args: ["-h"], on: false, needs: BOTH,
+               blurb: "archive link targets instead of the links"),
+      Flag.new(id: :onefs, label: "one file system", args: ["--one-file-system"], on: false, needs: BOTH,
+               blurb: "do not cross mount points"),
+      Flag.new(id: :numeric, label: "numeric owner", args: ["--numeric-owner"], on: false, needs: BOTH,
+               blurb: "store uid/gid, not names"),
+      Flag.new(id: :sparse, label: "sparse files", args: ["-S"], on: false, needs: GNU,
+               blurb: "store holes efficiently"),
+      Flag.new(id: :excl_vcs, label: "exclude VCS dirs", args: ["--exclude-vcs"], on: false, needs: GNU,
+               blurb: "skip .git, .hg, .svn"),
+      Flag.new(id: :excl_ign, label: "honour .gitignore", args: ["--exclude-vcs-ignores"], on: false, needs: GNU,
+               blurb: "skip files your VCS ignores"),
+      Flag.new(id: :sorted, label: "reproducible order", args: ["--sort=name"], on: false, needs: GNU,
+               blurb: "deterministic member order"),
+      Flag.new(id: :keepgoing, label: "ignore read errors", args: ["--ignore-failed-read"], on: false, needs: GNU,
+               blurb: "do not abort on unreadable files")
+    ].select { |f| f.supported?(flavor) }
+  end
+
+  def self.zip_flags
+    [
+      Flag.new(id: :verbose, label: "verbose", args: [], on: true,
+               blurb: "list each file (drives the progress readout)"),
+      Flag.new(id: :symlinks, label: "store symlinks", args: ["-y"], on: true,
+               blurb: "keep links as links, do not follow"),
+      Flag.new(id: :dirs, label: "directory entries", args: [], on: true,
+               blurb: "record folders explicitly"),
+      Flag.new(id: :junk, label: "junk paths", args: ["-j"], on: false,
+               blurb: "flatten everything into the root"),
+      Flag.new(id: :noextra, label: "strip extra attributes", args: ["-X"], on: false,
+               blurb: "no uid/gid or timestamps beyond the basics"),
+      Flag.new(id: :oldest, label: "archive time = newest entry", args: ["-o"], on: false,
+               blurb: "reproducible-ish archive mtime")
+    ]
+  end
+end

data/lib/rakpak/job.rb

@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+require_relative "plan"
+
+module Rakpak
+  # Runs a plan's steps on a worker thread, streaming output into a ring
+  # buffer. The UI polls it; backgrounding is just "stop looking at it".
+  class Job
+    KEEP = 400
+
+    attr_reader :plan, :state, :started_at, :finished_at, :exit_status,
+                :error, :step_index, :steps, :file_count, :current_file
+
+    def initialize(plan, total_files: nil)
+      @plan = plan
+      @steps = plan.steps
+      @total_files = total_files
+      @lines = []
+      @lock = Mutex.new
+      @state = :pending
+      @step_index = 0
+      @file_count = 0
+      @current_file = nil
+      @verbose = true
+      @cancel = false
+      @pid = nil
+      @spawned = nil # output path of the step currently being written
+      @started_at = nil
+      @finished_at = nil
+    end
+
+    def start
+      @started_at = now
+      @state = :running
+      @thread = Thread.new { run_all }
+      @thread.abort_on_exception = false
+      self
+    end
+
+    def running? = @state == :running
+    def done? = %i[done failed cancelled].include?(@state)
+    def ok? = @state == :done
+
+    def elapsed
+      return 0 unless @started_at
+
+      (@finished_at || now) - @started_at
+    end
+
+    def tail(n)
+      @lock.synchronize { @lines.last(n).dup }
+    end
+
+    def label
+      @lock.synchronize { @steps[[@step_index, @steps.size - 1].min]&.first || "archive" }
+    end
+
+    def total_files = @total_files
+
+    # nil when there is nothing to measure progress against: no file total,
+    # or a step whose tool is not listing members (verbose off).
+    def fraction
+      return nil unless @total_files&.positive? && @verbose
+
+      per = 1.0 / @steps.size
+      base = @step_index * per
+      [base + (per * [@file_count.to_f / @total_files, 1.0].min), 1.0].min
+    end
+
+    # Bytes on disk for whatever this step is writing right now.
+    def output_size
+      out = @plan.outputs[[@step_index, @plan.outputs.size - 1].min]
+      return nil unless out
+
+      File.size(out)
+    rescue StandardError
+      nil
+    end
+
+    def summary
+      @plan.outputs.map do |o|
+        size = begin
+          File.size(o)
+        rescue StandardError
+          nil
+        end
+        [File.basename(o), size]
+      end
+    end
+
+    def cancel
+      @cancel = true
+      kill_current
+    end
+
+    # Blocks until the worker is finished, for at most `secs`.
+    def wait(secs = nil) = @thread&.join(secs)
+
+    private
+
+    def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    def kill_current
+      pid = @pid
+      return unless pid
+
+      begin
+        Process.kill("TERM", -pid) # the whole group: tar and its compressor
+      rescue StandardError
+        nil
+      end
+      Thread.new do
+        sleep 2
+        # Once waitpid has reaped it the pid may belong to someone else.
+        next unless @pid == pid
+
+        begin
+          Process.kill("KILL", -pid)
+        rescue StandardError
+          nil
+        end
+      end
+    end
+
+    def push(line)
+      return if line.empty?
+
+      @lock.synchronize do
+        @lines << line
+        @lines.shift(@lines.size - KEEP) if @lines.size > KEEP
+      end
+    end
+
+    def run_all
+      @steps.each_with_index do |(label, argv, verbose, stdout), idx|
+        @step_index = idx
+        @file_count = 0
+        @verbose = verbose
+        # A cancel that lands between steps must not start the next one.
+        return finish(:cancelled) if @cancel
+
+        push("▸ #{label}: #{Plan.show_cmd(argv, stdout)}")
+        status = run_step(argv, stdout)
+        return finish(:cancelled) if @cancel
+
+        unless status&.success?
+          @error = failure_message(label, status)
+          return finish(:failed)
+        end
+        @spawned = nil
+      end
+      finish(:done)
+    rescue StandardError => e
+      @error = "#{e.class}: #{e.message}"
+      finish(:failed)
+    end
+
+    def failure_message(label, status)
+      # status is nil when the command could not be spawned at all; in that
+      # case run_step already recorded the useful message.
+      return @error || "#{label} could not be started" if status.nil?
+
+      if status.signaled?
+        name = Signal.signame(status.termsig) || status.termsig.to_s
+        return "#{label} killed by SIG#{name}"
+      end
+      @exit_status = status.exitstatus
+      "#{label} exited with status #{@exit_status}"
+    end
+
+    def finish(state)
+      cleanup_incomplete if %i[failed cancelled].include?(state)
+      @state = state
+      @finished_at = now
+    end
+
+    # A half-written archive is worse than no archive: it opens, lists a few
+    # members, then fails. Remove it, but only the output of the step that
+    # was actually interrupted. Archives finished by earlier steps are whole
+    # and must survive, and a step that never spawned wrote nothing.
+    def cleanup_incomplete
+      path = @spawned
+      return unless path && File.exist?(path)
+
+      File.unlink(path)
+      push("removed incomplete #{File.basename(path)}")
+    rescue StandardError => e
+      push("could not remove #{File.basename(path)}: #{e.message}")
+    end
+
+    # `stdout` names a file the command's output is the archive for (gzip -c);
+    # otherwise stdout joins stderr in the log.
+    def run_step(argv, stdout = nil)
+      out = @plan.outputs[@step_index]
+      # Every tool here is asked to create the archive, and the confirm
+      # screen has said an existing one will be overwritten. zip would
+      # otherwise update it in place, keeping members that no longer exist.
+      File.unlink(out) if out && File.exist?(out)
+      @spawned = out
+
+      rd, wr = IO.pipe
+      begin
+        pid = Process.spawn(*argv, out: stdout ? [stdout, "wb"] : wr, err: wr, in: File::NULL,
+                                   pgroup: true, chdir: @plan.base)
+      rescue StandardError => e
+        # Nothing was spawned, so nothing will close these for us.
+        rd.close
+        wr.close
+        @error = e.message
+        return nil
+      end
+      wr.close
+      @pid = pid
+      # A cancel that raced the spawn saw no pid to signal; do it now.
+      kill_current if @cancel
+
+      buf = String.new("", encoding: Encoding::UTF_8)
+      begin
+        loop do
+          chunk = begin
+            rd.read_nonblock(16_384)
+          rescue IO::WaitReadable
+            IO.select([rd], nil, nil, 0.25)
+            next
+          rescue EOFError
+            break
+          end
+          # A pipe hands back binary. Filenames are arbitrary bytes, so this
+          # has to be made printable before it can meet the UTF-8 frame.
+          buf << chunk.force_encoding(Encoding::UTF_8).scrub("·")
+          # tar emits newlines, zip rewrites lines with \r.
+          while (m = buf.match(/[\r\n]/))
+            line = buf.slice!(0, m.end(0)).chomp("\n").chomp("\r").rstrip
+            record(line)
+          end
+        end
+      ensure
+        record(buf.rstrip) unless buf.strip.empty?
+        rd.close unless rd.closed?
+      end
+
+      _, status = Process.waitpid2(@pid)
+      @pid = nil
+      status
+    end
+
+    def record(line)
+      return if line.empty?
+
+      case line
+      when /\A\s*(?:adding|updating|deflated|stored):\s*(.+?)(?:\s*\(|\z)/
+        @file_count += 1
+        @current_file = Regexp.last_match(1)
+      when /\A(?:tar|zip|gzip|zstd|xz|bzip2|brotli|lz4):/, /\A\s*(?:total bytes|zip warning)/
+        @current_file = line
+      else
+        # Without -v the only lines are diagnostics, not members.
+        @file_count += 1 if @verbose
+        @current_file = line
+      end
+      push(line)
+    end
+  end
+end