ibex 0.1.0 → 0.2.0

data/lib/ibex/generation_input.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "digest"
+
+module Ibex
+  # Digest and identity of source bytes actually consumed by one generation.
+  class GenerationInput
+    attr_reader :path #: String
+    attr_reader :sha256 #: String
+    attr_reader :bytesize #: Integer
+    attr_reader :dev #: Integer
+    attr_reader :ino #: Integer
+    attr_reader :access_paths #: Array[String]
+
+    # @rbs (String path, String source, ?access_path: String) -> void
+    def initialize(path, source, access_path: path)
+      @path = File.realpath(path).freeze
+      stat = File.stat(@path)
+      @sha256 = Digest::SHA256.hexdigest(source).freeze
+      @bytesize = source.bytesize
+      @dev = stat.dev
+      @ino = stat.ino
+      @access_paths = [File.expand_path(access_path)] #: Array[String]
+    end
+
+    # Record another lexical path that supplied these canonical bytes.
+    # @rbs (String path) -> void
+    def add_access_path(path)
+      expanded = File.expand_path(path)
+      @access_paths << expanded unless @access_paths.include?(expanded)
+    end
+
+    # @rbs () -> Hash[String, untyped]
+    def to_h
+      { "path" => @path, "sha256" => @sha256, "bytesize" => @bytesize }
+    end
+
+    # Detect content changes even when size and timestamps are preserved.
+    # @rbs () -> bool
+    def current?
+      source = File.binread(@path)
+      stat = File.stat(@path)
+      lexical_paths_current? && stat.dev == @dev && stat.ino == @ino &&
+        source.bytesize == @bytesize && Digest::SHA256.hexdigest(source) == @sha256
+    rescue SystemCallError
+      false
+    end
+
+    private
+
+    # @rbs () -> bool
+    def lexical_paths_current?
+      @access_paths.all? { |path| File.realpath(path) == @path }
+    end
+  end
+end

data/lib/ibex/generation_manifest.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "digest"
+require "json"
+
+module Ibex
+  # Deterministic description of one published parser generation.
+  module GenerationManifest
+    SCHEMA_VERSION = 1 #: Integer
+    IDENTIFIER = "generation" #: String
+    ROOT_KEYS = %w[ibex_manifest schema_version input options artifacts].freeze #: Array[String]
+    INPUT_KEYS = %w[root sha256 files].freeze #: Array[String]
+    FILE_KEYS = %w[path sha256 bytesize].freeze #: Array[String]
+    ARTIFACT_KEYS = %w[kind path sha256 bytesize].freeze #: Array[String]
+
+    module_function
+
+    # @rbs (ArtifactSet artifacts, source_records: Array[GenerationInput], options: Hash[String, untyped]) -> String
+    def render(artifacts, source_records:, options:)
+      files = source_records.map(&:to_h)
+      document = {
+        "ibex_manifest" => IDENTIFIER,
+        "schema_version" => SCHEMA_VERSION,
+        "input" => {
+          "root" => files.fetch(0).fetch("path"),
+          "sha256" => input_digest(files),
+          "files" => files
+        },
+        "options" => sorted_hash(options),
+        "artifacts" => artifacts.map { |artifact| artifact_entry(artifact) }
+      }
+      "#{JSON.pretty_generate(document)}\n"
+    end
+
+    # Validate the manifest shape and, by default, every published artifact digest.
+    # @rbs (String source, ?verify_artifacts: bool) -> Hash[String, untyped]
+    def validate(source, verify_artifacts: true)
+      document = JSON.parse(source)
+      validate_document(document)
+      verify_entries(document.fetch("artifacts")) if verify_artifacts
+      document
+    rescue JSON::ParserError, KeyError, TypeError, ArgumentError => e
+      raise Ibex::Error, "(manifest):1:1: invalid generation manifest: #{e.message}"
+    end
+
+    # @rbs (String path, ?verify_artifacts: bool) -> Hash[String, untyped]
+    def validate_file(path, verify_artifacts: true)
+      validate(File.binread(path), verify_artifacts: verify_artifacts)
+    rescue SystemCallError, ArgumentError => e
+      raise Ibex::Error, "#{path}:1:1: cannot read generation manifest: #{e.message}"
+    end
+
+    # @rbs (Artifact artifact) -> Hash[String, untyped]
+    def artifact_entry(artifact)
+      {
+        "kind" => artifact.kind.to_s,
+        "path" => File.expand_path(artifact.path),
+        "sha256" => Digest::SHA256.hexdigest(artifact.content),
+        "bytesize" => artifact.content.bytesize
+      }
+    end
+    private_class_method :artifact_entry
+
+    # @rbs (Array[Hash[String, untyped]] files) -> String
+    def input_digest(files)
+      digest = Digest::SHA256.new
+      files.each do |entry|
+        digest << entry.fetch("path") << "\0"
+        digest << entry.fetch("sha256") << "\0"
+        digest << entry.fetch("bytesize").to_s << "\0"
+      end
+      digest.hexdigest
+    end
+    private_class_method :input_digest
+
+    # @rbs (Hash[String, untyped] value) -> Hash[String, untyped]
+    def sorted_hash(value)
+      value.keys.sort.to_h do |key|
+        child = value.fetch(key)
+        normalized = child.is_a?(Hash) ? sorted_hash(child) : child
+        [key, normalized]
+      end
+    end
+    private_class_method :sorted_hash
+
+    # @rbs (untyped document) -> void
+    def validate_document(document)
+      raise TypeError, "document must be an object" unless document.is_a?(Hash)
+
+      exact_keys!(document, ROOT_KEYS, "document")
+      validate_identity(document)
+      validate_input(document.fetch("input"))
+      artifacts = document.fetch("artifacts")
+      raise TypeError, "options must be an object" unless document.fetch("options").is_a?(Hash)
+      raise TypeError, "artifacts must be an array" unless artifacts.is_a?(Array)
+      raise TypeError, "artifacts must not be empty" if artifacts.empty?
+
+      validate_entries(artifacts, "artifact", require_kind: true)
+    end
+    private_class_method :validate_document
+
+    # @rbs (Hash[untyped, untyped] document) -> void
+    def validate_identity(document)
+      raise TypeError, "unsupported manifest identifier" unless document["ibex_manifest"] == IDENTIFIER
+      raise TypeError, "unsupported schema_version" unless document["schema_version"] == SCHEMA_VERSION
+    end
+    private_class_method :validate_identity
+
+    # @rbs (untyped input) -> void
+    def validate_input(input)
+      raise TypeError, "input must be an object" unless input.is_a?(Hash)
+
+      exact_keys!(input, INPUT_KEYS, "input")
+
+      files = input.fetch("files")
+      raise TypeError, "input files must be a non-empty array" unless files.is_a?(Array) && !files.empty?
+
+      validate_entries(files, "input file")
+      expected = input_digest(files)
+      root = input.fetch("root")
+      raise TypeError, "input root must be a non-empty string" unless root.is_a?(String) && !root.empty?
+      raise TypeError, "input digest mismatch" unless input.fetch("sha256") == expected
+      raise TypeError, "input root mismatch" unless root == files.fetch(0).fetch("path")
+    end
+    private_class_method :validate_input
+
+    # @rbs (Array[untyped] entries, String label, ?require_kind: bool) -> void
+    def validate_entries(entries, label, require_kind: false)
+      entries.each { |entry| validate_entry(entry, label, require_kind) }
+    end
+    private_class_method :validate_entries
+
+    # @rbs (untyped entry, String label, bool require_kind) -> void
+    def validate_entry(entry, label, require_kind)
+      raise TypeError, "#{label} must be an object" unless entry.is_a?(Hash)
+
+      exact_keys!(entry, require_kind ? ARTIFACT_KEYS : FILE_KEYS, label)
+      validate_path(entry.fetch("path"), label)
+      validate_kind(entry.fetch("kind"), label) if require_kind
+      validate_digest(entry.fetch("sha256"), label)
+      bytesize = entry.fetch("bytesize")
+      return if bytesize.is_a?(Integer) && bytesize >= 0
+
+      raise TypeError, "#{label} bytesize must be non-negative"
+    end
+    private_class_method :validate_entry
+
+    # @rbs (untyped path, String label) -> void
+    def validate_path(path, label)
+      raise TypeError, "#{label} path must be a non-empty string" unless path.is_a?(String) && !path.empty?
+      return unless path.include?("\0") || path.match?(/[[:cntrl:]]/)
+
+      raise TypeError, "#{label} path must not contain control characters"
+    end
+    private_class_method :validate_path
+
+    # @rbs (untyped kind, String label) -> void
+    def validate_kind(kind, label)
+      return if kind.is_a?(String) && !kind.empty?
+
+      raise TypeError, "#{label} kind must be a non-empty string"
+    end
+    private_class_method :validate_kind
+
+    # @rbs (untyped digest, String label) -> void
+    def validate_digest(digest, label)
+      return if digest.is_a?(String) && digest.match?(/\A[0-9a-f]{64}\z/)
+
+      raise TypeError, "#{label} digest must be SHA-256"
+    end
+    private_class_method :validate_digest
+
+    # @rbs (Hash[untyped, untyped] value, Array[String] expected, String label) -> void
+    def exact_keys!(value, expected, label)
+      keys = value.keys
+      raise TypeError, "#{label} keys must be strings" unless keys.all?(String)
+
+      extras = keys - expected
+      missing = expected - keys
+      raise TypeError, "#{label} has unknown property #{extras.fetch(0)}" unless extras.empty?
+      raise KeyError, "#{label} is missing #{missing.fetch(0)}" unless missing.empty?
+    end
+    private_class_method :exact_keys!
+
+    # @rbs (Array[untyped] entries) -> void
+    def verify_entries(entries)
+      entries.each do |entry|
+        path = entry.fetch("path")
+        source = File.binread(path)
+        raise TypeError, "artifact bytesize mismatch for #{path}" unless source.bytesize == entry.fetch("bytesize")
+        raise TypeError, "artifact digest mismatch for #{path}" unless
+          Digest::SHA256.hexdigest(source) == entry.fetch("sha256")
+      end
+    rescue SystemCallError => e
+      raise TypeError, "cannot read artifact: #{e.message}"
+    end
+    private_class_method :verify_entries
+  end
+end

data/lib/ibex/generation_transaction.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "digest"
+require_relative "error"
+require_relative "generation_transaction_recovery"
+require_relative "generation_transaction_validation"
+
+module Ibex
+  # Publishes a rendered ArtifactSet with rollback and a manifest-last marker.
+  class GenerationTransaction
+    include GenerationTransactionRecovery
+    include GenerationTransactionValidation
+
+    class SourceChanged < Ibex::Error; end
+
+    class Error < Ibex::Error
+      attr_reader :rollback_failed #: bool
+      attr_reader :recovery_artifacts #: Array[String]
+
+      # @rbs (String message, ?rollback_failed: bool, ?recovery_artifacts: Array[String]) -> void
+      def initialize(message, rollback_failed: false, recovery_artifacts: [])
+        super(message)
+        @rollback_failed = rollback_failed
+        @recovery_artifacts = recovery_artifacts.freeze
+      end
+    end
+
+    # rubocop:disable Layout/LineLength
+    # @rbs (ArtifactSet artifacts, ?warning: ^(String) -> void, ?stability_check: ^() -> bool, ?source_records: Array[GenerationInput], ?lock_sleeper: ^(Float) -> void) -> void
+    def initialize(artifacts, warning: ->(_message) {}, stability_check: -> { true }, source_records: [],
+                   lock_sleeper: ->(seconds) { sleep(seconds) })
+      @artifacts = artifacts
+      @warning = warning
+      @stability_check = stability_check
+      @source_records = source_records
+      @lock_sleeper = lock_sleeper
+      @locks = [] #: Array[File]
+      @locked_paths = [] #: Array[String]
+      @records = [] #: Array[Hash[Symbol, untyped]]
+      @committed = false
+      @temporary_sequence = 0
+    end
+    # rubocop:enable Layout/LineLength
+
+    # @rbs () -> void
+    def commit
+      @committed = false
+      prepare_records
+      validate_source_boundaries!
+      acquire_locks
+      prepare_records
+      validate_source_boundaries!
+      validate_locked_targets!
+      ensure_sources_stable!("before staging outputs")
+      stage_records
+      ensure_sources_stable!("while staging outputs")
+      backup_records
+      sync_directories!
+      install_group(:non_marker)
+      sync_directories!
+      ensure_sources_stable!("before parser publication")
+      install_group(:parser)
+      sync_directories!
+      ensure_sources_stable!("after parser publication")
+      install_group(:manifest)
+      sync_directories!
+      ensure_sources_stable!("after manifest publication")
+      @committed = true
+      cleanup_success
+    rescue StandardError, SignalException => e
+      @committed ? report_post_commit_failure(e) : handle_failure(e)
+    ensure
+      release_locks
+    end
+
+    private
+
+    # @rbs () -> void
+    def prepare_records
+      @records = @artifacts.map do |artifact|
+        target = resolved_target(artifact.path)
+        {
+          artifact: artifact, target: target, directory: File.dirname(target),
+          stage: nil, backup: nil, backed_up: false, installed: false,
+          existed: File.exist?(target), mode: target_mode(artifact, target)
+        }
+      end
+      grouped = @records.group_by do |record|
+        portable_target_key(record.fetch(:target))
+      end
+      collision = grouped.find { |_key, records| records.length > 1 }
+      return unless collision
+
+      kinds = collision.fetch(1).map { |record| record.fetch(:artifact).kind }.join(", ")
+      targets = collision.fetch(1).map { |record| record.fetch(:target) }.join(", ")
+      raise Error, "(generation):1:1: artifact targets collide: #{targets} (#{kinds})"
+    end
+
+    # @rbs (Artifact artifact, String target) -> Integer
+    def target_mode(artifact, target)
+      return artifact.mode if artifact.mode
+      return File.stat(target).mode & 0o7777 if File.exist?(target)
+
+      mask = File.umask
+      File.umask(mask)
+      0o666 & ~mask
+    end
+
+    # @rbs () -> void
+    def acquire_locks
+      lock_paths = @records.map { |record| lock_path(record.fetch(:target)) }.uniq.sort
+      validate_lock_boundaries!(lock_paths)
+      @locked_paths = lock_paths
+      @locks = [] #: Array[File]
+      lock_paths.each do |path|
+        flags = File::RDWR | File::CREAT
+        flags |= File.const_get(:NOFOLLOW) if File.const_defined?(:NOFOLLOW)
+        lock = File.new(path, flags, 0o600)
+        @locks << lock
+        validate_lock_identity!(path, lock)
+        acquire_lock(lock, path)
+        validate_lock_identity!(path, lock)
+      end
+    end
+
+    # @rbs (File lock, String path) -> void
+    def acquire_lock(lock, path)
+      until lock.flock(File::LOCK_EX | File::LOCK_NB)
+        unless @stability_check.call
+          raise SourceChanged, "(generation):1:1: generation cancelled while waiting for lock: #{path}"
+        end
+
+        @lock_sleeper.call(0.05)
+      end
+    end
+
+    # @rbs (String target) -> String
+    def lock_path(target)
+      digest = Digest::SHA256.hexdigest(target).slice(0, 16)
+      File.join(File.dirname(target), ".ibex-generation-#{digest}.lock")
+    end
+
+    # @rbs () -> void
+    def stage_records
+      @records.each do |record|
+        temporary = create_temporary_file(record.fetch(:directory), ".stage")
+        begin
+          record[:stage] = temporary.path.dup
+          temporary.binmode
+          temporary.write(record.fetch(:artifact).content)
+          temporary.flush
+          temporary.chmod(record.fetch(:mode))
+          temporary.fsync
+        ensure
+          temporary.close
+        end
+      end
+    end
+
+    # @rbs () -> void
+    def backup_records
+      @records.each do |record|
+        target = record.fetch(:target)
+        next unless File.exist?(target)
+
+        validate_existing_target!(target)
+        backup = create_backup_link(target, record.fetch(:directory))
+        record[:backup] = backup
+        record[:backed_up] = true
+      end
+    end
+
+    # @rbs (String directory, String suffix) -> File
+    def create_temporary_file(directory, suffix)
+      with_temporary_path(directory, suffix) do |path|
+        flags = File::WRONLY | File::CREAT | File::EXCL
+        flags |= File.const_get(:NOFOLLOW) if File.const_defined?(:NOFOLLOW)
+        return File.new(path, flags, 0o600)
+      end
+      raise Error, "(generation):1:1: temporary file is unavailable in #{directory}"
+    end
+
+    # @rbs (String target, String directory) -> String
+    def create_backup_link(target, directory)
+      with_temporary_path(directory, ".backup") do |path|
+        File.link(target, path)
+        return path
+      end
+      raise Error, "(generation):1:1: backup path is unavailable in #{directory}"
+    end
+
+    # @rbs (String directory, String suffix) { (String) -> void } -> void
+    def with_temporary_path(directory, suffix)
+      100.times do
+        @temporary_sequence += 1
+        name = ".ibex-generation-#{Process.pid}-#{object_id.to_s(36)}-#{@temporary_sequence.to_s(36)}#{suffix}"
+        yield File.join(directory, name)
+      rescue Errno::EEXIST
+        next
+      end
+    end
+
+    # @rbs (Symbol group) -> void
+    def install_group(group)
+      selected = @records.select { |record| publication_group(record.fetch(:artifact)) == group }
+      selected.sort_by { |record| record.fetch(:target) }.each do |record|
+        File.rename(record.fetch(:stage), record.fetch(:target))
+        record[:stage] = nil
+        record[:installed] = true
+      end
+    end
+
+    # @rbs (Artifact artifact) -> Symbol
+    def publication_group(artifact)
+      return :manifest if artifact.kind == :manifest
+      return :parser if artifact.kind == :parser
+
+      :non_marker
+    end
+
+    # @rbs () -> void
+    def sync_directories!
+      @records.map { |record| record.fetch(:directory) }.uniq.sort.each do |directory|
+        File.open(directory, File::RDONLY, &:fsync)
+      end
+    end
+
+    # @rbs () -> void
+    def cleanup_success
+      failures = cleanup_artifacts(preserve_backups: false)
+      sync_failures = sync_directories_best_effort
+      messages = failures + sync_failures
+      warn_best_effort("generation cleanup incomplete: #{messages.join('; ')}") unless messages.empty?
+    end
+
+    # @rbs (Exception error) -> void
+    def report_post_commit_failure(error)
+      warn_best_effort("generation committed but cleanup failed: #{error.message}")
+    end
+
+    # @rbs (String message) -> void
+    def warn_best_effort(message)
+      @warning.call(message)
+    rescue StandardError
+      nil
+    end
+
+    # @rbs () -> void
+    def release_locks
+      @locks.reverse_each do |lock|
+        lock.flock(File::LOCK_UN)
+        lock.close
+      rescue StandardError
+        nil
+      end
+      @locks = [] #: Array[File]
+      @locked_paths = [] #: Array[String]
+    end
+  end
+end

data/lib/ibex/generation_transaction_recovery.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  # Failure recovery and best-effort cleanup for GenerationTransaction.
+  module GenerationTransactionRecovery
+    private
+
+    # @rbs (Exception cause) -> bot
+    def handle_failure(cause)
+      rollback_failures = rollback
+      cleanup_failures = cleanup_artifacts(preserve_backups: !rollback_failures.empty?)
+      sync_failures = sync_directories_best_effort
+      raise cause if source_change_clean?(cause, rollback_failures, cleanup_failures, sync_failures)
+
+      recovery = recovery_artifacts
+      details = failure_details(cause, rollback_failures, cleanup_failures, sync_failures, recovery)
+      raise GenerationTransaction::Error.new(
+        "(generation):1:1: #{details.join(' | ')}",
+        rollback_failed: !rollback_failures.empty?, recovery_artifacts: recovery
+      ), cause: cause
+    end
+
+    # @rbs (Exception cause, Array[String] rollback, Array[String] cleanup, Array[String] sync) -> bool
+    def source_change_clean?(cause, rollback, cleanup, sync)
+      cause.is_a?(GenerationTransaction::SourceChanged) && rollback.empty? && cleanup.empty? && sync.empty?
+    end
+
+    # rubocop:disable Layout/LineLength
+    # @rbs (Exception cause, Array[String] rollback, Array[String] cleanup, Array[String] sync, Array[String] recovery) -> Array[String]
+    def failure_details(cause, rollback, cleanup, sync, recovery)
+      details = [cause.message]
+      details << "rollback failed: #{rollback.join('; ')}" unless rollback.empty?
+      details << "cleanup failed: #{cleanup.join('; ')}" unless cleanup.empty?
+      details << "directory sync failed: #{sync.join('; ')}" unless sync.empty?
+      details << "recovery artifacts: #{recovery.join(', ')}" unless recovery.empty?
+      details
+    end
+    # rubocop:enable Layout/LineLength
+
+    # @rbs () -> Array[String]
+    def recovery_artifacts
+      @records.filter_map do |record|
+        backup = record[:backup]
+        backup if backup && File.exist?(backup)
+      end
+    end
+
+    # @rbs () -> Array[String]
+    def rollback
+      failures = [] #: Array[String]
+      @records.reverse_each do |record|
+        rollback_record(record)
+      rescue StandardError => e
+        failures << "#{record.fetch(:target)}: #{e.message}"
+      end
+      failures
+    end
+
+    # @rbs (Hash[Symbol, untyped] record) -> void
+    def rollback_record(record)
+      target = record.fetch(:target)
+      if record.fetch(:backed_up)
+        if record.fetch(:installed)
+          File.rename(record.fetch(:backup), target)
+        else
+          File.unlink(record.fetch(:backup))
+        end
+        record[:backup] = nil
+        record[:backed_up] = false
+        record[:installed] = false
+      elsif record.fetch(:installed)
+        File.unlink(target)
+        record[:installed] = false
+      end
+    end
+
+    # @rbs (preserve_backups: bool) -> Array[String]
+    def cleanup_artifacts(preserve_backups:)
+      failures = [] #: Array[String]
+      @records.each do |record|
+        paths = [record[:stage]]
+        paths << record[:backup] unless preserve_backups
+        paths.compact.each { |path| remove_artifact(path, failures) }
+      end
+      failures
+    end
+
+    # @rbs (String path, Array[String] failures) -> void
+    def remove_artifact(path, failures)
+      File.unlink(path)
+    rescue Errno::ENOENT
+      nil
+    rescue StandardError => e
+      failures << "#{path}: #{e.message}"
+    end
+
+    # @rbs () -> Array[String]
+    def sync_directories_best_effort
+      failures = [] #: Array[String]
+      @records.map { |record| record.fetch(:directory) }.uniq.sort.each do |directory|
+        File.open(directory, File::RDONLY, &:fsync)
+      rescue StandardError => e
+        failures << "#{directory}: #{e.message}"
+      end
+      failures
+    end
+  end
+end