rbe-tebako 0.15.0 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fad34d9072105f46cae716b4de5481d747661dca815b24a935fc8f5cad7036e5
-  data.tar.gz: 9714907bfc2b6e2daf38f691c690e3c522efdbd3625c81594b70494c68d6cd6b
+  metadata.gz: 3d051f3caca43f3080ba1793827fe2390b7a9410ed53cd8c5426856c70592cd9
+  data.tar.gz: 9466ae396ba0171287dd792fb99cc58995aa76b750772b146f92687f862e28e1
 SHA512:
-  metadata.gz: 2ec48c03294d5bc456cf0235f2b06e294d3911c746dadf9fda18aea2c6e857183cf5ca6e9c7c87c992e38a3e75720abdc372f2fd9a95ea9435aa9a19dbd58e12
-  data.tar.gz: 99413e19ef3c98d3fefbddf3476b94c9f136286b8e5ab427af380afe5ae0e6d51e2a10db919110e201c1077a5609d5ffdd7f315aa0bb3243ce364e4c2ae00be9
+  metadata.gz: d3bcdf7fd8770090d25ca313808091f3785c5fbac4015af47a6bd745acaae341efcd9381225eeae1dd1d986cfcb4ad895d92c4215586e338c1127be8a9004c15
+  data.tar.gz: 6151c0a0d960ef89d768a3510400a6bd4ba2fa01b4756e224222e026cc776ad2ddf17978bc9b561a230251c6ffa191ec0833c31cea717cd0db499507b1c705c8

data/CMakeLists.txt

@@ -436,12 +436,23 @@ if (${SETUP_MODE})
 # ...................................................................
 # Packaged filesystem
 
-  add_custom_target(packaged_filesystem
+  file(GLOB_RECURSE TEBAKO_PACKAGER_SOURCES CONFIGURE_DEPENDS
+    ${CMAKE_CURRENT_SOURCE_DIR}/lib/tebako/*.rb
+  )
+
+  add_custom_command(
+    OUTPUT ${DATA_BIN_FILE}
     COMMAND ruby ${DEPS_BIN_DIR}/deploy.rb ${RUBY_VER}
-    DEPENDS setup
-    BYPRODUCTS ${DATA_BIN_FILE}
+    DEPENDS ${DEPS_BIN_DIR}/deploy.rb ${PACKAGE_MANIFEST} ${TEBAKO_PACKAGER_SOURCES}
+    COMMENT "Building packaged filesystem from changed inputs"
+    VERBATIM
   )
 
+  add_custom_target(packaged_filesystem
+    DEPENDS ${DATA_BIN_FILE}
+  )
+  add_dependencies(packaged_filesystem setup)
+
   set(CMAKE_CXX_FLAGS "${RUBY_C_FLAGS}")
 
   add_library(tebako-fs STATIC
@@ -452,6 +463,11 @@ if (${SETUP_MODE})
     ${DEPS_INCLUDE_DIR}/tebako/tebako-version.h
   )
 
+  set_source_files_properties(
+    ${DEPS_SRC_DIR}/tebako/tebako-fs.cpp
+    PROPERTIES OBJECT_DEPENDS ${DATA_BIN_FILE}
+  )
+
   if(${RUBY_VER} VERSION_LESS "3.3.0" AND ("${OSTYPE_TXT}" MATCHES "^msys*" OR "${OSTYPE_TXT}" MATCHES "^cygwin*"))
     target_compile_definitions(tebako-fs PUBLIC RB_W32_PRE_33)
   endif(${RUBY_VER} VERSION_LESS "3.3.0" AND ("${OSTYPE_TXT}" MATCHES "^msys*" OR "${OSTYPE_TXT}" MATCHES "^cygwin*"))

data/README.adoc

@@ -294,7 +294,7 @@ single, performant, executable binary.
 - Packages all files and dependencies into a single binary
 - Supports multiple operating systems and architectures
 - Provides efficient packaging and execution with DwarFS
-- Offers security features like signing on macOS
+- Supports macOS signing with the legacy embedded bundle format
 - Simplifies distribution and deployment
 - Native running speed
 |
@@ -403,8 +403,9 @@ the packaging environment and collects the required dependencies. Subsequent
 invocations are much faster.
 ====
 
-Upon the next invocation, Tebako will use previously created packaging
-environment. The press process itself takes minutes.
+Upon the next invocation, Tebako reuses its packaging environment and
+content-addressed artifacts. An unchanged bundle normally returns after
+verification, and an application-only edit reuses the finalized Ruby runtime.
 
 You can manage setup of packaging environment manually; please refer to
 description of setup and clean commands below.
@@ -416,9 +417,15 @@ tebako press \
   -r|--root=<project-root-folder> \
   [-p|--prefix=<tebako-root-folder>] \
   [-R|--Ruby=<ruby-version>] \
-  [-o|--output=<packaged-file-name>] \
-  [-l|--log-level=<error|warn|debug|trace>] \
-  [-c|--cwd=<package-current-working-directory>] \
+  [-o|--output=<packaged-file-name>] \
+  [-l|--log-level=<error|warn|debug|trace>] \
+  [--compression-level=<0..9>] \
+  [--[no-]deployment-cache] \
+  [--explain] \
+  [--report=json] \
+  [--layer-strategy=<coarse|semantic>] \
+  [--bundle-format=<layered|legacy>] \
+  [-c|--cwd=<package-current-working-directory>] \
   [-D|--devmode] \
   [-P|--patchelf] \
   [-m|--mode=<bundle|both|application|runtime>] \
@@ -470,8 +477,82 @@ and the application package is named according to the `-o` option with the `.teb
 (optional, defaults to `error`)
 logging level for the Tebako built-in memory filesystem driver.
 
-`--cwd=<package-current-working-directory>`::
-(optional)
+`--compression-level=<0..9>`::
+(optional, defaults to `5`)
+DwarFS compression level for the packaged filesystem. Level `0` prioritizes
+packaging speed, while level `9` prioritizes the smallest package size.
+
+When an application has a `Gemfile`, Tebako caches its installed gems under the
+Tebako prefix. Later presses restore the matching dependency set and use
+`bundle check`, running `bundle install` only when gems are missing. The cache
+key includes the lockfile when one exists, plus the Gemfile, gemspec, Ruby ABI,
+platform, Bundler configuration, and native build inputs.
+
+`--deployment-cache`::
+(optional, enabled by default)
+Restores a verified, content-addressed deployment tree when the application,
+lockfile, Ruby ABI, platform, and packaging inputs are unchanged. Use
+`--no-deployment-cache` to force deployment work.
+
+`--explain`::
+(optional)
+Prints why each fast-build stage was rebuilt or reused.
+
+`--report=json`::
+(optional)
+Emits the same build decisions as structured JSON for benchmarks and CI.
+
+`--layer-strategy=<coarse|semantic>`::
+(optional, defaults to `coarse`)
+Controls application-layer planning where the selected runtime supports
+multiple secondary filesystems. The current default runtime uses one
+application capsule because its filesystem wrapper supports one secondary
+mount. The planner remains available for runtimes that advertise wider mount
+support.
+
+`--bundle-format=<layered|legacy>`::
+(optional, defaults to `layered`)
+Selects the internal representation of a one-file `bundle`.
++
+`layered` reuses a verified finalized Ruby runtime and combines it with one
+checksummed, compressed application capsule. Linux and Windows store the
+capsule in a versioned trailer. macOS performs a targeted final link that
+places the capsule in the `__TEBAKO,__app` Mach-O section, allowing the complete
+executable to be signed. Application edits never recompile the Ruby runtime.
+The deployed result is still exactly one executable and does not need Ruby on
+the destination machine.
++
+`legacy` embeds the complete filesystem during the native link. It is slower
+for application edits but remains available for compatibility.
++
+Layered macOS bundles support normal `codesign` replacement and strict
+verification. Sign with Hardened Runtime and Tebako's supplied entitlements:
++
+[source,shell]
+----
+codesign --force --timestamp --options runtime \
+  --entitlements docs/macos-entitlements.plist \
+  --sign "Developer ID Application: YOUR NAME (TEAMID)" \
+  ./my-tebako-app
+
+codesign --verify --deep --strict --verbose=4 ./my-tebako-app
+----
++
+Ruby's YJIT allocates executable memory without `MAP_JIT`, which requires
+`com.apple.security.cs.allow-unsigned-executable-memory` under Hardened Runtime.
+Packaged native gems are extracted as independently ad-hoc-signed `.bundle`
+files, which requires `com.apple.security.cs.disable-library-validation`.
+The supplied link:docs/macos-entitlements.plist[macOS entitlements file]
+contains those two runtime exceptions.
++
+Release CI should then submit a ZIP, DMG, or package containing the signed
+executable to `notarytool`, staple the accepted artifact where that container
+supports stapling, and execute the distributed result. Developer ID signing and
+the Apple notary service require project credentials and therefore remain
+release-CI checks.
+
+`--cwd=<package-current-working-directory>`::
+(optional)
 A folder within Tebako memfs where the packaged application will start. This
 folder should be specified relative to the memfs root.
 +
@@ -542,10 +623,118 @@ tebako press \
   --root='~/projects/myproject' \
   --entry=start.rb \
   --output=/temp/myproject.tebako
-----
-====
-
-==== Setup
+----
+====
+
+===== Incremental presses
+
+Tebako keeps content-addressed DwarFS images under the Tebako prefix. Cache keys
+include file contents and modes, the DwarFS writer, the compression level, and
+package metadata; timestamps alone do not invalidate an image. Generated files
+are updated only when their contents change, and CMake rebuilds or relinks the
+runtime only when the packaged filesystem or native inputs change.
+
+In the default `bundle` path, Tebako reuses an immutable finalized runtime and
+compresses the deployed app, executables, gems, and package metadata into one
+application capsule. The capsule is appended with a versioned, checksummed
+footer, producing exactly one executable. An application-only edit therefore
+does not rebuild the runtime filesystem or relink Ruby.
++
+`both` mode uses the same capsule representation for its separate application
+package. `application` mode continues to emit a monolithic image so it remains
+compatible with previously built runtime executables.
+
+The installed DwarFS writer does not expose a persistent compressed-chunk cache.
+Layer-sized, content-addressed images therefore provide incremental compression
+without changing the DwarFS format itself.
+
+To compare warm, unchanged, and application-changed presses on the same machine,
+run:
+
+[source,sh]
+----
+ruby benchmark/press.rb \
+  --root tests/test-00 \
+  --entry test.rb \
+  --prefix PWD \
+  --mode bundle
+----
+
+The command emits JSON timings, cache-hit counts, native-link reuse, and a
+runtime validation result. Use `--mode bundle` to measure the primary
+single-executable workflow or `--mode both` to measure split runtime/application
+packaging. Keep the mode, Ruby version, compression level, prefix, machine load,
+and application fixed when comparing revisions.
+
+===== Explicit fast-build commands
+
+The runtime and application phases can be invoked independently so application
+edits never enter the runtime build:
+
+[source,sh]
+----
+rbe-tebako runtime build \
+  --Ruby=4.0.6 \
+  --output=build/tebako-runtime \
+  --sdk-output=build/tebako-runtime-sdk.tar.gz
+
+SDK_SHA256="$(cut -d' ' -f1 build/tebako-runtime-sdk.tar.gz.sha256)"
+rbe-tebako runtime install build/tebako-runtime-sdk.tar.gz \
+  --destination=$HOME/.tebako/runtime-sdk \
+  --sha256="$SDK_SHA256"
+
+rbe-tebako application build \
+  --prefix=$HOME/.tebako/runtime-sdk \
+  --runtime=$HOME/.tebako/runtime-sdk/runtime/tebako-runtime \
+  --root=. \
+  --entry-point=bin/my-app \
+  --output=build/my-app \
+  --explain
+----
+
+`runtime build --sdk-output` writes the SDK and a conventional adjacent
+`.sha256` checksum file. Distribute or pin that digest through a trusted channel;
+a checksum downloaded from the same untrusted location as the SDK provides
+corruption detection, not publisher authentication.
++
+SDK installation checks the trusted SHA-256 before extracting anything, verifies
+the archive manifest and individual file checksums, and atomically replaces the
+selected installation. It relocates text-based build configuration from the
+producer's prefix to the installation prefix, registers the packaged runtime in
+the finalized-runtime cache, and includes the prepared Ruby link tree needed for
+fast signable single-file assembly on macOS.
++
+Set `TEBAKO_PREFIX` to the installation root to make ordinary `press` commands
+reuse the SDK runtime without adding SDK-specific arguments:
+
+[source,sh]
+----
+export TEBAKO_PREFIX=$HOME/.tebako/runtime-sdk
+rbe-tebako press \
+  --Ruby=4.0.6 \
+  --root=. \
+  --entry-point=bin/my-app \
+  --output=build/my-app \
+  --explain
+----
+
+The runtime is reused only when its descriptor matches the current platform,
+architecture, Ruby version, Tebako source, compression and logging options, and
+native build environment. A mismatch is safe: Tebako builds and caches a
+compatible runtime instead.
++
+Tebako also exposes cache and package diagnostics:
+
+[source,sh]
+----
+rbe-tebako cache stats
+rbe-tebako cache verify
+rbe-tebako cache prune --max-size=20GB
+rbe-tebako package inspect build/my-app.tebako
+rbe-tebako package verify build/my-app.tebako
+----
+
+==== Setup
 
 Tebako requires post-install setup after gem installation .
 Post-intall setup is called automatically during the first packaging.

data/benchmark/press.rb

@@ -0,0 +1,130 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "open3"
+require "optparse"
+require "rbconfig"
+require "tmpdir"
+
+# Measures warm, unchanged, and application-changed Tebako presses.
+class PressBenchmark
+  def initialize(options)
+    @options = options
+    @repo = File.expand_path("..", __dir__)
+  end
+
+  def run
+    Dir.mktmpdir("tebako-press-benchmark") do |directory|
+      puts JSON.pretty_generate(results_for(directory))
+    end
+  end
+
+  private
+
+  def results_for(directory)
+    root, output = prepare_workspace(directory)
+    results = { mode: @options[:mode], warm: press(root, output), unchanged: press(root, output) }
+    add_probe(root)
+    results[:application_changed] = press(root, output)
+    results[:runtime_validation] = validate(output) unless @options[:skip_run]
+    add_comparisons(results)
+    results
+  end
+
+  def prepare_workspace(directory)
+    root = File.join(directory, "application")
+    FileUtils.cp_r(File.join(File.expand_path(@options[:root]), "."), root)
+    [root, File.join(directory, "tebako-benchmark")]
+  end
+
+  def add_probe(root)
+    timestamp = Process.clock_gettime(Process::CLOCK_REALTIME).to_s
+    File.write(File.join(root, ".tebako-benchmark-probe"), timestamp)
+  end
+
+  def add_comparisons(results)
+    warm = results[:warm][:seconds]
+    results[:comparisons] = {
+      unchanged_vs_warm: ratio(results[:unchanged][:seconds], warm),
+      application_changed_vs_warm: ratio(results[:application_changed][:seconds], warm)
+    }
+  end
+
+  def ratio(value, baseline)
+    (value / baseline).round(3)
+  end
+
+  def press(root, output)
+    command = press_command(root, output)
+    warn "Running #{command.join(" ")}"
+    started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    log, status = Open3.capture2e(*command, chdir: @repo)
+    abort log unless status.success?
+
+    press_result(log, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
+  end
+
+  def press_result(log, elapsed)
+    report = parse_report(log)
+    {
+      seconds: elapsed.round(3),
+      reused_filesystems: log.scan("reusing packaged filesystem").length,
+      reused_native_link: log.include?("reusing Ruby executable"),
+      rebuilt_runtime_filesystem: log.include?("Building packaged filesystem from changed inputs"),
+      stages: report.fetch("events", []).map do |event|
+        event.slice("stage", "status", "duration_seconds", "reason", "details")
+      end
+    }
+  end
+
+  def parse_report(log)
+    line = log.lines.reverse.find { |candidate| candidate.start_with?("{\"schema_version\"") }
+    line ? JSON.parse(line) : {}
+  rescue JSON::ParserError
+    {}
+  end
+
+  def press_command(root, output)
+    [
+      RbConfig.ruby, "-I#{File.join(@repo, "lib")}", File.join(@repo, "exe", "rbe-tebako"),
+      "press", "--mode", @options[:mode], "--prefix", @options[:prefix], "--Ruby", @options[:ruby],
+      "--root", root, "--entry-point", @options[:entry], "--output", output,
+      "--compression-level", @options[:compression].to_s, "--report", "json"
+    ]
+  end
+
+  def validate(output)
+    command = @options[:mode] == "bundle" ? [output] : [output, "--tebako-run", "#{output}.tebako"]
+    log, status = Open3.capture2e(*command)
+    { success: status.success?, output: log }
+  end
+end
+
+options = {
+  root: File.expand_path("../tests/test-00", __dir__),
+  entry: "test.rb",
+  prefix: "PWD",
+  ruby: "4.0.6",
+  compression: 5,
+  mode: "both",
+  skip_run: false
+}
+
+OptionParser.new do |parser|
+  parser.banner = "Usage: benchmark/press.rb [options]"
+  parser.on("--root PATH", "Application root (default: tests/test-00)") { |value| options[:root] = value }
+  parser.on("--entry PATH", "Entry point relative to root (default: test.rb)") { |value| options[:entry] = value }
+  parser.on("--prefix PATH", "Tebako prefix (default: PWD)") { |value| options[:prefix] = value }
+  parser.on("--ruby VERSION", "Packaged Ruby version (default: 4.0.6)") { |value| options[:ruby] = value }
+  parser.on("--mode MODE", %w[bundle both], "Package mode: bundle or both (default: both)") do |value|
+    options[:mode] = value
+  end
+  parser.on("--compression N", Integer, "DwarFS compression level (default: 5)") do |value|
+    options[:compression] = value
+  end
+  parser.on("--skip-run", "Do not execute the resulting package") { options[:skip_run] = true }
+end.parse!
+
+PressBenchmark.new(options).run

data/docs/layered-package-format.md

@@ -0,0 +1,56 @@
+# Tebako layered application package format
+
+The source of truth for fixed fields is
+[`format/layered_package.json`](../format/layered_package.json). Run
+`ruby scripts/generate_layered_format.rb` after changing it.
+
+Version 1.0 uses little-endian integers and ends with the eight-byte magic
+`TEBAKOL1`. The version is encoded in that magic; readers must reject unknown
+magic values rather than guessing a compatible layout.
+
+The package layout is:
+
+1. Tebako application descriptor.
+2. One or more DwarFS layer payloads.
+3. Layer manifest.
+4. Footer.
+
+The footer contains a `uint64` manifest byte length followed by the magic.
+The manifest starts with a `uint32` layer count. Each record contains a
+`uint16` UTF-8 mount-point length, the mount-point bytes, a `uint64` payload
+offset, and a `uint64` payload length.
+
+Mount points are relative, non-empty paths. Empty components and the `.` and
+`..` components are forbidden. Mount points must be unique. Payload ranges
+must be non-empty, must end before the manifest, and must not overlap.
+
+Version 1 does not carry embedded payload checksums. `tebako package verify`
+therefore validates its complete structure, bounds, and mount rules and reports
+computed SHA-256 identities for inspection; it cannot authenticate a version 1
+payload against an embedded digest. A checksum-bearing format requires a new
+magic/version and corresponding runtime reader support.
+
+## One-file executable containers
+
+A layered package is wrapped in a single-file envelope containing the package
+bytes, a little-endian `uint64` package length, a 32-byte SHA-256 digest, and
+the eight-byte `TEBAKOB1` magic.
+
+Linux and Windows place that envelope after the native executable. macOS places
+the same envelope in the `__TEBAKO,__app` section of a dedicated `__TEBAKO`
+Mach-O segment. The section is created during a targeted final link so it is
+part of the signed Mach-O rather than unstructured trailing data. The runtime
+locates the section through the loaded Mach-O header, validates the envelope
+digest, and then parses the layered package.
+
+The macOS link reuses the already-built Ruby objects, static extensions, and
+runtime libraries. It does not compile Ruby or regenerate the runtime
+filesystem. The resulting executable supports signature replacement followed
+by `codesign --verify --deep --strict`.
+
+Hardened Runtime signing must use `docs/macos-entitlements.plist`. Ruby's YJIT
+requires executable anonymous memory, and native gem `.bundle` files are
+extracted with independent ad-hoc signatures. The signing profile therefore
+allows unsigned executable memory and disables library validation. Both pure
+Ruby and native-gem fixtures are exercised with that profile; credentialed
+Developer ID signing and notarization belong in release CI.

data/docs/macos-entitlements.plist

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+  <true/>
+  <key>com.apple.security.cs.disable-library-validation</key>
+  <true/>
+</dict>
+</plist>

data/format/layered_package.json

@@ -0,0 +1,20 @@
+{
+  "name": "Tebako layered application package",
+  "major_version": 1,
+  "minor_version": 0,
+  "magic": "TEBAKOL1",
+  "endianness": "little",
+  "footer": [
+    ["manifest_size", "uint64"],
+    ["magic", "bytes[8]"]
+  ],
+  "manifest_header": [
+    ["layer_count", "uint32"]
+  ],
+  "layer_record": [
+    ["mount_point_size", "uint16"],
+    ["mount_point", "bytes[mount_point_size]"],
+    ["payload_offset", "uint64"],
+    ["payload_size", "uint64"]
+  ]
+}

data/format/single_file_bundle.json

@@ -0,0 +1,12 @@
+{
+  "name": "Tebako single-file layered bundle",
+  "major_version": 1,
+  "minor_version": 0,
+  "magic": "TEBAKOB1",
+  "endianness": "little",
+  "footer": [
+    ["application_size", "uint64"],
+    ["application_sha256", "bytes[32]"],
+    ["magic", "bytes[8]"]
+  ]
+}

data/include/tebako/layered-format.h

@@ -0,0 +1,16 @@
+// Generated by scripts/generate_layered_format.rb. Do not edit.
+#ifndef TEBAKO_LAYERED_FORMAT_H
+#define TEBAKO_LAYERED_FORMAT_H
+
+#include <cstddef>
+#include <cstdint>
+
+namespace tebako::layered_format {
+inline constexpr std::uint16_t major_version = 1;
+inline constexpr std::uint16_t minor_version = 0;
+inline constexpr char magic[] = "TEBAKOL1";
+inline constexpr std::size_t magic_size = sizeof(magic) - 1;
+inline constexpr std::size_t footer_size = sizeof(std::uint64_t) + magic_size;
+}  // namespace tebako::layered_format
+
+#endif

data/include/tebako/single-file-bundle-format.h

@@ -0,0 +1,17 @@
+// Generated by scripts/generate_layered_format.rb. Do not edit.
+#ifndef TEBAKO_SINGLE_FILE_BUNDLE_FORMAT_H
+#define TEBAKO_SINGLE_FILE_BUNDLE_FORMAT_H
+
+#include <cstddef>
+#include <cstdint>
+
+namespace tebako::single_file_bundle_format {
+inline constexpr std::uint16_t major_version = 1;
+inline constexpr std::uint16_t minor_version = 0;
+inline constexpr char magic[] = "TEBAKOB1";
+inline constexpr std::size_t magic_size = sizeof(magic) - 1;
+inline constexpr std::size_t digest_size = 32;
+inline constexpr std::size_t footer_size = sizeof(std::uint64_t) + digest_size + magic_size;
+}  // namespace tebako::single_file_bundle_format
+
+#endif