@amritk/nish 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +553 -0
- package/bin/launcher.js +186 -0
- package/bin/nish +23 -0
- package/bin/packaging.js +220 -0
- package/docs/AI.md +1076 -0
- package/docs/INSTALL.md +468 -0
- package/llms.txt +49 -0
- package/package.json +87 -0
- package/runtime/nish.d.ts +290 -0
- package/runtime/nish.h +340 -0
- package/runtime/nish.mjs +143 -0
- package/runtime/runtime.c +1184 -0
- package/runtime/runtime_os.c +351 -0
- package/runtime/runtime_parallel.c +156 -0
- package/runtime/runtime_wasm.c +99 -0
- package/runtime/shim.mjs +672 -0
- package/scripts/bootstrap.sh +357 -0
- package/scripts/build.sh +279 -0
- package/scripts/changelog-gen.mjs +528 -0
- package/scripts/changelog-section.sh +28 -0
- package/scripts/ci-profile.mjs +187 -0
- package/scripts/codes-registry.js +73 -0
- package/scripts/gen-diagnostic-codes.mjs +199 -0
- package/scripts/gen-pow5-tables.py +45 -0
- package/scripts/nish-compiler.sh +17 -0
- package/scripts/platform-package.mjs +91 -0
- package/scripts/postinstall.mjs +133 -0
- package/scripts/size-report.sh +72 -0
- package/scripts/smoke.sh +94 -0
- package/scripts/verify-binaries.sh +213 -0
- package/std/README.md +185 -0
- package/std/json.ts +402 -0
- package/std/pair.ts +28 -0
- package/std/testing.ts +347 -0
- package/std/text.ts +193 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Put the native compiler on PATH directly, when this machine got one.
|
|
4
|
+
*
|
|
5
|
+
* `bin/nish` ships as a node shim that resolves the prebuilt binary and spawns
|
|
6
|
+
* it. That works everywhere and costs node's startup on every invocation --
|
|
7
|
+
* 94 ms against the binary's own 2.7 ms, which is nothing for a single build
|
|
8
|
+
* and is 80 seconds across a suite that spawns a compiler per case. So when a
|
|
9
|
+
* `@amritk/nish-<asset>` package did get installed, this replaces the shim
|
|
10
|
+
* with a one-line `exec` of that package's binary: 3.9 ms, and no node.
|
|
11
|
+
*
|
|
12
|
+
* **It execs the binary where it lies; it does not copy it here.** Copying was
|
|
13
|
+
* the first attempt and it is wrong, for a reason worth writing down because
|
|
14
|
+
* nothing in the test suite was looking for it. npm links the command as
|
|
15
|
+
* `node_modules/.bin/nish -> ../@amritk/nish/bin/nish`, so a user always
|
|
16
|
+
* invokes it through a symlink, and the native compiler resolves `build.sh`
|
|
17
|
+
* and `runtime/` from `argv[0]`'s directory without following one. A copy at
|
|
18
|
+
* `@amritk/nish/bin/nish` invoked through `.bin` therefore looks for them in
|
|
19
|
+
* `node_modules/`, finds neither, and every `--link` fails with "cannot find
|
|
20
|
+
* scripts/build.sh" -- while `--version` and plain `-o` keep working, which is
|
|
21
|
+
* what makes it a trap. The node shim never had the problem because node
|
|
22
|
+
* resolves `import.meta.url` to the realpath.
|
|
23
|
+
*
|
|
24
|
+
* Execing an absolute path sidesteps it entirely and is better on its own
|
|
25
|
+
* terms: the binary runs from inside its own package, next to the `runtime/`
|
|
26
|
+
* and `scripts/` that were staged and smoke-tested beside it by `release.yml`,
|
|
27
|
+
* rather than beside the main package's copies.
|
|
28
|
+
*
|
|
29
|
+
* **The compiler resolves the link itself as of 0.6.0** -- the real path of
|
|
30
|
+
* whatever `argv[0]` named is a candidate for the package root
|
|
31
|
+
* ([wp19 §5a](../docs/wp19-stage0-retirement.md) item 4, in `self/`) -- so this
|
|
32
|
+
* `exec` is no longer what makes a copy work, and the reason it is still here is
|
|
33
|
+
* the 91 ms above rather than the defect. It also still carries whoever never
|
|
34
|
+
* ran this script: a shim npm linked and nothing swapped now finds its own
|
|
35
|
+
* package too.
|
|
36
|
+
*
|
|
37
|
+
* **This script may never fail an install.** It exits 0 whatever happens --
|
|
38
|
+
* no prebuilt binary for this platform, a read-only `node_modules`, scripts
|
|
39
|
+
* disabled, a partially written package. Every one of those leaves the shim in
|
|
40
|
+
* place, and the shim is the mechanism this only optimises: where a prebuilt
|
|
41
|
+
* binary exists it finds it, and where none does it says so and exits 3. What
|
|
42
|
+
* it is *not*, since 0.6.0, is a compiler of its own -- `dist/` is no longer in
|
|
43
|
+
* the package -- so "the shim is still there" means the command still behaves
|
|
44
|
+
* correctly, not that it can still compile on a platform with no binary. A
|
|
45
|
+
* postinstall that can break `npm ci` is a worse bug than the startup cost it
|
|
46
|
+
* exists to remove.
|
|
47
|
+
*/
|
|
48
|
+
import fs from "node:fs";
|
|
49
|
+
import path from "node:path";
|
|
50
|
+
import { createRequire } from "node:module";
|
|
51
|
+
import { pathToFileURL } from "node:url";
|
|
52
|
+
|
|
53
|
+
const root = path.resolve(import.meta.dirname, "..");
|
|
54
|
+
|
|
55
|
+
const swap = async () => {
|
|
56
|
+
// A checkout, not an install. The repository is its own package, so `npm ci`
|
|
57
|
+
// here installs this package's own optionalDependencies -- and without this
|
|
58
|
+
// guard, the first `npm ci` after they are published would overwrite the
|
|
59
|
+
// tracked `bin/nish` with a binary and leave the working tree dirty. The
|
|
60
|
+
// landmark is `self/compile.ts`, which is the check `scripts/bootstrap.sh`
|
|
61
|
+
// makes and is not in `files`; it used to be `src/launcher.ts`, which stopped
|
|
62
|
+
// being a landmark when the launcher moved into `bin/` and ships.
|
|
63
|
+
if (fs.existsSync(path.join(root, "self", "compile.ts"))) return "a checkout, so the shim stays";
|
|
64
|
+
|
|
65
|
+
const shim = path.join(root, "bin", "nish");
|
|
66
|
+
if (!fs.existsSync(shim)) return "no bin/nish to replace";
|
|
67
|
+
|
|
68
|
+
const packaging = path.join(root, "bin", "packaging.js");
|
|
69
|
+
if (!fs.existsSync(packaging)) return "bin/packaging.js is missing";
|
|
70
|
+
const { assetFor, platformPackageName } = await import(pathToFileURL(packaging).href);
|
|
71
|
+
const asset = assetFor(process.platform, process.arch);
|
|
72
|
+
if (asset === null) return `no prebuilt binary for ${process.platform}/${process.arch}`;
|
|
73
|
+
|
|
74
|
+
const name = platformPackageName(
|
|
75
|
+
JSON.parse(fs.readFileSync(path.join(root, "package.json"), "utf8")).name,
|
|
76
|
+
asset
|
|
77
|
+
);
|
|
78
|
+
let binary;
|
|
79
|
+
try {
|
|
80
|
+
const manifest = createRequire(import.meta.url).resolve(`${name}/package.json`);
|
|
81
|
+
binary = path.join(path.dirname(manifest), "bin", "nish");
|
|
82
|
+
} catch {
|
|
83
|
+
return `${name} is not installed`;
|
|
84
|
+
}
|
|
85
|
+
if (!fs.existsSync(binary)) return `${name} carries no bin/nish`;
|
|
86
|
+
|
|
87
|
+
// The absolute path is baked in at install time, which is the one thing this
|
|
88
|
+
// gives up: a `node_modules` copied to a different path without a reinstall
|
|
89
|
+
// leaves a shim pointing at nothing. The `-x` test is what turns that from a
|
|
90
|
+
// confusing exec failure into a sentence naming the fix, and `npm rebuild`
|
|
91
|
+
// re-runs this script and rewrites the path.
|
|
92
|
+
//
|
|
93
|
+
// Written beside the target and renamed, so a process running `nish` while
|
|
94
|
+
// this runs never sees a half-written one; rename is atomic within a
|
|
95
|
+
// directory, and the temporary file is cleaned up if it is not.
|
|
96
|
+
//
|
|
97
|
+
// Single quotes, not `JSON.stringify`. JSON escaping is not shell escaping:
|
|
98
|
+
// it leaves `$` and a backtick alone, and inside the double quotes they would
|
|
99
|
+
// have ended up in, the shell would expand them. A home directory with a `$`
|
|
100
|
+
// in it is unusual and entirely legal. In POSIX sh nothing is special inside
|
|
101
|
+
// single quotes, so the only thing to handle is a single quote itself.
|
|
102
|
+
const shellQuote = (value) => `'${value.split("'").join(`'\\''`)}'`;
|
|
103
|
+
const target = shellQuote(binary);
|
|
104
|
+
const script =
|
|
105
|
+
"#!/bin/sh\n" +
|
|
106
|
+
"# Written by scripts/postinstall.mjs. `npm rebuild` regenerates it.\n" +
|
|
107
|
+
`if [ -x ${target} ]; then exec ${target} "$@"; fi\n` +
|
|
108
|
+
// Exit 3 is this project's toolchain code: the compiler could not be run,
|
|
109
|
+
// which is exactly what has happened (docs/wp12-release.md, "Exit codes").
|
|
110
|
+
'echo "nish: the prebuilt compiler is not where it was installed; run \'npm rebuild\' to repoint this" >&2\n' +
|
|
111
|
+
"exit 3\n";
|
|
112
|
+
const temp = `${shim}.${process.pid}.tmp`;
|
|
113
|
+
try {
|
|
114
|
+
fs.writeFileSync(temp, script);
|
|
115
|
+
fs.chmodSync(temp, 0o755);
|
|
116
|
+
fs.renameSync(temp, shim);
|
|
117
|
+
} catch (err) {
|
|
118
|
+
try {
|
|
119
|
+
fs.rmSync(temp, { force: true });
|
|
120
|
+
} catch {
|
|
121
|
+
// Nothing to do about it, and it is not worth a second message.
|
|
122
|
+
}
|
|
123
|
+
throw err;
|
|
124
|
+
}
|
|
125
|
+
return `bin/nish now execs the native compiler in ${name}`;
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
try {
|
|
129
|
+
console.log(`nish: ${await swap()}`);
|
|
130
|
+
} catch (err) {
|
|
131
|
+
// Deliberately not an error: the shim is still there and still works.
|
|
132
|
+
console.log(`nish: keeping the node launcher (${err instanceof Error ? err.message : String(err)})`);
|
|
133
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Before/after binary size report for an Nish module + driver + runtime.
|
|
3
|
+
# scripts/size-report.sh [--markdown] [module.ll] [driver.c]
|
|
4
|
+
#
|
|
5
|
+
# Prints the runtime.c budget row (its -Oz text size) and one row per build
|
|
6
|
+
# profile (debug, speed, size, and wasm when wasm-ld is available). --markdown
|
|
7
|
+
# emits a GitHub-flavoured table, used by CI for the job summary; the default
|
|
8
|
+
# is a plain aligned table. Linux and macOS.
|
|
9
|
+
set -euo pipefail
|
|
10
|
+
cd "$(dirname "$0")/.."
|
|
11
|
+
|
|
12
|
+
format=plain
|
|
13
|
+
args=()
|
|
14
|
+
for a in "$@"; do
|
|
15
|
+
case "$a" in
|
|
16
|
+
--markdown|--md) format=markdown ;;
|
|
17
|
+
-h|--help) sed -n '2,8p' "$0"; exit 0 ;;
|
|
18
|
+
*) args+=("$a") ;;
|
|
19
|
+
esac
|
|
20
|
+
done
|
|
21
|
+
ll=${args[0]:-build/add.ll}
|
|
22
|
+
driver=${args[1]:-examples/main.c}
|
|
23
|
+
mkdir -p build/size
|
|
24
|
+
|
|
25
|
+
bytes() { wc -c < "$1" | tr -d ' '; } # macOS wc pads with spaces
|
|
26
|
+
row() { # row <profile> <bytes> <command>
|
|
27
|
+
if [ "$format" = markdown ]; then
|
|
28
|
+
printf '| `%s` | %s | `%s` |\n' "$1" "$2" "$3"
|
|
29
|
+
else
|
|
30
|
+
printf '%-10s %10s %s\n' "$1" "$2" "$3"
|
|
31
|
+
fi
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if [ "$format" = markdown ]; then
|
|
35
|
+
printf '| Profile | Bytes | Command |\n| --- | ---: | --- |\n'
|
|
36
|
+
else
|
|
37
|
+
printf '%-10s %10s %s\n' PROFILE BYTES COMMAND
|
|
38
|
+
fi
|
|
39
|
+
# The runtime budgets (docs/wp7-runtime.md, "Runtime additions and budget"): each of the two
|
|
40
|
+
# runtime translation units compiled alone at -Oz, the sum of its `.text*` sections, against
|
|
41
|
+
# its own ceiling. The gate is `node tests/run.js budget`; this row is the same number for a
|
|
42
|
+
# reader. It was reported here as the `text` *column* of `size` instead, which also counts
|
|
43
|
+
# `.rodata` and the `.eh_frame` entries the size profile strips -- so this row read 4,696
|
|
44
|
+
# against a 4,096 budget while the code it is about was at 2,775. Read-only data ships too, so
|
|
45
|
+
# it gets its own row rather than being folded in: Ryu's two power-of-five tables are 9,888
|
|
46
|
+
# bytes of it, and section GC keeps them out of any binary that never formats a double (WP15).
|
|
47
|
+
# `.text.unlikely.` is summed with `.text` because a linked binary pays for both, and a ceiling
|
|
48
|
+
# on `.text` alone can be met by moving code to another section instead of by shrinking it.
|
|
49
|
+
text_sum() { # text_sum <object>: every .text* section, summed
|
|
50
|
+
size -A "$1" | awk '$1 ~ /^\.text/ { n += $2 } END { print n + 0 }'
|
|
51
|
+
}
|
|
52
|
+
sect() { size -A "$2" | awk -v s="$1" '$1 == s { print $2 }'; }
|
|
53
|
+
"${CC:-clang}" -Oz -c runtime/runtime.c -o build/size/runtime.o
|
|
54
|
+
"${CC:-clang}" -Oz -c runtime/runtime_os.c -o build/size/runtime_os.o
|
|
55
|
+
row runtime "$(text_sum build/size/runtime.o)" \
|
|
56
|
+
"clang -Oz -c runtime/runtime.c && size -A (.text*; budget 3584)"
|
|
57
|
+
row runtime_os "$(text_sum build/size/runtime_os.o)" \
|
|
58
|
+
"clang -Oz -c runtime/runtime_os.c && size -A (.text*; budget 1280)"
|
|
59
|
+
row rodata "$(sect .rodata build/size/runtime.o)" "the core object's .rodata (no budget; the Ryu tables live here)"
|
|
60
|
+
# `scripts/build.sh` compiles runtime_os.c beside any runtime.c it is handed, so naming the
|
|
61
|
+
# core here builds the whole runtime -- and `--gc-sections` then drops whatever this module
|
|
62
|
+
# never calls, which for `examples/add.ts` is all of it.
|
|
63
|
+
for p in debug speed size; do
|
|
64
|
+
scripts/build.sh "$ll" runtime/runtime.c "$driver" -o "build/size/app-$p" --profile "$p" >/dev/null
|
|
65
|
+
row "$p" "$(bytes "build/size/app-$p")" "scripts/build.sh $ll runtime/runtime.c $driver --profile $p"
|
|
66
|
+
done
|
|
67
|
+
# The wasm profile needs wasm-ld; ask clang where it would look (it checks its own
|
|
68
|
+
# bin dir before PATH). Skip the row when the linker is not installed.
|
|
69
|
+
if [ -x "$("${CC:-clang}" -print-prog-name=wasm-ld)" ]; then
|
|
70
|
+
scripts/build.sh "$ll" -o build/size/app.wasm --profile wasm >/dev/null
|
|
71
|
+
row wasm "$(bytes build/size/app.wasm)" "scripts/build.sh $ll --profile wasm"
|
|
72
|
+
fi
|
package/scripts/smoke.sh
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Smoke test: build every example program with `--link` in the `size` profile,
|
|
3
|
+
# run it, and print a table of binary sizes.
|
|
4
|
+
#
|
|
5
|
+
# scripts/smoke.sh [examples-dir] (default: examples/)
|
|
6
|
+
# npm run smoke (after `npm run build`)
|
|
7
|
+
# NISH=<nish> scripts/smoke.sh with that compiler instead
|
|
8
|
+
#
|
|
9
|
+
# NISH names the compiler, run as scripts/nish-compiler.sh says: a native
|
|
10
|
+
# `nish` directly and a Node entry point under node. Unset, it is build/nish,
|
|
11
|
+
# what `npm run build` leaves.
|
|
12
|
+
#
|
|
13
|
+
# A program is any examples/**/*.ts that declares `export const main`. It
|
|
14
|
+
# is expected to exit 0 unless it carries a `// smoke: exit <n>` comment; a
|
|
15
|
+
# `// smoke: argv <args>` comment passes those arguments on its command line
|
|
16
|
+
# (process.argv). The script exits non-zero if any program fails to compile,
|
|
17
|
+
# link, or run with the expected status. Binaries and IR go to build/smoke/.
|
|
18
|
+
# Needs clang on PATH.
|
|
19
|
+
set -uo pipefail
|
|
20
|
+
cd "$(dirname "$0")/.."
|
|
21
|
+
|
|
22
|
+
examples=${1:-examples}
|
|
23
|
+
out=build/smoke
|
|
24
|
+
mkdir -p "$out"
|
|
25
|
+
|
|
26
|
+
# shellcheck source=scripts/nish-compiler.sh
|
|
27
|
+
. scripts/nish-compiler.sh
|
|
28
|
+
nish_compiler "${NISH:-build/nish}"
|
|
29
|
+
|
|
30
|
+
if ! command -v clang >/dev/null 2>&1 && [ -z "${CC:-}" ]; then
|
|
31
|
+
echo "error: smoke test needs clang on PATH (see docs/INSTALL.md)" >&2
|
|
32
|
+
exit 3
|
|
33
|
+
fi
|
|
34
|
+
|
|
35
|
+
# Either spelling declares the entry (docs/wp22-arrow-functions.md): the arrow
|
|
36
|
+
# `export const main = (...) => ...` is the form the corpus is written in, and
|
|
37
|
+
# `export function main` is still legal.
|
|
38
|
+
#
|
|
39
|
+
# `mapfile` would say this in one line and is a bash 4 builtin: macOS ships
|
|
40
|
+
# bash 3.2 (Apple will not ship GPLv3) and this script runs there too, since
|
|
41
|
+
# WP19 G3 wants the matrix on both operating systems. `while read` over the
|
|
42
|
+
# same pipeline is the portable spelling, and the process substitution keeps
|
|
43
|
+
# the loop out of a subshell so `programs` survives it.
|
|
44
|
+
programs=()
|
|
45
|
+
while IFS= read -r program; do
|
|
46
|
+
programs+=("$program")
|
|
47
|
+
done < <(grep -rl --include='*.ts' -E '^export (function main\b|const main[[:space:]]*=)' "$examples" | sort)
|
|
48
|
+
if [ ${#programs[@]} -eq 0 ]; then
|
|
49
|
+
echo "error: no examples with \`export const main\` under $examples" >&2
|
|
50
|
+
exit 1
|
|
51
|
+
fi
|
|
52
|
+
|
|
53
|
+
failed=0
|
|
54
|
+
rows=()
|
|
55
|
+
for src in "${programs[@]}"; do
|
|
56
|
+
# examples/multi/main.ts -> multi_main; examples/hello.ts -> hello
|
|
57
|
+
name=$(printf '%s' "${src#"$examples"/}" | sed -e 's/\.ts$//' -e 's#/#_#g')
|
|
58
|
+
exe="$out/$name"
|
|
59
|
+
want=$(sed -n 's#^// smoke: exit \([0-9][0-9]*\).*#\1#p' "$src" | head -n 1)
|
|
60
|
+
want=${want:-0}
|
|
61
|
+
# `// smoke: args <flags>` passes extra compiler flags (e.g. --number-mode f64).
|
|
62
|
+
extra=$(sed -n 's#^// smoke: args \(.*\)#\1#p' "$src" | head -n 1)
|
|
63
|
+
# shellcheck disable=SC2086
|
|
64
|
+
if ! "${compiler[@]}" "$src" $extra --link "$exe" --profile size >"$out/$name.log" 2>&1; then
|
|
65
|
+
status="BUILD FAIL"
|
|
66
|
+
failed=1
|
|
67
|
+
sed 's/^/ /' "$out/$name.log" >&2
|
|
68
|
+
rows+=("$(printf '%-24s %10s %s' "$src" "-" "$status")")
|
|
69
|
+
continue
|
|
70
|
+
fi
|
|
71
|
+
bytes=$(wc -c < "$exe" | tr -d ' ') # macOS wc pads with spaces
|
|
72
|
+
|
|
73
|
+
# `// smoke: argv <args>` is the program's command line (WP7 process.argv).
|
|
74
|
+
argv=$(sed -n 's#^// smoke: argv \(.*\)#\1#p' "$src" | head -n 1)
|
|
75
|
+
# shellcheck disable=SC2086
|
|
76
|
+
"$exe" $argv >"$out/$name.out" 2>"$out/$name.err"
|
|
77
|
+
got=$?
|
|
78
|
+
if [ "$got" -eq "$want" ]; then
|
|
79
|
+
status="ok (exit $got)"
|
|
80
|
+
else
|
|
81
|
+
status="RUN FAIL (exit $got, expected $want)"
|
|
82
|
+
failed=1
|
|
83
|
+
sed 's/^/ /' "$out/$name.err" >&2
|
|
84
|
+
fi
|
|
85
|
+
rows+=("$(printf '%-24s %10s %s' "$src" "$bytes" "$status")")
|
|
86
|
+
done
|
|
87
|
+
|
|
88
|
+
printf '%-24s %10s %s\n' PROGRAM BYTES STATUS
|
|
89
|
+
printf '%s\n' "${rows[@]}"
|
|
90
|
+
if [ "$failed" -ne 0 ]; then
|
|
91
|
+
echo "smoke: FAILED" >&2
|
|
92
|
+
exit 1
|
|
93
|
+
fi
|
|
94
|
+
echo "smoke: ${#programs[@]} program(s) built (size profile) and ran"
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# `stage3 == stage2`: the last of the equalities `scripts/bootstrap.sh --verify`
|
|
3
|
+
# asserts, as a script of its own.
|
|
4
|
+
#
|
|
5
|
+
# scripts/verify-binaries.sh <stage2> <stage3>
|
|
6
|
+
#
|
|
7
|
+
# Exit 0 and one or more lines on stdout saying what was compared; exit 1 and
|
|
8
|
+
# the reason on stderr when the two binaries are not the same compiler.
|
|
9
|
+
#
|
|
10
|
+
# It is a file rather than a block inside bootstrap.sh for the reason
|
|
11
|
+
# .github/seed-matrix.sh is one: it is a gate, it has arms that only one
|
|
12
|
+
# operating system reaches, and a gate nothing can run is a gate nothing
|
|
13
|
+
# checks. `tests/run.js` drives this script with fabricated files and
|
|
14
|
+
# `NISH_UNAME_S` set, so every arm is reached from whatever machine the suite
|
|
15
|
+
# runs on. Read the caveat under WHAT THE TESTS PROVE before trusting that
|
|
16
|
+
# sentence further than it goes.
|
|
17
|
+
#
|
|
18
|
+
# THE COMPARISON. Two links of the same input produce the same bytes on ELF,
|
|
19
|
+
# and that is the strongest form of "stage2 reproduces itself": it is asserted
|
|
20
|
+
# there and nothing here relaxes it.
|
|
21
|
+
#
|
|
22
|
+
# On Mach-O they did not, and that was measured rather than expected: on
|
|
23
|
+
# macos-latest (2026-09-13, WP19 R2) stage3 and stage2 differed at identical
|
|
24
|
+
# size -- 597,048 bytes both -- while every IR equality in bootstrap.sh passed.
|
|
25
|
+
# Same size, differing bytes: something small and fixed-width was not
|
|
26
|
+
# reproducible, and it was the linker's rather than the compiler's.
|
|
27
|
+
#
|
|
28
|
+
# WHAT THAT SOMETHING WAS is now measured, and an earlier version of this
|
|
29
|
+
# header first claimed it wrongly and then said, correctly, that nobody had run
|
|
30
|
+
# it on a mac. Somebody has: a throwaway workflow ran release.yml's `binaries`
|
|
31
|
+
# steps on both darwin rows on 2026-09-19 and attributed every differing byte
|
|
32
|
+
# to the load command, section or linkedit blob it falls inside.
|
|
33
|
+
#
|
|
34
|
+
# x86_64-darwin 649,808 bytes both, 16 differing, all at 0x468..0x478:
|
|
35
|
+
# (macos-15-intel) LC_UUID's sixteen bytes of UUID, and nothing else in the
|
|
36
|
+
# file. `otool -l` diffed to the one `uuid` line.
|
|
37
|
+
#
|
|
38
|
+
# aarch64-darwin 646,696 bytes both, 48 differing: the same sixteen at
|
|
39
|
+
# (macos-latest) 0x468, plus 33 in the LC_CODE_SIGNATURE blob at 0x9ca81 --
|
|
40
|
+
# one SHA-256 code-directory slot. ld64 ad-hoc signs arm64
|
|
41
|
+
# (`codesign -dvvv`: adhoc,linker-signed) and that slot is
|
|
42
|
+
# the page hash of the page LC_UUID sits on, so the
|
|
43
|
+
# signature is the consequence and the UUID the cause. The
|
|
44
|
+
# Intel row, which ld64 does not sign at all, is the control
|
|
45
|
+
# for that claim.
|
|
46
|
+
#
|
|
47
|
+
# So LC_UUID it was, as this header had guessed. The remedy it proposed was
|
|
48
|
+
# wrong, and that is measured too: `-Wl,-no_uuid` does make the two files
|
|
49
|
+
# identical, and dyld on arm64 then refuses to load either of them -- `missing
|
|
50
|
+
# LC_UUID load command`, then SIGABRT -- so the stage links and will not start.
|
|
51
|
+
# An unloadable compiler is worse than any reproducibility, and ELF's
|
|
52
|
+
# `--build-id=none` has no Mach-O twin.
|
|
53
|
+
#
|
|
54
|
+
# WHAT SETTLED IT was one more measurement. Three links of one input by one
|
|
55
|
+
# compiler, on both rows -- twice to the same output path, once to a different
|
|
56
|
+
# one -- and what came back is that the UUID is STABLE ACROSS TWO LINKS TO ONE
|
|
57
|
+
# PATH and DIFFERS ON A LINK TO ANOTHER, with nothing else in the file moving.
|
|
58
|
+
# It is not a hash of the output's own content, then, which is what ld64
|
|
59
|
+
# classic did and what would have made any two links of one input agree.
|
|
60
|
+
#
|
|
61
|
+
# The output path is the leading explanation and it is not the only one the
|
|
62
|
+
# probe leaves standing. The two same-path links were invocations 1 and 2 and
|
|
63
|
+
# the differing-path link was invocation 3, and the probe never went back to
|
|
64
|
+
# the first path -- so path-identity and invocation-order are confounded, and
|
|
65
|
+
# anything that had changed by the third link (a coarse clock tick, a
|
|
66
|
+
# per-session counter, an intermediate name derived from the path rather than
|
|
67
|
+
# equal to it) fits the same numbers. A fourth link back to the first path,
|
|
68
|
+
# expected to reproduce the FIRST UUID, is the one line that would tell them
|
|
69
|
+
# apart, and it was not run.
|
|
70
|
+
#
|
|
71
|
+
# The remedy holds either way, which is why this is left as it is rather than
|
|
72
|
+
# guessed at: run 4 measured it end to end, with a real `bootstrap.sh --verify`
|
|
73
|
+
# on both darwin rows, and got byte-identical stages. Whatever LC_UUID is a
|
|
74
|
+
# function of, two stages linked at one path agree.
|
|
75
|
+
#
|
|
76
|
+
# That makes the failure the harness's rather than the toolchain's:
|
|
77
|
+
# `scripts/bootstrap.sh` linked stage2 at `$work/stage2` and stage3 at
|
|
78
|
+
# `$work/stage3`, so two compilers that agreed about every other byte were
|
|
79
|
+
# guaranteed two different UUIDs. It links both at `$work/stage` now and moves
|
|
80
|
+
# each into place afterwards, and both darwin rows reach the `cmp -s` above and
|
|
81
|
+
# stop at it. Nothing below was relaxed to get there.
|
|
82
|
+
#
|
|
83
|
+
# The debug map, which an even earlier version of this header named, was never
|
|
84
|
+
# it: bootstrap.sh defaults to `--profile speed` and never passes `-g`, so
|
|
85
|
+
# there is no DWARF in the .o files for ld64 to build a map from, and build.sh
|
|
86
|
+
# already dropped the local symbols at the link.
|
|
87
|
+
#
|
|
88
|
+
# WHAT IS ASSERTED ON DARWIN is therefore a net under a byte comparison that
|
|
89
|
+
# now holds, rather than the arm that decides a darwin row. It costs nothing
|
|
90
|
+
# while the two files are identical, and it is narrower than raw bytes and
|
|
91
|
+
# wider than nothing when some future toolchain starts varying something else:
|
|
92
|
+
#
|
|
93
|
+
# * the two files are the same SIZE. The measurement above is that they were,
|
|
94
|
+
# exactly, while differing -- so a difference in generated code, which is
|
|
95
|
+
# what this comparison exists to catch, has to keep the byte count to get
|
|
96
|
+
# past it. This is the assertion doing the work.
|
|
97
|
+
# * they are identical once debug information is stripped, where a tool on
|
|
98
|
+
# PATH can strip it. At `--profile speed` that is expected to remove
|
|
99
|
+
# nothing, per the debug-map paragraph above; it is kept because it costs
|
|
100
|
+
# nothing and because `--profile debug` does put DWARF in the .o files.
|
|
101
|
+
#
|
|
102
|
+
# A pair that is the same size and still differs after that is reported as
|
|
103
|
+
# **unattributed** and fails. It fails because the alternative is a blanket
|
|
104
|
+
# exemption that would accept a stage3 that is a different compiler, which is
|
|
105
|
+
# the one thing this comparison is for; and it says "unattributed" rather than
|
|
106
|
+
# "the code differs" because the one thing ld64 was measured to vary is already
|
|
107
|
+
# accounted for by the shared link path, so what is left is unknown rather than
|
|
108
|
+
# damning. The message carries the next step.
|
|
109
|
+
#
|
|
110
|
+
# WHAT THE TESTS PROVE. `tests/run.js` reaches every branch here, and for the
|
|
111
|
+
# Darwin branches it does so two ways: with ELF pairs, which establish the
|
|
112
|
+
# control flow only -- `NISH_UNAME_S=Darwin` changes which branch runs, not
|
|
113
|
+
# what format the files are -- and with a fabricated minimal Mach-O pair
|
|
114
|
+
# differing only in LC_UUID, which is the real format and the real field. That
|
|
115
|
+
# second one is what says the unattributed arm is reachable with a benign pair,
|
|
116
|
+
# and it is asserted as a known limitation rather than as a pass. It stays a
|
|
117
|
+
# limitation of this script, and it has stopped describing what a bootstrap
|
|
118
|
+
# produces: the stages still carry an LC_UUID each, and bootstrap.sh links them
|
|
119
|
+
# at one path, so the two agree and there is no such pair to forgive.
|
|
120
|
+
set -euo pipefail
|
|
121
|
+
|
|
122
|
+
if [ $# -ne 2 ]; then
|
|
123
|
+
echo "usage: scripts/verify-binaries.sh <stage2> <stage3>" >&2
|
|
124
|
+
exit 2
|
|
125
|
+
fi
|
|
126
|
+
a="$1"
|
|
127
|
+
b="$2"
|
|
128
|
+
for f in "$a" "$b"; do
|
|
129
|
+
[ -f "$f" ] || { echo "verify-binaries: $f does not exist" >&2; exit 2; }
|
|
130
|
+
done
|
|
131
|
+
|
|
132
|
+
# Overridden by tests/run.js so both platforms' branches can be driven from one
|
|
133
|
+
# machine. Announced when set, because a release build that had this in its
|
|
134
|
+
# environment would assert a different equality and the log should say so.
|
|
135
|
+
uname_s="$(uname -s)"
|
|
136
|
+
if [ -n "${NISH_UNAME_S:-}" ] && [ "${NISH_UNAME_S}" != "$uname_s" ]; then
|
|
137
|
+
echo "verify-binaries: NISH_UNAME_S=$NISH_UNAME_S overrides the real platform ($uname_s); this is a test hook" >&2
|
|
138
|
+
uname_s="$NISH_UNAME_S"
|
|
139
|
+
fi
|
|
140
|
+
|
|
141
|
+
if cmp -s "$a" "$b"; then
|
|
142
|
+
echo "stage3 == stage2: byte-identical binaries"
|
|
143
|
+
exit 0
|
|
144
|
+
fi
|
|
145
|
+
|
|
146
|
+
if [ "$uname_s" != "Darwin" ]; then
|
|
147
|
+
echo "bootstrap: stage3 is not byte-identical to stage2" >&2
|
|
148
|
+
exit 1
|
|
149
|
+
fi
|
|
150
|
+
|
|
151
|
+
size_a=$(wc -c < "$a" | tr -d ' ')
|
|
152
|
+
size_b=$(wc -c < "$b" | tr -d ' ')
|
|
153
|
+
if [ "$size_a" != "$size_b" ]; then
|
|
154
|
+
{
|
|
155
|
+
echo "bootstrap: stage3 differs from stage2 and the two are not even the same size"
|
|
156
|
+
echo "bootstrap: ($size_a vs $size_b bytes). Whatever is not reproducible about ld64 is"
|
|
157
|
+
echo "bootstrap: small and fixed-width -- the size is not it. This is a difference in"
|
|
158
|
+
echo "bootstrap: what the compiler emitted (docs/wp10-ci.md#ci-matrix)."
|
|
159
|
+
} >&2
|
|
160
|
+
exit 1
|
|
161
|
+
fi
|
|
162
|
+
|
|
163
|
+
# A scratch directory of its own, so this never writes next to the binaries it
|
|
164
|
+
# was handed -- bootstrap.sh's work directory is also where the next stage gets
|
|
165
|
+
# built, and tests/run.js hands it files in a fixture it then compares.
|
|
166
|
+
tmp="$(mktemp -d)"
|
|
167
|
+
trap 'rm -rf "$tmp"' EXIT
|
|
168
|
+
stripped=0
|
|
169
|
+
for tool in llvm-objcopy objcopy; do
|
|
170
|
+
if command -v "$tool" >/dev/null 2>&1 &&
|
|
171
|
+
"$tool" --strip-debug "$a" "$tmp/a" 2>/dev/null &&
|
|
172
|
+
"$tool" --strip-debug "$b" "$tmp/b" 2>/dev/null; then
|
|
173
|
+
stripped=1
|
|
174
|
+
break
|
|
175
|
+
fi
|
|
176
|
+
done
|
|
177
|
+
if [ "$stripped" -eq 0 ] && command -v strip >/dev/null 2>&1; then
|
|
178
|
+
cp "$a" "$tmp/a"
|
|
179
|
+
cp "$b" "$tmp/b"
|
|
180
|
+
if strip -x "$tmp/a" 2>/dev/null && strip -x "$tmp/b" 2>/dev/null; then
|
|
181
|
+
stripped=1
|
|
182
|
+
fi
|
|
183
|
+
fi
|
|
184
|
+
|
|
185
|
+
if [ "$stripped" -eq 1 ] && cmp -s "$tmp/a" "$tmp/b"; then
|
|
186
|
+
echo "stage3 == stage2: same size ($size_a bytes) and identical once debug information"
|
|
187
|
+
echo "was stripped; the raw bytes differ, which is ld64 and not the compiler"
|
|
188
|
+
exit 0
|
|
189
|
+
fi
|
|
190
|
+
|
|
191
|
+
{
|
|
192
|
+
echo "bootstrap: stage3 and stage2 are the same size ($size_a bytes) and still differ."
|
|
193
|
+
if [ "$stripped" -eq 1 ]; then
|
|
194
|
+
echo "bootstrap: Stripping debug information did not account for it -- which at"
|
|
195
|
+
echo "bootstrap: --profile speed is expected, because nothing put DWARF in the .o files"
|
|
196
|
+
echo "bootstrap: and build.sh already linked with -Wl,-x."
|
|
197
|
+
else
|
|
198
|
+
echo "bootstrap: No llvm-objcopy, objcopy or strip on PATH could strip either file, so"
|
|
199
|
+
echo "bootstrap: that half of the comparison did not run."
|
|
200
|
+
fi
|
|
201
|
+
echo "bootstrap:"
|
|
202
|
+
echo "bootstrap: This difference is UNATTRIBUTED. It is a failure rather than a warning,"
|
|
203
|
+
echo "bootstrap: because the alternative accepts a stage3 that is a different compiler."
|
|
204
|
+
echo "bootstrap: The one thing ld64 was measured to vary here is LC_UUID -- and on arm64"
|
|
205
|
+
echo "bootstrap: the code-directory slot that hashes the page it sits on. It was stable"
|
|
206
|
+
echo "bootstrap: across two links to one path, which is why scripts/bootstrap.sh links"
|
|
207
|
+
echo "bootstrap: every comparable stage at one path, and that was measured to hold end to"
|
|
208
|
+
echo "bootstrap: end. So either these two were not built that way, which \`otool -l\` will"
|
|
209
|
+
echo "bootstrap: show by disagreeing about the uuid, or ld64 is varying something new."
|
|
210
|
+
echo "bootstrap: Attribute the differing bytes before changing anything, the way the"
|
|
211
|
+
echo "bootstrap: first measurement was made (docs/wp10-ci.md#ci-matrix)."
|
|
212
|
+
} >&2
|
|
213
|
+
exit 1
|