@scriptc/compiler 0.0.33 → 0.0.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/cc-cache.test.js +1091 -7
- package/dist/backend/cc-cache.test.js.map +1 -1
- package/dist/backend/cc.d.ts +109 -18
- package/dist/backend/cc.js +1057 -289
- package/dist/backend/cc.js.map +1 -1
- package/dist/backend/emission/emit-exprs.js +26 -1
- package/dist/backend/emission/emit-exprs.js.map +1 -1
- package/dist/backend/emission/emit-shapes.js +37 -4
- package/dist/backend/emission/emit-shapes.js.map +1 -1
- package/dist/backend/emission/emitter.d.ts +11 -2
- package/dist/backend/emission/emitter.js +11 -5
- package/dist/backend/emission/emitter.js.map +1 -1
- package/dist/backend/library-identity.d.ts +22 -0
- package/dist/backend/library-identity.js +103 -0
- package/dist/backend/library-identity.js.map +1 -0
- package/dist/backend/llvm/emitter.d.ts +3 -0
- package/dist/backend/llvm/emitter.js +35 -5
- package/dist/backend/llvm/emitter.js.map +1 -1
- package/dist/backend/llvm/shapes.d.ts +1 -0
- package/dist/backend/llvm/shapes.js +25 -1
- package/dist/backend/llvm/shapes.js.map +1 -1
- package/dist/backend/llvm/split.d.ts +43 -0
- package/dist/backend/llvm/split.js +227 -0
- package/dist/backend/llvm/split.js.map +1 -0
- package/dist/backend/llvm/split.test.d.ts +1 -0
- package/dist/backend/llvm/split.test.js +102 -0
- package/dist/backend/llvm/split.test.js.map +1 -0
- package/dist/backend/mangle.d.ts +1 -0
- package/dist/backend/mangle.js +3 -0
- package/dist/backend/mangle.js.map +1 -1
- package/dist/backend/object-localize.d.ts +5 -5
- package/dist/backend/object-localize.js +19 -16
- package/dist/backend/object-localize.js.map +1 -1
- package/dist/backend/object-localize.test.js +57 -13
- package/dist/backend/object-localize.test.js.map +1 -1
- package/dist/executable/early-cache.d.ts +86 -0
- package/dist/executable/early-cache.js +460 -0
- package/dist/executable/early-cache.js.map +1 -0
- package/dist/executable/early-cache.test.d.ts +1 -0
- package/dist/executable/early-cache.test.js +300 -0
- package/dist/executable/early-cache.test.js.map +1 -0
- package/dist/ffi/profile.d.ts +7 -2
- package/dist/ffi/profile.js +1 -0
- package/dist/ffi/profile.js.map +1 -1
- package/dist/frontend/input-tracker.d.ts +82 -0
- package/dist/frontend/input-tracker.js +380 -0
- package/dist/frontend/input-tracker.js.map +1 -0
- package/dist/frontend/input-tracker.test.d.ts +1 -0
- package/dist/frontend/input-tracker.test.js +137 -0
- package/dist/frontend/input-tracker.test.js.map +1 -0
- package/dist/frontend/lowering/lower-builtins.js +4 -10
- package/dist/frontend/lowering/lower-builtins.js.map +1 -1
- package/dist/frontend/lowering/lower-calls.js +169 -144
- package/dist/frontend/lowering/lower-calls.js.map +1 -1
- package/dist/frontend/lowering/lower-classes.js +3 -0
- package/dist/frontend/lowering/lower-classes.js.map +1 -1
- package/dist/frontend/lowering/lower-containers.js +66 -32
- package/dist/frontend/lowering/lower-containers.js.map +1 -1
- package/dist/frontend/lowering/lower-exprs.js +89 -11
- package/dist/frontend/lowering/lower-exprs.js.map +1 -1
- package/dist/frontend/lowering/lower-inspect.js +45 -26
- package/dist/frontend/lowering/lower-inspect.js.map +1 -1
- package/dist/frontend/lowering/lower-island.js +3 -3
- package/dist/frontend/lowering/lower-island.js.map +1 -1
- package/dist/frontend/lowering/lower-mixins.js +1 -1
- package/dist/frontend/lowering/lower-mixins.js.map +1 -1
- package/dist/frontend/lowering/lower-modules.js +1 -1
- package/dist/frontend/lowering/lower-modules.js.map +1 -1
- package/dist/frontend/lowering/lower-stmts.js +49 -4
- package/dist/frontend/lowering/lower-stmts.js.map +1 -1
- package/dist/frontend/lowering/lowerer.d.ts +73 -29
- package/dist/frontend/lowering/lowerer.js +588 -116
- package/dist/frontend/lowering/lowerer.js.map +1 -1
- package/dist/frontend/lowering/surfaces.js +1 -1
- package/dist/frontend/lowering/surfaces.js.map +1 -1
- package/dist/frontend/npm-static-rewrite.js +25 -26
- package/dist/frontend/npm-static-rewrite.js.map +1 -1
- package/dist/frontend/npm-static.js +29 -30
- package/dist/frontend/npm-static.js.map +1 -1
- package/dist/frontend/npm-static.test.d.ts +1 -0
- package/dist/frontend/npm-static.test.js +15 -0
- package/dist/frontend/npm-static.test.js.map +1 -0
- package/dist/frontend/npm.js +5 -17
- package/dist/frontend/npm.js.map +1 -1
- package/dist/frontend/program.js +44 -19
- package/dist/frontend/program.js.map +1 -1
- package/dist/frontend/resolve.js +8 -29
- package/dist/frontend/resolve.js.map +1 -1
- package/dist/frontend/ts7/checker.d.ts +40 -2
- package/dist/frontend/ts7/checker.js +194 -32
- package/dist/frontend/ts7/checker.js.map +1 -1
- package/dist/frontend/ts7/program.js +17 -18
- package/dist/frontend/ts7/program.js.map +1 -1
- package/dist/frontend/types.d.ts +33 -0
- package/dist/frontend/types.js +134 -45
- package/dist/frontend/types.js.map +1 -1
- package/dist/index.d.ts +5 -2
- package/dist/index.js +458 -141
- package/dist/index.js.map +1 -1
- package/dist/ir/nodes.d.ts +27 -4
- package/dist/ir/nodes.js +28 -0
- package/dist/ir/nodes.js.map +1 -1
- package/dist/ir/serialize.d.ts +1 -1
- package/dist/ir/serialize.js +1 -1
- package/dist/ir/validate.js +37 -1
- package/dist/ir/validate.js.map +1 -1
- package/dist/library/early-cache.d.ts +65 -0
- package/dist/library/early-cache.js +361 -0
- package/dist/library/early-cache.js.map +1 -0
- package/dist/library/early-cache.test.d.ts +1 -0
- package/dist/library/early-cache.test.js +528 -0
- package/dist/library/early-cache.test.js.map +1 -0
- package/dist/library/implementation-identity.d.ts +23 -0
- package/dist/library/implementation-identity.js +84 -0
- package/dist/library/implementation-identity.js.map +1 -0
- package/dist/library/int-infer.js +14 -0
- package/dist/library/int-infer.js.map +1 -1
- package/dist/library/int-infer.test.js +21 -4
- package/dist/library/int-infer.test.js.map +1 -1
- package/dist/library/semantic-source.d.ts +13 -0
- package/dist/library/semantic-source.js +257 -0
- package/dist/library/semantic-source.js.map +1 -0
- package/dist/library/semantic-source.test.d.ts +1 -0
- package/dist/library/semantic-source.test.js +113 -0
- package/dist/library/semantic-source.test.js.map +1 -0
- package/dist/library/sidecar.d.ts +6 -0
- package/dist/library/sidecar.js +11 -0
- package/dist/library/sidecar.js.map +1 -1
- package/dist/startup-cache.d.ts +6 -0
- package/dist/startup-cache.js +6 -0
- package/dist/startup-cache.js.map +1 -0
- package/package.json +3 -2
- package/surface-manifest.json +1 -1
package/dist/backend/cc.js
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import { execFile, spawnSync } from "node:child_process";
|
|
2
|
-
import { createHash } from "node:crypto";
|
|
2
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
3
3
|
import { constants as fsConstants, existsSync, readdirSync } from "node:fs";
|
|
4
4
|
import { createRequire } from "node:module";
|
|
5
5
|
import { access, chmod, copyFile, link, lstat, mkdir, mkdtemp, readdir, readFile, realpath, rename, rm, stat, unlink, utimes, writeFile } from "node:fs/promises";
|
|
6
6
|
import { availableParallelism, homedir, tmpdir } from "node:os";
|
|
7
7
|
import { basename, delimiter, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
8
|
+
import { performance } from "node:perf_hooks";
|
|
9
|
+
import { fileURLToPath } from "node:url";
|
|
8
10
|
import { promisify } from "node:util";
|
|
9
11
|
import { localizeElfObject, mergeAndLocalizeCoffObjects } from "./object-localize.js";
|
|
10
12
|
const execFileAsync = promisify(execFile);
|
|
13
|
+
const CC_IMPLEMENTATION_PATH = fileURLToPath(import.meta.url);
|
|
11
14
|
/** Test lanes run against one immutable checkout and one immutable toolchain
|
|
12
15
|
* for the lifetime of each Vitest worker. Production deliberately rediscovers
|
|
13
16
|
* compiler/linker inputs on every invocation, but doing that thousands of
|
|
@@ -118,17 +121,6 @@ const MUTABLE_COMPILE_ENV_KEYS = [
|
|
|
118
121
|
"CCC_ADD_ARGS",
|
|
119
122
|
"CLANG_CONFIG_FILE_SYSTEM_DIR",
|
|
120
123
|
"CLANG_CONFIG_FILE_USER_DIR",
|
|
121
|
-
// The direct runtime build does not consume the conventional Make/CMake
|
|
122
|
-
// variables, but the lazily-built QuickJS prerequisite does. Their values
|
|
123
|
-
// can name mutable tools, headers, libraries, and toolchain files, so a
|
|
124
|
-
// stable spelling is not a safe persistent-cache identity.
|
|
125
|
-
"CC",
|
|
126
|
-
"CFLAGS",
|
|
127
|
-
"CPPFLAGS",
|
|
128
|
-
"LDFLAGS",
|
|
129
|
-
"AR",
|
|
130
|
-
"RANLIB",
|
|
131
|
-
"CMAKE_TOOLCHAIN_FILE",
|
|
132
124
|
// `zig cc` resolves its bundled headers/runtime through ZIG_LIB_DIR and a
|
|
133
125
|
// caller-selected native libc description through ZIG_LIBC. Both values name
|
|
134
126
|
// mutable compiler inputs whose contents can change behind a stable path.
|
|
@@ -155,6 +147,41 @@ export function toolchainEnvironmentFingerprint(env = process.env) {
|
|
|
155
147
|
}
|
|
156
148
|
return hash.digest("hex");
|
|
157
149
|
}
|
|
150
|
+
/** Inputs that can change which native tool/runtime implementation an
|
|
151
|
+
* executable build selects before compileC has a chance to rediscover it.
|
|
152
|
+
* The early whole-program cache keys this exact posture before restoring a
|
|
153
|
+
* final binary; compileC retains its deeper inode/content validation. */
|
|
154
|
+
export async function executableNativeEnvironmentFingerprint(env = process.env) {
|
|
155
|
+
const configuredCompiler = env["SCRIPTC_CC"] ?? "";
|
|
156
|
+
let compilerIdentity;
|
|
157
|
+
try {
|
|
158
|
+
compilerIdentity = await effectiveCompilerEnvironmentIdentity(resolveCc(env), env);
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
// A failed trace cannot safely describe a reusable native posture. Keep
|
|
162
|
+
// the build working, but make this invocation miss every persistent early
|
|
163
|
+
// entry so compileC performs its full discovery and validation.
|
|
164
|
+
compilerIdentity = `<unavailable:${configuredCompiler}:${randomUUID()}>`;
|
|
165
|
+
}
|
|
166
|
+
const hash = createHash("sha256")
|
|
167
|
+
.update("executable-native-environment-v2\0")
|
|
168
|
+
.update(toolchainEnvironmentFingerprint(env)).update("\0")
|
|
169
|
+
// PATH text alone is not a resolution proof, and on Darwin /usr/bin/clang
|
|
170
|
+
// is a stable shim whose selected Xcode compiler can change underneath it.
|
|
171
|
+
// Re-resolve and trace the effective driver on every early lookup.
|
|
172
|
+
.update(compilerIdentity).update("\0");
|
|
173
|
+
for (const name of [
|
|
174
|
+
"PATH",
|
|
175
|
+
"SCRIPTC_FETCH_CURL",
|
|
176
|
+
"SCRIPTC_TEST_RUNTIME_SRC_DIR",
|
|
177
|
+
"SCRIPTC_TEST_VENDOR_CACHE_DIR",
|
|
178
|
+
"SCRIPTC_TEST_TRUST_COMPILER_WRAPPER",
|
|
179
|
+
]) {
|
|
180
|
+
const value = env[name];
|
|
181
|
+
hash.update(name).update(value === undefined ? "\0unset\0" : "\0set\0").update(value ?? "").update("\0");
|
|
182
|
+
}
|
|
183
|
+
return hash.digest("hex");
|
|
184
|
+
}
|
|
158
185
|
/** Structured compiler-driver failure. Most callers still let this surface
|
|
159
186
|
* as an internal build error; the TypeScript compiler pipeline recognizes it
|
|
160
187
|
* when an outbound FFI profile is active and turns user-controlled native
|
|
@@ -505,14 +532,19 @@ export function cacheTargetIdentity(driver, hostPlatform = process.platform, hos
|
|
|
505
532
|
function vendorEngineDir() {
|
|
506
533
|
return join(runtimeSrcDir(), "..", "vendor", "quickjs-ng");
|
|
507
534
|
}
|
|
508
|
-
/** Vendor prerequisites
|
|
509
|
-
*
|
|
510
|
-
*
|
|
511
|
-
|
|
535
|
+
/** Vendor prerequisites follow the per-user build root when persistent cache
|
|
536
|
+
* validation succeeds. Keeping generated objects out of node_modules makes
|
|
537
|
+
* npm installations, read-only package stores, and explicit cache warming all
|
|
538
|
+
* share one writable cache. The test-only override remains the first choice;
|
|
539
|
+
* callers without a validated root retain the historical package-local
|
|
540
|
+
* default (ordinary builds give those callers a private transient root). */
|
|
541
|
+
function vendorBuildCacheRoot(buildRoot) {
|
|
512
542
|
const testRoot = process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"];
|
|
513
|
-
|
|
543
|
+
if (testRoot !== undefined)
|
|
544
|
+
return resolve(testRoot);
|
|
545
|
+
return buildRoot === undefined
|
|
514
546
|
? join(runtimeSrcDir(), "..", "vendor", ".cache")
|
|
515
|
-
:
|
|
547
|
+
: join(buildRoot, "vendor");
|
|
516
548
|
}
|
|
517
549
|
/** Cross targets already carry their architecture in the explicit triple.
|
|
518
550
|
* Host-native prerequisites need the same separation: a checkout/package can
|
|
@@ -521,27 +553,30 @@ export function vendorCacheTargetFlavor(driver, hostPlatform = process.platform,
|
|
|
521
553
|
return driver.target === null ? `native-${hostPlatform}-${hostArch}` : driver.target;
|
|
522
554
|
}
|
|
523
555
|
/** Vendor prerequisites live outside the content-addressed build root, so
|
|
524
|
-
* their directory name must carry the
|
|
525
|
-
* identity
|
|
526
|
-
*
|
|
527
|
-
|
|
556
|
+
* their directory name must carry the environment, resolved compiler, owned
|
|
557
|
+
* source bytes, and build-recipe identity of the artifacts that consume them.
|
|
558
|
+
* The short digest keeps the cache path readable while preventing build-order
|
|
559
|
+
* and cross-installation contamination. */
|
|
560
|
+
export function vendorCacheBuildIdentity(environmentFingerprint, compilerIdentity, nativeSourceIdentity) {
|
|
528
561
|
return createHash("sha256")
|
|
529
|
-
.update("vendor-toolchain-
|
|
562
|
+
.update("vendor-toolchain-v2\0")
|
|
530
563
|
.update(environmentFingerprint)
|
|
531
564
|
.update("\0")
|
|
532
565
|
.update(compilerIdentity)
|
|
566
|
+
.update("\0")
|
|
567
|
+
.update(nativeSourceIdentity)
|
|
533
568
|
.digest("hex")
|
|
534
569
|
.slice(0, 20);
|
|
535
570
|
}
|
|
536
571
|
const vendorCompilerIdentityFallbacks = new Map();
|
|
537
572
|
async function currentVendorCacheBuildIdentity(driver, environmentFingerprint) {
|
|
538
|
-
// Native vendor recipes additionally use bare clang/ar
|
|
573
|
+
// Native vendor recipes additionally use bare clang/ar; cross
|
|
539
574
|
// recipes use the zig driver for compilation and `zig ar`. Include every
|
|
540
|
-
// executable that can affect the
|
|
575
|
+
// executable that can affect the cached prerequisite, not just the
|
|
541
576
|
// final program's driver.
|
|
542
577
|
const commands = [
|
|
543
578
|
driver.argv[0] ?? "clang",
|
|
544
|
-
...(driver.target === null ? ["clang", "
|
|
579
|
+
...(driver.target === null ? ["clang", "ar"] : []),
|
|
545
580
|
].filter((command, index, all) => all.indexOf(command) === index);
|
|
546
581
|
const identities = await Promise.all(commands.map(async (command) => {
|
|
547
582
|
const spellingKey = `${environmentFingerprint}\0${command}`;
|
|
@@ -555,7 +590,7 @@ async function currentVendorCacheBuildIdentity(driver, environmentFingerprint) {
|
|
|
555
590
|
`<unresolved>\0${command}`,
|
|
556
591
|
].join("\0");
|
|
557
592
|
}));
|
|
558
|
-
return vendorCacheBuildIdentity(environmentFingerprint, identities.join("\x1f"));
|
|
593
|
+
return vendorCacheBuildIdentity(environmentFingerprint, identities.join("\x1f"), await runtimeFingerprint(runtimeSrcDir()));
|
|
559
594
|
}
|
|
560
595
|
function engineArchivePath(sanitize, driver, buildIdentity, cacheRoot = vendorBuildCacheRoot()) {
|
|
561
596
|
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
|
@@ -564,6 +599,18 @@ function engineArchivePath(sanitize, driver, buildIdentity, cacheRoot = vendorBu
|
|
|
564
599
|
async function fileExists(path) {
|
|
565
600
|
return stat(path).then((s) => s.isFile(), () => false);
|
|
566
601
|
}
|
|
602
|
+
/** Vendor prerequisites are shared persistent cache entries, so they carry
|
|
603
|
+
* the same adjacent content digest as runtime objects and complete artifacts.
|
|
604
|
+
* Missing/invalid pairs are rebuilt. Do not remove them during validation: a
|
|
605
|
+
* concurrent publisher installs data before its digest, and that temporary
|
|
606
|
+
* invalid window must not let a reader unlink the publisher's data. */
|
|
607
|
+
async function validVendorArtifact(path) {
|
|
608
|
+
return validCachedFile(path);
|
|
609
|
+
}
|
|
610
|
+
function protectCachedArtifact(paths, path) {
|
|
611
|
+
paths?.add(path);
|
|
612
|
+
paths?.add(cacheDigestPath(path));
|
|
613
|
+
}
|
|
567
614
|
function privateSiblingPath(destination, label) {
|
|
568
615
|
// Keep the temporary component independent of the caller's basename. A
|
|
569
616
|
// destination can validly consume the filesystem's entire NAME_MAX budget;
|
|
@@ -586,8 +633,73 @@ async function installArtifact(source, destination, mode) {
|
|
|
586
633
|
await rm(tmp, { force: true }).catch(() => undefined);
|
|
587
634
|
}
|
|
588
635
|
}
|
|
636
|
+
/** Publish one finished vendor prerequisite from disposable build scratch.
|
|
637
|
+
* The shared data and digest use the ordinary atomic cache publisher.
|
|
638
|
+
* Concurrent builders produce equivalent bytes; on platforms where rename
|
|
639
|
+
* cannot replace the winner, accept only a checksum-verified winner. */
|
|
640
|
+
async function publishVendorArtifact(source, destination) {
|
|
641
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
642
|
+
try {
|
|
643
|
+
await publishCachedFile(source, destination);
|
|
644
|
+
}
|
|
645
|
+
catch (error) {
|
|
646
|
+
// A concurrent equivalent publisher may have won on a platform where
|
|
647
|
+
// rename cannot replace an existing file. Accept only its verified pair.
|
|
648
|
+
if (await validCachedFile(destination))
|
|
649
|
+
return;
|
|
650
|
+
if (attempt > 0)
|
|
651
|
+
throw error;
|
|
652
|
+
// Otherwise the existing pair is genuinely stale/corrupt and blocked
|
|
653
|
+
// replacement (notably on Windows). Remove it only after a complete
|
|
654
|
+
// replacement is ready in private scratch, then retry once.
|
|
655
|
+
await Promise.all([
|
|
656
|
+
rm(destination, { force: true }).catch(() => undefined),
|
|
657
|
+
rm(cacheDigestPath(destination), { force: true }).catch(() => undefined),
|
|
658
|
+
]);
|
|
659
|
+
continue;
|
|
660
|
+
}
|
|
661
|
+
// Close publication races where another writer replaced only one member
|
|
662
|
+
// of the pair between this publisher's two atomic renames.
|
|
663
|
+
if (await validCachedFile(destination))
|
|
664
|
+
return;
|
|
665
|
+
}
|
|
666
|
+
throw new Error(`vendor cache publication failed integrity validation: ${destination}`);
|
|
667
|
+
}
|
|
668
|
+
/** Pin cache-backed vendor inputs under an invocation-private directory before
|
|
669
|
+
* linking/archiving. The shared LRU may unlink their cache names at any time;
|
|
670
|
+
* hard links (or copies where links are unavailable) keep active inputs alive.
|
|
671
|
+
* A sweep can race the initial pin, so rematerialize and retry once. */
|
|
672
|
+
async function stageVendorInputs(materialize, stageDir) {
|
|
673
|
+
let lastError = new Error("vendor cache input disappeared while staging");
|
|
674
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
675
|
+
await rm(stageDir, { recursive: true, force: true }).catch(() => undefined);
|
|
676
|
+
await mkdir(stageDir, { recursive: true });
|
|
677
|
+
const sources = await materialize();
|
|
678
|
+
try {
|
|
679
|
+
const now = new Date();
|
|
680
|
+
return await Promise.all(sources.map(async (source) => {
|
|
681
|
+
const destination = join(stageDir, basename(source));
|
|
682
|
+
try {
|
|
683
|
+
await link(source, destination);
|
|
684
|
+
}
|
|
685
|
+
catch {
|
|
686
|
+
await copyFile(source, destination);
|
|
687
|
+
}
|
|
688
|
+
// Vendor prerequisites share the cache root's mtime-based LRU. A
|
|
689
|
+
// successful stage is a cache read, so promote the shared source name
|
|
690
|
+
// best-effort (it may have raced an eviction after the hard link).
|
|
691
|
+
await utimes(source, now, now).catch(() => undefined);
|
|
692
|
+
return destination;
|
|
693
|
+
}));
|
|
694
|
+
}
|
|
695
|
+
catch (error) {
|
|
696
|
+
lastError = error;
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
throw lastError;
|
|
700
|
+
}
|
|
589
701
|
/** The engine archive for one flavor, built lazily on the first --dynamic
|
|
590
|
-
* compile
|
|
702
|
+
* compile and cached under <build-cache>/vendor/<commit>-<flavor>-<target>-<toolchain>/ — unlike
|
|
591
703
|
* the runtime's own sources (recompiled every build, ~100ms), the engine is
|
|
592
704
|
* far too big to rebuild per compile. The plain flavor is ALWAYS MinSizeRel
|
|
593
705
|
* regardless of the program's optimization level: measured binary delta is
|
|
@@ -599,67 +711,24 @@ async function installArtifact(source, destination, mode) {
|
|
|
599
711
|
* private temp dir and publish with an atomic rename — first one wins,
|
|
600
712
|
* losers discard their work and use the winner's archive.
|
|
601
713
|
*
|
|
602
|
-
*
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
714
|
+
* The qjs library target is just four TUs (QJS_ENGINE_SOURCES), so both host
|
|
715
|
+
* and cross builds use the same direct per-TU recipe. This removes CMake from
|
|
716
|
+
* the runtime dependency set and lets cache warming compile exactly the
|
|
717
|
+
* archive a later program consumes. Host builds use clang + ar; cross builds
|
|
718
|
+
* use the selected zig driver + zig ar. */
|
|
607
719
|
async function ensureEngineArchive(sanitize, driver, buildIdentity, cacheRoot = vendorBuildCacheRoot()) {
|
|
608
|
-
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
|
609
|
-
const vendor = vendorEngineDir();
|
|
610
720
|
const archive = engineArchivePath(sanitize, driver, buildIdentity, cacheRoot);
|
|
611
721
|
const cacheDir = dirname(archive);
|
|
612
|
-
if (await
|
|
722
|
+
if (await validVendorArtifact(archive))
|
|
613
723
|
return archive;
|
|
614
|
-
|
|
615
|
-
return buildEngineArchiveCross(sanitize, driver, cacheRoot, cacheDir);
|
|
616
|
-
await mkdir(cacheRoot, { recursive: true });
|
|
617
|
-
const buildDir = await mkdtemp(join(cacheRoot, `build-${flavor}-`));
|
|
618
|
-
const stageDir = `${buildDir}.stage`;
|
|
619
|
-
try {
|
|
620
|
-
const configureArgs = [
|
|
621
|
-
"-S", vendor,
|
|
622
|
-
"-B", buildDir,
|
|
623
|
-
...(sanitize
|
|
624
|
-
? ["-DCMAKE_BUILD_TYPE=Debug", "-DQJS_ENABLE_ASAN=ON"]
|
|
625
|
-
: ["-DCMAKE_BUILD_TYPE=MinSizeRel"]),
|
|
626
|
-
"-DQJS_BUILD_EXAMPLES=OFF",
|
|
627
|
-
"-DQJS_ENABLE_INSTALL=OFF",
|
|
628
|
-
];
|
|
629
|
-
try {
|
|
630
|
-
await execFileAsync("cmake", configureArgs);
|
|
631
|
-
}
|
|
632
|
-
catch (err) {
|
|
633
|
-
if (err.code === "ENOENT") {
|
|
634
|
-
throw new Error("--dynamic builds the embedded engine once with CMake, which was not found on PATH.\n" +
|
|
635
|
-
"Install cmake (e.g. `brew install cmake`) and retry. Static builds do not need it.");
|
|
636
|
-
}
|
|
637
|
-
throw new Error(`cmake failed configuring the embedded engine (${vendor}).\n\n` +
|
|
638
|
-
subprocessFailureDetail(err));
|
|
639
|
-
}
|
|
640
|
-
await execFileAsync("cmake", [
|
|
641
|
-
"--build", buildDir,
|
|
642
|
-
"--target", "qjs",
|
|
643
|
-
`-j${availableParallelism()}`,
|
|
644
|
-
]);
|
|
645
|
-
// Publish only the archive, atomically: rename fails if a concurrent
|
|
646
|
-
// build won the race, and the winner's archive is just as good.
|
|
647
|
-
await mkdir(stageDir);
|
|
648
|
-
await copyFile(join(buildDir, "libqjs.a"), join(stageDir, "libqjs.a"));
|
|
649
|
-
await rename(stageDir, cacheDir).catch(() => undefined);
|
|
650
|
-
}
|
|
651
|
-
finally {
|
|
652
|
-
await rm(buildDir, { recursive: true, force: true });
|
|
653
|
-
await rm(stageDir, { recursive: true, force: true });
|
|
654
|
-
}
|
|
655
|
-
return archive;
|
|
724
|
+
return buildEngineArchiveDirect(sanitize, driver, cacheRoot, cacheDir);
|
|
656
725
|
}
|
|
657
726
|
/** The qjs library target's source list (CMakeLists.txt `qjs_sources` with
|
|
658
727
|
* QJS_BUILD_LIBC off — cutils is header-only): the cross-build recipe
|
|
659
728
|
* compiles exactly these. */
|
|
660
729
|
const QJS_ENGINE_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"];
|
|
661
|
-
/** The engine archive for
|
|
662
|
-
*
|
|
730
|
+
/** The engine archive for one target, per-TU compiler invocations plus ar.
|
|
731
|
+
* The flags mirror what the former CMake
|
|
663
732
|
* configurations apply to the qjs library target: gnu11 (CMAKE_C_EXTENSIONS
|
|
664
733
|
* ON), hidden visibility, -funsigned-char, -DQUICKJS_NG_BUILD and
|
|
665
734
|
* -D_GNU_SOURCE (qjs_defines; linux targetArgs already carry the latter),
|
|
@@ -667,10 +736,11 @@ const QJS_ENGINE_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"
|
|
|
667
736
|
* (-O0 -ggdb -fno-omit-frame-pointer -fsanitize=address
|
|
668
737
|
* -fno-sanitize-recover=all) for asan. Same atomic-rename publish as every
|
|
669
738
|
* vendor cache. */
|
|
670
|
-
async function
|
|
739
|
+
async function buildEngineArchiveDirect(sanitize, driver, cacheRoot, cacheDir) {
|
|
671
740
|
const vendor = vendorEngineDir();
|
|
672
741
|
const archive = join(cacheDir, "libqjs.a");
|
|
673
|
-
const
|
|
742
|
+
const compileArgv = driver.target === null ? ["clang"] : driver.argv;
|
|
743
|
+
const arArgv = driver.target === null ? ["ar"] : [...driver.argv.slice(0, 1), "ar"];
|
|
674
744
|
const cflags = [
|
|
675
745
|
"-std=gnu11",
|
|
676
746
|
...driver.targetArgs,
|
|
@@ -687,21 +757,14 @@ async function buildEngineArchiveCross(sanitize, driver, cacheRoot, cacheDir) {
|
|
|
687
757
|
"-I", vendor,
|
|
688
758
|
];
|
|
689
759
|
await mkdir(cacheRoot, { recursive: true });
|
|
690
|
-
const buildDir = await mkdtemp(join(
|
|
760
|
+
const buildDir = await mkdtemp(join(tmpdir(), `scriptc-vendor-qjs-${driver.target ?? "host"}-`));
|
|
691
761
|
try {
|
|
692
762
|
const width = Math.min(QJS_ENGINE_SOURCES.length, availableParallelism());
|
|
693
763
|
for (let i = 0; i < QJS_ENGINE_SOURCES.length; i += width) {
|
|
694
|
-
await Promise.all(QJS_ENGINE_SOURCES.slice(i, i + width).map((src) => execFileAsync(
|
|
764
|
+
await Promise.all(QJS_ENGINE_SOURCES.slice(i, i + width).map((src) => execFileAsync(compileArgv[0] ?? "clang", [...compileArgv.slice(1), ...cflags, "-c", join(vendor, src), "-o", join(buildDir, `${basename(src, ".c")}.o`)], { cwd: buildDir })));
|
|
695
765
|
}
|
|
696
766
|
await execFileAsync(arArgv[0] ?? "ar", [...arArgv.slice(1), "rcs", join(buildDir, "libqjs.a"), ...QJS_ENGINE_SOURCES.map((s) => join(buildDir, `${basename(s, ".c")}.o`))]);
|
|
697
|
-
|
|
698
|
-
// the winner's archive is just as good.
|
|
699
|
-
const stageDir = `${buildDir}.stage`;
|
|
700
|
-
await mkdir(stageDir);
|
|
701
|
-
await copyFile(join(buildDir, "libqjs.a"), join(stageDir, "libqjs.a"));
|
|
702
|
-
await rename(stageDir, cacheDir).catch(async () => {
|
|
703
|
-
await rm(stageDir, { recursive: true, force: true });
|
|
704
|
-
});
|
|
767
|
+
await publishVendorArtifact(join(buildDir, "libqjs.a"), archive);
|
|
705
768
|
}
|
|
706
769
|
finally {
|
|
707
770
|
await rm(buildDir, { recursive: true, force: true });
|
|
@@ -722,7 +785,7 @@ function lreObjectPaths(sanitize, driver, buildIdentity, cacheRoot = vendorBuild
|
|
|
722
785
|
}
|
|
723
786
|
/** The libregexp objects for one flavor, compiled lazily on the first
|
|
724
787
|
* regex-using static build (~1s) and cached like the engine archive —
|
|
725
|
-
* vendor
|
|
788
|
+
* <build-cache>/vendor/<commit>-lre-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
|
|
726
789
|
* publish (parallel first builds race safely; losers discard their work).
|
|
727
790
|
* Plain is -Os (size class matters: the regex fence pins regex-using
|
|
728
791
|
* binaries well under the engine class); asan matches the final link so
|
|
@@ -741,11 +804,10 @@ async function ensureLreObjects(sanitize, driver, buildIdentity, cacheRoot = ven
|
|
|
741
804
|
`-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
|
742
805
|
const vendor = vendorEngineDir();
|
|
743
806
|
const objects = lreObjectPaths(sanitize, driver, buildIdentity, cacheRoot);
|
|
744
|
-
|
|
745
|
-
if ((await Promise.all(objects.map(fileExists))).every(Boolean))
|
|
807
|
+
if ((await Promise.all(objects.map(validVendorArtifact))).every(Boolean))
|
|
746
808
|
return objects;
|
|
747
809
|
await mkdir(cacheRoot, { recursive: true });
|
|
748
|
-
const buildDir = await mkdtemp(join(
|
|
810
|
+
const buildDir = await mkdtemp(join(tmpdir(), `scriptc-vendor-lre-${flavor}-`));
|
|
749
811
|
try {
|
|
750
812
|
// One -c per invocation: zig's COFF driver rejects multiple -c inputs
|
|
751
813
|
// in a single command ("coff does not support linking multiple
|
|
@@ -762,9 +824,7 @@ async function ensureLreObjects(sanitize, driver, buildIdentity, cacheRoot = ven
|
|
|
762
824
|
"-o", join(buildDir, f.replace(/\.c$/, ".o")),
|
|
763
825
|
], { cwd: buildDir });
|
|
764
826
|
}
|
|
765
|
-
|
|
766
|
-
// the winner's objects are just as good.
|
|
767
|
-
await rename(buildDir, cacheDir).catch(() => undefined);
|
|
827
|
+
await Promise.all(objects.map((destination) => publishVendorArtifact(join(buildDir, basename(destination)), destination)));
|
|
768
828
|
}
|
|
769
829
|
finally {
|
|
770
830
|
await rm(buildDir, { recursive: true, force: true });
|
|
@@ -788,7 +848,7 @@ function zlibObjectPaths(sanitize, driver, buildIdentity, cacheRoot = vendorBuil
|
|
|
788
848
|
}
|
|
789
849
|
/** The zlib objects for one flavor, compiled lazily on the first zlib-using
|
|
790
850
|
* CROSS build (~1s) and cached like the lre objects —
|
|
791
|
-
* vendor
|
|
851
|
+
* <build-cache>/vendor/zlib-<version>-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
|
|
792
852
|
* publish (parallel first builds race safely; losers discard their work).
|
|
793
853
|
* Plain is -Os, asan matches the final link so the sanitized lane
|
|
794
854
|
* instruments the codec too. The flavor keys the driver and target exactly
|
|
@@ -801,11 +861,10 @@ async function ensureZlibObjects(sanitize, driver, buildIdentity, cacheRoot = ve
|
|
|
801
861
|
`-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
|
802
862
|
const vendor = vendorZlibDir();
|
|
803
863
|
const objects = zlibObjectPaths(sanitize, driver, buildIdentity, cacheRoot);
|
|
804
|
-
|
|
805
|
-
if ((await Promise.all(objects.map(fileExists))).every(Boolean))
|
|
864
|
+
if ((await Promise.all(objects.map(validVendorArtifact))).every(Boolean))
|
|
806
865
|
return objects;
|
|
807
866
|
await mkdir(cacheRoot, { recursive: true });
|
|
808
|
-
const buildDir = await mkdtemp(join(
|
|
867
|
+
const buildDir = await mkdtemp(join(tmpdir(), `scriptc-vendor-zlib-${flavor}-`));
|
|
809
868
|
try {
|
|
810
869
|
// One -c per invocation, the lre recipe (zig's COFF driver rejects
|
|
811
870
|
// multiple -c inputs in a single command).
|
|
@@ -820,9 +879,7 @@ async function ensureZlibObjects(sanitize, driver, buildIdentity, cacheRoot = ve
|
|
|
820
879
|
"-o", join(buildDir, f.replace(/\.c$/, ".o")),
|
|
821
880
|
], { cwd: buildDir });
|
|
822
881
|
}
|
|
823
|
-
|
|
824
|
-
// the winner's objects are just as good.
|
|
825
|
-
await rename(buildDir, cacheDir).catch(() => undefined);
|
|
882
|
+
await Promise.all(objects.map((destination) => publishVendorArtifact(join(buildDir, basename(destination)), destination)));
|
|
826
883
|
}
|
|
827
884
|
finally {
|
|
828
885
|
await rm(buildDir, { recursive: true, force: true });
|
|
@@ -849,7 +906,7 @@ function curlStubDirPath(driver, buildIdentity, cacheRoot = vendorBuildCacheRoot
|
|
|
849
906
|
}
|
|
850
907
|
/** The libcurl import stub for one CROSS target, generated lazily on the
|
|
851
908
|
* first fetch-using cross build and cached like the zlib objects —
|
|
852
|
-
* vendor
|
|
909
|
+
* <build-cache>/vendor/curl-stub-<target>-<toolchain>/libcurl.so, atomic-rename publish. The
|
|
853
910
|
* stub is empty definitions of CURL_STUB_SYMBOLS compiled `-shared` with
|
|
854
911
|
* `-soname libcurl.so.4`: linking against it satisfies scr_fetch_curl.c's
|
|
855
912
|
* references and records a plain `DT_NEEDED libcurl.so.4`, which the
|
|
@@ -861,10 +918,10 @@ function curlStubDirPath(driver, buildIdentity, cacheRoot = vendorBuildCacheRoot
|
|
|
861
918
|
async function ensureCurlStub(driver, buildIdentity, cacheRoot = vendorBuildCacheRoot()) {
|
|
862
919
|
const cacheDir = curlStubDirPath(driver, buildIdentity, cacheRoot);
|
|
863
920
|
const lib = join(cacheDir, "libcurl.so");
|
|
864
|
-
if (await
|
|
921
|
+
if (await validVendorArtifact(lib))
|
|
865
922
|
return cacheDir;
|
|
866
923
|
await mkdir(cacheRoot, { recursive: true });
|
|
867
|
-
const buildDir = await mkdtemp(join(
|
|
924
|
+
const buildDir = await mkdtemp(join(tmpdir(), "scriptc-vendor-curl-stub-"));
|
|
868
925
|
try {
|
|
869
926
|
const src = join(buildDir, "stub.c");
|
|
870
927
|
await writeFile(src, CURL_STUB_SYMBOLS.map((s) => `void ${s}(void) {}\n`).join(""));
|
|
@@ -876,14 +933,7 @@ async function ensureCurlStub(driver, buildIdentity, cacheRoot = vendorBuildCach
|
|
|
876
933
|
src,
|
|
877
934
|
"-o", join(buildDir, "libcurl.so"),
|
|
878
935
|
]);
|
|
879
|
-
|
|
880
|
-
// the winner's stub is just as good.
|
|
881
|
-
const stageDir = `${buildDir}.stage`;
|
|
882
|
-
await mkdir(stageDir);
|
|
883
|
-
await copyFile(join(buildDir, "libcurl.so"), join(stageDir, "libcurl.so"));
|
|
884
|
-
await rename(stageDir, cacheDir).catch(async () => {
|
|
885
|
-
await rm(stageDir, { recursive: true, force: true });
|
|
886
|
-
});
|
|
936
|
+
await publishVendorArtifact(join(buildDir, "libcurl.so"), lib);
|
|
887
937
|
}
|
|
888
938
|
finally {
|
|
889
939
|
await rm(buildDir, { recursive: true, force: true });
|
|
@@ -899,7 +949,7 @@ function tlsArchivePath(sanitize, driver, buildIdentity, cacheRoot = vendorBuild
|
|
|
899
949
|
}
|
|
900
950
|
/** The mbedTLS archive for one flavor, compiled lazily on the first
|
|
901
951
|
* TLS-using build (~15s over ~110 TUs, parallelized) and cached like the
|
|
902
|
-
* engine archive — vendor
|
|
952
|
+
* engine archive — <build-cache>/vendor/mbedtls-<version>-<flavor>-<target>-<toolchain>/libmbedtls.a —
|
|
903
953
|
* with the same atomic-rename publish (parallel first builds race safely;
|
|
904
954
|
* losers discard their work). Plain is -Os (the TLS stack is link-gated
|
|
905
955
|
* but still wants its size class small); asan matches the final link so
|
|
@@ -918,13 +968,14 @@ async function ensureTlsArchive(sanitize, driver, buildIdentity, cacheRoot = ven
|
|
|
918
968
|
const arArgv = driver.target !== null ? [...driver.argv.slice(0, 1), "ar"] : ["ar"];
|
|
919
969
|
const vendor = vendorTlsDir();
|
|
920
970
|
const archive = tlsArchivePath(sanitize, driver, buildIdentity, cacheRoot);
|
|
921
|
-
|
|
922
|
-
if (await fileExists(archive))
|
|
971
|
+
if (await validVendorArtifact(archive))
|
|
923
972
|
return archive;
|
|
924
973
|
await mkdir(cacheRoot, { recursive: true });
|
|
925
|
-
const buildDir = await mkdtemp(join(
|
|
974
|
+
const buildDir = await mkdtemp(join(tmpdir(), `scriptc-vendor-mbedtls-${flavor}-`));
|
|
926
975
|
try {
|
|
927
|
-
const sources = (await readdir(join(vendor, "library")))
|
|
976
|
+
const sources = (await readdir(join(vendor, "library")))
|
|
977
|
+
.filter((name) => !name.startsWith(".") && name.endsWith(".c"))
|
|
978
|
+
.sort();
|
|
928
979
|
const cflags = [
|
|
929
980
|
"-std=c11",
|
|
930
981
|
...driver.targetArgs,
|
|
@@ -937,14 +988,7 @@ async function ensureTlsArchive(sanitize, driver, buildIdentity, cacheRoot = ven
|
|
|
937
988
|
await Promise.all(sources.slice(i, i + width).map((src) => execFileAsync(compileArgv[0] ?? "clang", [...compileArgv.slice(1), ...cflags, "-c", join(vendor, "library", src), "-o", join(buildDir, `${basename(src, ".c")}.o`)])));
|
|
938
989
|
}
|
|
939
990
|
await execFileAsync(arArgv[0] ?? "ar", [...arArgv.slice(1), "rcs", join(buildDir, "libmbedtls.a"), ...sources.map((s) => join(buildDir, `${basename(s, ".c")}.o`))]);
|
|
940
|
-
|
|
941
|
-
// the winner's archive is just as good.
|
|
942
|
-
const stageDir = `${buildDir}.stage`;
|
|
943
|
-
await mkdir(stageDir);
|
|
944
|
-
await copyFile(join(buildDir, "libmbedtls.a"), join(stageDir, "libmbedtls.a"));
|
|
945
|
-
await rename(stageDir, cacheDir).catch(async () => {
|
|
946
|
-
await rm(stageDir, { recursive: true, force: true });
|
|
947
|
-
});
|
|
991
|
+
await publishVendorArtifact(join(buildDir, "libmbedtls.a"), archive);
|
|
948
992
|
}
|
|
949
993
|
finally {
|
|
950
994
|
await rm(buildDir, { recursive: true, force: true });
|
|
@@ -958,6 +1002,11 @@ async function ensureTlsArchive(sanitize, driver, buildIdentity, cacheRoot = ven
|
|
|
958
1002
|
* objects apart from executable-lane objects). Persistent builds cache the
|
|
959
1003
|
* completed archive by program-TU content and cache runtime objects separately,
|
|
960
1004
|
* so an edit recompiles only the changed program object before re-archiving.
|
|
1005
|
+
* Large LLVM dev TUs split further into stable symbol-hash shards: compile
|
|
1006
|
+
* those in parallel, cache each object independently, then relocatably merge
|
|
1007
|
+
* them back into the archive's one canonical program member. A localized edit
|
|
1008
|
+
* recompiles only the changed buckets; exact repeats use the merged-object or
|
|
1009
|
+
* completed-archive tiers and never repeat the merge.
|
|
961
1010
|
* The base set narrows from the executable lane's unconditional sources:
|
|
962
1011
|
* scr_async.c (fibers, timers, the loop) and scr_child.c drop — the
|
|
963
1012
|
* async_free refusal already guarantees nothing references them — and
|
|
@@ -977,9 +1026,94 @@ const LIB_RUNTIME_SOURCES = [
|
|
|
977
1026
|
...RUNTIME_SOURCES.filter((f) => f !== "scr_async.c" && f !== "scr_child.c" && f !== "scr_ffi.c"),
|
|
978
1027
|
"scr_library.c",
|
|
979
1028
|
];
|
|
1029
|
+
function updateProgramShardCacheIdentity(hash, shards, publicSymbols, mergeIdentity) {
|
|
1030
|
+
hash.update("\0program-shards\0");
|
|
1031
|
+
if (shards === undefined) {
|
|
1032
|
+
hash.update("<none>\0");
|
|
1033
|
+
}
|
|
1034
|
+
else {
|
|
1035
|
+
hash.update("<present>\0");
|
|
1036
|
+
for (const shard of shards) {
|
|
1037
|
+
hash.update(shard.name).update("\0").update(shard.source).update("\0");
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
hash.update("\0program-public-symbols\0");
|
|
1041
|
+
if (publicSymbols === undefined) {
|
|
1042
|
+
hash.update("<none>\0");
|
|
1043
|
+
}
|
|
1044
|
+
else {
|
|
1045
|
+
hash.update("<present>\0");
|
|
1046
|
+
for (const symbol of publicSymbols)
|
|
1047
|
+
hash.update(symbol).update("\0");
|
|
1048
|
+
}
|
|
1049
|
+
hash.update("\0program-shard-merge\0");
|
|
1050
|
+
hash.update(mergeIdentity === undefined ? "<none>\0" : `<present>\0${mergeIdentity}\0`);
|
|
1051
|
+
}
|
|
1052
|
+
/** Identity of the machinery that turns LLVM shard objects back into the
|
|
1053
|
+
* canonical program member, or null when the target cannot do so. Sharding is
|
|
1054
|
+
* only a build optimization: a missing host tool or unsupported object class
|
|
1055
|
+
* retains the ordinary single-TU compile. The identity joins every cache tier
|
|
1056
|
+
* that contains merged bytes; raw shard-object keys deliberately omit it. */
|
|
1057
|
+
async function resolveProgramShardMergeIdentity(driver) {
|
|
1058
|
+
const platform = targetPlatform(driver);
|
|
1059
|
+
// Mach-O merging uses the host's ld64. A Darwin target produced from a
|
|
1060
|
+
// non-Darwin host can still compile as one canonical Zig TU, but the host's
|
|
1061
|
+
// ELF/COFF linker cannot combine those objects.
|
|
1062
|
+
if (platform === "darwin") {
|
|
1063
|
+
if (process.platform !== "darwin")
|
|
1064
|
+
return null;
|
|
1065
|
+
const ld = await resolvedTool("ld");
|
|
1066
|
+
return ld === null
|
|
1067
|
+
? null
|
|
1068
|
+
: rememberFingerprintDependencies(`program-shard-merge-darwin-v1\0${ld.cacheIdentity}`, [ld.canonicalPath]);
|
|
1069
|
+
}
|
|
1070
|
+
if (platform === "linux") {
|
|
1071
|
+
if (driver.target === null) {
|
|
1072
|
+
const [ld, objcopy] = await Promise.all([
|
|
1073
|
+
resolvedTool("ld"),
|
|
1074
|
+
resolvedTool("objcopy"),
|
|
1075
|
+
]);
|
|
1076
|
+
return ld === null || objcopy === null
|
|
1077
|
+
? null
|
|
1078
|
+
: rememberFingerprintDependencies(`program-shard-merge-linux-v1\0${ld.cacheIdentity}\0${objcopy.cacheIdentity}`, [ld.canonicalPath, objcopy.canonicalPath]);
|
|
1079
|
+
}
|
|
1080
|
+
const arch = driver.target.split("-", 1)[0];
|
|
1081
|
+
if (arch !== "x86_64" && arch !== "aarch64")
|
|
1082
|
+
return null;
|
|
1083
|
+
const compiler = await resolvedTool(driver.argv[0] ?? "zig");
|
|
1084
|
+
return compiler === null
|
|
1085
|
+
? null
|
|
1086
|
+
: rememberFingerprintDependencies(`program-shard-merge-cross-elf-v1\0${arch}\0${compiler.cacheIdentity}`, [compiler.canonicalPath]);
|
|
1087
|
+
}
|
|
1088
|
+
if (platform === "win32") {
|
|
1089
|
+
const arch = driver.target?.split("-", 1)[0] ?? process.arch;
|
|
1090
|
+
return arch === "x86_64" || arch === "x64"
|
|
1091
|
+
? `program-shard-merge-coff-v1\0${arch}`
|
|
1092
|
+
: null;
|
|
1093
|
+
}
|
|
1094
|
+
return null;
|
|
1095
|
+
}
|
|
980
1096
|
export async function compileLibArchive(opts) {
|
|
981
1097
|
const rtDir = runtimeSrcDir();
|
|
982
1098
|
const driver = resolveCc();
|
|
1099
|
+
const shardNames = new Set();
|
|
1100
|
+
const programShardsValid = opts.programShards?.every((shard) => {
|
|
1101
|
+
if (basename(shard.name) !== shard.name || !shard.name.endsWith(".ll") ||
|
|
1102
|
+
shardNames.has(shard.name))
|
|
1103
|
+
return false;
|
|
1104
|
+
shardNames.add(shard.name);
|
|
1105
|
+
return true;
|
|
1106
|
+
}) === true;
|
|
1107
|
+
const programShardsRequested = opts.cPath.endsWith(".ll") &&
|
|
1108
|
+
programShardsValid && opts.programShards !== undefined && opts.programShards.length > 1 &&
|
|
1109
|
+
opts.programPublicSymbols !== undefined
|
|
1110
|
+
? opts.programShards
|
|
1111
|
+
: null;
|
|
1112
|
+
const programShardMergeIdentity = programShardsRequested === null
|
|
1113
|
+
? null
|
|
1114
|
+
: await resolveProgramShardMergeIdentity(driver);
|
|
1115
|
+
const programShards = programShardMergeIdentity === null ? null : programShardsRequested;
|
|
1116
|
+
const programPublicSymbols = programShards === null ? undefined : opts.programPublicSymbols;
|
|
983
1117
|
const sanitize = opts.sanitize ?? false;
|
|
984
1118
|
const optimization = opts.optimization ?? "release";
|
|
985
1119
|
const regex = opts.regex ?? false;
|
|
@@ -1049,11 +1183,14 @@ export async function compileLibArchive(opts) {
|
|
|
1049
1183
|
}
|
|
1050
1184
|
}
|
|
1051
1185
|
let implicitToolchain = null;
|
|
1186
|
+
let implicitCompileToolchain = null;
|
|
1052
1187
|
let runtimeCompilerInvocation = null;
|
|
1053
1188
|
let programCompilerInvocation = null;
|
|
1054
1189
|
if (persistentDriverCache) {
|
|
1055
1190
|
try {
|
|
1056
|
-
|
|
1191
|
+
const fingerprints = await implicitToolchainFingerprints(driver, toolchainEnv);
|
|
1192
|
+
implicitToolchain = fingerprints.complete;
|
|
1193
|
+
implicitCompileToolchain = fingerprints.compile;
|
|
1057
1194
|
}
|
|
1058
1195
|
catch {
|
|
1059
1196
|
// An identity probe is cache machinery, never a reason a valid native
|
|
@@ -1087,34 +1224,38 @@ export async function compileLibArchive(opts) {
|
|
|
1087
1224
|
let archiverVersion = "";
|
|
1088
1225
|
let runtimeHash = "";
|
|
1089
1226
|
let programDependencyHash = "";
|
|
1090
|
-
let cachedProgramBytes =
|
|
1227
|
+
let cachedProgramBytes = opts.programSource === undefined
|
|
1228
|
+
? null
|
|
1229
|
+
: Buffer.from(opts.programSource, "utf8");
|
|
1230
|
+
const identityBytes = opts.identityCSource === undefined
|
|
1231
|
+
? null
|
|
1232
|
+
: Buffer.from(opts.identityCSource, "utf8");
|
|
1091
1233
|
if (root !== null) {
|
|
1092
1234
|
try {
|
|
1093
1235
|
const [cv, fingerprint, programBytes] = await Promise.all([
|
|
1094
1236
|
ccVersionOnce(driver.argv, toolchainEnv, true),
|
|
1095
1237
|
runtimeFingerprint(rtDir),
|
|
1096
|
-
readFile(opts.cPath),
|
|
1238
|
+
cachedProgramBytes === null ? readFile(opts.cPath) : Promise.resolve(cachedProgramBytes),
|
|
1097
1239
|
]);
|
|
1098
1240
|
compilerVersion = cv;
|
|
1099
1241
|
runtimeHash = fingerprint;
|
|
1100
1242
|
cachedProgramBytes = programBytes;
|
|
1243
|
+
programDependencyHash = await translationUnitDependencyFingerprint(driver, cflags, opts.cPath, programBytes, toolchainEnv);
|
|
1101
1244
|
if (cacheCompleteArchive) {
|
|
1102
|
-
const
|
|
1103
|
-
toolVersionOnce(arArgv, toolchainEnv, true),
|
|
1104
|
-
translationUnitDependencyFingerprint(driver, cflags, opts.cPath, programBytes, toolchainEnv),
|
|
1105
|
-
]);
|
|
1245
|
+
const av = await toolVersionOnce(arArgv, toolchainEnv, true);
|
|
1106
1246
|
archiverVersion = av;
|
|
1107
|
-
programDependencyHash = programDependencies;
|
|
1108
1247
|
const key = createHash("sha256")
|
|
1109
|
-
//
|
|
1110
|
-
//
|
|
1111
|
-
|
|
1248
|
+
// v10 adds the shard-merge implementation/tool identity. The
|
|
1249
|
+
// canonical TU still keys source semantics; shard, keep, and merge
|
|
1250
|
+
// bytes key the exact merged program object so ABI projections,
|
|
1251
|
+
// tool replacements, and single-/multi-TU producers never collide.
|
|
1252
|
+
.update("lib-v10\0")
|
|
1112
1253
|
.update(cacheTargetIdentity(driver)).update("\0")
|
|
1113
1254
|
.update(toolchainEnv).update("\0")
|
|
1114
1255
|
.update(implicitToolchain).update("\0")
|
|
1115
1256
|
.update(runtimeCompilerInvocation).update("\0")
|
|
1116
1257
|
.update(programCompilerInvocation).update("\0")
|
|
1117
|
-
.update(
|
|
1258
|
+
.update(programDependencyHash).update("\0")
|
|
1118
1259
|
.update(opts.cacheIdentity).update("\0")
|
|
1119
1260
|
.update(driver.argv.join("\x1f")).update("\0")
|
|
1120
1261
|
.update(cv).update("\0")
|
|
@@ -1128,9 +1269,14 @@ export async function compileLibArchive(opts) {
|
|
|
1128
1269
|
// latter. Archive members also inherit the TU's basename.
|
|
1129
1270
|
.update(opts.cPath).update("\0")
|
|
1130
1271
|
.update(resolve(opts.cPath)).update("\0")
|
|
1131
|
-
.update(programBytes)
|
|
1272
|
+
.update(programBytes);
|
|
1273
|
+
updateProgramShardCacheIdentity(key, programShards ?? undefined, programPublicSymbols, programShardMergeIdentity ?? undefined);
|
|
1274
|
+
const keyHex = key
|
|
1275
|
+
.update("\0identity\0")
|
|
1276
|
+
.update(identityBytes === null ? "<none>" : "<generated>").update("\0")
|
|
1277
|
+
.update(identityBytes ?? Buffer.alloc(0))
|
|
1132
1278
|
.digest("hex");
|
|
1133
|
-
cachedArchive = join(root, "lib",
|
|
1279
|
+
cachedArchive = join(root, "lib", keyHex);
|
|
1134
1280
|
const tmpOut = privateSiblingPath(opts.outPath, "lib-hit");
|
|
1135
1281
|
try {
|
|
1136
1282
|
await mkdir(dirname(opts.outPath), { recursive: true });
|
|
@@ -1155,19 +1301,19 @@ export async function compileLibArchive(opts) {
|
|
|
1155
1301
|
cachedArchive = null;
|
|
1156
1302
|
}
|
|
1157
1303
|
}
|
|
1158
|
-
const transientVendorRoot =
|
|
1304
|
+
const transientVendorRoot = root !== null && implicitToolchain !== null
|
|
1159
1305
|
? null
|
|
1160
1306
|
: join(tmpdir(), `scriptc-lib-vendor-${process.pid}-${Math.random().toString(36).slice(2)}`);
|
|
1161
|
-
const vendorCacheRoot = transientVendorRoot ?? vendorBuildCacheRoot();
|
|
1307
|
+
const vendorCacheRoot = transientVendorRoot ?? vendorBuildCacheRoot(root ?? undefined);
|
|
1162
1308
|
try {
|
|
1163
|
-
const lreObjects = regex
|
|
1164
|
-
? await ensureLreObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
|
1165
|
-
: [];
|
|
1166
|
-
const zlibObjects = opts.zlib
|
|
1167
|
-
? await ensureZlibObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
|
1168
|
-
: [];
|
|
1169
1309
|
const buildDir = await mkdtemp(join(tmpdir(), "scriptc-lib-"));
|
|
1170
1310
|
try {
|
|
1311
|
+
const lreObjects = regex
|
|
1312
|
+
? await stageVendorInputs(() => ensureLreObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot), join(buildDir, "vendor-lre"))
|
|
1313
|
+
: [];
|
|
1314
|
+
const zlibObjects = opts.zlib
|
|
1315
|
+
? await stageVendorInputs(() => ensureZlibObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot), join(buildDir, "vendor-zlib"))
|
|
1316
|
+
: [];
|
|
1171
1317
|
const compileOne = async (src, objName, compilerVisibleSource) => {
|
|
1172
1318
|
const obj = join(buildDir, objName);
|
|
1173
1319
|
const args = [
|
|
@@ -1198,9 +1344,171 @@ export async function compileLibArchive(opts) {
|
|
|
1198
1344
|
const programSource = cachedProgramBytes === null
|
|
1199
1345
|
? opts.cPath
|
|
1200
1346
|
: join(buildDir, `program${opts.cPath.endsWith(".ll") ? ".ll" : ".c"}`);
|
|
1201
|
-
if (cachedProgramBytes !== null)
|
|
1347
|
+
if (cachedProgramBytes !== null && programShards === null) {
|
|
1202
1348
|
await writeFile(programSource, cachedProgramBytes);
|
|
1203
|
-
|
|
1349
|
+
}
|
|
1350
|
+
let cachedProgramObject = null;
|
|
1351
|
+
if (root !== null && cachedProgramBytes !== null && compilerVersion !== "" &&
|
|
1352
|
+
implicitToolchain !== null && programCompilerInvocation !== null) {
|
|
1353
|
+
const programKey = createHash("sha256")
|
|
1354
|
+
.update("lib-program-obj-v3\0")
|
|
1355
|
+
.update(cacheTargetIdentity(driver)).update("\0")
|
|
1356
|
+
.update(toolchainEnv).update("\0")
|
|
1357
|
+
.update(implicitToolchain).update("\0")
|
|
1358
|
+
.update(programCompilerInvocation).update("\0")
|
|
1359
|
+
.update(opts.cacheIdentity).update("\0")
|
|
1360
|
+
.update(driver.argv.join("\x1f")).update("\0")
|
|
1361
|
+
.update(compilerVersion).update("\0")
|
|
1362
|
+
.update(runtimeHash).update("\0")
|
|
1363
|
+
.update(programDependencyHash).update("\0")
|
|
1364
|
+
.update(programCompilerArgs.join("\x1f")).update("\0")
|
|
1365
|
+
.update(opts.cPath).update("\0")
|
|
1366
|
+
.update(resolve(opts.cPath)).update("\0")
|
|
1367
|
+
.update(cachedProgramBytes);
|
|
1368
|
+
updateProgramShardCacheIdentity(programKey, programShards ?? undefined, programPublicSymbols, programShardMergeIdentity ?? undefined);
|
|
1369
|
+
const programKeyHex = programKey.digest("hex");
|
|
1370
|
+
cachedProgramObject = join(root, "program-obj", programKeyHex);
|
|
1371
|
+
}
|
|
1372
|
+
const stagedProgramObject = join(buildDir, `${stem}.program.o`);
|
|
1373
|
+
let programObject;
|
|
1374
|
+
let programShardFallback = false;
|
|
1375
|
+
if (cachedProgramObject !== null &&
|
|
1376
|
+
await copyValidCachedFile(cachedProgramObject, stagedProgramObject)) {
|
|
1377
|
+
programObject = stagedProgramObject;
|
|
1378
|
+
}
|
|
1379
|
+
else if (programShards !== null) {
|
|
1380
|
+
try {
|
|
1381
|
+
const shardEntries = programShards.map((shard, index) => {
|
|
1382
|
+
const sourcePath = join(buildDir, shard.name);
|
|
1383
|
+
const staged = join(buildDir, `${stem}.program-${index.toString().padStart(3, "0")}.o`);
|
|
1384
|
+
let cachePath = null;
|
|
1385
|
+
if (root !== null && compilerVersion !== "" && implicitCompileToolchain !== null &&
|
|
1386
|
+
programCompilerInvocation !== null) {
|
|
1387
|
+
const key = createHash("sha256")
|
|
1388
|
+
// v2 removes the broad implicit-toolchain fingerprint: it
|
|
1389
|
+
// includes the linker selected by the driver, but raw shard
|
|
1390
|
+
// objects are compile-only outputs. The compile-only toolchain
|
|
1391
|
+
// identity retains compiler/config/header/assembler inputs;
|
|
1392
|
+
// the effective program invocation pins this exact flag lane.
|
|
1393
|
+
// Merge-tool identity belongs only to the merged-object and
|
|
1394
|
+
// completed-archive tiers.
|
|
1395
|
+
.update("lib-program-shard-v2\0")
|
|
1396
|
+
.update(cacheTargetIdentity(driver)).update("\0")
|
|
1397
|
+
.update(toolchainEnv).update("\0")
|
|
1398
|
+
.update(implicitCompileToolchain).update("\0")
|
|
1399
|
+
.update(programCompilerInvocation).update("\0")
|
|
1400
|
+
.update(opts.cacheIdentity).update("\0")
|
|
1401
|
+
.update(driver.argv.join("\x1f")).update("\0")
|
|
1402
|
+
.update(compilerVersion).update("\0")
|
|
1403
|
+
.update(runtimeHash).update("\0")
|
|
1404
|
+
.update(programDependencyHash).update("\0")
|
|
1405
|
+
.update(programCompilerArgs.join("\x1f")).update("\0")
|
|
1406
|
+
.update(opts.cPath).update("\0")
|
|
1407
|
+
.update(resolve(opts.cPath)).update("\0")
|
|
1408
|
+
.update(shard.name).update("\0")
|
|
1409
|
+
.update(shard.source)
|
|
1410
|
+
.digest("hex");
|
|
1411
|
+
cachePath = join(root, "program-shard", key);
|
|
1412
|
+
}
|
|
1413
|
+
return { ...shard, sourcePath, staged, cachePath, missed: false };
|
|
1414
|
+
});
|
|
1415
|
+
const shardWidth = Math.min(8, availableParallelism());
|
|
1416
|
+
for (let i = 0; i < shardEntries.length; i += shardWidth) {
|
|
1417
|
+
await Promise.all(shardEntries.slice(i, i + shardWidth).map(async (entry) => {
|
|
1418
|
+
await writeFile(entry.sourcePath, entry.source);
|
|
1419
|
+
if (entry.cachePath !== null &&
|
|
1420
|
+
await copyValidCachedFile(entry.cachePath, entry.staged))
|
|
1421
|
+
return;
|
|
1422
|
+
entry.missed = true;
|
|
1423
|
+
await compileOne(entry.sourcePath, basename(entry.staged), opts.cPath);
|
|
1424
|
+
}));
|
|
1425
|
+
}
|
|
1426
|
+
const publishable = shardEntries.filter((entry) => entry.missed && entry.cachePath !== null);
|
|
1427
|
+
const mergedProgramObject = await localizeLibraryObjects(driver, arArgv, buildDir, shardEntries.map((entry) => entry.staged), [], programPublicSymbols, `${stem}.program`);
|
|
1428
|
+
// Keep the canonical archive member spelling. The fact that native
|
|
1429
|
+
// compilation used shards is an implementation detail; consumers and
|
|
1430
|
+
// deterministic cache tests continue to see `<stem>.program.o`.
|
|
1431
|
+
await rename(mergedProgramObject, stagedProgramObject);
|
|
1432
|
+
programObject = stagedProgramObject;
|
|
1433
|
+
if (cachedProgramObject !== null || publishable.length > 0) {
|
|
1434
|
+
try {
|
|
1435
|
+
const [currentRuntime, currentInvocation, currentDependencies, currentCompiler] = await Promise.all([
|
|
1436
|
+
runtimeFingerprint(rtDir),
|
|
1437
|
+
effectiveCompilerInvocationFingerprint(driver, toolchainEnv, programCompilerArgs, programSourceExtension),
|
|
1438
|
+
translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cachedProgramBytes, toolchainEnv),
|
|
1439
|
+
ccVersionOnce(driver.argv, toolchainEnv, true),
|
|
1440
|
+
]);
|
|
1441
|
+
const shardInputsStillMatch = currentRuntime === runtimeHash &&
|
|
1442
|
+
currentInvocation === programCompilerInvocation &&
|
|
1443
|
+
currentDependencies === programDependencyHash && currentCompiler === compilerVersion;
|
|
1444
|
+
const currentFingerprints = shardInputsStillMatch
|
|
1445
|
+
? await implicitToolchainFingerprints(driver, toolchainEnv)
|
|
1446
|
+
: null;
|
|
1447
|
+
const compileInputsStillMatch = shardInputsStillMatch &&
|
|
1448
|
+
currentFingerprints?.compile === implicitCompileToolchain;
|
|
1449
|
+
let mergedInputsStillMatch = false;
|
|
1450
|
+
if (compileInputsStillMatch && cachedProgramObject !== null) {
|
|
1451
|
+
const currentMerge = await resolveProgramShardMergeIdentity(driver).catch(() => null);
|
|
1452
|
+
mergedInputsStillMatch =
|
|
1453
|
+
currentFingerprints?.complete === implicitToolchain &&
|
|
1454
|
+
currentMerge === programShardMergeIdentity;
|
|
1455
|
+
}
|
|
1456
|
+
if (compileInputsStillMatch) {
|
|
1457
|
+
await Promise.all([
|
|
1458
|
+
...(mergedInputsStillMatch
|
|
1459
|
+
? [publishCachedFile(programObject, cachedProgramObject)]
|
|
1460
|
+
: []),
|
|
1461
|
+
...publishable.map((entry) => publishCachedFile(entry.staged, entry.cachePath)),
|
|
1462
|
+
]);
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
catch {
|
|
1466
|
+
// Best-effort: the merged program object is already valid.
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
catch {
|
|
1471
|
+
// Sharding is only an optimization. A present but incompatible or
|
|
1472
|
+
// failing merge tool (or a shard-only compiler failure) must not
|
|
1473
|
+
// turn a valid canonical LLVM TU into a failed library build. Do not
|
|
1474
|
+
// publish this retry under shard-derived object/archive cache keys.
|
|
1475
|
+
programShardFallback = true;
|
|
1476
|
+
if (cachedProgramBytes !== null)
|
|
1477
|
+
await writeFile(programSource, cachedProgramBytes);
|
|
1478
|
+
programObject = await compileOne(programSource, `${stem}.program.o`, cachedProgramBytes === null ? undefined : opts.cPath);
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
else {
|
|
1482
|
+
programObject = await compileOne(programSource, `${stem}.program.o`, cachedProgramBytes === null ? undefined : opts.cPath);
|
|
1483
|
+
if (cachedProgramObject !== null) {
|
|
1484
|
+
try {
|
|
1485
|
+
const [currentRuntime, currentImplicit, currentInvocation, currentDependencies, currentCompiler] = await Promise.all([
|
|
1486
|
+
runtimeFingerprint(rtDir),
|
|
1487
|
+
implicitToolchainFingerprint(driver, toolchainEnv),
|
|
1488
|
+
effectiveCompilerInvocationFingerprint(driver, toolchainEnv, programCompilerArgs, programSourceExtension),
|
|
1489
|
+
translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cachedProgramBytes, toolchainEnv),
|
|
1490
|
+
ccVersionOnce(driver.argv, toolchainEnv, true),
|
|
1491
|
+
]);
|
|
1492
|
+
if (currentRuntime === runtimeHash &&
|
|
1493
|
+
currentImplicit === implicitToolchain &&
|
|
1494
|
+
currentInvocation === programCompilerInvocation &&
|
|
1495
|
+
currentDependencies === programDependencyHash &&
|
|
1496
|
+
currentCompiler === compilerVersion) {
|
|
1497
|
+
await publishCachedFile(programObject, cachedProgramObject);
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
catch {
|
|
1501
|
+
// Best-effort: the archive build already owns a valid object.
|
|
1502
|
+
}
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
const identityObject = identityBytes === null
|
|
1506
|
+
? null
|
|
1507
|
+
: await (async () => {
|
|
1508
|
+
const source = join(buildDir, "identity.c");
|
|
1509
|
+
await writeFile(source, identityBytes);
|
|
1510
|
+
return compileOne(source, `${stem}.identity.o`);
|
|
1511
|
+
})();
|
|
1204
1512
|
let runtimeObjects = null;
|
|
1205
1513
|
let cacheInputsStable = true;
|
|
1206
1514
|
let objectImplicitVerification = null;
|
|
@@ -1233,14 +1541,14 @@ export async function compileLibArchive(opts) {
|
|
|
1233
1541
|
runtimeObjects.push(...(await Promise.all(sources.slice(i, i + width).map((f) => compileOne(join(rtDir, f), f.replace(/\.c$/, ".o"))))));
|
|
1234
1542
|
}
|
|
1235
1543
|
}
|
|
1236
|
-
const objects = [programObject, ...runtimeObjects, ...lreObjects, ...zlibObjects];
|
|
1544
|
+
const objects = [programObject, ...(identityObject === null ? [] : [identityObject]), ...runtimeObjects, ...lreObjects, ...zlibObjects];
|
|
1237
1545
|
// Multi-instance library mode: the archive's one member becomes the
|
|
1238
1546
|
// combined, symbol-localized object (cached vendor/runtime objects
|
|
1239
1547
|
// are read-only inputs here — the combine step never mutates them).
|
|
1240
1548
|
const archiveMembers = opts.localizeSymbols === undefined
|
|
1241
1549
|
? objects
|
|
1242
1550
|
: [
|
|
1243
|
-
await localizeLibraryObjects(driver, arArgv, buildDir, programObject, [...runtimeObjects, ...lreObjects, ...zlibObjects], opts.localizeSymbols, stem),
|
|
1551
|
+
await localizeLibraryObjects(driver, arArgv, buildDir, [programObject, ...(identityObject === null ? [] : [identityObject])], [...runtimeObjects, ...lreObjects, ...zlibObjects], opts.localizeSymbols, stem),
|
|
1244
1552
|
];
|
|
1245
1553
|
// A cacheable or runtime-localized build owns a private archive from
|
|
1246
1554
|
// `ar` through publication. Localized archives deliberately bypass the
|
|
@@ -1267,7 +1575,7 @@ export async function compileLibArchive(opts) {
|
|
|
1267
1575
|
programDependencyHash !== "" &&
|
|
1268
1576
|
compilerVersion !== "" &&
|
|
1269
1577
|
archiverVersion !== "") {
|
|
1270
|
-
const [currentRuntime, currentImplicit, currentRuntimeInvocation, currentProgramInvocation, currentProgramDependencies, currentCompiler, currentArchiver] = await Promise.all([
|
|
1578
|
+
const [currentRuntime, currentImplicit, currentRuntimeInvocation, currentProgramInvocation, currentProgramDependencies, currentCompiler, currentArchiver, currentProgramShardMerge] = await Promise.all([
|
|
1271
1579
|
runtimeFingerprint(rtDir).catch(() => null),
|
|
1272
1580
|
implicitToolchainFingerprint(driver, toolchainEnv).catch(() => null),
|
|
1273
1581
|
effectiveCompilerInvocationFingerprint(driver, toolchainEnv, cflags).catch(() => null),
|
|
@@ -1275,16 +1583,21 @@ export async function compileLibArchive(opts) {
|
|
|
1275
1583
|
translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cachedProgramBytes, toolchainEnv).catch(() => null),
|
|
1276
1584
|
ccVersionOnce(driver.argv, toolchainEnv, true).catch(() => null),
|
|
1277
1585
|
toolVersionOnce(arArgv, toolchainEnv, true).catch(() => null),
|
|
1586
|
+
programShards === null
|
|
1587
|
+
? Promise.resolve(null)
|
|
1588
|
+
: resolveProgramShardMergeIdentity(driver).catch(() => null),
|
|
1278
1589
|
]);
|
|
1279
1590
|
runtimeStillMatchesKey =
|
|
1280
1591
|
cacheInputsStable &&
|
|
1592
|
+
!programShardFallback &&
|
|
1281
1593
|
currentRuntime === runtimeHash &&
|
|
1282
1594
|
currentImplicit === implicitToolchain &&
|
|
1283
1595
|
currentRuntimeInvocation === runtimeCompilerInvocation &&
|
|
1284
1596
|
currentProgramInvocation === programCompilerInvocation &&
|
|
1285
1597
|
currentProgramDependencies === programDependencyHash &&
|
|
1286
1598
|
currentCompiler === compilerVersion &&
|
|
1287
|
-
currentArchiver === archiverVersion
|
|
1599
|
+
currentArchiver === archiverVersion &&
|
|
1600
|
+
currentProgramShardMerge === programShardMergeIdentity;
|
|
1288
1601
|
}
|
|
1289
1602
|
if (cachedArchive !== null && root !== null && runtimeStillMatchesKey) {
|
|
1290
1603
|
try {
|
|
@@ -1320,7 +1633,8 @@ export async function compileLibArchive(opts) {
|
|
|
1320
1633
|
* shared by design. Windows embedders additionally link advapi32, iphlpapi,
|
|
1321
1634
|
* and ws2_32.
|
|
1322
1635
|
*
|
|
1323
|
-
*
|
|
1636
|
+
* The generated program and optional identity objects are mandatory roots;
|
|
1637
|
+
* support-member selection still matters. A classic archive's unused members (and their
|
|
1324
1638
|
* undefined references to units library mode excludes, like the
|
|
1325
1639
|
* fs-promises unit's fiber symbols) never reach an embedder's link. A
|
|
1326
1640
|
* blind merge of every object would carry those references into the one
|
|
@@ -1329,7 +1643,7 @@ export async function compileLibArchive(opts) {
|
|
|
1329
1643
|
* members the program object transitively needs (the COFF arm implements
|
|
1330
1644
|
* the same member semantics in process).
|
|
1331
1645
|
*
|
|
1332
|
-
* Mach-O — one host-ld64 invocation: -r merges
|
|
1646
|
+
* Mach-O — one host-ld64 invocation: -r merges roots + needed members,
|
|
1333
1647
|
* -exported_symbols_list demotes every unlisted global to
|
|
1334
1648
|
* private extern, and -r without -keep_private_externs writes
|
|
1335
1649
|
* private externs out as non-external symbols. Apple ASan's
|
|
@@ -1360,7 +1674,7 @@ export async function compileLibArchive(opts) {
|
|
|
1360
1674
|
* mirrors MSVC link.exe; zig's COFF driver refuses multi-object
|
|
1361
1675
|
* merges) and llvm-objcopy rejects symbol-scope flags for
|
|
1362
1676
|
* COFF, so no tool pairing exists to shell out to. */
|
|
1363
|
-
async function localizeLibraryObjects(driver, arArgv, buildDir,
|
|
1677
|
+
async function localizeLibraryObjects(driver, arArgv, buildDir, rootObjects, supportObjects, keepSymbols, stem) {
|
|
1364
1678
|
const platform = targetPlatform(driver);
|
|
1365
1679
|
const combined = join(buildDir, `${stem}.localized.o`);
|
|
1366
1680
|
const staging = join(buildDir, `${stem}.localize-staging.a`);
|
|
@@ -1378,10 +1692,13 @@ async function localizeLibraryObjects(driver, arArgv, buildDir, programObject, s
|
|
|
1378
1692
|
if (platform === "win32") {
|
|
1379
1693
|
// COFF has no relocatable-link tool to stage through; the member
|
|
1380
1694
|
// selection and combine+demote happen in process over the object bytes.
|
|
1381
|
-
const [
|
|
1695
|
+
const [roots, support] = await Promise.all([
|
|
1696
|
+
Promise.all(rootObjects.map((path) => readFile(path))),
|
|
1697
|
+
Promise.all(supportObjects.map((path) => readFile(path))),
|
|
1698
|
+
]);
|
|
1382
1699
|
try {
|
|
1383
|
-
await writeFile(combined, mergeAndLocalizeCoffObjects(
|
|
1384
|
-
|
|
1700
|
+
await writeFile(combined, mergeAndLocalizeCoffObjects(roots, support, new Set(keepSymbols), {
|
|
1701
|
+
roots: rootObjects.map((path) => basename(path)),
|
|
1385
1702
|
support: supportObjects.map((path) => basename(path)),
|
|
1386
1703
|
}));
|
|
1387
1704
|
}
|
|
@@ -1390,14 +1707,17 @@ async function localizeLibraryObjects(driver, arArgv, buildDir, programObject, s
|
|
|
1390
1707
|
}
|
|
1391
1708
|
return combined;
|
|
1392
1709
|
}
|
|
1393
|
-
|
|
1710
|
+
const supportArgs = supportObjects.length === 0 ? [] : [staging];
|
|
1711
|
+
if (supportObjects.length > 0) {
|
|
1712
|
+
await run([arArgv[0] ?? "ar", ...arArgv.slice(1), "rcs", staging, ...supportObjects]);
|
|
1713
|
+
}
|
|
1394
1714
|
if (platform === "darwin") {
|
|
1395
1715
|
await writeFile(keepFile, keepSymbols.map((s) => `_${s}\n`).join(""));
|
|
1396
|
-
await run(["ld", "-r",
|
|
1716
|
+
await run(["ld", "-r", ...rootObjects, ...supportArgs, "-o", combined, "-exported_symbols_list", keepFile]);
|
|
1397
1717
|
}
|
|
1398
1718
|
else if (platform === "linux" && driver.target === null) {
|
|
1399
1719
|
await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join(""));
|
|
1400
|
-
await run(["ld", "-r", "--force-group-allocation",
|
|
1720
|
+
await run(["ld", "-r", "--force-group-allocation", ...rootObjects, ...supportArgs, "-o", combined]);
|
|
1401
1721
|
await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]);
|
|
1402
1722
|
}
|
|
1403
1723
|
else if (platform === "linux") {
|
|
@@ -1412,7 +1732,7 @@ async function localizeLibraryObjects(driver, arArgv, buildDir, programObject, s
|
|
|
1412
1732
|
...driver.argv.slice(1),
|
|
1413
1733
|
"-target", driver.zigTarget ?? driver.target,
|
|
1414
1734
|
"-nostdlib",
|
|
1415
|
-
"-r",
|
|
1735
|
+
"-r", ...rootObjects, ...supportArgs,
|
|
1416
1736
|
"-o", combined,
|
|
1417
1737
|
]);
|
|
1418
1738
|
try {
|
|
@@ -1431,7 +1751,7 @@ async function localizeLibraryObjects(driver, arArgv, buildDir, programObject, s
|
|
|
1431
1751
|
* Content-addressed caches that let repeat builds of unchanged programs skip
|
|
1432
1752
|
* payload code generation/linking — the test lanes' dominant cost. Executable
|
|
1433
1753
|
* lookups still run lightweight dependency and link-input metadata probes.
|
|
1434
|
-
*
|
|
1754
|
+
* Principal keyspaces under the cache root:
|
|
1435
1755
|
*
|
|
1436
1756
|
* bin/<key> — whole program binaries. key = sha256(resolved clang
|
|
1437
1757
|
* identity/version, target + compiler/linker environment,
|
|
@@ -1461,6 +1781,14 @@ async function localizeLibraryObjects(driver, arArgv, buildDir, programObject, s
|
|
|
1461
1781
|
* identity, TU path, and program-TU bytes.
|
|
1462
1782
|
* Checksum-verified hits skip native code generation and ar.
|
|
1463
1783
|
*
|
|
1784
|
+
* program-obj/<key> — the canonical library program object, after any LLVM
|
|
1785
|
+
* shard merge. Exact TU repeats reuse it directly.
|
|
1786
|
+
*
|
|
1787
|
+
* program-shard/<key> — stable LLVM dev-library buckets keyed by their IR
|
|
1788
|
+
* text and complete toolchain/TU identity. Local edits
|
|
1789
|
+
* retain every bucket whose definitions/declarations did
|
|
1790
|
+
* not change.
|
|
1791
|
+
*
|
|
1464
1792
|
* obj/<set>/<f>.o — per-flavor runtime objects for cache-miss builds. The
|
|
1465
1793
|
* historical single invocation recompiles every runtime
|
|
1466
1794
|
* TU per program (~1.3s at -O2); with cached objects a
|
|
@@ -1521,6 +1849,31 @@ export function resolveBuildCacheRoot(env = process.env, platform = process.plat
|
|
|
1521
1849
|
? resolve(userHome, "Library", "Caches", "scriptc", "build")
|
|
1522
1850
|
: resolve(userHome, ".cache", "scriptc", "build");
|
|
1523
1851
|
}
|
|
1852
|
+
/** Shared persistent-cache root for compiler-level tiers. The early library
|
|
1853
|
+
* cache deliberately follows the native cache's activation and hard-disable
|
|
1854
|
+
* contract, while native compilation retains ownership of toolchain safety. */
|
|
1855
|
+
export function buildCacheRoot() {
|
|
1856
|
+
return resolveBuildCacheRoot();
|
|
1857
|
+
}
|
|
1858
|
+
/** Harden/create a compiler-level cache root using the same privacy policy as
|
|
1859
|
+
* the artifact caches. Failure disables only the optional caller's tier. */
|
|
1860
|
+
export async function prepareBuildCacheRoot(root) {
|
|
1861
|
+
if (root === null)
|
|
1862
|
+
return null;
|
|
1863
|
+
try {
|
|
1864
|
+
await ensurePrivateCacheRoot(root, process.env["SCRIPTC_CACHE_DIR"] === undefined);
|
|
1865
|
+
return root;
|
|
1866
|
+
}
|
|
1867
|
+
catch {
|
|
1868
|
+
return null;
|
|
1869
|
+
}
|
|
1870
|
+
}
|
|
1871
|
+
/** Register a successful compiler-level cache write with the shared bounded
|
|
1872
|
+
* LRU policy. */
|
|
1873
|
+
export async function pruneBuildCache(root) {
|
|
1874
|
+
if (root !== null)
|
|
1875
|
+
await pruneCache(root).catch(() => undefined);
|
|
1876
|
+
}
|
|
1524
1877
|
function cacheRootDir() {
|
|
1525
1878
|
return resolveBuildCacheRoot();
|
|
1526
1879
|
}
|
|
@@ -1551,6 +1904,52 @@ async function ensurePrivateCacheRoot(root, hardenExisting) {
|
|
|
1551
1904
|
}
|
|
1552
1905
|
}
|
|
1553
1906
|
const ccVersionMemos = new Map();
|
|
1907
|
+
/** Identity of the compiler implementation and configuration selected by a
|
|
1908
|
+
* fresh driver invocation. In particular, Darwin's stable /usr/bin/clang shim
|
|
1909
|
+
* exposes the currently selected Xcode clang only in its `-###` trace. The
|
|
1910
|
+
* normalized trace also covers output-affecting default driver configuration
|
|
1911
|
+
* that can change while argv[0] and PATH retain the same spelling. */
|
|
1912
|
+
async function effectiveCompilerEnvironmentIdentity(driver, env) {
|
|
1913
|
+
const compiler = driver.argv[0] ?? "clang";
|
|
1914
|
+
const resolvedDriver = await resolvedTool(compiler, env);
|
|
1915
|
+
if (resolvedDriver === null)
|
|
1916
|
+
return `<unresolved:${compiler}>`;
|
|
1917
|
+
const probeDir = await mkdtemp(join(tmpdir(), "scriptc-early-cc-probe-"));
|
|
1918
|
+
try {
|
|
1919
|
+
const source = join(probeDir, "empty.c");
|
|
1920
|
+
const object = join(probeDir, "empty.o");
|
|
1921
|
+
await writeFile(source, "int scriptc_early_driver_probe;\n");
|
|
1922
|
+
const trace = await execFileAsync(compiler, [
|
|
1923
|
+
...driver.argv.slice(1),
|
|
1924
|
+
...driver.targetArgs,
|
|
1925
|
+
"-###",
|
|
1926
|
+
"-std=c11",
|
|
1927
|
+
"-c",
|
|
1928
|
+
source,
|
|
1929
|
+
"-o",
|
|
1930
|
+
object,
|
|
1931
|
+
], { cwd: probeDir, env, maxBuffer: 16 * 1024 * 1024 });
|
|
1932
|
+
const effectiveSpellings = [];
|
|
1933
|
+
for (const line of `${trace.stdout}\n${trace.stderr}`.split(/\r?\n/)) {
|
|
1934
|
+
const tokens = driverTraceCandidates(line);
|
|
1935
|
+
const cc1 = tokens.indexOf("-cc1");
|
|
1936
|
+
if (cc1 > 0)
|
|
1937
|
+
effectiveSpellings.push(tokens[cc1 - 1]);
|
|
1938
|
+
}
|
|
1939
|
+
const effective = effectiveSpellings.length === 1
|
|
1940
|
+
? await resolvedTool(effectiveSpellings[0], env)
|
|
1941
|
+
: null;
|
|
1942
|
+
return createHash("sha256")
|
|
1943
|
+
.update("effective-compiler-environment-v1\0")
|
|
1944
|
+
.update(resolvedDriver.cacheIdentity).update("\0")
|
|
1945
|
+
.update(normalizedProbeInvocation(trace, probeDir)).update("\0")
|
|
1946
|
+
.update(effective?.cacheIdentity ?? `<unresolved-effective:${effectiveSpellings.join("\x1f")}>`)
|
|
1947
|
+
.digest("hex");
|
|
1948
|
+
}
|
|
1949
|
+
finally {
|
|
1950
|
+
await rm(probeDir, { recursive: true, force: true }).catch(() => undefined);
|
|
1951
|
+
}
|
|
1952
|
+
}
|
|
1554
1953
|
/** Resolve the executable the OS will select for an argv[0] spelling. The
|
|
1555
1954
|
* path and inode metadata join the version output below: two PATH postures
|
|
1556
1955
|
* must not share cache entries merely because both drivers call themselves
|
|
@@ -2083,7 +2482,7 @@ function translationUnitDependencyFingerprint(driver, cflags, sourcePath, source
|
|
|
2083
2482
|
* change that redirects includes cannot hide behind an unchanged old path
|
|
2084
2483
|
* list. Dependency discovery must succeed on every cache-enabled invocation;
|
|
2085
2484
|
* a prior path list cannot reveal a new higher-priority header. */
|
|
2086
|
-
async function
|
|
2485
|
+
async function implicitToolchainFingerprintsFresh(driver, environmentFingerprint) {
|
|
2087
2486
|
let probe;
|
|
2088
2487
|
const compiler = driver.argv[0] ?? "clang";
|
|
2089
2488
|
const compilerIdentity = await resolvedToolIdentity(compiler);
|
|
@@ -2136,7 +2535,8 @@ async function implicitToolchainFingerprintFresh(driver, environmentFingerprint)
|
|
|
2136
2535
|
// executable/version and dependency set.
|
|
2137
2536
|
execFileAsync(compiler, [...prefix, "-std=c11", "-###", "-c", driverSource, "-o", driverOutput], { cwd: probeDir, maxBuffer: 16 * 1024 * 1024 }),
|
|
2138
2537
|
]);
|
|
2139
|
-
const
|
|
2538
|
+
const assemblerSpelling = assembler.stdout.trim();
|
|
2539
|
+
const toolSpellings = [linker.stdout.trim(), assemblerSpelling].filter((value, index, all) => value !== "" && all.indexOf(value) === index);
|
|
2140
2540
|
const sourceSet = new Set(sources);
|
|
2141
2541
|
const dependencyPaths = [
|
|
2142
2542
|
...new Set(dependencyResults
|
|
@@ -2157,44 +2557,56 @@ async function implicitToolchainFingerprintFresh(driver, environmentFingerprint)
|
|
|
2157
2557
|
path: resolved?.canonicalPath ?? null,
|
|
2158
2558
|
};
|
|
2159
2559
|
})),
|
|
2560
|
+
compileToolSpellings: assemblerSpelling === "" ? [] : [assemblerSpelling],
|
|
2160
2561
|
};
|
|
2161
2562
|
}
|
|
2162
2563
|
finally {
|
|
2163
2564
|
await rm(probeDir, { recursive: true, force: true }).catch(() => undefined);
|
|
2164
2565
|
}
|
|
2165
2566
|
}
|
|
2166
|
-
const
|
|
2167
|
-
|
|
2168
|
-
.
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
.update(probe.dependencies.join("\x1f"))
|
|
2179
|
-
.update("\0")
|
|
2180
|
-
.update(probe.dependencyFingerprint)
|
|
2181
|
-
.update("\0");
|
|
2182
|
-
for (const tool of probe.tools) {
|
|
2183
|
-
const currentIdentity = await resolvedToolIdentity(tool.spelling);
|
|
2184
|
-
hash
|
|
2185
|
-
.update(tool.spelling)
|
|
2567
|
+
const tools = await Promise.all(probe.tools.map(async (tool) => ({
|
|
2568
|
+
...tool,
|
|
2569
|
+
currentIdentity: await resolvedToolIdentity(tool.spelling),
|
|
2570
|
+
})));
|
|
2571
|
+
const fingerprint = (domain, selectedTools) => {
|
|
2572
|
+
const hash = createHash("sha256")
|
|
2573
|
+
.update(domain)
|
|
2574
|
+
.update(environmentFingerprint)
|
|
2575
|
+
.update("\0")
|
|
2576
|
+
.update(probe.compilerIdentity)
|
|
2577
|
+
.update("\0")
|
|
2578
|
+
.update(probe.compilerInvocation)
|
|
2186
2579
|
.update("\0")
|
|
2187
|
-
.update(
|
|
2580
|
+
.update(driver.argv.join("\x1f"))
|
|
2581
|
+
.update("\0")
|
|
2582
|
+
.update(driver.targetArgs.join("\x1f"))
|
|
2583
|
+
.update("\0")
|
|
2584
|
+
.update(probe.dependencies.join("\x1f"))
|
|
2585
|
+
.update("\0")
|
|
2586
|
+
.update(probe.dependencyFingerprint)
|
|
2188
2587
|
.update("\0");
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2588
|
+
for (const tool of selectedTools) {
|
|
2589
|
+
hash
|
|
2590
|
+
.update(tool.spelling)
|
|
2591
|
+
.update("\0")
|
|
2592
|
+
.update(tool.currentIdentity ?? tool.identity ?? "<unresolved>")
|
|
2593
|
+
.update("\0");
|
|
2594
|
+
}
|
|
2595
|
+
return rememberFingerprintDependencies(hash.digest("hex"), [
|
|
2596
|
+
...probe.dependencies,
|
|
2597
|
+
...probe.invocationPaths,
|
|
2598
|
+
...selectedTools.flatMap((tool) => tool.path === null ? [] : [tool.path]),
|
|
2599
|
+
], probe.dependencies, probe.dependencyFingerprint);
|
|
2600
|
+
};
|
|
2601
|
+
const compileSpellingSet = new Set(probe.compileToolSpellings);
|
|
2602
|
+
return {
|
|
2603
|
+
// Preserve the established complete fingerprint domain and byte stream.
|
|
2604
|
+
complete: fingerprint("implicit-toolchain-v2\0", tools),
|
|
2605
|
+
compile: fingerprint("implicit-compile-toolchain-v1\0", tools.filter((tool) => compileSpellingSet.has(tool.spelling))),
|
|
2606
|
+
};
|
|
2195
2607
|
}
|
|
2196
2608
|
const stableImplicitToolchainMemos = new Map();
|
|
2197
|
-
function
|
|
2609
|
+
function implicitToolchainFingerprints(driver, environmentFingerprint) {
|
|
2198
2610
|
const key = [
|
|
2199
2611
|
environmentFingerprint,
|
|
2200
2612
|
driver.argv.join("\x1f"),
|
|
@@ -2202,7 +2614,10 @@ function implicitToolchainFingerprint(driver, environmentFingerprint) {
|
|
|
2202
2614
|
driver.targetArgs.join("\x1f"),
|
|
2203
2615
|
runtimeSrcDir(),
|
|
2204
2616
|
].join("\0");
|
|
2205
|
-
return stableTestMemo(stableImplicitToolchainMemos, key, () =>
|
|
2617
|
+
return stableTestMemo(stableImplicitToolchainMemos, key, () => implicitToolchainFingerprintsFresh(driver, environmentFingerprint));
|
|
2618
|
+
}
|
|
2619
|
+
function implicitToolchainFingerprint(driver, environmentFingerprint) {
|
|
2620
|
+
return implicitToolchainFingerprints(driver, environmentFingerprint).then((fingerprints) => fingerprints.complete);
|
|
2206
2621
|
}
|
|
2207
2622
|
function linkTraceCandidate(line) {
|
|
2208
2623
|
const trimmed = line.trim().replace(/^(?:LOAD|load)\s+/, "");
|
|
@@ -2428,19 +2843,20 @@ function ccacheAvailable() {
|
|
|
2428
2843
|
ccacheMemo ??= execFileAsync("ccache", ["--version"]).then(() => true, () => false);
|
|
2429
2844
|
return ccacheMemo;
|
|
2430
2845
|
}
|
|
2431
|
-
/** Content hash of every
|
|
2432
|
-
*
|
|
2433
|
-
*
|
|
2434
|
-
*
|
|
2435
|
-
*
|
|
2436
|
-
*
|
|
2437
|
-
*
|
|
2438
|
-
* include without changing any previously selected dependency, while content
|
|
2439
|
-
* hashing catches same-size edits whose timestamp was preserved by a copy/sync
|
|
2440
|
-
* tool. */
|
|
2846
|
+
/** Content hash of every owned native source/header plus this backend's build
|
|
2847
|
+
* recipe implementation. It keys complete artifacts, runtime objects, and the
|
|
2848
|
+
* separately built vendor prerequisites, so two installed scriptc versions or
|
|
2849
|
+
* worktrees can share a user cache without exchanging outputs produced from
|
|
2850
|
+
* different vendored bytes or compile/archive recipes. Recursive enumeration
|
|
2851
|
+
* also catches a newly added nested header that begins shadowing a system
|
|
2852
|
+
* include, while content hashing catches same-size timestamp-preserving edits. */
|
|
2441
2853
|
async function runtimeFingerprintFresh(rtDir) {
|
|
2442
2854
|
const groups = await runtimeFingerprintInputGroups(rtDir);
|
|
2443
|
-
const h = createHash("sha256")
|
|
2855
|
+
const h = createHash("sha256")
|
|
2856
|
+
.update("native-owned-inputs-v2\0")
|
|
2857
|
+
.update(QJS_COMMIT).update(MBEDTLS_VERSION).update(ZLIB_VERSION)
|
|
2858
|
+
.update("\0backend-recipe\0")
|
|
2859
|
+
.update(await readFile(CC_IMPLEMENTATION_PATH));
|
|
2444
2860
|
for (const group of groups) {
|
|
2445
2861
|
for (const n of group.names) {
|
|
2446
2862
|
h.update(group.label).update("/").update(n).update("\0").update(await readFile(join(group.dir, n))).update("\0");
|
|
@@ -2452,6 +2868,10 @@ async function runtimeFingerprintInputGroups(rtDir) {
|
|
|
2452
2868
|
return Promise.all([
|
|
2453
2869
|
{ label: "runtime", dir: rtDir },
|
|
2454
2870
|
{ label: "ryu", dir: join(rtDir, "..", "vendor", "ryu") },
|
|
2871
|
+
{ label: "quickjs-ng", dir: join(rtDir, "..", "vendor", "quickjs-ng") },
|
|
2872
|
+
{ label: "mbedtls", dir: join(rtDir, "..", "vendor", "mbedtls") },
|
|
2873
|
+
{ label: "zlib", dir: join(rtDir, "..", "vendor", "zlib") },
|
|
2874
|
+
{ label: "curl", dir: join(rtDir, "..", "vendor", "curl") },
|
|
2455
2875
|
].map(async (group) => {
|
|
2456
2876
|
const names = (await nativeSourceFiles(group.dir, true))
|
|
2457
2877
|
.map((path) => relative(group.dir, path))
|
|
@@ -2460,7 +2880,10 @@ async function runtimeFingerprintInputGroups(rtDir) {
|
|
|
2460
2880
|
}));
|
|
2461
2881
|
}
|
|
2462
2882
|
async function runtimeFingerprintInputPaths(rtDir) {
|
|
2463
|
-
return
|
|
2883
|
+
return [
|
|
2884
|
+
...(await runtimeFingerprintInputGroups(rtDir)).flatMap((group) => group.names.map((name) => join(group.dir, name))),
|
|
2885
|
+
CC_IMPLEMENTATION_PATH,
|
|
2886
|
+
];
|
|
2464
2887
|
}
|
|
2465
2888
|
const stableRuntimeFingerprintMemos = new Map();
|
|
2466
2889
|
export function runtimeFingerprint(rtDir) {
|
|
@@ -2646,7 +3069,22 @@ async function snapshotLocalArtifactDependencies(dependencyPaths, recursiveDirec
|
|
|
2646
3069
|
const recursive = new Set(recursiveDirectories.map((path) => resolve(path)));
|
|
2647
3070
|
return Promise.all([...new Set(dependencyPaths)].sort().map((path) => snapshotLocalArtifactDependency(path, recursive.has(resolve(path)) ? recursiveExclusions : null)));
|
|
2648
3071
|
}
|
|
2649
|
-
async function
|
|
3072
|
+
export async function nativeArtifactDependenciesStillMatch(dependencies) {
|
|
3073
|
+
if (!dependencies.every((dependency) => dependency !== null && typeof dependency === "object" &&
|
|
3074
|
+
typeof dependency.path === "string" &&
|
|
3075
|
+
(dependency.kind === "file" || dependency.kind === "directory" || dependency.kind === "symlink") &&
|
|
3076
|
+
typeof dependency.dev === "number" && typeof dependency.ino === "number" &&
|
|
3077
|
+
typeof dependency.size === "number" && typeof dependency.mtimeMs === "number" &&
|
|
3078
|
+
typeof dependency.ctimeMs === "number" &&
|
|
3079
|
+
(dependency.treeDigest === undefined || typeof dependency.treeDigest === "string") &&
|
|
3080
|
+
(dependency.treeExclusions === undefined || (Array.isArray(dependency.treeExclusions) &&
|
|
3081
|
+
dependency.treeExclusions.every((path) => typeof path === "string"))) &&
|
|
3082
|
+
(dependency.kind !== "symlink" || (typeof dependency.targetPath === "string" &&
|
|
3083
|
+
(dependency.targetKind === "file" || dependency.targetKind === "directory") &&
|
|
3084
|
+
typeof dependency.targetDev === "number" && typeof dependency.targetIno === "number" &&
|
|
3085
|
+
typeof dependency.targetSize === "number" && typeof dependency.targetMtimeMs === "number" &&
|
|
3086
|
+
typeof dependency.targetCtimeMs === "number"))))
|
|
3087
|
+
return false;
|
|
2650
3088
|
return (await Promise.all(dependencies.map(async (dependency) => {
|
|
2651
3089
|
const current = await snapshotLocalArtifactDependency(dependency.path, dependency.treeDigest === undefined ? null : dependency.treeExclusions ?? []).catch(() => null);
|
|
2652
3090
|
return current !== null && JSON.stringify(current) === JSON.stringify(dependency);
|
|
@@ -2676,7 +3114,7 @@ async function readNativeMetadataStamp(root, key) {
|
|
|
2676
3114
|
values: stamp.values,
|
|
2677
3115
|
dependencies: stamp.dependencies,
|
|
2678
3116
|
}) !== stamp.integrity ||
|
|
2679
|
-
!(await
|
|
3117
|
+
!(await nativeArtifactDependenciesStillMatch(stamp.dependencies))) {
|
|
2680
3118
|
return null;
|
|
2681
3119
|
}
|
|
2682
3120
|
return stamp;
|
|
@@ -2722,11 +3160,14 @@ function nativeMetadataKey(kind, parts) {
|
|
|
2722
3160
|
* The generated TU bytes, every scriptc runtime source, the selected direct
|
|
2723
3161
|
* compiler inode, target/options/environment, and the output path all join the
|
|
2724
3162
|
* key. A digest rejects a modified/truncated output before the no-op hit. */
|
|
2725
|
-
function localArtifactIdentity(opts, driver, environmentFingerprint, compilerIdentity, runtimeHash, programBytes) {
|
|
3163
|
+
function localArtifactIdentity(opts, driver, environmentFingerprint, compilerIdentity, runtimeHash, programBytes, programShardMerge) {
|
|
2726
3164
|
const normalizedOptions = Object.fromEntries(Object.entries(opts)
|
|
2727
|
-
.
|
|
3165
|
+
// Shard source can be tens of megabytes. Hash it incrementally below
|
|
3166
|
+
// instead of materializing a second giant JSON string solely for this
|
|
3167
|
+
// output-local fast-path identity.
|
|
3168
|
+
.filter(([key, value]) => value !== undefined && key !== "programShards" && key !== "programPublicSymbols")
|
|
2728
3169
|
.sort(([a], [b]) => a.localeCompare(b)));
|
|
2729
|
-
|
|
3170
|
+
const hash = createHash("sha256")
|
|
2730
3171
|
.update("local-artifact-v1\0")
|
|
2731
3172
|
.update(cacheTargetIdentity(driver)).update("\0")
|
|
2732
3173
|
.update(environmentFingerprint).update("\0")
|
|
@@ -2734,7 +3175,11 @@ function localArtifactIdentity(opts, driver, environmentFingerprint, compilerIde
|
|
|
2734
3175
|
.update(runtimeHash).update("\0")
|
|
2735
3176
|
.update(driver.argv.join("\x1f")).update("\0")
|
|
2736
3177
|
.update(driver.targetArgs.join("\x1f")).update("\0")
|
|
2737
|
-
.update(driver.linkArgs.join("\x1f")).update("\0")
|
|
3178
|
+
.update(driver.linkArgs.join("\x1f")).update("\0");
|
|
3179
|
+
if (programShardMerge !== null) {
|
|
3180
|
+
updateProgramShardCacheIdentity(hash, opts.programShards, opts.programPublicSymbols, programShardMerge);
|
|
3181
|
+
}
|
|
3182
|
+
return hash
|
|
2738
3183
|
.update(process.env["SCRIPTC_FETCH_CURL"] === "1" ? "fetch-curl" : "fetch-native").update("\0")
|
|
2739
3184
|
.update(JSON.stringify(normalizedOptions)).update("\0")
|
|
2740
3185
|
.update(resolve(opts.cPath)).update("\0")
|
|
@@ -2789,16 +3234,16 @@ async function localArtifactHit(stampPath, outPath, key) {
|
|
|
2789
3234
|
typeof dependency.targetSize !== "number" ||
|
|
2790
3235
|
typeof dependency.targetMtimeMs !== "number" ||
|
|
2791
3236
|
typeof dependency.targetCtimeMs !== "number")) ||
|
|
2792
|
-
!(await
|
|
3237
|
+
!(await nativeArtifactDependenciesStillMatch(stamp.dependencies)) ||
|
|
2793
3238
|
await fileDigest(outPath) !== stamp.digest) {
|
|
2794
|
-
return
|
|
3239
|
+
return null;
|
|
2795
3240
|
}
|
|
2796
3241
|
const now = new Date();
|
|
2797
3242
|
await utimes(stampPath, now, now).catch(() => undefined);
|
|
2798
|
-
return
|
|
3243
|
+
return stamp;
|
|
2799
3244
|
}
|
|
2800
3245
|
catch {
|
|
2801
|
-
return
|
|
3246
|
+
return null;
|
|
2802
3247
|
}
|
|
2803
3248
|
}
|
|
2804
3249
|
async function publishLocalArtifactStamp(stampPath, outPath, key, dependencyPaths, recursiveDirectories = [], recursiveExclusions = []) {
|
|
@@ -2818,6 +3263,7 @@ async function publishLocalArtifactStamp(stampPath, outPath, key, dependencyPath
|
|
|
2818
3263
|
};
|
|
2819
3264
|
await writeFile(tmp, `${JSON.stringify(stamp)}\n`, { mode: 0o600 });
|
|
2820
3265
|
await rename(tmp, stampPath);
|
|
3266
|
+
return stamp;
|
|
2821
3267
|
}
|
|
2822
3268
|
finally {
|
|
2823
3269
|
await rm(tmp, { force: true }).catch(() => undefined);
|
|
@@ -2828,7 +3274,7 @@ async function publishLocalArtifactStamp(stampPath, outPath, key, dependencyPath
|
|
|
2828
3274
|
* per-file atomic renames make every winner equivalent. Publication is held
|
|
2829
3275
|
* until verifyInputs confirms that the source/header fingerprint used by the
|
|
2830
3276
|
* key still describes the bytes clang just read. */
|
|
2831
|
-
async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, verifyInputs) {
|
|
3277
|
+
async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, verifyInputs, protectedPaths) {
|
|
2832
3278
|
const setKey = createHash("sha256")
|
|
2833
3279
|
.update(keyPrefix)
|
|
2834
3280
|
.update(cflags.join("\x1f"))
|
|
@@ -2836,6 +3282,13 @@ async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, ve
|
|
|
2836
3282
|
.slice(0, 24);
|
|
2837
3283
|
const objDir = join(root, "obj", setKey);
|
|
2838
3284
|
const objOf = (src) => join(objDir, `${basename(src, ".c")}.o`);
|
|
3285
|
+
if (protectedPaths !== undefined) {
|
|
3286
|
+
for (const source of sources) {
|
|
3287
|
+
const object = objOf(source);
|
|
3288
|
+
protectedPaths.add(object);
|
|
3289
|
+
protectedPaths.add(cacheDigestPath(object));
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
2839
3292
|
const present = await Promise.all(sources.map((s) => validCachedFile(objOf(s))));
|
|
2840
3293
|
const missing = sources.filter((_, i) => !present[i]);
|
|
2841
3294
|
if (missing.length > 0) {
|
|
@@ -2846,7 +3299,7 @@ async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, ve
|
|
|
2846
3299
|
ccArgv[0] === "clang" &&
|
|
2847
3300
|
(await ccacheAvailable());
|
|
2848
3301
|
await mkdir(objDir, { recursive: true });
|
|
2849
|
-
const tmpDir = await mkdtemp(join(
|
|
3302
|
+
const tmpDir = await mkdtemp(join(tmpdir(), "scriptc-cache-obj-"));
|
|
2850
3303
|
try {
|
|
2851
3304
|
const compiled = new Map();
|
|
2852
3305
|
// Modest parallelism: a flavor's objects build once, but several cold
|
|
@@ -2872,7 +3325,7 @@ async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, ve
|
|
|
2872
3325
|
},
|
|
2873
3326
|
}
|
|
2874
3327
|
: undefined);
|
|
2875
|
-
compiled.set(src,
|
|
3328
|
+
compiled.set(src, tmpObj);
|
|
2876
3329
|
}));
|
|
2877
3330
|
}
|
|
2878
3331
|
// The fingerprint was computed before these subprocesses started. Do
|
|
@@ -2882,12 +3335,7 @@ async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, ve
|
|
|
2882
3335
|
throw new CacheInputsChangedError();
|
|
2883
3336
|
for (const [src, built] of compiled) {
|
|
2884
3337
|
const destination = objOf(src);
|
|
2885
|
-
|
|
2886
|
-
await writeFile(tmpDigest, `${built.digest}\n`);
|
|
2887
|
-
// Publish data first and its verifier second. A concurrent reader in
|
|
2888
|
-
// the tiny gap sees a missing/mismatched digest and falls back safely.
|
|
2889
|
-
await rename(built.object, destination);
|
|
2890
|
-
await rename(tmpDigest, cacheDigestPath(destination));
|
|
3338
|
+
await publishCachedFile(built, destination);
|
|
2891
3339
|
}
|
|
2892
3340
|
}
|
|
2893
3341
|
finally {
|
|
@@ -2934,7 +3382,7 @@ export async function stageRuntimeObjects(objects, stageDir) {
|
|
|
2934
3382
|
* mtimes. Active links use private staged names/hard links, so cache names can
|
|
2935
3383
|
* be unlinked safely. */
|
|
2936
3384
|
const rootWriteCounts = new Map();
|
|
2937
|
-
async function pruneCache(root) {
|
|
3385
|
+
async function pruneCache(root, protectedPaths) {
|
|
2938
3386
|
const configuredCap = process.env["SCRIPTC_CACHE_MAX_MB"];
|
|
2939
3387
|
const writes = (rootWriteCounts.get(root) ?? 0) + 1;
|
|
2940
3388
|
rootWriteCounts.set(root, writes);
|
|
@@ -2951,6 +3399,14 @@ async function pruneCache(root) {
|
|
|
2951
3399
|
if (ent.isDirectory())
|
|
2952
3400
|
await walk(p);
|
|
2953
3401
|
else if (ent.isFile()) {
|
|
3402
|
+
// Atomic publishers use private names until their data/digest or
|
|
3403
|
+
// metadata stamp is complete. They are active writes, not LRU entries.
|
|
3404
|
+
if (ent.name.startsWith(".scriptc-") ||
|
|
3405
|
+
ent.name.startsWith(".tmp-") ||
|
|
3406
|
+
ent.name.includes(".tmp-"))
|
|
3407
|
+
continue;
|
|
3408
|
+
if (protectedPaths?.has(p))
|
|
3409
|
+
continue;
|
|
2954
3410
|
const s = await stat(p).catch(() => null);
|
|
2955
3411
|
if (s !== null)
|
|
2956
3412
|
files.push({ path: p, size: s.size, mtimeMs: s.mtimeMs });
|
|
@@ -2988,9 +3444,10 @@ async function pruneCache(root) {
|
|
|
2988
3444
|
* unchanged programs skip payload code generation/linking via the binary
|
|
2989
3445
|
* cache after lightweight metadata probes, and misses link the program's own
|
|
2990
3446
|
* TU against cached per-flavor runtime objects. */
|
|
2991
|
-
|
|
3447
|
+
async function compileCInternal(opts, cacheWarmOnly, cacheWarmPaths) {
|
|
2992
3448
|
const rtDir = runtimeSrcDir();
|
|
2993
3449
|
const sanitize = opts.sanitize ?? false;
|
|
3450
|
+
const optimization = opts.optimization ?? "release";
|
|
2994
3451
|
const dynamic = opts.dynamic ?? false;
|
|
2995
3452
|
const regex = opts.regex ?? false;
|
|
2996
3453
|
// fetch's implementation switch: the default is the NATIVE bridge
|
|
@@ -3014,6 +3471,24 @@ export async function compileC(opts) {
|
|
|
3014
3471
|
const tls = (opts.tls ?? false) || nativeFetch || netIsland;
|
|
3015
3472
|
const tlsCa = (opts.tlsCa ?? false) || tls;
|
|
3016
3473
|
const driver = resolveCc();
|
|
3474
|
+
const shardNames = new Set();
|
|
3475
|
+
const programShardsValid = opts.programShards?.every((shard) => {
|
|
3476
|
+
if (basename(shard.name) !== shard.name || !shard.name.endsWith(".ll") ||
|
|
3477
|
+
shardNames.has(shard.name))
|
|
3478
|
+
return false;
|
|
3479
|
+
shardNames.add(shard.name);
|
|
3480
|
+
return true;
|
|
3481
|
+
}) === true;
|
|
3482
|
+
const programShardsRequested = optimization === "dev" && !sanitize && opts.cPath.endsWith(".ll") &&
|
|
3483
|
+
programShardsValid && opts.programShards !== undefined && opts.programShards.length > 1 &&
|
|
3484
|
+
opts.programPublicSymbols !== undefined
|
|
3485
|
+
? opts.programShards
|
|
3486
|
+
: null;
|
|
3487
|
+
const programShardMergeIdentity = programShardsRequested === null
|
|
3488
|
+
? null
|
|
3489
|
+
: await resolveProgramShardMergeIdentity(driver);
|
|
3490
|
+
const programShards = programShardMergeIdentity === null ? null : programShardsRequested;
|
|
3491
|
+
const programPublicSymbols = programShards === null ? undefined : opts.programPublicSymbols;
|
|
3017
3492
|
// Mobile triples produce library archives, never standalone executables:
|
|
3018
3493
|
// the executable-lane runtime (event loop, sockets, child processes) is
|
|
3019
3494
|
// not verified on those device classes. compile() reports the SC3002
|
|
@@ -3034,7 +3509,7 @@ export async function compileC(opts) {
|
|
|
3034
3509
|
// See the resolveCc block: these inputs are built on and for the HOST
|
|
3035
3510
|
// (vendored archives, system libs). Regex, zlib, and the engine archive
|
|
3036
3511
|
// are NOT here: their vendored sources are plain C that ensureLreObjects
|
|
3037
|
-
// / ensureZlibObjects /
|
|
3512
|
+
// / ensureZlibObjects / buildEngineArchiveDirect compile per target with
|
|
3038
3513
|
// the driver itself — win32 included (the Windows lane runs the
|
|
3039
3514
|
// @dynamic corpus and the zlib program against the box's Node). The
|
|
3040
3515
|
// NATIVE fetch rides the socket units and cross-compiles with them —
|
|
@@ -3075,11 +3550,19 @@ export async function compileC(opts) {
|
|
|
3075
3550
|
let root = opts.cacheIdentity === undefined || !persistentDriverCache
|
|
3076
3551
|
? null
|
|
3077
3552
|
: configuredCacheRoot;
|
|
3553
|
+
if (cacheWarmOnly && root === null) {
|
|
3554
|
+
throw new Error("native cache warming requires a persistently cacheable compiler environment");
|
|
3555
|
+
}
|
|
3078
3556
|
if (root !== null) {
|
|
3079
3557
|
try {
|
|
3080
3558
|
await ensurePrivateCacheRoot(root, process.env["SCRIPTC_CACHE_DIR"] === undefined);
|
|
3081
3559
|
}
|
|
3082
|
-
catch {
|
|
3560
|
+
catch (error) {
|
|
3561
|
+
if (cacheWarmOnly) {
|
|
3562
|
+
throw new Error("native cache warming could not prepare the persistent cache root", {
|
|
3563
|
+
cause: error,
|
|
3564
|
+
});
|
|
3565
|
+
}
|
|
3083
3566
|
root = null;
|
|
3084
3567
|
}
|
|
3085
3568
|
}
|
|
@@ -3089,7 +3572,8 @@ export async function compileC(opts) {
|
|
|
3089
3572
|
// broader cross-output CAS performs its compiler/SDK/linker rediscovery.
|
|
3090
3573
|
// FFI/native-input builds and the public arbitrary-C cache API stay on the
|
|
3091
3574
|
// strict path because their dependency graphs are caller-owned.
|
|
3092
|
-
if (
|
|
3575
|
+
if (!cacheWarmOnly &&
|
|
3576
|
+
root !== null &&
|
|
3093
3577
|
cachePolicy.completeArtifacts &&
|
|
3094
3578
|
opts.cacheIdentity === "scriptc-generated-v1" &&
|
|
3095
3579
|
(opts.linkInputs?.length ?? 0) === 0 &&
|
|
@@ -3103,7 +3587,7 @@ export async function compileC(opts) {
|
|
|
3103
3587
|
]);
|
|
3104
3588
|
if (compiler !== null) {
|
|
3105
3589
|
const effectiveCompiler = directCompilerSelections.get(compilerDriverProbeKey(driver, toolchainEnv)) ?? compiler;
|
|
3106
|
-
const key = localArtifactIdentity(opts, driver, toolchainEnv, `${compiler.cacheIdentity}\0${effectiveCompiler.cacheIdentity}`, runtimeHash, programBytes);
|
|
3590
|
+
const key = localArtifactIdentity(opts, driver, toolchainEnv, `${compiler.cacheIdentity}\0${effectiveCompiler.cacheIdentity}`, runtimeHash, programBytes, programShardMergeIdentity);
|
|
3107
3591
|
const stampPath = localArtifactStampPath(root, opts.outPath);
|
|
3108
3592
|
localArtifact = {
|
|
3109
3593
|
stampPath,
|
|
@@ -3112,8 +3596,11 @@ export async function compileC(opts) {
|
|
|
3112
3596
|
programBytes,
|
|
3113
3597
|
compilerPath: effectiveCompiler.canonicalPath,
|
|
3114
3598
|
};
|
|
3115
|
-
|
|
3599
|
+
const hit = await localArtifactHit(stampPath, opts.outPath, key);
|
|
3600
|
+
if (hit !== null) {
|
|
3601
|
+
await opts.onArtifactReady?.({ dependencies: hit.dependencies }).catch(() => undefined);
|
|
3116
3602
|
return;
|
|
3603
|
+
}
|
|
3117
3604
|
}
|
|
3118
3605
|
}
|
|
3119
3606
|
catch {
|
|
@@ -3123,6 +3610,7 @@ export async function compileC(opts) {
|
|
|
3123
3610
|
}
|
|
3124
3611
|
}
|
|
3125
3612
|
let implicitToolchain = null;
|
|
3613
|
+
let implicitCompileToolchain = null;
|
|
3126
3614
|
let toolchainMetadataStamp = null;
|
|
3127
3615
|
const metadataCompiler = root === null
|
|
3128
3616
|
? null
|
|
@@ -3148,8 +3636,16 @@ export async function compileC(opts) {
|
|
|
3148
3636
|
root === null || toolchainMetadataKey === null
|
|
3149
3637
|
? null
|
|
3150
3638
|
: await readNativeMetadataStamp(root, toolchainMetadataKey);
|
|
3151
|
-
implicitToolchain = toolchainMetadataStamp?.values["implicitToolchain"] ??
|
|
3152
|
-
|
|
3639
|
+
implicitToolchain = toolchainMetadataStamp?.values["implicitToolchain"] ?? null;
|
|
3640
|
+
implicitCompileToolchain =
|
|
3641
|
+
toolchainMetadataStamp?.values["implicitCompileToolchain"] ?? null;
|
|
3642
|
+
if (implicitToolchain === null ||
|
|
3643
|
+
(programShards !== null && implicitCompileToolchain === null)) {
|
|
3644
|
+
const fingerprints = await implicitToolchainFingerprints(driver, toolchainEnv);
|
|
3645
|
+
implicitToolchain = fingerprints.complete;
|
|
3646
|
+
implicitCompileToolchain = fingerprints.compile;
|
|
3647
|
+
toolchainMetadataStamp = null;
|
|
3648
|
+
}
|
|
3153
3649
|
let compilerVersion = toolchainMetadataStamp?.values["compilerVersion"];
|
|
3154
3650
|
if (compilerVersion === undefined) {
|
|
3155
3651
|
compilerVersion = await ccVersionOnce(driver.argv, toolchainEnv, true);
|
|
@@ -3158,73 +3654,122 @@ export async function compileC(opts) {
|
|
|
3158
3654
|
root !== null &&
|
|
3159
3655
|
toolchainMetadataKey !== null) {
|
|
3160
3656
|
if (metadataCompiler !== null && metadataEffectiveCompiler !== null) {
|
|
3161
|
-
toolchainMetadataStamp = await publishNativeMetadataStamp(root, toolchainMetadataKey, {
|
|
3657
|
+
toolchainMetadataStamp = await publishNativeMetadataStamp(root, toolchainMetadataKey, {
|
|
3658
|
+
implicitToolchain,
|
|
3659
|
+
...(implicitCompileToolchain === null ? {} : { implicitCompileToolchain }),
|
|
3660
|
+
compilerVersion,
|
|
3661
|
+
}, [
|
|
3162
3662
|
metadataCompiler.canonicalPath,
|
|
3163
3663
|
metadataEffectiveCompiler.canonicalPath,
|
|
3164
3664
|
...fingerprintDependencyPaths(implicitToolchain),
|
|
3165
|
-
|
|
3665
|
+
...(implicitCompileToolchain === null
|
|
3666
|
+
? []
|
|
3667
|
+
: fingerprintDependencyPaths(implicitCompileToolchain)),
|
|
3668
|
+
], [
|
|
3669
|
+
implicitToolchain,
|
|
3670
|
+
...(implicitCompileToolchain === null ? [] : [implicitCompileToolchain]),
|
|
3671
|
+
]);
|
|
3166
3672
|
}
|
|
3167
3673
|
}
|
|
3168
3674
|
}
|
|
3169
|
-
catch {
|
|
3675
|
+
catch (error) {
|
|
3170
3676
|
// Cache discovery is best-effort. In particular, a compiler wrapper can
|
|
3171
3677
|
// compile successfully without implementing the metadata probes.
|
|
3678
|
+
if (cacheWarmOnly) {
|
|
3679
|
+
throw new Error("native cache warming could not validate the compiler toolchain", {
|
|
3680
|
+
cause: error,
|
|
3681
|
+
});
|
|
3682
|
+
}
|
|
3172
3683
|
root = null;
|
|
3173
3684
|
}
|
|
3174
3685
|
}
|
|
3175
3686
|
const vendorBuildIdentity = await currentVendorCacheBuildIdentity(driver, `${toolchainEnv}\0${implicitToolchain ?? "<uncached>"}`);
|
|
3176
3687
|
// Mutable include/SDK/config inputs can change behind a stable environment
|
|
3177
3688
|
// spelling. The complete/runtime cache is disabled below; vendor objects
|
|
3178
|
-
// must follow the same rule instead of silently surviving in
|
|
3179
|
-
//
|
|
3689
|
+
// must follow the same rule instead of silently surviving in the user cache.
|
|
3690
|
+
// A private root gives this invocation the usual vendor build recipe
|
|
3180
3691
|
// without publishing or reusing those prerequisites.
|
|
3181
|
-
const transientVendorRoot =
|
|
3692
|
+
const transientVendorRoot = root !== null && implicitToolchain !== null
|
|
3182
3693
|
? null
|
|
3183
3694
|
: join(tmpdir(), `scriptc-vendor-${process.pid}-${Math.random().toString(36).slice(2)}`);
|
|
3184
|
-
const vendorCacheRoot = transientVendorRoot ?? vendorBuildCacheRoot();
|
|
3695
|
+
const vendorCacheRoot = transientVendorRoot ?? vendorBuildCacheRoot(root ?? undefined);
|
|
3185
3696
|
// Every vendor output path is deterministic from pins, flags, driver, and
|
|
3186
3697
|
// target. Build the command/key from those paths now, but do not materialize
|
|
3187
3698
|
// them until a complete-binary lookup has missed.
|
|
3188
|
-
|
|
3699
|
+
let engineArchive = dynamic
|
|
3189
3700
|
? engineArchivePath(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
|
3190
3701
|
: null;
|
|
3191
|
-
|
|
3702
|
+
let tlsArchive = tls
|
|
3192
3703
|
? tlsArchivePath(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
|
3193
3704
|
: null;
|
|
3194
3705
|
// --dynamic + regex shares the archive's libregexp (its host hooks and
|
|
3195
3706
|
// ours would collide; see scr_regex.c) — the standalone objects are for
|
|
3196
3707
|
// static builds only.
|
|
3197
|
-
|
|
3708
|
+
let lreObjects = regex && !dynamic
|
|
3198
3709
|
? lreObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
|
3199
3710
|
: [];
|
|
3200
3711
|
// Vendored zlib is the CROSS story only — host builds keep the exact
|
|
3201
3712
|
// historical `-lz` system link (see CcOptions.zlib). The native fetch's
|
|
3202
3713
|
// gzip decoder rides the same objects/link.
|
|
3203
|
-
|
|
3714
|
+
let zlibObjects = ((opts.zlib ?? false) || nativeFetch) && driver.target !== null
|
|
3204
3715
|
? zlibObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
|
3205
3716
|
: [];
|
|
3206
3717
|
// The libcurl import stub is likewise CROSS-only — host builds keep the
|
|
3207
3718
|
// exact historical system `-lcurl` link (see CcOptions.fetch). Curl
|
|
3208
3719
|
// reference builds only.
|
|
3209
|
-
|
|
3720
|
+
let curlStubDir = curlFetch && driver.target !== null
|
|
3210
3721
|
? curlStubDirPath(driver, vendorBuildIdentity, vendorCacheRoot)
|
|
3211
3722
|
: null;
|
|
3212
|
-
const materializeVendorPrerequisites = async () => {
|
|
3723
|
+
const materializeVendorPrerequisites = async (stageRoot, materializeCacheRoot = vendorCacheRoot) => {
|
|
3213
3724
|
// Preserve the historical order to avoid multiplying first-build resource
|
|
3214
3725
|
// pressure when several large vendor sets are cold simultaneously.
|
|
3215
3726
|
if (dynamic) {
|
|
3216
|
-
|
|
3727
|
+
const cachedArchive = engineArchivePath(sanitize, driver, vendorBuildIdentity, materializeCacheRoot);
|
|
3728
|
+
protectCachedArtifact(cacheWarmPaths, cachedArchive);
|
|
3729
|
+
const materialize = async () => [
|
|
3730
|
+
await ensureEngineArchive(sanitize, driver, vendorBuildIdentity, materializeCacheRoot),
|
|
3731
|
+
];
|
|
3732
|
+
const paths = stageRoot === undefined
|
|
3733
|
+
? await materialize()
|
|
3734
|
+
: await stageVendorInputs(materialize, join(stageRoot, "engine"));
|
|
3735
|
+
engineArchive = paths[0];
|
|
3736
|
+
}
|
|
3737
|
+
if (tls) {
|
|
3738
|
+
const cachedArchive = tlsArchivePath(sanitize, driver, vendorBuildIdentity, materializeCacheRoot);
|
|
3739
|
+
protectCachedArtifact(cacheWarmPaths, cachedArchive);
|
|
3740
|
+
const materialize = async () => [
|
|
3741
|
+
await ensureTlsArchive(sanitize, driver, vendorBuildIdentity, materializeCacheRoot),
|
|
3742
|
+
];
|
|
3743
|
+
const paths = stageRoot === undefined
|
|
3744
|
+
? await materialize()
|
|
3745
|
+
: await stageVendorInputs(materialize, join(stageRoot, "tls"));
|
|
3746
|
+
tlsArchive = paths[0];
|
|
3217
3747
|
}
|
|
3218
|
-
if (tls)
|
|
3219
|
-
await ensureTlsArchive(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
|
|
3220
3748
|
if (regex && !dynamic) {
|
|
3221
|
-
|
|
3749
|
+
for (const object of lreObjectPaths(sanitize, driver, vendorBuildIdentity, materializeCacheRoot))
|
|
3750
|
+
protectCachedArtifact(cacheWarmPaths, object);
|
|
3751
|
+
const materialize = async () => await ensureLreObjects(sanitize, driver, vendorBuildIdentity, materializeCacheRoot);
|
|
3752
|
+
lreObjects = stageRoot === undefined
|
|
3753
|
+
? await materialize()
|
|
3754
|
+
: await stageVendorInputs(materialize, join(stageRoot, "lre"));
|
|
3222
3755
|
}
|
|
3223
3756
|
if (zlibObjects.length > 0) {
|
|
3224
|
-
|
|
3757
|
+
for (const object of zlibObjectPaths(sanitize, driver, vendorBuildIdentity, materializeCacheRoot))
|
|
3758
|
+
protectCachedArtifact(cacheWarmPaths, object);
|
|
3759
|
+
const materialize = async () => await ensureZlibObjects(sanitize, driver, vendorBuildIdentity, materializeCacheRoot);
|
|
3760
|
+
zlibObjects = stageRoot === undefined
|
|
3761
|
+
? await materialize()
|
|
3762
|
+
: await stageVendorInputs(materialize, join(stageRoot, "zlib"));
|
|
3225
3763
|
}
|
|
3226
3764
|
if (curlStubDir !== null) {
|
|
3227
|
-
|
|
3765
|
+
protectCachedArtifact(cacheWarmPaths, join(curlStubDirPath(driver, vendorBuildIdentity, materializeCacheRoot), "libcurl.so"));
|
|
3766
|
+
const materialize = async () => [
|
|
3767
|
+
join(await ensureCurlStub(driver, vendorBuildIdentity, materializeCacheRoot), "libcurl.so"),
|
|
3768
|
+
];
|
|
3769
|
+
const paths = stageRoot === undefined
|
|
3770
|
+
? await materialize()
|
|
3771
|
+
: await stageVendorInputs(materialize, join(stageRoot, "curl"));
|
|
3772
|
+
curlStubDir = dirname(paths[0]);
|
|
3228
3773
|
}
|
|
3229
3774
|
};
|
|
3230
3775
|
// rt() maps each runtime source's path on the command line: identity for
|
|
@@ -3235,7 +3780,7 @@ export async function compileC(opts) {
|
|
|
3235
3780
|
...threadArgs,
|
|
3236
3781
|
...(sanitize
|
|
3237
3782
|
? ["-O1", "-fsanitize=address", "-DSCR_RC_AUDIT"]
|
|
3238
|
-
: ["-O2"]),
|
|
3783
|
+
: [optimization === "dev" ? "-O0" : "-O2"]),
|
|
3239
3784
|
...(opts.textDecoderLegacy ? ["-DSCR_TEXT_DECODER_LEGACY"] : []),
|
|
3240
3785
|
"-fno-math-errno",
|
|
3241
3786
|
// The emitted object model is deliberately type-punned C: a hierarchy
|
|
@@ -3429,7 +3974,7 @@ export async function compileC(opts) {
|
|
|
3429
3974
|
...threadArgs,
|
|
3430
3975
|
...(sanitize
|
|
3431
3976
|
? ["-O1", "-fsanitize=address", "-DSCR_RC_AUDIT"]
|
|
3432
|
-
: ["-O2"]),
|
|
3977
|
+
: [optimization === "dev" ? "-O0" : "-O2"]),
|
|
3433
3978
|
...(opts.textDecoderLegacy ? ["-DSCR_TEXT_DECODER_LEGACY"] : []),
|
|
3434
3979
|
"-fno-math-errno",
|
|
3435
3980
|
"-fno-strict-aliasing", // the emitted object model type-puns — see buildArgs
|
|
@@ -3462,6 +4007,16 @@ export async function compileC(opts) {
|
|
|
3462
4007
|
`${guidance}\n\n${stderr}`);
|
|
3463
4008
|
}
|
|
3464
4009
|
};
|
|
4010
|
+
const runUncachedBuild = async () => {
|
|
4011
|
+
const privateVendorRoot = transientVendorRoot ?? join(tmpdir(), `scriptc-vendor-fallback-${process.pid}-${Math.random().toString(36).slice(2)}`);
|
|
4012
|
+
try {
|
|
4013
|
+
await materializeVendorPrerequisites(undefined, privateVendorRoot);
|
|
4014
|
+
await runClang(buildArgs((p) => p));
|
|
4015
|
+
}
|
|
4016
|
+
finally {
|
|
4017
|
+
await rm(privateVendorRoot, { recursive: true, force: true }).catch(() => undefined);
|
|
4018
|
+
}
|
|
4019
|
+
};
|
|
3465
4020
|
let runtimeCompilerInvocation = null;
|
|
3466
4021
|
let programCompilerInvocation = null;
|
|
3467
4022
|
let payloadMetadata = null;
|
|
@@ -3522,23 +4077,20 @@ export async function compileC(opts) {
|
|
|
3522
4077
|
}
|
|
3523
4078
|
}
|
|
3524
4079
|
}
|
|
3525
|
-
catch {
|
|
4080
|
+
catch (error) {
|
|
3526
4081
|
// Preserve the uncached build for wrappers that compile successfully but
|
|
3527
4082
|
// cannot provide a dry-run trace for the real build flavor.
|
|
4083
|
+
if (cacheWarmOnly) {
|
|
4084
|
+
throw new Error("native cache warming could not validate the compiler invocation", {
|
|
4085
|
+
cause: error,
|
|
4086
|
+
});
|
|
4087
|
+
}
|
|
3528
4088
|
root = null;
|
|
3529
4089
|
}
|
|
3530
4090
|
}
|
|
3531
4091
|
if (root === null) {
|
|
3532
4092
|
// The exact historical command line, byte for byte.
|
|
3533
|
-
|
|
3534
|
-
await materializeVendorPrerequisites();
|
|
3535
|
-
await runClang(buildArgs((p) => p));
|
|
3536
|
-
}
|
|
3537
|
-
finally {
|
|
3538
|
-
if (transientVendorRoot !== null) {
|
|
3539
|
-
await rm(transientVendorRoot, { recursive: true, force: true }).catch(() => undefined);
|
|
3540
|
-
}
|
|
3541
|
-
}
|
|
4093
|
+
await runUncachedBuild();
|
|
3542
4094
|
return;
|
|
3543
4095
|
}
|
|
3544
4096
|
let cv;
|
|
@@ -3551,12 +4103,16 @@ export async function compileC(opts) {
|
|
|
3551
4103
|
readFile(opts.cPath),
|
|
3552
4104
|
]));
|
|
3553
4105
|
}
|
|
3554
|
-
catch {
|
|
4106
|
+
catch (error) {
|
|
3555
4107
|
// A version/fingerprint probe is an optimization boundary. If the compiler
|
|
3556
4108
|
// itself can still compile, preserve the pre-cache behavior instead of
|
|
3557
4109
|
// surfacing a metadata command's failure as the build result.
|
|
3558
|
-
|
|
3559
|
-
|
|
4110
|
+
if (cacheWarmOnly) {
|
|
4111
|
+
throw new Error("native cache warming could not validate cache inputs", {
|
|
4112
|
+
cause: error,
|
|
4113
|
+
});
|
|
4114
|
+
}
|
|
4115
|
+
await runUncachedBuild();
|
|
3560
4116
|
return;
|
|
3561
4117
|
}
|
|
3562
4118
|
// Caller-supplied native inputs can all hide mutable dependencies: `-l`
|
|
@@ -3564,7 +4120,8 @@ export async function compileC(opts) {
|
|
|
3564
4120
|
// script can retain identical top-level bytes as its referenced files are
|
|
3565
4121
|
// rebuilt. Keep the safe runtime-object cache, but force a fresh final link
|
|
3566
4122
|
// whenever the caller supplies either form.
|
|
3567
|
-
let cacheCompleteArtifact =
|
|
4123
|
+
let cacheCompleteArtifact = !cacheWarmOnly &&
|
|
4124
|
+
cachePolicy.completeArtifacts &&
|
|
3568
4125
|
(opts.linkInputs?.length ?? 0) === 0 &&
|
|
3569
4126
|
(opts.systemLibraries?.length ?? 0) === 0;
|
|
3570
4127
|
let programDependencies = null;
|
|
@@ -3623,6 +4180,7 @@ export async function compileC(opts) {
|
|
|
3623
4180
|
linkProbeArgs,
|
|
3624
4181
|
effectiveLinkInvocationArgs,
|
|
3625
4182
|
linkTraceInvocationArgs,
|
|
4183
|
+
...(programShardMergeIdentity === null ? [] : [programShardMergeIdentity]),
|
|
3626
4184
|
]);
|
|
3627
4185
|
if (cacheCompleteArtifact) {
|
|
3628
4186
|
try {
|
|
@@ -3654,6 +4212,9 @@ export async function compileC(opts) {
|
|
|
3654
4212
|
...(linkMetadataStamp?.dependencies.map((dependency) => dependency.path) ??
|
|
3655
4213
|
fingerprintDependencyPaths(implicitLinker)),
|
|
3656
4214
|
...fingerprintDependencyPaths(programDependencies),
|
|
4215
|
+
...(programShardMergeIdentity === null
|
|
4216
|
+
? []
|
|
4217
|
+
: fingerprintDependencyPaths(programShardMergeIdentity)),
|
|
3657
4218
|
]);
|
|
3658
4219
|
// Every content-bearing fingerprint was computed before this metadata
|
|
3659
4220
|
// snapshot. Re-read those exact files now so the snapshot cannot certify
|
|
@@ -3664,11 +4225,12 @@ export async function compileC(opts) {
|
|
|
3664
4225
|
programCompilerInvocation,
|
|
3665
4226
|
implicitLinker,
|
|
3666
4227
|
programDependencies,
|
|
4228
|
+
...(programShardMergeIdentity === null ? [] : [programShardMergeIdentity]),
|
|
3667
4229
|
])) ||
|
|
3668
4230
|
await runtimeFingerprint(rtDir).catch(() => null) !== fingerprint ||
|
|
3669
4231
|
!(await Promise.all([toolchainMetadataStamp, compileMetadataStamp, linkMetadataStamp]
|
|
3670
4232
|
.filter((stamp) => stamp !== null)
|
|
3671
|
-
.map((stamp) =>
|
|
4233
|
+
.map((stamp) => nativeArtifactDependenciesStillMatch(stamp.dependencies)))).every(Boolean)) {
|
|
3672
4234
|
throw new CacheInputsChangedError();
|
|
3673
4235
|
}
|
|
3674
4236
|
if (localArtifact !== null) {
|
|
@@ -3693,6 +4255,25 @@ export async function compileC(opts) {
|
|
|
3693
4255
|
localArtifactDependencyPaths = null;
|
|
3694
4256
|
}
|
|
3695
4257
|
}
|
|
4258
|
+
// Warm-only builds deliberately do not need linker/header identity for a
|
|
4259
|
+
// complete executable, but they still need a stable runtime-object key.
|
|
4260
|
+
// The ordinary complete-artifact path computes these values above.
|
|
4261
|
+
if (cacheWarmOnly && preBuildDependencies === null) {
|
|
4262
|
+
try {
|
|
4263
|
+
const [dependencies, invocation] = await Promise.all([
|
|
4264
|
+
translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cBytes, toolchainEnv),
|
|
4265
|
+
effectiveCompilerInvocationFingerprint(driver, toolchainEnv, cflags),
|
|
4266
|
+
]);
|
|
4267
|
+
programDependencies = dependencies;
|
|
4268
|
+
if (runtimeCompilerInvocation === null)
|
|
4269
|
+
runtimeCompilerInvocation = invocation;
|
|
4270
|
+
}
|
|
4271
|
+
catch (error) {
|
|
4272
|
+
throw new Error("native cache warming could not validate runtime-object inputs", {
|
|
4273
|
+
cause: error,
|
|
4274
|
+
});
|
|
4275
|
+
}
|
|
4276
|
+
}
|
|
3696
4277
|
const binDir = join(root, "bin");
|
|
3697
4278
|
let keyHex = null;
|
|
3698
4279
|
let cachedBin = null;
|
|
@@ -3704,9 +4285,10 @@ export async function compileC(opts) {
|
|
|
3704
4285
|
// the fingerprint and the pin.
|
|
3705
4286
|
const identityArgs = buildArgs((p) => p).map((a) => a === opts.cPath ? "<program.c>" : a === opts.outPath ? "<out>" : a);
|
|
3706
4287
|
const key = createHash("sha256")
|
|
3707
|
-
//
|
|
3708
|
-
//
|
|
3709
|
-
.
|
|
4288
|
+
// Sharded dev builds need the v10 identity below. Canonical single-TU
|
|
4289
|
+
// builds retain v9 so adding the opt-in mode does not evict release
|
|
4290
|
+
// binaries compiled by an earlier scriptc.
|
|
4291
|
+
.update(programShards === null ? "bin-v9\0" : "bin-v10\0")
|
|
3710
4292
|
.update(cacheTargetIdentity(driver)).update("\0")
|
|
3711
4293
|
.update(toolchainEnv).update("\0")
|
|
3712
4294
|
.update(implicitToolchain).update("\0")
|
|
@@ -3728,6 +4310,9 @@ export async function compileC(opts) {
|
|
|
3728
4310
|
.update(fingerprint).update("\0")
|
|
3729
4311
|
.update(identityArgs.join("\x1f")).update("\0")
|
|
3730
4312
|
.update(cBytes);
|
|
4313
|
+
if (programShards !== null) {
|
|
4314
|
+
updateProgramShardCacheIdentity(key, programShards, programPublicSymbols, programShardMergeIdentity ?? undefined);
|
|
4315
|
+
}
|
|
3731
4316
|
keyHex = key.digest("hex");
|
|
3732
4317
|
cachedBin = join(binDir, keyHex);
|
|
3733
4318
|
const tmpOut = privateSiblingPath(opts.outPath, "bin-hit");
|
|
@@ -3744,7 +4329,10 @@ export async function compileC(opts) {
|
|
|
3744
4329
|
await chmod(tmpOut, 0o777 & ~process.umask());
|
|
3745
4330
|
await rename(tmpOut, opts.outPath);
|
|
3746
4331
|
if (localArtifact !== null && localArtifactDependencyPaths !== null) {
|
|
3747
|
-
await publishLocalArtifactStamp(localArtifact.stampPath, opts.outPath, localArtifact.key, localArtifactDependencyPaths, [dirname(resolve(opts.cPath))], [root]).catch(() =>
|
|
4332
|
+
const stamp = await publishLocalArtifactStamp(localArtifact.stampPath, opts.outPath, localArtifact.key, localArtifactDependencyPaths, [dirname(resolve(opts.cPath))], [root]).catch(() => null);
|
|
4333
|
+
if (stamp !== null) {
|
|
4334
|
+
await opts.onArtifactReady?.({ dependencies: stamp.dependencies }).catch(() => undefined);
|
|
4335
|
+
}
|
|
3748
4336
|
}
|
|
3749
4337
|
return; // hit: the program/runtime payload compile and link were skipped
|
|
3750
4338
|
}
|
|
@@ -3753,7 +4341,6 @@ export async function compileC(opts) {
|
|
|
3753
4341
|
/* miss — build below, then publish */
|
|
3754
4342
|
}
|
|
3755
4343
|
}
|
|
3756
|
-
await materializeVendorPrerequisites();
|
|
3757
4344
|
// Miss: link the program's own TU against cached per-flavor runtime objects.
|
|
3758
4345
|
// Collect the runtime sources this build actually compiles (the same
|
|
3759
4346
|
// conditionals as the command line, by construction).
|
|
@@ -3764,6 +4351,7 @@ export async function compileC(opts) {
|
|
|
3764
4351
|
});
|
|
3765
4352
|
const buildDir = await mkdtemp(join(tmpdir(), "scriptc-cache-build-"));
|
|
3766
4353
|
try {
|
|
4354
|
+
await materializeVendorPrerequisites(join(buildDir, "vendor-inputs"));
|
|
3767
4355
|
// Freeze the exact bytes used for the key. Generated TUs live at stable
|
|
3768
4356
|
// paths under .scriptc/, so two builds can otherwise overwrite that path
|
|
3769
4357
|
// between hashing and clang and publish one invocation's code under the
|
|
@@ -3784,7 +4372,7 @@ export async function compileC(opts) {
|
|
|
3784
4372
|
let strictObjectVerification = null;
|
|
3785
4373
|
const objectImplicitToolchainStillMatches = () => {
|
|
3786
4374
|
if (preBuildDependencies !== null) {
|
|
3787
|
-
return
|
|
4375
|
+
return nativeArtifactDependenciesStillMatch(preBuildDependencies);
|
|
3788
4376
|
}
|
|
3789
4377
|
// Complete-artifact caching can be disabled by caller-owned native
|
|
3790
4378
|
// inputs while the safe runtime-object tier remains active. Preserve its
|
|
@@ -3799,15 +4387,116 @@ export async function compileC(opts) {
|
|
|
3799
4387
|
return strictObjectVerification;
|
|
3800
4388
|
};
|
|
3801
4389
|
try {
|
|
3802
|
-
const cached = await ensureRuntimeObjects(root, driver.argv, cflags, rtInputs, `obj-v5\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${implicitToolchain}\0${runtimeCompilerInvocation}\0${ccName}\0${cv}\0${fingerprint}\0`, async () => await objectImplicitToolchainStillMatches());
|
|
4390
|
+
const cached = await ensureRuntimeObjects(root, driver.argv, cflags, rtInputs, `obj-v5\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${implicitToolchain}\0${runtimeCompilerInvocation}\0${ccName}\0${cv}\0${fingerprint}\0`, async () => await objectImplicitToolchainStillMatches(), cacheWarmPaths);
|
|
3803
4391
|
objects = await stageRuntimeObjects(cached, join(buildDir, "runtime-objects"));
|
|
3804
4392
|
}
|
|
3805
4393
|
catch (err) {
|
|
3806
4394
|
if (err instanceof CacheInputsChangedError)
|
|
3807
4395
|
cacheInputsStable = false;
|
|
4396
|
+
if (cacheWarmOnly) {
|
|
4397
|
+
throw new Error("native cache warming could not persist runtime objects", {
|
|
4398
|
+
cause: err,
|
|
4399
|
+
});
|
|
4400
|
+
}
|
|
3808
4401
|
objects = null; // cache trouble is never a build failure
|
|
3809
4402
|
}
|
|
3810
|
-
|
|
4403
|
+
const compileShardedProgram = async () => {
|
|
4404
|
+
if (programShards === null || programPublicSymbols === undefined ||
|
|
4405
|
+
programCompilerInvocation === null || implicitCompileToolchain === null)
|
|
4406
|
+
return null;
|
|
4407
|
+
try {
|
|
4408
|
+
const programDependencyHash = programDependencies ??
|
|
4409
|
+
await translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cBytes, toolchainEnv);
|
|
4410
|
+
const stem = basename(opts.cPath, ".ll");
|
|
4411
|
+
const entries = programShards.map((shard, index) => {
|
|
4412
|
+
const sourcePath = join(buildDir, shard.name);
|
|
4413
|
+
const staged = join(buildDir, `${stem}.program-${index.toString().padStart(3, "0")}.o`);
|
|
4414
|
+
const key = createHash("sha256")
|
|
4415
|
+
.update("exe-program-shard-v1\0")
|
|
4416
|
+
.update(cacheTargetIdentity(driver)).update("\0")
|
|
4417
|
+
.update(toolchainEnv).update("\0")
|
|
4418
|
+
.update(implicitCompileToolchain).update("\0")
|
|
4419
|
+
.update(programCompilerInvocation).update("\0")
|
|
4420
|
+
.update(opts.cacheIdentity).update("\0")
|
|
4421
|
+
.update(driver.argv.join("\x1f")).update("\0")
|
|
4422
|
+
.update(cv).update("\0")
|
|
4423
|
+
.update(fingerprint).update("\0")
|
|
4424
|
+
.update(programDependencyHash).update("\0")
|
|
4425
|
+
.update(programCompilerArgs.join("\x1f")).update("\0")
|
|
4426
|
+
.update(opts.cPath).update("\0")
|
|
4427
|
+
.update(resolve(opts.cPath)).update("\0")
|
|
4428
|
+
.update(shard.name).update("\0")
|
|
4429
|
+
.update(shard.source)
|
|
4430
|
+
.digest("hex");
|
|
4431
|
+
return {
|
|
4432
|
+
...shard,
|
|
4433
|
+
sourcePath,
|
|
4434
|
+
staged,
|
|
4435
|
+
cachePath: join(root, "program-shard", key),
|
|
4436
|
+
missed: false,
|
|
4437
|
+
};
|
|
4438
|
+
});
|
|
4439
|
+
const shardWidth = Math.min(8, availableParallelism());
|
|
4440
|
+
for (let i = 0; i < entries.length; i += shardWidth) {
|
|
4441
|
+
await Promise.all(entries.slice(i, i + shardWidth).map(async (entry) => {
|
|
4442
|
+
await writeFile(entry.sourcePath, entry.source);
|
|
4443
|
+
if (await copyValidCachedFile(entry.cachePath, entry.staged))
|
|
4444
|
+
return;
|
|
4445
|
+
entry.missed = true;
|
|
4446
|
+
await runClang([
|
|
4447
|
+
...programCompilerArgs,
|
|
4448
|
+
`-ffile-prefix-map=${entry.sourcePath}=${opts.cPath}`,
|
|
4449
|
+
"-c",
|
|
4450
|
+
entry.sourcePath,
|
|
4451
|
+
"-o",
|
|
4452
|
+
entry.staged,
|
|
4453
|
+
]);
|
|
4454
|
+
}));
|
|
4455
|
+
}
|
|
4456
|
+
const arArgv = driver.argv[0] === "zig" ? [driver.argv[0], "ar"] : ["ar"];
|
|
4457
|
+
const merged = await localizeLibraryObjects(driver, arArgv, buildDir, entries.map((entry) => entry.staged), [], programPublicSymbols, `${stem}.program`);
|
|
4458
|
+
const publishable = entries.filter((entry) => entry.missed);
|
|
4459
|
+
if (publishable.length > 0) {
|
|
4460
|
+
try {
|
|
4461
|
+
const [currentRuntime, currentFingerprints, currentInvocation, currentDependencies, currentCompiler, currentMerge] = await Promise.all([
|
|
4462
|
+
runtimeFingerprint(rtDir),
|
|
4463
|
+
implicitToolchainFingerprints(driver, toolchainEnv),
|
|
4464
|
+
effectiveCompilerInvocationFingerprint(driver, toolchainEnv, programCompilerArgs, ".ll"),
|
|
4465
|
+
translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cBytes, toolchainEnv),
|
|
4466
|
+
ccVersionOnce(driver.argv, toolchainEnv, true),
|
|
4467
|
+
resolveProgramShardMergeIdentity(driver),
|
|
4468
|
+
]);
|
|
4469
|
+
if (currentRuntime === fingerprint &&
|
|
4470
|
+
currentFingerprints.compile === implicitCompileToolchain &&
|
|
4471
|
+
currentInvocation === programCompilerInvocation &&
|
|
4472
|
+
currentDependencies === programDependencyHash &&
|
|
4473
|
+
currentCompiler === cv &&
|
|
4474
|
+
currentMerge === programShardMergeIdentity) {
|
|
4475
|
+
await Promise.all(publishable.map((entry) => publishCachedFile(entry.staged, entry.cachePath)));
|
|
4476
|
+
}
|
|
4477
|
+
}
|
|
4478
|
+
catch {
|
|
4479
|
+
// The merged object is valid for this invocation; publication is
|
|
4480
|
+
// only an optimization for later edits.
|
|
4481
|
+
}
|
|
4482
|
+
}
|
|
4483
|
+
return merged;
|
|
4484
|
+
}
|
|
4485
|
+
catch {
|
|
4486
|
+
// Sharding is an optimization. Compile/link the canonical TU below if
|
|
4487
|
+
// a target tool, one shard, or relocatable merge is unavailable.
|
|
4488
|
+
return null;
|
|
4489
|
+
}
|
|
4490
|
+
};
|
|
4491
|
+
const shardedProgramObject = await compileShardedProgram();
|
|
4492
|
+
if (programShards !== null && shardedProgramObject === null) {
|
|
4493
|
+
// The canonical TU fallback is valid output, but it must not populate a
|
|
4494
|
+
// cache key describing a successful shard projection/merge.
|
|
4495
|
+
cacheInputsStable = false;
|
|
4496
|
+
}
|
|
4497
|
+
await runClang(buildArgs((p) => objects?.get(p) ?? p, shardedProgramObject === null
|
|
4498
|
+
? { programPath, outPath: privateOut, compilerVisibleSource: opts.cPath }
|
|
4499
|
+
: { programPath: shardedProgramObject, outPath: privateOut }));
|
|
3811
4500
|
await installArtifact(privateOut, opts.outPath);
|
|
3812
4501
|
if (cachedBin !== null && keyHex !== null) {
|
|
3813
4502
|
// Metadata comparison catches ordinary changes cheaply, but cannot by
|
|
@@ -3815,7 +4504,7 @@ export async function compileC(opts) {
|
|
|
3815
4504
|
// the key. Recompute every content-bearing identity after the final link
|
|
3816
4505
|
// so a header/SDK/compiler change in either pre-build gap cannot publish
|
|
3817
4506
|
// new output under an old key.
|
|
3818
|
-
const [currentRuntime, currentImplicit, currentRuntimeInvocation, currentProgramInvocation, currentProgramDependencies, currentLinker, currentCompiler] = await Promise.all([
|
|
4507
|
+
const [currentRuntime, currentImplicit, currentRuntimeInvocation, currentProgramInvocation, currentProgramDependencies, currentLinker, currentCompiler, currentProgramShardMerge] = await Promise.all([
|
|
3819
4508
|
runtimeFingerprint(rtDir).catch(() => null),
|
|
3820
4509
|
implicitToolchainFingerprint(driver, toolchainEnv).catch(() => null),
|
|
3821
4510
|
effectiveCompilerInvocationFingerprint(driver, toolchainEnv, cflags).catch(() => null),
|
|
@@ -3823,18 +4512,22 @@ export async function compileC(opts) {
|
|
|
3823
4512
|
translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cBytes, toolchainEnv).catch(() => null),
|
|
3824
4513
|
implicitLinkerFingerprint(driver, toolchainEnv, linkProbeArgs, effectiveLinkInvocationArgs, linkTraceInvocationArgs).catch(() => null),
|
|
3825
4514
|
ccVersionOnce(driver.argv, toolchainEnv, true).catch(() => null),
|
|
4515
|
+
programShards === null
|
|
4516
|
+
? Promise.resolve(null)
|
|
4517
|
+
: resolveProgramShardMergeIdentity(driver).catch(() => null),
|
|
3826
4518
|
]);
|
|
3827
4519
|
cacheInputsStable =
|
|
3828
4520
|
cacheInputsStable &&
|
|
3829
4521
|
preBuildDependencies !== null &&
|
|
3830
|
-
await
|
|
4522
|
+
await nativeArtifactDependenciesStillMatch(preBuildDependencies) &&
|
|
3831
4523
|
currentRuntime === fingerprint &&
|
|
3832
4524
|
currentImplicit === implicitToolchain &&
|
|
3833
4525
|
currentRuntimeInvocation === runtimeCompilerInvocation &&
|
|
3834
4526
|
currentProgramInvocation === programCompilerInvocation &&
|
|
3835
4527
|
currentProgramDependencies === programDependencies &&
|
|
3836
4528
|
currentLinker === implicitLinker &&
|
|
3837
|
-
currentCompiler === cv
|
|
4529
|
+
currentCompiler === cv &&
|
|
4530
|
+
currentProgramShardMerge === programShardMergeIdentity;
|
|
3838
4531
|
}
|
|
3839
4532
|
if (cachedBin !== null && keyHex !== null && cacheInputsStable) {
|
|
3840
4533
|
try {
|
|
@@ -3851,7 +4544,10 @@ export async function compileC(opts) {
|
|
|
3851
4544
|
localArtifactDependencyPaths !== null &&
|
|
3852
4545
|
cacheCompleteArtifact &&
|
|
3853
4546
|
cacheInputsStable) {
|
|
3854
|
-
await publishLocalArtifactStamp(localArtifact.stampPath, opts.outPath, localArtifact.key, localArtifactDependencyPaths, [dirname(resolve(opts.cPath))], [root]).catch(() =>
|
|
4547
|
+
const stamp = await publishLocalArtifactStamp(localArtifact.stampPath, opts.outPath, localArtifact.key, localArtifactDependencyPaths, [dirname(resolve(opts.cPath))], [root]).catch(() => null);
|
|
4548
|
+
if (stamp !== null) {
|
|
4549
|
+
await opts.onArtifactReady?.({ dependencies: stamp.dependencies }).catch(() => undefined);
|
|
4550
|
+
}
|
|
3855
4551
|
}
|
|
3856
4552
|
}
|
|
3857
4553
|
finally {
|
|
@@ -3859,6 +4555,78 @@ export async function compileC(opts) {
|
|
|
3859
4555
|
}
|
|
3860
4556
|
// Runtime-object population is itself a cache write, including on builds
|
|
3861
4557
|
// whose native link inputs disable complete-artifact publication.
|
|
3862
|
-
await pruneCache(root).catch(() => undefined);
|
|
4558
|
+
await pruneCache(root, cacheWarmPaths).catch(() => undefined);
|
|
4559
|
+
}
|
|
4560
|
+
export async function compileC(opts) {
|
|
4561
|
+
await compileCInternal(opts, false);
|
|
4562
|
+
}
|
|
4563
|
+
export function supportedNativeCacheWarmProfiles(driver) {
|
|
4564
|
+
if (isMobileTarget(driver.target) || targetPlatform(driver) === "wasi")
|
|
4565
|
+
return [];
|
|
4566
|
+
return ["runtime", "tls", "dynamic"];
|
|
4567
|
+
}
|
|
4568
|
+
/** Populate expensive native prerequisites against the current compiler,
|
|
4569
|
+
* target, SDK, and environment. The resulting entries use the exact same
|
|
4570
|
+
* identities and validators as ordinary builds; this merely pays their cost
|
|
4571
|
+
* before a developer's first program asks for them. Synthetic link products
|
|
4572
|
+
* are discarded and never enter the complete-executable cache. */
|
|
4573
|
+
export async function warmNativeCaches(options = {}) {
|
|
4574
|
+
const cacheRoot = cacheRootDir();
|
|
4575
|
+
if (cacheRoot === null) {
|
|
4576
|
+
throw new Error("the native build cache is disabled (unset SCRIPTC_NO_CACHE and use a non-empty SCRIPTC_CACHE_DIR)");
|
|
4577
|
+
}
|
|
4578
|
+
await ensurePrivateCacheRoot(cacheRoot, process.env["SCRIPTC_CACHE_DIR"] === undefined);
|
|
4579
|
+
const known = new Set(["runtime", "tls", "dynamic"]);
|
|
4580
|
+
for (const profile of options.profiles ?? []) {
|
|
4581
|
+
if (!known.has(profile))
|
|
4582
|
+
throw new Error(`unknown native cache warm profile '${profile}'`);
|
|
4583
|
+
}
|
|
4584
|
+
if (options.profiles?.length === 0)
|
|
4585
|
+
return { cacheRoot, profiles: [] };
|
|
4586
|
+
const mobileTarget = mobileLibraryTarget();
|
|
4587
|
+
if (mobileTarget !== null) {
|
|
4588
|
+
throw new Error(`native cache warming targets executable builds and is unsupported for SCRIPTC_TARGET=${mobileTarget}`);
|
|
4589
|
+
}
|
|
4590
|
+
const driver = resolveCc();
|
|
4591
|
+
const supported = supportedNativeCacheWarmProfiles(driver);
|
|
4592
|
+
if (supported.length === 0) {
|
|
4593
|
+
throw new Error(`native cache warming targets persistently cached native executables and is unsupported for SCRIPTC_TARGET=${driver.target}`);
|
|
4594
|
+
}
|
|
4595
|
+
const profiles = [...new Set(options.profiles ?? supported)];
|
|
4596
|
+
for (const profile of profiles) {
|
|
4597
|
+
if (!supported.includes(profile)) {
|
|
4598
|
+
throw new Error(`native cache warm profile '${profile}' is unsupported for SCRIPTC_TARGET=${driver.target}`);
|
|
4599
|
+
}
|
|
4600
|
+
}
|
|
4601
|
+
const workDir = await mkdtemp(join(tmpdir(), "scriptc-cache-warm-"));
|
|
4602
|
+
const cPath = join(workDir, "warm.c");
|
|
4603
|
+
await writeFile(cPath, "int main(void) { return 0; }\n");
|
|
4604
|
+
const protectedPaths = new Set();
|
|
4605
|
+
try {
|
|
4606
|
+
const results = await Promise.all(profiles.map(async (profile) => {
|
|
4607
|
+
const started = performance.now();
|
|
4608
|
+
await compileCInternal({
|
|
4609
|
+
cPath,
|
|
4610
|
+
outPath: join(workDir, process.platform === "win32" ? `${profile}.exe` : profile),
|
|
4611
|
+
cacheIdentity: "scriptc-native-cache-warm-v1",
|
|
4612
|
+
optimization: options.optimization ?? "release",
|
|
4613
|
+
sanitize: options.sanitize ?? false,
|
|
4614
|
+
...(profile === "tls" ? { fetch: true } : {}),
|
|
4615
|
+
...(profile === "dynamic" ? { dynamic: true } : {}),
|
|
4616
|
+
}, true, protectedPaths);
|
|
4617
|
+
return {
|
|
4618
|
+
profile,
|
|
4619
|
+
elapsedMs: Math.round((performance.now() - started) * 10) / 10,
|
|
4620
|
+
};
|
|
4621
|
+
}));
|
|
4622
|
+
await pruneCache(cacheRoot).catch(() => undefined);
|
|
4623
|
+
if (!(await Promise.all([...protectedPaths].map(fileExists))).every(Boolean)) {
|
|
4624
|
+
throw new Error(`SCRIPTC_CACHE_MAX_MB is too small to retain the requested native cache warm profiles (${profiles.join(", ")})`);
|
|
4625
|
+
}
|
|
4626
|
+
return { cacheRoot, profiles: results };
|
|
4627
|
+
}
|
|
4628
|
+
finally {
|
|
4629
|
+
await rm(workDir, { recursive: true, force: true }).catch(() => undefined);
|
|
4630
|
+
}
|
|
3863
4631
|
}
|
|
3864
4632
|
//# sourceMappingURL=cc.js.map
|