@scriptc/compiler 0.0.33 → 0.0.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/dist/backend/cc-cache.test.js +1091 -7
  2. package/dist/backend/cc-cache.test.js.map +1 -1
  3. package/dist/backend/cc.d.ts +109 -18
  4. package/dist/backend/cc.js +1057 -289
  5. package/dist/backend/cc.js.map +1 -1
  6. package/dist/backend/emission/emit-exprs.js +26 -1
  7. package/dist/backend/emission/emit-exprs.js.map +1 -1
  8. package/dist/backend/emission/emit-shapes.js +37 -4
  9. package/dist/backend/emission/emit-shapes.js.map +1 -1
  10. package/dist/backend/emission/emitter.d.ts +11 -2
  11. package/dist/backend/emission/emitter.js +11 -5
  12. package/dist/backend/emission/emitter.js.map +1 -1
  13. package/dist/backend/library-identity.d.ts +22 -0
  14. package/dist/backend/library-identity.js +103 -0
  15. package/dist/backend/library-identity.js.map +1 -0
  16. package/dist/backend/llvm/emitter.d.ts +3 -0
  17. package/dist/backend/llvm/emitter.js +35 -5
  18. package/dist/backend/llvm/emitter.js.map +1 -1
  19. package/dist/backend/llvm/shapes.d.ts +1 -0
  20. package/dist/backend/llvm/shapes.js +25 -1
  21. package/dist/backend/llvm/shapes.js.map +1 -1
  22. package/dist/backend/llvm/split.d.ts +43 -0
  23. package/dist/backend/llvm/split.js +227 -0
  24. package/dist/backend/llvm/split.js.map +1 -0
  25. package/dist/backend/llvm/split.test.d.ts +1 -0
  26. package/dist/backend/llvm/split.test.js +102 -0
  27. package/dist/backend/llvm/split.test.js.map +1 -0
  28. package/dist/backend/mangle.d.ts +1 -0
  29. package/dist/backend/mangle.js +3 -0
  30. package/dist/backend/mangle.js.map +1 -1
  31. package/dist/backend/object-localize.d.ts +5 -5
  32. package/dist/backend/object-localize.js +19 -16
  33. package/dist/backend/object-localize.js.map +1 -1
  34. package/dist/backend/object-localize.test.js +57 -13
  35. package/dist/backend/object-localize.test.js.map +1 -1
  36. package/dist/executable/early-cache.d.ts +86 -0
  37. package/dist/executable/early-cache.js +460 -0
  38. package/dist/executable/early-cache.js.map +1 -0
  39. package/dist/executable/early-cache.test.d.ts +1 -0
  40. package/dist/executable/early-cache.test.js +300 -0
  41. package/dist/executable/early-cache.test.js.map +1 -0
  42. package/dist/ffi/profile.d.ts +7 -2
  43. package/dist/ffi/profile.js +1 -0
  44. package/dist/ffi/profile.js.map +1 -1
  45. package/dist/frontend/input-tracker.d.ts +82 -0
  46. package/dist/frontend/input-tracker.js +380 -0
  47. package/dist/frontend/input-tracker.js.map +1 -0
  48. package/dist/frontend/input-tracker.test.d.ts +1 -0
  49. package/dist/frontend/input-tracker.test.js +137 -0
  50. package/dist/frontend/input-tracker.test.js.map +1 -0
  51. package/dist/frontend/lowering/lower-builtins.js +4 -10
  52. package/dist/frontend/lowering/lower-builtins.js.map +1 -1
  53. package/dist/frontend/lowering/lower-calls.js +169 -144
  54. package/dist/frontend/lowering/lower-calls.js.map +1 -1
  55. package/dist/frontend/lowering/lower-classes.js +3 -0
  56. package/dist/frontend/lowering/lower-classes.js.map +1 -1
  57. package/dist/frontend/lowering/lower-containers.js +66 -32
  58. package/dist/frontend/lowering/lower-containers.js.map +1 -1
  59. package/dist/frontend/lowering/lower-exprs.js +89 -11
  60. package/dist/frontend/lowering/lower-exprs.js.map +1 -1
  61. package/dist/frontend/lowering/lower-inspect.js +45 -26
  62. package/dist/frontend/lowering/lower-inspect.js.map +1 -1
  63. package/dist/frontend/lowering/lower-island.js +3 -3
  64. package/dist/frontend/lowering/lower-island.js.map +1 -1
  65. package/dist/frontend/lowering/lower-mixins.js +1 -1
  66. package/dist/frontend/lowering/lower-mixins.js.map +1 -1
  67. package/dist/frontend/lowering/lower-modules.js +1 -1
  68. package/dist/frontend/lowering/lower-modules.js.map +1 -1
  69. package/dist/frontend/lowering/lower-stmts.js +49 -4
  70. package/dist/frontend/lowering/lower-stmts.js.map +1 -1
  71. package/dist/frontend/lowering/lowerer.d.ts +73 -29
  72. package/dist/frontend/lowering/lowerer.js +588 -116
  73. package/dist/frontend/lowering/lowerer.js.map +1 -1
  74. package/dist/frontend/lowering/surfaces.js +1 -1
  75. package/dist/frontend/lowering/surfaces.js.map +1 -1
  76. package/dist/frontend/npm-static-rewrite.js +25 -26
  77. package/dist/frontend/npm-static-rewrite.js.map +1 -1
  78. package/dist/frontend/npm-static.js +29 -30
  79. package/dist/frontend/npm-static.js.map +1 -1
  80. package/dist/frontend/npm-static.test.d.ts +1 -0
  81. package/dist/frontend/npm-static.test.js +15 -0
  82. package/dist/frontend/npm-static.test.js.map +1 -0
  83. package/dist/frontend/npm.js +5 -17
  84. package/dist/frontend/npm.js.map +1 -1
  85. package/dist/frontend/program.js +44 -19
  86. package/dist/frontend/program.js.map +1 -1
  87. package/dist/frontend/resolve.js +8 -29
  88. package/dist/frontend/resolve.js.map +1 -1
  89. package/dist/frontend/ts7/checker.d.ts +40 -2
  90. package/dist/frontend/ts7/checker.js +194 -32
  91. package/dist/frontend/ts7/checker.js.map +1 -1
  92. package/dist/frontend/ts7/program.js +17 -18
  93. package/dist/frontend/ts7/program.js.map +1 -1
  94. package/dist/frontend/types.d.ts +33 -0
  95. package/dist/frontend/types.js +134 -45
  96. package/dist/frontend/types.js.map +1 -1
  97. package/dist/index.d.ts +5 -2
  98. package/dist/index.js +458 -141
  99. package/dist/index.js.map +1 -1
  100. package/dist/ir/nodes.d.ts +27 -4
  101. package/dist/ir/nodes.js +28 -0
  102. package/dist/ir/nodes.js.map +1 -1
  103. package/dist/ir/serialize.d.ts +1 -1
  104. package/dist/ir/serialize.js +1 -1
  105. package/dist/ir/validate.js +37 -1
  106. package/dist/ir/validate.js.map +1 -1
  107. package/dist/library/early-cache.d.ts +65 -0
  108. package/dist/library/early-cache.js +361 -0
  109. package/dist/library/early-cache.js.map +1 -0
  110. package/dist/library/early-cache.test.d.ts +1 -0
  111. package/dist/library/early-cache.test.js +528 -0
  112. package/dist/library/early-cache.test.js.map +1 -0
  113. package/dist/library/implementation-identity.d.ts +23 -0
  114. package/dist/library/implementation-identity.js +84 -0
  115. package/dist/library/implementation-identity.js.map +1 -0
  116. package/dist/library/int-infer.js +14 -0
  117. package/dist/library/int-infer.js.map +1 -1
  118. package/dist/library/int-infer.test.js +21 -4
  119. package/dist/library/int-infer.test.js.map +1 -1
  120. package/dist/library/semantic-source.d.ts +13 -0
  121. package/dist/library/semantic-source.js +257 -0
  122. package/dist/library/semantic-source.js.map +1 -0
  123. package/dist/library/semantic-source.test.d.ts +1 -0
  124. package/dist/library/semantic-source.test.js +113 -0
  125. package/dist/library/semantic-source.test.js.map +1 -0
  126. package/dist/library/sidecar.d.ts +6 -0
  127. package/dist/library/sidecar.js +11 -0
  128. package/dist/library/sidecar.js.map +1 -1
  129. package/dist/startup-cache.d.ts +6 -0
  130. package/dist/startup-cache.js +6 -0
  131. package/dist/startup-cache.js.map +1 -0
  132. package/package.json +3 -2
  133. package/surface-manifest.json +1 -1
@@ -1,13 +1,16 @@
1
1
  import { execFile, spawnSync } from "node:child_process";
2
- import { createHash } from "node:crypto";
2
+ import { createHash, randomUUID } from "node:crypto";
3
3
  import { constants as fsConstants, existsSync, readdirSync } from "node:fs";
4
4
  import { createRequire } from "node:module";
5
5
  import { access, chmod, copyFile, link, lstat, mkdir, mkdtemp, readdir, readFile, realpath, rename, rm, stat, unlink, utimes, writeFile } from "node:fs/promises";
6
6
  import { availableParallelism, homedir, tmpdir } from "node:os";
7
7
  import { basename, delimiter, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
8
+ import { performance } from "node:perf_hooks";
9
+ import { fileURLToPath } from "node:url";
8
10
  import { promisify } from "node:util";
9
11
  import { localizeElfObject, mergeAndLocalizeCoffObjects } from "./object-localize.js";
10
12
  const execFileAsync = promisify(execFile);
13
+ const CC_IMPLEMENTATION_PATH = fileURLToPath(import.meta.url);
11
14
  /** Test lanes run against one immutable checkout and one immutable toolchain
12
15
  * for the lifetime of each Vitest worker. Production deliberately rediscovers
13
16
  * compiler/linker inputs on every invocation, but doing that thousands of
@@ -118,17 +121,6 @@ const MUTABLE_COMPILE_ENV_KEYS = [
118
121
  "CCC_ADD_ARGS",
119
122
  "CLANG_CONFIG_FILE_SYSTEM_DIR",
120
123
  "CLANG_CONFIG_FILE_USER_DIR",
121
- // The direct runtime build does not consume the conventional Make/CMake
122
- // variables, but the lazily-built QuickJS prerequisite does. Their values
123
- // can name mutable tools, headers, libraries, and toolchain files, so a
124
- // stable spelling is not a safe persistent-cache identity.
125
- "CC",
126
- "CFLAGS",
127
- "CPPFLAGS",
128
- "LDFLAGS",
129
- "AR",
130
- "RANLIB",
131
- "CMAKE_TOOLCHAIN_FILE",
132
124
  // `zig cc` resolves its bundled headers/runtime through ZIG_LIB_DIR and a
133
125
  // caller-selected native libc description through ZIG_LIBC. Both values name
134
126
  // mutable compiler inputs whose contents can change behind a stable path.
@@ -155,6 +147,41 @@ export function toolchainEnvironmentFingerprint(env = process.env) {
155
147
  }
156
148
  return hash.digest("hex");
157
149
  }
150
+ /** Inputs that can change which native tool/runtime implementation an
151
+ * executable build selects before compileC has a chance to rediscover it.
152
+ * The early whole-program cache keys this exact posture before restoring a
153
+ * final binary; compileC retains its deeper inode/content validation. */
154
+ export async function executableNativeEnvironmentFingerprint(env = process.env) {
155
+ const configuredCompiler = env["SCRIPTC_CC"] ?? "";
156
+ let compilerIdentity;
157
+ try {
158
+ compilerIdentity = await effectiveCompilerEnvironmentIdentity(resolveCc(env), env);
159
+ }
160
+ catch {
161
+ // A failed trace cannot safely describe a reusable native posture. Keep
162
+ // the build working, but make this invocation miss every persistent early
163
+ // entry so compileC performs its full discovery and validation.
164
+ compilerIdentity = `<unavailable:${configuredCompiler}:${randomUUID()}>`;
165
+ }
166
+ const hash = createHash("sha256")
167
+ .update("executable-native-environment-v2\0")
168
+ .update(toolchainEnvironmentFingerprint(env)).update("\0")
169
+ // PATH text alone is not a resolution proof, and on Darwin /usr/bin/clang
170
+ // is a stable shim whose selected Xcode compiler can change underneath it.
171
+ // Re-resolve and trace the effective driver on every early lookup.
172
+ .update(compilerIdentity).update("\0");
173
+ for (const name of [
174
+ "PATH",
175
+ "SCRIPTC_FETCH_CURL",
176
+ "SCRIPTC_TEST_RUNTIME_SRC_DIR",
177
+ "SCRIPTC_TEST_VENDOR_CACHE_DIR",
178
+ "SCRIPTC_TEST_TRUST_COMPILER_WRAPPER",
179
+ ]) {
180
+ const value = env[name];
181
+ hash.update(name).update(value === undefined ? "\0unset\0" : "\0set\0").update(value ?? "").update("\0");
182
+ }
183
+ return hash.digest("hex");
184
+ }
158
185
  /** Structured compiler-driver failure. Most callers still let this surface
159
186
  * as an internal build error; the TypeScript compiler pipeline recognizes it
160
187
  * when an outbound FFI profile is active and turns user-controlled native
@@ -505,14 +532,19 @@ export function cacheTargetIdentity(driver, hostPlatform = process.platform, hos
505
532
  function vendorEngineDir() {
506
533
  return join(runtimeSrcDir(), "..", "vendor", "quickjs-ng");
507
534
  }
508
- /** Vendor prerequisites have historically lived beside their source trees.
509
- * The test-only override gives cache-ordering tests a disposable root without
510
- * renaming package-local artifacts that parallel workers may be using. */
511
- function vendorBuildCacheRoot() {
535
+ /** Vendor prerequisites follow the per-user build root when persistent cache
536
+ * validation succeeds. Keeping generated objects out of node_modules makes
537
+ * npm installations, read-only package stores, and explicit cache warming all
538
+ * share one writable cache. The test-only override remains the first choice;
539
+ * callers without a validated root retain the historical package-local
540
+ * default (ordinary builds give those callers a private transient root). */
541
+ function vendorBuildCacheRoot(buildRoot) {
512
542
  const testRoot = process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"];
513
- return testRoot === undefined
543
+ if (testRoot !== undefined)
544
+ return resolve(testRoot);
545
+ return buildRoot === undefined
514
546
  ? join(runtimeSrcDir(), "..", "vendor", ".cache")
515
- : resolve(testRoot);
547
+ : join(buildRoot, "vendor");
516
548
  }
517
549
  /** Cross targets already carry their architecture in the explicit triple.
518
550
  * Host-native prerequisites need the same separation: a checkout/package can
@@ -521,27 +553,30 @@ export function vendorCacheTargetFlavor(driver, hostPlatform = process.platform,
521
553
  return driver.target === null ? `native-${hostPlatform}-${hostArch}` : driver.target;
522
554
  }
523
555
  /** Vendor prerequisites live outside the content-addressed build root, so
524
- * their directory name must carry the same environment and resolved compiler
525
- * identity as the artifacts that consume them. The short digest keeps the
526
- * package-local path readable while preventing build-order contamination. */
527
- export function vendorCacheBuildIdentity(environmentFingerprint, compilerIdentity) {
556
+ * their directory name must carry the environment, resolved compiler, owned
557
+ * source bytes, and build-recipe identity of the artifacts that consume them.
558
+ * The short digest keeps the cache path readable while preventing build-order
559
+ * and cross-installation contamination. */
560
+ export function vendorCacheBuildIdentity(environmentFingerprint, compilerIdentity, nativeSourceIdentity) {
528
561
  return createHash("sha256")
529
- .update("vendor-toolchain-v1\0")
562
+ .update("vendor-toolchain-v2\0")
530
563
  .update(environmentFingerprint)
531
564
  .update("\0")
532
565
  .update(compilerIdentity)
566
+ .update("\0")
567
+ .update(nativeSourceIdentity)
533
568
  .digest("hex")
534
569
  .slice(0, 20);
535
570
  }
536
571
  const vendorCompilerIdentityFallbacks = new Map();
537
572
  async function currentVendorCacheBuildIdentity(driver, environmentFingerprint) {
538
- // Native vendor recipes additionally use bare clang/ar and CMake; cross
573
+ // Native vendor recipes additionally use bare clang/ar; cross
539
574
  // recipes use the zig driver for compilation and `zig ar`. Include every
540
- // executable that can affect the package-local prerequisite, not just the
575
+ // executable that can affect the cached prerequisite, not just the
541
576
  // final program's driver.
542
577
  const commands = [
543
578
  driver.argv[0] ?? "clang",
544
- ...(driver.target === null ? ["clang", "cc", "ar", "ranlib", "cmake"] : []),
579
+ ...(driver.target === null ? ["clang", "ar"] : []),
545
580
  ].filter((command, index, all) => all.indexOf(command) === index);
546
581
  const identities = await Promise.all(commands.map(async (command) => {
547
582
  const spellingKey = `${environmentFingerprint}\0${command}`;
@@ -555,7 +590,7 @@ async function currentVendorCacheBuildIdentity(driver, environmentFingerprint) {
555
590
  `<unresolved>\0${command}`,
556
591
  ].join("\0");
557
592
  }));
558
- return vendorCacheBuildIdentity(environmentFingerprint, identities.join("\x1f"));
593
+ return vendorCacheBuildIdentity(environmentFingerprint, identities.join("\x1f"), await runtimeFingerprint(runtimeSrcDir()));
559
594
  }
560
595
  function engineArchivePath(sanitize, driver, buildIdentity, cacheRoot = vendorBuildCacheRoot()) {
561
596
  const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
@@ -564,6 +599,18 @@ function engineArchivePath(sanitize, driver, buildIdentity, cacheRoot = vendorBu
564
599
  async function fileExists(path) {
565
600
  return stat(path).then((s) => s.isFile(), () => false);
566
601
  }
602
+ /** Vendor prerequisites are shared persistent cache entries, so they carry
603
+ * the same adjacent content digest as runtime objects and complete artifacts.
604
+ * Missing/invalid pairs are rebuilt. Do not remove them during validation: a
605
+ * concurrent publisher installs data before its digest, and that temporary
606
+ * invalid window must not let a reader unlink the publisher's data. */
607
+ async function validVendorArtifact(path) {
608
+ return validCachedFile(path);
609
+ }
610
+ function protectCachedArtifact(paths, path) {
611
+ paths?.add(path);
612
+ paths?.add(cacheDigestPath(path));
613
+ }
567
614
  function privateSiblingPath(destination, label) {
568
615
  // Keep the temporary component independent of the caller's basename. A
569
616
  // destination can validly consume the filesystem's entire NAME_MAX budget;
@@ -586,8 +633,73 @@ async function installArtifact(source, destination, mode) {
586
633
  await rm(tmp, { force: true }).catch(() => undefined);
587
634
  }
588
635
  }
636
+ /** Publish one finished vendor prerequisite from disposable build scratch.
637
+ * The shared data and digest use the ordinary atomic cache publisher.
638
+ * Concurrent builders produce equivalent bytes; on platforms where rename
639
+ * cannot replace the winner, accept only a checksum-verified winner. */
640
+ async function publishVendorArtifact(source, destination) {
641
+ for (let attempt = 0; attempt < 2; attempt++) {
642
+ try {
643
+ await publishCachedFile(source, destination);
644
+ }
645
+ catch (error) {
646
+ // A concurrent equivalent publisher may have won on a platform where
647
+ // rename cannot replace an existing file. Accept only its verified pair.
648
+ if (await validCachedFile(destination))
649
+ return;
650
+ if (attempt > 0)
651
+ throw error;
652
+ // Otherwise the existing pair is genuinely stale/corrupt and blocked
653
+ // replacement (notably on Windows). Remove it only after a complete
654
+ // replacement is ready in private scratch, then retry once.
655
+ await Promise.all([
656
+ rm(destination, { force: true }).catch(() => undefined),
657
+ rm(cacheDigestPath(destination), { force: true }).catch(() => undefined),
658
+ ]);
659
+ continue;
660
+ }
661
+ // Close publication races where another writer replaced only one member
662
+ // of the pair between this publisher's two atomic renames.
663
+ if (await validCachedFile(destination))
664
+ return;
665
+ }
666
+ throw new Error(`vendor cache publication failed integrity validation: ${destination}`);
667
+ }
668
+ /** Pin cache-backed vendor inputs under an invocation-private directory before
669
+ * linking/archiving. The shared LRU may unlink their cache names at any time;
670
+ * hard links (or copies where links are unavailable) keep active inputs alive.
671
+ * A sweep can race the initial pin, so rematerialize and retry once. */
672
+ async function stageVendorInputs(materialize, stageDir) {
673
+ let lastError = new Error("vendor cache input disappeared while staging");
674
+ for (let attempt = 0; attempt < 2; attempt++) {
675
+ await rm(stageDir, { recursive: true, force: true }).catch(() => undefined);
676
+ await mkdir(stageDir, { recursive: true });
677
+ const sources = await materialize();
678
+ try {
679
+ const now = new Date();
680
+ return await Promise.all(sources.map(async (source) => {
681
+ const destination = join(stageDir, basename(source));
682
+ try {
683
+ await link(source, destination);
684
+ }
685
+ catch {
686
+ await copyFile(source, destination);
687
+ }
688
+ // Vendor prerequisites share the cache root's mtime-based LRU. A
689
+ // successful stage is a cache read, so promote the shared source name
690
+ // best-effort (it may have raced an eviction after the hard link).
691
+ await utimes(source, now, now).catch(() => undefined);
692
+ return destination;
693
+ }));
694
+ }
695
+ catch (error) {
696
+ lastError = error;
697
+ }
698
+ }
699
+ throw lastError;
700
+ }
589
701
  /** The engine archive for one flavor, built lazily on the first --dynamic
590
- * compile (~10s) and cached under vendor/.cache/<commit>-<flavor>-<target>-<toolchain>/ — unlike
702
+ * compile and cached under <build-cache>/vendor/<commit>-<flavor>-<target>-<toolchain>/ — unlike
591
703
  * the runtime's own sources (recompiled every build, ~100ms), the engine is
592
704
  * far too big to rebuild per compile. The plain flavor is ALWAYS MinSizeRel
593
705
  * regardless of the program's optimization level: measured binary delta is
@@ -599,67 +711,24 @@ async function installArtifact(source, destination, mode) {
599
711
  * private temp dir and publish with an atomic rename — first one wins,
600
712
  * losers discard their work and use the winner's archive.
601
713
  *
602
- * SCRIPTC_TARGET adds a per-target cache flavor built WITHOUT CMake (the
603
- * mbedTLS recipe): the qjs library target is just four TUs
604
- * (QJS_ENGINE_SOURCES) compiled with `zig cc -target <triple>` under the
605
- * same flags CMake's MinSizeRel/Debug+ASan configurations apply, packed
606
- * with `zig ar`. Host builds keep the exact historical CMake recipe. */
714
+ * The qjs library target is just four TUs (QJS_ENGINE_SOURCES), so both host
715
+ * and cross builds use the same direct per-TU recipe. This removes CMake from
716
+ * the runtime dependency set and lets cache warming compile exactly the
717
+ * archive a later program consumes. Host builds use clang + ar; cross builds
718
+ * use the selected zig driver + zig ar. */
607
719
  async function ensureEngineArchive(sanitize, driver, buildIdentity, cacheRoot = vendorBuildCacheRoot()) {
608
- const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
609
- const vendor = vendorEngineDir();
610
720
  const archive = engineArchivePath(sanitize, driver, buildIdentity, cacheRoot);
611
721
  const cacheDir = dirname(archive);
612
- if (await fileExists(archive))
722
+ if (await validVendorArtifact(archive))
613
723
  return archive;
614
- if (driver.target !== null)
615
- return buildEngineArchiveCross(sanitize, driver, cacheRoot, cacheDir);
616
- await mkdir(cacheRoot, { recursive: true });
617
- const buildDir = await mkdtemp(join(cacheRoot, `build-${flavor}-`));
618
- const stageDir = `${buildDir}.stage`;
619
- try {
620
- const configureArgs = [
621
- "-S", vendor,
622
- "-B", buildDir,
623
- ...(sanitize
624
- ? ["-DCMAKE_BUILD_TYPE=Debug", "-DQJS_ENABLE_ASAN=ON"]
625
- : ["-DCMAKE_BUILD_TYPE=MinSizeRel"]),
626
- "-DQJS_BUILD_EXAMPLES=OFF",
627
- "-DQJS_ENABLE_INSTALL=OFF",
628
- ];
629
- try {
630
- await execFileAsync("cmake", configureArgs);
631
- }
632
- catch (err) {
633
- if (err.code === "ENOENT") {
634
- throw new Error("--dynamic builds the embedded engine once with CMake, which was not found on PATH.\n" +
635
- "Install cmake (e.g. `brew install cmake`) and retry. Static builds do not need it.");
636
- }
637
- throw new Error(`cmake failed configuring the embedded engine (${vendor}).\n\n` +
638
- subprocessFailureDetail(err));
639
- }
640
- await execFileAsync("cmake", [
641
- "--build", buildDir,
642
- "--target", "qjs",
643
- `-j${availableParallelism()}`,
644
- ]);
645
- // Publish only the archive, atomically: rename fails if a concurrent
646
- // build won the race, and the winner's archive is just as good.
647
- await mkdir(stageDir);
648
- await copyFile(join(buildDir, "libqjs.a"), join(stageDir, "libqjs.a"));
649
- await rename(stageDir, cacheDir).catch(() => undefined);
650
- }
651
- finally {
652
- await rm(buildDir, { recursive: true, force: true });
653
- await rm(stageDir, { recursive: true, force: true });
654
- }
655
- return archive;
724
+ return buildEngineArchiveDirect(sanitize, driver, cacheRoot, cacheDir);
656
725
  }
657
726
  /** The qjs library target's source list (CMakeLists.txt `qjs_sources` with
658
727
  * QJS_BUILD_LIBC off — cutils is header-only): the cross-build recipe
659
728
  * compiles exactly these. */
660
729
  const QJS_ENGINE_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"];
661
- /** The engine archive for a CROSS target, per-TU `zig cc` compiles + `zig
662
- * ar` (no CMake — the mbedTLS recipe). The flags mirror what the CMake
730
+ /** The engine archive for one target, per-TU compiler invocations plus ar.
731
+ * The flags mirror what the former CMake
663
732
  * configurations apply to the qjs library target: gnu11 (CMAKE_C_EXTENSIONS
664
733
  * ON), hidden visibility, -funsigned-char, -DQUICKJS_NG_BUILD and
665
734
  * -D_GNU_SOURCE (qjs_defines; linux targetArgs already carry the latter),
@@ -667,10 +736,11 @@ const QJS_ENGINE_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"
667
736
  * (-O0 -ggdb -fno-omit-frame-pointer -fsanitize=address
668
737
  * -fno-sanitize-recover=all) for asan. Same atomic-rename publish as every
669
738
  * vendor cache. */
670
- async function buildEngineArchiveCross(sanitize, driver, cacheRoot, cacheDir) {
739
+ async function buildEngineArchiveDirect(sanitize, driver, cacheRoot, cacheDir) {
671
740
  const vendor = vendorEngineDir();
672
741
  const archive = join(cacheDir, "libqjs.a");
673
- const arArgv = [...driver.argv.slice(0, 1), "ar"];
742
+ const compileArgv = driver.target === null ? ["clang"] : driver.argv;
743
+ const arArgv = driver.target === null ? ["ar"] : [...driver.argv.slice(0, 1), "ar"];
674
744
  const cflags = [
675
745
  "-std=gnu11",
676
746
  ...driver.targetArgs,
@@ -687,21 +757,14 @@ async function buildEngineArchiveCross(sanitize, driver, cacheRoot, cacheDir) {
687
757
  "-I", vendor,
688
758
  ];
689
759
  await mkdir(cacheRoot, { recursive: true });
690
- const buildDir = await mkdtemp(join(cacheRoot, `build-qjs-${driver.target ?? "host"}-`));
760
+ const buildDir = await mkdtemp(join(tmpdir(), `scriptc-vendor-qjs-${driver.target ?? "host"}-`));
691
761
  try {
692
762
  const width = Math.min(QJS_ENGINE_SOURCES.length, availableParallelism());
693
763
  for (let i = 0; i < QJS_ENGINE_SOURCES.length; i += width) {
694
- await Promise.all(QJS_ENGINE_SOURCES.slice(i, i + width).map((src) => execFileAsync(driver.argv[0] ?? "clang", [...driver.argv.slice(1), ...cflags, "-c", join(vendor, src), "-o", join(buildDir, `${basename(src, ".c")}.o`)], { cwd: buildDir })));
764
+ await Promise.all(QJS_ENGINE_SOURCES.slice(i, i + width).map((src) => execFileAsync(compileArgv[0] ?? "clang", [...compileArgv.slice(1), ...cflags, "-c", join(vendor, src), "-o", join(buildDir, `${basename(src, ".c")}.o`)], { cwd: buildDir })));
695
765
  }
696
766
  await execFileAsync(arArgv[0] ?? "ar", [...arArgv.slice(1), "rcs", join(buildDir, "libqjs.a"), ...QJS_ENGINE_SOURCES.map((s) => join(buildDir, `${basename(s, ".c")}.o`))]);
697
- // Atomic publish: rename fails if a concurrent build won the race, and
698
- // the winner's archive is just as good.
699
- const stageDir = `${buildDir}.stage`;
700
- await mkdir(stageDir);
701
- await copyFile(join(buildDir, "libqjs.a"), join(stageDir, "libqjs.a"));
702
- await rename(stageDir, cacheDir).catch(async () => {
703
- await rm(stageDir, { recursive: true, force: true });
704
- });
767
+ await publishVendorArtifact(join(buildDir, "libqjs.a"), archive);
705
768
  }
706
769
  finally {
707
770
  await rm(buildDir, { recursive: true, force: true });
@@ -722,7 +785,7 @@ function lreObjectPaths(sanitize, driver, buildIdentity, cacheRoot = vendorBuild
722
785
  }
723
786
  /** The libregexp objects for one flavor, compiled lazily on the first
724
787
  * regex-using static build (~1s) and cached like the engine archive —
725
- * vendor/.cache/<commit>-lre-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
788
+ * <build-cache>/vendor/<commit>-lre-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
726
789
  * publish (parallel first builds race safely; losers discard their work).
727
790
  * Plain is -Os (size class matters: the regex fence pins regex-using
728
791
  * binaries well under the engine class); asan matches the final link so
@@ -741,11 +804,10 @@ async function ensureLreObjects(sanitize, driver, buildIdentity, cacheRoot = ven
741
804
  `-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
742
805
  const vendor = vendorEngineDir();
743
806
  const objects = lreObjectPaths(sanitize, driver, buildIdentity, cacheRoot);
744
- const cacheDir = dirname(objects[0]);
745
- if ((await Promise.all(objects.map(fileExists))).every(Boolean))
807
+ if ((await Promise.all(objects.map(validVendorArtifact))).every(Boolean))
746
808
  return objects;
747
809
  await mkdir(cacheRoot, { recursive: true });
748
- const buildDir = await mkdtemp(join(cacheRoot, `build-lre-${flavor}-`));
810
+ const buildDir = await mkdtemp(join(tmpdir(), `scriptc-vendor-lre-${flavor}-`));
749
811
  try {
750
812
  // One -c per invocation: zig's COFF driver rejects multiple -c inputs
751
813
  // in a single command ("coff does not support linking multiple
@@ -762,9 +824,7 @@ async function ensureLreObjects(sanitize, driver, buildIdentity, cacheRoot = ven
762
824
  "-o", join(buildDir, f.replace(/\.c$/, ".o")),
763
825
  ], { cwd: buildDir });
764
826
  }
765
- // Atomic publish: rename fails if a concurrent build won the race, and
766
- // the winner's objects are just as good.
767
- await rename(buildDir, cacheDir).catch(() => undefined);
827
+ await Promise.all(objects.map((destination) => publishVendorArtifact(join(buildDir, basename(destination)), destination)));
768
828
  }
769
829
  finally {
770
830
  await rm(buildDir, { recursive: true, force: true });
@@ -788,7 +848,7 @@ function zlibObjectPaths(sanitize, driver, buildIdentity, cacheRoot = vendorBuil
788
848
  }
789
849
  /** The zlib objects for one flavor, compiled lazily on the first zlib-using
790
850
  * CROSS build (~1s) and cached like the lre objects —
791
- * vendor/.cache/zlib-<version>-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
851
+ * <build-cache>/vendor/zlib-<version>-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
792
852
  * publish (parallel first builds race safely; losers discard their work).
793
853
  * Plain is -Os, asan matches the final link so the sanitized lane
794
854
  * instruments the codec too. The flavor keys the driver and target exactly
@@ -801,11 +861,10 @@ async function ensureZlibObjects(sanitize, driver, buildIdentity, cacheRoot = ve
801
861
  `-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
802
862
  const vendor = vendorZlibDir();
803
863
  const objects = zlibObjectPaths(sanitize, driver, buildIdentity, cacheRoot);
804
- const cacheDir = dirname(objects[0]);
805
- if ((await Promise.all(objects.map(fileExists))).every(Boolean))
864
+ if ((await Promise.all(objects.map(validVendorArtifact))).every(Boolean))
806
865
  return objects;
807
866
  await mkdir(cacheRoot, { recursive: true });
808
- const buildDir = await mkdtemp(join(cacheRoot, `build-zlib-${flavor}-`));
867
+ const buildDir = await mkdtemp(join(tmpdir(), `scriptc-vendor-zlib-${flavor}-`));
809
868
  try {
810
869
  // One -c per invocation, the lre recipe (zig's COFF driver rejects
811
870
  // multiple -c inputs in a single command).
@@ -820,9 +879,7 @@ async function ensureZlibObjects(sanitize, driver, buildIdentity, cacheRoot = ve
820
879
  "-o", join(buildDir, f.replace(/\.c$/, ".o")),
821
880
  ], { cwd: buildDir });
822
881
  }
823
- // Atomic publish: rename fails if a concurrent build won the race, and
824
- // the winner's objects are just as good.
825
- await rename(buildDir, cacheDir).catch(() => undefined);
882
+ await Promise.all(objects.map((destination) => publishVendorArtifact(join(buildDir, basename(destination)), destination)));
826
883
  }
827
884
  finally {
828
885
  await rm(buildDir, { recursive: true, force: true });
@@ -849,7 +906,7 @@ function curlStubDirPath(driver, buildIdentity, cacheRoot = vendorBuildCacheRoot
849
906
  }
850
907
  /** The libcurl import stub for one CROSS target, generated lazily on the
851
908
  * first fetch-using cross build and cached like the zlib objects —
852
- * vendor/.cache/curl-stub-<target>-<toolchain>/libcurl.so, atomic-rename publish. The
909
+ * <build-cache>/vendor/curl-stub-<target>-<toolchain>/libcurl.so, atomic-rename publish. The
853
910
  * stub is empty definitions of CURL_STUB_SYMBOLS compiled `-shared` with
854
911
  * `-soname libcurl.so.4`: linking against it satisfies scr_fetch_curl.c's
855
912
  * references and records a plain `DT_NEEDED libcurl.so.4`, which the
@@ -861,10 +918,10 @@ function curlStubDirPath(driver, buildIdentity, cacheRoot = vendorBuildCacheRoot
861
918
  async function ensureCurlStub(driver, buildIdentity, cacheRoot = vendorBuildCacheRoot()) {
862
919
  const cacheDir = curlStubDirPath(driver, buildIdentity, cacheRoot);
863
920
  const lib = join(cacheDir, "libcurl.so");
864
- if (await fileExists(lib))
921
+ if (await validVendorArtifact(lib))
865
922
  return cacheDir;
866
923
  await mkdir(cacheRoot, { recursive: true });
867
- const buildDir = await mkdtemp(join(cacheRoot, `build-curl-stub-`));
924
+ const buildDir = await mkdtemp(join(tmpdir(), "scriptc-vendor-curl-stub-"));
868
925
  try {
869
926
  const src = join(buildDir, "stub.c");
870
927
  await writeFile(src, CURL_STUB_SYMBOLS.map((s) => `void ${s}(void) {}\n`).join(""));
@@ -876,14 +933,7 @@ async function ensureCurlStub(driver, buildIdentity, cacheRoot = vendorBuildCach
876
933
  src,
877
934
  "-o", join(buildDir, "libcurl.so"),
878
935
  ]);
879
- // Atomic publish: rename fails if a concurrent build won the race, and
880
- // the winner's stub is just as good.
881
- const stageDir = `${buildDir}.stage`;
882
- await mkdir(stageDir);
883
- await copyFile(join(buildDir, "libcurl.so"), join(stageDir, "libcurl.so"));
884
- await rename(stageDir, cacheDir).catch(async () => {
885
- await rm(stageDir, { recursive: true, force: true });
886
- });
936
+ await publishVendorArtifact(join(buildDir, "libcurl.so"), lib);
887
937
  }
888
938
  finally {
889
939
  await rm(buildDir, { recursive: true, force: true });
@@ -899,7 +949,7 @@ function tlsArchivePath(sanitize, driver, buildIdentity, cacheRoot = vendorBuild
899
949
  }
900
950
  /** The mbedTLS archive for one flavor, compiled lazily on the first
901
951
  * TLS-using build (~15s over ~110 TUs, parallelized) and cached like the
902
- * engine archive — vendor/.cache/mbedtls-<version>-<flavor>-<target>-<toolchain>/libmbedtls.a —
952
+ * engine archive — <build-cache>/vendor/mbedtls-<version>-<flavor>-<target>-<toolchain>/libmbedtls.a —
903
953
  * with the same atomic-rename publish (parallel first builds race safely;
904
954
  * losers discard their work). Plain is -Os (the TLS stack is link-gated
905
955
  * but still wants its size class small); asan matches the final link so
@@ -918,13 +968,14 @@ async function ensureTlsArchive(sanitize, driver, buildIdentity, cacheRoot = ven
918
968
  const arArgv = driver.target !== null ? [...driver.argv.slice(0, 1), "ar"] : ["ar"];
919
969
  const vendor = vendorTlsDir();
920
970
  const archive = tlsArchivePath(sanitize, driver, buildIdentity, cacheRoot);
921
- const cacheDir = dirname(archive);
922
- if (await fileExists(archive))
971
+ if (await validVendorArtifact(archive))
923
972
  return archive;
924
973
  await mkdir(cacheRoot, { recursive: true });
925
- const buildDir = await mkdtemp(join(cacheRoot, `build-mbedtls-${flavor}-`));
974
+ const buildDir = await mkdtemp(join(tmpdir(), `scriptc-vendor-mbedtls-${flavor}-`));
926
975
  try {
927
- const sources = (await readdir(join(vendor, "library"))).filter((n) => n.endsWith(".c")).sort();
976
+ const sources = (await readdir(join(vendor, "library")))
977
+ .filter((name) => !name.startsWith(".") && name.endsWith(".c"))
978
+ .sort();
928
979
  const cflags = [
929
980
  "-std=c11",
930
981
  ...driver.targetArgs,
@@ -937,14 +988,7 @@ async function ensureTlsArchive(sanitize, driver, buildIdentity, cacheRoot = ven
937
988
  await Promise.all(sources.slice(i, i + width).map((src) => execFileAsync(compileArgv[0] ?? "clang", [...compileArgv.slice(1), ...cflags, "-c", join(vendor, "library", src), "-o", join(buildDir, `${basename(src, ".c")}.o`)])));
938
989
  }
939
990
  await execFileAsync(arArgv[0] ?? "ar", [...arArgv.slice(1), "rcs", join(buildDir, "libmbedtls.a"), ...sources.map((s) => join(buildDir, `${basename(s, ".c")}.o`))]);
940
- // Atomic publish: rename fails if a concurrent build won the race, and
941
- // the winner's archive is just as good.
942
- const stageDir = `${buildDir}.stage`;
943
- await mkdir(stageDir);
944
- await copyFile(join(buildDir, "libmbedtls.a"), join(stageDir, "libmbedtls.a"));
945
- await rename(stageDir, cacheDir).catch(async () => {
946
- await rm(stageDir, { recursive: true, force: true });
947
- });
991
+ await publishVendorArtifact(join(buildDir, "libmbedtls.a"), archive);
948
992
  }
949
993
  finally {
950
994
  await rm(buildDir, { recursive: true, force: true });
@@ -958,6 +1002,11 @@ async function ensureTlsArchive(sanitize, driver, buildIdentity, cacheRoot = ven
958
1002
  * objects apart from executable-lane objects). Persistent builds cache the
959
1003
  * completed archive by program-TU content and cache runtime objects separately,
960
1004
  * so an edit recompiles only the changed program object before re-archiving.
1005
+ * Large LLVM dev TUs split further into stable symbol-hash shards: compile
1006
+ * those in parallel, cache each object independently, then relocatably merge
1007
+ * them back into the archive's one canonical program member. A localized edit
1008
+ * recompiles only the changed buckets; exact repeats use the merged-object or
1009
+ * completed-archive tiers and never repeat the merge.
961
1010
  * The base set narrows from the executable lane's unconditional sources:
962
1011
  * scr_async.c (fibers, timers, the loop) and scr_child.c drop — the
963
1012
  * async_free refusal already guarantees nothing references them — and
@@ -977,9 +1026,94 @@ const LIB_RUNTIME_SOURCES = [
977
1026
  ...RUNTIME_SOURCES.filter((f) => f !== "scr_async.c" && f !== "scr_child.c" && f !== "scr_ffi.c"),
978
1027
  "scr_library.c",
979
1028
  ];
1029
+ function updateProgramShardCacheIdentity(hash, shards, publicSymbols, mergeIdentity) {
1030
+ hash.update("\0program-shards\0");
1031
+ if (shards === undefined) {
1032
+ hash.update("<none>\0");
1033
+ }
1034
+ else {
1035
+ hash.update("<present>\0");
1036
+ for (const shard of shards) {
1037
+ hash.update(shard.name).update("\0").update(shard.source).update("\0");
1038
+ }
1039
+ }
1040
+ hash.update("\0program-public-symbols\0");
1041
+ if (publicSymbols === undefined) {
1042
+ hash.update("<none>\0");
1043
+ }
1044
+ else {
1045
+ hash.update("<present>\0");
1046
+ for (const symbol of publicSymbols)
1047
+ hash.update(symbol).update("\0");
1048
+ }
1049
+ hash.update("\0program-shard-merge\0");
1050
+ hash.update(mergeIdentity === undefined ? "<none>\0" : `<present>\0${mergeIdentity}\0`);
1051
+ }
1052
+ /** Identity of the machinery that turns LLVM shard objects back into the
1053
+ * canonical program member, or null when the target cannot do so. Sharding is
1054
+ * only a build optimization: a missing host tool or unsupported object class
1055
+ * retains the ordinary single-TU compile. The identity joins every cache tier
1056
+ * that contains merged bytes; raw shard-object keys deliberately omit it. */
1057
+ async function resolveProgramShardMergeIdentity(driver) {
1058
+ const platform = targetPlatform(driver);
1059
+ // Mach-O merging uses the host's ld64. A Darwin target produced from a
1060
+ // non-Darwin host can still compile as one canonical Zig TU, but the host's
1061
+ // ELF/COFF linker cannot combine those objects.
1062
+ if (platform === "darwin") {
1063
+ if (process.platform !== "darwin")
1064
+ return null;
1065
+ const ld = await resolvedTool("ld");
1066
+ return ld === null
1067
+ ? null
1068
+ : rememberFingerprintDependencies(`program-shard-merge-darwin-v1\0${ld.cacheIdentity}`, [ld.canonicalPath]);
1069
+ }
1070
+ if (platform === "linux") {
1071
+ if (driver.target === null) {
1072
+ const [ld, objcopy] = await Promise.all([
1073
+ resolvedTool("ld"),
1074
+ resolvedTool("objcopy"),
1075
+ ]);
1076
+ return ld === null || objcopy === null
1077
+ ? null
1078
+ : rememberFingerprintDependencies(`program-shard-merge-linux-v1\0${ld.cacheIdentity}\0${objcopy.cacheIdentity}`, [ld.canonicalPath, objcopy.canonicalPath]);
1079
+ }
1080
+ const arch = driver.target.split("-", 1)[0];
1081
+ if (arch !== "x86_64" && arch !== "aarch64")
1082
+ return null;
1083
+ const compiler = await resolvedTool(driver.argv[0] ?? "zig");
1084
+ return compiler === null
1085
+ ? null
1086
+ : rememberFingerprintDependencies(`program-shard-merge-cross-elf-v1\0${arch}\0${compiler.cacheIdentity}`, [compiler.canonicalPath]);
1087
+ }
1088
+ if (platform === "win32") {
1089
+ const arch = driver.target?.split("-", 1)[0] ?? process.arch;
1090
+ return arch === "x86_64" || arch === "x64"
1091
+ ? `program-shard-merge-coff-v1\0${arch}`
1092
+ : null;
1093
+ }
1094
+ return null;
1095
+ }
980
1096
  export async function compileLibArchive(opts) {
981
1097
  const rtDir = runtimeSrcDir();
982
1098
  const driver = resolveCc();
1099
+ const shardNames = new Set();
1100
+ const programShardsValid = opts.programShards?.every((shard) => {
1101
+ if (basename(shard.name) !== shard.name || !shard.name.endsWith(".ll") ||
1102
+ shardNames.has(shard.name))
1103
+ return false;
1104
+ shardNames.add(shard.name);
1105
+ return true;
1106
+ }) === true;
1107
+ const programShardsRequested = opts.cPath.endsWith(".ll") &&
1108
+ programShardsValid && opts.programShards !== undefined && opts.programShards.length > 1 &&
1109
+ opts.programPublicSymbols !== undefined
1110
+ ? opts.programShards
1111
+ : null;
1112
+ const programShardMergeIdentity = programShardsRequested === null
1113
+ ? null
1114
+ : await resolveProgramShardMergeIdentity(driver);
1115
+ const programShards = programShardMergeIdentity === null ? null : programShardsRequested;
1116
+ const programPublicSymbols = programShards === null ? undefined : opts.programPublicSymbols;
983
1117
  const sanitize = opts.sanitize ?? false;
984
1118
  const optimization = opts.optimization ?? "release";
985
1119
  const regex = opts.regex ?? false;
@@ -1049,11 +1183,14 @@ export async function compileLibArchive(opts) {
1049
1183
  }
1050
1184
  }
1051
1185
  let implicitToolchain = null;
1186
+ let implicitCompileToolchain = null;
1052
1187
  let runtimeCompilerInvocation = null;
1053
1188
  let programCompilerInvocation = null;
1054
1189
  if (persistentDriverCache) {
1055
1190
  try {
1056
- implicitToolchain = await implicitToolchainFingerprint(driver, toolchainEnv);
1191
+ const fingerprints = await implicitToolchainFingerprints(driver, toolchainEnv);
1192
+ implicitToolchain = fingerprints.complete;
1193
+ implicitCompileToolchain = fingerprints.compile;
1057
1194
  }
1058
1195
  catch {
1059
1196
  // An identity probe is cache machinery, never a reason a valid native
@@ -1087,34 +1224,38 @@ export async function compileLibArchive(opts) {
1087
1224
  let archiverVersion = "";
1088
1225
  let runtimeHash = "";
1089
1226
  let programDependencyHash = "";
1090
- let cachedProgramBytes = null;
1227
+ let cachedProgramBytes = opts.programSource === undefined
1228
+ ? null
1229
+ : Buffer.from(opts.programSource, "utf8");
1230
+ const identityBytes = opts.identityCSource === undefined
1231
+ ? null
1232
+ : Buffer.from(opts.identityCSource, "utf8");
1091
1233
  if (root !== null) {
1092
1234
  try {
1093
1235
  const [cv, fingerprint, programBytes] = await Promise.all([
1094
1236
  ccVersionOnce(driver.argv, toolchainEnv, true),
1095
1237
  runtimeFingerprint(rtDir),
1096
- readFile(opts.cPath),
1238
+ cachedProgramBytes === null ? readFile(opts.cPath) : Promise.resolve(cachedProgramBytes),
1097
1239
  ]);
1098
1240
  compilerVersion = cv;
1099
1241
  runtimeHash = fingerprint;
1100
1242
  cachedProgramBytes = programBytes;
1243
+ programDependencyHash = await translationUnitDependencyFingerprint(driver, cflags, opts.cPath, programBytes, toolchainEnv);
1101
1244
  if (cacheCompleteArchive) {
1102
- const [av, programDependencies] = await Promise.all([
1103
- toolVersionOnce(arArgv, toolchainEnv, true),
1104
- translationUnitDependencyFingerprint(driver, cflags, opts.cPath, programBytes, toolchainEnv),
1105
- ]);
1245
+ const av = await toolVersionOnce(arArgv, toolchainEnv, true);
1106
1246
  archiverVersion = av;
1107
- programDependencyHash = programDependencies;
1108
1247
  const key = createHash("sha256")
1109
- // v7 adds effective compiler-wrapper invocations for the real runtime
1110
- // and program compile flavors.
1111
- .update("lib-v7\0")
1248
+ // v10 adds the shard-merge implementation/tool identity. The
1249
+ // canonical TU still keys source semantics; shard, keep, and merge
1250
+ // bytes key the exact merged program object so ABI projections,
1251
+ // tool replacements, and single-/multi-TU producers never collide.
1252
+ .update("lib-v10\0")
1112
1253
  .update(cacheTargetIdentity(driver)).update("\0")
1113
1254
  .update(toolchainEnv).update("\0")
1114
1255
  .update(implicitToolchain).update("\0")
1115
1256
  .update(runtimeCompilerInvocation).update("\0")
1116
1257
  .update(programCompilerInvocation).update("\0")
1117
- .update(programDependencies).update("\0")
1258
+ .update(programDependencyHash).update("\0")
1118
1259
  .update(opts.cacheIdentity).update("\0")
1119
1260
  .update(driver.argv.join("\x1f")).update("\0")
1120
1261
  .update(cv).update("\0")
@@ -1128,9 +1269,14 @@ export async function compileLibArchive(opts) {
1128
1269
  // latter. Archive members also inherit the TU's basename.
1129
1270
  .update(opts.cPath).update("\0")
1130
1271
  .update(resolve(opts.cPath)).update("\0")
1131
- .update(programBytes)
1272
+ .update(programBytes);
1273
+ updateProgramShardCacheIdentity(key, programShards ?? undefined, programPublicSymbols, programShardMergeIdentity ?? undefined);
1274
+ const keyHex = key
1275
+ .update("\0identity\0")
1276
+ .update(identityBytes === null ? "<none>" : "<generated>").update("\0")
1277
+ .update(identityBytes ?? Buffer.alloc(0))
1132
1278
  .digest("hex");
1133
- cachedArchive = join(root, "lib", key);
1279
+ cachedArchive = join(root, "lib", keyHex);
1134
1280
  const tmpOut = privateSiblingPath(opts.outPath, "lib-hit");
1135
1281
  try {
1136
1282
  await mkdir(dirname(opts.outPath), { recursive: true });
@@ -1155,19 +1301,19 @@ export async function compileLibArchive(opts) {
1155
1301
  cachedArchive = null;
1156
1302
  }
1157
1303
  }
1158
- const transientVendorRoot = persistentDriverCache && implicitToolchain !== null
1304
+ const transientVendorRoot = root !== null && implicitToolchain !== null
1159
1305
  ? null
1160
1306
  : join(tmpdir(), `scriptc-lib-vendor-${process.pid}-${Math.random().toString(36).slice(2)}`);
1161
- const vendorCacheRoot = transientVendorRoot ?? vendorBuildCacheRoot();
1307
+ const vendorCacheRoot = transientVendorRoot ?? vendorBuildCacheRoot(root ?? undefined);
1162
1308
  try {
1163
- const lreObjects = regex
1164
- ? await ensureLreObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
1165
- : [];
1166
- const zlibObjects = opts.zlib
1167
- ? await ensureZlibObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
1168
- : [];
1169
1309
  const buildDir = await mkdtemp(join(tmpdir(), "scriptc-lib-"));
1170
1310
  try {
1311
+ const lreObjects = regex
1312
+ ? await stageVendorInputs(() => ensureLreObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot), join(buildDir, "vendor-lre"))
1313
+ : [];
1314
+ const zlibObjects = opts.zlib
1315
+ ? await stageVendorInputs(() => ensureZlibObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot), join(buildDir, "vendor-zlib"))
1316
+ : [];
1171
1317
  const compileOne = async (src, objName, compilerVisibleSource) => {
1172
1318
  const obj = join(buildDir, objName);
1173
1319
  const args = [
@@ -1198,9 +1344,171 @@ export async function compileLibArchive(opts) {
1198
1344
  const programSource = cachedProgramBytes === null
1199
1345
  ? opts.cPath
1200
1346
  : join(buildDir, `program${opts.cPath.endsWith(".ll") ? ".ll" : ".c"}`);
1201
- if (cachedProgramBytes !== null)
1347
+ if (cachedProgramBytes !== null && programShards === null) {
1202
1348
  await writeFile(programSource, cachedProgramBytes);
1203
- const programObject = await compileOne(programSource, `${stem}.program.o`, cachedProgramBytes === null ? undefined : opts.cPath);
1349
+ }
1350
+ let cachedProgramObject = null;
1351
+ if (root !== null && cachedProgramBytes !== null && compilerVersion !== "" &&
1352
+ implicitToolchain !== null && programCompilerInvocation !== null) {
1353
+ const programKey = createHash("sha256")
1354
+ .update("lib-program-obj-v3\0")
1355
+ .update(cacheTargetIdentity(driver)).update("\0")
1356
+ .update(toolchainEnv).update("\0")
1357
+ .update(implicitToolchain).update("\0")
1358
+ .update(programCompilerInvocation).update("\0")
1359
+ .update(opts.cacheIdentity).update("\0")
1360
+ .update(driver.argv.join("\x1f")).update("\0")
1361
+ .update(compilerVersion).update("\0")
1362
+ .update(runtimeHash).update("\0")
1363
+ .update(programDependencyHash).update("\0")
1364
+ .update(programCompilerArgs.join("\x1f")).update("\0")
1365
+ .update(opts.cPath).update("\0")
1366
+ .update(resolve(opts.cPath)).update("\0")
1367
+ .update(cachedProgramBytes);
1368
+ updateProgramShardCacheIdentity(programKey, programShards ?? undefined, programPublicSymbols, programShardMergeIdentity ?? undefined);
1369
+ const programKeyHex = programKey.digest("hex");
1370
+ cachedProgramObject = join(root, "program-obj", programKeyHex);
1371
+ }
1372
+ const stagedProgramObject = join(buildDir, `${stem}.program.o`);
1373
+ let programObject;
1374
+ let programShardFallback = false;
1375
+ if (cachedProgramObject !== null &&
1376
+ await copyValidCachedFile(cachedProgramObject, stagedProgramObject)) {
1377
+ programObject = stagedProgramObject;
1378
+ }
1379
+ else if (programShards !== null) {
1380
+ try {
1381
+ const shardEntries = programShards.map((shard, index) => {
1382
+ const sourcePath = join(buildDir, shard.name);
1383
+ const staged = join(buildDir, `${stem}.program-${index.toString().padStart(3, "0")}.o`);
1384
+ let cachePath = null;
1385
+ if (root !== null && compilerVersion !== "" && implicitCompileToolchain !== null &&
1386
+ programCompilerInvocation !== null) {
1387
+ const key = createHash("sha256")
1388
+ // v2 removes the broad implicit-toolchain fingerprint: it
1389
+ // includes the linker selected by the driver, but raw shard
1390
+ // objects are compile-only outputs. The compile-only toolchain
1391
+ // identity retains compiler/config/header/assembler inputs;
1392
+ // the effective program invocation pins this exact flag lane.
1393
+ // Merge-tool identity belongs only to the merged-object and
1394
+ // completed-archive tiers.
1395
+ .update("lib-program-shard-v2\0")
1396
+ .update(cacheTargetIdentity(driver)).update("\0")
1397
+ .update(toolchainEnv).update("\0")
1398
+ .update(implicitCompileToolchain).update("\0")
1399
+ .update(programCompilerInvocation).update("\0")
1400
+ .update(opts.cacheIdentity).update("\0")
1401
+ .update(driver.argv.join("\x1f")).update("\0")
1402
+ .update(compilerVersion).update("\0")
1403
+ .update(runtimeHash).update("\0")
1404
+ .update(programDependencyHash).update("\0")
1405
+ .update(programCompilerArgs.join("\x1f")).update("\0")
1406
+ .update(opts.cPath).update("\0")
1407
+ .update(resolve(opts.cPath)).update("\0")
1408
+ .update(shard.name).update("\0")
1409
+ .update(shard.source)
1410
+ .digest("hex");
1411
+ cachePath = join(root, "program-shard", key);
1412
+ }
1413
+ return { ...shard, sourcePath, staged, cachePath, missed: false };
1414
+ });
1415
+ const shardWidth = Math.min(8, availableParallelism());
1416
+ for (let i = 0; i < shardEntries.length; i += shardWidth) {
1417
+ await Promise.all(shardEntries.slice(i, i + shardWidth).map(async (entry) => {
1418
+ await writeFile(entry.sourcePath, entry.source);
1419
+ if (entry.cachePath !== null &&
1420
+ await copyValidCachedFile(entry.cachePath, entry.staged))
1421
+ return;
1422
+ entry.missed = true;
1423
+ await compileOne(entry.sourcePath, basename(entry.staged), opts.cPath);
1424
+ }));
1425
+ }
1426
+ const publishable = shardEntries.filter((entry) => entry.missed && entry.cachePath !== null);
1427
+ const mergedProgramObject = await localizeLibraryObjects(driver, arArgv, buildDir, shardEntries.map((entry) => entry.staged), [], programPublicSymbols, `${stem}.program`);
1428
+ // Keep the canonical archive member spelling. The fact that native
1429
+ // compilation used shards is an implementation detail; consumers and
1430
+ // deterministic cache tests continue to see `<stem>.program.o`.
1431
+ await rename(mergedProgramObject, stagedProgramObject);
1432
+ programObject = stagedProgramObject;
1433
+ if (cachedProgramObject !== null || publishable.length > 0) {
1434
+ try {
1435
+ const [currentRuntime, currentInvocation, currentDependencies, currentCompiler] = await Promise.all([
1436
+ runtimeFingerprint(rtDir),
1437
+ effectiveCompilerInvocationFingerprint(driver, toolchainEnv, programCompilerArgs, programSourceExtension),
1438
+ translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cachedProgramBytes, toolchainEnv),
1439
+ ccVersionOnce(driver.argv, toolchainEnv, true),
1440
+ ]);
1441
+ const shardInputsStillMatch = currentRuntime === runtimeHash &&
1442
+ currentInvocation === programCompilerInvocation &&
1443
+ currentDependencies === programDependencyHash && currentCompiler === compilerVersion;
1444
+ const currentFingerprints = shardInputsStillMatch
1445
+ ? await implicitToolchainFingerprints(driver, toolchainEnv)
1446
+ : null;
1447
+ const compileInputsStillMatch = shardInputsStillMatch &&
1448
+ currentFingerprints?.compile === implicitCompileToolchain;
1449
+ let mergedInputsStillMatch = false;
1450
+ if (compileInputsStillMatch && cachedProgramObject !== null) {
1451
+ const currentMerge = await resolveProgramShardMergeIdentity(driver).catch(() => null);
1452
+ mergedInputsStillMatch =
1453
+ currentFingerprints?.complete === implicitToolchain &&
1454
+ currentMerge === programShardMergeIdentity;
1455
+ }
1456
+ if (compileInputsStillMatch) {
1457
+ await Promise.all([
1458
+ ...(mergedInputsStillMatch
1459
+ ? [publishCachedFile(programObject, cachedProgramObject)]
1460
+ : []),
1461
+ ...publishable.map((entry) => publishCachedFile(entry.staged, entry.cachePath)),
1462
+ ]);
1463
+ }
1464
+ }
1465
+ catch {
1466
+ // Best-effort: the merged program object is already valid.
1467
+ }
1468
+ }
1469
+ }
1470
+ catch {
1471
+ // Sharding is only an optimization. A present but incompatible or
1472
+ // failing merge tool (or a shard-only compiler failure) must not
1473
+ // turn a valid canonical LLVM TU into a failed library build. Do not
1474
+ // publish this retry under shard-derived object/archive cache keys.
1475
+ programShardFallback = true;
1476
+ if (cachedProgramBytes !== null)
1477
+ await writeFile(programSource, cachedProgramBytes);
1478
+ programObject = await compileOne(programSource, `${stem}.program.o`, cachedProgramBytes === null ? undefined : opts.cPath);
1479
+ }
1480
+ }
1481
+ else {
1482
+ programObject = await compileOne(programSource, `${stem}.program.o`, cachedProgramBytes === null ? undefined : opts.cPath);
1483
+ if (cachedProgramObject !== null) {
1484
+ try {
1485
+ const [currentRuntime, currentImplicit, currentInvocation, currentDependencies, currentCompiler] = await Promise.all([
1486
+ runtimeFingerprint(rtDir),
1487
+ implicitToolchainFingerprint(driver, toolchainEnv),
1488
+ effectiveCompilerInvocationFingerprint(driver, toolchainEnv, programCompilerArgs, programSourceExtension),
1489
+ translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cachedProgramBytes, toolchainEnv),
1490
+ ccVersionOnce(driver.argv, toolchainEnv, true),
1491
+ ]);
1492
+ if (currentRuntime === runtimeHash &&
1493
+ currentImplicit === implicitToolchain &&
1494
+ currentInvocation === programCompilerInvocation &&
1495
+ currentDependencies === programDependencyHash &&
1496
+ currentCompiler === compilerVersion) {
1497
+ await publishCachedFile(programObject, cachedProgramObject);
1498
+ }
1499
+ }
1500
+ catch {
1501
+ // Best-effort: the archive build already owns a valid object.
1502
+ }
1503
+ }
1504
+ }
1505
+ const identityObject = identityBytes === null
1506
+ ? null
1507
+ : await (async () => {
1508
+ const source = join(buildDir, "identity.c");
1509
+ await writeFile(source, identityBytes);
1510
+ return compileOne(source, `${stem}.identity.o`);
1511
+ })();
1204
1512
  let runtimeObjects = null;
1205
1513
  let cacheInputsStable = true;
1206
1514
  let objectImplicitVerification = null;
@@ -1233,14 +1541,14 @@ export async function compileLibArchive(opts) {
1233
1541
  runtimeObjects.push(...(await Promise.all(sources.slice(i, i + width).map((f) => compileOne(join(rtDir, f), f.replace(/\.c$/, ".o"))))));
1234
1542
  }
1235
1543
  }
1236
- const objects = [programObject, ...runtimeObjects, ...lreObjects, ...zlibObjects];
1544
+ const objects = [programObject, ...(identityObject === null ? [] : [identityObject]), ...runtimeObjects, ...lreObjects, ...zlibObjects];
1237
1545
  // Multi-instance library mode: the archive's one member becomes the
1238
1546
  // combined, symbol-localized object (cached vendor/runtime objects
1239
1547
  // are read-only inputs here — the combine step never mutates them).
1240
1548
  const archiveMembers = opts.localizeSymbols === undefined
1241
1549
  ? objects
1242
1550
  : [
1243
- await localizeLibraryObjects(driver, arArgv, buildDir, programObject, [...runtimeObjects, ...lreObjects, ...zlibObjects], opts.localizeSymbols, stem),
1551
+ await localizeLibraryObjects(driver, arArgv, buildDir, [programObject, ...(identityObject === null ? [] : [identityObject])], [...runtimeObjects, ...lreObjects, ...zlibObjects], opts.localizeSymbols, stem),
1244
1552
  ];
1245
1553
  // A cacheable or runtime-localized build owns a private archive from
1246
1554
  // `ar` through publication. Localized archives deliberately bypass the
@@ -1267,7 +1575,7 @@ export async function compileLibArchive(opts) {
1267
1575
  programDependencyHash !== "" &&
1268
1576
  compilerVersion !== "" &&
1269
1577
  archiverVersion !== "") {
1270
- const [currentRuntime, currentImplicit, currentRuntimeInvocation, currentProgramInvocation, currentProgramDependencies, currentCompiler, currentArchiver] = await Promise.all([
1578
+ const [currentRuntime, currentImplicit, currentRuntimeInvocation, currentProgramInvocation, currentProgramDependencies, currentCompiler, currentArchiver, currentProgramShardMerge] = await Promise.all([
1271
1579
  runtimeFingerprint(rtDir).catch(() => null),
1272
1580
  implicitToolchainFingerprint(driver, toolchainEnv).catch(() => null),
1273
1581
  effectiveCompilerInvocationFingerprint(driver, toolchainEnv, cflags).catch(() => null),
@@ -1275,16 +1583,21 @@ export async function compileLibArchive(opts) {
1275
1583
  translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cachedProgramBytes, toolchainEnv).catch(() => null),
1276
1584
  ccVersionOnce(driver.argv, toolchainEnv, true).catch(() => null),
1277
1585
  toolVersionOnce(arArgv, toolchainEnv, true).catch(() => null),
1586
+ programShards === null
1587
+ ? Promise.resolve(null)
1588
+ : resolveProgramShardMergeIdentity(driver).catch(() => null),
1278
1589
  ]);
1279
1590
  runtimeStillMatchesKey =
1280
1591
  cacheInputsStable &&
1592
+ !programShardFallback &&
1281
1593
  currentRuntime === runtimeHash &&
1282
1594
  currentImplicit === implicitToolchain &&
1283
1595
  currentRuntimeInvocation === runtimeCompilerInvocation &&
1284
1596
  currentProgramInvocation === programCompilerInvocation &&
1285
1597
  currentProgramDependencies === programDependencyHash &&
1286
1598
  currentCompiler === compilerVersion &&
1287
- currentArchiver === archiverVersion;
1599
+ currentArchiver === archiverVersion &&
1600
+ currentProgramShardMerge === programShardMergeIdentity;
1288
1601
  }
1289
1602
  if (cachedArchive !== null && root !== null && runtimeStillMatchesKey) {
1290
1603
  try {
@@ -1320,7 +1633,8 @@ export async function compileLibArchive(opts) {
1320
1633
  * shared by design. Windows embedders additionally link advapi32, iphlpapi,
1321
1634
  * and ws2_32.
1322
1635
  *
1323
- * Member selection matters: a classic archive's unused members (and their
1636
+ * The generated program and optional identity objects are mandatory roots;
1637
+ * support-member selection still matters. A classic archive's unused members (and their
1324
1638
  * undefined references to units library mode excludes, like the
1325
1639
  * fs-promises unit's fiber symbols) never reach an embedder's link. A
1326
1640
  * blind merge of every object would carry those references into the one
@@ -1329,7 +1643,7 @@ export async function compileLibArchive(opts) {
1329
1643
  * members the program object transitively needs (the COFF arm implements
1330
1644
  * the same member semantics in process).
1331
1645
  *
1332
- * Mach-O — one host-ld64 invocation: -r merges program + needed members,
1646
+ * Mach-O — one host-ld64 invocation: -r merges roots + needed members,
1333
1647
  * -exported_symbols_list demotes every unlisted global to
1334
1648
  * private extern, and -r without -keep_private_externs writes
1335
1649
  * private externs out as non-external symbols. Apple ASan's
@@ -1360,7 +1674,7 @@ export async function compileLibArchive(opts) {
1360
1674
  * mirrors MSVC link.exe; zig's COFF driver refuses multi-object
1361
1675
  * merges) and llvm-objcopy rejects symbol-scope flags for
1362
1676
  * COFF, so no tool pairing exists to shell out to. */
1363
- async function localizeLibraryObjects(driver, arArgv, buildDir, programObject, supportObjects, keepSymbols, stem) {
1677
+ async function localizeLibraryObjects(driver, arArgv, buildDir, rootObjects, supportObjects, keepSymbols, stem) {
1364
1678
  const platform = targetPlatform(driver);
1365
1679
  const combined = join(buildDir, `${stem}.localized.o`);
1366
1680
  const staging = join(buildDir, `${stem}.localize-staging.a`);
@@ -1378,10 +1692,13 @@ async function localizeLibraryObjects(driver, arArgv, buildDir, programObject, s
1378
1692
  if (platform === "win32") {
1379
1693
  // COFF has no relocatable-link tool to stage through; the member
1380
1694
  // selection and combine+demote happen in process over the object bytes.
1381
- const [program, ...support] = await Promise.all([programObject, ...supportObjects].map((path) => readFile(path)));
1695
+ const [roots, support] = await Promise.all([
1696
+ Promise.all(rootObjects.map((path) => readFile(path))),
1697
+ Promise.all(supportObjects.map((path) => readFile(path))),
1698
+ ]);
1382
1699
  try {
1383
- await writeFile(combined, mergeAndLocalizeCoffObjects(program, support, new Set(keepSymbols), {
1384
- program: basename(programObject),
1700
+ await writeFile(combined, mergeAndLocalizeCoffObjects(roots, support, new Set(keepSymbols), {
1701
+ roots: rootObjects.map((path) => basename(path)),
1385
1702
  support: supportObjects.map((path) => basename(path)),
1386
1703
  }));
1387
1704
  }
@@ -1390,14 +1707,17 @@ async function localizeLibraryObjects(driver, arArgv, buildDir, programObject, s
1390
1707
  }
1391
1708
  return combined;
1392
1709
  }
1393
- await run([arArgv[0] ?? "ar", ...arArgv.slice(1), "rcs", staging, ...supportObjects]);
1710
+ const supportArgs = supportObjects.length === 0 ? [] : [staging];
1711
+ if (supportObjects.length > 0) {
1712
+ await run([arArgv[0] ?? "ar", ...arArgv.slice(1), "rcs", staging, ...supportObjects]);
1713
+ }
1394
1714
  if (platform === "darwin") {
1395
1715
  await writeFile(keepFile, keepSymbols.map((s) => `_${s}\n`).join(""));
1396
- await run(["ld", "-r", programObject, staging, "-o", combined, "-exported_symbols_list", keepFile]);
1716
+ await run(["ld", "-r", ...rootObjects, ...supportArgs, "-o", combined, "-exported_symbols_list", keepFile]);
1397
1717
  }
1398
1718
  else if (platform === "linux" && driver.target === null) {
1399
1719
  await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join(""));
1400
- await run(["ld", "-r", "--force-group-allocation", programObject, staging, "-o", combined]);
1720
+ await run(["ld", "-r", "--force-group-allocation", ...rootObjects, ...supportArgs, "-o", combined]);
1401
1721
  await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]);
1402
1722
  }
1403
1723
  else if (platform === "linux") {
@@ -1412,7 +1732,7 @@ async function localizeLibraryObjects(driver, arArgv, buildDir, programObject, s
1412
1732
  ...driver.argv.slice(1),
1413
1733
  "-target", driver.zigTarget ?? driver.target,
1414
1734
  "-nostdlib",
1415
- "-r", programObject, staging,
1735
+ "-r", ...rootObjects, ...supportArgs,
1416
1736
  "-o", combined,
1417
1737
  ]);
1418
1738
  try {
@@ -1431,7 +1751,7 @@ async function localizeLibraryObjects(driver, arArgv, buildDir, programObject, s
1431
1751
  * Content-addressed caches that let repeat builds of unchanged programs skip
1432
1752
  * payload code generation/linking — the test lanes' dominant cost. Executable
1433
1753
  * lookups still run lightweight dependency and link-input metadata probes.
1434
- * Three keyspaces under the cache root:
1754
+ * Principal keyspaces under the cache root:
1435
1755
  *
1436
1756
  * bin/<key> — whole program binaries. key = sha256(resolved clang
1437
1757
  * identity/version, target + compiler/linker environment,
@@ -1461,6 +1781,14 @@ async function localizeLibraryObjects(driver, arArgv, buildDir, programObject, s
1461
1781
  * identity, TU path, and program-TU bytes.
1462
1782
  * Checksum-verified hits skip native code generation and ar.
1463
1783
  *
1784
+ * program-obj/<key> — the canonical library program object, after any LLVM
1785
+ * shard merge. Exact TU repeats reuse it directly.
1786
+ *
1787
+ * program-shard/<key> — stable LLVM dev-library buckets keyed by their IR
1788
+ * text and complete toolchain/TU identity. Local edits
1789
+ * retain every bucket whose definitions/declarations did
1790
+ * not change.
1791
+ *
1464
1792
  * obj/<set>/<f>.o — per-flavor runtime objects for cache-miss builds. The
1465
1793
  * historical single invocation recompiles every runtime
1466
1794
  * TU per program (~1.3s at -O2); with cached objects a
@@ -1521,6 +1849,31 @@ export function resolveBuildCacheRoot(env = process.env, platform = process.plat
1521
1849
  ? resolve(userHome, "Library", "Caches", "scriptc", "build")
1522
1850
  : resolve(userHome, ".cache", "scriptc", "build");
1523
1851
  }
1852
+ /** Shared persistent-cache root for compiler-level tiers. The early library
1853
+ * cache deliberately follows the native cache's activation and hard-disable
1854
+ * contract, while native compilation retains ownership of toolchain safety. */
1855
+ export function buildCacheRoot() {
1856
+ return resolveBuildCacheRoot();
1857
+ }
1858
+ /** Harden/create a compiler-level cache root using the same privacy policy as
1859
+ * the artifact caches. Failure disables only the optional caller's tier. */
1860
+ export async function prepareBuildCacheRoot(root) {
1861
+ if (root === null)
1862
+ return null;
1863
+ try {
1864
+ await ensurePrivateCacheRoot(root, process.env["SCRIPTC_CACHE_DIR"] === undefined);
1865
+ return root;
1866
+ }
1867
+ catch {
1868
+ return null;
1869
+ }
1870
+ }
1871
+ /** Register a successful compiler-level cache write with the shared bounded
1872
+ * LRU policy. */
1873
+ export async function pruneBuildCache(root) {
1874
+ if (root !== null)
1875
+ await pruneCache(root).catch(() => undefined);
1876
+ }
1524
1877
  function cacheRootDir() {
1525
1878
  return resolveBuildCacheRoot();
1526
1879
  }
@@ -1551,6 +1904,52 @@ async function ensurePrivateCacheRoot(root, hardenExisting) {
1551
1904
  }
1552
1905
  }
1553
1906
  const ccVersionMemos = new Map();
1907
+ /** Identity of the compiler implementation and configuration selected by a
1908
+ * fresh driver invocation. In particular, Darwin's stable /usr/bin/clang shim
1909
+ * exposes the currently selected Xcode clang only in its `-###` trace. The
1910
+ * normalized trace also covers output-affecting default driver configuration
1911
+ * that can change while argv[0] and PATH retain the same spelling. */
1912
+ async function effectiveCompilerEnvironmentIdentity(driver, env) {
1913
+ const compiler = driver.argv[0] ?? "clang";
1914
+ const resolvedDriver = await resolvedTool(compiler, env);
1915
+ if (resolvedDriver === null)
1916
+ return `<unresolved:${compiler}>`;
1917
+ const probeDir = await mkdtemp(join(tmpdir(), "scriptc-early-cc-probe-"));
1918
+ try {
1919
+ const source = join(probeDir, "empty.c");
1920
+ const object = join(probeDir, "empty.o");
1921
+ await writeFile(source, "int scriptc_early_driver_probe;\n");
1922
+ const trace = await execFileAsync(compiler, [
1923
+ ...driver.argv.slice(1),
1924
+ ...driver.targetArgs,
1925
+ "-###",
1926
+ "-std=c11",
1927
+ "-c",
1928
+ source,
1929
+ "-o",
1930
+ object,
1931
+ ], { cwd: probeDir, env, maxBuffer: 16 * 1024 * 1024 });
1932
+ const effectiveSpellings = [];
1933
+ for (const line of `${trace.stdout}\n${trace.stderr}`.split(/\r?\n/)) {
1934
+ const tokens = driverTraceCandidates(line);
1935
+ const cc1 = tokens.indexOf("-cc1");
1936
+ if (cc1 > 0)
1937
+ effectiveSpellings.push(tokens[cc1 - 1]);
1938
+ }
1939
+ const effective = effectiveSpellings.length === 1
1940
+ ? await resolvedTool(effectiveSpellings[0], env)
1941
+ : null;
1942
+ return createHash("sha256")
1943
+ .update("effective-compiler-environment-v1\0")
1944
+ .update(resolvedDriver.cacheIdentity).update("\0")
1945
+ .update(normalizedProbeInvocation(trace, probeDir)).update("\0")
1946
+ .update(effective?.cacheIdentity ?? `<unresolved-effective:${effectiveSpellings.join("\x1f")}>`)
1947
+ .digest("hex");
1948
+ }
1949
+ finally {
1950
+ await rm(probeDir, { recursive: true, force: true }).catch(() => undefined);
1951
+ }
1952
+ }
1554
1953
  /** Resolve the executable the OS will select for an argv[0] spelling. The
1555
1954
  * path and inode metadata join the version output below: two PATH postures
1556
1955
  * must not share cache entries merely because both drivers call themselves
@@ -2083,7 +2482,7 @@ function translationUnitDependencyFingerprint(driver, cflags, sourcePath, source
2083
2482
  * change that redirects includes cannot hide behind an unchanged old path
2084
2483
  * list. Dependency discovery must succeed on every cache-enabled invocation;
2085
2484
  * a prior path list cannot reveal a new higher-priority header. */
2086
- async function implicitToolchainFingerprintFresh(driver, environmentFingerprint) {
2485
+ async function implicitToolchainFingerprintsFresh(driver, environmentFingerprint) {
2087
2486
  let probe;
2088
2487
  const compiler = driver.argv[0] ?? "clang";
2089
2488
  const compilerIdentity = await resolvedToolIdentity(compiler);
@@ -2136,7 +2535,8 @@ async function implicitToolchainFingerprintFresh(driver, environmentFingerprint)
2136
2535
  // executable/version and dependency set.
2137
2536
  execFileAsync(compiler, [...prefix, "-std=c11", "-###", "-c", driverSource, "-o", driverOutput], { cwd: probeDir, maxBuffer: 16 * 1024 * 1024 }),
2138
2537
  ]);
2139
- const toolSpellings = [linker.stdout.trim(), assembler.stdout.trim()].filter((value, index, all) => value !== "" && all.indexOf(value) === index);
2538
+ const assemblerSpelling = assembler.stdout.trim();
2539
+ const toolSpellings = [linker.stdout.trim(), assemblerSpelling].filter((value, index, all) => value !== "" && all.indexOf(value) === index);
2140
2540
  const sourceSet = new Set(sources);
2141
2541
  const dependencyPaths = [
2142
2542
  ...new Set(dependencyResults
@@ -2157,44 +2557,56 @@ async function implicitToolchainFingerprintFresh(driver, environmentFingerprint)
2157
2557
  path: resolved?.canonicalPath ?? null,
2158
2558
  };
2159
2559
  })),
2560
+ compileToolSpellings: assemblerSpelling === "" ? [] : [assemblerSpelling],
2160
2561
  };
2161
2562
  }
2162
2563
  finally {
2163
2564
  await rm(probeDir, { recursive: true, force: true }).catch(() => undefined);
2164
2565
  }
2165
2566
  }
2166
- const hash = createHash("sha256")
2167
- .update("implicit-toolchain-v2\0")
2168
- .update(environmentFingerprint)
2169
- .update("\0")
2170
- .update(probe.compilerIdentity)
2171
- .update("\0")
2172
- .update(probe.compilerInvocation)
2173
- .update("\0")
2174
- .update(driver.argv.join("\x1f"))
2175
- .update("\0")
2176
- .update(driver.targetArgs.join("\x1f"))
2177
- .update("\0")
2178
- .update(probe.dependencies.join("\x1f"))
2179
- .update("\0")
2180
- .update(probe.dependencyFingerprint)
2181
- .update("\0");
2182
- for (const tool of probe.tools) {
2183
- const currentIdentity = await resolvedToolIdentity(tool.spelling);
2184
- hash
2185
- .update(tool.spelling)
2567
+ const tools = await Promise.all(probe.tools.map(async (tool) => ({
2568
+ ...tool,
2569
+ currentIdentity: await resolvedToolIdentity(tool.spelling),
2570
+ })));
2571
+ const fingerprint = (domain, selectedTools) => {
2572
+ const hash = createHash("sha256")
2573
+ .update(domain)
2574
+ .update(environmentFingerprint)
2575
+ .update("\0")
2576
+ .update(probe.compilerIdentity)
2577
+ .update("\0")
2578
+ .update(probe.compilerInvocation)
2186
2579
  .update("\0")
2187
- .update(currentIdentity ?? tool.identity ?? "<unresolved>")
2580
+ .update(driver.argv.join("\x1f"))
2581
+ .update("\0")
2582
+ .update(driver.targetArgs.join("\x1f"))
2583
+ .update("\0")
2584
+ .update(probe.dependencies.join("\x1f"))
2585
+ .update("\0")
2586
+ .update(probe.dependencyFingerprint)
2188
2587
  .update("\0");
2189
- }
2190
- return rememberFingerprintDependencies(hash.digest("hex"), [
2191
- ...probe.dependencies,
2192
- ...probe.invocationPaths,
2193
- ...probe.tools.flatMap((tool) => tool.path === null ? [] : [tool.path]),
2194
- ], probe.dependencies, probe.dependencyFingerprint);
2588
+ for (const tool of selectedTools) {
2589
+ hash
2590
+ .update(tool.spelling)
2591
+ .update("\0")
2592
+ .update(tool.currentIdentity ?? tool.identity ?? "<unresolved>")
2593
+ .update("\0");
2594
+ }
2595
+ return rememberFingerprintDependencies(hash.digest("hex"), [
2596
+ ...probe.dependencies,
2597
+ ...probe.invocationPaths,
2598
+ ...selectedTools.flatMap((tool) => tool.path === null ? [] : [tool.path]),
2599
+ ], probe.dependencies, probe.dependencyFingerprint);
2600
+ };
2601
+ const compileSpellingSet = new Set(probe.compileToolSpellings);
2602
+ return {
2603
+ // Preserve the established complete fingerprint domain and byte stream.
2604
+ complete: fingerprint("implicit-toolchain-v2\0", tools),
2605
+ compile: fingerprint("implicit-compile-toolchain-v1\0", tools.filter((tool) => compileSpellingSet.has(tool.spelling))),
2606
+ };
2195
2607
  }
2196
2608
  const stableImplicitToolchainMemos = new Map();
2197
- function implicitToolchainFingerprint(driver, environmentFingerprint) {
2609
+ function implicitToolchainFingerprints(driver, environmentFingerprint) {
2198
2610
  const key = [
2199
2611
  environmentFingerprint,
2200
2612
  driver.argv.join("\x1f"),
@@ -2202,7 +2614,10 @@ function implicitToolchainFingerprint(driver, environmentFingerprint) {
2202
2614
  driver.targetArgs.join("\x1f"),
2203
2615
  runtimeSrcDir(),
2204
2616
  ].join("\0");
2205
- return stableTestMemo(stableImplicitToolchainMemos, key, () => implicitToolchainFingerprintFresh(driver, environmentFingerprint));
2617
+ return stableTestMemo(stableImplicitToolchainMemos, key, () => implicitToolchainFingerprintsFresh(driver, environmentFingerprint));
2618
+ }
2619
+ function implicitToolchainFingerprint(driver, environmentFingerprint) {
2620
+ return implicitToolchainFingerprints(driver, environmentFingerprint).then((fingerprints) => fingerprints.complete);
2206
2621
  }
2207
2622
  function linkTraceCandidate(line) {
2208
2623
  const trimmed = line.trim().replace(/^(?:LOAD|load)\s+/, "");
@@ -2428,19 +2843,20 @@ function ccacheAvailable() {
2428
2843
  ccacheMemo ??= execFileAsync("ccache", ["--version"]).then(() => true, () => false);
2429
2844
  return ccacheMemo;
2430
2845
  }
2431
- /** Content hash of every .c/.h in the runtime src tree, the Ryū sources
2432
- * textually included by scr_number.c, and the separately-built vendor pins —
2433
- * everything a binary links that the emitted C bytes don't already cover
2434
- * (npm-embedded C rides inside the emitted C; the engine and standalone
2435
- * vendor archives are pinned by their version constants). The small source
2436
- * tree is hashed on every identity calculation: recursive enumeration is
2437
- * required because a newly added nested header can begin shadowing a system
2438
- * include without changing any previously selected dependency, while content
2439
- * hashing catches same-size edits whose timestamp was preserved by a copy/sync
2440
- * tool. */
2846
+ /** Content hash of every owned native source/header plus this backend's build
2847
+ * recipe implementation. It keys complete artifacts, runtime objects, and the
2848
+ * separately built vendor prerequisites, so two installed scriptc versions or
2849
+ * worktrees can share a user cache without exchanging outputs produced from
2850
+ * different vendored bytes or compile/archive recipes. Recursive enumeration
2851
+ * also catches a newly added nested header that begins shadowing a system
2852
+ * include, while content hashing catches same-size timestamp-preserving edits. */
2441
2853
  async function runtimeFingerprintFresh(rtDir) {
2442
2854
  const groups = await runtimeFingerprintInputGroups(rtDir);
2443
- const h = createHash("sha256").update(QJS_COMMIT).update(MBEDTLS_VERSION).update(ZLIB_VERSION);
2855
+ const h = createHash("sha256")
2856
+ .update("native-owned-inputs-v2\0")
2857
+ .update(QJS_COMMIT).update(MBEDTLS_VERSION).update(ZLIB_VERSION)
2858
+ .update("\0backend-recipe\0")
2859
+ .update(await readFile(CC_IMPLEMENTATION_PATH));
2444
2860
  for (const group of groups) {
2445
2861
  for (const n of group.names) {
2446
2862
  h.update(group.label).update("/").update(n).update("\0").update(await readFile(join(group.dir, n))).update("\0");
@@ -2452,6 +2868,10 @@ async function runtimeFingerprintInputGroups(rtDir) {
2452
2868
  return Promise.all([
2453
2869
  { label: "runtime", dir: rtDir },
2454
2870
  { label: "ryu", dir: join(rtDir, "..", "vendor", "ryu") },
2871
+ { label: "quickjs-ng", dir: join(rtDir, "..", "vendor", "quickjs-ng") },
2872
+ { label: "mbedtls", dir: join(rtDir, "..", "vendor", "mbedtls") },
2873
+ { label: "zlib", dir: join(rtDir, "..", "vendor", "zlib") },
2874
+ { label: "curl", dir: join(rtDir, "..", "vendor", "curl") },
2455
2875
  ].map(async (group) => {
2456
2876
  const names = (await nativeSourceFiles(group.dir, true))
2457
2877
  .map((path) => relative(group.dir, path))
@@ -2460,7 +2880,10 @@ async function runtimeFingerprintInputGroups(rtDir) {
2460
2880
  }));
2461
2881
  }
2462
2882
  async function runtimeFingerprintInputPaths(rtDir) {
2463
- return (await runtimeFingerprintInputGroups(rtDir)).flatMap((group) => group.names.map((name) => join(group.dir, name)));
2883
+ return [
2884
+ ...(await runtimeFingerprintInputGroups(rtDir)).flatMap((group) => group.names.map((name) => join(group.dir, name))),
2885
+ CC_IMPLEMENTATION_PATH,
2886
+ ];
2464
2887
  }
2465
2888
  const stableRuntimeFingerprintMemos = new Map();
2466
2889
  export function runtimeFingerprint(rtDir) {
@@ -2646,7 +3069,22 @@ async function snapshotLocalArtifactDependencies(dependencyPaths, recursiveDirec
2646
3069
  const recursive = new Set(recursiveDirectories.map((path) => resolve(path)));
2647
3070
  return Promise.all([...new Set(dependencyPaths)].sort().map((path) => snapshotLocalArtifactDependency(path, recursive.has(resolve(path)) ? recursiveExclusions : null)));
2648
3071
  }
2649
- async function localArtifactDependenciesStillMatch(dependencies) {
3072
+ export async function nativeArtifactDependenciesStillMatch(dependencies) {
3073
+ if (!dependencies.every((dependency) => dependency !== null && typeof dependency === "object" &&
3074
+ typeof dependency.path === "string" &&
3075
+ (dependency.kind === "file" || dependency.kind === "directory" || dependency.kind === "symlink") &&
3076
+ typeof dependency.dev === "number" && typeof dependency.ino === "number" &&
3077
+ typeof dependency.size === "number" && typeof dependency.mtimeMs === "number" &&
3078
+ typeof dependency.ctimeMs === "number" &&
3079
+ (dependency.treeDigest === undefined || typeof dependency.treeDigest === "string") &&
3080
+ (dependency.treeExclusions === undefined || (Array.isArray(dependency.treeExclusions) &&
3081
+ dependency.treeExclusions.every((path) => typeof path === "string"))) &&
3082
+ (dependency.kind !== "symlink" || (typeof dependency.targetPath === "string" &&
3083
+ (dependency.targetKind === "file" || dependency.targetKind === "directory") &&
3084
+ typeof dependency.targetDev === "number" && typeof dependency.targetIno === "number" &&
3085
+ typeof dependency.targetSize === "number" && typeof dependency.targetMtimeMs === "number" &&
3086
+ typeof dependency.targetCtimeMs === "number"))))
3087
+ return false;
2650
3088
  return (await Promise.all(dependencies.map(async (dependency) => {
2651
3089
  const current = await snapshotLocalArtifactDependency(dependency.path, dependency.treeDigest === undefined ? null : dependency.treeExclusions ?? []).catch(() => null);
2652
3090
  return current !== null && JSON.stringify(current) === JSON.stringify(dependency);
@@ -2676,7 +3114,7 @@ async function readNativeMetadataStamp(root, key) {
2676
3114
  values: stamp.values,
2677
3115
  dependencies: stamp.dependencies,
2678
3116
  }) !== stamp.integrity ||
2679
- !(await localArtifactDependenciesStillMatch(stamp.dependencies))) {
3117
+ !(await nativeArtifactDependenciesStillMatch(stamp.dependencies))) {
2680
3118
  return null;
2681
3119
  }
2682
3120
  return stamp;
@@ -2722,11 +3160,14 @@ function nativeMetadataKey(kind, parts) {
2722
3160
  * The generated TU bytes, every scriptc runtime source, the selected direct
2723
3161
  * compiler inode, target/options/environment, and the output path all join the
2724
3162
  * key. A digest rejects a modified/truncated output before the no-op hit. */
2725
- function localArtifactIdentity(opts, driver, environmentFingerprint, compilerIdentity, runtimeHash, programBytes) {
3163
+ function localArtifactIdentity(opts, driver, environmentFingerprint, compilerIdentity, runtimeHash, programBytes, programShardMerge) {
2726
3164
  const normalizedOptions = Object.fromEntries(Object.entries(opts)
2727
- .filter(([, value]) => value !== undefined)
3165
+ // Shard source can be tens of megabytes. Hash it incrementally below
3166
+ // instead of materializing a second giant JSON string solely for this
3167
+ // output-local fast-path identity.
3168
+ .filter(([key, value]) => value !== undefined && key !== "programShards" && key !== "programPublicSymbols")
2728
3169
  .sort(([a], [b]) => a.localeCompare(b)));
2729
- return createHash("sha256")
3170
+ const hash = createHash("sha256")
2730
3171
  .update("local-artifact-v1\0")
2731
3172
  .update(cacheTargetIdentity(driver)).update("\0")
2732
3173
  .update(environmentFingerprint).update("\0")
@@ -2734,7 +3175,11 @@ function localArtifactIdentity(opts, driver, environmentFingerprint, compilerIde
2734
3175
  .update(runtimeHash).update("\0")
2735
3176
  .update(driver.argv.join("\x1f")).update("\0")
2736
3177
  .update(driver.targetArgs.join("\x1f")).update("\0")
2737
- .update(driver.linkArgs.join("\x1f")).update("\0")
3178
+ .update(driver.linkArgs.join("\x1f")).update("\0");
3179
+ if (programShardMerge !== null) {
3180
+ updateProgramShardCacheIdentity(hash, opts.programShards, opts.programPublicSymbols, programShardMerge);
3181
+ }
3182
+ return hash
2738
3183
  .update(process.env["SCRIPTC_FETCH_CURL"] === "1" ? "fetch-curl" : "fetch-native").update("\0")
2739
3184
  .update(JSON.stringify(normalizedOptions)).update("\0")
2740
3185
  .update(resolve(opts.cPath)).update("\0")
@@ -2789,16 +3234,16 @@ async function localArtifactHit(stampPath, outPath, key) {
2789
3234
  typeof dependency.targetSize !== "number" ||
2790
3235
  typeof dependency.targetMtimeMs !== "number" ||
2791
3236
  typeof dependency.targetCtimeMs !== "number")) ||
2792
- !(await localArtifactDependenciesStillMatch(stamp.dependencies)) ||
3237
+ !(await nativeArtifactDependenciesStillMatch(stamp.dependencies)) ||
2793
3238
  await fileDigest(outPath) !== stamp.digest) {
2794
- return false;
3239
+ return null;
2795
3240
  }
2796
3241
  const now = new Date();
2797
3242
  await utimes(stampPath, now, now).catch(() => undefined);
2798
- return true;
3243
+ return stamp;
2799
3244
  }
2800
3245
  catch {
2801
- return false;
3246
+ return null;
2802
3247
  }
2803
3248
  }
2804
3249
  async function publishLocalArtifactStamp(stampPath, outPath, key, dependencyPaths, recursiveDirectories = [], recursiveExclusions = []) {
@@ -2818,6 +3263,7 @@ async function publishLocalArtifactStamp(stampPath, outPath, key, dependencyPath
2818
3263
  };
2819
3264
  await writeFile(tmp, `${JSON.stringify(stamp)}\n`, { mode: 0o600 });
2820
3265
  await rename(tmp, stampPath);
3266
+ return stamp;
2821
3267
  }
2822
3268
  finally {
2823
3269
  await rm(tmp, { force: true }).catch(() => undefined);
@@ -2828,7 +3274,7 @@ async function publishLocalArtifactStamp(stampPath, outPath, key, dependencyPath
2828
3274
  * per-file atomic renames make every winner equivalent. Publication is held
2829
3275
  * until verifyInputs confirms that the source/header fingerprint used by the
2830
3276
  * key still describes the bytes clang just read. */
2831
- async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, verifyInputs) {
3277
+ async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, verifyInputs, protectedPaths) {
2832
3278
  const setKey = createHash("sha256")
2833
3279
  .update(keyPrefix)
2834
3280
  .update(cflags.join("\x1f"))
@@ -2836,6 +3282,13 @@ async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, ve
2836
3282
  .slice(0, 24);
2837
3283
  const objDir = join(root, "obj", setKey);
2838
3284
  const objOf = (src) => join(objDir, `${basename(src, ".c")}.o`);
3285
+ if (protectedPaths !== undefined) {
3286
+ for (const source of sources) {
3287
+ const object = objOf(source);
3288
+ protectedPaths.add(object);
3289
+ protectedPaths.add(cacheDigestPath(object));
3290
+ }
3291
+ }
2839
3292
  const present = await Promise.all(sources.map((s) => validCachedFile(objOf(s))));
2840
3293
  const missing = sources.filter((_, i) => !present[i]);
2841
3294
  if (missing.length > 0) {
@@ -2846,7 +3299,7 @@ async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, ve
2846
3299
  ccArgv[0] === "clang" &&
2847
3300
  (await ccacheAvailable());
2848
3301
  await mkdir(objDir, { recursive: true });
2849
- const tmpDir = await mkdtemp(join(root, "obj", "build-"));
3302
+ const tmpDir = await mkdtemp(join(tmpdir(), "scriptc-cache-obj-"));
2850
3303
  try {
2851
3304
  const compiled = new Map();
2852
3305
  // Modest parallelism: a flavor's objects build once, but several cold
@@ -2872,7 +3325,7 @@ async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, ve
2872
3325
  },
2873
3326
  }
2874
3327
  : undefined);
2875
- compiled.set(src, { object: tmpObj, digest: await fileDigest(tmpObj) });
3328
+ compiled.set(src, tmpObj);
2876
3329
  }));
2877
3330
  }
2878
3331
  // The fingerprint was computed before these subprocesses started. Do
@@ -2882,12 +3335,7 @@ async function ensureRuntimeObjects(root, ccArgv, cflags, sources, keyPrefix, ve
2882
3335
  throw new CacheInputsChangedError();
2883
3336
  for (const [src, built] of compiled) {
2884
3337
  const destination = objOf(src);
2885
- const tmpDigest = `${built.object}.sha256`;
2886
- await writeFile(tmpDigest, `${built.digest}\n`);
2887
- // Publish data first and its verifier second. A concurrent reader in
2888
- // the tiny gap sees a missing/mismatched digest and falls back safely.
2889
- await rename(built.object, destination);
2890
- await rename(tmpDigest, cacheDigestPath(destination));
3338
+ await publishCachedFile(built, destination);
2891
3339
  }
2892
3340
  }
2893
3341
  finally {
@@ -2934,7 +3382,7 @@ export async function stageRuntimeObjects(objects, stageDir) {
2934
3382
  * mtimes. Active links use private staged names/hard links, so cache names can
2935
3383
  * be unlinked safely. */
2936
3384
  const rootWriteCounts = new Map();
2937
- async function pruneCache(root) {
3385
+ async function pruneCache(root, protectedPaths) {
2938
3386
  const configuredCap = process.env["SCRIPTC_CACHE_MAX_MB"];
2939
3387
  const writes = (rootWriteCounts.get(root) ?? 0) + 1;
2940
3388
  rootWriteCounts.set(root, writes);
@@ -2951,6 +3399,14 @@ async function pruneCache(root) {
2951
3399
  if (ent.isDirectory())
2952
3400
  await walk(p);
2953
3401
  else if (ent.isFile()) {
3402
+ // Atomic publishers use private names until their data/digest or
3403
+ // metadata stamp is complete. They are active writes, not LRU entries.
3404
+ if (ent.name.startsWith(".scriptc-") ||
3405
+ ent.name.startsWith(".tmp-") ||
3406
+ ent.name.includes(".tmp-"))
3407
+ continue;
3408
+ if (protectedPaths?.has(p))
3409
+ continue;
2954
3410
  const s = await stat(p).catch(() => null);
2955
3411
  if (s !== null)
2956
3412
  files.push({ path: p, size: s.size, mtimeMs: s.mtimeMs });
@@ -2988,9 +3444,10 @@ async function pruneCache(root) {
2988
3444
  * unchanged programs skip payload code generation/linking via the binary
2989
3445
  * cache after lightweight metadata probes, and misses link the program's own
2990
3446
  * TU against cached per-flavor runtime objects. */
2991
- export async function compileC(opts) {
3447
+ async function compileCInternal(opts, cacheWarmOnly, cacheWarmPaths) {
2992
3448
  const rtDir = runtimeSrcDir();
2993
3449
  const sanitize = opts.sanitize ?? false;
3450
+ const optimization = opts.optimization ?? "release";
2994
3451
  const dynamic = opts.dynamic ?? false;
2995
3452
  const regex = opts.regex ?? false;
2996
3453
  // fetch's implementation switch: the default is the NATIVE bridge
@@ -3014,6 +3471,24 @@ export async function compileC(opts) {
3014
3471
  const tls = (opts.tls ?? false) || nativeFetch || netIsland;
3015
3472
  const tlsCa = (opts.tlsCa ?? false) || tls;
3016
3473
  const driver = resolveCc();
3474
+ const shardNames = new Set();
3475
+ const programShardsValid = opts.programShards?.every((shard) => {
3476
+ if (basename(shard.name) !== shard.name || !shard.name.endsWith(".ll") ||
3477
+ shardNames.has(shard.name))
3478
+ return false;
3479
+ shardNames.add(shard.name);
3480
+ return true;
3481
+ }) === true;
3482
+ const programShardsRequested = optimization === "dev" && !sanitize && opts.cPath.endsWith(".ll") &&
3483
+ programShardsValid && opts.programShards !== undefined && opts.programShards.length > 1 &&
3484
+ opts.programPublicSymbols !== undefined
3485
+ ? opts.programShards
3486
+ : null;
3487
+ const programShardMergeIdentity = programShardsRequested === null
3488
+ ? null
3489
+ : await resolveProgramShardMergeIdentity(driver);
3490
+ const programShards = programShardMergeIdentity === null ? null : programShardsRequested;
3491
+ const programPublicSymbols = programShards === null ? undefined : opts.programPublicSymbols;
3017
3492
  // Mobile triples produce library archives, never standalone executables:
3018
3493
  // the executable-lane runtime (event loop, sockets, child processes) is
3019
3494
  // not verified on those device classes. compile() reports the SC3002
@@ -3034,7 +3509,7 @@ export async function compileC(opts) {
3034
3509
  // See the resolveCc block: these inputs are built on and for the HOST
3035
3510
  // (vendored archives, system libs). Regex, zlib, and the engine archive
3036
3511
  // are NOT here: their vendored sources are plain C that ensureLreObjects
3037
- // / ensureZlibObjects / buildEngineArchiveCross compile per target with
3512
+ // / ensureZlibObjects / buildEngineArchiveDirect compile per target with
3038
3513
  // the driver itself — win32 included (the Windows lane runs the
3039
3514
  // @dynamic corpus and the zlib program against the box's Node). The
3040
3515
  // NATIVE fetch rides the socket units and cross-compiles with them —
@@ -3075,11 +3550,19 @@ export async function compileC(opts) {
3075
3550
  let root = opts.cacheIdentity === undefined || !persistentDriverCache
3076
3551
  ? null
3077
3552
  : configuredCacheRoot;
3553
+ if (cacheWarmOnly && root === null) {
3554
+ throw new Error("native cache warming requires a persistently cacheable compiler environment");
3555
+ }
3078
3556
  if (root !== null) {
3079
3557
  try {
3080
3558
  await ensurePrivateCacheRoot(root, process.env["SCRIPTC_CACHE_DIR"] === undefined);
3081
3559
  }
3082
- catch {
3560
+ catch (error) {
3561
+ if (cacheWarmOnly) {
3562
+ throw new Error("native cache warming could not prepare the persistent cache root", {
3563
+ cause: error,
3564
+ });
3565
+ }
3083
3566
  root = null;
3084
3567
  }
3085
3568
  }
@@ -3089,7 +3572,8 @@ export async function compileC(opts) {
3089
3572
  // broader cross-output CAS performs its compiler/SDK/linker rediscovery.
3090
3573
  // FFI/native-input builds and the public arbitrary-C cache API stay on the
3091
3574
  // strict path because their dependency graphs are caller-owned.
3092
- if (root !== null &&
3575
+ if (!cacheWarmOnly &&
3576
+ root !== null &&
3093
3577
  cachePolicy.completeArtifacts &&
3094
3578
  opts.cacheIdentity === "scriptc-generated-v1" &&
3095
3579
  (opts.linkInputs?.length ?? 0) === 0 &&
@@ -3103,7 +3587,7 @@ export async function compileC(opts) {
3103
3587
  ]);
3104
3588
  if (compiler !== null) {
3105
3589
  const effectiveCompiler = directCompilerSelections.get(compilerDriverProbeKey(driver, toolchainEnv)) ?? compiler;
3106
- const key = localArtifactIdentity(opts, driver, toolchainEnv, `${compiler.cacheIdentity}\0${effectiveCompiler.cacheIdentity}`, runtimeHash, programBytes);
3590
+ const key = localArtifactIdentity(opts, driver, toolchainEnv, `${compiler.cacheIdentity}\0${effectiveCompiler.cacheIdentity}`, runtimeHash, programBytes, programShardMergeIdentity);
3107
3591
  const stampPath = localArtifactStampPath(root, opts.outPath);
3108
3592
  localArtifact = {
3109
3593
  stampPath,
@@ -3112,8 +3596,11 @@ export async function compileC(opts) {
3112
3596
  programBytes,
3113
3597
  compilerPath: effectiveCompiler.canonicalPath,
3114
3598
  };
3115
- if (await localArtifactHit(stampPath, opts.outPath, key))
3599
+ const hit = await localArtifactHit(stampPath, opts.outPath, key);
3600
+ if (hit !== null) {
3601
+ await opts.onArtifactReady?.({ dependencies: hit.dependencies }).catch(() => undefined);
3116
3602
  return;
3603
+ }
3117
3604
  }
3118
3605
  }
3119
3606
  catch {
@@ -3123,6 +3610,7 @@ export async function compileC(opts) {
3123
3610
  }
3124
3611
  }
3125
3612
  let implicitToolchain = null;
3613
+ let implicitCompileToolchain = null;
3126
3614
  let toolchainMetadataStamp = null;
3127
3615
  const metadataCompiler = root === null
3128
3616
  ? null
@@ -3148,8 +3636,16 @@ export async function compileC(opts) {
3148
3636
  root === null || toolchainMetadataKey === null
3149
3637
  ? null
3150
3638
  : await readNativeMetadataStamp(root, toolchainMetadataKey);
3151
- implicitToolchain = toolchainMetadataStamp?.values["implicitToolchain"] ??
3152
- await implicitToolchainFingerprint(driver, toolchainEnv);
3639
+ implicitToolchain = toolchainMetadataStamp?.values["implicitToolchain"] ?? null;
3640
+ implicitCompileToolchain =
3641
+ toolchainMetadataStamp?.values["implicitCompileToolchain"] ?? null;
3642
+ if (implicitToolchain === null ||
3643
+ (programShards !== null && implicitCompileToolchain === null)) {
3644
+ const fingerprints = await implicitToolchainFingerprints(driver, toolchainEnv);
3645
+ implicitToolchain = fingerprints.complete;
3646
+ implicitCompileToolchain = fingerprints.compile;
3647
+ toolchainMetadataStamp = null;
3648
+ }
3153
3649
  let compilerVersion = toolchainMetadataStamp?.values["compilerVersion"];
3154
3650
  if (compilerVersion === undefined) {
3155
3651
  compilerVersion = await ccVersionOnce(driver.argv, toolchainEnv, true);
@@ -3158,73 +3654,122 @@ export async function compileC(opts) {
3158
3654
  root !== null &&
3159
3655
  toolchainMetadataKey !== null) {
3160
3656
  if (metadataCompiler !== null && metadataEffectiveCompiler !== null) {
3161
- toolchainMetadataStamp = await publishNativeMetadataStamp(root, toolchainMetadataKey, { implicitToolchain, compilerVersion }, [
3657
+ toolchainMetadataStamp = await publishNativeMetadataStamp(root, toolchainMetadataKey, {
3658
+ implicitToolchain,
3659
+ ...(implicitCompileToolchain === null ? {} : { implicitCompileToolchain }),
3660
+ compilerVersion,
3661
+ }, [
3162
3662
  metadataCompiler.canonicalPath,
3163
3663
  metadataEffectiveCompiler.canonicalPath,
3164
3664
  ...fingerprintDependencyPaths(implicitToolchain),
3165
- ], [implicitToolchain]);
3665
+ ...(implicitCompileToolchain === null
3666
+ ? []
3667
+ : fingerprintDependencyPaths(implicitCompileToolchain)),
3668
+ ], [
3669
+ implicitToolchain,
3670
+ ...(implicitCompileToolchain === null ? [] : [implicitCompileToolchain]),
3671
+ ]);
3166
3672
  }
3167
3673
  }
3168
3674
  }
3169
- catch {
3675
+ catch (error) {
3170
3676
  // Cache discovery is best-effort. In particular, a compiler wrapper can
3171
3677
  // compile successfully without implementing the metadata probes.
3678
+ if (cacheWarmOnly) {
3679
+ throw new Error("native cache warming could not validate the compiler toolchain", {
3680
+ cause: error,
3681
+ });
3682
+ }
3172
3683
  root = null;
3173
3684
  }
3174
3685
  }
3175
3686
  const vendorBuildIdentity = await currentVendorCacheBuildIdentity(driver, `${toolchainEnv}\0${implicitToolchain ?? "<uncached>"}`);
3176
3687
  // Mutable include/SDK/config inputs can change behind a stable environment
3177
3688
  // spelling. The complete/runtime cache is disabled below; vendor objects
3178
- // must follow the same rule instead of silently surviving in package-local
3179
- // .cache. A private root gives this invocation the usual vendor build recipe
3689
+ // must follow the same rule instead of silently surviving in the user cache.
3690
+ // A private root gives this invocation the usual vendor build recipe
3180
3691
  // without publishing or reusing those prerequisites.
3181
- const transientVendorRoot = persistentDriverCache && implicitToolchain !== null
3692
+ const transientVendorRoot = root !== null && implicitToolchain !== null
3182
3693
  ? null
3183
3694
  : join(tmpdir(), `scriptc-vendor-${process.pid}-${Math.random().toString(36).slice(2)}`);
3184
- const vendorCacheRoot = transientVendorRoot ?? vendorBuildCacheRoot();
3695
+ const vendorCacheRoot = transientVendorRoot ?? vendorBuildCacheRoot(root ?? undefined);
3185
3696
  // Every vendor output path is deterministic from pins, flags, driver, and
3186
3697
  // target. Build the command/key from those paths now, but do not materialize
3187
3698
  // them until a complete-binary lookup has missed.
3188
- const engineArchive = dynamic
3699
+ let engineArchive = dynamic
3189
3700
  ? engineArchivePath(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
3190
3701
  : null;
3191
- const tlsArchive = tls
3702
+ let tlsArchive = tls
3192
3703
  ? tlsArchivePath(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
3193
3704
  : null;
3194
3705
  // --dynamic + regex shares the archive's libregexp (its host hooks and
3195
3706
  // ours would collide; see scr_regex.c) — the standalone objects are for
3196
3707
  // static builds only.
3197
- const lreObjects = regex && !dynamic
3708
+ let lreObjects = regex && !dynamic
3198
3709
  ? lreObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
3199
3710
  : [];
3200
3711
  // Vendored zlib is the CROSS story only — host builds keep the exact
3201
3712
  // historical `-lz` system link (see CcOptions.zlib). The native fetch's
3202
3713
  // gzip decoder rides the same objects/link.
3203
- const zlibObjects = ((opts.zlib ?? false) || nativeFetch) && driver.target !== null
3714
+ let zlibObjects = ((opts.zlib ?? false) || nativeFetch) && driver.target !== null
3204
3715
  ? zlibObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
3205
3716
  : [];
3206
3717
  // The libcurl import stub is likewise CROSS-only — host builds keep the
3207
3718
  // exact historical system `-lcurl` link (see CcOptions.fetch). Curl
3208
3719
  // reference builds only.
3209
- const curlStubDir = curlFetch && driver.target !== null
3720
+ let curlStubDir = curlFetch && driver.target !== null
3210
3721
  ? curlStubDirPath(driver, vendorBuildIdentity, vendorCacheRoot)
3211
3722
  : null;
3212
- const materializeVendorPrerequisites = async () => {
3723
+ const materializeVendorPrerequisites = async (stageRoot, materializeCacheRoot = vendorCacheRoot) => {
3213
3724
  // Preserve the historical order to avoid multiplying first-build resource
3214
3725
  // pressure when several large vendor sets are cold simultaneously.
3215
3726
  if (dynamic) {
3216
- await ensureEngineArchive(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
3727
+ const cachedArchive = engineArchivePath(sanitize, driver, vendorBuildIdentity, materializeCacheRoot);
3728
+ protectCachedArtifact(cacheWarmPaths, cachedArchive);
3729
+ const materialize = async () => [
3730
+ await ensureEngineArchive(sanitize, driver, vendorBuildIdentity, materializeCacheRoot),
3731
+ ];
3732
+ const paths = stageRoot === undefined
3733
+ ? await materialize()
3734
+ : await stageVendorInputs(materialize, join(stageRoot, "engine"));
3735
+ engineArchive = paths[0];
3736
+ }
3737
+ if (tls) {
3738
+ const cachedArchive = tlsArchivePath(sanitize, driver, vendorBuildIdentity, materializeCacheRoot);
3739
+ protectCachedArtifact(cacheWarmPaths, cachedArchive);
3740
+ const materialize = async () => [
3741
+ await ensureTlsArchive(sanitize, driver, vendorBuildIdentity, materializeCacheRoot),
3742
+ ];
3743
+ const paths = stageRoot === undefined
3744
+ ? await materialize()
3745
+ : await stageVendorInputs(materialize, join(stageRoot, "tls"));
3746
+ tlsArchive = paths[0];
3217
3747
  }
3218
- if (tls)
3219
- await ensureTlsArchive(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
3220
3748
  if (regex && !dynamic) {
3221
- await ensureLreObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
3749
+ for (const object of lreObjectPaths(sanitize, driver, vendorBuildIdentity, materializeCacheRoot))
3750
+ protectCachedArtifact(cacheWarmPaths, object);
3751
+ const materialize = async () => await ensureLreObjects(sanitize, driver, vendorBuildIdentity, materializeCacheRoot);
3752
+ lreObjects = stageRoot === undefined
3753
+ ? await materialize()
3754
+ : await stageVendorInputs(materialize, join(stageRoot, "lre"));
3222
3755
  }
3223
3756
  if (zlibObjects.length > 0) {
3224
- await ensureZlibObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
3757
+ for (const object of zlibObjectPaths(sanitize, driver, vendorBuildIdentity, materializeCacheRoot))
3758
+ protectCachedArtifact(cacheWarmPaths, object);
3759
+ const materialize = async () => await ensureZlibObjects(sanitize, driver, vendorBuildIdentity, materializeCacheRoot);
3760
+ zlibObjects = stageRoot === undefined
3761
+ ? await materialize()
3762
+ : await stageVendorInputs(materialize, join(stageRoot, "zlib"));
3225
3763
  }
3226
3764
  if (curlStubDir !== null) {
3227
- await ensureCurlStub(driver, vendorBuildIdentity, vendorCacheRoot);
3765
+ protectCachedArtifact(cacheWarmPaths, join(curlStubDirPath(driver, vendorBuildIdentity, materializeCacheRoot), "libcurl.so"));
3766
+ const materialize = async () => [
3767
+ join(await ensureCurlStub(driver, vendorBuildIdentity, materializeCacheRoot), "libcurl.so"),
3768
+ ];
3769
+ const paths = stageRoot === undefined
3770
+ ? await materialize()
3771
+ : await stageVendorInputs(materialize, join(stageRoot, "curl"));
3772
+ curlStubDir = dirname(paths[0]);
3228
3773
  }
3229
3774
  };
3230
3775
  // rt() maps each runtime source's path on the command line: identity for
@@ -3235,7 +3780,7 @@ export async function compileC(opts) {
3235
3780
  ...threadArgs,
3236
3781
  ...(sanitize
3237
3782
  ? ["-O1", "-fsanitize=address", "-DSCR_RC_AUDIT"]
3238
- : ["-O2"]),
3783
+ : [optimization === "dev" ? "-O0" : "-O2"]),
3239
3784
  ...(opts.textDecoderLegacy ? ["-DSCR_TEXT_DECODER_LEGACY"] : []),
3240
3785
  "-fno-math-errno",
3241
3786
  // The emitted object model is deliberately type-punned C: a hierarchy
@@ -3429,7 +3974,7 @@ export async function compileC(opts) {
3429
3974
  ...threadArgs,
3430
3975
  ...(sanitize
3431
3976
  ? ["-O1", "-fsanitize=address", "-DSCR_RC_AUDIT"]
3432
- : ["-O2"]),
3977
+ : [optimization === "dev" ? "-O0" : "-O2"]),
3433
3978
  ...(opts.textDecoderLegacy ? ["-DSCR_TEXT_DECODER_LEGACY"] : []),
3434
3979
  "-fno-math-errno",
3435
3980
  "-fno-strict-aliasing", // the emitted object model type-puns — see buildArgs
@@ -3462,6 +4007,16 @@ export async function compileC(opts) {
3462
4007
  `${guidance}\n\n${stderr}`);
3463
4008
  }
3464
4009
  };
4010
+ const runUncachedBuild = async () => {
4011
+ const privateVendorRoot = transientVendorRoot ?? join(tmpdir(), `scriptc-vendor-fallback-${process.pid}-${Math.random().toString(36).slice(2)}`);
4012
+ try {
4013
+ await materializeVendorPrerequisites(undefined, privateVendorRoot);
4014
+ await runClang(buildArgs((p) => p));
4015
+ }
4016
+ finally {
4017
+ await rm(privateVendorRoot, { recursive: true, force: true }).catch(() => undefined);
4018
+ }
4019
+ };
3465
4020
  let runtimeCompilerInvocation = null;
3466
4021
  let programCompilerInvocation = null;
3467
4022
  let payloadMetadata = null;
@@ -3522,23 +4077,20 @@ export async function compileC(opts) {
3522
4077
  }
3523
4078
  }
3524
4079
  }
3525
- catch {
4080
+ catch (error) {
3526
4081
  // Preserve the uncached build for wrappers that compile successfully but
3527
4082
  // cannot provide a dry-run trace for the real build flavor.
4083
+ if (cacheWarmOnly) {
4084
+ throw new Error("native cache warming could not validate the compiler invocation", {
4085
+ cause: error,
4086
+ });
4087
+ }
3528
4088
  root = null;
3529
4089
  }
3530
4090
  }
3531
4091
  if (root === null) {
3532
4092
  // The exact historical command line, byte for byte.
3533
- try {
3534
- await materializeVendorPrerequisites();
3535
- await runClang(buildArgs((p) => p));
3536
- }
3537
- finally {
3538
- if (transientVendorRoot !== null) {
3539
- await rm(transientVendorRoot, { recursive: true, force: true }).catch(() => undefined);
3540
- }
3541
- }
4093
+ await runUncachedBuild();
3542
4094
  return;
3543
4095
  }
3544
4096
  let cv;
@@ -3551,12 +4103,16 @@ export async function compileC(opts) {
3551
4103
  readFile(opts.cPath),
3552
4104
  ]));
3553
4105
  }
3554
- catch {
4106
+ catch (error) {
3555
4107
  // A version/fingerprint probe is an optimization boundary. If the compiler
3556
4108
  // itself can still compile, preserve the pre-cache behavior instead of
3557
4109
  // surfacing a metadata command's failure as the build result.
3558
- await materializeVendorPrerequisites();
3559
- await runClang(buildArgs((p) => p));
4110
+ if (cacheWarmOnly) {
4111
+ throw new Error("native cache warming could not validate cache inputs", {
4112
+ cause: error,
4113
+ });
4114
+ }
4115
+ await runUncachedBuild();
3560
4116
  return;
3561
4117
  }
3562
4118
  // Caller-supplied native inputs can all hide mutable dependencies: `-l`
@@ -3564,7 +4120,8 @@ export async function compileC(opts) {
3564
4120
  // script can retain identical top-level bytes as its referenced files are
3565
4121
  // rebuilt. Keep the safe runtime-object cache, but force a fresh final link
3566
4122
  // whenever the caller supplies either form.
3567
- let cacheCompleteArtifact = cachePolicy.completeArtifacts &&
4123
+ let cacheCompleteArtifact = !cacheWarmOnly &&
4124
+ cachePolicy.completeArtifacts &&
3568
4125
  (opts.linkInputs?.length ?? 0) === 0 &&
3569
4126
  (opts.systemLibraries?.length ?? 0) === 0;
3570
4127
  let programDependencies = null;
@@ -3623,6 +4180,7 @@ export async function compileC(opts) {
3623
4180
  linkProbeArgs,
3624
4181
  effectiveLinkInvocationArgs,
3625
4182
  linkTraceInvocationArgs,
4183
+ ...(programShardMergeIdentity === null ? [] : [programShardMergeIdentity]),
3626
4184
  ]);
3627
4185
  if (cacheCompleteArtifact) {
3628
4186
  try {
@@ -3654,6 +4212,9 @@ export async function compileC(opts) {
3654
4212
  ...(linkMetadataStamp?.dependencies.map((dependency) => dependency.path) ??
3655
4213
  fingerprintDependencyPaths(implicitLinker)),
3656
4214
  ...fingerprintDependencyPaths(programDependencies),
4215
+ ...(programShardMergeIdentity === null
4216
+ ? []
4217
+ : fingerprintDependencyPaths(programShardMergeIdentity)),
3657
4218
  ]);
3658
4219
  // Every content-bearing fingerprint was computed before this metadata
3659
4220
  // snapshot. Re-read those exact files now so the snapshot cannot certify
@@ -3664,11 +4225,12 @@ export async function compileC(opts) {
3664
4225
  programCompilerInvocation,
3665
4226
  implicitLinker,
3666
4227
  programDependencies,
4228
+ ...(programShardMergeIdentity === null ? [] : [programShardMergeIdentity]),
3667
4229
  ])) ||
3668
4230
  await runtimeFingerprint(rtDir).catch(() => null) !== fingerprint ||
3669
4231
  !(await Promise.all([toolchainMetadataStamp, compileMetadataStamp, linkMetadataStamp]
3670
4232
  .filter((stamp) => stamp !== null)
3671
- .map((stamp) => localArtifactDependenciesStillMatch(stamp.dependencies)))).every(Boolean)) {
4233
+ .map((stamp) => nativeArtifactDependenciesStillMatch(stamp.dependencies)))).every(Boolean)) {
3672
4234
  throw new CacheInputsChangedError();
3673
4235
  }
3674
4236
  if (localArtifact !== null) {
@@ -3693,6 +4255,25 @@ export async function compileC(opts) {
3693
4255
  localArtifactDependencyPaths = null;
3694
4256
  }
3695
4257
  }
4258
+ // Warm-only builds deliberately do not need linker/header identity for a
4259
+ // complete executable, but they still need a stable runtime-object key.
4260
+ // The ordinary complete-artifact path computes these values above.
4261
+ if (cacheWarmOnly && preBuildDependencies === null) {
4262
+ try {
4263
+ const [dependencies, invocation] = await Promise.all([
4264
+ translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cBytes, toolchainEnv),
4265
+ effectiveCompilerInvocationFingerprint(driver, toolchainEnv, cflags),
4266
+ ]);
4267
+ programDependencies = dependencies;
4268
+ if (runtimeCompilerInvocation === null)
4269
+ runtimeCompilerInvocation = invocation;
4270
+ }
4271
+ catch (error) {
4272
+ throw new Error("native cache warming could not validate runtime-object inputs", {
4273
+ cause: error,
4274
+ });
4275
+ }
4276
+ }
3696
4277
  const binDir = join(root, "bin");
3697
4278
  let keyHex = null;
3698
4279
  let cachedBin = null;
@@ -3704,9 +4285,10 @@ export async function compileC(opts) {
3704
4285
  // the fingerprint and the pin.
3705
4286
  const identityArgs = buildArgs((p) => p).map((a) => a === opts.cPath ? "<program.c>" : a === opts.outPath ? "<out>" : a);
3706
4287
  const key = createHash("sha256")
3707
- // v9 adds effective compiler-wrapper invocations for the real runtime,
3708
- // program, and link flavors.
3709
- .update("bin-v9\0")
4288
+ // Sharded dev builds need the v10 identity below. Canonical single-TU
4289
+ // builds retain v9 so adding the opt-in mode does not evict release
4290
+ // binaries compiled by an earlier scriptc.
4291
+ .update(programShards === null ? "bin-v9\0" : "bin-v10\0")
3710
4292
  .update(cacheTargetIdentity(driver)).update("\0")
3711
4293
  .update(toolchainEnv).update("\0")
3712
4294
  .update(implicitToolchain).update("\0")
@@ -3728,6 +4310,9 @@ export async function compileC(opts) {
3728
4310
  .update(fingerprint).update("\0")
3729
4311
  .update(identityArgs.join("\x1f")).update("\0")
3730
4312
  .update(cBytes);
4313
+ if (programShards !== null) {
4314
+ updateProgramShardCacheIdentity(key, programShards, programPublicSymbols, programShardMergeIdentity ?? undefined);
4315
+ }
3731
4316
  keyHex = key.digest("hex");
3732
4317
  cachedBin = join(binDir, keyHex);
3733
4318
  const tmpOut = privateSiblingPath(opts.outPath, "bin-hit");
@@ -3744,7 +4329,10 @@ export async function compileC(opts) {
3744
4329
  await chmod(tmpOut, 0o777 & ~process.umask());
3745
4330
  await rename(tmpOut, opts.outPath);
3746
4331
  if (localArtifact !== null && localArtifactDependencyPaths !== null) {
3747
- await publishLocalArtifactStamp(localArtifact.stampPath, opts.outPath, localArtifact.key, localArtifactDependencyPaths, [dirname(resolve(opts.cPath))], [root]).catch(() => undefined);
4332
+ const stamp = await publishLocalArtifactStamp(localArtifact.stampPath, opts.outPath, localArtifact.key, localArtifactDependencyPaths, [dirname(resolve(opts.cPath))], [root]).catch(() => null);
4333
+ if (stamp !== null) {
4334
+ await opts.onArtifactReady?.({ dependencies: stamp.dependencies }).catch(() => undefined);
4335
+ }
3748
4336
  }
3749
4337
  return; // hit: the program/runtime payload compile and link were skipped
3750
4338
  }
@@ -3753,7 +4341,6 @@ export async function compileC(opts) {
3753
4341
  /* miss — build below, then publish */
3754
4342
  }
3755
4343
  }
3756
- await materializeVendorPrerequisites();
3757
4344
  // Miss: link the program's own TU against cached per-flavor runtime objects.
3758
4345
  // Collect the runtime sources this build actually compiles (the same
3759
4346
  // conditionals as the command line, by construction).
@@ -3764,6 +4351,7 @@ export async function compileC(opts) {
3764
4351
  });
3765
4352
  const buildDir = await mkdtemp(join(tmpdir(), "scriptc-cache-build-"));
3766
4353
  try {
4354
+ await materializeVendorPrerequisites(join(buildDir, "vendor-inputs"));
3767
4355
  // Freeze the exact bytes used for the key. Generated TUs live at stable
3768
4356
  // paths under .scriptc/, so two builds can otherwise overwrite that path
3769
4357
  // between hashing and clang and publish one invocation's code under the
@@ -3784,7 +4372,7 @@ export async function compileC(opts) {
3784
4372
  let strictObjectVerification = null;
3785
4373
  const objectImplicitToolchainStillMatches = () => {
3786
4374
  if (preBuildDependencies !== null) {
3787
- return localArtifactDependenciesStillMatch(preBuildDependencies);
4375
+ return nativeArtifactDependenciesStillMatch(preBuildDependencies);
3788
4376
  }
3789
4377
  // Complete-artifact caching can be disabled by caller-owned native
3790
4378
  // inputs while the safe runtime-object tier remains active. Preserve its
@@ -3799,15 +4387,116 @@ export async function compileC(opts) {
3799
4387
  return strictObjectVerification;
3800
4388
  };
3801
4389
  try {
3802
- const cached = await ensureRuntimeObjects(root, driver.argv, cflags, rtInputs, `obj-v5\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${implicitToolchain}\0${runtimeCompilerInvocation}\0${ccName}\0${cv}\0${fingerprint}\0`, async () => await objectImplicitToolchainStillMatches());
4390
+ const cached = await ensureRuntimeObjects(root, driver.argv, cflags, rtInputs, `obj-v5\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${implicitToolchain}\0${runtimeCompilerInvocation}\0${ccName}\0${cv}\0${fingerprint}\0`, async () => await objectImplicitToolchainStillMatches(), cacheWarmPaths);
3803
4391
  objects = await stageRuntimeObjects(cached, join(buildDir, "runtime-objects"));
3804
4392
  }
3805
4393
  catch (err) {
3806
4394
  if (err instanceof CacheInputsChangedError)
3807
4395
  cacheInputsStable = false;
4396
+ if (cacheWarmOnly) {
4397
+ throw new Error("native cache warming could not persist runtime objects", {
4398
+ cause: err,
4399
+ });
4400
+ }
3808
4401
  objects = null; // cache trouble is never a build failure
3809
4402
  }
3810
- await runClang(buildArgs((p) => objects?.get(p) ?? p, { programPath, outPath: privateOut, compilerVisibleSource: opts.cPath }));
4403
+ const compileShardedProgram = async () => {
4404
+ if (programShards === null || programPublicSymbols === undefined ||
4405
+ programCompilerInvocation === null || implicitCompileToolchain === null)
4406
+ return null;
4407
+ try {
4408
+ const programDependencyHash = programDependencies ??
4409
+ await translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cBytes, toolchainEnv);
4410
+ const stem = basename(opts.cPath, ".ll");
4411
+ const entries = programShards.map((shard, index) => {
4412
+ const sourcePath = join(buildDir, shard.name);
4413
+ const staged = join(buildDir, `${stem}.program-${index.toString().padStart(3, "0")}.o`);
4414
+ const key = createHash("sha256")
4415
+ .update("exe-program-shard-v1\0")
4416
+ .update(cacheTargetIdentity(driver)).update("\0")
4417
+ .update(toolchainEnv).update("\0")
4418
+ .update(implicitCompileToolchain).update("\0")
4419
+ .update(programCompilerInvocation).update("\0")
4420
+ .update(opts.cacheIdentity).update("\0")
4421
+ .update(driver.argv.join("\x1f")).update("\0")
4422
+ .update(cv).update("\0")
4423
+ .update(fingerprint).update("\0")
4424
+ .update(programDependencyHash).update("\0")
4425
+ .update(programCompilerArgs.join("\x1f")).update("\0")
4426
+ .update(opts.cPath).update("\0")
4427
+ .update(resolve(opts.cPath)).update("\0")
4428
+ .update(shard.name).update("\0")
4429
+ .update(shard.source)
4430
+ .digest("hex");
4431
+ return {
4432
+ ...shard,
4433
+ sourcePath,
4434
+ staged,
4435
+ cachePath: join(root, "program-shard", key),
4436
+ missed: false,
4437
+ };
4438
+ });
4439
+ const shardWidth = Math.min(8, availableParallelism());
4440
+ for (let i = 0; i < entries.length; i += shardWidth) {
4441
+ await Promise.all(entries.slice(i, i + shardWidth).map(async (entry) => {
4442
+ await writeFile(entry.sourcePath, entry.source);
4443
+ if (await copyValidCachedFile(entry.cachePath, entry.staged))
4444
+ return;
4445
+ entry.missed = true;
4446
+ await runClang([
4447
+ ...programCompilerArgs,
4448
+ `-ffile-prefix-map=${entry.sourcePath}=${opts.cPath}`,
4449
+ "-c",
4450
+ entry.sourcePath,
4451
+ "-o",
4452
+ entry.staged,
4453
+ ]);
4454
+ }));
4455
+ }
4456
+ const arArgv = driver.argv[0] === "zig" ? [driver.argv[0], "ar"] : ["ar"];
4457
+ const merged = await localizeLibraryObjects(driver, arArgv, buildDir, entries.map((entry) => entry.staged), [], programPublicSymbols, `${stem}.program`);
4458
+ const publishable = entries.filter((entry) => entry.missed);
4459
+ if (publishable.length > 0) {
4460
+ try {
4461
+ const [currentRuntime, currentFingerprints, currentInvocation, currentDependencies, currentCompiler, currentMerge] = await Promise.all([
4462
+ runtimeFingerprint(rtDir),
4463
+ implicitToolchainFingerprints(driver, toolchainEnv),
4464
+ effectiveCompilerInvocationFingerprint(driver, toolchainEnv, programCompilerArgs, ".ll"),
4465
+ translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cBytes, toolchainEnv),
4466
+ ccVersionOnce(driver.argv, toolchainEnv, true),
4467
+ resolveProgramShardMergeIdentity(driver),
4468
+ ]);
4469
+ if (currentRuntime === fingerprint &&
4470
+ currentFingerprints.compile === implicitCompileToolchain &&
4471
+ currentInvocation === programCompilerInvocation &&
4472
+ currentDependencies === programDependencyHash &&
4473
+ currentCompiler === cv &&
4474
+ currentMerge === programShardMergeIdentity) {
4475
+ await Promise.all(publishable.map((entry) => publishCachedFile(entry.staged, entry.cachePath)));
4476
+ }
4477
+ }
4478
+ catch {
4479
+ // The merged object is valid for this invocation; publication is
4480
+ // only an optimization for later edits.
4481
+ }
4482
+ }
4483
+ return merged;
4484
+ }
4485
+ catch {
4486
+ // Sharding is an optimization. Compile/link the canonical TU below if
4487
+ // a target tool, one shard, or relocatable merge is unavailable.
4488
+ return null;
4489
+ }
4490
+ };
4491
+ const shardedProgramObject = await compileShardedProgram();
4492
+ if (programShards !== null && shardedProgramObject === null) {
4493
+ // The canonical TU fallback is valid output, but it must not populate a
4494
+ // cache key describing a successful shard projection/merge.
4495
+ cacheInputsStable = false;
4496
+ }
4497
+ await runClang(buildArgs((p) => objects?.get(p) ?? p, shardedProgramObject === null
4498
+ ? { programPath, outPath: privateOut, compilerVisibleSource: opts.cPath }
4499
+ : { programPath: shardedProgramObject, outPath: privateOut }));
3811
4500
  await installArtifact(privateOut, opts.outPath);
3812
4501
  if (cachedBin !== null && keyHex !== null) {
3813
4502
  // Metadata comparison catches ordinary changes cheaply, but cannot by
@@ -3815,7 +4504,7 @@ export async function compileC(opts) {
3815
4504
  // the key. Recompute every content-bearing identity after the final link
3816
4505
  // so a header/SDK/compiler change in either pre-build gap cannot publish
3817
4506
  // new output under an old key.
3818
- const [currentRuntime, currentImplicit, currentRuntimeInvocation, currentProgramInvocation, currentProgramDependencies, currentLinker, currentCompiler] = await Promise.all([
4507
+ const [currentRuntime, currentImplicit, currentRuntimeInvocation, currentProgramInvocation, currentProgramDependencies, currentLinker, currentCompiler, currentProgramShardMerge] = await Promise.all([
3819
4508
  runtimeFingerprint(rtDir).catch(() => null),
3820
4509
  implicitToolchainFingerprint(driver, toolchainEnv).catch(() => null),
3821
4510
  effectiveCompilerInvocationFingerprint(driver, toolchainEnv, cflags).catch(() => null),
@@ -3823,18 +4512,22 @@ export async function compileC(opts) {
3823
4512
  translationUnitDependencyFingerprint(driver, cflags, opts.cPath, cBytes, toolchainEnv).catch(() => null),
3824
4513
  implicitLinkerFingerprint(driver, toolchainEnv, linkProbeArgs, effectiveLinkInvocationArgs, linkTraceInvocationArgs).catch(() => null),
3825
4514
  ccVersionOnce(driver.argv, toolchainEnv, true).catch(() => null),
4515
+ programShards === null
4516
+ ? Promise.resolve(null)
4517
+ : resolveProgramShardMergeIdentity(driver).catch(() => null),
3826
4518
  ]);
3827
4519
  cacheInputsStable =
3828
4520
  cacheInputsStable &&
3829
4521
  preBuildDependencies !== null &&
3830
- await localArtifactDependenciesStillMatch(preBuildDependencies) &&
4522
+ await nativeArtifactDependenciesStillMatch(preBuildDependencies) &&
3831
4523
  currentRuntime === fingerprint &&
3832
4524
  currentImplicit === implicitToolchain &&
3833
4525
  currentRuntimeInvocation === runtimeCompilerInvocation &&
3834
4526
  currentProgramInvocation === programCompilerInvocation &&
3835
4527
  currentProgramDependencies === programDependencies &&
3836
4528
  currentLinker === implicitLinker &&
3837
- currentCompiler === cv;
4529
+ currentCompiler === cv &&
4530
+ currentProgramShardMerge === programShardMergeIdentity;
3838
4531
  }
3839
4532
  if (cachedBin !== null && keyHex !== null && cacheInputsStable) {
3840
4533
  try {
@@ -3851,7 +4544,10 @@ export async function compileC(opts) {
3851
4544
  localArtifactDependencyPaths !== null &&
3852
4545
  cacheCompleteArtifact &&
3853
4546
  cacheInputsStable) {
3854
- await publishLocalArtifactStamp(localArtifact.stampPath, opts.outPath, localArtifact.key, localArtifactDependencyPaths, [dirname(resolve(opts.cPath))], [root]).catch(() => undefined);
4547
+ const stamp = await publishLocalArtifactStamp(localArtifact.stampPath, opts.outPath, localArtifact.key, localArtifactDependencyPaths, [dirname(resolve(opts.cPath))], [root]).catch(() => null);
4548
+ if (stamp !== null) {
4549
+ await opts.onArtifactReady?.({ dependencies: stamp.dependencies }).catch(() => undefined);
4550
+ }
3855
4551
  }
3856
4552
  }
3857
4553
  finally {
@@ -3859,6 +4555,78 @@ export async function compileC(opts) {
3859
4555
  }
3860
4556
  // Runtime-object population is itself a cache write, including on builds
3861
4557
  // whose native link inputs disable complete-artifact publication.
3862
- await pruneCache(root).catch(() => undefined);
4558
+ await pruneCache(root, cacheWarmPaths).catch(() => undefined);
4559
+ }
4560
+ export async function compileC(opts) {
4561
+ await compileCInternal(opts, false);
4562
+ }
4563
+ export function supportedNativeCacheWarmProfiles(driver) {
4564
+ if (isMobileTarget(driver.target) || targetPlatform(driver) === "wasi")
4565
+ return [];
4566
+ return ["runtime", "tls", "dynamic"];
4567
+ }
4568
+ /** Populate expensive native prerequisites against the current compiler,
4569
+ * target, SDK, and environment. The resulting entries use the exact same
4570
+ * identities and validators as ordinary builds; this merely pays their cost
4571
+ * before a developer's first program asks for them. Synthetic link products
4572
+ * are discarded and never enter the complete-executable cache. */
4573
+ export async function warmNativeCaches(options = {}) {
4574
+ const cacheRoot = cacheRootDir();
4575
+ if (cacheRoot === null) {
4576
+ throw new Error("the native build cache is disabled (unset SCRIPTC_NO_CACHE and use a non-empty SCRIPTC_CACHE_DIR)");
4577
+ }
4578
+ await ensurePrivateCacheRoot(cacheRoot, process.env["SCRIPTC_CACHE_DIR"] === undefined);
4579
+ const known = new Set(["runtime", "tls", "dynamic"]);
4580
+ for (const profile of options.profiles ?? []) {
4581
+ if (!known.has(profile))
4582
+ throw new Error(`unknown native cache warm profile '${profile}'`);
4583
+ }
4584
+ if (options.profiles?.length === 0)
4585
+ return { cacheRoot, profiles: [] };
4586
+ const mobileTarget = mobileLibraryTarget();
4587
+ if (mobileTarget !== null) {
4588
+ throw new Error(`native cache warming targets executable builds and is unsupported for SCRIPTC_TARGET=${mobileTarget}`);
4589
+ }
4590
+ const driver = resolveCc();
4591
+ const supported = supportedNativeCacheWarmProfiles(driver);
4592
+ if (supported.length === 0) {
4593
+ throw new Error(`native cache warming targets persistently cached native executables and is unsupported for SCRIPTC_TARGET=${driver.target}`);
4594
+ }
4595
+ const profiles = [...new Set(options.profiles ?? supported)];
4596
+ for (const profile of profiles) {
4597
+ if (!supported.includes(profile)) {
4598
+ throw new Error(`native cache warm profile '${profile}' is unsupported for SCRIPTC_TARGET=${driver.target}`);
4599
+ }
4600
+ }
4601
+ const workDir = await mkdtemp(join(tmpdir(), "scriptc-cache-warm-"));
4602
+ const cPath = join(workDir, "warm.c");
4603
+ await writeFile(cPath, "int main(void) { return 0; }\n");
4604
+ const protectedPaths = new Set();
4605
+ try {
4606
+ const results = await Promise.all(profiles.map(async (profile) => {
4607
+ const started = performance.now();
4608
+ await compileCInternal({
4609
+ cPath,
4610
+ outPath: join(workDir, process.platform === "win32" ? `${profile}.exe` : profile),
4611
+ cacheIdentity: "scriptc-native-cache-warm-v1",
4612
+ optimization: options.optimization ?? "release",
4613
+ sanitize: options.sanitize ?? false,
4614
+ ...(profile === "tls" ? { fetch: true } : {}),
4615
+ ...(profile === "dynamic" ? { dynamic: true } : {}),
4616
+ }, true, protectedPaths);
4617
+ return {
4618
+ profile,
4619
+ elapsedMs: Math.round((performance.now() - started) * 10) / 10,
4620
+ };
4621
+ }));
4622
+ await pruneCache(cacheRoot).catch(() => undefined);
4623
+ if (!(await Promise.all([...protectedPaths].map(fileExists))).every(Boolean)) {
4624
+ throw new Error(`SCRIPTC_CACHE_MAX_MB is too small to retain the requested native cache warm profiles (${profiles.join(", ")})`);
4625
+ }
4626
+ return { cacheRoot, profiles: results };
4627
+ }
4628
+ finally {
4629
+ await rm(workDir, { recursive: true, force: true }).catch(() => undefined);
4630
+ }
3863
4631
  }
3864
4632
  //# sourceMappingURL=cc.js.map