@scriptc/compiler 0.0.34 → 0.0.35

package/dist/index.js

@@ -1,8 +1,9 @@
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { basename, dirname, join, resolve } from "node:path";
-import { buildCacheRoot, CcCompileError, compileC, compileLibArchive, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform } from "./backend/cc.js";
+import { buildCacheRoot, CcCompileError, compileC, compileLibArchive, executableNativeEnvironmentFingerprint, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform } from "./backend/cc.js";
 import { emitModule } from "./backend/emission/emitter.js";
 import { emitLlvmModule, LlvmUnsupportedError } from "./backend/llvm/emitter.js";
+import { splitLlvmLibraryProgram, splitLlvmProgram } from "./backend/llvm/split.js";
 import { rebaseLibrarySourceComments, replaceLibraryIdentity, stripLibraryIdentity, stripLibrarySourceComments } from "./backend/library-identity.js";
 import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES } from "./diagnostics/diagnostic.js";
 import { checkLibraryIntegerSlots, classSeed, hasIntSlots, numberCarrierKind } from "./library/int-infer.js";
@@ -27,8 +28,10 @@ import { hasForeignFfiCallback } from "./backend/ffi-callbacks.js";
 import { FrontendInputTracker, trackedReadFile } from "./frontend/input-tracker.js";
 import { libraryFrontendImplementationFingerprint, publishEarlyLibraryCache, readEarlyLibraryCache, readSemanticLibraryCache } from "./library/early-cache.js";
 import { createSourceLineRebaser } from "./library/semantic-source.js";
+import { publishEarlyExecutableCache, publishEarlyExecutableRoute, readEarlyExecutableCache } from "./executable/early-cache.js";
+import { compilerImplementationIdentity } from "./library/implementation-identity.js";
 export const VERSION = "0.0.1";
-export { compileC, runtimeSrcDir } from "./backend/cc.js";
+export { compileC, runtimeSrcDir, warmNativeCaches, } from "./backend/cc.js";
 export { ANDROID_MIN_API, IPHONEOS_MIN_VERSION, isAndroidTarget, isIosTarget, isMobileTarget, mobileLibraryTarget, mobileTargetRefusal } from "./backend/cc.js";
 export { emitModule } from "./backend/emission/emitter.js";
 export { renderAll, renderDiagnostic } from "./diagnostics/render.js";
@@ -629,13 +632,112 @@ export function analyze(entryPath, opts = {}) {
 }
 /** The whole pipeline: load → preflight → lower → validate → emit C → clang. */
 export async function compile(entryPath, opts) {
+    const frontendInputs = new FrontendInputTracker();
+    return frontendInputs.run(() => compileTracked(entryPath, opts, frontendInputs));
+}
+function executableNativeFeatures(mod, backend, dynamic, optimization, llvmRefusal) {
+    return {
+        backend,
+        ...(optimization === "dev" ? { optimization: "dev" } : {}),
+        ...(llvmRefusal === undefined ? {} : { llvmRefusal }),
+        dynamic,
+        regex: moduleUsesRegex(mod),
+        copying: moduleUsesCopying(mod),
+        textDecoderLegacy: moduleUsesLegacyTextDecoder(mod),
+        fileHandle: moduleUsesFileHandle(mod),
+        fetch: moduleUsesFetch(mod),
+        netIsland: moduleEmbedsBuiltin(mod, "node:http") ||
+            moduleEmbedsBuiltin(mod, "node:https") ||
+            moduleEmbedsBuiltin(mod, "node:net") ||
+            moduleEmbedsBuiltin(mod, "node:tls"),
+        zlib: moduleUsesZlib(mod) || moduleEmbedsCompressedNpm(mod),
+        assert: moduleUsesAssert(mod),
+        inspect: moduleUsesInspect(mod),
+        dynInvoke: moduleUsesDynInvoke(mod),
+        dc: moduleUsesDc(mod),
+        dynAsync: moduleUsesDynAsync(mod),
+        events: moduleUsesProcessEvents(mod),
+        emitter: moduleUsesEmitter(mod),
+        symbol: moduleUsesSymbol(mod),
+        searchParams: moduleUsesSearchParams(mod),
+        qs: moduleUsesQs(mod),
+        parseArgs: moduleUsesParseArgs(mod),
+        stream: moduleUsesStream(mod),
+        net: moduleUsesNet(mod),
+        http: moduleUsesHttpServer(mod),
+        http2: moduleUsesHttp2(mod),
+        dgram: moduleUsesDgram(mod),
+        watch: moduleUsesFsWatch(mod),
+        foreignFfi: hasForeignFfiCallback(mod.ffiImports ?? []),
+        nodeTest: moduleUsesNodeTest(mod),
+        tls: moduleUsesTls(mod),
+        tlsCa: moduleUsesTlsCa(mod),
+    };
+}
+async function compileExecutableNative(features, cPath, outPath, sanitize, ffi, programSplit = null, onArtifactReady) {
+    const effectiveProgramSplit = programSplit ??
+        (features.optimization === "dev" && features.backend === "llvm" && !sanitize
+            ? splitLlvmProgram(await readFile(cPath, "utf8"))
+            : null);
+    await compileC({
+        cPath,
+        outPath,
+        cacheIdentity: "scriptc-generated-v1",
+        ...(features.optimization === "dev" ? { optimization: "dev" } : {}),
+        ...(effectiveProgramSplit === null
+            ? {}
+            : {
+                programShards: effectiveProgramSplit.shards,
+                programPublicSymbols: effectiveProgramSplit.publicSymbols,
+            }),
+        sanitize,
+        dynamic: features.dynamic,
+        regex: features.regex,
+        copying: features.copying,
+        textDecoderLegacy: features.textDecoderLegacy,
+        fileHandle: features.fileHandle,
+        fetch: features.fetch,
+        netIsland: features.netIsland,
+        zlib: features.zlib,
+        assert: features.assert,
+        inspect: features.inspect,
+        dynInvoke: features.dynInvoke,
+        dc: features.dc,
+        dynAsync: features.dynAsync,
+        events: features.events,
+        emitter: features.emitter,
+        symbol: features.symbol,
+        searchParams: features.searchParams,
+        qs: features.qs,
+        parseArgs: features.parseArgs,
+        stream: features.stream,
+        net: features.net,
+        http: features.http,
+        http2: features.http2,
+        dgram: features.dgram,
+        watch: features.watch,
+        foreignFfi: features.foreignFfi,
+        nodeTest: features.nodeTest,
+        tls: features.tls,
+        tlsCa: features.tlsCa,
+        ...(onArtifactReady === undefined ? {} : { onArtifactReady }),
+        ...(ffi === null
+            ? {}
+            : { linkInputs: ffi.libraries, systemLibraries: ffi.systemLibraries }),
+    });
+}
+async function compileTracked(entryPath, opts, frontendInputs) {
+    entryPath = resolve(entryPath);
     let ffi = null;
+    let ffiProfileBytes = null;
     if (opts.ffiProfilePath !== undefined) {
-        const loaded = loadFfiProfile(opts.ffiProfilePath);
+        const ffiProfilePath = resolve(opts.ffiProfilePath);
+        const loaded = loadFfiProfile(ffiProfilePath);
         if (!loaded.ok) {
             return { ok: false, diagnostics: loaded.diagnostics, sourceTexts: new Map() };
         }
         ffi = loaded.profile;
+        ffiProfileBytes = loaded.profileBytes;
     }
     const buildPlatform = buildTargetPlatform();
     // Mobile triples are library-mode targets: the archive an embedding app
@@ -664,6 +766,81 @@ export async function compile(entryPath, opts) {
             };
         }
     }
+    const cacheRoot = provenanceSources() === null
+        ? await prepareBuildCacheRoot(buildCacheRoot())
+        : null;
+    const implementation = await compilerImplementationIdentity();
+    const earlyCacheOptions = {
+        entryPath,
+        outDir: opts.outDir,
+        outPath: opts.outPath,
+        emitIr: opts.emitIr ?? false,
+        sanitize: opts.sanitize ?? false,
+        dynamic: opts.dynamic ?? false,
+        backend: opts.backend ?? "auto",
+        ...(opts.optimization === "dev" ? { optimization: "dev" } : {}),
+        npmStatic: opts.npmStatic ?? null,
+        ffiProfile: opts.ffiProfilePath === undefined || ffiProfileBytes === null
+            ? null
+            : { path: opts.ffiProfilePath, bytes: ffiProfileBytes },
+        target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${process.arch}`,
+        compiler: [process.env["SCRIPTC_CC"] ?? "clang"],
+        nativeEnvironment: await executableNativeEnvironmentFingerprint(),
+        nodeVersion: process.version,
+        implementation: implementation.digest,
+        implementationDependencies: implementation.dependencies,
+    };
+    const earlyHit = await readEarlyExecutableCache(cacheRoot, earlyCacheOptions);
+    if (earlyHit !== null) {
+        // Route/proof metadata is independently evictable. A full-compiler
+        // fallback that still finds the validated payload repairs that lightweight
+        // index so the next identical CLI invocation can avoid this module graph.
+        await publishEarlyExecutableRoute(cacheRoot, earlyCacheOptions).catch(() => undefined);
+        if (earlyHit.executableRestored) {
+            await pruneBuildCache(cacheRoot);
+            return {
+                ok: true,
+                binaryPath: opts.outPath,
+                cPath: earlyHit.cPath,
+                backend: earlyHit.native.backend,
+                ...(earlyHit.irPath === undefined ? {} : { irPath: earlyHit.irPath }),
+                ...(earlyHit.native.llvmRefusal === undefined
+                    ? {}
+                    : { llvmRefusal: earlyHit.native.llvmRefusal }),
+            };
+        }
+        try {
+            await compileExecutableNative(earlyHit.native, earlyHit.cPath, opts.outPath, opts.sanitize ?? false, ffi, null, async ({ dependencies }) => {
+                await publishEarlyExecutableCache(cacheRoot, earlyCacheOptions, {
+                    ...earlyHit,
+                    executableRestored: true,
+                    nativeDependencies: dependencies,
+                    frontend: earlyHit.frontend,
+                });
+            });
+        }
+        catch (err) {
+            if (ffi !== null && err instanceof CcCompileError) {
+                return {
+                    ok: false,
+                    diagnostics: [ffiNativeBuildDiag(ffiNativeBuildDetail(err), opts.ffiProfilePath ?? entryPath)],
+                    sourceTexts: new Map(),
+                };
+            }
+            throw err;
+        }
+        await pruneBuildCache(cacheRoot);
+        return {
+            ok: true,
+            binaryPath: opts.outPath,
+            cPath: earlyHit.cPath,
+            backend: earlyHit.native.backend,
+            ...(earlyHit.irPath === undefined ? {} : { irPath: earlyHit.irPath }),
+            ...(earlyHit.native.llvmRefusal === undefined
+                ? {}
+                : { llvmRefusal: earlyHit.native.llvmRefusal }),
+        };
+    }
     const fe = runFrontend(entryPath, opts.npmStatic);
     let lowered;
     let entryText;
@@ -737,6 +914,7 @@ export async function compile(entryPath, opts) {
     // (the frontend ran once, the IR is backend-agnostic — nothing recompiles).
     let cPath = join(opts.outDir, `${stem}.c`);
     let backend = "c";
+    let llvmSource = null;
     let llvmRefusal;
     if (opts.backend !== "c") {
         try {
@@ -746,6 +924,7 @@ export async function compile(entryPath, opts) {
             });
             cPath = join(opts.outDir, `${stem}.ll`);
             await writeFile(cPath, ll);
+            llvmSource = ll;
             backend = "llvm";
         }
         catch (err) {
@@ -772,105 +951,24 @@ export async function compile(entryPath, opts) {
         irPath = join(opts.outDir, `${stem}.ir.json`);
         await writeFile(irPath, serializeModule(lowered.module));
     }
+    const nativeFeatures = executableNativeFeatures(lowered.module, backend, opts.dynamic ?? false, opts.optimization ?? "release", llvmRefusal);
+    const programSplit = backend === "llvm" && (opts.optimization ?? "release") === "dev" &&
+        !(opts.sanitize ?? false) && llvmSource !== null
+        ? splitLlvmProgram(llvmSource)
+        : null;
     await mkdir(dirname(opts.outPath), { recursive: true });
+    let publishedExecutable = false;
     try {
-        await compileC({
-            cPath,
-            outPath: opts.outPath,
-            // The emitted TU's non-system dependencies are the runtime/vendor tree,
-            // which cc.ts fingerprints separately. Low-level arbitrary C omits this
-            // opt-in so same-path header edits can never hit a stale artifact.
-            cacheIdentity: "scriptc-generated-v1",
-            sanitize: opts.sanitize ?? false,
-            dynamic: opts.dynamic ?? false,
-            // The link switch for scr_regex.c + libregexp: detected on the IR, so
-            // regex-free programs keep the historical (pinned) command line.
-            regex: moduleUsesRegex(lowered.module),
-            // Array copying methods and the typed-array bridges live in one
-            // optional TU, linked only when one of their IR intrinsics survives.
-            copying: moduleUsesCopying(lowered.module),
-            textDecoderLegacy: moduleUsesLegacyTextDecoder(lowered.module),
-            // FileHandle owns a native descriptor and its settled-promise adapters;
-            // keep that optional TU out of programs that never call fsp.open.
-            fileHandle: moduleUsesFileHandle(lowered.module),
-            // The link switch for scr_fetch.c (the native bridge over scr_net +
-            // scr_tls + scr_http's client parser + zlib — cc.ts implies those
-            // units into the link): embedded npm code that references fetch gets
-            // the bridge; everything else keeps its exact link line.
-            fetch: moduleUsesFetch(lowered.module),
-            // The island's node:http/https client bridge: embedded graphs that
-            // import those builtins pull scr_net_island.c + the socket units.
-            netIsland: moduleEmbedsBuiltin(lowered.module, "node:http") ||
-                moduleEmbedsBuiltin(lowered.module, "node:https") ||
-                moduleEmbedsBuiltin(lowered.module, "node:net") ||
-                moduleEmbedsBuiltin(lowered.module, "node:tls"),
-            // The link switch for scr_zlib.c + libz: zlib.* libCalls on the IR,
-            // node:zlib in the embedded graph, or COMPRESSED embedded module text
-            // (emit-island.ts stores big npm sources as raw DEFLATE; the emitted
-            // main installs scr_zlib_inflate_exact on the same predicate).
-            zlib: moduleUsesZlib(lowered.module) || moduleEmbedsCompressedNpm(lowered.module),
-            // The link switch for scr_assert.c: assert.* libCalls on the IR (the
-            // regex switch also pulls it — scr_regex.c calls the assert helpers).
-            assert: moduleUsesAssert(lowered.module),
-            // The link switch for scr_inspect.c: insp.* libCalls on the IR.
-            inspect: moduleUsesInspect(lowered.module),
-            // The link switch for scr_dyn_invoke.c: dynInvoke nodes or
-            // dyn.defineProps libCalls on the IR.
-            dynInvoke: moduleUsesDynInvoke(lowered.module),
-            // The link switch for scr_dc.c: dc.* libCalls on the IR (the
-            // diagnostics_channel registry and pub/sub).
-            dc: moduleUsesDc(lowered.module),
-            // The link switch for scr_async_dyn.c: the checked-dynamic async
-            // surfaces (cc.ts also pulls it under the dynInvoke/dc gates).
-            dynAsync: moduleUsesDynAsync(lowered.module),
-            // The link switch for scr_events.c: process signal/exit listeners and
-            // the stdin event surface on the IR.
-            events: moduleUsesProcessEvents(lowered.module),
-            // The link switch for scr_events_emitter.c: the node:events
-            // EventEmitter surface on the IR (emitter.* libCalls or the
-            // %EventEmitter class def).
-            emitter: moduleUsesEmitter(lowered.module),
-            // The link switch for scr_symbol.c: sym.* libCalls or a symbol-kind
-            // type anywhere on the IR.
-            symbol: moduleUsesSymbol(lowered.module),
-            // The link switch for scr_url_params.c: sp.* libCalls, the
-            // url.searchParams getter, or a searchParams-kind type on the IR.
-            searchParams: moduleUsesSearchParams(lowered.module),
-            // The link switch for scr_qs.c: the qs.* libCalls that live there
-            // (parse/stringify/unescape; escape rides the always-linked encoder).
-            qs: moduleUsesQs(lowered.module),
-            // The static node:util parseArgs implementation (scr_util.c).
-            parseArgs: moduleUsesParseArgs(lowered.module),
-            // The link switch for scr_stream.c: the node:stream class surface on
-            // the IR (stream libCalls or the %Readable-family class defs).
-            stream: moduleUsesStream(lowered.module),
-            // The link switch for scr_net.c: net.* (or http.* — http rides on
-            // net) libCalls on the IR.
-            net: moduleUsesNet(lowered.module),
-            // The link switch for scr_http.c: http.* libCalls on the IR.
-            http: moduleUsesHttpServer(lowered.module),
-            http2: moduleUsesHttp2(lowered.module),
-            // The link switch for scr_dgram.c: dgram.* or dns.* libCalls on the IR.
-            dgram: moduleUsesDgram(lowered.module),
-            // The link switch for scr_watch.c: fs.watch/watcher.* libCalls on the IR.
-            watch: moduleUsesFsWatch(lowered.module),
-            foreignFfi: hasForeignFfiCallback(lowered.module.ffiImports ?? []),
-            // The link switch for scr_test.c: test.* libCalls on the IR.
-            nodeTest: moduleUsesNodeTest(lowered.module),
-            // The link switch for scr_tls.c + the vendored mbedTLS archive:
-            // tls.* or https.* libCalls on the IR.
-            tls: moduleUsesTls(lowered.module),
-            // The link switch for scr_tls_ca.c (the CA-store introspection unit
-            // — plain PEM bookkeeping, no mbedTLS): tlsca.* libCalls on the IR.
-            // cc.ts also compiles it under the tls gate (scr_tls.c consults the
-            // unit's default-set override for its trust anchors).
-            tlsCa: moduleUsesTlsCa(lowered.module),
-            ...(ffi !== null
-                ? {
-                    linkInputs: ffi.libraries,
-                    systemLibraries: ffi.systemLibraries,
-                }
-                : {}),
+        await compileExecutableNative(nativeFeatures, cPath, opts.outPath, opts.sanitize ?? false, ffi, programSplit, async ({ dependencies }) => {
+            await publishEarlyExecutableCache(cacheRoot, earlyCacheOptions, {
+                cPath,
+                native: nativeFeatures,
+                executableRestored: true,
+                nativeDependencies: dependencies,
+                frontend: frontendInputs.snapshot(),
+                ...(irPath === undefined ? {} : { irPath }),
+            });
+            publishedExecutable = true;
         });
     }
     catch (err) {
@@ -885,6 +983,16 @@ export async function compile(entryPath, opts) {
         }
         throw err;
     }
+    if (!publishedExecutable) {
+        await publishEarlyExecutableCache(cacheRoot, earlyCacheOptions, {
+            cPath,
+            native: nativeFeatures,
+            executableRestored: false,
+            frontend: frontendInputs.snapshot(),
+            ...(irPath === undefined ? {} : { irPath }),
+        }).catch(() => undefined);
+    }
+    await pruneBuildCache(cacheRoot);
     return {
         ok: true,
         binaryPath: opts.outPath,
@@ -1245,7 +1353,8 @@ async function compileLibraryNative(profile, cPath, archivePath, sanitize, featu
     const localizeSymbols = libraryLocalizeSymbols(profile);
     let identityCSource;
     let programSource;
-    if (profile.sidecar !== null || profile.emission === "c") {
+    if (profile.sidecar !== null || profile.emission === "c" ||
+        (profile.emission === "llvm" && profile.optimization === "dev" && !sanitize)) {
         const publicSource = await readFile(cPath, "utf8");
         programSource = publicSource;
     }
@@ -1268,10 +1377,19 @@ async function compileLibraryNative(profile, cPath, archivePath, sanitize, featu
     if (profile.emission === "c") {
         programSource = stripLibrarySourceComments(programSource, profile.entry);
     }
+    const llvmSplit = profile.emission === "llvm" && profile.optimization === "dev" && !sanitize && programSource !== undefined
+        ? splitLlvmLibraryProgram(programSource)
+        : null;
     await compileLibArchive({
         cPath,
         ...(programSource !== undefined ? { programSource } : {}),
         ...(identityCSource !== undefined ? { identityCSource } : {}),
+        ...(llvmSplit !== null
+            ? {
+                programShards: llvmSplit.shards,
+                programPublicSymbols: llvmSplit.publicSymbols,
+            }
+            : {}),
         outPath: archivePath,
         cacheIdentity: "scriptc-generated-library-v1",
         sanitize,