@scriptc/compiler 0.0.32 → 0.0.34

package/dist/index.js

@@ -1,14 +1,15 @@
-import { mkdir, rm, writeFile } from "node:fs/promises";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { basename, dirname, join, resolve } from "node:path";
-import { CcCompileError, compileC, compileLibArchive, mobileLibraryTarget, mobileTargetRefusal, resolveCc, targetPlatform } from "./backend/cc.js";
+import { buildCacheRoot, CcCompileError, compileC, compileLibArchive, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform } from "./backend/cc.js";
 import { emitModule } from "./backend/emission/emitter.js";
 import { emitLlvmModule, LlvmUnsupportedError } from "./backend/llvm/emitter.js";
+import { rebaseLibrarySourceComments, replaceLibraryIdentity, stripLibraryIdentity, stripLibrarySourceComments } from "./backend/library-identity.js";
 import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES } from "./diagnostics/diagnostic.js";
 import { checkLibraryIntegerSlots, classSeed, hasIntSlots, numberCarrierKind } from "./library/int-infer.js";
 import { loadLibraryProfile, profileRemediation, profileTeaching } from "./library/profile.js";
 import { decorateLibraryRefusals, evaluateLibraryFences } from "./library/fence-eval.js";
 import { assembleTrapTeaching } from "./library/trap-teaching.js";
-import { buildSidecar, canonicalModuleGraph, canonicalPath, compilerReleaseVersion, libraryIdentityHashes, } from "./library/sidecar.js";
+import { buildSidecar, canonicalModuleGraph, canonicalPath, compilerReleaseVersion, libraryIdentityHashes, updateSidecarIdentity, } from "./library/sidecar.js";
 import { validateSidecar } from "./library/sidecar-validate.js";
 import { entryFunctionExports } from "./frontend/lib-exports.js";
 import { entryContractFacts } from "./frontend/lib-contract.js";
@@ -18,10 +19,14 @@ import { validateModule } from "./ir/validate.js";
 import { canonicalBuiltinModule, checkPreflight, isNodeTypesPath, loadProgram, locOf, requiresOf, resolveNpmImport } from "./frontend/program.js";
 import { npmStaticIneligibleReason, npmStaticOffenders, npmStaticPackageOfPath } from "./frontend/npm-static.js";
 import { provenanceSources } from "./frontend/provenance-registry.js";
-import { resolveBareModule } from "./frontend/resolve.js";
+import { clearResolveCaches, resolveBareModule } from "./frontend/resolve.js";
 import { isJsSourceFileName, isRelativeSpecifier } from "./frontend/shared.js";
 import { lowerToIr } from "./frontend/lowering/lowerer.js";
 import { loadFfiProfile } from "./ffi/profile.js";
+import { hasForeignFfiCallback } from "./backend/ffi-callbacks.js";
+import { FrontendInputTracker, trackedReadFile } from "./frontend/input-tracker.js";
+import { libraryFrontendImplementationFingerprint, publishEarlyLibraryCache, readEarlyLibraryCache, readSemanticLibraryCache } from "./library/early-cache.js";
+import { createSourceLineRebaser } from "./library/semantic-source.js";
 export const VERSION = "0.0.1";
 export { compileC, runtimeSrcDir } from "./backend/cc.js";
 export { ANDROID_MIN_API, IPHONEOS_MIN_VERSION, isAndroidTarget, isIosTarget, isMobileTarget, mobileLibraryTarget, mobileTargetRefusal } from "./backend/cc.js";
@@ -316,6 +321,11 @@ function packageNameOfBareSpecifier(specifier) {
  * static-or-refuse teaching, and the detection closes over the opted-in
  * packages' own bare edges (no island exists to serve a dep from). */
 function runFrontend(entryPath, npmStatic, externalTypes) {
+    // Resolver package/workspace metadata is intentionally shared across the
+    // several load attempts of ONE auto-detection fixpoint, but never across
+    // separate compiles in a long-lived process.  A cache miss must observe
+    // package.json edits before it can publish a new early-library entry.
+    clearResolveCaches();
     const statuses = [];
     const npmSites = new Map();
     const judged = new Set();
@@ -844,6 +854,7 @@ export async function compile(entryPath, opts) {
             dgram: moduleUsesDgram(lowered.module),
             // The link switch for scr_watch.c: fs.watch/watcher.* libCalls on the IR.
             watch: moduleUsesFsWatch(lowered.module),
+            foreignFfi: hasForeignFfiCallback(lowered.module.ffiImports ?? []),
             // The link switch for scr_test.c: test.* libCalls on the IR.
             nodeTest: moduleUsesNodeTest(lowered.module),
             // The link switch for scr_tls.c + the vendored mbedTLS archive:
@@ -1200,14 +1211,205 @@ function mergeSidecarIntSlots(cfg, facts, mod) {
     }
     return { ok: true, config: cfg };
 }
+function libraryNativeFeatures(mod, backend) {
+    return {
+        backend,
+        regex: moduleUsesRegex(mod),
+        assert: moduleUsesAssert(mod),
+        inspect: moduleUsesInspect(mod),
+        symbol: moduleUsesSymbol(mod),
+        searchParams: moduleUsesSearchParams(mod),
+        emitter: moduleUsesEmitter(mod),
+        zlib: moduleUsesZlib(mod),
+        copying: moduleUsesCopying(mod),
+        textDecoderLegacy: moduleUsesLegacyTextDecoder(mod),
+        ...(mod.lib?.identity !== undefined ? { buildId: mod.lib.identity.buildId } : {}),
+    };
+}
+function libraryLocalizeSymbols(profile) {
+    return profile.localizeRuntime
+        ? [
+            profile.initSymbol,
+            profile.sinkRegisterSymbol,
+            ...(profile.collectSymbol !== null ? [profile.collectSymbol] : []),
+            ...(profile.resultResetSymbol !== null ? [profile.resultResetSymbol] : []),
+            ...(profile.callbackRegisterSymbol !== null ? [profile.callbackRegisterSymbol] : []),
+            ...(profile.sidecar !== null
+                ? [profile.sidecar.buildIdSymbol, profile.sidecar.abiVersionSymbol]
+                : []),
+            ...profile.exports.map((entry) => entry.symbol),
+        ]
+        : undefined;
+}
+async function compileLibraryNative(profile, cPath, archivePath, sanitize, features) {
+    const localizeSymbols = libraryLocalizeSymbols(profile);
+    let identityCSource;
+    let programSource;
+    if (profile.sidecar !== null || profile.emission === "c") {
+        const publicSource = await readFile(cPath, "utf8");
+        programSource = publicSource;
+    }
+    if (profile.sidecar !== null) {
+        if (features.buildId === undefined)
+            throw new Error("library identity TU has no build id");
+        const withoutIdentity = stripLibraryIdentity(programSource, profile.emission);
+        if (withoutIdentity === programSource) {
+            throw new Error("generated public library TU has no identity region");
+        }
+        programSource = withoutIdentity;
+        identityCSource = [
+            "#include <stdint.h>",
+            "#include <inttypes.h>",
+            `uint64_t ${profile.sidecar.buildIdSymbol}(void) { return UINT64_C(0x${features.buildId}); }`,
+            `uint32_t ${profile.sidecar.abiVersionSymbol}(void) { return ${profile.sidecar.abiVersion}u; }`,
+            "",
+        ].join("\n");
+    }
+    if (profile.emission === "c") {
+        programSource = stripLibrarySourceComments(programSource, profile.entry);
+    }
+    await compileLibArchive({
+        cPath,
+        ...(programSource !== undefined ? { programSource } : {}),
+        ...(identityCSource !== undefined ? { identityCSource } : {}),
+        outPath: archivePath,
+        cacheIdentity: "scriptc-generated-library-v1",
+        sanitize,
+        optimization: profile.optimization,
+        ...(localizeSymbols !== undefined ? { localizeSymbols } : {}),
+        ...(profile.instancePerThread ? { threadInstances: true } : {}),
+        regex: features.regex,
+        assert: features.assert,
+        inspect: features.inspect,
+        symbol: features.symbol,
+        searchParams: features.searchParams,
+        emitter: features.emitter,
+        zlib: features.zlib,
+        copying: features.copying,
+        textDecoderLegacy: features.textDecoderLegacy,
+    });
+}
+async function emitSemanticLibraryHit(hit, profile, opts, archivePath, cacheRoot, cacheOptions, timing) {
+    const mod = hit.mod;
+    const rootDir = dirname(resolve(opts.profilePath));
+    let sidecarJson = hit.sidecarJson;
+    if (profile.sidecar !== null) {
+        if (mod.lib?.identity === undefined || sidecarJson === null) {
+            throw new Error("semantic library cache lost sidecar identity metadata");
+        }
+        const modules = canonicalModuleGraph(rootDir, hit.sourceTexts);
+        const { buildId, sourceHash } = libraryIdentityHashes(compilerReleaseVersion(), profile.profileBytes, modules);
+        mod.lib.identity.buildId = buildId;
+        hit.native.buildId = buildId;
+        sidecarJson = updateSidecarIdentity(sidecarJson, buildId, sourceHash);
+    }
+    const validation = validateModule(mod);
+    if (validation.length > 0) {
+        return {
+            ok: false,
+            diagnostics: validation.map((violation) => iceDiag(violation.message, violation.loc)),
+            sourceTexts: hit.sourceTexts,
+        };
+    }
+    await mkdir(opts.outDir, { recursive: true });
+    const stem = basename(profile.entry).replace(/\.(ts|js|mjs|cjs)$/, "");
+    const cPath = join(opts.outDir, `${stem}.lib.${profile.emission === "llvm" ? "ll" : "c"}`);
+    let translationUnit = hit.translationUnit;
+    if (profile.sidecar !== null) {
+        translationUnit = replaceLibraryIdentity(translationUnit, profile.emission, mod.lib.identity);
+    }
+    if (profile.emission === "llvm") {
+        await writeFile(cPath, translationUnit);
+    }
+    else {
+        const previous = hit.previousSources.get(mod.sourceFile);
+        const current = hit.sourceTexts.get(mod.sourceFile);
+        if (previous === undefined || current === undefined) {
+            throw new Error("semantic library cache lost the entry source text");
+        }
+        translationUnit = rebaseLibrarySourceComments(translationUnit, mod.sourceFile, createSourceLineRebaser(mod.sourceFile, previous, current));
+        await writeFile(cPath, translationUnit);
+    }
+    timing("semantic-tu-restore", { output_bytes: Buffer.byteLength(translationUnit) });
+    await rm(join(opts.outDir, `${stem}.lib.${profile.emission === "llvm" ? "c" : "ll"}`), { force: true });
+    let irPath;
+    if (opts.emitIr) {
+        irPath = join(opts.outDir, `${stem}.lib.ir.json`);
+        await writeFile(irPath, serializeModule(mod));
+    }
+    await compileLibraryNative(profile, cPath, archivePath, opts.sanitize ?? false, hit.native);
+    timing("native-archive");
+    let sidecarPath;
+    if (sidecarJson !== null) {
+        sidecarPath = profile.sidecar.path !== null
+            ? resolve(dirname(archivePath), profile.sidecar.path)
+            : `${archivePath}.contract.json`;
+        await writeFile(sidecarPath, sidecarJson);
+    }
+    await publishEarlyLibraryCache(cacheRoot, cacheOptions, {
+        cPath,
+        native: hit.native,
+        frontend: hit.frontend,
+        semantic: { mod, sources: hit.sourceTexts },
+        ...(irPath !== undefined ? { irPath } : {}),
+        ...(sidecarPath !== undefined ? { sidecarPath } : {}),
+    }).catch(() => undefined);
+    await pruneBuildCache(cacheRoot);
+    timing("semantic-cache-publish");
+    timing("complete");
+    return {
+        ok: true,
+        archivePath,
+        cPath,
+        backend: profile.emission,
+        ...(irPath !== undefined ? { irPath } : {}),
+        ...(sidecarPath !== undefined ? { sidecarPath } : {}),
+    };
+}
 export async function compileLibrary(opts) {
+    const frontendInputs = new FrontendInputTracker();
+    return frontendInputs.run(() => compileLibraryTracked(opts, frontendInputs));
+}
+async function compileLibraryTracked(opts, frontendInputs) {
+    const timingOn = process.env["SCRIPTC_TIMING"] === "1";
+    const timingStart = performance.now();
+    let timingLast = timingStart;
+    const timing = (phase, detail = {}) => {
+        if (!timingOn)
+            return;
+        const now = performance.now();
+        process.stderr.write(`scriptc timing ${JSON.stringify({
+            phase,
+            phase_ms: Math.round((now - timingLast) * 10) / 10,
+            total_ms: Math.round((now - timingStart) * 10) / 10,
+            rss_mb: Math.round(process.memoryUsage().rss / 1024 / 1024),
+            ...detail,
+        })}\n`);
+        timingLast = now;
+    };
     const loadedProfile = loadLibraryProfile(resolve(opts.profilePath));
+    timing("profile-load");
     if (!loadedProfile.ok) {
         return { ok: false, diagnostics: loadedProfile.diagnostics, sourceTexts: new Map() };
     }
     const profile = loadedProfile.profile;
     const entryPath = profile.entry;
     const buildPlatform = buildTargetPlatform();
+    const profileDir = dirname(resolve(opts.profilePath));
+    for (let directory = dirname(entryPath);; directory = dirname(directory)) {
+        for (const name of ["tsconfig.json", "package.json"]) {
+            // Project configuration and package-realm metadata can affect the
+            // frontend even when TypeScript did not request their bytes through
+            // its delegated filesystem callbacks (tsgo may read them server-side).
+            frontendInputs.run(() => {
+                const path = join(directory, name);
+                trackedReadFile(path);
+            });
+        }
+        if (directory === profileDir || dirname(directory) === directory)
+            break;
+    }
+    const archivePath = opts.outPath ?? join(opts.outDir, `${basename(entryPath).replace(/\.(ts|js|mjs|cjs)$/, "")}.lib.a`);
     // Mobile-target admission first — a pure env/host check, so a refused
     // pairing never reaches toolchain discovery. iOS targets (device and
     // simulator) build on darwin hosts only: the Apple SDK sysroot and the
@@ -1264,6 +1466,44 @@ export async function compileLibrary(opts) {
             };
         }
     }
+    const cacheRoot = provenanceSources() === null
+        ? await prepareBuildCacheRoot(buildCacheRoot())
+        : null;
+    const earlyCacheOptions = {
+        profilePath: opts.profilePath,
+        profileBytes: profile.profileBytes,
+        entryPath,
+        outDir: opts.outDir,
+        ...(opts.outPath !== undefined ? { outPath: opts.outPath } : {}),
+        emitIr: opts.emitIr ?? false,
+        sanitize: opts.sanitize ?? false,
+        target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${process.arch}`,
+        compiler: [process.env["SCRIPTC_CC"] ?? "clang"],
+        nodeVersion: process.version,
+        implementation: await libraryFrontendImplementationFingerprint(),
+    };
+    const earlyHit = await readEarlyLibraryCache(cacheRoot, earlyCacheOptions, profile.sidecar === null ? undefined : profile.sidecar.path);
+    if (earlyHit !== null) {
+        timing("early-cache-hit");
+        await compileLibraryNative(profile, earlyHit.cPath, archivePath, opts.sanitize ?? false, earlyHit.native);
+        timing("native-archive");
+        timing("complete");
+        return {
+            ok: true,
+            archivePath,
+            cPath: earlyHit.cPath,
+            backend: earlyHit.native.backend,
+            ...(earlyHit.irPath !== undefined ? { irPath: earlyHit.irPath } : {}),
+            ...(earlyHit.sidecarPath !== undefined ? { sidecarPath: earlyHit.sidecarPath } : {}),
+        };
+    }
+    timing("early-cache-miss");
+    const semanticHit = await readSemanticLibraryCache(cacheRoot, earlyCacheOptions, profile.sidecar === null ? undefined : profile.sidecar.path);
+    if (semanticHit !== null) {
+        timing("semantic-cache-hit", { changed_sources: semanticHit.changedSources.length });
+        return emitSemanticLibraryHit(semanticHit, profile, opts, archivePath, cacheRoot, earlyCacheOptions, timing);
+    }
+    timing("semantic-cache-miss");
     // Bare npm specifiers in a library graph take the STATIC-OR-REFUSE
     // posture: "lib" runs the same auto-detection and eligibility bar as
     // the executable lane's --npm-static (own .d.ts, unminified shipped JS,
@@ -1271,6 +1511,10 @@ export async function compileLibrary(opts) {
     // island/dynamic tier to offer (SC4006's ground), so eligibility needs
     // no flag and a miss is a refusal, never a fallback.
     const fe = runFrontend(entryPath, "lib");
+    timing("frontend-load", {
+        entry_bytes: fe.entryText().length,
+        source_files: fe.sourceTexts().size,
+    });
     let lowered;
     let entryText;
     let sourceTexts;
@@ -1372,6 +1616,9 @@ export async function compileLibrary(opts) {
                     ]),
                 ],
             });
+            timing("lower", {
+                lib_roots: profile.exports.length + contractSurfaceRoots.length,
+            });
         }
         catch (e) {
             if (!isCheckerPanic(e))
@@ -1388,6 +1635,7 @@ export async function compileLibrary(opts) {
         fe.dispose();
     }
     const mod = lowered.module;
+    timing("frontend-dispose");
     const fail = (diagnostics) => ({
         ok: false,
         diagnostics: decorateLibraryRefusals(diagnostics, profile),
@@ -1418,9 +1666,9 @@ export async function compileLibrary(opts) {
     // The ask-2 contract sidecar rides the same invocation. Identity first
     // (schema §2's worked build_id definition over compiler version, profile
     // bytes, and the sorted canonical module graph; source_hash per the
-    // profile's "module-graph" contract) — the u64 lands on the IR so both
-    // backends emit the identity getters from the ONE value the sidecar
-    // records (V12's coherence by construction), then the projection into
+    // profile's "module-graph" contract) — the u64 lands on the IR so native
+    // archive assembly emits the identity getters from the ONE value the
+    // sidecar records (V12's coherence by construction), then the projection into
     // the schema (declaration orders from the AST) and the V1–V14
     // self-check before anything is written.
     let sidecarJson = null;
@@ -1457,6 +1705,7 @@ export async function compileLibrary(opts) {
             return fail([merged.diagnostic]);
         intCfg = merged.config;
     }
+    timing("contract-sidecar", { source_files: sourceTexts.size });
     // Ask 4: the integer-boundary inference — every value that can reach a
     // profile-declared i64/u64 slot must PROVE representability, wholeness,
     // and range, or the build refuses with the failed obligation, the
@@ -1471,17 +1720,21 @@ export async function compileLibrary(opts) {
             return fail(refusals.map((v) => libIntBoundaryDiag(v.path, v.cls, v.obligation, v.detail, v.fix, v.loc)));
         }
     }
+    timing("integer-proof");
     const validation = validateModule(mod);
     if (validation.length > 0)
         return fail(validation.map((v) => iceDiag(v.message, v.loc)));
+    timing("ir-validate");
     await mkdir(opts.outDir, { recursive: true });
     const stem = basename(entryPath).replace(/\.(ts|js|mjs|cjs)$/, "");
     let cPath;
     if (profile.emission === "llvm") {
         try {
             const ll = emitLlvmModule(mod);
+            timing("llvm-emit", { output_bytes: Buffer.byteLength(ll) });
             cPath = join(opts.outDir, `${stem}.lib.ll`);
             await writeFile(cPath, ll);
+            timing("llvm-write");
         }
         catch (err) {
             if (!(err instanceof LlvmUnsupportedError))
@@ -1500,40 +1753,9 @@ export async function compileLibrary(opts) {
         irPath = join(opts.outDir, `${stem}.lib.ir.json`);
         await writeFile(irPath, serializeModule(mod));
     }
-    const archivePath = opts.outPath ?? join(opts.outDir, `${stem}.lib.a`);
-    // Multi-instance library mode: the profile-declared external surface —
-    // the mode-provided entries, the identity getters, and the export map —
-    // is exactly the set the localization step keeps global.
-    const localizeSymbols = profile.localizeRuntime
-        ? [
-            profile.initSymbol,
-            profile.sinkRegisterSymbol,
-            ...(profile.collectSymbol !== null ? [profile.collectSymbol] : []),
-            ...(profile.resultResetSymbol !== null ? [profile.resultResetSymbol] : []),
-            ...(profile.callbackRegisterSymbol !== null ? [profile.callbackRegisterSymbol] : []),
-            ...(profile.sidecar !== null
-                ? [profile.sidecar.buildIdSymbol, profile.sidecar.abiVersionSymbol]
-                : []),
-            ...profile.exports.map((e) => e.symbol),
-        ]
-        : undefined;
-    await compileLibArchive({
-        cPath,
-        outPath: archivePath,
-        cacheIdentity: "scriptc-generated-library-v1",
-        sanitize: opts.sanitize ?? false,
-        ...(localizeSymbols !== undefined ? { localizeSymbols } : {}),
-        ...(profile.instancePerThread ? { threadInstances: true } : {}),
-        regex: moduleUsesRegex(mod),
-        assert: moduleUsesAssert(mod),
-        inspect: moduleUsesInspect(mod),
-        symbol: moduleUsesSymbol(mod),
-        searchParams: moduleUsesSearchParams(mod),
-        emitter: moduleUsesEmitter(mod),
-        zlib: moduleUsesZlib(mod),
-        copying: moduleUsesCopying(mod),
-        textDecoderLegacy: moduleUsesLegacyTextDecoder(mod),
-    });
+    const nativeFeatures = libraryNativeFeatures(mod, profile.emission);
+    await compileLibraryNative(profile, cPath, archivePath, opts.sanitize ?? false, nativeFeatures);
+    timing("native-archive");
     // The sidecar lands beside the compiled object, written by the same
     // invocation (profile-declared name; the neutral default when the
     // profile states none is <out>.contract.json).
@@ -1545,6 +1767,18 @@ export async function compileLibrary(opts) {
                 : `${archivePath}.contract.json`;
         await writeFile(sidecarPath, sidecarJson);
     }
+    const earlyPublish = {
+        cPath,
+        native: nativeFeatures,
+        frontend: frontendInputs.snapshot(),
+        semantic: { mod, sources: sourceTexts },
+        ...(irPath !== undefined ? { irPath } : {}),
+        ...(sidecarPath !== undefined ? { sidecarPath } : {}),
+    };
+    await publishEarlyLibraryCache(cacheRoot, earlyCacheOptions, earlyPublish).catch(() => undefined);
+    await pruneBuildCache(cacheRoot);
+    timing("early-cache-publish");
+    timing("complete");
     return {
         ok: true,
         archivePath,