modality-ts 0.0.10 → 0.0.11

package/dist/cli/features/extract/command.js

@@ -4,20 +4,20 @@ import { dirname, extname, join, parse, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 import * as ts from "typescript";
 import { runExtractionPipeline } from "modality-ts/extract";
-import { canonicalJson, parseModelArtifact, } from "modality-ts/core";
-import { routerSource } from "modality-ts/extract/sources/router";
+import { canonicalJson, collectTokenDomainPaths, domainCardinality, parseModelArtifact, } from "modality-ts/core";
+import { parseReactRouterRoutes, routerSource, } from "modality-ts/extract/sources/router";
 import { emitAppModel } from "../../codegen/model.js";
 import { loadAndApplyOverlay, loadOverlaySpec } from "../../overlay.js";
 import { createBuiltinModalityRegistry } from "../../registry/index.js";
 export async function runExtractCommand(options) {
     const sourcePaths = normalizedSourcePaths(options);
-    const project = await loadExtractionProject(sourcePaths);
-    const config = await loadModalityConfig(options.configPath ?? (await findNearestConfig(project.configStartDir)));
+    const projectBase = await loadExtractionProject(sourcePaths);
+    const config = await loadModalityConfig(options.configPath ?? (await findNearestConfig(projectBase.configStartDir)));
     const route = options.route ?? config.route ?? "/";
     const appModelPath = options.appModelPath ?? `${dirname(options.modelPath)}/app.model.ts`;
     const packageJsonPath = options.packageJsonPath ??
         config.packageJsonPath ??
-        (await findNearestPackageJson(project.configStartDir));
+        (await findNearestPackageJson(projectBase.configStartDir));
     const dependencies = await readPackageDependencies(packageJsonPath);
     const registry = createBuiltinModalityRegistry({
         dependencies,
@@ -31,6 +31,9 @@ export async function runExtractCommand(options) {
         ],
         routerPlugin: options.routerPlugin ?? config.routerPlugin,
     });
+    const routerAdapter = registry.routerPlugin ?? routerSource();
+    const project = await attachRouteInventory(projectBase, routerAdapter);
+    const routePatterns = project.inventory.routes.map((node) => node.pattern);
     const effectApis = uniqueStrings([
         ...(config.effectApis ?? []),
         ...(options.effectApis ?? []),
@@ -47,27 +50,15 @@ export async function runExtractCommand(options) {
         sourceText: project.sourceText,
         fileName: project.entryFile,
         route,
-        routePatterns: project.routes,
+        routePatterns,
         effectApis,
         sourcePlugins: registry.sourcePlugins,
         routerPlugin: registry.routerPlugin,
+        inventory: project.inventory,
     });
     const transitions = [...pipeline.transitions];
-    const discoveredRoutes = uniqueStrings([
-        route,
-        ...project.routes,
-        ...transitionNavigatedRoutes(transitions),
-    ]);
-    const defaultRouter = routerSource();
-    const routeVars = registry.routerPlugin
-        ? registry.routerPlugin.routeVars(discoveredRoutes, {
-            route,
-            bounds: { maxHistory: 4 },
-        })
-        : defaultRouter.routeVars(discoveredRoutes, {
-            route,
-            bounds: { maxHistory: 4 },
-        });
+    const lowering = buildLocationLowering(transitions, routerAdapter, project.inventory);
+    const routeVars = routerAdapter.locationVars(project.inventory, { route, bounds: { maxHistory: 4 } }, lowering);
     const templateVars = pipeline.templateFragments.flatMap((fragment) => fragment.vars);
     const stateVars = refineAssignedLiteralDomains([...pipeline.stateVars, ...templateVars], transitions);
     const extractedModel = {
@@ -111,7 +102,7 @@ export async function runExtractCommand(options) {
             extractionCaveats,
         },
     };
-    const report = createExtractionReport(project.sourceFiles, model, warnings, overlay.ignoredVars, options.now ?? new Date());
+    const report = createExtractionReport(project.sourceFiles, model, warnings, overlay.ignoredVars, options.now ?? new Date(), project.inventory);
     await mkdir(dirname(options.modelPath), { recursive: true });
     await writeFile(options.modelPath, `${canonicalJson(model)}\n`, "utf8");
     await mkdir(dirname(appModelPath), { recursive: true });
@@ -123,11 +114,42 @@ export async function runExtractCommand(options) {
     if (options.expectModelPath) {
         await assertMatchesExpectedModel(model, options.expectModelPath);
     }
+    const stateSpaceLine = (() => {
+        const contributors = report.stateContributors;
+        if (!contributors)
+            return undefined;
+        const { totalBits, topVars } = contributors;
+        const top = topVars
+            .slice(0, 3)
+            .map((v) => `${v.varId}(${v.bits.toFixed(1)})`)
+            .join(",");
+        return `state-space≈${totalBits.toFixed(1)}bits top:${top}`;
+    })();
+    const coarseDomainsLine = (() => {
+        const entries = report.coarseDomains ?? [];
+        if (entries.length === 0)
+            return undefined;
+        const count = entries.reduce((sum, entry) => sum + entry.paths.length, 0);
+        const first = entries[0];
+        if (!first)
+            return undefined;
+        const examplePath = first.paths[0];
+        return `coarse-domains=${count} e.g. ${first.varId}[${examplePath ?? ""}]`;
+    })();
+    const routeCoverageLine = (() => {
+        const coverage = report.routeCoverage;
+        if (!coverage || coverage.configured === 0)
+            return undefined;
+        return formatRouteCoverageLine(coverage);
+    })();
     return {
         model,
         report,
         lines: [
             `extracted vars=${pipeline.stateVars.length + pipeline.templateFragments.flatMap((fragment) => fragment.vars).length} transitions=${transitions.length}`,
+            ...(stateSpaceLine ? [stateSpaceLine] : []),
+            ...(routeCoverageLine ? [routeCoverageLine] : []),
+            ...(coarseDomainsLine ? [coarseDomainsLine] : []),
             `plugins=${registry.plugins.map((plugin) => `${plugin.kind}:${plugin.id}@${plugin.version}`).join(",") || "none"}`,
             `model=${options.modelPath}`,
             `appModel=${appModelPath}`,
@@ -159,6 +181,8 @@ async function loadExtractionProject(sourcePaths) {
     if (sourcePaths.length > 1)
         return loadMultiFileExtractionProject(sourcePaths);
     const resolved = sourcePaths[0];
+    if (!resolved)
+        throw new Error("extract requires at least one source path");
     const info = await stat(resolved);
     if (!info.isDirectory()) {
         const source = await readFile(resolved, "utf8");
@@ -169,7 +193,7 @@ async function loadExtractionProject(sourcePaths) {
             sourceText: imported.sources.map((entry) => entry.text).join("\n"),
             sourceFiles: imported.sources.map((entry) => entry.path),
             sources: imported.sources,
-            routes: [],
+            inventory: { routes: [] },
             effectApis: fetchEffectApis(imported.sources.map((entry) => entry.text).join("\n")),
             configStartDir: dirname(resolved),
         };
@@ -179,6 +203,7 @@ async function loadExtractionProject(sourcePaths) {
     const rootPath = join(resolved, "app", "root.tsx");
     const roots = await existingFiles([rootPath]);
     const entries = [
+        { path: routesPath, text: await readFile(routesPath, "utf8") },
         ...(await Promise.all(roots.map(async (path) => ({ path, text: await readFile(path, "utf8") })))),
         ...(await Promise.all(routeEntries.map(async (entry) => ({
             path: resolve(dirname(routesPath), entry.file),
@@ -193,7 +218,7 @@ async function loadExtractionProject(sourcePaths) {
         sourceText,
         sourceFiles: imported.sources.map((entry) => entry.path),
         sources: imported.sources,
-        routes: uniqueStrings(routeEntries.map((entry) => entry.pattern)),
+        inventory: { routes: [] },
         effectApis: fetchEffectApis(sourceText),
         configStartDir: resolved,
     };
@@ -213,7 +238,7 @@ async function loadMultiFileExtractionProject(sourcePaths) {
         sourceText,
         sourceFiles: sources.map((entry) => entry.path),
         sources,
-        routes: uniqueStrings(projects.flatMap((project) => project.routes)),
+        inventory: { routes: [] },
         effectApis: uniqueStrings([
             ...projects.flatMap((project) => project.effectApis),
             ...fetchEffectApis(sourceText),
@@ -221,12 +246,51 @@ async function loadMultiFileExtractionProject(sourcePaths) {
         configStartDir: commonAncestor(projects.map((project) => project.configStartDir)),
     };
 }
+async function attachRouteInventory(project, adapter) {
+    const files = [...project.sources];
+    const manifestPath = files.find((file) => file.path.endsWith("routes.ts"))?.path ??
+        (await findNearestRoutesManifest(project.configStartDir));
+    if (manifestPath &&
+        !files.some((file) => resolve(file.path) === resolve(manifestPath))) {
+        files.push({
+            path: manifestPath,
+            text: await readFile(manifestPath, "utf8"),
+        });
+    }
+    const inventory = await adapter.discoverRoutes({
+        rootDir: project.configStartDir,
+        files,
+        readFile: (path) => readFile(path, "utf8"),
+    });
+    return { ...project, inventory };
+}
+async function findNearestRoutesManifest(startDir) {
+    let current = resolve(startDir);
+    for (let depth = 0; depth < 8; depth += 1) {
+        const candidate = join(current, "app", "routes.ts");
+        try {
+            const info = await stat(candidate);
+            if (info.isFile())
+                return candidate;
+        }
+        catch {
+            // keep walking upward
+        }
+        const parent = dirname(current);
+        if (parent === current)
+            break;
+        current = parent;
+    }
+    return undefined;
+}
 async function sourceWithLocalImports(entries, tsconfig) {
     const seen = new Set();
     const sources = [];
     const queue = [...entries];
     while (queue.length > 0) {
         const next = queue.shift();
+        if (!next)
+            break;
         const canonical = resolve(next.path);
         if (seen.has(canonical))
             continue;
@@ -372,33 +436,6 @@ async function existingFiles(paths) {
     }
     return found;
 }
-function parseReactRouterRoutes(source) {
-    const routes = [];
-    const parsed = tsCreateSourceFile(source);
-    const visit = (node) => {
-        if (ts.isCallExpression(node) && ts.isIdentifier(node.expression)) {
-            if (node.expression.text === "index" &&
-                ts.isStringLiteral(node.arguments[0])) {
-                routes.push({ pattern: "/", file: node.arguments[0].text });
-            }
-            if (node.expression.text === "route" &&
-                ts.isStringLiteral(node.arguments[0]) &&
-                ts.isStringLiteral(node.arguments[1])) {
-                routes.push({
-                    pattern: reactRouterPathPattern(node.arguments[0].text),
-                    file: node.arguments[1].text,
-                });
-            }
-        }
-        tsForEachChild(node, visit);
-    };
-    visit(parsed);
-    return routes;
-}
-function reactRouterPathPattern(pattern) {
-    const normalized = pattern.startsWith("/") ? pattern : `/${pattern}`;
-    return normalized.replace(/\$([A-Za-z0-9_]+)/g, ":$1").replace(/\*$/, "*");
-}
 function sourceHashes(sources) {
     return Object.fromEntries(sources.map((source) => [source.path, sha256(source.text)]));
 }
@@ -536,7 +573,11 @@ function commonAncestor(paths) {
         }
     }
     const prefix = first.slice(0, length).join("/");
-    return prefix === "" ? parse(paths[0]).root : prefix;
+    if (prefix === "") {
+        const firstPath = paths[0];
+        return firstPath ? parse(firstPath).root : process.cwd();
+    }
+    return prefix;
 }
 async function findNearestPackageJson(startDir) {
     let dir = startDir;
@@ -577,21 +618,50 @@ async function assertMatchesExpectedModel(model, expectedModelPath) {
         throw new Error(`Extracted model differs from expected snapshot ${expectedModelPath}`);
     }
 }
-function createExtractionReport(sourceFiles, model, warnings, ignoredVars, now) {
+function round2(n) {
+    return Math.round(n * 100) / 100;
+}
+function buildStateContributors(model, limit = 20) {
+    const contributors = model.vars.map((decl) => {
+        const cardinality = domainCardinality(decl.domain);
+        const bits = cardinality < 1 ? 0 : round2(Math.log2(cardinality));
+        const scope = decl.scope.kind === "global" ? "global" : decl.scope.route;
+        const origin = typeof decl.origin === "string" ? decl.origin : decl.origin.file;
+        return {
+            varId: decl.id,
+            domainKind: decl.domain.kind,
+            bits,
+            scope,
+            origin,
+        };
+    });
+    const totalBits = round2(contributors.reduce((sum, c) => sum + c.bits, 0));
+    const topVars = [...contributors]
+        .sort((a, b) => b.bits - a.bits || a.varId.localeCompare(b.varId))
+        .slice(0, limit);
+    const bySourceMap = new Map();
+    for (const c of contributors) {
+        bySourceMap.set(c.origin, round2((bySourceMap.get(c.origin) ?? 0) + c.bits));
+    }
+    const bySource = [...bySourceMap.entries()]
+        .map(([source, bits]) => ({ source, bits }))
+        .sort((a, b) => b.bits - a.bits || a.source.localeCompare(b.source));
+    return { totalBits, topVars, bySource };
+}
+function createExtractionReport(sourceFiles, model, warnings, ignoredVars, now, inventory) {
     const caveats = model.metadata?.extractionCaveats ?? emptyExtractionCaveats();
+    const varDomains = new Map(model.vars.map((decl) => [decl.id, decl.domain]));
     const transitionHandlers = model.transitions.map((transition) => ({
         id: transition.id,
         classification: transition.confidence === "manual"
             ? "overlay"
             : transition.confidence,
         reasons: transition.confidence === "over-approx"
-            ? overApproxReasons(transition)
+            ? overApproxReasons(transition, varDomains)
             : [],
     }));
     const transitionIds = new Set(transitionHandlers.map((handler) => handler.id));
-    const unextractableHandlers = warnings
-        .map(unextractableHandlerFromWarning)
-        .filter((handler) => Boolean(handler))
+    const unextractableHandlers = dedupeUnextractableHandlers(warnings)
         .filter((handler) => !transitionIds.has(handler.id))
         .map((handler) => ({
         id: handler.id,
@@ -602,6 +672,14 @@ function createExtractionReport(sourceFiles, model, warnings, ignoredVars, now)
     const exactOrOverlay = handlers.filter((handler) => handler.classification === "exact" ||
         handler.classification === "overlay").length;
     const unextractable = handlers.filter((handler) => handler.classification === "unextractable").length;
+    const coarseDomains = model.vars
+        .map((decl) => ({
+        varId: decl.id,
+        paths: collectTokenDomainPaths(decl.domain),
+    }))
+        .filter((entry) => entry.paths.length > 0)
+        .sort((a, b) => a.varId.localeCompare(b.varId));
+    const routeCoverage = buildRouteCoverage(inventory, model);
     return {
         schemaVersion: 1,
         kind: "extraction-report",
@@ -624,6 +702,9 @@ function createExtractionReport(sourceFiles, model, warnings, ignoredVars, now)
                             ? "default-token"
                             : "type-derived"),
         })),
+        ...(coarseDomains.length > 0 ? { coarseDomains } : {}),
+        stateContributors: buildStateContributors(model),
+        ...(routeCoverage ? { routeCoverage } : {}),
         coverage: {
             handlersTotal: handlers.length,
             exactOrOverlay,
@@ -634,6 +715,106 @@ function createExtractionReport(sourceFiles, model, warnings, ignoredVars, now)
         warnings,
     };
 }
+function buildRouteCoverage(inventory, model) {
+    if (!inventory || inventory.routes.length === 0)
+        return undefined;
+    const routeVar = model.vars.find((decl) => decl.id === "sys:route");
+    const modeledValues = new Set(routeVar?.domain.kind === "enum" ? routeVar.domain.values : []);
+    const routes = inventory.routes
+        .map((node) => {
+        const modeled = modeledValues.has(node.pattern);
+        if (modeled)
+            return { pattern: node.pattern, modeled: true };
+        let classification;
+        let reason;
+        if (node.kind === "resource") {
+            classification = "api";
+            reason = "API/resource route excluded from client state";
+        }
+        else if (node.redirectTo) {
+            classification = "redirect-only";
+            reason = "Redirect-only route excluded from client state";
+        }
+        else if (node.pattern.includes("*")) {
+            classification = "unsupported";
+            reason = "Splat/wildcard route pattern not modeled";
+        }
+        else {
+            classification = "no-client-state";
+            reason = "No client-side state modeled for this route";
+        }
+        return { pattern: node.pattern, modeled: false, classification, reason };
+    })
+        .sort((left, right) => left.pattern.localeCompare(right.pattern));
+    const modeled = routes.filter((entry) => entry.modeled).length;
+    return { configured: inventory.routes.length, modeled, routes };
+}
+function formatRouteCoverageLine(coverage) {
+    const omitted = coverage.configured - coverage.modeled;
+    const counts = new Map();
+    for (const entry of coverage.routes) {
+        if (entry.modeled || !entry.classification)
+            continue;
+        counts.set(entry.classification, (counts.get(entry.classification) ?? 0) + 1);
+    }
+    const parts = [...counts.entries()]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([classification, count]) => `${classification}=${count}`);
+    const suffix = parts.length > 0 ? ` [${parts.join(",")}]` : "";
+    return `routes configured=${coverage.configured} modeled=${coverage.modeled} omitted=${omitted}${suffix}`;
+}
+function buildLocationLowering(transitions, adapter, inventory) {
+    const pushTargets = new Set();
+    const pushOrigins = new Set();
+    let hasUnboundPush = false;
+    for (const transition of transitions) {
+        if (transition.id.startsWith("route:"))
+            continue;
+        const navigations = collectPushReplaceNavigations(transition.effect);
+        if (navigations.length === 0)
+            continue;
+        const component = transition.id.split(".")[0] ?? "";
+        const origin = adapter.routeForComponent?.(component, inventory);
+        for (const navigation of navigations) {
+            if (navigation.to)
+                pushTargets.add(navigation.to);
+            if (!origin)
+                hasUnboundPush = true;
+            else
+                pushOrigins.add(origin);
+        }
+    }
+    return {
+        pushTargets: [...pushTargets].sort(),
+        pushOrigins: [...pushOrigins].sort(),
+        hasUnboundPush,
+    };
+}
+function collectPushReplaceNavigations(effect) {
+    const navigations = [];
+    const visit = (current) => {
+        if (current.kind === "navigate" &&
+            (current.mode === "push" || current.mode === "replace")) {
+            const to = current.to?.kind === "lit" && typeof current.to.value === "string"
+                ? current.to.value
+                : undefined;
+            navigations.push({
+                mode: current.mode,
+                ...(to !== undefined ? { to } : {}),
+            });
+        }
+        if (current.kind === "seq") {
+            for (const child of current.effects)
+                visit(child);
+        }
+        if (current.kind === "if") {
+            visit(current.then);
+            visit(current.else);
+        }
+    };
+    visit(effect);
+    return navigations;
+}
 function emptyExtractionCaveats() {
     return {
         globalTaints: [],
@@ -656,10 +837,7 @@ function createExtractionCaveats(warnings) {
             .map(unhandledRejectionFromWarning)
             .filter(isCaveat)
             .sort(compareCaveats),
-        unextractableHandlers: warnings
-            .map(unextractableHandlerFromWarning)
-            .filter(isCaveat)
-            .sort(compareCaveats),
+        unextractableHandlers: dedupeUnextractableHandlers(warnings),
     };
 }
 function globalTaintFromWarning(warning) {
@@ -723,32 +901,15 @@ function firstSemverMajor(range) {
 function pluginProvenance(plugins) {
     return [...plugins.sources, ...(plugins.router ? [plugins.router] : [])].sort((left, right) => left.kind.localeCompare(right.kind) || left.id.localeCompare(right.id));
 }
-function transitionNavigatedRoutes(transitions) {
-    const routes = new Set();
-    const visit = (effect) => {
-        if (effect.kind === "navigate" &&
-            effect.to?.kind === "lit" &&
-            typeof effect.to.value === "string")
-            routes.add(effect.to.value);
-        if (effect.kind === "seq") {
-            for (const child of effect.effects)
-                visit(child);
-        }
-        if (effect.kind === "if") {
-            visit(effect.then);
-            visit(effect.else);
-        }
-    };
-    for (const transition of transitions)
-        visit(transition.effect);
-    return [...routes].sort();
-}
-function overApproxReasons(transition) {
+function overApproxReasons(transition, varDomains = new Map()) {
     const reasons = new Set();
     if (transition.id.endsWith(".escaped"))
         reasons.add("setter escaped to unanalyzed call");
-    for (const variable of havocWrites(transition.effect))
-        reasons.add(`havoc write to ${variable}`);
+    for (const variable of havocWrites(transition.effect)) {
+        const domain = varDomains.get(variable);
+        const prefix = domain?.kind === "bool" ? "safe local toggle" : "domain-wide havoc";
+        reasons.add(`${prefix}: havoc write to ${variable}`);
+    }
     if (reasons.size === 0)
         reasons.add("transition confidence is over-approx");
     return [...reasons].sort();
@@ -799,17 +960,28 @@ function normalizeId(id) {
 function editDistance(left, right) {
     const previous = Array.from({ length: right.length + 1 }, (_, index) => index);
     for (let leftIndex = 1; leftIndex <= left.length; leftIndex += 1) {
-        let diagonal = previous[0];
+        const startDiagonal = previous[0];
+        if (startDiagonal === undefined)
+            break;
+        let diagonal = startDiagonal;
         previous[0] = leftIndex;
         for (let rightIndex = 1; rightIndex <= right.length; rightIndex += 1) {
-            const up = previous[rightIndex] + 1;
-            const leftCost = previous[rightIndex - 1] + 1;
+            const upCell = previous[rightIndex];
+            const leftCell = previous[rightIndex - 1];
+            if (upCell === undefined || leftCell === undefined)
+                break;
+            const up = upCell + 1;
+            const leftCost = leftCell + 1;
             const subst = diagonal + (left[leftIndex - 1] === right[rightIndex - 1] ? 0 : 1);
-            diagonal = previous[rightIndex];
+            const corner = previous[rightIndex];
+            if (corner === undefined)
+                break;
+            diagonal = corner;
             previous[rightIndex] = Math.min(up, leftCost, subst);
         }
     }
-    return previous[right.length];
+    const distance = previous[right.length];
+    return distance ?? 0;
 }
 function havocWrites(effect) {
     if (effect.kind === "havoc")
@@ -820,9 +992,47 @@ function havocWrites(effect) {
         return [...havocWrites(effect.then), ...havocWrites(effect.else)];
     return [];
 }
+const GENERIC_UNEXTRACTABLE_CATEGORIES = new Set([
+    "no-extractable-effect",
+    "unextractable",
+]);
+function dedupeUnextractableHandlers(warnings) {
+    const parsed = warnings.map(unextractableHandlerFromWarning).filter((handler) => Boolean(handler));
+    const byId = new Map();
+    for (const handler of parsed) {
+        const existing = byId.get(handler.id);
+        if (!existing) {
+            byId.set(handler.id, handler);
+            continue;
+        }
+        const existingIsGeneric = GENERIC_UNEXTRACTABLE_CATEGORIES.has(existing.category);
+        const incomingIsGeneric = GENERIC_UNEXTRACTABLE_CATEGORIES.has(handler.category);
+        if (existingIsGeneric && !incomingIsGeneric)
+            byId.set(handler.id, handler);
+    }
+    return [...byId.values()]
+        .map(({ id, reason, source }) => source ? { id, reason, source } : { id, reason })
+        .sort(compareCaveats);
+}
 function unextractableHandlerFromWarning(warning) {
-    const match = /^Unextractable handler (.+)$/.exec(warning);
-    return match?.[1] ? { id: match[1], reason: warning } : undefined;
+    const rich = /^Unextractable handler (\S+) \[([^\]]+)\] \((.+)\)$/.exec(warning);
+    if (rich) {
+        const id = rich[1];
+        const category = rich[2];
+        const source = rich[3];
+        if (!id || !category || !source)
+            return undefined;
+        return {
+            id,
+            category,
+            reason: `${category} at ${source}`,
+            source,
+        };
+    }
+    const bare = /^Unextractable handler (\S+)$/.exec(warning);
+    return bare?.[1]
+        ? { id: bare[1], category: "unextractable", reason: bare[0] }
+        : undefined;
 }
 function pendingVars(effectApis, transitions = [], vars = [], maxPending = 3) {
     const enqueues = transitions.flatMap((transition) => enqueueOps(transition.effect));