modality-ts 0.0.16 → 0.0.18

package/dist/cli/features/extract/command.js

@@ -3,12 +3,13 @@ import { createHash } from "node:crypto";
 import { basename, dirname, extname, join, parse, relative, resolve, } from "node:path";
 import { pathToFileURL } from "node:url";
 import * as ts from "typescript";
-import { runExtractionPipeline } from "modality-ts/extract";
+import { runExtractionPipeline, } from "modality-ts/extract";
 import { canonicalJson, collectTokenDomainPaths, domainCardinality, parseModelArtifact, } from "modality-ts/core";
 import { parseReactRouterRoutes, routerSource, } from "modality-ts/extract/sources/router";
 import { emitAppModel } from "../../codegen/model.js";
 import { loadAndApplyOverlay, loadOverlaySpec } from "../../overlay.js";
 import { createBuiltinModalityRegistry } from "../../registry/index.js";
+import { sourceWithReachableImports, } from "./project.js";
 export async function runExtractCommand(options) {
     const sourcePaths = normalizedSourcePaths(options);
     const projectBase = await loadExtractionProject(sourcePaths);
@@ -31,7 +32,8 @@ export async function runExtractCommand(options) {
         routerPlugin: options.routerPlugin ?? config.routerPlugin,
     });
     const routerAdapter = registry.routerPlugin ?? routerSource();
-    const project = await attachRouteInventory(projectBase, routerAdapter);
+    const projectWithInventory = await attachRouteInventory(projectBase, routerAdapter);
+    const project = await buildClientProjectSurface(projectWithInventory, routerAdapter);
     const route = resolveExtractionRoute(project, config, options, sourcePaths);
     const routePatterns = project.inventory.routes.map((node) => node.pattern);
     const effectApis = uniqueStrings([
@@ -46,9 +48,7 @@ export async function runExtractCommand(options) {
         ...(config.bounds ?? {}),
         ...(options.bounds ?? {}),
     };
-    const pipeline = runExtractionPipeline({
-        sourceText: project.sourceText,
-        fileName: project.entryFile,
+    const pipeline = runProjectExtractionPipeline(project, {
         route,
         routePatterns,
         effectApis,
@@ -90,6 +90,7 @@ export async function runExtractCommand(options) {
         ].join("\n"));
     }
     const warnings = [
+        ...project.surfaceWarnings,
         ...pipeline.warnings,
         ...overlay.warnings,
         ...pluginConformanceWarnings(registry.sourcePlugins, dependencies),
@@ -102,7 +103,7 @@ export async function runExtractCommand(options) {
             extractionCaveats,
         },
     };
-    const report = createExtractionReport(project.sourceFiles, model, warnings, overlay.ignoredVars, options.now ?? new Date(), project.inventory);
+    const report = createExtractionReport(project.sourceFiles, model, warnings, overlay.ignoredVars, options.now ?? new Date(), project.inventory, buildEffectOperations(project.effectApiProvenance, config.effectApis, options.effectApis));
     await mkdir(dirname(options.modelPath), { recursive: true });
     await writeFile(options.modelPath, `${canonicalJson(model)}\n`, "utf8");
     await mkdir(dirname(appModelPath), { recursive: true });
@@ -204,25 +205,22 @@ async function loadExtractionProject(sourcePaths) {
     if (!resolved)
         throw new Error("extract requires at least one source path");
     const info = await stat(resolved);
+    const tsconfig = await readTsConfigResolution(info.isDirectory() ? resolved : dirname(resolved));
     if (!info.isDirectory()) {
         const source = await readFile(resolved, "utf8");
-        const tsconfig = await readTsConfigResolution(dirname(resolved));
-        const imported = await sourceWithLocalImports([{ path: resolved, text: source }], tsconfig);
-        return {
+        const rawEntries = [{ path: resolved, text: source }];
+        return emptySurfaceProject({
             entryFile: resolved,
-            sourceText: imported.sources.map((entry) => entry.text).join("\n"),
-            sourceFiles: imported.sources.map((entry) => entry.path),
-            sources: imported.sources,
-            inventory: { routes: [] },
-            effectApis: fetchEffectApis(imported.sources.map((entry) => entry.text).join("\n")),
+            rawEntries,
+            tsconfig,
             configStartDir: dirname(resolved),
-        };
+        });
     }
     const routesPath = join(resolved, "app", "routes.ts");
     const routeEntries = parseReactRouterRoutes(await readFile(routesPath, "utf8"));
     const rootPath = join(resolved, "app", "root.tsx");
     const roots = await existingFiles([rootPath]);
-    const entries = [
+    const rawEntries = [
         { path: routesPath, text: await readFile(routesPath, "utf8") },
         ...(await Promise.all(roots.map(async (path) => ({ path, text: await readFile(path, "utf8") })))),
         ...(await Promise.all(routeEntries.map(async (entry) => ({
@@ -230,44 +228,159 @@ async function loadExtractionProject(sourcePaths) {
             text: await readFile(resolve(dirname(routesPath), entry.file), "utf8"),
         })))),
     ];
-    const tsconfig = await readTsConfigResolution(resolved);
-    const imported = await sourceWithLocalImports(entries, tsconfig);
-    const sourceText = imported.sources.map((entry) => entry.text).join("\n");
-    return {
+    return emptySurfaceProject({
         entryFile: routesPath,
-        sourceText,
-        sourceFiles: imported.sources.map((entry) => entry.path),
-        sources: imported.sources,
-        inventory: { routes: [] },
-        effectApis: fetchEffectApis(sourceText),
+        rawEntries,
+        tsconfig,
         configStartDir: resolved,
+    });
+}
+function emptySurfaceProject(input) {
+    return {
+        entryFile: input.entryFile,
+        sourceText: "",
+        interactionSources: [],
+        sourceFiles: [],
+        sources: [],
+        inventory: { routes: [] },
+        effectApis: [],
+        effectApiProvenance: [],
+        surfaceWarnings: [],
+        configStartDir: input.configStartDir,
+        rawEntries: input.rawEntries,
+        tsconfig: input.tsconfig,
+    };
+}
+async function buildClientProjectSurface(project, adapter) {
+    const reachable = await sourceWithReachableImports(project.rawEntries, project.tsconfig, adapter, project.inventory);
+    const includedSources = reachable.sources.filter((entry) => entry.included);
+    const interactionSources = includedSources
+        .filter((entry) => entry.interactionText.trim().length > 0)
+        .map((entry) => ({ path: entry.path, text: entry.interactionText }))
+        .sort((left, right) => left.path.localeCompare(right.path));
+    const interactionSourcePaths = new Set(interactionSources.map((entry) => entry.path));
+    const reportSources = includedSources.filter((entry) => interactionSourcePaths.has(entry.path) ||
+        entry.path.endsWith("routes.ts"));
+    return {
+        ...project,
+        sourceText: interactionSources.map((entry) => entry.text).join("\n"),
+        interactionSources,
+        sourceFiles: reportSources
+            .map((entry) => entry.path)
+            .sort((left, right) => left.localeCompare(right)),
+        sources: reportSources.map((entry) => ({
+            path: entry.path,
+            text: entry.text,
+        })),
+        effectApis: reachable.effectApis,
+        effectApiProvenance: reachable.effectApiProvenance,
+        surfaceWarnings: reachable.warnings,
+    };
+}
+function runProjectExtractionPipeline(project, options) {
+    const fragments = project.interactionSources.length > 0
+        ? project.interactionSources
+        : project.sourceText.trim().length > 0
+            ? [{ path: project.entryFile, text: project.sourceText }]
+            : [];
+    if (fragments.length === 0) {
+        return runExtractionPipeline({
+            sourceText: "",
+            fileName: project.entryFile,
+            ...options,
+        });
+    }
+    const discoverFragments = fragments.map((entry) => ({
+        sourceText: entry.text,
+        fileName: entry.path,
+    }));
+    if (fragments.length === 1) {
+        const fragment = fragments[0];
+        return runExtractionPipeline({
+            sourceText: fragment.text,
+            fileName: fragment.path,
+            discoverFragments,
+            ...options,
+        });
+    }
+    return mergeExtractionPipelineResults(fragments.map((fragment) => runExtractionPipeline({
+        sourceText: fragment.text,
+        fileName: fragment.path,
+        discoverFragments,
+        ...options,
+        inventory: { routes: [] },
+    })), runExtractionPipeline({
+        sourceText: "",
+        fileName: project.entryFile,
+        ...options,
+    }));
+}
+function mergeExtractionPipelineResults(fragmentResults, inventoryResult) {
+    const transitionIds = new Set();
+    const transitions = [];
+    for (const result of fragmentResults) {
+        for (const transition of result.transitions) {
+            if (transitionIds.has(transition.id))
+                continue;
+            transitionIds.add(transition.id);
+            transitions.push(transition);
+        }
+    }
+    for (const transition of inventoryResult.transitions) {
+        if (transitionIds.has(transition.id))
+            continue;
+        transitionIds.add(transition.id);
+        transitions.push(transition);
+    }
+    const varIds = new Set();
+    const stateVars = [];
+    for (const result of fragmentResults) {
+        for (const decl of result.stateVars) {
+            if (varIds.has(decl.id))
+                continue;
+            varIds.add(decl.id);
+            stateVars.push(decl);
+        }
+    }
+    const writeChannelIds = new Set();
+    const writeChannels = [];
+    for (const result of fragmentResults) {
+        for (const channel of result.writeChannels) {
+            if (writeChannelIds.has(channel.id))
+                continue;
+            writeChannelIds.add(channel.id);
+            writeChannels.push(channel);
+        }
+    }
+    const templateFragments = fragmentResults.flatMap((result) => result.templateFragments);
+    const warnings = uniqueStrings(fragmentResults.flatMap((result) => [...result.warnings]));
+    return {
+        transitions,
+        warnings,
+        stateVars,
+        templateFragments,
+        routeVars: inventoryResult.routeVars,
+        writeChannels,
+        plugins: inventoryResult.plugins,
     };
 }
 async function loadMultiFileExtractionProject(sourcePaths) {
     const projects = await Promise.all(sourcePaths.map((sourcePath) => loadExtractionProject([sourcePath])));
-    const sourcesByPath = new Map();
+    const rawEntriesByPath = new Map();
     for (const project of projects) {
-        for (const source of project.sources) {
-            sourcesByPath.set(source.path, source);
+        for (const entry of project.rawEntries) {
+            rawEntriesByPath.set(resolve(entry.path), entry);
         }
     }
-    const sources = [...sourcesByPath.values()].sort((left, right) => left.path.localeCompare(right.path));
-    const sourceText = sources.map((entry) => entry.text).join("\n");
-    return {
+    return emptySurfaceProject({
         entryFile: projects.map((project) => project.entryFile).join(","),
-        sourceText,
-        sourceFiles: sources.map((entry) => entry.path),
-        sources,
-        inventory: { routes: [] },
-        effectApis: uniqueStrings([
-            ...projects.flatMap((project) => project.effectApis),
-            ...fetchEffectApis(sourceText),
-        ]),
+        rawEntries: [...rawEntriesByPath.values()].sort((left, right) => left.path.localeCompare(right.path)),
+        tsconfig: projects[0]?.tsconfig ?? { paths: [] },
         configStartDir: commonAncestor(projects.map((project) => project.configStartDir)),
-    };
+    });
 }
 async function attachRouteInventory(project, adapter) {
-    const files = [...project.sources];
+    const files = [...project.rawEntries];
     const manifestPath = files.find((file) => file.path.endsWith("routes.ts"))?.path ??
         (await findNearestRoutesManifest(project.configStartDir));
     if (manifestPath &&
@@ -282,10 +395,11 @@ async function attachRouteInventory(project, adapter) {
         files,
         readFile: (path) => readFile(path, "utf8"),
     });
-    return { ...project, sources: files, inventory };
+    return { ...project, rawEntries: files, inventory };
 }
 function findManifestPath(project) {
-    return project.sources.find((file) => file.path.endsWith("routes.ts"))?.path;
+    return project.rawEntries.find((file) => file.path.endsWith("routes.ts"))
+        ?.path;
 }
 function resolveProjectRoot(manifestPath, fallback) {
     if (!manifestPath)
@@ -317,7 +431,7 @@ function manifestRouteFiles(project) {
     const manifestPath = findManifestPath(project);
     if (!manifestPath)
         return [];
-    const manifest = project.sources.find((file) => resolve(file.path) === resolve(manifestPath));
+    const manifest = project.rawEntries.find((file) => resolve(file.path) === resolve(manifestPath));
     if (!manifest)
         return [];
     const manifestDir = dirname(manifestPath);
@@ -384,105 +498,6 @@ async function findNearestRoutesManifest(startDir) {
     }
     return undefined;
 }
-async function sourceWithLocalImports(entries, tsconfig) {
-    const seen = new Set();
-    const sources = [];
-    const queue = [...entries];
-    while (queue.length > 0) {
-        const next = queue.shift();
-        if (!next)
-            break;
-        const canonical = resolve(next.path);
-        if (seen.has(canonical))
-            continue;
-        seen.add(canonical);
-        sources.push({ path: canonical, text: next.text });
-        for (const specifier of localImportSpecifiers(next.text)) {
-            const imported = await resolveImportPath(dirname(canonical), specifier, tsconfig);
-            if (imported)
-                queue.push({ path: imported, text: await readFile(imported, "utf8") });
-        }
-    }
-    return { sources };
-}
-function localImportSpecifiers(source) {
-    const specs = [];
-    const parsed = tsCreateSourceFile(source);
-    const visit = (node) => {
-        if (tsIsImportDeclaration(node) &&
-            tsIsStringLiteral(node.moduleSpecifier) &&
-            isLocalImportSpecifier(node.moduleSpecifier.text)) {
-            specs.push(node.moduleSpecifier.text);
-        }
-        tsForEachChild(node, visit);
-    };
-    visit(parsed);
-    return specs;
-}
-function tsCreateSourceFile(source) {
-    return ts.createSourceFile("imports.tsx", source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX);
-}
-function tsIsImportDeclaration(node) {
-    return ts.isImportDeclaration(node);
-}
-function tsIsStringLiteral(node) {
-    return ts.isStringLiteral(node);
-}
-function tsForEachChild(node, cb) {
-    ts.forEachChild(node, cb);
-}
-function isLocalImportSpecifier(specifier) {
-    return specifier.startsWith(".") || specifier.startsWith("~/");
-}
-async function resolveImportPath(baseDir, specifier, tsconfig) {
-    if (specifier.startsWith("./+types/") || specifier.startsWith("../+types/"))
-        return undefined;
-    const bases = importBases(baseDir, specifier, tsconfig);
-    for (const base of bases) {
-        const resolved = await firstExistingModulePath(base);
-        if (resolved)
-            return resolved;
-    }
-    return undefined;
-}
-function importBases(baseDir, specifier, tsconfig) {
-    if (specifier.startsWith("."))
-        return [resolve(baseDir, specifier)];
-    const matches = tsconfig.paths.flatMap((entry) => {
-        if (!specifier.startsWith(entry.prefix) ||
-            !specifier.endsWith(entry.suffix))
-            return [];
-        const star = specifier.slice(entry.prefix.length, specifier.length - entry.suffix.length);
-        return entry.targets.map((target) => resolve(target.replace("*", star)));
-    });
-    if (matches.length > 0)
-        return matches;
-    return tsconfig.baseUrl ? [resolve(tsconfig.baseUrl, specifier)] : [];
-}
-async function firstExistingModulePath(base) {
-    const candidates = /\.[cm]?[jt]sx?$/.test(base)
-        ? [base]
-        : [
-            `${base}.ts`,
-            `${base}.tsx`,
-            `${base}.mts`,
-            `${base}.cts`,
-            join(base, "index.ts"),
-            join(base, "index.tsx"),
-        ];
-    for (const candidate of candidates) {
-        try {
-            const candidateStat = await stat(candidate);
-            if (candidateStat.isFile())
-                return candidate;
-        }
-        catch (error) {
-            if (error.code !== "ENOENT")
-                throw error;
-        }
-    }
-    return undefined;
-}
 async function readTsConfigResolution(startDir) {
     const tsconfigPath = await findNearestTsConfig(startDir);
     if (!tsconfigPath)
@@ -540,55 +555,6 @@ async function existingFiles(paths) {
 function sourceHashes(sources) {
     return Object.fromEntries(sources.map((source) => [source.path, sha256(source.text)]));
 }
-function fetchEffectApis(sourceText) {
-    const source = tsCreateSourceFile(sourceText);
-    const ops = new Set();
-    const visit = (node) => {
-        if (ts.isCallExpression(node) &&
-            ts.isIdentifier(node.expression) &&
-            node.expression.text === "fetch") {
-            const op = fetchOpId(node);
-            if (op)
-                ops.add(op);
-        }
-        tsForEachChild(node, visit);
-    };
-    visit(source);
-    return [...ops].sort();
-}
-function fetchOpId(call) {
-    const first = call.arguments[0];
-    if (!first)
-        return undefined;
-    const path = fetchPathValue(first);
-    if (!path)
-        return undefined;
-    const method = fetchMethodValue(call.arguments[1]) ?? "GET";
-    return `${method} ${path}`;
-}
-function fetchPathValue(expression) {
-    if (ts.isStringLiteral(expression) ||
-        ts.isNoSubstitutionTemplateLiteral(expression))
-        return normalizeFetchPath(expression.text);
-    if (ts.isTemplateExpression(expression)) {
-        let value = expression.head.text;
-        for (const span of expression.templateSpans)
-            value += `:id${span.literal.text}`;
-        return normalizeFetchPath(value);
-    }
-    return undefined;
-}
-function normalizeFetchPath(path) {
-    return (path.startsWith("/") ? path : `/${path}`).replace(/\/:param(?=\/|$)/g, "/:id");
-}
-function fetchMethodValue(expression) {
-    if (!expression || !ts.isObjectLiteralExpression(expression))
-        return undefined;
-    const method = expression.properties.find((property) => ts.isPropertyAssignment(property) &&
-        propertyName(property.name) === "method");
-    const value = method ? literalString(method.initializer) : undefined;
-    return value?.toUpperCase();
-}
 function literalString(expression) {
     return ts.isStringLiteral(expression) ||
         ts.isNoSubstitutionTemplateLiteral(expression)
@@ -749,7 +715,7 @@ function buildStateContributors(model, limit = 20) {
         .sort((a, b) => b.bits - a.bits || a.source.localeCompare(b.source));
     return { totalBits, topVars, bySource };
 }
-function createExtractionReport(sourceFiles, model, warnings, ignoredVars, now, inventory) {
+function createExtractionReport(sourceFiles, model, warnings, ignoredVars, now, inventory, effectOperations) {
     const caveats = model.metadata?.extractionCaveats ?? emptyExtractionCaveats();
     const varDomains = new Map(model.vars.map((decl) => [decl.id, decl.domain]));
     const transitionHandlers = model.transitions.map((transition) => ({
@@ -814,8 +780,29 @@ function createExtractionReport(sourceFiles, model, warnings, ignoredVars, now,
             percentExactOrOverlay: handlers.length === 0 ? 1 : exactOrOverlay / handlers.length,
         },
         warnings,
+        ...(effectOperations && effectOperations.length > 0
+            ? { effectOperations }
+            : {}),
     };
 }
+function buildEffectOperations(provenance, configApis, optionApis) {
+    const entries = provenance.map((entry) => ({
+        opId: entry.opId,
+        source: entry.source.file,
+        line: entry.source.line,
+        column: entry.source.column,
+        origin: "source",
+    }));
+    for (const opId of configApis ?? []) {
+        entries.push({ opId, origin: "config" });
+    }
+    for (const opId of optionApis ?? []) {
+        entries.push({ opId, origin: "option" });
+    }
+    return entries.sort((left, right) => left.opId.localeCompare(right.opId) ||
+        (left.origin ?? "").localeCompare(right.origin ?? "") ||
+        (left.source ?? "").localeCompare(right.source ?? ""));
+}
 function buildRouteCoverage(inventory, model) {
     if (!inventory || inventory.routes.length === 0)
         return undefined;