carrick 0.3.92 → 0.3.94

package/sidecar/dist/src/retype.js

@@ -25,6 +25,7 @@
  * project other requests read is the project the scan loaded.
  */
 import { Node, SyntaxKind, ts } from 'ts-morph';
+import { fileDiagnostics } from './unwidened.js';
 /** Names appended to a file that is not ours carry this prefix. */
 const PREFIX = '__carrick_';
 const WIRE = `${PREFIX}Wire`;
@@ -32,16 +33,19 @@ export class Retyper {
     project;
     inferrer;
     jsonWire;
+    topTypes;
     /**
      * `jsonWire` is the check phase's own JSON wire transform
-     * (`jsonWireDeclarations` in the capture bundle), handed in by the entry
-     * point so both judges read the wire the same way without this file
-     * crossing the bundle seam.
+     * (`jsonWireDeclarations` in the capture bundle) and `topTypes` its
+     * any/unknown walk, handed in by the entry point so both judges read the
+     * wire and a top type the same way without this file crossing the bundle
+     * seam.
      */
-    constructor(project, inferrer, jsonWire) {
+    constructor(project, inferrer, jsonWire, topTypes) {
         this.project = project;
         this.inferrer = inferrer;
         this.jsonWire = jsonWire;
+        this.topTypes = topTypes;
     }
     /**
      * Judge every item, spending at most `budgetMs`. Each item rebuilds the
@@ -93,6 +97,8 @@ export class Retyper {
         const plan = this.plan(call, producer, item.wire);
         if (typeof plan === 'string')
             return abstain(item, plan);
+        // Planned now: a check forgets every node of the file, `call` included.
+        const unwidened = this.unwidenedPlan(call, item);
         const original = sourceFile.getFullText();
         let pre = before.get(sourceFile);
         if (!pre) {
@@ -119,10 +125,21 @@ export class Retyper {
                 messages = new Map(plain.added.map((d) => [key(d), d.message]));
             }
         }
+        // carrick#1516: the published type is what the compiler inferred, with
+        // any literal the handler returns widened. When the handler's own return,
+        // read before that widening, raises nothing, the producer's type is wider
+        // than what it sends: its own verdict class, not a break. Anything short
+        // of a clean check (a diagnostic, an abstain) leaves the mismatch.
+        let outcome = 'mismatch';
+        if (unwidened) {
+            const narrowed = this.check(sourceFile, original, unwidened(true), pre);
+            if (narrowed.kind === 'checked' && narrowed.added.length === 0)
+                outcome = 'wider';
+        }
         const lineOf = lineIndex(original);
         return {
             item_id: item.item_id,
-            outcome: 'mismatch',
+            outcome,
             diagnostics: decisive.added.map((d) => ({
                 line: lineOf(d.start),
                 code: d.code,
@@ -130,6 +147,17 @@ export class Retyper {
             })),
         };
     }
+    /**
+     * The rewrite that states the producer's UNWIDENED return (carrick#1516),
+     * when the item carries one that can be stated at this call.
+     */
+    unwidenedPlan(call, item) {
+        const text = oneLine(item.producer_unwidened_type ?? '');
+        if (!text || text.includes('//') || text.includes('/*'))
+            return undefined;
+        const plan = this.plan(call, text, item.wire);
+        return typeof plan === 'string' ? undefined : plan;
+    }
     /**
      * How to state the producer's type at this call, or why it cannot be done.
      * Returns a builder so the wire and the declared form share one decision.
@@ -150,7 +178,8 @@ export class Retyper {
                 const text = stated(useWire);
                 const delta = text.length - (argEnd - argStart);
                 return {
-                    edits: [{ start: argStart, end: argEnd, text }],
+                    edits: [{ start: argStart, end: argEnd, text, statedAt: 0 }],
+                    stated: text,
                     wire: useWire && wire,
                     typeArgumentCall: { start: callStart, end: callEnd + delta },
                 };
@@ -164,7 +193,8 @@ export class Retyper {
             return (useWire) => {
                 const text = `<${stated(useWire)}>`;
                 return {
-                    edits: [{ start: at, end: at, text }],
+                    edits: [{ start: at, end: at, text, statedAt: 1 }],
+                    stated: stated(useWire),
                     wire: useWire && wire,
                     typeArgumentCall: { start: callStart, end: callEnd + text.length },
                 };
@@ -190,6 +220,7 @@ export class Retyper {
                 const text = stated(useWire);
                 return {
                     edits: edits.flatMap((edit) => edit(text)),
+                    stated: text,
                     wire: useWire && wire,
                 };
             };
@@ -249,12 +280,76 @@ export class Retyper {
                 added.push(mapped);
             }
             added.sort((a, b) => a.start - b.start || a.code - b.code);
+            const unresolved = this.unresolvedMembers(sourceFile, rewrite, added.length > 0);
+            if (unresolved)
+                return { kind: 'abstain', reason: unresolved };
             return { kind: 'checked', added };
         }
         finally {
             sourceFile.replaceWithText(original);
         }
     }
+    /**
+     * Why the stated type, as the consumer's program reads it, cannot be
+     * compared, or `undefined` when it can (carrick#1514). Read after the
+     * diagnostics diff, because the two top types fail in opposite directions.
+     *
+     * The producer's text is sent only when it holds no `any`/`unknown`, but
+     * the consumer's program is where it is read: under its compiler options,
+     * through the wire transform (a lib type whose `toJSON()` returns `any`),
+     * with its own declarations.
+     *
+     * - A member that reads as `unknown` makes reads fail that the producer's
+     *   type would pass, so any diagnostic may be ours. It is never compared.
+     * - A member that reads as `any` can hide a failure but never make one, so
+     *   the diagnostics the diff found stand; only an empty diff is not
+     *   compared, since it may be the `any` agreeing.
+     *
+     * Either way the reason names the member, as the check phase does for a
+     * published type.
+     *
+     * The walk's budget sentinel is dropped. The scanner's text screen
+     * (`contains_disqualifying_top_type`, which has no budget) found no
+     * `any`/`unknown` in the producer's text before sending it, so a walk that
+     * runs out of budget can only miss one the consumer's program made deeper
+     * than the budget allows; the compiler's diagnostics stand.
+     */
+    unresolvedMembers(sourceFile, rewrite, diagnosed) {
+        let shift = 0;
+        let at;
+        for (const edit of [...rewrite.edits].sort((a, b) => a.start - b.start)) {
+            if (at === undefined && edit.statedAt !== undefined) {
+                at = edit.start + shift + edit.statedAt;
+            }
+            shift += edit.text.length - (edit.end - edit.start);
+        }
+        const node = at === undefined
+            ? undefined
+            : sourceFile.getDescendantAtStartWithWidth(at, rewrite.stated.length);
+        // Every rewrite states the type at a recorded offset, so this is only
+        // reached through a wrong offset; it abstains rather than skip the walk.
+        if (!node)
+            return 'the stated producer type could not be found again after the rewrite';
+        const type = node.getType().compilerType;
+        let found;
+        if (type.flags & ts.TypeFlags.Unknown)
+            found = [{ kind: 'unknown', path: '' }];
+        else if (type.flags & ts.TypeFlags.Any)
+            found = [{ kind: 'any', path: '' }];
+        else {
+            const program = this.project.getProgram().compilerObject;
+            found = this.topTypes(type, program, program.getTypeChecker(), node.compilerNode);
+        }
+        const unknowns = found.filter((finding) => finding.kind === 'unknown');
+        const anys = found.filter((finding) => finding.kind === 'any');
+        const deciding = unknowns.length > 0 ? unknowns : diagnosed ? [] : anys;
+        if (deciding.length === 0)
+            return undefined;
+        const members = deciding
+            .map(({ kind, path }) => (path === '' ? `'${kind}'` : `'${kind}' at '${path}'`))
+            .join(', ');
+        return `the producer's response reads as ${members} in the consumer's program, so it was not compared`;
+    }
     /**
      * The stated type argument must be what the call RETURNS (or a type
      * argument or member of it): the rewrite is only a statement about the
@@ -738,7 +833,11 @@ function bodyReadEdit(read) {
             const type = parent.getTypeNodeOrThrow();
             const start = type.getStart();
             const end = type.getEnd();
-            return (stated) => [{ start, end, text: awaited ? stated : `Promise<${stated}>` }];
+            return (stated) => [
+                awaited
+                    ? { start, end, text: stated, statedAt: 0 }
+                    : { start, end, text: `Promise<${stated}>`, statedAt: 'Promise<'.length },
+            ];
         }
         else if (!Node.isParenthesizedExpression(parent))
             break;
@@ -750,22 +849,12 @@ function bodyReadEdit(read) {
         // The compiler reports past a parenthesis, so a finding on the cast read
         // lands on `res`, an original position, never on this one.
         { start, end: start, text: '(' },
-        { start: end, end, text: ` as Promise<${stated}>)` },
+        { start: end, end, text: ` as Promise<${stated}>)`, statedAt: ' as Promise<'.length },
     ];
 }
 function declaresTypeParameters(call) {
     return (resolvedDeclaration(call)?.typeParameters?.length ?? 0) > 0;
 }
-function fileDiagnostics(sourceFile) {
-    const program = sourceFile.getProject().getProgram().compilerObject;
-    const node = sourceFile.compilerNode;
-    return [...program.getSyntacticDiagnostics(node), ...program.getSemanticDiagnostics(node)]
-        .filter((d) => d.file === node && d.start !== undefined)
-        .map((d) => ({ start: d.start, code: d.code, message: flatten(d.messageText) }));
-}
-function flatten(text) {
-    return ts.flattenDiagnosticMessageText(text, ' ');
-}
 function key(d) {
     return `${d.start}:${d.code}`;
 }

package/sidecar/dist/src/type-inferrer.d.ts

@@ -57,6 +57,11 @@ export interface TypeInferrerOptions {
      * `node_modules` symlink from an installed dependency (carrick#1264).
      */
     repoRoot: string;
+    /**
+     * How long the unwidened reading of one `infer` batch may take
+     * (carrick#1516). Defaults to `UNWIDENED_BUDGET_MS`.
+     */
+    unwidenedBudgetMs?: number;
 }
 /**
  * TypeInferrer - Extracts types from source code, both explicit and inferred
@@ -69,6 +74,13 @@ export declare class TypeInferrer {
     private readonly project;
     private readonly packageOf;
     private readonly repoRoot;
+    /**
+     * The node each inferred type was read from, keyed by the location object
+     * `getNodeLocation` built for it (the one an `InferredType` carries). The
+     * unwidened reading re-reads from here (carrick#1516).
+     */
+    private readonly readNodes;
+    private readonly unwidenedBudgetMs;
     constructor(options: TypeInferrerOptions);
     /**
      * What the structural printer needs to tell the user's own declarations from
@@ -85,6 +97,27 @@ export declare class TypeInferrer {
      * @returns InferResult with inferred types or errors
      */
     infer(requests: InferRequestItem[], extractionConfig?: ExtractionConfig): InferResult;
+    /**
+     * carrick#1516: read each response inference again with the literals on its
+     * handler's path marked `as const`, and record the narrower type the handler
+     * really returns beside the published one (`unwidened_type_string`). See
+     * `unwidened.ts` for what is marked and why the reading is sound.
+     *
+     * One rewrite for the whole batch: every file on any request's path is
+     * rewritten once, type-checked before and after, the requests re-inferred,
+     * and every file restored before returning, so the project other requests
+     * read is the project the scan loaded.
+     *
+     * Nothing is read past `deadline`: a batch that runs out keeps the readings
+     * it made and publishes none for the rest.
+     */
+    private addUnwidenedReadings;
+    /**
+     * The request as it locates in the rewritten files: a span moved past the
+     * insertions before it, an expression text replaced by the rewritten text of
+     * the node it named. Lines do not move (an insertion holds no newline).
+     */
+    private mappedRequest;
     /**
      * Infer a single type from a request
      */

package/sidecar/dist/src/type-inferrer.js

@@ -21,6 +21,7 @@ import * as path from 'node:path';
 import { Node, SyntaxKind, ts, } from 'ts-morph';
 import { validateInferRequestItem } from './validators.js';
 import { isExternalOrigin } from './origin.js';
+import { addedDiagnostics, applyInsertions, fileDiagnostics, literalInsertions, mapBack, mapForward, normalise, pathOf, } from './unwidened.js';
 import { expandTypeStructural, } from './type-structural-expander.js';
 /**
  * TS/lib globals and primitives that must never be emitted as a deterministic
@@ -230,6 +231,18 @@ const TYPE_TEXT_FLAGS = ts.TypeFormatFlags.NoTruncation | ts.TypeFormatFlags.InT
 function typeText(type, enclosingNode) {
     return type.getText(enclosingNode, TYPE_TEXT_FLAGS);
 }
+/**
+ * How long the unwidened reading of one batch may take by default. The
+ * reading adds a field and never an answer, so running out costs only the
+ * readings not yet made; the rewrite is always undone.
+ */
+const UNWIDENED_BUDGET_MS = 120_000;
+/**
+ * No reading starts or continues past this long after the batch began: the
+ * scanner's read deadline for one request is 900s, and the inferences the
+ * batch already made must reach it.
+ */
+const UNWIDENED_LATEST_MS = 600_000;
 /**
  * TypeInferrer - Extracts types from source code, both explicit and inferred
  *
@@ -241,10 +254,18 @@ export class TypeInferrer {
     project;
     packageOf;
     repoRoot;
+    /**
+     * The node each inferred type was read from, keyed by the location object
+     * `getNodeLocation` built for it (the one an `InferredType` carries). The
+     * unwidened reading re-reads from here (carrick#1516).
+     */
+    readNodes = new WeakMap();
+    unwidenedBudgetMs;
     constructor(options) {
         this.project = options.project;
         this.packageOf = options.packageOf;
         this.repoRoot = options.repoRoot;
+        this.unwidenedBudgetMs = options.unwidenedBudgetMs ?? UNWIDENED_BUDGET_MS;
     }
     /**
      * What the structural printer needs to tell the user's own declarations from
@@ -266,8 +287,11 @@ export class TypeInferrer {
      * @returns InferResult with inferred types or errors
      */
     infer(requests, extractionConfig) {
+        const started = performance.now();
         const inferredTypes = [];
         const errors = [];
+        /** Response inferences the unwidened reading re-reads (carrick#1516). */
+        const responses = [];
         for (const request of requests) {
             // Plain JavaScript has no type annotations to extract, and `checkJs` is
             // off, so inferring against a `.js` file yields nothing useful — it only
@@ -287,6 +311,9 @@ export class TypeInferrer {
                 const result = this.inferSingle(request, extractionConfig);
                 if (result) {
                     inferredTypes.push(result);
+                    if (request.infer_kind === 'response_body' || request.infer_kind === 'function_return') {
+                        responses.push({ request, result });
+                    }
                 }
                 else {
                     errors.push(`Could not infer type at ${request.file_path}:${loc} (${request.infer_kind})`);
@@ -298,12 +325,193 @@ export class TypeInferrer {
                 errors.push(`Error inferring type at ${request.file_path}:${loc}: ${error}`);
             }
         }
+        try {
+            const deadline = Math.min(performance.now() + this.unwidenedBudgetMs, started + UNWIDENED_LATEST_MS);
+            this.addUnwidenedReadings(responses, extractionConfig, deadline);
+        }
+        catch (err) {
+            // The reading only ever adds a field; it never costs an answer.
+            this.logError(`Unwidened reading failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
         return {
             success: errors.length === 0 || inferredTypes.length > 0,
             inferred_types: inferredTypes.length > 0 ? inferredTypes : undefined,
             errors: errors.length > 0 ? errors : undefined,
         };
     }
+    /**
+     * carrick#1516: read each response inference again with the literals on its
+     * handler's path marked `as const`, and record the narrower type the handler
+     * really returns beside the published one (`unwidened_type_string`). See
+     * `unwidened.ts` for what is marked and why the reading is sound.
+     *
+     * One rewrite for the whole batch: every file on any request's path is
+     * rewritten once, type-checked before and after, the requests re-inferred,
+     * and every file restored before returning, so the project other requests
+     * read is the project the scan loaded.
+     *
+     * Nothing is read past `deadline`: a batch that runs out keeps the readings
+     * it made and publishes none for the rest.
+     */
+    addUnwidenedReadings(responses, extractionConfig, deadline) {
+        const outOfTime = (stage) => {
+            if (performance.now() <= deadline)
+                return false;
+            this.log(`Unwidened reading stopped ${stage}: it ran out of its budget`);
+            return true;
+        };
+        const plans = [];
+        const insertionsByFile = new Map();
+        const checkedFiles = new Set();
+        // Every position is read before the first rewrite: a rewrite forgets
+        // every node of its file.
+        for (const { request, result } of responses) {
+            // A declared return type is the contract; nothing was widened.
+            if (result.is_explicit)
+                continue;
+            const read = this.readNodes.get(result.source_location);
+            if (!read)
+                continue;
+            const functions = pathOf(read);
+            let marked = 0;
+            for (const fn of functions) {
+                const insertions = literalInsertions(fn);
+                marked += insertions.length;
+                if (insertions.length === 0)
+                    continue;
+                const file = fn.getSourceFile();
+                insertionsByFile.set(file, [...(insertionsByFile.get(file) ?? []), ...insertions]);
+            }
+            // Nothing on the path to mark: the reading would equal the published type.
+            if (marked === 0)
+                continue;
+            const path = functions.map((fn) => ({
+                file: fn.getSourceFile(),
+                start: fn.getStart(),
+                end: fn.getEnd(),
+            }));
+            for (const fn of path)
+                checkedFiles.add(fn.file);
+            let locatedByText;
+            if (request.expression_text && request.span_start === undefined) {
+                const sourceFile = this.getSourceFile(request.file_path);
+                const located = sourceFile
+                    ? this.findNodeByText(sourceFile, request.expression_text, request.expression_line)
+                    : undefined;
+                if (located)
+                    locatedByText = { start: located.getStart(), end: located.getEnd() };
+            }
+            plans.push({
+                request,
+                result,
+                read: { file: read.getSourceFile(), start: read.getStart(), end: read.getEnd() },
+                path,
+                locatedByText,
+            });
+        }
+        if (plans.length === 0)
+            return;
+        const insertions = new Map();
+        for (const [file, list] of insertionsByFile)
+            insertions.set(file, normalise(list));
+        const before = new Map();
+        for (const file of checkedFiles) {
+            if (outOfTime('before the rewrite'))
+                return;
+            before.set(file, fileDiagnostics(file));
+        }
+        const originals = new Map();
+        for (const file of insertions.keys())
+            originals.set(file, file.getFullText());
+        try {
+            for (const [file, list] of insertions) {
+                file.replaceWithText(applyInsertions(originals.get(file), list));
+            }
+            // A diagnostic the marking added sits in a function on some request's
+            // path (or elsewhere in that function's file): every request whose path
+            // holds it loses its reading.
+            const tainted = new Set();
+            for (const file of checkedFiles) {
+                if (outOfTime('while checking the rewrite'))
+                    return;
+                const list = insertions.get(file) ?? [];
+                for (const at of addedDiagnostics(before.get(file), fileDiagnostics(file), list)) {
+                    const holders = plans.flatMap((plan) => plan.path).filter((fn) => fn.file === file && fn.start <= at && at < fn.end);
+                    for (const plan of plans) {
+                        const onPath = plan.path.some((fn) => holders.length > 0 ? holders.includes(fn) : fn.file === file);
+                        if (onPath)
+                            tainted.add(plan);
+                    }
+                }
+            }
+            for (const plan of plans) {
+                if (outOfTime('while re-reading'))
+                    return;
+                if (tainted.has(plan)) {
+                    this.log(`Unwidened reading for ${plan.request.file_path}:${plan.request.line_number} ` +
+                        'dropped: marking its literals added a diagnostic on its path');
+                    continue;
+                }
+                const request = this.mappedRequest(plan.request, plan.locatedByText, insertions);
+                let again = null;
+                try {
+                    again = this.inferSingle(request, extractionConfig);
+                }
+                catch {
+                    again = null;
+                }
+                if (!again || again.type_string === plan.result.type_string)
+                    continue;
+                // An invariant, not a case: `mappedRequest` is what keeps the re-read
+                // on the node the inference read, so no fixture reaches this. It is
+                // what turns a locator the mapping got wrong into no reading instead
+                // of another node's type published as this handler's.
+                const readAgain = this.readNodes.get(again.source_location);
+                const list = insertions.get(plan.read.file) ?? [];
+                const sameNode = readAgain !== undefined &&
+                    readAgain.getSourceFile() === plan.read.file &&
+                    mapBack(readAgain.getStart(), list) === plan.read.start &&
+                    mapBack(readAgain.getEnd(), list) === plan.read.end;
+                if (!sameNode) {
+                    this.log(`Unwidened reading for ${plan.request.file_path}:${plan.request.line_number} ` +
+                        'dropped: the re-read did not read the node the inference read');
+                    continue;
+                }
+                plan.result.unwidened_type_string = again.type_string;
+            }
+        }
+        finally {
+            for (const [file, text] of originals)
+                file.replaceWithText(text);
+        }
+    }
+    /**
+     * The request as it locates in the rewritten files: a span moved past the
+     * insertions before it, an expression text replaced by the rewritten text of
+     * the node it named. Lines do not move (an insertion holds no newline).
+     */
+    mappedRequest(request, locatedByText, insertions) {
+        const sourceFile = this.getSourceFile(request.file_path);
+        const list = sourceFile ? insertions.get(sourceFile) ?? [] : [];
+        if (list.length === 0)
+            return request;
+        if (request.span_start !== undefined && request.span_end !== undefined) {
+            return {
+                ...request,
+                span_start: mapForward(request.span_start, list, 'start'),
+                span_end: mapForward(request.span_end, list, 'end'),
+            };
+        }
+        if (locatedByText && sourceFile) {
+            return {
+                ...request,
+                expression_text: sourceFile
+                    .getFullText()
+                    .slice(mapForward(locatedByText.start, list, 'start'), mapForward(locatedByText.end, list, 'end')),
+            };
+        }
+        return request;
+    }
     /**
      * Infer a single type from a request
      */
@@ -5421,13 +5629,15 @@ export class TypeInferrer {
     getNodeLocation(node) {
         const startLinePos = node.getStartLineNumber();
         const endLinePos = node.getEndLineNumber();
-        return {
+        const location = {
             file_path: node.getSourceFile().getFilePath(),
             start_line: startLinePos,
             end_line: endLinePos,
             start_column: node.getStart() - node.getStartLinePos(),
             end_column: node.getEnd() - node.getStartLinePos(),
         };
+        this.readNodes.set(location, node);
+        return location;
     }
     createInferredType(request, typeString, isExplicit, sourceLocation, payloadTypeString, primaryTypeSymbol, arrayDepth, primaryTypeSymbolSource) {
         const alias = request.alias ||

package/sidecar/dist/src/types.d.ts

@@ -300,6 +300,12 @@ export interface RetypeItem {
     expression_line?: number;
     /** The producer's response type as TypeScript text, fully inlined. */
     producer_type: string;
+    /**
+     * The producer's response as its handler returns it, literals read before
+     * TypeScript widens them (carrick#1516), when that differs from
+     * `producer_type`. Asked only when `producer_type` raised diagnostics.
+     */
+    producer_unwidened_type?: string;
     /** Judge the form JSON puts on the wire (an `http` response). */
     wire: boolean;
 }
@@ -511,11 +517,14 @@ export interface RetypeDiagnostic {
  * - `mismatch`: the rewrite added diagnostics; each is a place the consumer
  *   uses something the producer's response does not provide.
  * - `agrees`: it added none.
+ * - `wider`: the published type added diagnostics and the handler's
+ *   unwidened return added none (carrick#1516): the producer's type is wider
+ *   than what it sends. `diagnostics` are the published type's.
  * - `abstain`: the check could not be made; `reason` says why.
  */
 export interface RetypeOutcome {
     item_id: string;
-    outcome: 'mismatch' | 'agrees' | 'abstain';
+    outcome: 'mismatch' | 'agrees' | 'wider' | 'abstain';
     diagnostics: RetypeDiagnostic[];
     reason?: string;
 }
@@ -615,6 +624,16 @@ export interface InferredType {
      * Sorted by `path`; absent (not empty) when the type carries no top type.
      */
     any_provenance?: TypeProvenance[];
+    /**
+     * carrick#1516, response inferences only: the same inference re-read with
+     * every literal on the handler's path kept at its literal type
+     * (`unwidened.ts`). `type_string` is what the compiler infers and what the
+     * index publishes; this is what the handler actually sends when TypeScript
+     * widened a literal in it (`scope: string` published, `scope: 'all' |
+     * 'specific'` sent). Absent when the two are the same, or when the reading
+     * was dropped as unsound.
+     */
+    unwidened_type_string?: string;
 }
 /**
  * Why a type carries `any`/`unknown` at a position, and where.