carrick 0.3.106 → 0.3.107

package/sidecar/dist/src/function-line-index.js

@@ -0,0 +1,162 @@
+/**
+ * The function a line names, from an index built once per source file
+ * (carrick#1915).
+ *
+ * A signature request carries only the line its function starts on, and a
+ * signature pass sends one request per unannotated slot: thousands of
+ * lookups, several per function and many per file. Answering each by walking
+ * the file's every node cost the file's size per slot, and was 81% of the
+ * pass on a 2,345-file program. Here the file is walked once, over the
+ * compiler's own nodes, and each lookup reads the few entries near its line.
+ *
+ * The index belongs to the compiler's source file node, not to the file's
+ * name: replacing a file's text gives it a new node, so a file rewritten for
+ * one reading (the unwidened reading, the retype check) and then restored is
+ * indexed again each time, and never answered from positions it no longer has.
+ */
+import { SyntaxKind, ts, } from 'ts-morph';
+import { lineIndex } from './line-index.js';
+/** How far, in lines, a function may start from the line that names it. */
+const LINE_TOLERANCE = 2;
+const FUNCTION_KINDS = new Set([
+    SyntaxKind.FunctionDeclaration,
+    SyntaxKind.ArrowFunction,
+    SyntaxKind.FunctionExpression,
+    SyntaxKind.MethodDeclaration,
+]);
+/** The kinds ts-morph's `Node.isStatement` accepts. */
+const STATEMENT_KINDS = new Set([
+    SyntaxKind.Block,
+    SyntaxKind.BreakStatement,
+    SyntaxKind.ClassDeclaration,
+    SyntaxKind.ContinueStatement,
+    SyntaxKind.DebuggerStatement,
+    SyntaxKind.DoStatement,
+    SyntaxKind.EmptyStatement,
+    SyntaxKind.EnumDeclaration,
+    SyntaxKind.ExportAssignment,
+    SyntaxKind.ExportDeclaration,
+    SyntaxKind.ExpressionStatement,
+    SyntaxKind.ForInStatement,
+    SyntaxKind.ForOfStatement,
+    SyntaxKind.ForStatement,
+    SyntaxKind.FunctionDeclaration,
+    SyntaxKind.IfStatement,
+    SyntaxKind.ImportDeclaration,
+    SyntaxKind.ImportEqualsDeclaration,
+    SyntaxKind.InterfaceDeclaration,
+    SyntaxKind.LabeledStatement,
+    SyntaxKind.ModuleBlock,
+    SyntaxKind.ModuleDeclaration,
+    SyntaxKind.NotEmittedStatement,
+    SyntaxKind.ReturnStatement,
+    SyntaxKind.SwitchStatement,
+    SyntaxKind.ThrowStatement,
+    SyntaxKind.TryStatement,
+    SyntaxKind.TypeAliasDeclaration,
+    SyntaxKind.VariableStatement,
+    SyntaxKind.WhileStatement,
+    SyntaxKind.WithStatement,
+]);
+const indexes = new WeakMap();
+/**
+ * The function whose declaration starts at, or within `LINE_TOLERANCE` lines
+ * of, the given line: the closest, then the smallest, then the first in the
+ * file. A function starting after the line is passed over when a statement
+ * that opens on the line or after it, on a line before the function's, does
+ * not contain the function. Undefined when no function is left.
+ *
+ * These are `TypeInferrer.findFunctionByLine`'s rules, which says why each is
+ * there; this is where they are computed.
+ */
+export function functionAtLine(sourceFile, line) {
+    if (!Number.isFinite(line))
+        return undefined;
+    const index = indexOf(sourceFile.compilerNode);
+    /** Statements opening inside the forward window. */
+    const windowStatements = [];
+    for (let at = Math.ceil(line); at <= line + LINE_TOLERANCE; at++) {
+        const opening = index.statementsByLine.get(at);
+        if (opening)
+            windowStatements.push(...opening);
+    }
+    const separatedFromAnchor = (fn) => windowStatements.some((statement) => statement.line < fn.line && !(statement.start <= fn.start && statement.end >= fn.end));
+    let best;
+    for (let at = Math.ceil(line - LINE_TOLERANCE); at <= line + LINE_TOLERANCE; at++) {
+        for (const fn of index.functionsByLine.get(at) ?? []) {
+            if (fn.line > line && separatedFromAnchor(fn))
+                continue;
+            if (best === undefined || isBetter(fn, best, line))
+                best = fn;
+        }
+    }
+    return best ? wrapped(sourceFile, best) : undefined;
+}
+/** Closer to the line, then smaller, then earlier in the file. */
+function isBetter(fn, best, line) {
+    const delta = Math.abs(fn.line - line);
+    const bestDelta = Math.abs(best.line - line);
+    if (delta !== bestDelta)
+        return delta < bestDelta;
+    const size = fn.end - fn.start;
+    const bestSize = best.end - best.start;
+    if (size !== bestSize)
+        return size < bestSize;
+    return fn.order < best.order;
+}
+function indexOf(file) {
+    let index = indexes.get(file);
+    if (!index) {
+        index = buildIndex(file);
+        indexes.set(file, index);
+    }
+    return index;
+}
+function buildIndex(file) {
+    // Lines as ts-morph's `getStartLineNumber` counts them, which is what the
+    // scanner's line numbers were matched against.
+    const lineAt = lineIndex(file.text);
+    const functionsByLine = new Map();
+    const statementsByLine = new Map();
+    let order = 0;
+    const visit = (node) => {
+        const isFunction = FUNCTION_KINDS.has(node.kind);
+        const isStatement = STATEMENT_KINDS.has(node.kind);
+        if (isFunction || isStatement) {
+            const start = node.getStart(file);
+            const extent = { start, end: node.end, line: lineAt(start) };
+            if (isFunction) {
+                pushAt(functionsByLine, extent.line, { ...extent, node, order: order++ });
+            }
+            if (isStatement) {
+                pushAt(statementsByLine, extent.line, extent);
+            }
+        }
+        ts.forEachChild(node, visit);
+    };
+    ts.forEachChild(file, visit);
+    return { functionsByLine, statementsByLine };
+}
+function pushAt(byLine, line, entry) {
+    const entries = byLine.get(line);
+    if (entries)
+        entries.push(entry);
+    else
+        byLine.set(line, [entry]);
+}
+/**
+ * The ts-morph node for an indexed function. Only this one is wrapped: the
+ * deepest node at the function's first character is the function or sits
+ * inside it, so its ancestors lead there, and the rest of the file's nodes are
+ * never given wrappers they would keep for the life of the project.
+ */
+function wrapped(sourceFile, fn) {
+    let node = sourceFile.getDescendantAtPos(fn.start);
+    while (node && node.compilerNode !== fn.node) {
+        node = node.getParent();
+    }
+    // An invariant, not a case: no shape is known where the climb misses. If
+    // one exists, the answer is still the function the index chose.
+    node ??= sourceFile.getFirstDescendant((descendant) => descendant.compilerNode === fn.node);
+    return node;
+}

package/sidecar/dist/src/index.d.ts

@@ -10,5 +10,10 @@
  * - stdout is ONLY for JSON responses
  * - stderr is for logging
  * - Process stays alive between requests (warm standby)
+ * - One request at a time. A handler that runs the compiler blocks the event
+ *   loop until it returns, so a request sent meanwhile waits in the pipe, and
+ *   nothing can cancel the one that is running. The scanner kills a process
+ *   that goes silent past its deadline; a long handler stays alive by writing
+ *   `progress` frames as it finishes units of work (writeProgress).
  */
 export {};

package/sidecar/dist/src/index.js

@@ -10,6 +10,11 @@
  * - stdout is ONLY for JSON responses
  * - stderr is for logging
  * - Process stays alive between requests (warm standby)
+ * - One request at a time. A handler that runs the compiler blocks the event
+ *   loop until it returns, so a request sent meanwhile waits in the pipe, and
+ *   nothing can cancel the one that is running. The scanner kills a process
+ *   that goes silent past its deadline; a long handler stays alive by writing
+ *   `progress` frames as it finishes units of work (writeProgress).
  */
 import * as path from 'node:path';
 import * as readline from 'node:readline';
@@ -21,6 +26,7 @@ import { DefinitionResolver } from './definition-resolver.js';
 import { captureStub, findDisqualifyingTopTypes, jsonWireDeclarations, runCheck, } from './capture/index.js';
 import { Retyper } from './retype.js';
 import { LibraryClaimsVerifier, httpCheck } from './library-claims.js';
+import { PROGRESS_INTERVAL_MS, atMostEvery } from './progress.js';
 // ===========================================================================
 // Module-level state
 // ===========================================================================
@@ -168,7 +174,11 @@ function handleBundle(request) {
         log(`Bundling ${request.symbols.length} symbol(s)`);
         // A symbol named by a path is bundled from the program of the project
         // that owns that file (carrick#1604).
-        const results = [...byProject(request.symbols, (symbol) => symbol.source_file)].map(([key, symbols]) => projectComponents(key).typeBundler.bundle(symbols));
+        const results = [...byProject(request.symbols, (symbol) => symbol.source_file)].map(([key, symbols]) => {
+            const { typeBundler } = projectComponents(key);
+            writeProgress(request.request_id, 'bundle', 'program ready');
+            return typeBundler.bundle(symbols);
+        });
         const result = results.length === 1 ? results[0] : mergeBundles(results);
         if (!result.success) {
             return {
@@ -284,7 +294,13 @@ async function handleCheckV2Async(request) {
 function handleInfer(request) {
     try {
         log(`Inferring ${request.requests.length} type(s)`);
-        const results = [...byProject(request.requests, (item) => item.file_path)].map(([key, items]) => projectComponents(key).typeInferrer.infer(items, request.extraction_config));
+        // One count for the request, whichever programs its items span.
+        let done = 0;
+        const report = atMostEvery(PROGRESS_INTERVAL_MS, () => writeProgress(request.request_id, 'infer', `${done} of ${request.requests.length}`));
+        const results = [...byProject(request.requests, (item) => item.file_path)].map(([key, items]) => projectComponents(key).typeInferrer.infer(items, request.extraction_config, () => {
+            done += 1;
+            report();
+        }));
         const result = results.length === 1
             ? results[0]
             : (() => {
@@ -388,6 +404,8 @@ function handleVerifyLibraryClaims(request) {
     const started = performance.now();
     try {
         const { claimsVerifier } = projectComponents();
+        // The verifier's budget starts now, and so does the scanner's wait.
+        writeProgress(request.request_id, 'verify_library_claims', 'program ready');
         const fromDir = path.resolve(projectLoader.getRepoRoot(), request.from_dir);
         log(`Verifying ${request.checks.length} library claim(s) from ${fromDir}`);
         const { semantics, modules } = claimsVerifier.run(fromDir, request.checks, request.budget_ms ?? SEMANTICS_BUDGET_MS);
@@ -530,9 +548,15 @@ function writeResponse(response) {
     process.stdout.write(json + '\n');
 }
 /**
- * Write a non-terminal progress/keepalive frame (async install protocol).
- * Distinct `status: 'progress'` so clients skip it and wait for the terminal
- * success/error frame.
+ * Write a non-terminal progress frame. Distinct `status: 'progress'` so
+ * clients skip it and wait for the terminal success/error frame; the scanner
+ * restarts its deadline for the request on each one (carrick#1914).
+ *
+ * A synchronous handler calls this between units of its work, with the event
+ * loop blocked. The frame still leaves at once: a write this small to a pipe
+ * with room in it completes inside the call, and the reader on the other end
+ * drains the pipe on its own thread, so there is room. It is written through
+ * the same stream as every answer, so a frame can never land inside one.
  */
 function writeProgress(requestId, phase, message) {
     process.stdout.write(JSON.stringify({ request_id: requestId, status: 'progress', phase, message }) + '\n');

package/sidecar/dist/src/line-index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * The 1-based line of a position in a text, counted by line feeds alone: one
+ * more than the `\n`s before the position. That is the count ts-morph's
+ * `getStartLineNumber` makes. A lone carriage return or a Unicode line
+ * separator ends a line for the compiler and not here.
+ *
+ * The text is read once; each lookup after that is a binary search.
+ */
+export declare function lineIndex(text: string): (pos: number) => number;

package/sidecar/dist/src/line-index.js

@@ -0,0 +1,26 @@
+/**
+ * The 1-based line of a position in a text, counted by line feeds alone: one
+ * more than the `\n`s before the position. That is the count ts-morph's
+ * `getStartLineNumber` makes. A lone carriage return or a Unicode line
+ * separator ends a line for the compiler and not here.
+ *
+ * The text is read once; each lookup after that is a binary search.
+ */
+export function lineIndex(text) {
+    const starts = [0];
+    for (let i = 0; i < text.length; i++)
+        if (text[i] === '\n')
+            starts.push(i + 1);
+    return (pos) => {
+        let lo = 0;
+        let hi = starts.length - 1;
+        while (lo < hi) {
+            const mid = (lo + hi + 1) >> 1;
+            if (starts[mid] <= pos)
+                lo = mid;
+            else
+                hi = mid - 1;
+        }
+        return lo + 1;
+    };
+}

package/sidecar/dist/src/progress.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Pacing for the `progress` frames a long request writes (carrick#1914).
+ */
+/**
+ * The least time between two progress frames of one request.
+ *
+ * The scanner's deadline for a request is how long the sidecar may stay
+ * silent about it, so a frame has to say that a unit of work finished, and
+ * has to be far more frequent than that deadline. It does not have to follow
+ * every unit: a batch can finish thousands in a second.
+ */
+export declare const PROGRESS_INTERVAL_MS = 1500;
+/**
+ * `report`, held to one call per `intervalMs`: the first call goes through at
+ * once, and a call made sooner than `intervalMs` after the last one that went
+ * through is dropped.
+ *
+ * It is called by the work, between units, and never by a timer. A handler
+ * that stops finishing units stops reporting, which is what the reader on the
+ * other end is waiting to notice.
+ */
+export declare function atMostEvery<Args extends unknown[]>(intervalMs: number, report: (...args: Args) => void, now?: () => number): (...args: Args) => void;

package/sidecar/dist/src/progress.js

@@ -0,0 +1,31 @@
+/**
+ * Pacing for the `progress` frames a long request writes (carrick#1914).
+ */
+/**
+ * The least time between two progress frames of one request.
+ *
+ * The scanner's deadline for a request is how long the sidecar may stay
+ * silent about it, so a frame has to say that a unit of work finished, and
+ * has to be far more frequent than that deadline. It does not have to follow
+ * every unit: a batch can finish thousands in a second.
+ */
+export const PROGRESS_INTERVAL_MS = 1500;
+/**
+ * `report`, held to one call per `intervalMs`: the first call goes through at
+ * once, and a call made sooner than `intervalMs` after the last one that went
+ * through is dropped.
+ *
+ * It is called by the work, between units, and never by a timer. A handler
+ * that stops finishing units stops reporting, which is what the reader on the
+ * other end is waiting to notice.
+ */
+export function atMostEvery(intervalMs, report, now = () => performance.now()) {
+    let last = Number.NEGATIVE_INFINITY;
+    return (...args) => {
+        const at = now();
+        if (at - last < intervalMs)
+            return;
+        last = at;
+        report(...args);
+    };
+}

package/sidecar/dist/src/retype.js

@@ -26,6 +26,7 @@
  */
 import { Node, SyntaxKind, ts } from 'ts-morph';
 import { readsResponseStatus, statusesIn, SUCCEEDED, testsOnPath, } from './failure-path.js';
+import { lineIndex } from './line-index.js';
 import { fileDiagnostics } from './unwidened.js';
 /** Names appended to a file that is not ours carry this prefix. */
 const PREFIX = '__carrick_';
@@ -842,22 +843,3 @@ function mapBack(pos, edits) {
     }
     return { kind: 'original', pos: pos - shift };
 }
-/** 1-based line of an original-file position. */
-function lineIndex(text) {
-    const starts = [0];
-    for (let i = 0; i < text.length; i++)
-        if (text[i] === '\n')
-            starts.push(i + 1);
-    return (pos) => {
-        let lo = 0;
-        let hi = starts.length - 1;
-        while (lo < hi) {
-            const mid = (lo + hi + 1) >> 1;
-            if (starts[mid] <= pos)
-                lo = mid;
-            else
-                hi = mid - 1;
-        }
-        return lo + 1;
-    };
-}

package/sidecar/dist/src/type-inferrer.d.ts

@@ -63,6 +63,11 @@ export interface TypeInferrerOptions {
      */
     unwidenedBudgetMs?: number;
 }
+/**
+ * Called each time an `infer` batch is done with one of its requests,
+ * whatever that request answered: skipped, refused, failed or inferred.
+ */
+export type InferRequestDone = () => void;
 /**
  * TypeInferrer - Extracts types from source code, both explicit and inferred
  *
@@ -94,9 +99,10 @@ export declare class TypeInferrer {
      *
      * @param requests - Array of inference requests
      * @param extractionConfig - Agent-generated extraction config for payload unwrapping
+     * @param onRequestDone - Called once per request, as the batch is done with it
      * @returns InferResult with inferred types or errors
      */
-    infer(requests: InferRequestItem[], extractionConfig?: ExtractionConfig): InferResult;
+    infer(requests: InferRequestItem[], extractionConfig?: ExtractionConfig, onRequestDone?: InferRequestDone): InferResult;
     /**
      * carrick#1836: list on `result` the declarations behind the names its text
      * prints that the request's file cannot resolve (`PrintedTypes.namesIn`).
@@ -350,12 +356,54 @@ export declare class TypeInferrer {
      */
     private resultCarrierPayload;
     /**
-     * `Future<T>` -> `T` for a promise-like of the source's own making, read off
-     * the await protocol rather than a name: a `then` whose first parameter is a
-     * callback, whose own first parameter is the value awaiting it yields.
-     * `Promise` and `PromiseLike` are peeled by `unwrapPromiseType` before this.
+     * The shape test of `resultCarrierPayload`: the carrier `type` is, once a
+     * promise-like around it is peeled, and the type arguments a branch of it
+     * holds as a member. `undefined` when `type` is not a carrier.
+     */
+    private resultCarrierArguments;
+    /**
+     * The decided abstain of a call whose result carries transport the
+     * service's wrapper rules verify and read no payload out of (carrick#1841,
+     * carrick#1843): `unknown` with `machinery_envelope` at the root and no
+     * anchor. The root reason is what keeps the capture's own locator from
+     * re-reading the raw call (`inference_decided_no_contract`,
+     * engine/type_compat_v2.rs).
+     */
+    private transportAbstain;
+    /**
+     * `Future<T>` -> `T` for a thenable, read off the await protocol rather
+     * than a name: a `then` whose first parameter is a callback, whose own
+     * first parameter is the value awaiting it yields. `Promise` and
+     * `PromiseLike` are peeled by `unwrapPromiseType` before this.
+     *
+     * The callback is read through `null` and `undefined` (carrick#1877). A
+     * `then` of the source's own making declares `(value: T) => void`; the
+     * platform's declares `onfulfilled?: ((value: T) => ...) | null`, and that
+     * is the `then` a subclass of `Promise` inherits. Read as written, an
+     * optional, nullable callback has no call signature, and the subclass was
+     * not seen as a thenable at all.
+     *
+     * A `then` that cannot be called, or whose first parameter is no callback,
+     * is no protocol, and the type is returned as it is.
+     *
+     * The compiler's own awaited type arbitrates. This walk reads the first
+     * signature of `then`; the language reads all of them. Where awaiting the
+     * type and awaiting what this walk found are not the same thing to the
+     * compiler (an overloaded `then` whose first signature is not the one
+     * `await` takes), the walk did not read the protocol, and the type is
+     * returned as it is rather than published as a guess.
      */
     private unwrapThenableType;
+    /** The value `then`'s first signature hands its callback, read until it stops changing. */
+    private firstThenValue;
+    /**
+     * What `await` yields for `type` where the names do not say: `type` is,
+     * once `Promise` and `PromiseLike` are peeled, a thenable by the protocol
+     * (a subclass of `Promise`, a class with a `then` of its own). `undefined`
+     * where the names say it all or `type` is no thenable, so a caller keeps
+     * the reading it had (carrick#1877).
+     */
+    private awaitedBeyondPromise;
     /**
      * The platform's error shape, in full: `name` and `message` strings AND a
      * `stack`, which is what the `Error` interface declares and every subclass
@@ -456,7 +504,17 @@ export declare class TypeInferrer {
      */
     private castsOfUnreadParameter;
     /**
-     * The zero-argument whole-body read that takes `identifier` as its receiver,
+     * The zero-argument whole-body read taken in place on the value `callExpr`
+     * yields, or `undefined` (carrick#1851): `(await fetch(url)).text()`. The
+     * receiver is the call itself, through the wrappers that leave a value as
+     * it is (parentheses, `await`, `!`), so it is the same read
+     * `bodyReadOnReceiver` finds on a binding of that value. A call that is not
+     * awaited first is read the same way: a request that is a promise and reads
+     * its own body (`send(url).json()`) yields the body from that read too.
+     */
+    private bodyReadOnCallValue;
+    /**
+     * The zero-argument whole-body read that takes `receiver` as its receiver,
      * `res.json()` or `res.text()`, or `undefined`. A text read is a body read
      * like a json one (carrick#1842): without it, `return res.text()` left the
      * walk on the response binding and published the transport object.
@@ -518,12 +576,27 @@ export declare class TypeInferrer {
      * Try to unwrap a type using a single ExtractionRule.
      */
     private tryUnwrapWithRule;
+    /**
+     * The names `type` goes by, own symbol first (carrick#1843).
+     *
+     * `type Task<A> = __Task<A>` has the class's symbol and arguments, and the
+     * alias's beside them. `type Reply<T> = { ... }` has the anonymous `__type`
+     * with no arguments of its own. `type Outcome<A, E> = Done<A, E> |
+     * Failed<A, E>` has no symbol of its own at all. In each, the alias and its
+     * arguments are what a rule naming `Task`, `Reply` or `Outcome` describes.
+     */
+    private wrapperReadings;
     /**
      * Extract the payload type from a matched wrapper. Returns null when the
      * rule matched the wrapper but no payload is recoverable from generics or
      * property paths — the caller decides what a payload-less match means
      * (verified machinery collapses to `unknown` after every rule has run;
      * a name-only match leaves the type untouched).
+     *
+     * `payloadGenericIndex` counts the arguments of `reading`, the name the
+     * rule matched (carrick#1843): an alias is free to order its parameters
+     * differently from the type it stands for, so the same index into the
+     * other list is a different argument.
      */
     private extractPayloadFromWrapper;
     /**
@@ -920,11 +993,19 @@ export declare class TypeInferrer {
     /**
      * Declaration file (absolute path) of the anchor symbol
      * `primaryTypeSymbol` reports for this type, or `undefined` when the type
-     * has no user-facing anchor or no source declaration. The scanner's
-     * pub/sub two-anchor arbitration (carrick#413) uses this to re-aim a
-     * demoted explicit bundle request: the bundler resolves a `SymbolRequest`
-     * only against declarations IN its `source_file`, so the request must
-     * point at the file that actually declares the tsc-witnessed payload type.
+     * has no user-facing anchor or no source declaration. Every path that
+     * reports the symbol reports this beside it (carrick#1819): it is where a
+     * reader finds the type an anchor names. The scanner's pub/sub two-anchor
+     * arbitration (carrick#413) also uses it to re-aim a demoted explicit
+     * bundle request: the bundler resolves a `SymbolRequest` only against
+     * declarations IN its `source_file`, so the request must point at the file
+     * that actually declares the tsc-witnessed payload type.
+     *
+     * The declarations read are those of the type's own symbol, so a name
+     * imported through a barrel reports the file that declares it, and a name
+     * two files declare reports the one this type resolves to. A declaration in
+     * an installed package or a TypeScript lib is reported as it is found, as
+     * an absolute path; what a reader is shown for it is the scanner's call.
      *
      * Only declaration kinds the bundler's `validateSymbols` can resolve
      * (interface, type alias, class, enum, function, variable) count. A