carrick 0.3.107 → 0.3.108

package/sidecar/dist/src/definition-resolver.js

@@ -26,19 +26,17 @@
  * inference path in `type-inferrer.ts`), which walks the resolved `Type` and
  * rebuilds the inlined text.
  *
- * Each resolve call builds its own throwaway in-memory project over the stub
- * tree, so the warm sidecar's long-lived project never sees stub files and
- * cannot accumulate stale trees across requests.
+ * Each resolve call builds its own throwaway project over the stub tree and
+ * reads nothing else: not the service's project, and not whether the sidecar
+ * was ever initialised. So the request costs what the stub costs, in a process
+ * that has built nothing as in one that has (carrick#1927), and the sidecar's
+ * long-lived project never sees stub files.
  */
 import * as path from 'node:path';
 import * as fs from 'node:fs';
 import { Project, Node } from 'ts-morph';
 import { expandTypeStructural, } from './type-structural-expander.js';
 export class DefinitionResolver {
-    project;
-    constructor(options) {
-        this.project = options.project;
-    }
     /**
      * Resolve surface aliases from a capture stub package directory
      * (`<stub_dir>/types/surface.d.ts` + its declaration tree).

package/sidecar/dist/src/index.js

@@ -27,6 +27,7 @@ import { captureStub, findDisqualifyingTopTypes, jsonWireDeclarations, runCheck,
 import { Retyper } from './retype.js';
 import { LibraryClaimsVerifier, httpCheck } from './library-claims.js';
 import { PROGRESS_INTERVAL_MS, atMostEvery } from './progress.js';
+import { mergeInferTimings } from './infer-timing.js';
 // ===========================================================================
 // Module-level state
 // ===========================================================================
@@ -38,6 +39,13 @@ let initTimeMs = null;
  * the service's files (carrick#1604).
  */
 let components = new Map();
+/**
+ * Reads a capture stub's own declaration tree and nothing of the init'd
+ * project, so it is not one of the project's components: taking it from them
+ * would build the service's whole program to answer about the stub
+ * (carrick#1927).
+ */
+const definitionResolver = new DefinitionResolver();
 /**
  * Get the project-backed components, building the project if this is the
  * first request that needs it.
@@ -67,7 +75,6 @@ function projectComponents(key = '') {
         built = {
             typeBundler: new TypeBundler({ project, repoRoot }),
             typeInferrer,
-            definitionResolver: new DefinitionResolver({ project }),
             // Locates calls exactly as `infer` does, so it rewrites the node the
             // consumer's published type came from (carrick#1491).
             // The walk is typed against the capture bundle's compiler copy and
@@ -218,6 +225,10 @@ function handleBundle(request) {
 function handleCaptureV2(request) {
     try {
         log(`capture_v2 for service '${request.service_name}' (${request.anchors.length} anchor(s))`);
+        // A frame for each stage as the capture reaches it, and within a stage
+        // (one report per anchor) at most one per interval.
+        let stage;
+        const withinStage = atMostEvery(PROGRESS_INTERVAL_MS, (phase, message) => writeProgress(request.request_id, phase, message));
         const result = captureStub({
             repoRoot: request.repo_root,
             serviceName: request.service_name,
@@ -225,6 +236,12 @@ function handleCaptureV2(request) {
             outDir: request.out_dir,
             tsconfigPath: request.tsconfig_path,
             scanRoot: request.scan_root,
+            onProgress: (phase, message) => {
+                if (phase === stage)
+                    return withinStage(phase, message);
+                stage = phase;
+                writeProgress(request.request_id, phase, message);
+            },
         });
         return {
             request_id: request.request_id,
@@ -316,6 +333,9 @@ function handleInfer(request) {
             request_id: request.request_id,
             status: result.success ? 'success' : 'error',
             inferred_types: result.inferred_types,
+            // Beside the answers and never in them: what the requests cost, for
+            // the caller's log (carrick#1985).
+            infer_timing: mergeInferTimings(results.map((r) => r.timing)),
             errors: result.errors,
         };
     }
@@ -345,11 +365,19 @@ function handleRetypeCheck(request) {
         const groups = byProject(request.items.map((item, index) => ({ item, index })), ({ item }) => item.file_path);
         const deadline = performance.now() + budget;
         const outcomes = new Array(request.items.length);
+        // One count for the request, whichever programs its items span, reported
+        // as `infer` reports its batch (carrick#1914, carrick#1945).
+        let judged = 0;
+        const report = atMostEvery(PROGRESS_INTERVAL_MS, () => writeProgress(request.request_id, 'retype', `${judged} of ${request.items.length}`));
+        const onJudged = () => {
+            judged += 1;
+            report();
+        };
         for (const [key, entries] of groups) {
             // One budget for the request, whichever programs it spans.
             const remaining = groups.size === 1 ? budget : Math.max(0, deadline - performance.now());
             projectComponents(key)
-                .retyper.run(entries.map(({ item }) => item), remaining)
+                .retyper.run(entries.map(({ item }) => item), remaining, onJudged)
                 .forEach((outcome, position) => {
                 outcomes[entries[position].index] = outcome;
             });
@@ -446,12 +474,13 @@ function handleListLibrarySurface(request) {
 }
 /**
  * Handle the 'resolve_definitions' action - resolve surface aliases from a
- * v2 capture stub package's declaration tree.
+ * v2 capture stub package's declaration tree. Stateless, as `capture_v2` is:
+ * it needs no init and builds no project but the stub's own.
  */
 function handleResolveDefinitions(request) {
     try {
         log(`Resolving ${request.aliases.length} type alias(es) from ${request.stub_dir}`);
-        const results = projectComponents().definitionResolver.resolveFromStub(request.stub_dir, request.aliases);
+        const results = definitionResolver.resolveFromStub(request.stub_dir, request.aliases);
         return {
             request_id: request.request_id,
             status: 'success',

package/sidecar/dist/src/infer-timing.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Which requests of an `infer` batch were slow, and at what (carrick#1985).
+ *
+ * A pass can send thousands of requests and spend most of its time in a few
+ * of them. The inferrer times every request; this module turns those timings
+ * into what the batch's answer carries: how many there were and what they
+ * add up to, the slowest few by name, the request that printed the longest
+ * type, and the requests that were the first asked of their file, apart.
+ *
+ * A request's time is also told apart by what it went on: the compiler call
+ * that computes the type, and the print of that type to text. The two have
+ * different remedies, and a printed length alone cannot say which one a slow
+ * request was: a type of four characters can take seconds to compute, and a
+ * type of a million can take half a second to print.
+ *
+ * It measures and names. It changes no answer and holds no type text.
+ */
+import type { InferSlotTiming, InferTiming } from './types.js';
+/**
+ * How many of its slowest requests a batch's answer names.
+ *
+ * Enough for a reader adding batches up to say what the slowest hundredth of
+ * a whole pass took: a pass's slowest requests are among its batches'
+ * slowest, and the last one a batch names is the most any it left out can
+ * have taken.
+ */
+export declare const SLOWEST_SLOTS = 25;
+/** How long something that started at `startedMs` has taken by `nowMs`. */
+export declare function elapsedMs(startedMs: number, nowMs: number): number;
+/**
+ * The parts of a request's time that are told apart: `type` is the compiler
+ * call that computes a type, `print` is the print of a type to text.
+ */
+export type TimedPhase = 'type' | 'print';
+/**
+ * Run `work`, and count how long it took as `phase`.
+ *
+ * The count is the process's, not a request's: whoever wants one request's
+ * share reads `phaseClock` before the request and after it. One request runs
+ * at a time, so the difference is that request's and nobody else's.
+ */
+export declare function timedPhase<T>(phase: TimedPhase, work: () => T): T;
+/** What each phase has come to so far, in milliseconds. */
+export declare function phaseClock(): Readonly<Record<TimedPhase, number>>;
+/**
+ * What a batch's answer says of the time its requests took. `slots` holds one
+ * timing per request, in the order the batch was done with them.
+ */
+export declare function inferTiming(slots: readonly InferSlotTiming[]): InferTiming;
+/**
+ * One timing for a request several projects answered, each with its own:
+ * the counts and the times added up, and the slowest of all of them.
+ */
+export declare function mergeInferTimings(parts: readonly InferTiming[]): InferTiming;

package/sidecar/dist/src/infer-timing.js

@@ -0,0 +1,124 @@
+/**
+ * Which requests of an `infer` batch were slow, and at what (carrick#1985).
+ *
+ * A pass can send thousands of requests and spend most of its time in a few
+ * of them. The inferrer times every request; this module turns those timings
+ * into what the batch's answer carries: how many there were and what they
+ * add up to, the slowest few by name, the request that printed the longest
+ * type, and the requests that were the first asked of their file, apart.
+ *
+ * A request's time is also told apart by what it went on: the compiler call
+ * that computes the type, and the print of that type to text. The two have
+ * different remedies, and a printed length alone cannot say which one a slow
+ * request was: a type of four characters can take seconds to compute, and a
+ * type of a million can take half a second to print.
+ *
+ * It measures and names. It changes no answer and holds no type text.
+ */
+/**
+ * How many of its slowest requests a batch's answer names.
+ *
+ * Enough for a reader adding batches up to say what the slowest hundredth of
+ * a whole pass took: a pass's slowest requests are among its batches'
+ * slowest, and the last one a batch names is the most any it left out can
+ * have taken.
+ */
+export const SLOWEST_SLOTS = 25;
+/** Milliseconds to the microsecond: sums of these stay readable. */
+const rounded = (ms) => Math.round(ms * 1000) / 1000;
+/** How long something that started at `startedMs` has taken by `nowMs`. */
+export function elapsedMs(startedMs, nowMs) {
+    return rounded(nowMs - startedMs);
+}
+/** What this process has spent in each phase since it started, in milliseconds. */
+const spent = { type: 0, print: 0 };
+/**
+ * Run `work`, and count how long it took as `phase`.
+ *
+ * The count is the process's, not a request's: whoever wants one request's
+ * share reads `phaseClock` before the request and after it. One request runs
+ * at a time, so the difference is that request's and nobody else's.
+ */
+export function timedPhase(phase, work) {
+    const started = performance.now();
+    try {
+        return work();
+    }
+    finally {
+        spent[phase] += performance.now() - started;
+    }
+}
+/** What each phase has come to so far, in milliseconds. */
+export function phaseClock() {
+    return { ...spent };
+}
+/**
+ * The `count` slowest of `slots`, slowest first. Requests that took the same
+ * time keep the order they are in.
+ */
+function slowestOf(slots, count) {
+    return [...slots].sort((a, b) => b.ms - a.ms).slice(0, count);
+}
+/**
+ * The one of `slots` that printed the longest type, the earliest of those
+ * that tie; undefined when none printed anything.
+ */
+function longestPrintedOf(slots) {
+    let longest;
+    for (const slot of slots) {
+        if (slot.printed_length > (longest?.printed_length ?? 0))
+            longest = slot;
+    }
+    return longest;
+}
+/**
+ * What a batch's answer says of the time its requests took. `slots` holds one
+ * timing per request, in the order the batch was done with them.
+ */
+export function inferTiming(slots) {
+    let slotsMs = 0;
+    let typeMs = 0;
+    let printMs = 0;
+    let firstInFile = 0;
+    let firstInFileMs = 0;
+    for (const slot of slots) {
+        slotsMs += slot.ms;
+        typeMs += slot.type_ms;
+        printMs += slot.print_ms;
+        if (slot.first_in_file) {
+            firstInFile += 1;
+            firstInFileMs += slot.ms;
+        }
+    }
+    const longest = longestPrintedOf(slots);
+    return {
+        slots: slots.length,
+        slots_ms: rounded(slotsMs),
+        type_ms: rounded(typeMs),
+        print_ms: rounded(printMs),
+        first_in_file_slots: firstInFile,
+        first_in_file_ms: rounded(firstInFileMs),
+        slowest: slowestOf(slots, SLOWEST_SLOTS),
+        ...(longest ? { longest_printed: longest } : {}),
+    };
+}
+/**
+ * One timing for a request several projects answered, each with its own:
+ * the counts and the times added up, and the slowest of all of them.
+ */
+export function mergeInferTimings(parts) {
+    if (parts.length === 1)
+        return parts[0];
+    const sum = (read) => parts.reduce((total, part) => total + read(part), 0);
+    const longest = longestPrintedOf(parts.flatMap((part) => part.longest_printed ?? []));
+    return {
+        slots: sum((part) => part.slots),
+        slots_ms: rounded(sum((part) => part.slots_ms)),
+        type_ms: rounded(sum((part) => part.type_ms)),
+        print_ms: rounded(sum((part) => part.print_ms)),
+        first_in_file_slots: sum((part) => part.first_in_file_slots),
+        first_in_file_ms: rounded(sum((part) => part.first_in_file_ms)),
+        slowest: slowestOf(parts.flatMap((part) => part.slowest), SLOWEST_SLOTS),
+        ...(longest ? { longest_printed: longest } : {}),
+    };
+}

package/sidecar/dist/src/retype.d.ts

@@ -53,9 +53,11 @@ export declare class Retyper {
      * Judge every item, spending at most `budgetMs`. Each item rebuilds the
      * program at least twice, so a consumer with many calls could otherwise
      * outrun the caller's read deadline and lose every answer; the items the
-     * budget does not reach abstain and say so.
+     * budget does not reach abstain and say so. `onJudged` is called after each
+     * item the budget reached, so the caller can report progress while the
+     * event loop is blocked (carrick#1945).
      */
-    run(items: RetypeItem[], budgetMs: number): RetypeOutcome[];
+    run(items: RetypeItem[], budgetMs: number, onJudged?: () => void): RetypeOutcome[];
     private runOne;
     /**
      * The rewrite that states the producer's UNWIDENED return (carrick#1516),

package/sidecar/dist/src/retype.js

@@ -53,9 +53,11 @@ export class Retyper {
      * Judge every item, spending at most `budgetMs`. Each item rebuilds the
      * program at least twice, so a consumer with many calls could otherwise
      * outrun the caller's read deadline and lose every answer; the items the
-     * budget does not reach abstain and say so.
+     * budget does not reach abstain and say so. `onJudged` is called after each
+     * item the budget reached, so the caller can report progress while the
+     * event loop is blocked (carrick#1945).
      */
-    run(items, budgetMs) {
+    run(items, budgetMs, onJudged) {
         const deadline = performance.now() + budgetMs;
         // What each file said before any rewrite. Every rewrite is undone, so it
         // is the same for every item in the file.
@@ -64,12 +66,15 @@ export class Retyper {
             if (performance.now() >= deadline) {
                 return abstain(item, `the retype check ran out of its ${budgetMs}ms budget`);
             }
+            let outcome;
             try {
-                return this.runOne(item, before);
+                outcome = this.runOne(item, before);
             }
             catch (err) {
-                return abstain(item, `the retype check failed: ${err instanceof Error ? err.message : String(err)}`);
+                outcome = abstain(item, `the retype check failed: ${err instanceof Error ? err.message : String(err)}`);
             }
+            onJudged?.();
+            return outcome;
         });
     }
     runOne(item, before) {

package/sidecar/dist/src/type-inferrer.d.ts

@@ -86,6 +86,12 @@ export declare class TypeInferrer {
      */
     private readonly readNodes;
     private readonly unwidenedBudgetMs;
+    /**
+     * The files a request has named to this inferrer, as the requests named
+     * them (carrick#1985). It lives as long as the project does, so the first
+     * request of a file is the first of the process, not of a batch.
+     */
+    private readonly filesAsked;
     constructor(options: TypeInferrerOptions);
     /**
      * What the structural printer needs to tell the user's own declarations from
@@ -193,6 +199,61 @@ export declare class TypeInferrer {
      */
     private resolveParamTarget;
     private inferResponseBody;
+    /**
+     * The handler a module exports on `line`: a function that opens on exactly
+     * that line, exported at the top of its module, where no call on the line
+     * registers a route (carrick#807).
+     *
+     * That is the anchor of a route its module's place in the project states:
+     * the row's line is the handler's own export, and the module's exports are
+     * how the framework finds it. Each condition keeps another kind of row out:
+     *
+     *  - the line has to be the function's first, with no tolerance. A model
+     *    row that names no site of its own can sit on any line of a handler,
+     *    and the nearest function to such a line is whatever is declared
+     *    beside it;
+     *  - a method of a class is a controller's handler, whose rows are
+     *    anchored on what a model located before the method's own return;
+     *  - a line that registers a route has a registration to read.
+     */
+    private handlerDeclaredAtLine;
+    /**
+     * `handlerDeclaredAtLine` without its registration test, for a caller that
+     * has already looked for a registration on the line.
+     */
+    private exportedFunctionOpeningOn;
+    /**
+     * The response of a handler the request's line declares, for a request
+     * that also located an expression inside it (carrick#807).
+     *
+     * The handler's own return is read first, exactly as a `function_return`
+     * request at the line reads it, and that reading stands: a body it
+     * recovered, and a decision it made not to publish one. The located
+     * expression is consulted in one case only. Where the handler returns a
+     * call nothing resolves (its library is not installed), the return walk
+     * reads no argument the source does not annotate, because it cannot tell a
+     * response sender from a query. A located payload that is an ARGUMENT of
+     * such a returned call says which it is, so the same walk runs again with
+     * that callee taken as a sender: every success body the handler returns
+     * through it, error branches dropped as before. The located expression's
+     * own type is never what is published here.
+     *
+     * Returns `undefined` when the located expression is something the return
+     * walk never looked at, so it is left to the readings below:
+     *  - the handler returns nothing (`void`): it sends through a parameter;
+     *  - the handler returns transport the walk read no body out of, and the
+     *    located expression is not part of what it returns. A body built before
+     *    the response is returned (`const res = send(body); ...; return res`)
+     *    is one. Named on a send that states an error or redirect status, it is
+     *    no success body and the row abstains.
+     */
+    private responseOfDeclaredHandler;
+    /**
+     * The call inside one of `returned` that the located expression is, or is
+     * an argument of: the send a located payload was handed to. `undefined`
+     * when the located expression is not part of what the handler returns.
+     */
+    private returnedSendHolding;
     /**
      * True when a located call's own result is the route's payload, so the
      * transitional drill into its first argument must not run (carrick#1732).
@@ -297,6 +358,14 @@ export declare class TypeInferrer {
      * else, and is not reported.
      */
     private statedBodyAtRead;
+    /**
+     * The type node `statedBodyAtRead` reads, for a caller that wants the
+     * annotation itself: a consumer's response read reports its root, and a
+     * handler's request read prints it (carrick#807). One reading of "stated at
+     * the read" for both, so the two cannot disagree about which annotation
+     * belongs to a read.
+     */
+    private typeNodeStatedAtRead;
     /**
      * A stated type with `any` or `unknown` written anywhere in it (`unknown`,
      * `Record<string, unknown>`, `{ items: any[] }`) leaves a position open: it
@@ -892,9 +961,22 @@ export declare class TypeInferrer {
      *
      * Under `statedOnly` the annotation is the ONLY thing that counts, so an
      * unresolvable callee's arguments never become a contract by accident.
+     *
+     * `asSent` judges the object in the form it is sent in, which is what an
+     * ARGUMENT handed to a sender is asked; see `typeIsObjectShaped`.
      */
     private nodeCarriesPayloadContract;
-    /** Object, array-of-object, or a union/intersection containing one. */
+    /**
+     * Object, array-of-object, or a union/intersection containing one.
+     *
+     * With `asSent` the object is judged in the form it is SENT in
+     * (carrick#1163): a value that declares `toJSON()` travels as what that
+     * returns. A `URL` or a `Date` is therefore the string it serialises to,
+     * the bare primitive this rule already refuses, and `redirect(new URL(path,
+     * base))` hands over a location exactly as `redirect("/next")` does
+     * (carrick#807). A value whose JSON form is itself an object is a body like
+     * any other.
+     */
     private typeIsObjectShaped;
     /**
      * The `satisfies X` / `as X` / `<X>` annotation node on an expression, when
@@ -1275,6 +1357,46 @@ export declare class TypeInferrer {
      * Returns null (not a spurious type) for a genuinely payload-less handler.
      */
     private inferRequestReadFromHandler;
+    /**
+     * The call in a handler that reads the request's body off the platform
+     * request the handler was handed, or `undefined` (carrick#807).
+     *
+     * Read by shape, with no method name consulted:
+     *
+     *  - a call that takes nothing, on a member of a value whose type is request
+     *    machinery (`typeIsFrameworkMachinery`: declared by the platform or an
+     *    installed library, and carrying its body readers);
+     *  - that value is rooted at one of the handler's OWN parameters, named or
+     *    destructured. A response read off an outbound call inside the handler
+     *    (`(await upstream.json()) as Rate`) has the same shape one variable
+     *    away, and is the opposite side of a different exchange;
+     *  - and the call's result, awaited, is `any` or `unknown`: the platform's
+     *    untyped parse. A read that states its own type (`formData()`, a typed
+     *    `json<T>()`) is not this shape and is left to the readers that
+     *    already handle it.
+     *
+     * The first such call in source order: a body is read once.
+     */
+    private platformBodyReadIn;
+    /**
+     * The request contract the source states AT a body read, or null when it
+     * states none there (carrick#807).
+     *
+     * Three statements count, each of them on the read itself:
+     *
+     *  - a cast on it: `(await request.json()) as NewWidget`;
+     *  - the annotation of the binding it initialises:
+     *    `const input: NewWidget = await request.json()`;
+     *  - a schema the read, or the binding that holds it, is handed to
+     *    (`schemaConsumingRead`): the schema's input is what a caller may send.
+     *
+     * The first two are `typeNodeStatedAtRead`, the reading a consumer's
+     * response read already gets. An annotation further out is not one of them:
+     * `const saved: Saved = await save(await request.json())` types what `save`
+     * returned, and reading it as the body would publish the wrong side of the
+     * handler. Nor is a placeholder (`as unknown`, `Record<string, unknown>`).
+     */
+    private requestStatedAtBodyRead;
     /**
      * Resolve a type-annotation/type-argument node to fully-structural text,
      * dropping any `Promise<…>` wrapper (`c.req.json<T>()` returns `Promise<T>`,
@@ -1418,8 +1540,31 @@ export declare class TypeInferrer {
      * request type resolves `body` to `unknown`/`any`, which the useless-type
      * guard rejects. So a handler that declares nothing yields null and the next
      * anchor runs.
+     *
+     * The member has to be one the route's own annotation can have filled
+     * (carrick#807). The platform `Request` declares `body` too, as the byte
+     * stream every request carries, and so does each library type that extends
+     * it. That is concrete, so it passed the useless-type guard and a handler
+     * taking the platform request published a stream as its request contract,
+     * ahead of the body read inside it. A `body` a library declares with one
+     * fixed type says nothing about this route and is skipped.
      */
     private requestBodyFromHandlerParams;
+    /**
+     * True when a member is declared by a library, or by the platform, with a
+     * type that names none of its declaring type's parameters: the same type on
+     * every value, whichever route the value belongs to.
+     *
+     * `body: ReqBody` on a request type generic in its body is a slot, and the
+     * handler's annotation fills it. `readonly body: ReadableStream<Uint8Array>
+     * | null` on the platform request is not. A member the repo declares itself
+     * is never fixed in this sense: writing `body: NewWidget` on the handler's
+     * own request type is the annotation, and one declaration of the repo's
+     * among several (an intersection with the platform type) decides it. A
+     * declaration that states no type at all is left to the guards that read
+     * the type.
+     */
+    private memberIsFixedByItsLibrary;
     /**
      * Anchor (b2): the contract declared by a VALIDATOR MIDDLEWARE on the
      * registration (carrick#964).