@telorun/kernel 0.85.0 → 0.87.0

package/dist/evaluation-context.js

@@ -2,8 +2,9 @@ import { AsyncLocalStorage } from "node:async_hooks";
 import { formatSpanCounter } from "./logging/span-id.js";
 import { deriveContext, getRefIdentity, isCompiledValue, isInvokeError, isCancellationError, isSuspension, resourceKey, UNCANCELLABLE_CONTEXT, } from "@telorun/sdk";
 import { RuntimeError } from "@telorun/sdk";
-import { evalPathCovers } from "@telorun/analyzer";
+import { celResourceReads, evalPathCovers } from "@telorun/analyzer";
 import { effectOwnerOf, executeReturnedChain } from "./effect-scope.js";
+import { impactClosure, reverseTopologicalOrder } from "./resource-edges.js";
 import { REDACTED, redactSensitive, sensitivePathsOfInstance, } from "./instance-sensitive-paths.js";
 import { classifyInitFailures, isDeferral, renderInitFailureText, summarizeInitFailures, } from "./init-failure-diagnostics.js";
 import { acceptReportedStatus, buildPublishedProps, diagnoseObservedStateAccess, } from "./observed-state.js";
@@ -432,6 +433,24 @@ export class EvaluationContext {
      * they are reported with both the init error and the refusing inverse.
      */
     withheldResources = new Set();
+    /**
+     * Keys whose instance was REMOVED BY AN UNWIND — the fact a dispatch that
+     * misses actually needs, as opposed to the context-wide state that only
+     * approximates it.
+     *
+     * A miss because the resource went away is the runtime withdrawing, not a
+     * manifest defect, and the two want opposite follow-ups (see `invoke`). Both
+     * paths that withdraw one go through `unwindEach`, the single removal site, so
+     * recording it there covers a shutdown AND a reconciliation — where a check
+     * against `state` would have converted the first and missed the second, since
+     * `unwindResources` deliberately leaves the state untouched. The second is the
+     * one that fires on every watch-session save.
+     *
+     * Cleared when the key is registered again: after a reconcile the name means a
+     * live resource, and a stale entry would report a genuine missing-resource
+     * defect as a cancellation.
+     */
+    unwoundResources = new Set();
     /** Resources discarded after a failed `init()` and re-queued for creation.
      *  Their re-creation is not progress — see the create sub-phase. */
     recreatedResources = new Set();
@@ -460,6 +479,7 @@ export class EvaluationContext {
      */
     adoptBorrowedResource(name, resource, instance, owner) {
         this.resourceInstances.set(name, { resource, instance });
+        this.unwoundResources.delete(name);
         this.borrowedResources.add(name);
         this.declaredManifests.set(name, resource);
         const unmirror = owner.mirrorPublications(resource.metadata.name, (props) => this.onResourceSnapshotted(name, props));
@@ -521,9 +541,29 @@ export class EvaluationContext {
     declaredManifests = new Map();
     /** Per-resource dependency names, captured at create() time — BEFORE Phase-5
      *  injection swaps refs for live instances, so the walk sees plain objects and
-     *  cannot wander into a controller's (possibly cyclic) object graph. Read only
-     *  when init fails, to attribute each failure to its cause. */
+     *  cannot wander into a controller's (possibly cyclic) object graph.
+     *
+     *  Read twice, and RETAINED for the context's lifetime because of the second
+     *  reader: to attribute an init failure to its cause, and to order teardown
+     *  (`teardownOrder`) so a consumer's inverses run while the resources it holds
+     *  are still alive. */
     resourceDependencies = new Map();
+    /**
+     * Names resolved by NAME during initialization, rather than through a
+     * declared reference slot — so a resource somebody may be holding, with no
+     * edge recording who.
+     *
+     * The set is of TARGETS, not of pairs: the door that records
+     * (`ModuleContext.getInstance`) is reached as `ctx.moduleContext`, which every
+     * resource of the module shares, so there is no caller to attribute the read
+     * to. That is enough for the one decision that depends on it — see
+     * {@link impactedBy}.
+     */
+    opaquelyRead = new Set();
+    /** Record a by-name resolution. Called by the recording door only. */
+    recordOpaqueRead(name) {
+        this.opaquelyRead.add(name);
+    }
     /**
      * Optional hook called between create() and init() for each resource.
      * Set by the kernel to inject live instances into reference fields.
@@ -730,6 +770,32 @@ export class EvaluationContext {
         this.pendingResources.push(resource);
         this.declaredManifests.set(name, resource);
     }
+    /**
+     * Forget a declaration entirely — the inverse of {@link registerManifest}.
+     *
+     * Reconciliation's other half: {@link unwindResources} disposes the INSTANCE,
+     * and this clears everything keyed by the name so the same name can be
+     * declared again. Without it `registerManifest` refuses with
+     * `ERR_DUPLICATE_RESOURCE`, which is the right answer for a manifest
+     * declaring one name twice and the wrong one for a second load of the same
+     * manifest.
+     *
+     * Every per-name record goes, not just the declaration: a resource left in
+     * `withheldResources` would be skipped by the init loop for the life of the
+     * kernel, and a stale `createdInstances` entry would have the loop initialize
+     * the object built from the PREVIOUS declaration.
+     */
+    deregisterManifest(name) {
+        this.declaredManifests.delete(name);
+        this.resourceDependencies.delete(name);
+        this.createdInstances.delete(name);
+        this.withheldResources.delete(name);
+        this.recreatedResources.delete(name);
+        this.opaquelyRead.delete(name);
+        const pending = this.pendingResources.findIndex((r) => r.metadata?.name === name);
+        if (pending >= 0)
+            this.pendingResources.splice(pending, 1);
+    }
     /**
      * The manifest a name was DECLARED with, resolved scope-local first and then
      * up the enclosing chain — the order `getInstance` and the CEL `resources`
@@ -890,7 +956,12 @@ export class EvaluationContext {
                             progress = true;
                         const createdRes = created.resource;
                         const refs = collectResourceRefs(createdRes);
-                        this.resourceDependencies.set(name, localDependencyNames(refs));
+                        // `resource` rather than `createdRes`: the registered declaration
+                        // still holds its expressions, while the created copy holds the
+                        // values they were expanded to.
+                        this.resourceDependencies.set(name, [
+                            ...new Set([...localDependencyNames(refs), ...celResourceReads(resource)]),
+                        ]);
                         const payload = {
                             resource: {
                                 kind: createdRes.kind,
@@ -1000,10 +1071,11 @@ export class EvaluationContext {
                     // so it resolves fine from here.
                     await this.publishSnapshot(name);
                     this.resourceInstances.set(name, { resource, instance });
+                    // Live again: a reconcile re-initializes what it unwound, and leaving
+                    // the mark would report a genuine missing-resource defect at this name
+                    // as a cancellation for the rest of the process.
+                    this.unwoundResources.delete(name);
                     this.createdInstances.delete(name);
-                    // Read only on failure, and this one succeeded — drop it rather than
-                    // holding a dep-name array per resource for the context's lifetime.
-                    this.resourceDependencies.delete(name);
                     errors.delete(name);
                     progress = true;
                     await this.emit(`${resource.kind}.${resource.metadata.name}.Initialized`, {
@@ -1173,15 +1245,32 @@ export class EvaluationContext {
         };
     }
     /**
-     * Cascade teardown depth-first through the tree:
-     *   1. Tear down child contexts in reverse registration order.
-     *   2. Tear down own resource instances in reverse registration order,
-     *      emitting a Teardown event for each via the injected emit callback.
+     * Cascade teardown through the tree:
+     *   1. Tear down own resource instances in {@link teardownOrder}, emitting a
+     *      Teardown event for each via the injected emit callback.
+     *   2. Sweep any child context still standing, in {@link childTeardownOrder}.
+     *
+     * **Own resources go FIRST, and the child sweep is a backstop.** A child
+     * context that belongs to a resource is torn down by that resource's own
+     * inverse — an import's `init()` returns `child.teardownResources()`, and so
+     * does a template's — so it already unwinds at its owner's position in step 1,
+     * which is the position the edges put it at. Running a child cascade ahead of
+     * step 1 tore every imported library down before any resource of THIS context,
+     * so an app resource holding `!ref Alias.name` unwound after its provider was
+     * already gone, and the owner's inverse then found nothing left to do.
+     *
+     * Sweeping afterwards rather than not at all is what still reclaims a context
+     * no inverse claims: a `lifecycle: shared` library, which is spawned under the
+     * root and deliberately gives no importer a claim on it, and an import whose
+     * `init()` never ran to register one. `teardownResources` is idempotent, so a
+     * context already taken down in step 1 costs the sweep nothing.
      */
     // eslint-disable-next-line @typescript-eslint/member-ordering
     async teardownResources() {
         this.state = "Draining";
-        const failures = [];
+        const failures = await this.unwindEach(this.teardownOrder());
+        // The backstop: whatever no resource's inverse claimed. Everything an import
+        // or a template owns is already down, so this is a no-op for it.
         for (const child of this.childTeardownOrder()) {
             try {
                 await child.teardownResources();
@@ -1190,7 +1279,48 @@ export class EvaluationContext {
                 failures.push({ resource: "(child context)", error: err });
             }
         }
-        for (const [key, { resource, instance }] of this.teardownOrder()) {
+        this.state = "Teardown";
+        this.raiseTeardownFailures(failures);
+    }
+    /**
+     * Unwind SOME of this context's resources and leave the rest running.
+     *
+     * The reconciliation half of teardown: a host that has decided which
+     * declarations moved unwinds exactly {@link impactedBy}'s answer, then
+     * re-registers and re-initializes. Ordering is {@link teardownOrder}
+     * restricted to the selection, so a consumer still unwinds before what it
+     * holds — and the selection being closed under holders is what makes that
+     * true of the resources left standing as well, since none of them holds
+     * anything in it.
+     *
+     * The context keeps its state: it is neither draining nor torn down, and no
+     * child context is swept, because a child belonging to an unwound import goes
+     * down with that import's own inverse exactly as it does at teardown.
+     *
+     * A name with no live instance is skipped rather than reported — a resource
+     * that failed to initialize has nothing to unwind, and a host asking for it is
+     * asking about a declaration, not about an instance.
+     */
+    // eslint-disable-next-line @typescript-eslint/member-ordering
+    async unwindResources(names) {
+        const selected = this.teardownOrder().filter(([, entry]) => names.has(entry.resource.metadata.name));
+        this.raiseTeardownFailures(await this.unwindEach(selected));
+    }
+    raiseTeardownFailures(failures) {
+        if (failures.length === 0)
+            return;
+        throw new RuntimeError("ERR_TEARDOWN_FAILED", `${failures.length} resource(s) failed during teardown`, failures.map(({ resource, error }) => ({
+            severity: "error",
+            message: error instanceof Error ? error.message : String(error),
+            resource,
+        })));
+    }
+    /** Unwind the given entries in the order supplied, aggregating what refused.
+     *  Failures are returned rather than thrown so one refusing inverse cannot
+     *  strand the resources after it — the log sinks above all. */
+    async unwindEach(entries) {
+        const failures = [];
+        for (const [key, { resource, instance }] of entries) {
             const label = `${resource.kind}.${resource.metadata.name}`;
             // A reading belongs to the run that produced it. The WeakMap would drop it
             // with the instance anyway; clearing here also covers an instance something
@@ -1242,27 +1372,30 @@ export class EvaluationContext {
                 failures.push({ resource: `${label} (Teardown event)`, error: err });
             }
             this.resourceInstances.delete(key);
+            // The single removal site, so it is also where "this went away because the
+            // runtime withdrew it" is recorded — the fact `invoke` reports a miss by.
+            this.unwoundResources.add(key);
+            // A torn-down resource must stop being readable: a CEL expansion that still
+            // found its reading would bake a value nothing is serving any more.
+            this.clearPublishedReading(resource.metadata.name);
         }
-        this.state = "Teardown";
-        if (failures.length > 0) {
-            throw new RuntimeError("ERR_TEARDOWN_FAILED", `${failures.length} resource(s) failed during teardown`, failures.map(({ resource, error }) => ({
-                severity: "error",
-                message: error instanceof Error ? error.message : String(error),
-                resource,
-            })));
-        }
+        return failures;
     }
     /**
-     * Child contexts in teardown order: ascending `teardownPriority` (default 0),
-     * with the base reverse-registration order preserved within each tier.
+     * Child contexts for the backstop sweep: ascending `teardownPriority`
+     * (default 0), reverse registration within a tier.
      *
-     * The same rule `teardownOrder` applies to resource instances, and for the
-     * same reason: reverse registration is reverse init order in the happy path,
-     * but a node that must reliably outlive the rest has to say so rather than
-     * depend on when it happened to be created. A `lifecycle: shared` library is
-     * registered when the FIRST import reaches it — which, for an import declared
-     * inside another library, is after that library's own context — so reverse
-     * registration would tear the singleton down while a borrower still holds it.
+     * What reaches the sweep is a context no resource's inverse claimed, which in
+     * practice is the `lifecycle: shared` libraries. Those are spawned under the
+     * ROOT and give no importer a claim, so nothing but this orders them — and
+     * reverse registration alone does not, since a singleton is registered when
+     * the FIRST import reaches it, which for an import declared inside another
+     * library is after that library's own context. `TEARDOWN_LAST` on the context
+     * is what keeps a singleton alive until the libraries borrowing it have gone.
+     *
+     * Its protection now stops at library-against-library. Every resource of the
+     * context that owns the sweep has already unwound by the time it runs, which
+     * is the ordering the sweep used to invert.
      */
     childTeardownOrder() {
         return [...this.children]
@@ -1271,17 +1404,17 @@ export class EvaluationContext {
             (b.teardownPriority ?? 0));
     }
     /**
-     * Resource instances in teardown order: ascending `teardownPriority`, with the
-     * base reverse-insertion order preserved within each priority tier.
+     * Resource instances in teardown order: ascending `teardownPriority` as a hard
+     * tier, and within a tier a consumer before every resource it holds
+     * ({@link reverseTopologicalOrder} over the create-time edges), with reverse
+     * insertion as the tiebreak.
      *
-     * The base order is reverse *insertion*, which is reverse init order in the
-     * happy path — but the init loop is a multi-pass retry, so a resource that
-     * failed its first pass lands later in the map than its topological rank
-     * implies. That makes the dependency graph an unreliable way to say "last".
-     * A resource that must reliably outlive the rest declares it directly via
-     * `teardownPriority` (log sinks set `TEARDOWN_LAST`), so the generic teardown
-     * path orders by a declared number rather than sniffing any one subsystem's
-     * instance shape.
+     * `teardownPriority` is a TIER rather than another edge because it is the
+     * author's statement about an edge nothing captured: a log sink is reached
+     * through `ctx.log` rather than through a ref slot, so no walk of the manifest
+     * can find the resources that will log on the way down. Letting topology
+     * reorder across tiers would let one discovered edge override a declaration
+     * made precisely because the edges are not all discoverable.
      */
     teardownOrder() {
         // A borrowed instance is torn down by the scope that declared it, never
@@ -1289,10 +1422,93 @@ export class EvaluationContext {
         const entries = [...this.resourceInstances.entries()]
             .filter(([name]) => !this.borrowedResources.has(name))
             .reverse();
-        // Stable sort by priority (default 0); Array.prototype.sort is stable, so
-        // the reverse-insertion order survives within each tier.
-        return entries.sort(([, a], [, b]) => (a.instance?.teardownPriority ?? 0) -
-            (b.instance?.teardownPriority ?? 0));
+        const tiers = new Map();
+        for (const entry of entries) {
+            const priority = entry[1].instance?.teardownPriority ?? 0;
+            const tier = tiers.get(priority);
+            if (tier)
+                tier.push(entry);
+            else
+                tiers.set(priority, [entry]);
+        }
+        return [...tiers.keys()]
+            .sort((a, b) => a - b)
+            .flatMap((priority) => reverseTopologicalOrder(tiers.get(priority), (value) => value.resource.metadata.name, (name) => this.resourceDependencies.get(name)));
+    }
+    /**
+     * Every resource of this context that becomes invalid when `names` do — the
+     * named resources plus everything that transitively holds one
+     * ({@link impactClosure} over the same create-time edges teardown reads).
+     *
+     * What a reconciliation unwinds. A cross-module reference projects onto the
+     * local `Telo.Import` (`localDependencyNames`), so a change inside an
+     * imported library reaches this context as its import being impacted, and the
+     * library goes down with that import's own inverse — which is why this
+     * answers for one context rather than walking the tree.
+     *
+     * **`opaque` is what the answer cannot cover.** A name resolved by NAME during
+     * initialization ({@link opaquelyRead}) may be held by a resource no edge
+     * names, so a closure reaching one is not an answer at all. Those names are
+     * reported rather than absorbed: expanding the set to "every resource here"
+     * would sweep in the module document, which is not a resource a caller can
+     * unwind and re-register, and would present a whole-context rebuild as a
+     * narrowing. The caller escalates, and can say which resource forced it.
+     */
+    // eslint-disable-next-line @typescript-eslint/member-ordering
+    impactedBy(names) {
+        const impacted = impactClosure(names, this.resourceDependencies);
+        const opaque = [...impacted].filter((name) => this.opaquelyRead.has(name));
+        return { impacted, opaque };
+    }
+    /** Whether this resource's `run()` has been dispatched. A rebuilt resource
+     *  that had been started is one nothing will start again — boot targets run
+     *  once — so a caller reconciling has to escalate rather than leave it
+     *  constructed and idle. */
+    wasStarted(name) {
+        const instance = this.resourceInstances.get(name)?.instance;
+        return instance !== undefined && startedInstances.has(instance);
+    }
+    /** Drop a resource's published reading. A context with no `resources` scope of
+     *  its own has nothing to drop; `ModuleContext` overrides. */
+    clearPublishedReading(name) {
+        void name;
+    }
+    /** The declaration registered under `name` in THIS context, without the
+     *  walk up the enclosing chain a name lookup does. */
+    declaredManifestFor(name) {
+        return this.declaredManifests.get(name);
+    }
+    /** Replace a declaration in place, without queueing the resource for
+     *  creation. For a survivor of a reconciliation: its declaration is
+     *  content-identical by construction, but the object carries fresh loader
+     *  stamps (`metadata.sourceLine` above all), and a diagnostic anchored on the
+     *  stale one points at a pre-edit line. */
+    refreshManifest(name, resource) {
+        if (this.declaredManifests.has(name))
+            this.declaredManifests.set(name, resource);
+    }
+    /** Re-open an initialized context for another initialization pass.
+     *
+     *  The transition is the context's own, not a field a caller assigns: while it
+     *  is open, a reference to a resource that has not been rebuilt yet must
+     *  produce the deferral the init loop retries on rather than a hard
+     *  not-found, and leaving it open after a failed pass turns every later
+     *  lookup into that deferral with no pass coming. */
+    reopenForInitialization() {
+        if (this.state === "Initialized")
+            this.state = "Validated";
+    }
+    /** Close a pass opened by {@link reopenForInitialization} that did not reach
+     *  the end of `initializeResources`. */
+    closeInitialization() {
+        if (this.state === "Validated")
+            this.state = "Initialized";
+    }
+    /** Names a resource resolved by NAME while this context was initializing, so
+     *  no edge records who is holding them. Read by {@link impactedBy}; exposed so
+     *  a host can report why a reconciliation could not be narrowed. */
+    opaqueReads() {
+        return this.opaquelyRead;
     }
     transientChild(context) {
         return new EvaluationContext(this.source, { ...this.context, ...context }, this._createInstance, this._secretValues, this.emit);
@@ -1306,6 +1522,45 @@ export class EvaluationContext {
     async invoke(kind, name, inputs, ctx) {
         const entry = this.resourceInstances.get(name);
         if (!entry) {
+            // A MISS ON A RESOURCE THE RUNTIME WITHDREW is the runtime going away, not
+            // a manifest defect, and the two want opposite follow-ups. An unwind
+            // removes each instance as it goes, so work still in flight — a detached
+            // task the kernel waited for and then abandoned, above all — finds an
+            // emptying map and would otherwise be told its target does not exist. That
+            // verdict is durable where the withdrawal is not: a durable run records it
+            // as `failed`, which is terminal, so one ordinary Ctrl-C leaves a run id
+            // nothing will ever pick up again — from the one feature whose whole
+            // purpose is surviving that.
+            //
+            // Reported as a cancellation because that is what it is, and because every
+            // consumer already handles one correctly: a durable body leaves the run
+            // `running` for the resumer, and a step's retry budget is not spent
+            // re-issuing a call the runtime has no intention of answering.
+            //
+            // Keyed on `unwoundResources` — the recorded FACT — rather than on this
+            // context's state, which is only a proxy for it and misses the
+            // reconciliation path entirely: `unwindResources` withdraws an instance and
+            // deliberately leaves the state alone, and that is the path a watch session
+            // takes on every save. Only the MISS is converted; a resource still in the
+            // map is dispatched as before, because a teardown-time flush is legitimate
+            // work and refusing it would break shutdown to protect it.
+            if (this.unwoundResources.has(name)) {
+                const reason = this.state === "Draining" || this.state === "Teardown"
+                    ? "the runtime is shutting down"
+                    : "the resource was unwound while the runtime reconciled";
+                // The SPAN payload, not an ad-hoc object: every consumer of this event
+                // reads `ref.kind` (the debug UI's kind facet) and `outcome` (its graph
+                // nodes and outcome tally) off the trace shape, so a payload carrying
+                // neither is an event that fires and is invisible — which is worse than
+                // no event, and precisely the cancellation an operator is looking for
+                // during a watch-session save. No span ids: this refusal happens before
+                // any span is opened, and `tracePayload` omits an undefined one.
+                await this.emit(`${name}.InvokeCancelled`, this.tracePayload(kind, name, undefined, undefined, undefined, "invoke", "end", "cancelled", {
+                    reason,
+                }));
+                throw new RuntimeError("ERR_INVOKE_CANCELLED", `Invoke ${kind}.${name} was cancelled: ${reason} and the resource has already ` +
+                    `been torn down.`);
+            }
             throw new RuntimeError("ERR_RESOURCE_NOT_FOUND", `Resource not found for invocation: ${kind}.${name}. Available resources: ${[...this.resourceInstances.keys()].join(", ")}`);
         }
         // A `!ref` whose kind couldn't be determined at resolve time (e.g. a