@bluewombat/runtime 0.10.0 → 0.11.1

package/dist/loop/features.js

@@ -2,9 +2,11 @@ import { existsSync } from "node:fs";
 import { rm } from "node:fs/promises";
 import { join } from "node:path";
 import { pathSegment } from "@bluewombat/conductor";
+import { CLI_NAME, } from "@bluewombat/manager-kit";
 import { publish, refreshWorkLine } from "./authority.js";
 import { cancelTicket, handleCancel as cancelTicketByKey, handleResume as resumeTicketByKey, } from "./deliveries.js";
 import { driveUntilBlocked } from "./drive.js";
+import { passReportsOf } from "./journal.js";
 import { pushReport } from "./report.js";
 /** Where a Ticket is while it still counts against its Feature. */
 const LIVE = new Set([
@@ -73,16 +75,20 @@ async function admitFeature(input, feature) {
     // about work already made: nothing here restarts it.
     const before = await input.ledger.getFeature(feature.key);
     if (before.ok && before.aggregate.state !== "waiting") {
-        // The label that leaves the e2e check out is the one thing a human can still change: it is
+        // The label that leaves the feature-test check out is the one thing a human can still change: it is
         // honoured until the Feature is submitted, and read again by the check each time it runs.
-        if (before.aggregate.intention.skip_e2e !== feature.skip_e2e) {
-            const moved = await input.ledger.setFeatureSkipE2e({
+        if (before.aggregate.intention.skip_feature_test !== feature.skip_feature_test) {
+            const moved = await input.ledger.setFeatureSkipFeatureTest({
                 key: feature.key,
-                skip: feature.skip_e2e,
+                skip: feature.skip_feature_test,
             });
             if (moved.ok) {
-                input.journal.append({ event: "skip-e2e", key: feature.key, skip: feature.skip_e2e });
-                input.trace(`feature    ${feature.key} ${feature.skip_e2e ? "now" : "no longer"} skips the e2e check`);
+                input.journal.append({
+                    event: "skip-feature-test",
+                    key: feature.key,
+                    skip: feature.skip_feature_test,
+                });
+                input.trace(`feature    ${feature.key} ${feature.skip_feature_test ? "now" : "no longer"} skips the feature-test check`);
                 return;
             }
         }
@@ -99,7 +105,7 @@ async function admitFeature(input, feature) {
         external_id: feature.external_id,
         blocked_by: feature.blocked_by,
         merge_when_ready: feature.merge_when_ready,
-        ...(feature.skip_e2e ? { skip_e2e: true } : {}),
+        ...(feature.skip_feature_test ? { skip_feature_test: true } : {}),
         ...(feature.title === undefined ? {} : { title: feature.title }),
         ...(feature.source === undefined ? {} : { source: feature.source }),
     };
@@ -197,7 +203,7 @@ export async function advanceProject(input, project) {
 }
 /** What the tracker says of a Ticket beside its words. */
 function marksOf(entry) {
-    return { noTests: entry.no_tests, e2e: entry.e2e === true };
+    return { featureTest: entry.featureTest === true };
 }
 function refOf(feature) {
     const { intention } = feature;
@@ -227,41 +233,42 @@ function admissionOf(ticket, feature, among, marks) {
         ...(feature.line === undefined ? {} : { line: feature.line }),
         ...(ticket.title === undefined ? {} : { title: ticket.title }),
         ...(ticket.source === undefined ? {} : { source: ticket.source }),
-        // The Ticket that holds the acceptance suite writes the tests itself: none are written first.
-        ...(marks.noTests || marks.e2e ? { no_tests: true } : {}),
-        ...(marks.e2e ? { e2e: true } : {}),
+        ...(marks.featureTest ? { featureTest: true } : {}),
     };
 }
 /**
- * What is wrong with a Feature of a Project that proves its Features by an e2e suite: it holds
- * no e2e Ticket and is not marked to skip the check, holds several, or its e2e Ticket may start
+ * What is wrong with a Feature of a Project that proves its Features by a feature-test suite: it holds
+ * no feature-test Ticket and is not marked to skip the check, holds several, or its feature-test Ticket may start
  * before the others have landed. Nothing: a Project without the check, or a Feature that skips it.
  */
-function e2eRefusal(input, feature, live) {
-    if (input.options.builder.e2e === undefined || feature.intention.skip_e2e === true) {
+function featureTestRefusal(input, feature, live) {
+    if (input.options.builder.featureTest === undefined ||
+        feature.intention.skip_feature_test === true) {
         return undefined;
     }
-    const holding = live.filter((entry) => entry.e2e);
-    const head = "This Project proves each Feature by an e2e acceptance suite, run once on the whole Feature.";
+    const holding = live.filter((entry) => entry.featureTest);
+    const head = "This Project proves each Feature by a feature tests, run once on the whole Feature.";
     if (holding.length === 0) {
-        return `${head} The Feature has no e2e Ticket and is not marked skip-test-e2e: add the e2e Ticket (the scenarios agreed with the operator, blocked by every other Ticket), or mark the Feature to skip the check.`;
+        return `${head} The Feature has no feature-test Ticket and is not marked skip-feature-test: add the feature-test Ticket (the scenarios agreed with the operator, blocked by every other Ticket), or mark the Feature to skip the check.`;
     }
     if (holding.length > 1) {
-        return `${head} The Feature holds ${holding.length} e2e Tickets: it holds one, with all its scenarios.`;
+        return `${head} The Feature holds ${holding.length} feature-test Tickets: it holds one, with all its scenarios.`;
     }
     const only = holding[0];
-    const others = live.filter((entry) => !entry.e2e).map((entry) => entry.ticket.key);
+    const others = live.filter((entry) => !entry.featureTest).map((entry) => entry.ticket.key);
     const missing = others.filter((key) => !only?.ticket.blocked_by.includes(key));
     if (only !== undefined && missing.length > 0) {
-        return `${head} The e2e Ticket ${only.ticket.key} must be blocked by every other Ticket of the Feature, so that it lands last. It is not blocked by: ${missing.join(", ")}.`;
+        return `${head} The feature-test Ticket ${only.ticket.key} must be blocked by every other Ticket of the Feature, so that it lands last. It is not blocked by: ${missing.join(", ")}.`;
     }
     return undefined;
 }
-/** Whether a Feature still holds an e2e Ticket that was not cancelled. */
-async function holdsLiveE2eTicket(input, feature) {
+/** Whether a Feature still holds a feature-test Ticket that was not cancelled. */
+async function holdsLiveFeatureTestTicket(input, feature) {
     for (const key of feature.tickets) {
         const got = await input.ledger.get(key);
-        if (got.ok && got.aggregate.intention.e2e === true && got.aggregate.state !== "cancelled") {
+        if (got.ok &&
+            got.aggregate.intention.featureTest === true &&
+            got.aggregate.state !== "cancelled") {
             return true;
         }
     }
@@ -295,14 +302,14 @@ async function openNextFeature(input, project) {
         });
         return;
     }
-    const refusal = e2eRefusal(input, feature, live);
+    const refusal = featureTestRefusal(input, feature, live);
     if (refusal !== undefined) {
         input.journal.append({ event: "invalid", key: feature.intention.key, reason: refusal });
         await pushReport(input, {
             event: "invalid",
             key: feature.intention.key,
             project,
-            eventId: `${feature.intention.key}:invalid:e2e:${refusal}`,
+            eventId: `${feature.intention.key}:invalid:feature-test:${refusal}`,
             fields: { reason: refusal },
         }, TO);
         return;
@@ -434,25 +441,23 @@ async function syncTicket(input, feature, entry, known) {
     }
     // Only a Ticket that has not started takes the new words: the tracker's
     // fingerprint moves with every comment, and a unit in flight is not rewritten.
-    // The label that says no tests are written first is not in the words, so it moves no
+    // The label that marks the feature-test Ticket is not in the words, so it moves no
     // fingerprint: it is compared on its own.
     const marks = marksOf(entry);
-    const flagged = current.aggregate.intention.no_tests === true;
-    const holdsE2e = current.aggregate.intention.e2e === true;
+    const holdsFeatureTest = current.aggregate.intention.featureTest === true;
     if (state === "received" &&
         (current.aggregate.intention.fingerprint !== ticket.fingerprint ||
-            flagged !== (marks.noTests || marks.e2e) ||
-            holdsE2e !== marks.e2e)) {
+            holdsFeatureTest !== marks.featureTest)) {
         await input.ledger.admit(admissionOf(ticket, feature, known, marks));
     }
-    // A Feature that is not put through the e2e check has no use of the Ticket that holds it.
-    if (feature.intention.skip_e2e === true && marks.e2e && LIVE.has(state)) {
+    // A Feature that is not put through the feature-test check has no use of the Ticket that holds it.
+    if (feature.intention.skip_feature_test === true && marks.featureTest && LIVE.has(state)) {
         input.journal.append({
-            event: "e2e-ticket-cancelled",
+            event: "feature-test-ticket-cancelled",
             key: ticket.key,
             feature: feature.intention.key,
         });
-        input.trace(`feature    ${feature.intention.key} skips the e2e check: ticket ${ticket.key} is cancelled`);
+        input.trace(`feature    ${feature.intention.key} skips the feature-test check: ticket ${ticket.key} is cancelled`);
         await cancelTicketByKey({ key: ticket.key, project: ticket.project }, input);
         return;
     }
@@ -485,7 +490,7 @@ async function advanceOpenFeature(input, feature) {
         return run;
     }
     if (run?.outcome === "idle" && tickets.some((summary) => summary.state === "received")) {
-        await escalateFeature(input, feature, "No Ticket can start: each waits on a Ticket that is not done.");
+        await escalateFeature(input, feature, "No Ticket can start: each waits on a Ticket that is not done.", { kind: "tickets" });
     }
     return run;
 }
@@ -499,13 +504,16 @@ const DRIVEN = new Set([
 /**
  * Work the Tickets of an open Feature, up to `limit` at a time.
  *
- * Each place is a worker: it drives a Ticket the Project already holds, else
- * takes the next one that may start, and goes on to another when its own is
- * done or is only waiting for the outside. A Ticket waiting on a check holds a
- * place but not a worker, so the others do not wait with it. With a limit of 1
- * this is one Ticket after the other.
+ * A Ticket the Project already holds is driven first, then the next one that
+ * may start. Whenever a drive ends or a place is free, the places are filled
+ * again, so a Ticket that was blocked at the start is taken as soon as what
+ * blocked it is done and not at the next pass. A Ticket waiting on a check
+ * holds a place but is not driven, so the others do not wait with it. Nothing
+ * new is taken once a run is refused, and the call returns when nothing is
+ * driven and nothing can start. With a limit of 1 this is one Ticket after the
+ * other.
  */
-async function driveTickets(input, feature, project, limit) {
+export async function driveTickets(input, feature, project, limit) {
     // Once, before anything runs at the same time: it clears what a dead process
     // left, which must not be done under a Ticket that is being driven.
     await input.conductor.reconcile(project);
@@ -514,21 +522,35 @@ async function driveTickets(input, feature, project, limit) {
         .filter((summary) => DRIVEN.has(summary.state))
         .map((summary) => summary.key);
     let last;
-    const worker = async () => {
-        while (!interrupted()) {
+    let refused = false;
+    const places = Math.max(1, limit);
+    const inFlight = new Set();
+    const drive = async (key) => {
+        const run = await driveUntilBlocked(input, project, key, true);
+        last = run;
+        // A refusal is about the run, not the Ticket: the next one would meet it too.
+        if (run.outcome === "refused") {
+            refused = true;
+        }
+    };
+    // Starts drives until the places are full or nothing can start.
+    const fill = async () => {
+        while (!interrupted() && !refused && inFlight.size < places) {
             const key = queue.shift() ?? (await input.conductor.claimNext(project))?.key;
             if (key === undefined) {
                 return;
             }
-            const run = await driveUntilBlocked(input, project, key, true);
-            last = run;
-            // A refusal is about the run, not the Ticket: the next one would meet it too.
-            if (run.outcome === "refused") {
-                return;
-            }
+            const started = drive(key).finally(() => {
+                inFlight.delete(started);
+            });
+            inFlight.add(started);
         }
     };
-    await Promise.all(Array.from({ length: Math.max(1, limit) }, worker));
+    await fill();
+    while (inFlight.size > 0) {
+        await Promise.race(inFlight);
+        await fill();
+    }
     if (last === undefined && (await input.ledger.activeTickets(project)).length === 0) {
         const waiting = (await ticketsOf(input, feature)).some((s) => s.state === "received");
         // Nothing held and nothing could start: what remains waits on what cannot finish.
@@ -552,8 +574,20 @@ async function reportLanded(input, feature, tickets) {
         }, TO);
     }
 }
-async function escalateFeature(input, feature, report) {
-    const escalated = await input.ledger.escalateFeature({ key: feature.intention.key, report });
+const STAGE_OF = {
+    tickets: "tickets",
+    assembly: "integrating",
+    submitted: "submitting",
+};
+async function escalateFeature(input, feature, report, 
+/** Where the Feature stopped, and who refused it when something named itself. */
+origin) {
+    const escalated = await input.ledger.escalateFeature({
+        key: feature.intention.key,
+        report,
+        kind: origin.kind,
+        ...(origin.refusedBy === undefined ? {} : { refusedBy: origin.refusedBy }),
+    });
     if (!escalated.ok) {
         return;
     }
@@ -565,7 +599,8 @@ async function escalateFeature(input, feature, report) {
         key: feature.intention.key,
         project: feature.intention.project,
         eventId: `${feature.intention.key}:escalated:${round}`,
-        fields: { reason: report, stage: "submitting" },
+        // Each stop says where it happened: Tickets, the assembled line, or the offer.
+        fields: { reason: report, stage: STAGE_OF[origin.kind] },
     }, TO);
 }
 /** Where a Feature's own work is done: apart from every Ticket's workspace, which Host clears of strays. */
@@ -605,13 +640,12 @@ function featureStore(input, feature) {
                 spent: (aggregate.submission?.refusals ?? 0) +
                     (aggregate.parked_refusals ?? 0) +
                     (aggregate.submission?.realigns ?? 0),
-                submission: aggregate.submission === undefined
-                    ? undefined
-                    : { reference: aggregate.submission.reference },
+                realigned: aggregate.submission?.realigns ?? 0,
+                submission: aggregate.submission === undefined ? undefined : { id: aggregate.submission.id },
             };
         },
         async freeze(frozen) {
-            await escalateFeature(input, feature, frozen.report ?? "The line was stopped, with no reason given.");
+            await escalateFeature(input, feature, frozen.report ?? "The line was stopped, with no reason given.", { kind: frozen.kind, refusedBy: frozen.refusedBy });
             return { outcome: "escalated", key };
         },
         async park(why) {
@@ -650,17 +684,19 @@ async function closeFeature(input, feature, tickets) {
     const key = feature.intention.key;
     const project = feature.intention.project;
     if (!tickets.some((summary) => summary.state === "done")) {
-        await escalateFeature(input, feature, "Every Ticket was abandoned, so nothing was made.");
+        await escalateFeature(input, feature, "Every Ticket was abandoned, so nothing was made.", {
+            kind: "tickets",
+        });
         return;
     }
-    if (input.options.builder.e2e !== undefined &&
-        feature.intention.skip_e2e !== true &&
-        !(await holdsLiveE2eTicket(input, feature))) {
-        await escalateFeature(input, feature, "This Feature is not marked skip-test-e2e and holds no e2e Ticket: it was cancelled while the Feature skipped the check, and the mark was taken off since. Mark the Feature skip-test-e2e again to leave the check out.");
+    if (input.options.builder.featureTest !== undefined &&
+        feature.intention.skip_feature_test !== true &&
+        !(await holdsLiveFeatureTestTicket(input, feature))) {
+        await escalateFeature(input, feature, "This Feature is not marked skip-feature-test and holds no feature-test Ticket: it was cancelled while the Feature skipped the check, and the mark was taken off since. Mark the Feature skip-feature-test again to leave the check out.", { kind: "tickets" });
         return;
     }
     if (input.authority === undefined || feature.line === undefined) {
-        await input.ledger.markFeatureSubmitted({ key, reference: "local" });
+        await input.ledger.markFeatureSubmitted({ key, reference: "local", id: "local" });
         await input.ledger.markFeatureDone(key);
         await reportDone(input, feature, undefined, undefined);
         return;
@@ -676,8 +712,15 @@ async function closeFeature(input, feature, tickets) {
         await rm(unit.workspace, { recursive: true, force: true });
     };
     // The Feature's work is cut from its line the way a Ticket's is, once; a
-    // pass that sent it to the fix finds it where it was left.
-    if (!existsSync(unit.workspace)) {
+    // pass that sent it to the fix finds it where it was left. A resume is the
+    // one time the line may have moved under it, by a person: what it gained is
+    // then merged into the work, which keeps what it holds.
+    if (existsSync(unit.workspace)) {
+        if (feature.workspace_stale === true && !(await takeInLine(input, feature, unit))) {
+            return;
+        }
+    }
+    else {
         if (!(await refreshOnto(input, feature.line))) {
             return;
         }
@@ -703,6 +746,9 @@ async function closeFeature(input, feature, tickets) {
             return;
         }
     }
+    if (feature.workspace_stale === true) {
+        await input.ledger.markFeatureWorkspaceCurrent(key);
+    }
     const prepared = await delivery.delivery.prepare(unit, featureStore(input, feature));
     if (prepared !== "ready") {
         return;
@@ -723,7 +769,9 @@ async function closeFeature(input, feature, tickets) {
         return;
     }
     if (folded.outcome !== "integrated") {
-        await escalateFeature(input, feature, "The Feature's work could not be folded into its line.");
+        await escalateFeature(input, feature, "The Feature's work could not be folded into its line.", {
+            kind: "assembly",
+        });
         return;
     }
     const published = await publish({
@@ -732,13 +780,16 @@ async function closeFeature(input, feature, tickets) {
         id: key,
         ref: feature.line,
         target,
+        kind: "line",
         timeoutMs: input.options.timeoutMs,
         env: input.workLine.env,
     });
     if (!published.ok) {
-        await escalateFeature(input, feature, published.reason);
+        await escalateFeature(input, feature, published.reason, { kind: "submitted" });
         return;
     }
+    // What the Feature's Gates said when they passed it goes with the Submission.
+    const checks = passReportsOf(input.options.ledgerRoot, key);
     const submitted = await input.authority.submit({
         key,
         project,
@@ -748,15 +799,20 @@ async function closeFeature(input, feature, tickets) {
         workspace: input.workLine.stable,
         ...(feature.intention.title === undefined ? {} : { title: feature.intention.title }),
         intention: feature.intention.intention,
+        ...(checks.length === 0 ? {} : { checks }),
     });
     if (submitted.outcome === "unavailable") {
         return;
     }
     if (submitted.outcome === "refused") {
-        await escalateFeature(input, feature, submitted.reason);
+        await escalateFeature(input, feature, submitted.reason, { kind: "submitted" });
         return;
     }
-    await input.ledger.markFeatureSubmitted({ key, reference: submitted.reference });
+    await input.ledger.markFeatureSubmitted({
+        key,
+        reference: submitted.reference,
+        id: submitted.id,
+    });
     await clean();
     await pushReport(input, {
         event: "submitted",
@@ -766,11 +822,48 @@ async function closeFeature(input, feature, tickets) {
         fields: { reference: submitted.reference },
     }, TO);
 }
+/**
+ * Bring what a resumed Feature's line gained since its work was cut into that
+ * work, without discarding what the work holds (the fix's own commits, even
+ * refused ones: a person may want them). Answers whether the pass goes on:
+ * not when the line could not be reached or the merge was interrupted (the next
+ * pass tries again), nor when it conflicts — that is for a person, and the
+ * work is left as it was.
+ */
+async function takeInLine(input, feature, unit) {
+    if (!(await refreshOnto(input, feature.line))) {
+        return false;
+    }
+    const taken = await input.transformers.integrate({
+        id: `${unit.key}:resume`,
+        context: unit.key,
+        parent: unit.workspace,
+        child: input.workLine.stable,
+        durationMs: input.options.timeoutMs,
+        subject: "Work left when the Feature stopped",
+        mergeSubject: `Merge ${feature.line} into the feature`,
+    });
+    switch (taken.outcome) {
+        case "integrated":
+            return true;
+        case "interrupted":
+            return false;
+        case "conflict":
+            await escalateFeature(input, feature, `The line ${feature.line} gained commits since the Feature's work was made, and they conflict with it: both changed the same places. The work was left as it was. Settle it on the line, or run \`${CLI_NAME} restart ${unit.key}\` to make the work again from the line, then resume.`, { kind: "assembly" });
+            return false;
+        default:
+            await escalateFeature(input, feature, `What the line ${feature.line} gained could not be merged into the Feature's work. The work was left as it was.`, { kind: "assembly" });
+            return false;
+    }
+}
+/** How long a Submission's state may stay unreadable before its Feature escalates. */
+const DEFAULT_SETTLED_TIMEOUT_MS = 4 * 60 * 60 * 1000;
 /**
  * The line is with the Authority. If the Feature merges by itself, its gates
  * decide, and a line the target moved on from goes back to be brought up to
  * date, exactly as a Ticket does; otherwise a human folds it and the system
- * waits, starting nothing.
+ * waits, starting nothing. A wait on a Submission whose state cannot be read
+ * ends in an escalation after `feature.settledTimeoutMs`.
  */
 async function settleFeature(input, feature) {
     const submission = feature.submission;
@@ -779,13 +872,19 @@ async function settleFeature(input, feature) {
     }
     const key = feature.intention.key;
     if (!feature.intention.merge_when_ready) {
-        const settled = (await input.tickets.settled?.({ reference: submission.reference })) ?? "unavailable";
+        const settled = (await input.tickets.settled?.({ id: submission.id })) ?? "unavailable";
         if (settled === "folded") {
             await input.ledger.markFeatureDone(key);
             await reportDone(input, feature, submission.reference, undefined);
         }
         else if (settled === "closed") {
-            await escalateFeature(input, feature, `The Submission ${submission.reference} was closed without being folded.`);
+            await escalateFeature(input, feature, `The Submission ${submission.reference} was closed without being folded.`, { kind: "submitted" });
+        }
+        else if (settled === "unavailable") {
+            const limit = input.options.feature?.settledTimeoutMs ?? DEFAULT_SETTLED_TIMEOUT_MS;
+            if (Date.now() - submission.submitted_at > limit) {
+                await escalateFeature(input, feature, `The state of the Submission ${submission.reference} could not be read for over ${Math.round(limit / 60_000)} minutes. Merge or close it, then resume the Feature.`, { kind: "submitted" });
+            }
         }
         return;
     }
@@ -816,6 +915,29 @@ async function reportDone(input, feature, reference, integrationReference) {
         },
     }, TO);
 }
+/**
+ * Restart an escalated Feature with a fresh workspace: what it holds is removed
+ * and the Feature is resumed, so the next pass cuts its work again from the
+ * line and judges that. Nothing is said to the tracker and nothing is fetched.
+ */
+export async function restartFeature(input, key) {
+    const got = await input.ledger.getFeature(key);
+    if (!got.ok) {
+        return { ok: false, code: "not-found" };
+    }
+    if (got.aggregate.state !== "escalated") {
+        return { ok: false, code: "not-escalated", state: got.aggregate.state };
+    }
+    if (input.featureDelivery !== undefined) {
+        await rm(featureWorkspace(input.featureDelivery, key), { recursive: true, force: true });
+    }
+    const resumed = await input.ledger.resumeFeature(key);
+    if (!resumed.ok) {
+        return { ok: false, code: "not-escalated", state: got.aggregate.state };
+    }
+    input.journal.append({ event: "restarted", key });
+    return { ok: true };
+}
 /** Abandon a Feature or a Ticket by key: whichever the ledger knows. */
 export async function cancelKey(input, key) {
     const feature = await input.ledger.getFeature(key);