@aglyn/plugins-ai 1.0.0-beta.237 → 1.0.0-beta.238

package/src/lib/jobs/ai-jobs.js

@@ -46,6 +46,7 @@ import { aiJobAutoConfirms } from "./ai-job-auto-confirm.js";
 import { AI_SITE_GUIDED_BUILD_FAILED_COPY, aiCustomerSafeCopy, aiCustomerSafeItem, aiCustomerSafeReview } from "../model/ai-job-failure-copy.js";
 import { aiSiteStartInputsOf } from "../model/ai-site-start.js";
 import { addAdminAudit } from "@aglyn/tenant-data-admin/server/admin-audit-write";
+import { AI_JOB_CANCELED_ITEM_NOTE, aiJobCancelPending, isAiJobCanceledError, watchAiJobCancel } from "./ai-job-cancel.js";
 /**
  * AI generation jobs — the Firestore state machine (AGL-2904).
  *
@@ -524,6 +525,8 @@ export async function getAiJob(firestore, orgId, jobId) {
         orchestration: job.orchestration
     } : {}, job.kind === 'build' && ((_job_inputs1 = job.inputs) == null ? void 0 : _job_inputs1['publish']) === true ? {
         publishAsked: true
+    } : {}, aiJobCancelPending(job) ? {
+        cancelRequested: true
     } : {});
 }
 function leaseIsLive(job, owner, nowMs) {
@@ -533,6 +536,52 @@ function leaseIsLive(job, owner, nowMs) {
     const until = toMillis(lease.until);
     return until !== null && until > nowMs;
 }
+/** Whether ANY owner's lease on the job is still live: a step is in flight. */ function anyLeaseIsLive(job, nowMs) {
+    var _job_lease;
+    const until = toMillis((_job_lease = job.lease) == null ? void 0 : _job_lease.until);
+    return Boolean(job.lease) && until !== null && until > nowMs;
+}
+/**
+ * The write that ends a job `canceled` (AGL-3616): it holds nothing and no
+ * lease, a step left running goes back to `pending` (it did not finish), and
+ * every item not yet built is skipped with the cancel's note. What the job
+ * built, spent and was given back stays as it is: drafts stay drafts, and
+ * credits for what ran stay charged.
+ */ function aiJobCanceledPatch(job, now, request) {
+    var _job_steps;
+    var _job_items;
+    return _extends({
+        status: 'canceled',
+        cancelRequested: request,
+        creditsReserved: 0,
+        lease: null,
+        error: null,
+        steps: ((_job_steps = job.steps) != null ? _job_steps : []).map((step)=>step.status === 'running' ? _extends({}, step, {
+                status: 'pending',
+                endedAt: null
+            }) : step)
+    }, ((_job_items = job.items) == null ? void 0 : _job_items.length) ? {
+        items: aiCanceledItems(job.items)
+    } : {}, {
+        updatedAt: now
+    });
+}
+function aiCanceledItems(items) {
+    return items.map((row)=>row.status === 'pending' || row.status === 'running' ? _extends({}, row, {
+            status: 'skipped',
+            note: AI_JOB_CANCELED_ITEM_NOTE
+        }) : row);
+}
+/** The cancel a pending-cancel job carries, or one made now for a job that has none. */ function cancelRequestOf(job, now) {
+    var _job_cancelRequested;
+    return (_job_cancelRequested = job.cancelRequested) != null ? _job_cancelRequested : aiJobCancelRequest(now, null);
+}
+/** A cancel asked now by `by`, as the job stores it (the instant is written as a Date). */ function aiJobCancelRequest(now, by) {
+    return {
+        at: now,
+        by
+    };
+}
 function nextStepIndex(job) {
     var _job_steps;
     return ((_job_steps = job.steps) != null ? _job_steps : []).findIndex((step)=>step.status === 'pending' || step.status === 'running');
@@ -561,6 +610,16 @@ function nextStepIndex(job) {
     return firestore.runTransaction(async (tx)=>{
         const job = jobFrom(await tx.get(ref));
         if (!job) return null;
+        // A cancel asked while a step ran, whose runner never came back to end
+        // it (AGL-3616): the lease ran out, so the job ends here, unclaimed.
+        if (aiJobCancelPending(job)) {
+            if (!leaseIsLive(job, owner, now.getTime())) {
+                tx.set(ref, aiJobCanceledPatch(job, now, cancelRequestOf(job, now)), {
+                    merge: true
+                });
+            }
+            return null;
+        }
         const stepIndex = claimableStepIndex(job, owner, now.getTime());
         if (stepIndex === -1) return null;
         const steps = job.steps.map((step, index)=>{
@@ -632,7 +691,17 @@ function nextStepIndex(job) {
     const ref = jobsCollection(firestore, orgId).doc(jobId);
     return firestore.runTransaction(async (tx)=>{
         const job = jobFrom(await tx.get(ref));
-        if (!job || claimableStepIndex(job, owner, now.getTime()) === -1) return null;
+        if (!job) return null;
+        // A paused workspace's job a person canceled ends canceled (AGL-3616).
+        if (aiJobCancelPending(job)) {
+            if (!leaseIsLive(job, owner, now.getTime())) {
+                tx.set(ref, aiJobCanceledPatch(job, now, cancelRequestOf(job, now)), {
+                    merge: true
+                });
+            }
+            return null;
+        }
+        if (claimableStepIndex(job, owner, now.getTime()) === -1) return null;
         tx.set(ref, {
             updatedAt: now
         }, {
@@ -734,8 +803,8 @@ function nextStepIndex(job) {
  */ export async function recordStep(firestore, orgId, jobId, owner, stepIndex, input, now = new Date()) {
     const ref = jobsCollection(firestore, orgId).doc(jobId);
     const recorded = await firestore.runTransaction(async (tx)=>{
-        var _ref, _input_items, _job_outputs, _input_outputs, _job_creditsSpent, _job_creditsReserved, _job_creditsReserved1, _job_refundedCredits;
-        var _job_lease, _input_item;
+        var _ref, _input_items, _job_outputs, _input_outputs, _job_creditsSpent, _job_creditsReserved, _job_creditsReserved1, _job_refundedCredits, _ref1;
+        var _job_lease, _input_item, _this;
         const job = jobFrom(await tx.get(ref));
         if (!job) throw new Error(`ai job ${orgId}/${jobId} vanished`);
         const steps = job.steps.map((step, index)=>{
@@ -758,8 +827,11 @@ function nextStepIndex(job) {
         const items = ledger && input.item ? aiApplyJobItemRecord(ledger, input.item, input.creditsSpent, now) : input.items ? ledger : null;
         const settled = input.status !== 'pending';
         const ownsLease = ((_job_lease = job.lease) == null ? void 0 : _job_lease.owner) === owner;
-        const parksForReview = Boolean(input.review) && !isAiJobTerminal(job.status);
-        const status = isAiJobTerminal(job.status) ? job.status : job.status === 'needs_input' ? job.status : parksForReview ? 'needs_review' : input.status === 'failed' ? 'running' : remaining > 0 ? 'queued' : 'running';
+        // A cancel asked while this step ran ends the job here (AGL-3616), in the
+        // write that records what the step cost and built: nothing after it runs.
+        const canceling = aiJobCancelPending(job);
+        const parksForReview = Boolean(input.review) && !isAiJobTerminal(job.status) && !canceling;
+        const status = isAiJobTerminal(job.status) ? job.status : canceling ? 'canceled' : job.status === 'needs_input' ? job.status : parksForReview ? 'needs_review' : input.status === 'failed' ? 'running' : remaining > 0 ? 'queued' : 'running';
         const patch = _extends({
             status,
             steps,
@@ -785,7 +857,16 @@ function nextStepIndex(job) {
         } : {}, parksForReview && input.review ? {
             review: input.review,
             error: input.review.reason === 'plan' ? null : input.review.message
-        } : {});
+        } : {}, canceling ? _extends({
+            creditsReserved: 0,
+            lease: null,
+            error: null,
+            steps: steps.map((step)=>step.status === 'running' ? _extends({}, step, {
+                    status: 'pending'
+                }) : step)
+        }, ((_this = items != null ? items : job.items) == null ? void 0 : _this.length) ? {
+            items: aiCanceledItems((_ref1 = items != null ? items : job.items) != null ? _ref1 : [])
+        } : {}) : {});
         tx.set(ref, patch, {
             merge: true
         });
@@ -823,14 +904,14 @@ async function transition(firestore, orgId, jobId, apply) {
     });
 }
 /** Every step done: the job is `done` and holds nothing. */ export async function completeAiJob(firestore, orgId, jobId, now = new Date()) {
-    const { job, changed } = await transition(firestore, orgId, jobId, (current)=>isAiJobTerminal(current.status) ? null : {
+    const { job, changed } = await transition(firestore, orgId, jobId, (current)=>isAiJobTerminal(current.status) ? null : aiJobCancelPending(current) ? aiJobCanceledPatch(current, now, cancelRequestOf(current, now)) : {
             status: 'done',
             creditsReserved: 0,
             lease: null,
             error: null,
             updatedAt: now
         });
-    if (changed) await announceAiJobTransition(job, 'done');
+    if (changed && job.status === 'done') await announceAiJobTransition(job, 'done');
     return job;
 }
 /**
@@ -845,7 +926,7 @@ async function transition(firestore, orgId, jobId, apply) {
         stepIndex: (_ref = detail == null ? void 0 : detail.stepIndex) != null ? _ref : null,
         error: (detail == null ? void 0 : detail.error) instanceof Error ? detail.error.message : (_ref1 = detail == null ? void 0 : detail.error) != null ? _ref1 : null
     });
-    const { job, changed } = await transition(firestore, orgId, jobId, (current)=>isAiJobTerminal(current.status) ? null : {
+    const { job, changed } = await transition(firestore, orgId, jobId, (current)=>isAiJobTerminal(current.status) ? null : aiJobCancelPending(current) ? aiJobCanceledPatch(current, now, cancelRequestOf(current, now)) : {
             status: 'failed',
             error: message,
             creditsReserved: 0,
@@ -857,7 +938,7 @@ async function transition(firestore, orgId, jobId, apply) {
                 }) : step),
             updatedAt: now
         });
-    if (changed) {
+    if (changed && job.status === 'failed') {
         var _ref2, _ref3;
         await announceAiJobTransition(job, 'failed', {
             ours: (_ref2 = detail == null ? void 0 : detail.ours) != null ? _ref2 : false,
@@ -872,7 +953,7 @@ async function transition(firestore, orgId, jobId, apply) {
  * step goes back to `pending` so the same step is what runs when the beat
  * tries again; its attempt is not counted, because nothing was attempted.
  */ export async function markAiJobNeedsInput(firestore, orgId, jobId, reason, now = new Date()) {
-    const { job } = await transition(firestore, orgId, jobId, (current)=>isAiJobTerminal(current.status) ? null : {
+    const { job } = await transition(firestore, orgId, jobId, (current)=>isAiJobTerminal(current.status) ? null : aiJobCancelPending(current) ? aiJobCanceledPatch(current, now, cancelRequestOf(current, now)) : {
             status: 'needs_input',
             error: reason,
             lease: null,
@@ -888,23 +969,46 @@ async function transition(firestore, orgId, jobId, apply) {
     return job;
 }
 /**
- * Cancel. Idempotent — a terminal job is returned unchanged — and it does
- * not wait for a step in flight: the runner holding the lease finishes its
- * provider call, records what it cost, and finds the job canceled when it
- * goes to complete it.
+ * Cancel (AGL-2904, AGL-3616). One transaction decides it against whatever
+ * else commits, so a cancel racing a finish resolves to whichever committed
+ * first: a job already `done`, `failed` or `canceled` is returned unchanged.
  *
- * The activity row is written only when the cancel changed something: a
- * second cancel of the same job is not a second act. `actor` is the member
- * who canceled; a cancel with nobody named is recorded as nobody's rather
- * than as the creator's.
+ * - **No step in flight** (queued, parked for a person or the meter, or a
+ *   lease that ran out): the job ends `canceled` in this write. It holds
+ *   nothing, its unbuilt items are skipped, and nothing it built is touched.
+ * - **A step in flight** (a live lease): the cancel is recorded as
+ *   `cancelRequested` and the job keeps its status until the runner holding
+ *   the lease comes back. The runner watches for it and aborts its provider
+ *   call; whatever the step had spent is metered, and the machine's next
+ *   write ends the job `canceled` (`recordStep`, or the finish or failure it
+ *   was about to write, which a pending cancel turns into the cancel).
+ *
+ * Either way a canceled job is never failed: no `ai.jobFailed` alert, no
+ * failure notice, no give-back (what ran stays charged), no publish.
+ *
+ * The activity row is written once, when the cancel is first asked: a second
+ * cancel of the same job is not a second act. `actor` is the member who
+ * canceled; a cancel with nobody named is recorded as nobody's rather than
+ * as the creator's.
  */ export async function cancelAiJob(firestore, orgId, jobId, now = new Date(), actor = null) {
-    const result = await transition(firestore, orgId, jobId, (current)=>isAiJobTerminal(current.status) ? null : {
-            status: 'canceled',
-            creditsReserved: 0,
-            lease: null,
+    var _ref;
+    const request = aiJobCancelRequest(now, (_ref = actor == null ? void 0 : actor.uid) != null ? _ref : null);
+    let firstAsk = false;
+    const result = await transition(firestore, orgId, jobId, (current)=>{
+        if (isAiJobTerminal(current.status)) return null;
+        const inFlight = anyLeaseIsLive(current, now.getTime());
+        if (current.cancelRequested) {
+            // Asked already: nothing new while its step is still in flight; a
+            // step whose runner never came back is ended now.
+            return inFlight ? null : aiJobCanceledPatch(current, now, current.cancelRequested);
+        }
+        firstAsk = true;
+        return inFlight ? {
+            cancelRequested: request,
             updatedAt: now
-        });
-    if (result.changed) {
+        } : aiJobCanceledPatch(current, now, request);
+    });
+    if (result.changed && firstAsk) {
         await logAiJobCanceled(orgId, actor != null ? actor : {
             uid: null
         }, {
@@ -975,6 +1079,13 @@ async function transition(firestore, orgId, jobId, apply) {
     var _detail_ours;
     var _current_items;
     const current = await getAiJob(firestore, orgId, jobId);
+    // A person's cancel asked while the step ran is the job's end (AGL-3616):
+    // not our failure, so nothing is given back and nobody is told.
+    if (current && aiJobCancelPending(current)) {
+        return failAiJob(firestore, orgId, jobId, message, _extends({}, detail, {
+            ours: false
+        }), now);
+    }
     // A build that got as far as its items settles item by item (AGL-3616):
     // what it delivered stands, and what it did not is given back per item.
     if (current && aiJobSettlesByItem(current.kind) && ((_current_items = current.items) == null ? void 0 : _current_items.length) && !isAiJobTerminal(current.status)) {
@@ -1090,6 +1201,8 @@ async function transition(firestore, orgId, jobId, apply) {
     const { now } = input;
     let job = await getAiJob(firestore, orgId, jobId);
     if (!job || isAiJobTerminal(job.status)) return job;
+    // A cancel asked before the build settled is how it ends (AGL-3616).
+    if (aiJobCancelPending(job)) return completeAiJob(firestore, orgId, jobId, now);
     const stopped = input.stopped;
     if (stopped && ((_job_items = job.items) != null ? _job_items : []).some((row)=>row.status === 'pending' || row.status === 'running')) {
         var _job_items4;
@@ -1422,7 +1535,7 @@ async function transition(firestore, orgId, jobId, apply) {
  * round.
  */ export async function runAiJobStep(firestore, orgId, jobId, options) {
     var _options_now, _step_attempts, _options_org, _data, _ref, _ref1, _outcome_effort, _step_passes, _options_actor;
-    var _reservation_allotment, _reservation_allotment1, _outcome_review, _itemRecord_failure, _recorded_job_items;
+    var _reservation_allotment, _reservation_allotment1, _options_signal, _options_signal1, _outcome_review, _itemRecord_failure, _recorded_job_items;
     const now = (_options_now = options.now) != null ? _options_now : new Date();
     // A reservation the caller took before the job existed is a message
     // counted against the org; a step that fails before the provider is
@@ -1539,6 +1652,11 @@ async function transition(firestore, orgId, jobId, apply) {
     if (!reservation.allowed) {
         const refusal = aiJobRefusalText(org, reservation);
         const parked = await markAiJobNeedsInput(firestore, orgId, jobId, refusal, now);
+        // A cancel asked meanwhile ended the job instead of parking it (AGL-3616).
+        if (parked.status === 'canceled') return {
+            outcome: 'canceled',
+            job: parked
+        };
         // Nobody parked the job — the meter did — so the row carries no actor
         // rather than the creator's name on an act they did not perform. Only
         // when the job was not already parked for this: the reason the last
@@ -1577,6 +1695,21 @@ async function transition(firestore, orgId, jobId, apply) {
         var _resolveAiModelChoice;
         return (_resolveAiModelChoice = resolveAiModelChoice(kind, (_job_model = job.model) != null ? _job_model : null, bounds)) == null ? void 0 : _resolveAiModelChoice.model;
     };
+    // A person's cancel stops the step (AGL-3616): the step's signal is the
+    // caller's budget AND a watch on the job, so a cancel aborts the provider
+    // call in flight, stops one about to start, and stops a write the step
+    // asks `throwIfAiJobCanceled` about first.
+    const stepSignal = new AbortController();
+    const forwardBudget = ()=>{
+        var _options_signal;
+        return stepSignal.abort((_options_signal = options.signal) == null ? void 0 : _options_signal.reason);
+    };
+    if ((_options_signal = options.signal) == null ? void 0 : _options_signal.aborted) forwardBudget();
+    else (_options_signal1 = options.signal) == null ? void 0 : _options_signal1.addEventListener('abort', forwardBudget, {
+        once: true
+    });
+    const stopWatch = watchAiJobCancel(()=>getAiJob(firestore, orgId, jobId), stepSignal, options.cancelPollMs);
+    const canceledMidStep = ()=>stepSignal.signal.aborted && isAiJobCanceledError(stepSignal.signal.reason);
     let outcome;
     // The runner's wall-clock time, recorded on the step beside its tokens
     // (AGL-2937): measured here, around the call, rather than asked of it.
@@ -1586,7 +1719,7 @@ async function transition(firestore, orgId, jobId, apply) {
             job,
             stepIndex,
             now,
-            signal: options.signal,
+            signal: stepSignal.signal,
             firestore,
             org,
             modelFor
@@ -1602,6 +1735,18 @@ async function transition(firestore, orgId, jobId, apply) {
                 jobId,
                 releaseError
             }));
+        // Canceled mid-step (AGL-3616): the step is handed back unrun and the
+        // record ends the job canceled — never failed, never retried.
+        if (isAiJobCanceledError(error) || canceledMidStep()) {
+            const { job: canceled } = await recordStep(firestore, orgId, jobId, options.owner, stepIndex, {
+                status: 'pending',
+                creditsSpent: 0
+            }, now);
+            return {
+                outcome: 'canceled',
+                job: canceled
+            };
+        }
         const retryable = error instanceof AiUpstreamError && error.retryable || isAbort(error);
         if (retryable) {
             const { job: requeued } = await recordStep(firestore, orgId, jobId, options.owner, stepIndex, {
@@ -1624,6 +1769,10 @@ async function transition(firestore, orgId, jobId, apply) {
                 free: (_reservation_free = reservation.free) != null ? _reservation_free : null
             }, now)
         };
+    } finally{
+        var _options_signal2;
+        stopWatch();
+        (_options_signal2 = options.signal) == null ? void 0 : _options_signal2.removeEventListener('abort', forwardBudget);
     }
     const latencyMs = Math.max(0, Date.now() - runStarted);
     // A step that failed before it reached the provider — the site it builds
@@ -1774,7 +1923,9 @@ async function transition(firestore, orgId, jobId, apply) {
     // A build item that failed on our side gives back its own attempt's spend
     // before its row is written (AGL-3616), so the row records what was given.
     let itemRecord = aiJobSettlesByItem(job.kind) ? outcome.item : undefined;
-    if ((itemRecord == null ? void 0 : itemRecord.status) === 'failed' && ((_itemRecord_failure = itemRecord.failure) == null ? void 0 : _itemRecord_failure.ours)) {
+    // An item a person's cancel cut short is not our failure (AGL-3616): it is
+    // given nothing back, and its row is the cancel's.
+    if ((itemRecord == null ? void 0 : itemRecord.status) === 'failed' && ((_itemRecord_failure = itemRecord.failure) == null ? void 0 : _itemRecord_failure.ours) && !canceledMidStep()) {
         var _ref2, _outcome_items, _ref3, _ref4, _reservation_free5;
         const slot = itemRecord.slot;
         const row = ((_ref2 = (_outcome_items = outcome.items) != null ? _outcome_items : job.items) != null ? _ref2 : []).find((one)=>one.slot === slot);
@@ -1855,6 +2006,11 @@ async function transition(firestore, orgId, jobId, apply) {
             }
         });
     }
+    // A cancel asked while the step ran ended the job in that record (AGL-3616).
+    if (recorded.job.status === 'canceled') return {
+        outcome: 'canceled',
+        job: recorded.job
+    };
     if ((review == null ? void 0 : review.reason) === 'plan' && recorded.job.status === 'needs_review' && aiJobAutoConfirms(job)) {
         // A guided site start confirms its own plan (AGL-3594): the resume
         // door's confirmation, by the job's creator, and the build queued.