@3minapi/cli 0.0.0-stage → 0.22.0

package/dist/modules/workers/task-runner.js

@@ -0,0 +1,892 @@
+import { RepeatScheduler, parseSettings } from "./repeat-scheduler.js";
+import { ApiError } from "../../core/api.js";
+import { ERROR_MESSAGE_MAX_CHARS, EVENT_MESSAGE_MAX_CHARS, REPORT_ENVELOPE_RESERVE, REPEAT_FAIL_STREAK_MAX, REPORT_MAX_BYTES, RESULT_RESERVED_KEY, STATUS_MESSAGE_MAX_CHARS, StopError, TASK_CODES, TASK_ERROR_CODE } from "../../core/domain/index.js";
+import { computeShape } from "./results/shape.js";
+import { substituteResult, substituteStatusMessage } from "./results/substitute.js";
+import { formatCount, oneLine, stripControl, truncate } from "../../core/text.js";
+export const CANCEL = {
+    server: 'cancelled',
+    superseded: 'superseded',
+    shutdown: 'host_stopped'
+};
+const STOPPED_PREFIX = 'stopped: ';
+const PENDING_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+export class TaskRunner {
+    opts;
+    inflight = new Map();
+    pending = [];
+    repeatAcks = 0;
+    flushing = null;
+    flushAgain = null;
+    scheduler;
+    constructor(opts) {
+        this.opts = opts;
+        this.scheduler = new RepeatScheduler({
+            db: opts.db,
+            logger: opts.logger,
+            now: opts.now,
+            run: (taskId, runNo) => this.runRepeat(taskId, runNo),
+            every: opts.repeatEvery
+        });
+    }
+    get free() {
+        return Math.max(0, this.opts.executor.concurrency - this.inflight.size - this.repeatAcks);
+    }
+    get busy() {
+        return this.inflight.size;
+    }
+    get repeating() {
+        return this.scheduler.size;
+    }
+    startRepeats() {
+        this.scheduler.load();
+    }
+    dropRepeats(workers) {
+        const gone = new Set(workers);
+        const ids = new Set(this.scheduler.dropWorkers(workers));
+        for (const row of this.opts.db.repeatTasks())
+            if (gone.has(row.worker))
+                ids.add(row.task_id);
+        for (const id of ids) {
+            this.scheduler.remove(id);
+            this.opts.logger.info('repeating task stopped; its worker was deleted on the server', {
+                task_id: id
+            });
+            this.opts.db.markFinished(id, 'cancelled', this.opts.now(), 'worker deleted on the server');
+            this.opts.executor.cancel(id, CANCEL.server);
+        }
+    }
+    handle(task, reporter, { ack }) {
+        return this.process(task, reporter, ack).catch((err) => {
+            this.opts.logger.error('task handling failed', {
+                task_id: task.task_id,
+                worker: task.worker,
+                message: err?.stack ?? String(err)
+            });
+        });
+    }
+    cancel(taskId) {
+        const repeat = this.scheduler.has(taskId) || this.isWaitingRepeat(taskId);
+        if (repeat) {
+            this.scheduler.remove(taskId);
+            this.opts.db.markFinished(taskId, 'cancelled', this.opts.now());
+            this.event(`${this.workerOf(taskId)}  ${shortId(taskId)}  repeat cancelled`);
+        }
+        if (this.opts.executor.cancel(taskId, CANCEL.server) || repeat) {
+            this.opts.logger.info('task cancelled by the server', { task_id: taskId });
+        }
+    }
+    isWaitingRepeat(taskId) {
+        const row = this.opts.db.getTask(taskId);
+        return Boolean(row?.repeat) && (row.state === 'received' || row.state === 'running');
+    }
+    async shutdown() {
+        this.scheduler.stop();
+        await this.opts.executor.cancelAll(CANCEL.shutdown);
+        await this.settled();
+    }
+    async halt(reason) {
+        this.scheduler.stop();
+        await (this.flushAgain ?? this.flushing);
+        this.pending = [];
+        await this.opts.executor.haltAll(STOPPED_PREFIX + reason);
+        await this.settled();
+    }
+    flushPending() {
+        if (this.flushing) {
+            this.flushAgain ??= this.flushing.then(() => {
+                this.flushAgain = null;
+                return this.flushPending();
+            });
+            return this.flushAgain;
+        }
+        this.flushing = this.flushQueue().finally(() => (this.flushing = null));
+        return this.flushing;
+    }
+    async flushQueue() {
+        const queue = this.pending;
+        this.pending = [];
+        for (const [i, p] of queue.entries()) {
+            const { r, at } = await this.deliver(p.task, p.reporter, p.replies);
+            if (r === 'stop') {
+                this.pending.push({ ...p, replies: p.replies.slice(at) }, ...queue.slice(i + 1));
+                return;
+            }
+            if (r === 'retry') {
+                if (this.opts.now() - p.since < PENDING_MAX_AGE_MS) {
+                    this.pending.push({ ...p, replies: p.replies.slice(at) });
+                }
+                else {
+                    this.setReply(p.task, 'rejected');
+                    this.finish(p.task, p.replies[at].state, 'reply was not delivered within 24 hours');
+                }
+                continue;
+            }
+            this.settle(p.task, r, p.replies[at]);
+        }
+    }
+    async recover(reporter) {
+        const rows = this.opts.db.unfinishedTasks();
+        for (const row of rows)
+            await this.recoverOne(row, reporter);
+        for (const row of this.opts.db.repeatTasks())
+            await this.recoverRepeat(row);
+        return rows.length;
+    }
+    async recoverRepeat(row) {
+        const task = this.repeatEnvelope(row);
+        if (!task)
+            return;
+        const runNo = row.run_no ?? 0;
+        if (runNo <= (row.runs ?? 0)) {
+            await this.closeIfDone(task, tallyOf(row), row.last_run_at ?? this.opts.now(), null);
+            return;
+        }
+        this.opts.logger.warn('a run of a repeating task was cut off by the previous run', {
+            task_id: row.task_id,
+            worker: row.worker,
+            run_no: runNo
+        });
+        const message = "the 3minapi CLI on the user's PC stopped during this run — it is unknown whether it finished";
+        await this.recordRun(task, runNo, tallyOf(row), {
+            ok: false,
+            reason: 'host_restarted',
+            message,
+            payload: JSON.stringify(failedBody('host_restarted', message).error),
+            report: false,
+            close: true
+        });
+    }
+    async recoverOne(row, reporter) {
+        const { logger } = this.opts;
+        const pull = Boolean(row.lease_id);
+        if (row.repeat) {
+            logger.info('dropping an unacknowledged repeating task; the server will deliver it again', {
+                task_id: row.task_id,
+                worker: row.worker
+            });
+            return this.opts.db.markFinished(row.task_id, 'superseded', this.opts.now(), 'not acknowledged before the restart; the server will deliver it again');
+        }
+        const task = {
+            task_id: row.task_id,
+            worker: row.worker,
+            input: null,
+            callback_url: row.callback_url ?? null,
+            lease_id: pull ? row.lease_id : null,
+            delivery_attempt: row.delivery_attempt,
+            source: 'pull'
+        };
+        const id = `${row.worker}  ${shortId(row.task_id)}`;
+        const attrs = { task_id: row.task_id, worker: row.worker, state: row.state };
+        const message = "the 3minapi CLI on the user's PC stopped before this task finished — it is unknown whether it ran";
+        if (!pull) {
+            logger.warn('closing a pushed task left unfinished by the previous run', attrs);
+            return this.finish(task, 'failed', message);
+        }
+        const stored = this.storedResult(row.task_id);
+        if (stored && stored.reply !== 'pending') {
+            return this.finish(task, stored.status);
+        }
+        if (stored) {
+            logger.info('resending the reply of a task the previous run finished', attrs);
+            if (stored.status === 'completed')
+                return this.completed(task, reporter, stored.payload, 0, stored);
+            const failed = parseStoredError(stored.payload);
+            if (failed) {
+                this.event(`${id}  failed earlier (${failed.reason}); resent its reply`);
+                return this.terminal(task, reporter, this.failedReplies(failed.reason, failed.message));
+            }
+        }
+        if (row.state === 'running' || row.acked_at != null) {
+            logger.warn('closing a task left unfinished by the previous run', attrs);
+            this.event(`${id}  failed (host_restarted): the previous run stopped before it finished`);
+            if (task.callback_url === null)
+                return this.finish(task, 'failed', message);
+            this.store(task, 'failed', JSON.stringify(failedBody('host_restarted', message).error));
+            return this.terminal(task, reporter, this.failedReplies('host_restarted', message));
+        }
+        logger.info('dropping an unacknowledged task; the server will deliver it again', attrs);
+        this.finish(task, 'superseded', 'not acknowledged before the restart; the server will deliver it again');
+    }
+    storedResult(taskId) {
+        if (!this.opts.results)
+            return null;
+        try {
+            return this.opts.results.get(taskId);
+        }
+        catch (err) {
+            this.opts.logger.warn('could not read the saved result', {
+                task_id: taskId,
+                message: err.message
+            });
+            return null;
+        }
+    }
+    async settled() {
+        while (this.inflight.size > 0) {
+            await Promise.all([...this.inflight.values()].map((f) => f.done));
+        }
+    }
+    async process(task, reporter, ack) {
+        const { executor, db, logger } = this.opts;
+        if (task.repeat && task.lease_id !== null)
+            return this.receiveRepeat(task, reporter, ack);
+        const prev = this.inflight.get(task.task_id);
+        if (prev) {
+            if (prev.leaseId === task.lease_id)
+                return;
+            executor.cancel(task.task_id, CANCEL.superseded);
+            await prev.done;
+        }
+        if (db.receive(this.received(task), this.opts.now()) === 'duplicate-done') {
+            logger.info('task already finished here; not running it again', { task_id: task.task_id });
+            return;
+        }
+        let release;
+        const entry = { leaseId: task.lease_id, done: new Promise((r) => (release = r)) };
+        this.inflight.set(task.task_id, entry);
+        try {
+            await this.execute(task, reporter, ack);
+        }
+        finally {
+            this.releaseSlot(() => {
+                if (this.inflight.get(task.task_id) === entry)
+                    this.inflight.delete(task.task_id);
+            });
+            release();
+        }
+    }
+    async execute(task, reporter, ack) {
+        const { executor, db, logger, now } = this.opts;
+        const attrs = { task_id: task.task_id, worker: task.worker };
+        if (ack) {
+            const r = await this.send(task, reporter, { status: 'working' });
+            if (r === 'ok')
+                db.markAcked(task.task_id, now());
+            if (r === 'stop')
+                return this.finish(task, 'cancelled', 'stopped before it started');
+            if (r === 'drop')
+                return this.finish(task, 'superseded', 'the server no longer accepts this task');
+            if (r === 'retry' || r === 'too_large') {
+                logger.warn('could not confirm the acknowledgement; running the task anyway', attrs);
+            }
+        }
+        let startedAt = null;
+        const outcome = await executor.run({ id: task.task_id, worker: task.worker, input: task.input }, {
+            onStart: () => {
+                startedAt = now();
+                db.markRunning(task.task_id, startedAt);
+                logger.info('task started', attrs);
+                this.event(`${task.worker}  ${shortId(task.task_id)}  started`);
+            },
+            onReport: async (message) => {
+                const r = await this.send(task, reporter, {
+                    status: 'working',
+                    status_message: clip(message, STATUS_MESSAGE_MAX_CHARS)
+                });
+                if (r === 'drop')
+                    executor.cancel(task.task_id, CANCEL.superseded);
+            }
+        });
+        const ms = startedAt === null ? 0 : now() - startedAt;
+        await this.conclude(task, reporter, outcome, ms);
+    }
+    async conclude(task, reporter, outcome, ms) {
+        const { logger } = this.opts;
+        const attrs = { task_id: task.task_id, worker: task.worker, duration_ms: ms };
+        if (outcome.status === 'cancelled') {
+            if (outcome.reason.startsWith(STOPPED_PREFIX)) {
+                logger.warn('task aborted by the stop; not reported', attrs);
+                this.event(`${task.worker}  ${shortId(task.task_id)}  aborted (not reported — calls are stopped)`);
+                return this.finish(task, 'cancelled', outcome.reason);
+            }
+            if (outcome.reason === CANCEL.shutdown) {
+                logger.info('task aborted because the CLI is stopping', attrs);
+                this.event(`${task.worker}  ${shortId(task.task_id)}  aborted (CLI stopping)`);
+                const message = "the 3minapi CLI on the user's PC stopped before the task finished";
+                this.store(task, 'failed', JSON.stringify(failedBody('host_stopped', message).error));
+                return this.terminal(task, reporter, [
+                    { body: failedBody('host_stopped', message), state: 'failed', reply: 'failed' }
+                ]);
+            }
+            const state = outcome.reason === CANCEL.superseded ? 'superseded' : 'cancelled';
+            logger.info(`task ${state}`, attrs);
+            this.event(`${task.worker}  ${shortId(task.task_id)}  ${state}`);
+            return this.finish(task, state);
+        }
+        if (outcome.status === 'completed')
+            return this.completed(task, reporter, outcome.resultJson, ms);
+        const builtin = this.opts.isBuiltin?.(task.worker) ?? false;
+        logger.warn('task failed', {
+            ...attrs,
+            reason: outcome.reason,
+            ...(builtin ? {} : { message: outcome.message })
+        });
+        this.event(`${task.worker}  ${shortId(task.task_id)}  failed (${outcome.reason}): ${clip(outcome.message, EVENT_MESSAGE_MAX_CHARS)}`);
+        this.store(task, 'failed', JSON.stringify(failedBody(outcome.reason, outcome.detail ?? outcome.message).error));
+        return this.terminal(task, reporter, [
+            {
+                body: failedBody(outcome.reason, outcome.message),
+                state: 'failed',
+                reply: 'failed',
+                error: builtin ? outcome.reason : outcome.message
+            }
+        ]);
+    }
+    completed(task, reporter, resultJson, ms, resend = null) {
+        const { logger } = this.opts;
+        const attrs = { task_id: task.task_id, worker: task.worker, duration_ms: ms };
+        const id = `${task.worker}  ${shortId(task.task_id)}`;
+        const done = resend ? 'completed earlier; resent its reply' : `completed in ${ms} ms`;
+        const value = JSON.parse(resultJson);
+        const bytes = Buffer.byteLength(resultJson);
+        let shape = null;
+        let saved;
+        if (resend) {
+            shape =
+                resend.shape && resend.shape_hash ? { shape: resend.shape, hash: resend.shape_hash } : null;
+            saved = true;
+        }
+        else {
+            if (this.opts.results && !this.opts.isBuiltin?.(task.worker))
+                shape = computeShape(value);
+            saved = this.store(task, 'completed', resultJson, shape);
+        }
+        const where = saved
+            ? "it was saved on the user's PC (3minapi worker results show " + task.task_id + ')'
+            : "it could not be saved on the user's PC";
+        if (isReserved(value)) {
+            const message = `the result has a top-level "${RESULT_RESERVED_KEY}" key, which 3minapi reserves — rename it; ${where}`;
+            logger.warn('task failed', { ...attrs, reason: 'reserved_result_key' });
+            this.event(`${id}  failed (reserved_result_key): rename the top-level "${RESULT_RESERVED_KEY}" key`);
+            return this.terminal(task, reporter, this.failedReplies('reserved_result_key', message));
+        }
+        const tooLarge = this.failedReplies('result_too_large', `the result is ${formatCount(bytes)} bytes, over the 100 KiB reply limit; ${where}`);
+        const substitute = task.source === 'pull' && saved && shape
+            ? [
+                {
+                    body: {
+                        status: 'completed',
+                        status_message: substituteStatusMessage(bytes),
+                        result: substituteResult({
+                            taskId: task.task_id,
+                            worker: task.worker,
+                            bytes,
+                            shape: shape.shape,
+                            shapeHash: shape.hash,
+                            reader: this.opts.reader?.() ?? null
+                        })
+                    },
+                    state: 'completed',
+                    reply: 'substitute'
+                }
+            ]
+            : [];
+        const max = this.opts.maxResultBytes ?? REPORT_MAX_BYTES - REPORT_ENVELOPE_RESERVE;
+        if (bytes <= max) {
+            logger.info('task completed', attrs);
+            this.event(`${id}  ${done}`);
+            return this.terminal(task, reporter, [
+                { body: { status: 'completed', result: value }, state: 'completed', reply: 'result' },
+                ...substitute,
+                ...tooLarge
+            ]);
+        }
+        if (substitute.length > 0) {
+            logger.info('task completed; the result is stored locally and a summary was sent', {
+                ...attrs,
+                bytes
+            });
+            this.event(`${id}  ${done} — ${formatCount(bytes)} bytes saved in data.db, sent a summary`);
+            return this.terminal(task, reporter, [...substitute, ...tooLarge]);
+        }
+        logger.warn('task failed', { ...attrs, reason: 'result_too_large', bytes, saved });
+        this.event(`${id}  failed (result_too_large): ${formatCount(bytes)} bytes${saved ? ' — saved in data.db' : ''}`);
+        return this.terminal(task, reporter, tooLarge);
+    }
+    failedReplies(reason, message) {
+        return [
+            { body: failedBody(reason, message), state: 'failed', reply: 'failed', error: message }
+        ];
+    }
+    store(task, status, payload, shape = null, runNo) {
+        const { results } = this.opts;
+        if (!results || this.opts.isBuiltin?.(task.worker))
+            return false;
+        try {
+            results.save({
+                taskId: task.task_id,
+                worker: task.worker,
+                createdAt: this.opts.now(),
+                source: task.source,
+                status,
+                reply: task.source === 'serve' || runNo !== undefined ? 'none' : 'pending',
+                payload,
+                shape: shape?.shape ?? null,
+                shapeHash: shape?.hash ?? null,
+                ...(runNo !== undefined ? { runNo } : {})
+            });
+            return true;
+        }
+        catch (err) {
+            const message = err.message;
+            this.opts.logger.warn('could not save the result to data.db', {
+                task_id: task.task_id,
+                message
+            });
+            this.event(`${task.worker}  ${shortId(task.task_id)}  result not saved to data.db: ${message}`);
+            return false;
+        }
+    }
+    setReply(task, reply) {
+        if (task.repeat)
+            return;
+        if (!this.opts.results || task.source === 'serve' || this.opts.isBuiltin?.(task.worker))
+            return;
+        try {
+            this.opts.results.setReply(task.task_id, reply);
+        }
+        catch (err) {
+            this.opts.logger.warn('could not update the saved result', {
+                task_id: task.task_id,
+                message: err.message
+            });
+        }
+    }
+    async terminal(task, reporter, replies) {
+        const { r, at } = await this.deliver(task, reporter, replies);
+        if (r === 'retry') {
+            this.opts.logger.warn('reply not delivered; will retry later', {
+                task_id: task.task_id
+            });
+            this.pending.push({
+                task,
+                reporter: reporter,
+                replies: replies.slice(at),
+                since: this.opts.now()
+            });
+            return;
+        }
+        this.settle(task, r, replies[at]);
+    }
+    async deliver(task, reporter, replies) {
+        let at = 0;
+        let r = await this.send(task, reporter, replies[0].body);
+        while (r === 'too_large' && at < replies.length - 1) {
+            at++;
+            r = await this.send(task, reporter, replies[at].body);
+        }
+        return { r, at };
+    }
+    settle(task, r, reply) {
+        switch (r) {
+            case 'ok':
+                this.setReply(task, reply.reply);
+                return this.finish(task, reply.state, reply.error);
+            case 'skip':
+                return this.finish(task, reply.state, reply.error);
+            case 'drop':
+                this.setReply(task, 'rejected');
+                return this.finish(task, reply.state, 'the server no longer accepts this task');
+            case 'stop':
+                this.setReply(task, 'rejected');
+                return this.finish(task, reply.state, 'not reported — calls are stopped');
+            case 'too_large':
+                this.setReply(task, 'rejected');
+                return this.finish(task, 'failed', 'the server rejected every reply as too large');
+        }
+    }
+    async send(task, reporter, body, invalidAsDrop = false, runNo) {
+        if (!reporter || task.callback_url === null)
+            return 'skip';
+        const withLease = task.lease_id === null ? body : { ...body, lease_id: task.lease_id };
+        try {
+            await reporter.report(task.callback_url, withLease);
+            return 'ok';
+        }
+        catch (err) {
+            if (err instanceof StopError) {
+                this.opts.onStop(err);
+                return 'stop';
+            }
+            if (err instanceof ApiError) {
+                const attrs = {
+                    task_id: task.task_id,
+                    ...(runNo !== undefined ? { run_no: runNo } : {}),
+                    status: err.status,
+                    code: err.code,
+                    message: err.message
+                };
+                if (err.code !== null && TASK_CODES.includes(err.code)) {
+                    this.opts.logger.info('the server dropped the task', attrs);
+                    return 'drop';
+                }
+                if (invalidAsDrop && err.code === 'invalid_request') {
+                    this.opts.logger.error('the server rejected a repeat report (a CLI bug)', attrs);
+                    return 'drop';
+                }
+                if (err.code === 'payload_too_large' || err.code === 'unsupported_media_type') {
+                    this.opts.logger.warn('the server rejected the reply', attrs);
+                    return 'too_large';
+                }
+                this.opts.logger.warn('reply failed', attrs);
+                return 'retry';
+            }
+            throw err;
+        }
+    }
+    async receiveRepeat(task, reporter, ack) {
+        const { db, logger, now } = this.opts;
+        const attrs = { task_id: task.task_id, worker: task.worker };
+        if (db.receive(this.received(task), now()) === 'duplicate-done') {
+            logger.info('task already finished here; not running it again', attrs);
+            return;
+        }
+        if (this.opts.isBuiltin?.(task.worker) || !this.opts.repeatReporter) {
+            const message = this.opts.isBuiltin?.(task.worker)
+                ? 'the built-in results reader does not repeat — create a single task for each read'
+                : 'this 3minapi CLI cannot run repeating tasks here';
+            logger.warn('repeating task rejected', attrs);
+            return this.terminal(task, reporter, this.failedReplies('invalid_input', message));
+        }
+        this.repeatAcks++;
+        let r = 'skip';
+        try {
+            if (ack)
+                r = await this.send(task, reporter, { status: 'working' });
+        }
+        finally {
+            this.releaseSlot(() => this.repeatAcks--);
+        }
+        if (r === 'ok')
+            db.markAcked(task.task_id, now());
+        if (r === 'stop')
+            return this.finish(task, 'cancelled', 'stopped before it started');
+        if (r === 'drop')
+            return this.finish(task, 'superseded', 'the server no longer accepts this task');
+        if (r === 'retry' || r === 'too_large') {
+            logger.warn('could not confirm the acknowledgement; running the repeating task anyway', attrs);
+        }
+        if (this.scheduler.has(task.task_id)) {
+            logger.info('repeating task delivered again; continuing with the new lease', attrs);
+            return;
+        }
+        const every = task.repeat.max_runs === undefined ? '' : `, ${task.repeat.max_runs} runs`;
+        logger.info('repeating task accepted', {
+            ...attrs,
+            interval_seconds: task.repeat.interval_seconds,
+            max_runs: task.repeat.max_runs ?? null
+        });
+        this.event(`${task.worker}  ${shortId(task.task_id)}  repeating every ${task.repeat.interval_seconds}s${every}`);
+        const row = db.getTask(task.task_id);
+        this.scheduler.add(task.task_id, task.worker, task.repeat, Math.max(row?.run_no ?? 0, row?.runs ?? 0));
+    }
+    async runRepeat(taskId, runNo) {
+        const { executor, db, logger, now } = this.opts;
+        const row = db.getTask(taskId);
+        const task = row && (row.state === 'received' || row.state === 'running')
+            ? this.repeatEnvelope(row)
+            : null;
+        if (!row || !task) {
+            this.scheduler.remove(taskId);
+            return;
+        }
+        let release;
+        const entry = { leaseId: task.lease_id, done: new Promise((r) => (release = r)) };
+        this.inflight.set(taskId, entry);
+        try {
+            let startedAt = null;
+            const outcome = await executor.run({ id: taskId, worker: task.worker, input: task.input, run: runNo }, {
+                onStart: () => {
+                    startedAt = now();
+                    db.startRun(taskId, runNo, startedAt);
+                    logger.info('repeat run started', {
+                        task_id: taskId,
+                        worker: task.worker,
+                        run_no: runNo
+                    });
+                },
+                onReport: async (message) => {
+                    const r = await this.send(this.withCurrentLease(task), this.opts.repeatReporter, { status: 'working', status_message: clip(message, STATUS_MESSAGE_MAX_CHARS) }, false, runNo);
+                    if (r === 'drop')
+                        executor.cancel(taskId, CANCEL.superseded);
+                }
+            });
+            const ms = startedAt === null ? 0 : now() - startedAt;
+            const current = db.getTask(taskId) ?? row;
+            await this.concludeRun(this.withCurrentLease(task, current), runNo, tallyOf(current), outcome, ms, startedAt !== null);
+        }
+        catch (err) {
+            logger.error('repeat run failed', {
+                task_id: taskId,
+                run_no: runNo,
+                message: err?.stack ?? String(err)
+            });
+        }
+        finally {
+            this.releaseSlot(() => {
+                if (this.inflight.get(taskId) === entry)
+                    this.inflight.delete(taskId);
+            });
+            release();
+        }
+    }
+    releaseSlot(release) {
+        const wasFull = this.free === 0;
+        release();
+        if (wasFull && this.free > 0)
+            this.opts.onSlotFree?.();
+    }
+    async concludeRun(task, runNo, tally, outcome, ms, started) {
+        const { logger } = this.opts;
+        const attrs = { task_id: task.task_id, worker: task.worker, run_no: runNo, duration_ms: ms };
+        if (!this.scheduler.has(task.task_id))
+            return;
+        if (outcome.status === 'cancelled') {
+            if (outcome.reason === CANCEL.server || outcome.reason === CANCEL.superseded) {
+                this.scheduler.remove(task.task_id);
+                logger.info('repeating task closed', { ...attrs, reason: outcome.reason });
+                return this.finish(task, 'cancelled', outcome.reason === CANCEL.superseded
+                    ? 'the server no longer accepts this task'
+                    : undefined);
+            }
+            if (!started) {
+                logger.info('queued repeat run dropped; the repeating task resumes on the next start', attrs);
+                return;
+            }
+            const message = outcome.reason.startsWith(STOPPED_PREFIX)
+                ? `the run was aborted because calls are ${outcome.reason}`
+                : "the 3minapi CLI on the user's PC stopped during this run";
+            logger.info('repeat run aborted; the repeating task resumes on the next start', attrs);
+            return this.recordRun(task, runNo, tally, {
+                ok: false,
+                reason: 'host_stopped',
+                message,
+                payload: JSON.stringify(failedBody('host_stopped', message).error),
+                report: false,
+                close: false
+            });
+        }
+        if (outcome.status === 'completed') {
+            const shape = this.opts.results ? computeShape(JSON.parse(outcome.resultJson)) : null;
+            logger.info('repeat run completed', attrs);
+            return this.recordRun(task, runNo, tally, {
+                ok: true,
+                payload: outcome.resultJson,
+                shape,
+                report: true,
+                close: true
+            });
+        }
+        logger.warn('repeat run failed', {
+            ...attrs,
+            reason: outcome.reason,
+            message: outcome.message
+        });
+        return this.recordRun(task, runNo, tally, {
+            ok: false,
+            reason: outcome.reason,
+            message: outcome.message,
+            payload: JSON.stringify(failedBody(outcome.reason, outcome.detail ?? outcome.message).error),
+            report: true,
+            close: true
+        });
+    }
+    async recordRun(task, runNo, tally, run) {
+        const { db, logger, now } = this.opts;
+        this.store(task, run.ok ? 'completed' : 'failed', run.payload, run.ok ? run.shape : null, runNo);
+        const next = {
+            runs: tally.runs + 1,
+            failed: tally.failed + (run.ok ? 0 : 1),
+            streak: run.ok ? 0 : tally.streak + 1
+        };
+        const lastRunAt = now();
+        db.recordRun(task.task_id, {
+            runs: next.runs,
+            failed: next.failed,
+            failStreak: next.streak,
+            lastRunAt
+        });
+        const settings = task.repeat;
+        const of = settings.max_runs === undefined ? '' : ` of ${settings.max_runs}`;
+        this.event(run.ok
+            ? `${task.worker}  ${shortId(task.task_id)}  run ${runNo}${of} completed`
+            : `${task.worker}  ${shortId(task.task_id)}  run ${runNo}${of} failed (${run.reason}): ${clip(run.message, EVENT_MESSAGE_MAX_CHARS)}`);
+        if (!run.close)
+            return;
+        if (await this.closeIfDone(task, next, lastRunAt, run.ok ? null : run.message))
+            return;
+        if (!run.report)
+            return;
+        const progress = {
+            runs: next.runs,
+            failed: next.failed,
+            last_run_at: new Date(lastRunAt).toISOString()
+        };
+        const r = await this.send(task, this.opts.repeatReporter ?? null, {
+            status: 'working',
+            status_message: clip(run.ok ? `Run ${runNo}${of} completed.` : `Run ${runNo}${of} failed: ${run.message}`, STATUS_MESSAGE_MAX_CHARS),
+            repeat: progress
+        }, true, runNo);
+        if (r === 'ok') {
+            if (this.opts.db.getTask(task.task_id)?.acked_at == null) {
+                this.opts.db.markAcked(task.task_id, this.opts.now());
+            }
+        }
+        else if (r === 'drop') {
+            this.scheduler.remove(task.task_id);
+            this.event(`${task.worker}  ${shortId(task.task_id)}  repeat ended on the server`);
+            this.finish(task, 'cancelled', 'the server no longer accepts this task');
+        }
+        else if (r === 'retry') {
+            logger.warn('repeat report not delivered; the next run reports the totals', {
+                task_id: task.task_id,
+                run_no: runNo
+            });
+        }
+    }
+    async closeIfDone(task, tally, lastRunAt, lastMessage) {
+        const { db, logger } = this.opts;
+        const settings = task.repeat;
+        const progress = {
+            runs: tally.runs,
+            failed: tally.failed,
+            last_run_at: new Date(lastRunAt).toISOString()
+        };
+        const reporter = this.opts.repeatReporter ?? null;
+        if (tally.streak >= REPEAT_FAIL_STREAK_MAX) {
+            this.scheduler.remove(task.task_id);
+            logger.warn('repeating task failed; too many runs failed in a row', {
+                task_id: task.task_id,
+                runs: tally.runs,
+                failed: tally.failed
+            });
+            this.event(`${task.worker}  ${shortId(task.task_id)}  repeat failed: ${REPEAT_FAIL_STREAK_MAX} runs in a row failed`);
+            const message = `${REPEAT_FAIL_STREAK_MAX} runs in a row failed` +
+                (lastMessage ? `; the last one: ${lastMessage}` : '');
+            const body = failedBody('repeat_failed', message);
+            body.error.data = { reason: 'repeat_failed', runs: tally.runs, failed: tally.failed };
+            await this.terminal(task, reporter, [
+                { body: { ...body, repeat: progress }, state: 'failed', reply: 'failed', error: message }
+            ]);
+            return true;
+        }
+        if (settings.max_runs !== undefined && tally.runs >= settings.max_runs) {
+            this.scheduler.remove(task.task_id);
+            logger.info('repeating task completed', {
+                task_id: task.task_id,
+                runs: tally.runs,
+                failed: tally.failed
+            });
+            this.event(`${task.worker}  ${shortId(task.task_id)}  repeat completed (${tally.runs} runs, ${tally.failed} failed)`);
+            const firstRunAt = db.getTask(task.task_id)?.started_at ?? lastRunAt;
+            await this.terminal(task, reporter, [
+                {
+                    body: {
+                        status: 'completed',
+                        status_message: `All ${tally.runs} runs finished; ${tally.failed} failed.`,
+                        result: {
+                            runs: tally.runs,
+                            failed: tally.failed,
+                            first_run_at: new Date(firstRunAt).toISOString(),
+                            last_run_at: progress.last_run_at
+                        },
+                        repeat: progress
+                    },
+                    state: 'completed',
+                    reply: 'result'
+                }
+            ]);
+            return true;
+        }
+        return false;
+    }
+    repeatEnvelope(row) {
+        const repeat = parseSettings(row.repeat);
+        let input = null;
+        try {
+            input = JSON.parse(row.input_json ?? 'null');
+        }
+        catch {
+        }
+        if (!repeat || row.input_json == null || !row.lease_id) {
+            this.opts.logger.warn('closing a repeating task that cannot be resumed', {
+                task_id: row.task_id
+            });
+            this.scheduler.remove(row.task_id);
+            this.opts.db.markFinished(row.task_id, 'failed', this.opts.now(), 'its saved settings could not be read');
+            return null;
+        }
+        return {
+            task_id: row.task_id,
+            worker: row.worker,
+            input,
+            callback_url: row.callback_url ?? null,
+            lease_id: row.lease_id,
+            delivery_attempt: row.delivery_attempt,
+            source: 'pull',
+            repeat
+        };
+    }
+    withCurrentLease(task, row = this.opts.db.getTask(task.task_id)) {
+        if (!row?.lease_id || row.lease_id === task.lease_id)
+            return task;
+        return { ...task, lease_id: row.lease_id, callback_url: row.callback_url ?? task.callback_url };
+    }
+    workerOf(taskId) {
+        return this.opts.db.getTask(taskId)?.worker ?? '';
+    }
+    finish(task, state, error) {
+        this.opts.db.markFinished(task.task_id, state, this.opts.now(), error);
+    }
+    received(task) {
+        return {
+            task_id: task.task_id,
+            worker: task.worker,
+            lease_id: task.lease_id ?? '',
+            delivery_attempt: task.delivery_attempt,
+            callback_url: task.callback_url,
+            ...(task.repeat ? { repeat: task.repeat, input: task.input } : {})
+        };
+    }
+    event(line) {
+        this.opts.onEvent?.(stripControl(line));
+    }
+}
+export function failedBody(reason, message) {
+    return {
+        status: 'failed',
+        status_message: clip(message, STATUS_MESSAGE_MAX_CHARS),
+        error: { code: TASK_ERROR_CODE, message, data: { reason } }
+    };
+}
+function parseStoredError(payload) {
+    try {
+        const error = JSON.parse(payload);
+        const reason = error?.data?.reason;
+        if (typeof reason !== 'string' || typeof error.message !== 'string')
+            return null;
+        return {
+            reason: reason,
+            message: truncate(error.message, ERROR_MESSAGE_MAX_CHARS)
+        };
+    }
+    catch {
+        return null;
+    }
+}
+function isReserved(value) {
+    return (value !== null &&
+        typeof value === 'object' &&
+        !Array.isArray(value) &&
+        Object.hasOwn(value, RESULT_RESERVED_KEY));
+}
+function clip(message, max) {
+    return truncate(oneLine(message), max);
+}
+function shortId(id) {
+    return id.length > 13 ? id.slice(-12) : id;
+}
+function tallyOf(row) {
+    return { runs: row.runs ?? 0, failed: row.failed ?? 0, streak: row.fail_streak ?? 0 };
+}