melchizedek-agents 1.0.2 → 1.1.0

package/dist/lib/runtime/turnControl.d.ts

@@ -35,6 +35,12 @@ export interface TurnControl {
     thinkingTokens: number;
     /** Ceiling on model calls for the whole turn; undefined = no ceiling. */
     readonly maxLlmCalls?: number;
+    /**
+     * The turn opted out of tracing (`runSyndicateTurn({ trace: false })`):
+     * nothing it runs opens a span or starts the tracer, so it reaches no
+     * sink. The budget and the token charge still apply.
+     */
+    readonly untraced?: boolean;
     /** Set once, by whichever control stopped the turn first. */
     stopReason?: TurnStopReason;
     /** Stop the turn. The first reason wins. */
@@ -46,6 +52,8 @@ export interface TurnControlOptions {
     deadlineMs?: number;
     /** An outer signal (a cancel request); aborting it cancels the turn. */
     signal?: AbortSignal;
+    /** The turn records no spans (TurnControl.untraced). */
+    untraced?: boolean;
 }
 /**
  * A control for one turn. Call `dispose()` when the turn ends so the
@@ -60,6 +68,13 @@ export declare function stopMessage(reason: TurnStopReason, control?: Pick<TurnC
 export declare function runWithTurnControl<T>(control: TurnControl, fn: () => T): T;
 /** The current turn's control, if this code is running inside a turn. */
 export declare function currentTurnControl(): TurnControl | undefined;
+/**
+ * Whether the code running now belongs to a turn that opted out of tracing.
+ * Every span the engine opens checks this first (lib/observability/
+ * tracer.ts, lib/runtime/native/telemetry.ts): an untraced turn opens no
+ * span and does not start the tracer. Outside a turn: false.
+ */
+export declare function turnUntraced(): boolean;
 /** The current turn's abort signal, for adapters to pass to provider SDKs. */
 export declare function currentTurnSignal(): AbortSignal | undefined;
 /**

package/dist/lib/runtime/turnControl.js

@@ -38,6 +38,7 @@ export function createTurnControl(opts = {}) {
         outputTokens: 0,
         thinkingTokens: 0,
         maxLlmCalls: opts.maxLlmCalls && opts.maxLlmCalls > 0 ? opts.maxLlmCalls : undefined,
+        ...(opts.untraced ? { untraced: true } : {}),
         stopReason: undefined,
         stop(reason) {
             if (control.stopReason)
@@ -82,6 +83,15 @@ export function runWithTurnControl(control, fn) {
 export function currentTurnControl() {
     return storage.getStore();
 }
+/**
+ * Whether the code running now belongs to a turn that opted out of tracing.
+ * Every span the engine opens checks this first (lib/observability/
+ * tracer.ts, lib/runtime/native/telemetry.ts): an untraced turn opens no
+ * span and does not start the tracer. Outside a turn: false.
+ */
+export function turnUntraced() {
+    return storage.getStore()?.untraced === true;
+}
 /** The current turn's abort signal, for adapters to pass to provider SDKs. */
 export function currentTurnSignal() {
     return storage.getStore()?.signal;

package/dist/lib/storage/postgres/index.d.ts

@@ -99,12 +99,17 @@ export interface PostgresStorage {
     turnLock: TurnLock;
     /** The task tools' lists and job queue (lib/tools/taskTools.ts setTaskBackend). */
     taskQueue: TaskBackend;
-    /** Task leases: renew this instance's, fail other instances' expired ones. */
+    /**
+     * Task leases: renew this instance's, fail other instances' expired ones,
+     * and list this instance's running tasks a caller asked another instance
+     * to cancel (migration 0014, ADR 0113).
+     */
     leases: {
         instanceId: string;
         ttlMs: number;
         renew: () => Promise<number>;
         reap: () => Promise<number>;
+        cancelRequested?: () => Promise<string[]>;
     };
     /** Closes the pool when this module created it, and the lock pool. */
     close: () => Promise<void>;

package/dist/lib/storage/postgres/index.js

@@ -16,7 +16,7 @@ import { SupabaseVectorMemoryService } from '../../memory/supabaseMemoryService.
 import { ERASE_STORES } from '../../memory/erase.js';
 import { postgresMemoryStore } from './memoryStore.js';
 import { PostgresSessionService } from './sessionService.js';
-import { PostgresTaskStore, reapExpiredTasks, renewTaskLeases } from './taskStore.js';
+import { PostgresTaskStore, cancelRequestedTasks, reapExpiredTasks, renewTaskLeases } from './taskStore.js';
 import { postgresTaskBackend } from './taskQueue.js';
 import { postgresCredentialStore } from './credentialStore.js';
 import { searchPathOption } from '../schema.js';
@@ -114,7 +114,12 @@ export function postgresStorage(options) {
         ...(credentials ? { credentials } : {}),
         ...(memoryService ? { memoryService } : {}),
         taskStore: (agentId) => new PostgresTaskStore(pool, agentId, { ttlDays: options.ttlDays, ownerResolver: options.taskOwner, lease }),
-        leases: { ...lease, renew: () => renewTaskLeases(pool, lease), reap: () => reapExpiredTasks(pool) },
+        leases: {
+            ...lease,
+            renew: () => renewTaskLeases(pool, lease),
+            reap: () => reapExpiredTasks(pool),
+            cancelRequested: () => cancelRequestedTasks(pool, lease),
+        },
         taskQueue: postgresTaskBackend(pool),
         async schemaVersion() {
             try {

package/dist/lib/storage/postgres/taskQueue.d.ts

@@ -1,6 +1,6 @@
 /**
  * lib/storage/postgres/taskQueue.ts — the task tools' TaskBackend on
- * Postgres (migration 0009, ADR 0021).
+ * Postgres (migrations 0009 and 0014, ADR 0021, ADR 0113).
  *
  * Every rule stays in lib/tools/taskTools.ts and runs on a TaskStore
  * snapshot; this file only loads one owner's snapshot and writes the change
@@ -14,8 +14,20 @@
  *   claim   takes the oldest queued background job of ANY owner with
  *           FOR UPDATE SKIP LOCKED, so two workers never take one job, and
  *           leases it to the worker until the lease runs out.
+ *   renew   extends the lease while the job is still this worker's and
+ *           running; resolves false otherwise, so the worker stops the run
+ *           (a task_update cancel of a running job lands here).
  *   recover finds running jobs whose lease ran out (the worker died) and
- *           applies the same interrupted-job rule the file store uses.
+ *           applies the same interrupted-job rule the file store uses. It
+ *           never touches a queued job or a running one whose lease is live.
+ *
+ * Checkpoints (ADR 0113): a running job's latest step checkpoint is the
+ * `checkpoint` column beside `record`, never inside it, so the tools never
+ * read it. Only the worker holding the lease writes it (saveCheckpoint). It
+ * survives while the job is running or queued: recover's requeue keeps it,
+ * so the next claim resumes from it, and claimNext leaves it; a mutate that
+ * moves the job to any other status, or recover failing it, clears it; a
+ * deleted row takes it along.
  */
 import type { Pool } from 'pg';
 import type { TaskBackend } from '../../tools/taskTools.ts';

package/dist/lib/storage/postgres/taskQueue.js

@@ -39,12 +39,15 @@ export function postgresTaskBackend(pool) {
                     if (before.get(t.id) === JSON.stringify(t))
                         continue;
                     // Leaving `running` ends the job's lease; staying in it keeps it.
+                    // A checkpoint survives only in running or queued.
                     await client.query(`INSERT INTO melchizedek_tasks (owner, id, seq, kind, status, record, updated_at)
              VALUES ($1, $2, $3, $4, $5, $6::jsonb, NOW())
              ON CONFLICT (owner, id) DO UPDATE
                SET kind = EXCLUDED.kind, status = EXCLUDED.status, record = EXCLUDED.record, updated_at = NOW(),
                    lease_owner = CASE WHEN EXCLUDED.status = 'running' THEN melchizedek_tasks.lease_owner END,
-                   lease_until = CASE WHEN EXCLUDED.status = 'running' THEN melchizedek_tasks.lease_until END`, [owner, t.id, seqOf(t.id), t.kind, t.status, JSON.stringify(t)]);
+                   lease_until = CASE WHEN EXCLUDED.status = 'running' THEN melchizedek_tasks.lease_until END,
+                   checkpoint = CASE WHEN EXCLUDED.status IN ('running', 'queued') THEN melchizedek_tasks.checkpoint END,
+                   checkpoint_at = CASE WHEN EXCLUDED.status IN ('running', 'queued') THEN melchizedek_tasks.checkpoint_at END`, [owner, t.id, seqOf(t.id), t.kind, t.status, JSON.stringify(t)]);
                 }
                 await client.query('UPDATE melchizedek_task_owners SET next_id = $2 WHERE owner = $1', [owner, store.next_id]);
                 return out;
@@ -74,8 +77,19 @@ export function postgresTaskBackend(pool) {
             return row ? { ...row.record, owner: row.owner } : null;
         },
         async renew(worker, job) {
-            await pool.query(`UPDATE melchizedek_tasks SET lease_until = NOW() + ($4::int * interval '1 millisecond')
+            const r = await pool.query(`UPDATE melchizedek_tasks SET lease_until = NOW() + ($4::int * interval '1 millisecond')
           WHERE owner = $1 AND id = $2 AND lease_owner = $3 AND status = 'running'`, [job.owner, job.id, worker.workerId, worker.leaseMs]);
+            return (r.rowCount ?? 0) > 0;
+        },
+        async saveCheckpoint(worker, job, checkpoint) {
+            const r = await pool.query(`UPDATE melchizedek_tasks SET checkpoint = $4::jsonb, checkpoint_at = NOW()
+          WHERE owner = $1 AND id = $2 AND lease_owner = $3 AND status = 'running'`, [job.owner, job.id, worker.workerId, JSON.stringify(checkpoint)]);
+            return (r.rowCount ?? 0) > 0;
+        },
+        async loadCheckpoint(job) {
+            const r = await pool.query(`SELECT checkpoint FROM melchizedek_tasks
+          WHERE owner = $1 AND id = $2 AND status IN ('running', 'queued')`, [job.owner, job.id]);
+            return r.rows[0]?.checkpoint ?? null;
         },
         async finish(job, outcome) {
             await this.mutate(job.owner, (store) => applyFinish(store, job.id, outcome));
@@ -91,7 +105,10 @@ export function postgresTaskBackend(pool) {
                     const t = row.record;
                     const outcome = applyInterrupted(t);
                     (outcome === 'failed' ? failed : requeued).push(`${row.owner || '(shared)'}:${t.id}`);
-                    await client.query(`UPDATE melchizedek_tasks SET status = $3, record = $4::jsonb, updated_at = NOW(), lease_owner = NULL, lease_until = NULL
+                    // Requeued keeps the checkpoint (the next claim resumes); failed clears it.
+                    await client.query(`UPDATE melchizedek_tasks SET status = $3, record = $4::jsonb, updated_at = NOW(), lease_owner = NULL, lease_until = NULL,
+                    checkpoint = CASE WHEN $3::text = 'queued' THEN checkpoint END,
+                    checkpoint_at = CASE WHEN $3::text = 'queued' THEN checkpoint_at END
               WHERE owner = $1 AND id = $2`, [row.owner, row.id, t.status, JSON.stringify(t)]);
                 }
                 return { requeued, failed };

package/dist/lib/storage/postgres/taskStore.d.ts

@@ -8,6 +8,16 @@
  * default resolver reads the authenticated user name), so one caller cannot
  * load or list another's task. `list` mirrors the SDK's InMemoryTaskStore:
  * same filters, same newest-first order, same page-token format.
+ *
+ * Cancel across replicas (migration 0014, ADR 0113). The SDK cancels a task
+ * through its own event bus when the task runs on this instance; otherwise
+ * it writes `canceled` into this store itself. When the row is still leased
+ * to another live instance, that write also stamps `cancel_requested_at`
+ * and leaves the lease with the instance running the task: its heartbeat
+ * (`cancelRequestedTasks`) finds the request and aborts the run. From then
+ * on the row stays `canceled` — the running instance's later saves renew or
+ * clear its lease but never overwrite the state it reported — so a client
+ * told `canceled` is never later shown `completed`.
  */
 import type { Pool } from 'pg';
 import type { TaskStore } from '@a2a-js/sdk/server';
@@ -31,13 +41,19 @@ export interface TaskLease {
     instanceId: string;
     ttlMs: number;
 }
+/**
+ * Ids of the running tasks leased to this instance that a caller asked,
+ * through another instance, to cancel. The lease heartbeat aborts each.
+ */
+export declare function cancelRequestedTasks(pool: Pool, lease: TaskLease): Promise<string[]>;
 /** Extends every lease this instance holds. Returns how many it renewed. */
 export declare function renewTaskLeases(pool: Pool, lease: TaskLease): Promise<number>;
 /**
  * Marks running tasks whose lease expired (their instance died) as failed,
  * with a message saying so. Any instance may run it: rows are claimed with
- * SKIP LOCKED, so two reapers never fail the same task twice. Returns how
- * many it failed.
+ * SKIP LOCKED, so two reapers never fail the same task twice. A task whose
+ * cancel was already reported keeps `canceled`: only its lease is cleared.
+ * Returns how many it failed.
  */
 export declare function reapExpiredTasks(pool: Pool, limit?: number): Promise<number>;
 export declare class PostgresTaskStore implements TaskStore {
@@ -49,6 +65,14 @@ export declare class PostgresTaskStore implements TaskStore {
     constructor(pool: Pool, agentId: string, options?: PostgresTaskStoreOptions);
     private scope;
     save(task: Task, context: Context): Promise<void>;
+    /**
+     * Asks the instance running a task to cancel it, without changing the
+     * task: stamps `cancel_requested_at` on the caller's own row (scoped as
+     * `load` is) while it is running and leased. The running instance's lease
+     * heartbeat aborts the run, which then ends `canceled`. Returns whether a
+     * running task of this caller's was found.
+     */
+    requestCancel(taskId: string, context: Context): Promise<boolean>;
     load(taskId: string, context: Context): Promise<Task | undefined>;
     list(params: ListParams, context: Context): Promise<ListResult>;
 }

package/dist/lib/storage/postgres/taskStore.js

@@ -8,6 +8,16 @@
  * default resolver reads the authenticated user name), so one caller cannot
  * load or list another's task. `list` mirrors the SDK's InMemoryTaskStore:
  * same filters, same newest-first order, same page-token format.
+ *
+ * Cancel across replicas (migration 0014, ADR 0113). The SDK cancels a task
+ * through its own event bus when the task runs on this instance; otherwise
+ * it writes `canceled` into this store itself. When the row is still leased
+ * to another live instance, that write also stamps `cancel_requested_at`
+ * and leaves the lease with the instance running the task: its heartbeat
+ * (`cancelRequestedTasks`) finds the request and aborts the run. From then
+ * on the row stays `canceled` — the running instance's later saves renew or
+ * clear its lease but never overwrite the state it reported — so a client
+ * told `canceled` is never later shown `completed`.
  */
 import { randomUUID } from 'node:crypto';
 import { Role, TaskState } from '@a2a-js/sdk';
@@ -15,6 +25,24 @@ import { resolveUserScope } from '@a2a-js/sdk/server';
 const DEFAULT_PAGE_SIZE = 50;
 /** States in which a task is being worked on by some instance. */
 const RUNNING = new Set([TaskState.TASK_STATE_SUBMITTED, TaskState.TASK_STATE_WORKING]);
+/** SQL: the stored row is a running task leased to another live instance. */
+const HELD_ELSEWHERE = `(adk_a2a_tasks.lease_owner IS NOT NULL
+   AND adk_a2a_tasks.lease_owner IS DISTINCT FROM $12::text
+   AND adk_a2a_tasks.lease_until > now()
+   AND adk_a2a_tasks.state = ANY($14::int[]))`;
+/** SQL: this save is a cancel, written by an instance not running the task. */
+const CANCEL_FROM_ELSEWHERE = `($6::int = $13::int AND ${HELD_ELSEWHERE})`;
+/** SQL: a cancel was reported for the row; its state stays `canceled`. */
+const CANCEL_REPORTED = `(adk_a2a_tasks.cancel_requested_at IS NOT NULL AND adk_a2a_tasks.state = $13::int)`;
+/**
+ * Ids of the running tasks leased to this instance that a caller asked,
+ * through another instance, to cancel. The lease heartbeat aborts each.
+ */
+export async function cancelRequestedTasks(pool, lease) {
+    const r = await pool.query(`SELECT id FROM adk_a2a_tasks
+     WHERE lease_owner = $1 AND lease_until IS NOT NULL AND cancel_requested_at IS NOT NULL`, [lease.instanceId]);
+    return r.rows.map((row) => String(row.id));
+}
 /** Extends every lease this instance holds. Returns how many it renewed. */
 export async function renewTaskLeases(pool, lease) {
     const r = await pool.query(`UPDATE adk_a2a_tasks SET lease_until = now() + ($2::int * interval '1 millisecond')
@@ -26,17 +54,25 @@ const ORPHANED = 'The server running this task stopped before it finished; send
 /**
  * Marks running tasks whose lease expired (their instance died) as failed,
  * with a message saying so. Any instance may run it: rows are claimed with
- * SKIP LOCKED, so two reapers never fail the same task twice. Returns how
- * many it failed.
+ * SKIP LOCKED, so two reapers never fail the same task twice. A task whose
+ * cancel was already reported keeps `canceled`: only its lease is cleared.
+ * Returns how many it failed.
  */
 export async function reapExpiredTasks(pool, limit = 100) {
     const client = await pool.connect();
     try {
         await client.query('BEGIN');
-        const r = await client.query(`SELECT tenant, owner, agent_id, id, task FROM adk_a2a_tasks
+        const r = await client.query(`SELECT tenant, owner, agent_id, id, task, state FROM adk_a2a_tasks
        WHERE lease_until IS NOT NULL AND lease_until < now()
        ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED`, [limit]);
+        let failed = 0;
         for (const row of r.rows) {
+            if (!RUNNING.has(Number(row.state))) {
+                await client.query(`UPDATE adk_a2a_tasks SET lease_owner = NULL, lease_until = NULL, cancel_requested_at = NULL
+           WHERE tenant = $1 AND owner = $2 AND agent_id = $3 AND id = $4`, [row.tenant, row.owner, row.agent_id, row.id]);
+                continue;
+            }
+            failed += 1;
             const task = row.task;
             const timestamp = new Date().toISOString();
             task.status = {
@@ -53,11 +89,11 @@ export async function reapExpiredTasks(pool, limit = 100) {
                 },
             };
             await client.query(`UPDATE adk_a2a_tasks SET task = $5::jsonb, state = $6, status_ts = $7, updated_at = now(),
-                lease_owner = NULL, lease_until = NULL
+                lease_owner = NULL, lease_until = NULL, cancel_requested_at = NULL
          WHERE tenant = $1 AND owner = $2 AND agent_id = $3 AND id = $4`, [row.tenant, row.owner, row.agent_id, row.id, JSON.stringify(task), TaskState.TASK_STATE_FAILED, timestamp]);
         }
         await client.query('COMMIT');
-        return r.rows.length;
+        return failed;
     }
     catch (err) {
         await client.query('ROLLBACK').catch(() => { });
@@ -97,13 +133,26 @@ export class PostgresTaskStore {
         const t = task;
         // A running task is leased to this instance; any other state holds none.
         const leased = !!this.lease && RUNNING.has(t.status?.state ?? -1);
+        // On conflict, three cases (see the header): a cancel written by an
+        // instance not running the task records the request and leaves the lease
+        // where it is; a row whose cancel was reported keeps its content while
+        // the running instance renews or clears the lease; anything else is the
+        // plain upsert. A pending request survives while the task runs and is
+        // cleared when it stops.
         await this.pool.query(`INSERT INTO adk_a2a_tasks (tenant, owner, agent_id, id, context_id, state, status_ts, task, updated_at, expire_at, lease_owner, lease_until)
        VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, now(), $9, $10,
                CASE WHEN $10::text IS NULL THEN NULL ELSE now() + ($11::int * interval '1 millisecond') END)
        ON CONFLICT (tenant, owner, agent_id, id) DO UPDATE
-         SET context_id = EXCLUDED.context_id, state = EXCLUDED.state, status_ts = EXCLUDED.status_ts,
-             task = EXCLUDED.task, updated_at = now(), expire_at = EXCLUDED.expire_at,
-             lease_owner = EXCLUDED.lease_owner, lease_until = EXCLUDED.lease_until`, [
+         SET context_id = CASE WHEN ${CANCEL_REPORTED} THEN adk_a2a_tasks.context_id ELSE EXCLUDED.context_id END,
+             state = CASE WHEN ${CANCEL_REPORTED} THEN adk_a2a_tasks.state ELSE EXCLUDED.state END,
+             status_ts = CASE WHEN ${CANCEL_REPORTED} THEN adk_a2a_tasks.status_ts ELSE EXCLUDED.status_ts END,
+             task = CASE WHEN ${CANCEL_REPORTED} THEN adk_a2a_tasks.task ELSE EXCLUDED.task END,
+             updated_at = now(), expire_at = EXCLUDED.expire_at,
+             lease_owner = CASE WHEN ${CANCEL_FROM_ELSEWHERE} THEN adk_a2a_tasks.lease_owner ELSE EXCLUDED.lease_owner END,
+             lease_until = CASE WHEN ${CANCEL_FROM_ELSEWHERE} THEN adk_a2a_tasks.lease_until ELSE EXCLUDED.lease_until END,
+             cancel_requested_at = CASE WHEN ${CANCEL_FROM_ELSEWHERE} THEN now()
+                                        WHEN EXCLUDED.lease_owner IS NOT NULL THEN adk_a2a_tasks.cancel_requested_at
+                                        ELSE NULL END`, [
             tenant,
             owner,
             this.agentId,
@@ -115,8 +164,25 @@ export class PostgresTaskStore {
             new Date(Date.now() + this.ttlMs).toISOString(),
             leased ? this.lease.instanceId : null,
             this.lease?.ttlMs ?? 0,
+            this.lease?.instanceId ?? null,
+            TaskState.TASK_STATE_CANCELED,
+            [...RUNNING],
         ]);
     }
+    /**
+     * Asks the instance running a task to cancel it, without changing the
+     * task: stamps `cancel_requested_at` on the caller's own row (scoped as
+     * `load` is) while it is running and leased. The running instance's lease
+     * heartbeat aborts the run, which then ends `canceled`. Returns whether a
+     * running task of this caller's was found.
+     */
+    async requestCancel(taskId, context) {
+        const [tenant, owner] = this.scope(context);
+        const r = await this.pool.query(`UPDATE adk_a2a_tasks SET cancel_requested_at = now()
+       WHERE tenant = $1 AND owner = $2 AND agent_id = $3 AND id = $4
+         AND lease_until IS NOT NULL AND state = ANY($5::int[])`, [tenant, owner, this.agentId, taskId, [...RUNNING]]);
+        return (r.rowCount ?? 0) > 0;
+    }
     async load(taskId, context) {
         const [tenant, owner] = this.scope(context);
         const r = await this.pool.query('SELECT task FROM adk_a2a_tasks WHERE tenant = $1 AND owner = $2 AND agent_id = $3 AND id = $4', [tenant, owner, this.agentId, taskId]);