@kubun/plugin-workflow 0.15.0 → 0.16.0

package/lib/control-plane.d.ts

@@ -0,0 +1,16 @@
+import type { WorkflowCommandRefusalReason, WorkflowControlPlaneAPI, WorkflowDiscoverOptions, WorkflowDiscoverReply } from '@kubun/plugin-p2p-api';
+import type { EnqueueOptions, InstanceStatus } from '@kubun/plugin-workflow-api';
+export declare class WorkflowCommandRefusedError extends Error {
+    #private;
+    constructor(reason: WorkflowCommandRefusalReason);
+    get reason(): WorkflowCommandRefusalReason;
+}
+export type WorkflowControlPlaneHelpers = {
+    discoverWorkflows(groupID: string, options?: WorkflowDiscoverOptions): Promise<Array<WorkflowDiscoverReply>>;
+    listWorkflows(groupID: string, targetDID: string): Promise<Array<InstanceStatus>>;
+    workflowStatus(groupID: string, targetDID: string, instanceID: string): Promise<InstanceStatus>;
+    enqueueWorkflow(groupID: string, targetDID: string, name: string, params?: unknown, opts?: EnqueueOptions): Promise<InstanceStatus>;
+    cancelWorkflow(groupID: string, targetDID: string, instanceID: string): Promise<InstanceStatus>;
+    retryWorkflow(groupID: string, targetDID: string, instanceID: string): Promise<InstanceStatus>;
+};
+export declare function createWorkflowControlPlaneHelpers(getP2P: () => Promise<WorkflowControlPlaneAPI>): WorkflowControlPlaneHelpers;

package/lib/control-plane.js

@@ -0,0 +1,51 @@
+// A directed command refused by the remote peer. The p2p API returns refusals as
+// DATA (enkaku collapses a handler throw to one generic error); this helper turns
+// them back into a typed local throw, so app code sees a thrown `reason`.
+export class WorkflowCommandRefusedError extends Error {
+    #reason;
+    constructor(reason){
+        super(refusalMessage(reason));
+        this.name = 'WorkflowCommandRefusedError';
+        this.#reason = reason;
+    }
+    get reason() {
+        return this.#reason;
+    }
+}
+// `not_found` avoids "no such instance": the remote refuses a missing instance and
+// a non-commandable one identically, so the message must not claim it doesn't
+// exist — only that it could not be commanded.
+function refusalMessage(reason) {
+    switch(reason){
+        case 'not_commandable':
+            return 'workflow did not opt into remote command';
+        case 'not_found':
+            return 'workflow instance is not commandable (missing or not opted in)';
+        case 'not_failed':
+            return 'workflow instance is not in a failed state';
+        case 'no_next_action':
+            return 'workflow instance has no resumable action to retry';
+        case 'singleton':
+            return 'a live instance already holds the same singleton key';
+    }
+}
+function unwrap(result) {
+    if (!result.ok) {
+        throw new WorkflowCommandRefusedError(result.reason);
+    }
+    return result.status;
+}
+// `getP2P` resolves the p2p control plane lazily: the workflow API is built before
+// every plugin factory returns, so `engine.getAPI('p2p')` cannot be awaited at
+// build time. A device with no p2p plugin rejects here — the correct failure for a
+// remote op with no transport.
+export function createWorkflowControlPlaneHelpers(getP2P) {
+    return {
+        discoverWorkflows: async (groupID, options)=>(await getP2P()).discoverWorkflows(groupID, options),
+        listWorkflows: async (groupID, targetDID)=>(await getP2P()).listWorkflows(groupID, targetDID),
+        workflowStatus: async (groupID, targetDID, instanceID)=>(await getP2P()).workflowStatus(groupID, targetDID, instanceID),
+        enqueueWorkflow: async (groupID, targetDID, name, params, opts)=>unwrap(await (await getP2P()).enqueueWorkflow(groupID, targetDID, name, params, opts)),
+        cancelWorkflow: async (groupID, targetDID, instanceID)=>unwrap(await (await getP2P()).cancelWorkflow(groupID, targetDID, instanceID)),
+        retryWorkflow: async (groupID, targetDID, instanceID)=>unwrap(await (await getP2P()).retryWorkflow(groupID, targetDID, instanceID))
+    };
+}

package/lib/definition.d.ts

@@ -1,3 +1,5 @@
 import type { WorkflowDefinition } from '@kubun/plugin-workflow-api';
 import type { HandlersRecord } from '@sozai/flow';
 export declare function defineWorkflow<State extends Record<string, unknown>, Handlers extends HandlersRecord<State> = HandlersRecord<State>>(def: WorkflowDefinition<State, Handlers>): WorkflowDefinition<State, Handlers>;
+export declare function isRemotelyObservable(def: Pick<WorkflowDefinition, 'remote'>): boolean;
+export declare function isRemotelyCommandable(def: Pick<WorkflowDefinition, 'remote'>): boolean;

package/lib/definition.js

@@ -6,3 +6,13 @@ export function defineWorkflow(def) {
     }
     return def;
 }
+// Single normalize point for def.remote: absent field/sub-field, or a
+// non-true value (including null, which optional booleans can read as),
+// default closed. Typed on just the `remote` field so any WorkflowDefinition
+// (whatever its State/Handlers) is accepted.
+export function isRemotelyObservable(def) {
+    return def.remote?.observe === true;
+}
+export function isRemotelyCommandable(def) {
+    return def.remote?.command === true;
+}

package/lib/engine.d.ts

@@ -1,20 +1,29 @@
 import type { Logger } from '@kubun/logger';
-import type { RetryPolicy, WorkflowAPI } from '@kubun/plugin-workflow-api';
+import type { RetryPolicy, RetryRefusalReason, WorkflowAPI } from '@kubun/plugin-workflow-api';
 import type { WorkflowStoreAPI } from '@kubun/store-workflow';
 import type { GetRandomID } from '@sozai/runtime';
 export declare function backoff(attempts: number, policy?: RetryPolicy): number;
 export declare function parseEvery(spec: string): number;
+export declare class WorkflowRetryError extends Error {
+    #private;
+    constructor(reason: RetryRefusalReason, message: string, options?: {
+        cause?: unknown;
+    });
+    get reason(): RetryRefusalReason;
+}
 export type WorkflowEngineDeps = {
     getStore: () => Promise<WorkflowStoreAPI>;
     logger: Logger;
     now?: () => number;
     getRandomID?: GetRandomID;
     tickMs?: number;
+    disposeTimeoutMs?: number;
+    ready?: () => Promise<void>;
 };
 export type WorkflowEngine = {
     api: WorkflowAPI;
     recover(): Promise<Array<string>>;
     tick(): Promise<void>;
-    dispose(): void;
+    dispose(): Promise<void>;
 };
 export declare function createWorkflowEngine(deps: WorkflowEngineDeps): WorkflowEngine;

package/lib/engine.js

@@ -6,6 +6,8 @@ import { computeNextDelay, validateAdaptivePolicy } from './adaptive.js';
 const LEASE_MS = 30_000;
 // Default scheduler cadence; injectable via deps so tests drive `tick()` directly.
 const DEFAULT_TICK_MS = 1_000;
+// Default upper bound for dispose()'s best-effort drain of in-flight drives.
+const DEFAULT_DISPOSE_TIMEOUT_MS = 5_000;
 // Backoff for a failed attempt (1-based). `fixed` returns `baseMs`; `exponential`
 // doubles per attempt, capped at `maxMs`. Pure so it is unit-testable.
 export function backoff(attempts, policy) {
@@ -32,6 +34,19 @@ const TERMINAL = new Set([
     'failed',
     'cancelled'
 ]);
+// Typed refusal thrown by `retry` when it does not re-arm. `reason` lets a caller
+// (or a GraphQL error consumer) branch on why, rather than parsing a message.
+export class WorkflowRetryError extends Error {
+    #reason;
+    constructor(reason, message, options){
+        super(message, options);
+        this.name = 'WorkflowRetryError';
+        this.#reason = reason;
+    }
+    get reason() {
+        return this.#reason;
+    }
+}
 function toStatus(row) {
     return {
         instanceID: row.instance_id,
@@ -86,6 +101,13 @@ export function createWorkflowEngine(deps) {
     const running = new Map();
     // In-flight drives, tracked so nudges are not lost and errors are logged.
     const inflight = new Set();
+    // Set by dispose(): stops the scheduler from claiming/driving new work while
+    // the existing in-flight drives are drained.
+    let disposed = false;
+    // Cleared once the host engine's identity readiness gate resolves. While
+    // false the scheduler claims nothing; a mismatched identity leaves it false
+    // forever (see the ready() wiring below).
+    let identityReady = deps.ready == null;
     let storePromise;
     function store() {
         storePromise ??= deps.getStore();
@@ -342,13 +364,25 @@ export function createWorkflowEngine(deps) {
         }
     }
     async function tick() {
+        // After dispose() no new work is claimed: a straggler tick (a queued nudge or
+        // the interval firing before it was cleared) must not start fresh drives while
+        // shutdown drains the in-flight ones. `identityReady` gates the same way — the
+        // scheduler claims nothing until the host engine verifies its database identity,
+        // and a mismatch bars it permanently.
+        if (disposed || !identityReady) return;
         try {
             const s = await store();
+            // Re-check after every await that precedes a claim: dispose() may have run
+            // during the gap. Bailing here, not only at entry, keeps a tick already
+            // in-flight when dispose() fired from claiming fresh work after shutdown began.
+            if (disposed) return;
             const at = now();
             // Promote due schedules first so their instances are claimable this pass.
             await promoteSchedules(s, at);
+            if (disposed) return;
             const lanes = await s.listRunnableQueues(at);
             for (const lane of lanes){
+                if (disposed) return;
                 const cap = laneConcurrency(lane) - await s.countRunning(lane, at);
                 if (cap <= 0) continue;
                 // Clamp to available tokens; consume only for rows actually claimed
@@ -357,6 +391,9 @@ export function createWorkflowEngine(deps) {
                 if (grant <= 0) continue;
                 const rows = await s.claimDue(lane, grant, bootID, LEASE_MS, at);
                 consumeTokens(lane, rows.length);
+                // If dispose() fired during claimDue, leave the just-claimed rows for
+                // crash-reclaim rather than starting drives no abort sweep will cancel.
+                if (disposed) return;
                 for (const row of rows)track(run(row));
             }
         } catch (cause) {
@@ -378,12 +415,32 @@ export function createWorkflowEngine(deps) {
         });
     }
     const tickMs = deps.tickMs ?? DEFAULT_TICK_MS;
+    const disposeTimeoutMs = deps.disposeTimeoutMs ?? DEFAULT_DISPOSE_TIMEOUT_MS;
     let interval = setInterval(()=>{
-        void tick();
+        // Tracked, not fire-and-forget: dispose()'s drain must be able to await an
+        // interval tick that is mid-pass, or its claim/promote writes could land
+        // after dispose() resolves and the store closes.
+        track(tick());
     }, tickMs);
     // Allow the process to exit even while the scheduler interval is pending.
     const intervalHandle = interval;
     if (typeof intervalHandle?.unref === 'function') intervalHandle.unref();
+    // Wire the identity readiness gate. On resolve, open the gate and drive any work
+    // enqueued while it was closed. On rejection (an identity mismatch) keep the gate
+    // shut forever AND drop the interval, so the engine is quiescent rather than merely
+    // barred at the HLC mint — which remains the fail-closed backstop if a drive slips
+    // through.
+    if (deps.ready != null) {
+        void deps.ready().then(()=>{
+            identityReady = true;
+            scheduleTick();
+        }, ()=>{
+            if (interval != null) {
+                clearInterval(interval);
+                interval = null;
+            }
+        });
+    }
     const api = {
         register (def) {
             const erased = def;
@@ -401,6 +458,12 @@ export function createWorkflowEngine(deps) {
         isManagedWorkflow (name) {
             return registry.get(name)?.managed === true;
         },
+        isRemotelyObservable (name) {
+            return registry.get(name)?.remote?.observe === true;
+        },
+        isRemotelyCommandable (name) {
+            return registry.get(name)?.remote?.command === true;
+        },
         defineQueue,
         async schedule (name, params, when) {
             const s = await store();
@@ -504,23 +567,10 @@ export function createWorkflowEngine(deps) {
                 runAfter: createdAt,
                 createdAt
             };
-            let result;
-            try {
-                result = await s.insertInstance(input);
-            } catch (cause) {
-                // A concurrent insert may lose the singleton race and surface the
-                // partial-unique constraint as a thrown error. Treat it as "singleton
-                // already active": return the existing active instance rather than throw.
-                if (opts?.singletonKey != null) {
-                    const active = (await s.list({
-                        name
-                    })).find((r)=>r.singleton_key === opts.singletonKey && (r.status === 'pending' || r.status === 'running'));
-                    if (active != null) return {
-                        instanceID: active.instance_id
-                    };
-                }
-                throw cause;
-            }
+            // insertInstance is atomic against a concurrent racer: an idempotency_key
+            // or active-singleton collision returns the winner's row as created:false
+            // rather than throwing, so no constraint-race catch is needed here.
+            const result = await s.insertInstance(input);
             // Idempotent or singleton hit: return the existing instance, drive nothing.
             if (!result.created) {
                 return {
@@ -554,6 +604,54 @@ export function createWorkflowEngine(deps) {
                 name: row.name
             });
         },
+        async retry (instanceID) {
+            const s = await store();
+            // Read first to classify a would-be no-op into a precise refusal; the
+            // store's guarded UPDATE remains the source of truth (a status flipped
+            // between this read and the write surfaces as an unapplied re-arm below).
+            const row = await s.get(instanceID);
+            if (row == null) {
+                throw new WorkflowRetryError('not_found', `Workflow instance "${instanceID}" not found`);
+            }
+            if (row.status !== 'failed') {
+                throw new WorkflowRetryError('not_failed', `Workflow instance "${instanceID}" is ${row.status}, not failed`);
+            }
+            if (row.next_action == null) {
+                throw new WorkflowRetryError('no_next_action', `Workflow instance "${instanceID}" has no resumable action to retry`);
+            }
+            if (row.singleton_key != null) {
+                const live = (await s.list({
+                    name: row.name
+                })).find((r)=>r.singleton_key === row.singleton_key && r.instance_id !== instanceID && (r.status === 'pending' || r.status === 'running'));
+                if (live != null) {
+                    throw new WorkflowRetryError('singleton', `Workflow "${row.name}" already has a live instance for singleton "${row.singleton_key}"`);
+                }
+            }
+            let rearmed;
+            try {
+                rearmed = await s.retryFailed(instanceID, now());
+            } catch (cause) {
+                // The only constraint the guarded UPDATE can violate is the singleton
+                // partial-unique index: a concurrent enqueue won the race after our
+                // pre-check. Surface it as the singleton refusal, not a raw driver error.
+                if (row.singleton_key != null) {
+                    // biome-ignore lint/style/useErrorCause: cause is forwarded to super via the third options arg
+                    throw new WorkflowRetryError('singleton', `Workflow "${row.name}" already has a live instance for singleton "${row.singleton_key}"`, {
+                        cause
+                    });
+                }
+                throw cause;
+            }
+            if (rearmed == null) {
+                // The status guard did not match: a concurrent retry (or reclaim) flipped
+                // it out of `failed` first. Never a false success.
+                throw new WorkflowRetryError('not_failed', `Workflow instance "${instanceID}" was no longer failed when re-armed`);
+            }
+            // Wake the scheduler so the re-armed instance is claimed without waiting
+            // for the poll interval, exactly as enqueue does.
+            scheduleTick();
+            return toStatus(rearmed);
+        },
         async get (instanceID) {
             const s = await store();
             const row = await s.get(instanceID);
@@ -606,17 +704,67 @@ export function createWorkflowEngine(deps) {
             // resumes them. The returned ids are the pre-tick stranded set.
             const s = await store();
             const stranded = await s.listRecoverable(now());
-            await tick();
+            // Tracked like every other tick so a concurrent dispose() drains this
+            // pass's store writes rather than resolving out from under them.
+            const p = tick();
+            track(p);
+            await p;
             return stranded.map((r)=>r.instance_id);
         },
-        tick,
-        dispose () {
+        // Tracked so dispose()'s drain awaits a manual tick that is mid-pass; an
+        // untracked one could issue promote/claim writes after dispose() resolved.
+        tick: ()=>{
+            const p = tick();
+            track(p);
+            return p;
+        },
+        async dispose () {
+            disposed = true;
             if (interval != null) {
                 clearInterval(interval);
                 interval = null;
             }
+            // Abort every live drive, then drain to quiescence: an aborted run still
+            // issues its `cancelled` terminal write, and aborting frees a lane so a
+            // straggler drive may appear — so loop until nothing is in flight rather than
+            // awaiting a single snapshot. `disposed` keeps tick() from adding new work, so
+            // a cooperative drive set terminates and the caller can then safely close the
+            // store.
             for (const controller of running.values())controller.abort();
             running.clear();
+            let drainedDone = false;
+            const drained = (async ()=>{
+                while(inflight.size > 0)await Promise.allSettled([
+                    ...inflight
+                ]);
+                drainedDone = true;
+            })();
+            // Bound the wait: a handler that never observes its abort signal leaves an
+            // un-settling drive in `inflight`, and an unbounded drain would wedge shutdown
+            // forever. Resolve best-effort on the deadline instead; the timer is unref'd so
+            // it never holds the process open. On timeout genuine writes may still be in
+            // flight, so a caller that then closes the store can race them — the deadline
+            // trades that residual for a bounded shutdown, and only fires for a
+            // non-settling drive (logged).
+            let timer;
+            const deadline = new Promise((resolve)=>{
+                timer = setTimeout(resolve, disposeTimeoutMs);
+                timer.unref?.();
+            });
+            try {
+                await Promise.race([
+                    drained,
+                    deadline
+                ]);
+            } finally{
+                if (timer != null) clearTimeout(timer);
+            }
+            if (!drainedDone) {
+                logger.warn('workflow dispose timed out draining in-flight drives', {
+                    inflight: inflight.size,
+                    disposeTimeoutMs
+                });
+            }
         }
     };
 }

package/lib/index.d.ts

@@ -2,8 +2,9 @@ import type { KubunPlugin, PluginFactoryParams } from '@kubun/engine';
 export type WorkflowPluginOptions = Record<string, never>;
 export declare function createWorkflowPlugin(_options?: WorkflowPluginOptions): (params: PluginFactoryParams) => KubunPlugin;
 export { computeNextDelay, validateAdaptivePolicy } from './adaptive.js';
-export { defineWorkflow } from './definition.js';
+export { createWorkflowControlPlaneHelpers, WorkflowCommandRefusedError, type WorkflowControlPlaneHelpers, } from './control-plane.js';
+export { defineWorkflow, isRemotelyCommandable, isRemotelyObservable } from './definition.js';
 export type { WorkflowEngine, WorkflowEngineDeps } from './engine.js';
-export { createWorkflowEngine, parseEvery } from './engine.js';
+export { createWorkflowEngine, parseEvery, WorkflowRetryError } from './engine.js';
 export type { WorkflowContext, WorkflowEventResult } from './schema.js';
 export { createWorkflowContextFactory, createWorkflowSchemaExtension, subscribeToWorkflowEvents, } from './schema.js';

package/lib/index.js

@@ -1,4 +1,5 @@
 import { getWorkflowStore, workflowStoreDefinition } from '@kubun/store-workflow';
+import { createWorkflowControlPlaneHelpers } from './control-plane.js';
 import { createWorkflowEngine } from './engine.js';
 import { createWorkflowContextFactory, createWorkflowSchemaExtension } from './schema.js';
 export function createWorkflowPlugin(_options = {}) {
@@ -8,17 +9,31 @@ export function createWorkflowPlugin(_options = {}) {
         const engine = createWorkflowEngine({
             getStore: ()=>getWorkflowStore(params.db),
             logger,
-            getRandomID: ()=>params.runtime.getRandomID()
+            getRandomID: ()=>params.runtime.getRandomID(),
+            // Gate the scheduler on the host engine's identity verification: nudges and
+            // the interval claim no work until `ready()` resolves, and an identity
+            // mismatch leaves the engine inert rather than driving jobs on a database it
+            // was rejected from.
+            ready: ()=>params.engine.ready()
         });
-        // No scheduler in Phase 1: re-drive stranded instances once on boot.
-        void engine.recover().catch((error)=>{
+        // Re-drive stranded instances once on boot, but only after the host engine
+        // verifies its database identity — a re-driven instance mints under the engine's
+        // clock, so recovery waits behind `ready()` rather than tripping the mint gate.
+        void params.engine.ready().then(()=>engine.recover()).catch((error)=>{
             logger.error('workflow recovery failed', {
                 error
             });
         });
+        // Caller-side control-plane helpers over the p2p plugin's typed API. Resolved
+        // lazily (getAPI cannot be awaited while factories are still returning); a device
+        // with no p2p plugin rejects on first use.
+        const controlPlane = createWorkflowControlPlaneHelpers(()=>params.engine.getAPI('p2p'));
         return {
             name: 'workflow',
-            api: engine.api,
+            api: {
+                ...engine.api,
+                ...controlPlane
+            },
             schemaExtension: ()=>createWorkflowSchemaExtension(),
             createContextFactory: ()=>createWorkflowContextFactory(engine.api),
             dispose: ()=>engine.dispose()
@@ -26,6 +41,7 @@ export function createWorkflowPlugin(_options = {}) {
     };
 }
 export { computeNextDelay, validateAdaptivePolicy } from './adaptive.js';
-export { defineWorkflow } from './definition.js';
-export { createWorkflowEngine, parseEvery } from './engine.js';
+export { createWorkflowControlPlaneHelpers, WorkflowCommandRefusedError } from './control-plane.js';
+export { defineWorkflow, isRemotelyCommandable, isRemotelyObservable } from './definition.js';
+export { createWorkflowEngine, parseEvery, WorkflowRetryError } from './engine.js';
 export { createWorkflowContextFactory, createWorkflowSchemaExtension, subscribeToWorkflowEvents } from './schema.js';

package/lib/schema.d.ts

@@ -9,14 +9,13 @@ export type WorkflowEventResult = {
 /**
  * Subscribe to a workflow's lifecycle events, yielding GraphQL result shapes.
  *
- * Implemented as a manual async iterator rather than an `async function*` so a
- * consumer's `return()` removes all listeners immediately. A `for await` wrapper
- * would queue the `return()` behind a parked `next()` between events, leaking the
- * listeners until the next event — for an idle subscription, potentially never.
+ * A manual async iterator, not an `async function*`, so a consumer's `return()`
+ * removes all listeners immediately; a `for await` wrapper would queue `return()`
+ * behind a parked `next()` and leak the listeners until the next event — for an
+ * idle subscription, potentially never.
  *
- * `fromEmitter` (the connector template) handles a single event name over an
- * `EventEmitter`; `WorkflowEvents` has five keys and the API exposes only the
- * listen-only `EventsSource`, so the fan-in is done by hand over `on`.
+ * `fromEmitter` handles a single event name; `WorkflowEvents` has five keys and the
+ * API exposes only the listen-only `EventsSource`, so the fan-in is done by hand.
  */
 export declare function subscribeToWorkflowEvents(events: EventsSource<WorkflowEvents>, instanceID?: string | null): AsyncIterableIterator<WorkflowEventResult>;
 /**
@@ -30,6 +29,8 @@ export declare function subscribeToWorkflowEvents(events: EventsSource<WorkflowE
 export type WorkflowContext = {
     get(instanceID: string): Promise<InstanceStatus | null>;
     list(filter: ListFilter): Promise<Array<InstanceStatus>>;
+    retry(instanceID: string): Promise<InstanceStatus>;
+    cancel(instanceID: string): Promise<void>;
     subscribeToWorkflowEvents(instanceID?: string | null): AsyncIterableIterator<WorkflowEventResult>;
     getPeriodicSync(scheduleID: string): Promise<PeriodicSync | null>;
     listPeriodicSyncs(workflowName?: string): Promise<Array<PeriodicSync>>;

package/lib/schema.js

@@ -14,14 +14,13 @@ const EVENT_TYPES = [
 /**
  * Subscribe to a workflow's lifecycle events, yielding GraphQL result shapes.
  *
- * Implemented as a manual async iterator rather than an `async function*` so a
- * consumer's `return()` removes all listeners immediately. A `for await` wrapper
- * would queue the `return()` behind a parked `next()` between events, leaking the
- * listeners until the next event — for an idle subscription, potentially never.
+ * A manual async iterator, not an `async function*`, so a consumer's `return()`
+ * removes all listeners immediately; a `for await` wrapper would queue `return()`
+ * behind a parked `next()` and leak the listeners until the next event — for an
+ * idle subscription, potentially never.
  *
- * `fromEmitter` (the connector template) handles a single event name over an
- * `EventEmitter`; `WorkflowEvents` has five keys and the API exposes only the
- * listen-only `EventsSource`, so the fan-in is done by hand over `on`.
+ * `fromEmitter` handles a single event name; `WorkflowEvents` has five keys and the
+ * API exposes only the listen-only `EventsSource`, so the fan-in is done by hand.
  */ export function subscribeToWorkflowEvents(events, instanceID) {
     const filter = instanceID != null ? (data)=>data.instanceID === instanceID : undefined;
     let isDone = false;
@@ -118,6 +117,8 @@ export function createWorkflowContextFactory(api) {
     return ()=>({
             get: (instanceID)=>api.get(instanceID),
             list: (filter)=>api.list(filter),
+            retry: (instanceID)=>api.retry(instanceID),
+            cancel: (instanceID)=>api.cancel(instanceID),
             subscribeToWorkflowEvents: (instanceID)=>subscribeToWorkflowEvents(api.events, instanceID),
             getPeriodicSync: async (scheduleID)=>{
                 const sync = await api.getPeriodicSync(scheduleID);
@@ -182,6 +183,13 @@ extend type Query {
 extend type Mutation {
   setPeriodicSyncEnabled(id: ID!, enabled: Boolean!): PeriodicSync!
   setPeriodicSyncPolicy(id: ID!, policy: JSON!): PeriodicSync!
+  # Operator re-arm of a failed instance. Errors (not a null result) when the
+  # instance is missing, not failed, has no resumable action, or a live
+  # singleton already holds its key.
+  retryWorkflow(instanceID: ID!): WorkflowInstanceStatus!
+  # Operator-initiated cancel. Async: cancel() only requests cancellation, so the
+  # returned status is a re-fetch right after and may not yet be terminal.
+  cancelWorkflow(instanceID: ID!): WorkflowInstanceStatus!
 }
 
 extend type Subscription {
@@ -204,7 +212,18 @@ export function createWorkflowSchemaExtension() {
             },
             mutationFields: {
                 setPeriodicSyncEnabled: (_source, args, context)=>requireWorkflow(context).setPeriodicSyncEnabled(decodeScheduleID(args.id), args.enabled),
-                setPeriodicSyncPolicy: (_source, args, context)=>requireWorkflow(context).setPeriodicSyncPolicy(decodeScheduleID(args.id), args.policy)
+                setPeriodicSyncPolicy: (_source, args, context)=>requireWorkflow(context).setPeriodicSyncPolicy(decodeScheduleID(args.id), args.policy),
+                retryWorkflow: (_source, args, context)=>requireWorkflow(context).retry(args.instanceID),
+                cancelWorkflow: async (_source, args, context)=>{
+                    const ctx = requireWorkflow(context);
+                    const instanceID = args.instanceID;
+                    await ctx.cancel(instanceID);
+                    const status = await ctx.get(instanceID);
+                    if (status == null) {
+                        throw new Error(`Workflow instance "${instanceID}" not found`);
+                    }
+                    return status;
+                }
             },
             typeFields: {
                 PeriodicSync: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kubun/plugin-workflow",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "keywords": [],
   "license": "see LICENSE.md",
   "sideEffects": false,
@@ -15,21 +15,22 @@
     "LICENSE.md"
   ],
   "dependencies": {
-    "@kubun/db": "^0.15.0",
-    "@kubun/engine": "^0.15.0",
-    "@kubun/graphql": "^0.15.0",
-    "@kubun/id": "^0.15.0",
-    "@kubun/logger": "^0.15.0",
-    "@kubun/plugin-workflow-api": "^0.15.0",
-    "@kubun/store-workflow": "^0.15.0",
+    "@kubun/db": "^0.16.0",
+    "@kubun/engine": "^0.16.0",
+    "@kubun/graphql": "^0.16.0",
+    "@kubun/id": "^0.16.0",
+    "@kubun/logger": "^0.16.0",
+    "@kubun/plugin-p2p-api": "^0.16.0",
+    "@kubun/plugin-workflow-api": "^0.16.0",
+    "@kubun/store-workflow": "^0.16.0",
     "@sozai/event": "^0.1.3",
     "@sozai/flow": "^0.3.1",
     "@sozai/runtime": "^0.1.0",
     "@sozai/schema": "^0.1.2"
   },
   "devDependencies": {
-    "@kubun/db-better-sqlite": "^0.15.0",
-    "@kubun/db-postgres": "^0.15.0",
+    "@kubun/db-better-sqlite": "^0.16.0",
+    "@kubun/db-postgres": "^0.16.0",
     "@testcontainers/postgresql": "^12.1.0"
   },
   "publishConfig": {