@convex-dev/workpool 0.1.3-alpha.0 → 0.2.0-beta.0

package/dist/esm/component/recovery.d.ts

@@ -0,0 +1,8 @@
+export declare const recover: import("convex/server").RegisteredMutation<"internal", {
+    jobs: {
+        workId: import("convex/values").GenericId<"work">;
+        scheduledId: import("convex/values").GenericId<"_scheduled_functions">;
+        started: number;
+    }[];
+}, Promise<void>>;
+//# sourceMappingURL=recovery.d.ts.map

package/dist/esm/component/recovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"recovery.d.ts","sourceRoot":"","sources":["../../../src/component/recovery.ts"],"names":[],"mappings":"AAOA,eAAO,MAAM,OAAO;;;;;;iBAuElB,CAAC"}

package/dist/esm/component/recovery.js

@@ -0,0 +1,74 @@
+import { internalMutation } from "./_generated/server.js";
+import { kickMainLoop } from "./kick.js";
+import { createLogger } from "./logging.js";
+import schema from "./schema.js";
+import { nextSegment } from "./shared.js";
+export const recover = internalMutation({
+    args: {
+        jobs: schema.tables.internalState.validator.fields.running,
+    },
+    handler: async (ctx, { jobs }) => {
+        const globals = await ctx.db.query("globals").unique();
+        const console = createLogger(globals?.logLevel);
+        const completed = [];
+        let didAnything = false;
+        const segment = nextSegment();
+        await Promise.all(jobs.map(async (job) => {
+            const scheduled = await ctx.db.system.get(job.scheduledId);
+            const preamble = `[recovery] Scheduled job ${job.scheduledId} for work ${job.workId}`;
+            if (scheduled === null) {
+                console.warn(`${preamble} not found`);
+                completed.push({
+                    workId: job.workId,
+                    runResult: { kind: "failed", error: `Scheduled job not found` },
+                });
+                return;
+            }
+            // This will find everything that timed out, failed ungracefully, was
+            // canceled, or succeeded without a return value.
+            switch (scheduled.state.kind) {
+                case "failed": {
+                    console.debug(`${preamble} failed and detected in recovery`);
+                    const pendingCompletion = await ctx.db
+                        .query("pendingCompletion")
+                        .withIndex("workId", (q) => q.eq("workId", job.workId))
+                        .first();
+                    if (pendingCompletion) {
+                        console.debug(`${preamble} already in pendingCompletion, not reporting`);
+                    }
+                    else {
+                        await ctx.db.insert("pendingCompletion", {
+                            runResult: scheduled.state,
+                            workId: job.workId,
+                            segment,
+                        });
+                        didAnything = true;
+                    }
+                    break;
+                }
+                case "canceled": {
+                    console.debug(`${preamble} was canceled and detected in recovery`);
+                    const pendingCancelation = await ctx.db
+                        .query("pendingCancelation")
+                        .withIndex("workId", (q) => q.eq("workId", job.workId))
+                        .first();
+                    if (pendingCancelation) {
+                        console.debug(`${preamble} already in pendingCancelation, not reporting`);
+                    }
+                    else {
+                        await ctx.db.insert("pendingCancelation", {
+                            workId: job.workId,
+                            segment,
+                        });
+                        didAnything = true;
+                    }
+                    break;
+                }
+            }
+        }));
+        if (didAnything) {
+            await kickMainLoop(ctx, "recovery");
+        }
+    },
+});
+//# sourceMappingURL=recovery.js.map

package/dist/esm/component/recovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recovery.js","sourceRoot":"","sources":["../../../src/component/recovery.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAa,WAAW,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,CAAC,MAAM,OAAO,GAAG,gBAAgB,CAAC;IACtC,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO;KAC3D;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QAC/B,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;QACvD,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,SAAS,GAAmD,EAAE,CAAC;QACrE,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,MAAM,OAAO,GAAG,WAAW,EAAE,CAAC;QAC9B,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACrB,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAC3D,MAAM,QAAQ,GAAG,4BAA4B,GAAG,CAAC,WAAW,aAAa,GAAG,CAAC,MAAM,EAAE,CAAC;YACtF,IAAI,SAAS,KAAK,IAAI,EAAE;gBACtB,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,YAAY,CAAC,CAAC;gBACtC,SAAS,CAAC,IAAI,CAAC;oBACb,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,yBAAyB,EAAE;iBAChE,CAAC,CAAC;gBACH,OAAO;aACR;YACD,qEAAqE;YACrE,iDAAiD;YACjD,QAAQ,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE;gBAC5B,KAAK,QAAQ,CAAC,CAAC;oBACb,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,kCAAkC,CAAC,CAAC;oBAC7D,MAAM,iBAAiB,GAAG,MAAM,GAAG,CAAC,EAAE;yBACnC,KAAK,CAAC,mBAAmB,CAAC;yBAC1B,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;yBACtD,KAAK,EAAE,CAAC;oBACX,IAAI,iBAAiB,EAAE;wBACrB,OAAO,CAAC,KAAK,CACX,GAAG,QAAQ,8CAA8C,CAC1D,CAAC;qBACH;yBAAM;wBACL,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,mBAAmB,EAAE;4BACvC,SAAS,EAAE,SAAS,CAAC,KAAK;4BAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;4BAClB,OAAO;yBACR,CAAC,CAAC;wBACH,WAAW,GAAG,IAAI,CAAC;qBACpB;oBACD,MAAM;iBACP;gBACD,KAAK,UAAU,CAAC,CAAC;oBACf,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,wCAAwC,CAAC,CAAC;oBACnE,MAAM,kBAAkB,GAAG,MAAM,GAAG,CAAC,EAAE;yBACpC,KAAK,CAAC,oBAAoB,CAAC;yBAC3B,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;yBACtD,KAAK,EAAE,CAAC;oBACX,IAAI,kBAAkB,EAAE;wBACtB,OAAO,CAAC,KAAK,CACX,GAAG,QAAQ,+CAA+C,CAC3D,CAAC;qBACH;yBAAM;wBACL,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,EAAE;4BACxC,MAAM,EAAE,GAAG,CAAC,MAAM;4BAClB,OAAO;yBACR,CAAC,CAAC;wBACH,WAAW,GAAG,IAAI,CAAC;qBACpB;oBACD,MAAM;iBACP;aACF;QACH,CAAC,CAAC,CACH,CAAC;QACF,IAAI,WAAW,EAAE;YACf,MAAM,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;SACrC;IACH,CAAC;CACF,CAAC,CAAC"}

package/dist/esm/component/schema.d.ts

@@ -1,154 +1,224 @@
-import { Infer } from "convex/values";
-export declare const completionStatus: import("convex/values").VUnion<"success" | "error" | "canceled" | "timeout", [import("convex/values").VLiteral<"success", "required">, import("convex/values").VLiteral<"error", "required">, import("convex/values").VLiteral<"canceled", "required">, import("convex/values").VLiteral<"timeout", "required">], "required", never>;
-export type CompletionStatus = Infer<typeof completionStatus>;
-/**
-Data flow:
-
-- The mutation `mainLoop` runs periodically and serially.
-- Several tables act as queues, with client-driven mutations enqueueing at high
-  timestamps and `mainLoop` popping at low timestamps:
-  pendingStart, pendingCompletion, and pendingCancelation.
-  - The `enqueue` mutation writes to pendingStart.
-  - The `cancel` mutation writes to pendingCancelation.
-  - The `saveResult` mutation, run as part of scheduled work, writes to pendingCompletion.
-- mainLoop processes the queues:
-  - pendingStart => inProgressWork.
-  - pendingCompletion and pendingCancelation => completedWork.
-  - inProgressWork that finishes uncleanly (timeout or system failure) => completedWork.
-- `mainLoop` schedules itself to run.
-- `enqueue`, `cancel`, and `saveResult` mutations check when `mainLoop` is scheduled to run,
-  and if it's too far in the future, they schedule it to run sooner.
-- `status` query reads from pendingWork and completedWork.
-- `cleanup` mutation deletes old rows from completedWork.
-
-To avoid OCCs, we restrict which mutations can read and write from each table:
-- pools: read by all, written only when static Workpool options change.
-- mainLoop (table): read by all, written mostly by `mainLoop`.
-  If `mainLoop` will not run for a while, mainLoop table is written by `enqueue`, `cancel`, or `saveResult`.
-- pendingWork: `enqueue` inserts at high timestamps, `mainLoop` pops at low timestamps. `status` query does point-reads.
-- pendingCompletion: `saveResult` inserts at high timestamps, `mainLoop` pops at low timestamps.
-- pendingCancelation: `cancel` inserts at high timestamps, `mainLoop` pops at low timestamps.
-- inProgressWork: `mainLoop` inserts, reads all, and deletes.
-- completedWork: `mainLoop` inserts at hight timestamps, `status` query reads, `cleanup` deletes at low timestamps.
-
- */
 declare const _default: import("convex/server").SchemaDefinition<{
-    pool: import("convex/server").TableDefinition<import("convex/values").VObject<{
+    globals: import("convex/server").TableDefinition<import("convex/values").VObject<{
         maxParallelism: number;
-        statusTtl: number;
         logLevel: "DEBUG" | "INFO" | "WARN" | "ERROR";
     }, {
         maxParallelism: import("convex/values").VFloat64<number, "required">;
-        statusTtl: import("convex/values").VFloat64<number, "required">;
         logLevel: import("convex/values").VUnion<"DEBUG" | "INFO" | "WARN" | "ERROR", [import("convex/values").VLiteral<"DEBUG", "required">, import("convex/values").VLiteral<"INFO", "required">, import("convex/values").VLiteral<"WARN", "required">, import("convex/values").VLiteral<"ERROR", "required">], "required", never>;
-    }, "required", "maxParallelism" | "statusTtl" | "logLevel">, {}, {}, {}>;
-    mainLoop: import("convex/server").TableDefinition<import("convex/values").VObject<{
+    }, "required", "maxParallelism" | "logLevel">, {}, {}, {}>;
+    internalState: import("convex/server").TableDefinition<import("convex/values").VObject<{
+        running: {
+            workId: import("convex/values").GenericId<"work">;
+            scheduledId: import("convex/values").GenericId<"_scheduled_functions">;
+            started: number;
+        }[];
+        lastRecovery: bigint;
+        generation: bigint;
+        segmentCursors: {
+            incoming: bigint;
+            completion: bigint;
+            cancelation: bigint;
+        };
+        report: {
+            failed: number;
+            canceled: number;
+            completed: number;
+            succeeded: number;
+            retries: number;
+            lastReportTs: number;
+        };
+    }, {
+        generation: import("convex/values").VInt64<bigint, "required">;
+        segmentCursors: import("convex/values").VObject<{
+            incoming: bigint;
+            completion: bigint;
+            cancelation: bigint;
+        }, {
+            incoming: import("convex/values").VInt64<bigint, "required">;
+            completion: import("convex/values").VInt64<bigint, "required">;
+            cancelation: import("convex/values").VInt64<bigint, "required">;
+        }, "required", "incoming" | "completion" | "cancelation">;
+        lastRecovery: import("convex/values").VInt64<bigint, "required">;
+        report: import("convex/values").VObject<{
+            failed: number;
+            canceled: number;
+            completed: number;
+            succeeded: number;
+            retries: number;
+            lastReportTs: number;
+        }, {
+            completed: import("convex/values").VFloat64<number, "required">;
+            succeeded: import("convex/values").VFloat64<number, "required">;
+            failed: import("convex/values").VFloat64<number, "required">;
+            retries: import("convex/values").VFloat64<number, "required">;
+            canceled: import("convex/values").VFloat64<number, "required">;
+            lastReportTs: import("convex/values").VFloat64<number, "required">;
+        }, "required", "failed" | "canceled" | "completed" | "succeeded" | "retries" | "lastReportTs">;
+        running: import("convex/values").VArray<{
+            workId: import("convex/values").GenericId<"work">;
+            scheduledId: import("convex/values").GenericId<"_scheduled_functions">;
+            started: number;
+        }[], import("convex/values").VObject<{
+            workId: import("convex/values").GenericId<"work">;
+            scheduledId: import("convex/values").GenericId<"_scheduled_functions">;
+            started: number;
+        }, {
+            workId: import("convex/values").VId<import("convex/values").GenericId<"work">, "required">;
+            scheduledId: import("convex/values").VId<import("convex/values").GenericId<"_scheduled_functions">, "required">;
+            started: import("convex/values").VFloat64<number, "required">;
+        }, "required", "workId" | "scheduledId" | "started">, "required">;
+    }, "required", "running" | "lastRecovery" | "generation" | "segmentCursors" | "report" | "segmentCursors.incoming" | "segmentCursors.completion" | "segmentCursors.cancelation" | "report.failed" | "report.canceled" | "report.completed" | "report.succeeded" | "report.retries" | "report.lastReportTs">, {}, {}, {}>;
+    runStatus: import("convex/server").TableDefinition<import("convex/values").VObject<{
         state: {
             kind: "running";
         } | {
             kind: "scheduled";
-            runAtTime: number;
-            fn: import("convex/values").GenericId<"_scheduled_functions">;
+            generation: bigint;
+            scheduledId: import("convex/values").GenericId<"_scheduled_functions">;
+            segment: bigint;
+            saturated: boolean;
         } | {
             kind: "idle";
+            generation: bigint;
         };
     }, {
         state: import("convex/values").VUnion<{
             kind: "running";
         } | {
             kind: "scheduled";
-            runAtTime: number;
-            fn: import("convex/values").GenericId<"_scheduled_functions">;
+            generation: bigint;
+            scheduledId: import("convex/values").GenericId<"_scheduled_functions">;
+            segment: bigint;
+            saturated: boolean;
         } | {
             kind: "idle";
+            generation: bigint;
         }, [import("convex/values").VObject<{
             kind: "running";
         }, {
             kind: import("convex/values").VLiteral<"running", "required">;
         }, "required", "kind">, import("convex/values").VObject<{
             kind: "scheduled";
-            runAtTime: number;
-            fn: import("convex/values").GenericId<"_scheduled_functions">;
+            generation: bigint;
+            scheduledId: import("convex/values").GenericId<"_scheduled_functions">;
+            segment: bigint;
+            saturated: boolean;
         }, {
             kind: import("convex/values").VLiteral<"scheduled", "required">;
-            runAtTime: import("convex/values").VFloat64<number, "required">;
-            fn: import("convex/values").VId<import("convex/values").GenericId<"_scheduled_functions">, "required">;
-        }, "required", "kind" | "runAtTime" | "fn">, import("convex/values").VObject<{
+            segment: import("convex/values").VInt64<bigint, "required">;
+            scheduledId: import("convex/values").VId<import("convex/values").GenericId<"_scheduled_functions">, "required">;
+            saturated: import("convex/values").VBoolean<boolean, "required">;
+            generation: import("convex/values").VInt64<bigint, "required">;
+        }, "required", "kind" | "generation" | "scheduledId" | "segment" | "saturated">, import("convex/values").VObject<{
             kind: "idle";
+            generation: bigint;
         }, {
             kind: import("convex/values").VLiteral<"idle", "required">;
-        }, "required", "kind">], "required", "kind" | "runAtTime" | "fn">;
-    }, "required", "state" | "state.kind" | "state.runAtTime" | "state.fn">, {}, {}, {}>;
+            generation: import("convex/values").VInt64<bigint, "required">;
+        }, "required", "kind" | "generation">], "required", "kind" | "generation" | "scheduledId" | "segment" | "saturated">;
+    }, "required", "state" | "state.kind" | "state.generation" | "state.scheduledId" | "state.segment" | "state.saturated">, {}, {}, {}>;
     work: import("convex/server").TableDefinition<import("convex/values").VObject<{
-        fnType: "action" | "mutation";
+        onComplete?: {
+            context?: any;
+            fnHandle: string;
+        } | undefined;
+        retryBehavior?: {
+            maxAttempts: number;
+            initialBackoffMs: number;
+            base: number;
+        } | undefined;
         fnHandle: string;
+        fnType: "action" | "mutation";
         fnName: string;
         fnArgs: any;
+        attempts: number;
     }, {
         fnType: import("convex/values").VUnion<"action" | "mutation", [import("convex/values").VLiteral<"action", "required">, import("convex/values").VLiteral<"mutation", "required">], "required", never>;
         fnHandle: import("convex/values").VString<string, "required">;
         fnName: import("convex/values").VString<string, "required">;
         fnArgs: import("convex/values").VAny<any, "required", string>;
-    }, "required", "fnType" | "fnHandle" | "fnName" | "fnArgs" | `fnArgs.${string}`>, {}, {}, {}>;
+        attempts: import("convex/values").VFloat64<number, "required">;
+        onComplete: import("convex/values").VObject<{
+            context?: any;
+            fnHandle: string;
+        } | undefined, {
+            fnHandle: import("convex/values").VString<string, "required">;
+            context: import("convex/values").VAny<any, "optional", string>;
+        }, "optional", "fnHandle" | "context" | `context.${string}`>;
+        retryBehavior: import("convex/values").VObject<{
+            maxAttempts: number;
+            initialBackoffMs: number;
+            base: number;
+        } | undefined, {
+            maxAttempts: import("convex/values").VFloat64<number, "required">;
+            initialBackoffMs: import("convex/values").VFloat64<number, "required">;
+            base: import("convex/values").VFloat64<number, "required">;
+        }, "optional", "maxAttempts" | "initialBackoffMs" | "base">;
+    }, "required", "fnHandle" | "fnType" | "fnName" | "fnArgs" | "attempts" | "onComplete" | "retryBehavior" | `fnArgs.${string}` | "onComplete.fnHandle" | "onComplete.context" | `onComplete.context.${string}` | "retryBehavior.maxAttempts" | "retryBehavior.initialBackoffMs" | "retryBehavior.base">, {}, {}, {}>;
     pendingStart: import("convex/server").TableDefinition<import("convex/values").VObject<{
         workId: import("convex/values").GenericId<"work">;
+        segment: bigint;
     }, {
         workId: import("convex/values").VId<import("convex/values").GenericId<"work">, "required">;
-    }, "required", "workId">, {
+        segment: import("convex/values").VInt64<bigint, "required">;
+    }, "required", "workId" | "segment">, {
         workId: ["workId", "_creationTime"];
+        segment: ["segment", "_creationTime"];
     }, {}, {}>;
     pendingCompletion: import("convex/server").TableDefinition<import("convex/values").VObject<{
         workId: import("convex/values").GenericId<"work">;
-        generation: number;
-        completionStatus: "success" | "error" | "canceled" | "timeout";
+        segment: bigint;
+        runResult: {
+            kind: "success";
+            returnValue: any;
+        } | {
+            kind: "failed";
+            error: string;
+        } | {
+            kind: "canceled";
+        };
     }, {
-        generation: import("convex/values").VFloat64<number, "required">;
-        completionStatus: import("convex/values").VUnion<"success" | "error" | "canceled" | "timeout", [import("convex/values").VLiteral<"success", "required">, import("convex/values").VLiteral<"error", "required">, import("convex/values").VLiteral<"canceled", "required">, import("convex/values").VLiteral<"timeout", "required">], "required", never>;
+        segment: import("convex/values").VInt64<bigint, "required">;
+        runResult: import("convex/values").VUnion<{
+            kind: "success";
+            returnValue: any;
+        } | {
+            kind: "failed";
+            error: string;
+        } | {
+            kind: "canceled";
+        }, [import("convex/values").VObject<{
+            kind: "success";
+            returnValue: any;
+        }, {
+            kind: import("convex/values").VLiteral<"success", "required">;
+            returnValue: import("convex/values").VAny<any, "required", string>;
+        }, "required", "kind" | "returnValue" | `returnValue.${string}`>, import("convex/values").VObject<{
+            kind: "failed";
+            error: string;
+        }, {
+            kind: import("convex/values").VLiteral<"failed", "required">;
+            error: import("convex/values").VString<string, "required">;
+        }, "required", "kind" | "error">, import("convex/values").VObject<{
+            kind: "canceled";
+        }, {
+            kind: import("convex/values").VLiteral<"canceled", "required">;
+        }, "required", "kind">], "required", "kind" | "returnValue" | `returnValue.${string}` | "error">;
         workId: import("convex/values").VId<import("convex/values").GenericId<"work">, "required">;
-    }, "required", "workId" | "generation" | "completionStatus">, {
+    }, "required", "workId" | "segment" | "runResult" | "runResult.kind" | "runResult.returnValue" | `runResult.returnValue.${string}` | "runResult.error">, {
         workId: ["workId", "_creationTime"];
-        generation: ["generation", "_creationTime"];
+        segment: ["segment", "_creationTime"];
     }, {}, {}>;
     pendingCancelation: import("convex/server").TableDefinition<import("convex/values").VObject<{
         workId: import("convex/values").GenericId<"work">;
+        segment: bigint;
     }, {
+        segment: import("convex/values").VInt64<bigint, "required">;
         workId: import("convex/values").VId<import("convex/values").GenericId<"work">, "required">;
-    }, "required", "workId">, {}, {}, {}>;
-    inProgressWork: import("convex/server").TableDefinition<import("convex/values").VObject<{
-        running: import("convex/values").GenericId<"_scheduled_functions">;
-        workId: import("convex/values").GenericId<"work">;
-        timeoutMs: number | null;
-    }, {
-        running: import("convex/values").VId<import("convex/values").GenericId<"_scheduled_functions">, "required">;
-        timeoutMs: import("convex/values").VUnion<number | null, [import("convex/values").VFloat64<number, "required">, import("convex/values").VNull<null, "required">], "required", never>;
-        workId: import("convex/values").VId<import("convex/values").GenericId<"work">, "required">;
-    }, "required", "running" | "workId" | "timeoutMs">, {
-        workId: ["workId", "_creationTime"];
-    }, {}, {}>;
-    inProgressCount: import("convex/server").TableDefinition<import("convex/values").VObject<{
-        count: number;
-    }, {
-        count: import("convex/values").VFloat64<number, "required">;
-    }, "required", "count">, {}, {}, {}>;
-    completedWork: import("convex/server").TableDefinition<import("convex/values").VObject<{
-        workId: import("convex/values").GenericId<"work">;
-        completionStatus: "success" | "error" | "canceled" | "timeout";
-    }, {
-        completionStatus: import("convex/values").VUnion<"success" | "error" | "canceled" | "timeout", [import("convex/values").VLiteral<"success", "required">, import("convex/values").VLiteral<"error", "required">, import("convex/values").VLiteral<"canceled", "required">, import("convex/values").VLiteral<"timeout", "required">], "required", never>;
-        workId: import("convex/values").VId<import("convex/values").GenericId<"work">, "required">;
-    }, "required", "workId" | "completionStatus">, {
+    }, "required", "workId" | "segment">, {
         workId: ["workId", "_creationTime"];
+        segment: ["segment", "_creationTime"];
     }, {}, {}>;
-    completionGeneration: import("convex/server").TableDefinition<import("convex/values").VObject<{
-        generation: number;
-    }, {
-        generation: import("convex/values").VFloat64<number, "required">;
-    }, "required", "generation">, {}, {}, {}>;
-    pendingStartCursor: import("convex/server").TableDefinition<import("convex/values").VObject<{
-        cursor: number;
-    }, {
-        cursor: import("convex/values").VFloat64<number, "required">;
-    }, "required", "cursor">, {}, {}, {}>;
 }, true>;
 export default _default;
 //# sourceMappingURL=schema.d.ts.map

package/dist/esm/component/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/component/schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAK,MAAM,eAAe,CAAC;AAGzC,eAAO,MAAM,gBAAgB,sUAK5B,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,wBA8DG"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/component/schema.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAOA,wBAiFG"}

package/dist/esm/component/schema.js

@@ -1,88 +1,77 @@
 import { defineSchema, defineTable } from "convex/server";
 import { v } from "convex/values";
-import { logLevel } from "./logging.js";
-export const completionStatus = v.union(v.literal("success"), v.literal("error"), v.literal("canceled"), v.literal("timeout"));
-/**
-Data flow:
-
-- The mutation `mainLoop` runs periodically and serially.
-- Several tables act as queues, with client-driven mutations enqueueing at high
-  timestamps and `mainLoop` popping at low timestamps:
-  pendingStart, pendingCompletion, and pendingCancelation.
-  - The `enqueue` mutation writes to pendingStart.
-  - The `cancel` mutation writes to pendingCancelation.
-  - The `saveResult` mutation, run as part of scheduled work, writes to pendingCompletion.
-- mainLoop processes the queues:
-  - pendingStart => inProgressWork.
-  - pendingCompletion and pendingCancelation => completedWork.
-  - inProgressWork that finishes uncleanly (timeout or system failure) => completedWork.
-- `mainLoop` schedules itself to run.
-- `enqueue`, `cancel`, and `saveResult` mutations check when `mainLoop` is scheduled to run,
-  and if it's too far in the future, they schedule it to run sooner.
-- `status` query reads from pendingWork and completedWork.
-- `cleanup` mutation deletes old rows from completedWork.
-
-To avoid OCCs, we restrict which mutations can read and write from each table:
-- pools: read by all, written only when static Workpool options change.
-- mainLoop (table): read by all, written mostly by `mainLoop`.
-  If `mainLoop` will not run for a while, mainLoop table is written by `enqueue`, `cancel`, or `saveResult`.
-- pendingWork: `enqueue` inserts at high timestamps, `mainLoop` pops at low timestamps. `status` query does point-reads.
-- pendingCompletion: `saveResult` inserts at high timestamps, `mainLoop` pops at low timestamps.
-- pendingCancelation: `cancel` inserts at high timestamps, `mainLoop` pops at low timestamps.
-- inProgressWork: `mainLoop` inserts, reads all, and deletes.
-- completedWork: `mainLoop` inserts at hight timestamps, `status` query reads, `cleanup` deletes at low timestamps.
-
- */
+import { config, onComplete, retryBehavior, runResult } from "./shared.js";
+// Represents a slice of time to process work.
+const segment = v.int64();
 export default defineSchema({
-    // Statically configured, singleton.
-    pool: defineTable({
-        maxParallelism: v.number(),
-        statusTtl: v.number(),
-        logLevel,
+    // Written from kickLoop, read everywhere.
+    globals: defineTable(config),
+    // Singleton, only read & written by `main`.
+    internalState: defineTable({
+        // Ensure that only one main is running at a time.
+        generation: v.int64(),
+        segmentCursors: v.object({
+            incoming: segment,
+            completion: segment,
+            cancelation: segment,
+        }),
+        lastRecovery: segment,
+        report: v.object({
+            completed: v.number(),
+            succeeded: v.number(),
+            failed: v.number(),
+            retries: v.number(),
+            canceled: v.number(),
+            lastReportTs: v.number(),
+        }),
+        running: v.array(v.object({
+            workId: v.id("work"),
+            scheduledId: v.id("_scheduled_functions"),
+            started: v.number(),
+        })),
     }),
-    mainLoop: defineTable({
+    // Singleton, written by `updateRunStatus` when running, by client or worker otherwise.
+    // Safe to read from kickLoop, since it should update infrequently.
+    runStatus: defineTable({
         state: v.union(v.object({ kind: v.literal("running") }), v.object({
             kind: v.literal("scheduled"),
-            runAtTime: v.number(),
-            fn: v.id("_scheduled_functions"),
-        }), v.object({ kind: v.literal("idle") })),
+            segment,
+            scheduledId: v.id("_scheduled_functions"),
+            saturated: v.boolean(),
+            generation: v.int64(),
+        }), v.object({ kind: v.literal("idle"), generation: v.int64() })),
     }),
+    // Written on enqueue. Safe to read. Deleted by `complete`.
     work: defineTable({
         fnType: v.union(v.literal("action"), v.literal("mutation")),
         fnHandle: v.string(),
         fnName: v.string(),
         fnArgs: v.any(),
+        attempts: v.number(),
+        onComplete: v.optional(onComplete),
+        retryBehavior: v.optional(retryBehavior),
     }),
+    // Written on enqueue, read & deleted by `main`.
     pendingStart: defineTable({
         workId: v.id("work"),
-    }).index("workId", ["workId"]),
+        segment,
+    })
+        .index("workId", ["workId"])
+        .index("segment", ["segment"]),
+    // Written by job, read & deleted by `main`.
     pendingCompletion: defineTable({
-        generation: v.number(),
-        completionStatus,
+        segment,
+        runResult,
         workId: v.id("work"),
     })
         .index("workId", ["workId"])
-        .index("generation", ["generation"]),
+        .index("segment", ["segment"]),
+    // Written on cancelation, read & deleted by `main`.
     pendingCancelation: defineTable({
+        segment,
         workId: v.id("work"),
-    }),
-    inProgressWork: defineTable({
-        running: v.id("_scheduled_functions"),
-        timeoutMs: v.union(v.number(), v.null()),
-        workId: v.id("work"),
-    }).index("workId", ["workId"]),
-    inProgressCount: defineTable({
-        count: v.number(),
-    }),
-    completedWork: defineTable({
-        completionStatus,
-        workId: v.id("work"),
-    }).index("workId", ["workId"]),
-    completionGeneration: defineTable({
-        generation: v.number(),
-    }),
-    pendingStartCursor: defineTable({
-        cursor: v.number(),
-    }),
+    })
+        .index("workId", ["workId"])
+        .index("segment", ["segment"]),
 });
 //# sourceMappingURL=schema.js.map

package/dist/esm/component/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/component/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EAAS,CAAC,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAExC,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CACrC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EACpB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAClB,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EACrB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CACrB,CAAC;AAGF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,eAAe,YAAY,CAAC;IAC1B,oCAAoC;IACpC,IAAI,EAAE,WAAW,CAAC;QAChB,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;QAC1B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,QAAQ;KACT,CAAC;IAEF,QAAQ,EAAE,WAAW,CAAC;QACpB,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,EACxC,CAAC,CAAC,MAAM,CAAC;YACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;YAC5B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;YACrB,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,sBAAsB,CAAC;SACjC,CAAC,EACF,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CACtC;KACF,CAAC;IAEF,IAAI,EAAE,WAAW,CAAC;QAChB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC3D,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;QAClB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE;KAChB,CAAC;IAEF,YAAY,EAAE,WAAW,CAAC;QACxB,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;KACrB,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC9B,iBAAiB,EAAE,WAAW,CAAC;QAC7B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;QACtB,gBAAgB;QAChB,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;KACrB,CAAC;SACC,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;SAC3B,KAAK,CAAC,YAAY,EAAE,CAAC,YAAY,CAAC,CAAC;IACtC,kBAAkB,EAAE,WAAW,CAAC;QAC9B,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;KACrB,CAAC;IAEF,cAAc,EAAE,WAAW,CAAC;QAC1B,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,sBAAsB,CAAC;QACrC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;KACrB,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC9B,eAAe,EAAE,WAAW,CAAC;QAC3B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;KAClB,CAAC;IAEF,aAAa,EAAE,WAAW,CAAC;QACzB,gBAAgB;QAChB,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;KACrB,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;IAE9B,oBAAoB,EAAE,WAAW,CAAC;QAChC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;KACvB,CAAC;IAEF,kBAAkB,EAAE,WAAW,CAAC;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;KACnB,CAAC;CACH,CAAC,CAAC"}
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/component/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE3E,8CAA8C;AAC9C,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;AAE1B,eAAe,YAAY,CAAC;IAC1B,0CAA0C;IAC1C,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC;IAC5B,4CAA4C;IAC5C,aAAa,EAAE,WAAW,CAAC;QACzB,kDAAkD;QAClD,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE;QACrB,cAAc,EAAE,CAAC,CAAC,MAAM,CAAC;YACvB,QAAQ,EAAE,OAAO;YACjB,UAAU,EAAE,OAAO;YACnB,WAAW,EAAE,OAAO;SACrB,CAAC;QACF,YAAY,EAAE,OAAO;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;YACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;YACrB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;YAClB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;YACnB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;YACpB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;SACzB,CAAC;QACF,OAAO,EAAE,CAAC,CAAC,KAAK,CACd,CAAC,CAAC,MAAM,CAAC;YACP,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;YACpB,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,sBAAsB,CAAC;YACzC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;SACpB,CAAC,CACH;KACF,CAAC;IAEF,uFAAuF;IACvF,mEAAmE;IACnE,SAAS,EAAE,WAAW,CAAC;QACrB,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,EACxC,CAAC,CAAC,MAAM,CAAC;YACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;YAC5B,OAAO;YACP,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,sBAAsB,CAAC;YACzC,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;YACtB,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE;SACtB,CAAC,EACF,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAC7D;KACF,CAAC;IAEF,2DAA2D;IAC3D,IAAI,EAAE,WAAW,CAAC;QAChB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC3D,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;QAClB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE;QACf,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;QAClC,aAAa,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC;KACzC,CAAC;IAEF,gDAAgD;IAChD,YAAY,EAAE,WAAW,CAAC;QACxB,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,OAAO;KACR,CAAC;SACC,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;SAC3B,KAAK,CAAC,SAAS,EAAE,CAAC,SAAS,CAAC,CAAC;IAEhC,4CAA4C;IAC5C,iBAAiB,EAAE,WAAW,CAAC;QAC7B,OAAO;QACP,SAAS;QACT,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;KACrB,CAAC;SACC,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;SAC3B,KAAK,CAAC,SAAS,EAAE,CAAC,SAAS,CAAC,CAAC;IAEhC,oDAAoD;IACpD,kBAAkB,EAAE,WAAW,CAAC;QAC9B,OAAO;QACP,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;KACrB,CAAC;SACC,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;SAC3B,KAAK,CAAC,SAAS,EAAE,CAAC,SAAS,CAAC,CAAC;CACjC,CAAC,CAAC"}