@convex-dev/workpool 0.2.0-beta.0 → 0.2.0

package/dist/esm/component/stats.js

@@ -1,5 +1,6 @@
 import { v } from "convex/values";
-import { internalQuery } from "./_generated/server.js";
+import { internalQuery, query } from "./_generated/server.js";
+import { DEFAULT_MAX_PARALLELISM } from "./shared.js";
 /**
  * Record stats about work execution. Intended to be queried by Axiom or Datadog.
  */
@@ -12,36 +13,39 @@ workpool
     parse_json(trim("'", tostring(["data.message"]))),
     parse_json('{}')
 )
-| extend lagSinceEnqueued = parsed_message["lagSinceEnqueued"]
+| extend startLag = parsed_message["startLag"]
 | extend fnName = parsed_message["fnName"]
-| summarize avg(todouble(lagSinceEnqueued)) by bin_auto(_time), tostring(fnName)
+| summarize avg(todouble(startLag)) by bin_auto(_time), tostring(fnName)
 
  */
-export function recordStarted(work) {
-    return JSON.stringify({
+export function recordEnqueued(console, data) {
+    console.event("enqueued", {
+        ...data,
+        enqueuedAt: Date.now(),
+    });
+}
+export function recordStarted(console, work, lagMs) {
+    console.event("started", {
         workId: work._id,
-        event: "started",
         fnName: work.fnName,
         enqueuedAt: work._creationTime,
         startedAt: Date.now(),
-        lagSinceEnqueued: Date.now() - work._creationTime,
+        startLag: lagMs,
     });
 }
-export function recordCompleted(work, status) {
-    return JSON.stringify({
+export function recordCompleted(console, work, status) {
+    console.event("completed", {
         workId: work._id,
-        event: "completed",
         fnName: work.fnName,
         completedAt: Date.now(),
+        attempts: work.attempts,
         status,
-        lagSinceEnqueued: Date.now() - work._creationTime,
     });
 }
-export function recordReport(state) {
+export function recordReport(console, state) {
     const { completed, succeeded, failed, retries, canceled } = state.report;
     const withoutRetries = completed - retries;
-    return JSON.stringify({
-        event: "report",
+    console.event("report", {
         completed,
         succeeded,
         failed,
@@ -55,7 +59,7 @@ export function recordReport(state) {
  * Warning: this should not be used from a mutation, as it will cause conflicts.
  * Use this to debug or diagnose your queue length when it's backed up.
  */
-export const queueLength = internalQuery({
+export const queueLength = query({
     args: {},
     returns: v.number(),
     handler: async (ctx) => {
@@ -67,22 +71,28 @@ export const queueLength = internalQuery({
  * Warning: this should not be used from a mutation, as it will cause conflicts.
  * Use this while developing to see the state of the queue.
  */
-export const debugCounts = internalQuery({
+export const diagnostic = internalQuery({
     args: {},
     returns: v.any(),
     handler: async (ctx) => {
-        const inProgressWork = (await ctx.db.query("internalState").unique())?.running.length ?? 0;
+        const global = await ctx.db.query("globals").unique();
+        const internalState = await ctx.db.query("internalState").unique();
+        const inProgressWork = internalState?.running.length ?? 0;
+        const maxParallelism = global?.maxParallelism ?? DEFAULT_MAX_PARALLELISM;
         /* eslint-disable @typescript-eslint/no-explicit-any */
         const pendingStart = await ctx.db.query("pendingStart").count();
         const pendingCompletion = await ctx.db.query("pendingCompletion").count();
         const pendingCancelation = await ctx.db.query("pendingCancelation").count();
+        const runStatus = await ctx.db.query("runStatus").unique();
         /* eslint-enable @typescript-eslint/no-explicit-any */
         return {
-            pendingStart,
-            inProgressWork,
-            pendingCompletion,
-            pendingCancelation,
-            active: inProgressWork - pendingCompletion,
+            canceling: pendingCancelation,
+            waiting: pendingStart,
+            running: inProgressWork - pendingCompletion,
+            completing: pendingCompletion,
+            spareCapacity: maxParallelism - inProgressWork,
+            runStatus: runStatus?.state.kind,
+            generation: internalState?.generation,
         };
     },
 });

package/dist/esm/component/stats.js.map

@@ -1 +1 @@
-{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../../src/component/stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD;;GAEG;AAEH;;;;;;;;;;;;;GAaG;AAEH,MAAM,UAAU,aAAa,CAAC,IAAiB;IAC7C,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,KAAK,EAAE,SAAS;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,aAAa;QAC9B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,gBAAgB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa;KAClD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,IAAiB,EACjB,MAAyC;IAEzC,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,KAAK,EAAE,WAAW;QAClB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,MAAM;QACN,gBAAgB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa;KAClD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAA2B;IACtD,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC;IACzE,MAAM,cAAc,GAAG,SAAS,GAAG,OAAO,CAAC;IAC3C,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,KAAK,EAAE,QAAQ;QACf,SAAS;QACT,SAAS;QACT,MAAM;QACN,OAAO;QACP,QAAQ;QACR,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC3D,oBAAoB,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;KACnE,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;IACvC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,8DAA8D;QAC9D,OAAQ,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAS,CAAC,KAAK,EAAE,CAAC;IACvD,CAAC;CACF,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;IACvC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE;IAChB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,cAAc,GAClB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QACtE,uDAAuD;QACvD,MAAM,YAAY,GAAG,MAAO,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAS,CAAC,KAAK,EAAE,CAAC;QACzE,MAAM,iBAAiB,GAAG,MACxB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,mBAAmB,CACjC,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,kBAAkB,GAAG,MACzB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,oBAAoB,CAClC,CAAC,KAAK,EAAE,CAAC;QACV,sDAAsD;QACtD,OAAO;YACL,YAAY;YACZ,cAAc;YACd,iBAAiB;YACjB,kBAAkB;YAClB,MAAM,EAAE,cAAc,GAAG,iBAAiB;SAC3C,CAAC;IACJ,CAAC;CACF,CAAC,CAAC"}
+{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../../src/component/stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAGtD;;GAEG;AAEH;;;;;;;;;;;;;GAaG;AAEH,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,IAIC;IAED,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE;QACxB,GAAG,IAAI;QACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;KACvB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,OAAe,EACf,IAAiB,EACjB,KAAa;IAEb,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE;QACvB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,aAAa;QAC9B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,IAAiB,EACjB,MAAsD;IAEtD,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE;QACzB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM;KACP,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,KAA2B;IACvE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC;IACzE,MAAM,cAAc,GAAG,SAAS,GAAG,OAAO,CAAC;IAC3C,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE;QACtB,SAAS;QACT,SAAS;QACT,MAAM;QACN,OAAO;QACP,QAAQ;QACR,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC3D,oBAAoB,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;KACnE,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,CAAC;IAC/B,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,8DAA8D;QAC9D,OAAQ,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAS,CAAC,KAAK,EAAE,CAAC;IACvD,CAAC;CACF,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,aAAa,CAAC;IACtC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE;IAChB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;QACtD,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC;QACnE,MAAM,cAAc,GAAG,aAAa,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAC1D,MAAM,cAAc,GAAG,MAAM,EAAE,cAAc,IAAI,uBAAuB,CAAC;QACzE,uDAAuD;QACvD,MAAM,YAAY,GAAG,MAAO,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAS,CAAC,KAAK,EAAE,CAAC;QACzE,MAAM,iBAAiB,GAAG,MACxB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,mBAAmB,CACjC,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,kBAAkB,GAAG,MACzB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,oBAAoB,CAClC,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3D,sDAAsD;QACtD,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,OAAO,EAAE,YAAY;YACrB,OAAO,EAAE,cAAc,GAAG,iBAAiB;YAC3C,UAAU,EAAE,iBAAiB;YAC7B,aAAa,EAAE,cAAc,GAAG,cAAc;YAC9C,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI;YAChC,UAAU,EAAE,aAAa,EAAE,UAAU;SACtC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC"}

package/dist/esm/component/worker.d.ts

@@ -3,23 +3,13 @@ export declare const runMutationWrapper: import("convex/server").RegisteredMutat
     fnHandle: string;
     workId: import("convex/values").GenericId<"work">;
     fnArgs: any;
+    attempt: number;
 }, Promise<void>>;
 export declare const runActionWrapper: import("convex/server").RegisteredAction<"internal", {
     logLevel: "DEBUG" | "INFO" | "WARN" | "ERROR";
     fnHandle: string;
     workId: import("convex/values").GenericId<"work">;
     fnArgs: any;
-}, Promise<void>>;
-export declare const saveResult: import("convex/server").RegisteredMutation<"internal", {
-    workId: import("convex/values").GenericId<"work">;
-    runResult: {
-        kind: "success";
-        returnValue: any;
-    } | {
-        kind: "failed";
-        error: string;
-    } | {
-        kind: "canceled";
-    };
+    attempt: number;
 }, Promise<void>>;
 //# sourceMappingURL=worker.d.ts.map

package/dist/esm/component/worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../../src/component/worker.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,kBAAkB;;;;;iBA0B7B,CAAC;AASH,eAAO,MAAM,gBAAgB;;;;;iBA2B3B,CAAC;AAEH,eAAO,MAAM,UAAU;;;;;;;;;;;iBAarB,CAAC"}
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../../src/component/worker.ts"],"names":[],"mappings":"AAYA,eAAO,MAAM,kBAAkB;;;;;;iBA4B7B,CAAC;AASH,eAAO,MAAM,gBAAgB;;;;;;iBA4B3B,CAAC"}

package/dist/esm/component/worker.js

@@ -1,33 +1,33 @@
 import { v } from "convex/values";
 import { internal } from "./_generated/api.js";
 import { internalAction, internalMutation } from "./_generated/server.js";
-import { kickMainLoop } from "./kick.js";
 import { createLogger, logLevel } from "./logging.js";
-import { nextSegment, runResult } from "./shared.js";
 export const runMutationWrapper = internalMutation({
     args: {
         workId: v.id("work"),
         fnHandle: v.string(),
         fnArgs: v.any(),
         logLevel,
+        attempt: v.number(),
     },
-    handler: async (ctx, { workId, fnHandle: handleStr, fnArgs, logLevel }) => {
-        const console = createLogger(logLevel);
-        const fnHandle = handleStr;
+    handler: async (ctx, { workId, attempt, ...args }) => {
+        const console = createLogger(args.logLevel);
+        const fnHandle = args.fnHandle;
         try {
-            const returnValue = await ctx.runMutation(fnHandle, fnArgs);
+            const returnValue = await ctx.runMutation(fnHandle, args.fnArgs);
             // NOTE: we could run the `saveResult` handler here, or call `ctx.runMutation`,
             // but we want the mutation to be a separate transaction to reduce the window for OCCs.
-            await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-                workId,
-                runResult: { kind: "success", returnValue },
+            await ctx.scheduler.runAfter(0, internal.complete.complete, {
+                jobs: [
+                    { workId, runResult: { kind: "success", returnValue }, attempt },
+                ],
             });
         }
         catch (e) {
             console.error(e);
-            await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-                workId,
-                runResult: { kind: "failed", error: formatError(e) },
+            const runResult = { kind: "failed", error: formatError(e) };
+            await ctx.scheduler.runAfter(0, internal.complete.complete, {
+                jobs: [{ workId, runResult, attempt }],
             });
         }
     },
@@ -44,43 +44,30 @@ export const runActionWrapper = internalAction({
         fnHandle: v.string(),
         fnArgs: v.any(),
         logLevel,
+        attempt: v.number(),
     },
-    handler: async (ctx, { workId, fnHandle: handleStr, fnArgs, logLevel }) => {
-        const console = createLogger(logLevel);
-        const fnHandle = handleStr;
+    handler: async (ctx, { workId, attempt, ...args }) => {
+        const console = createLogger(args.logLevel);
+        const fnHandle = args.fnHandle;
         try {
-            const returnValue = await ctx.runAction(fnHandle, fnArgs);
+            const returnValue = await ctx.runAction(fnHandle, args.fnArgs);
             // NOTE: we could run `ctx.runMutation`, but we want to guarantee execution,
             // and `ctx.scheduler.runAfter` won't OCC.
-            await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-                workId,
-                runResult: { kind: "success", returnValue },
+            const runResult = { kind: "success", returnValue };
+            await ctx.scheduler.runAfter(0, internal.complete.complete, {
+                jobs: [{ workId, runResult, attempt }],
             });
         }
         catch (e) {
             console.error(e);
             // We let the main loop handle the retries.
-            await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-                workId,
-                runResult: { kind: "failed", error: formatError(e) },
+            const runResult = { kind: "failed", error: formatError(e) };
+            await ctx.scheduler.runAfter(0, internal.complete.complete, {
+                jobs: [{ workId, runResult, attempt }],
             });
         }
     },
 });
-export const saveResult = internalMutation({
-    args: {
-        workId: v.id("work"),
-        runResult,
-    },
-    handler: async (ctx, { workId, runResult }) => {
-        await ctx.db.insert("pendingCompletion", {
-            runResult,
-            workId,
-            segment: nextSegment(),
-        });
-        await kickMainLoop(ctx, "saveResult");
-    },
-});
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const console = "THIS IS A REMINDER TO USE createLogger";
 //# sourceMappingURL=worker.js.map

package/dist/esm/component/worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/component/worker.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,CAAC,MAAM,kBAAkB,GAAG,gBAAgB,CAAC;IACjD,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE;QACf,QAAQ;KACT;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;QACxE,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,QAAQ,GAAG,SAAuC,CAAC;QACzD,IAAI;YACF,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC5D,+EAA+E;YAC/E,uFAAuF;YACvF,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE;gBAC1D,MAAM;gBACN,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE;aAC5C,CAAC,CAAC;SACJ;QAAC,OAAO,CAAU,EAAE;YACnB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjB,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE;gBAC1D,MAAM;gBACN,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE;aACrD,CAAC,CAAC;SACJ;IACH,CAAC;CACF,CAAC,CAAC;AAEH,SAAS,WAAW,CAAC,CAAU;IAC7B,IAAI,CAAC,YAAY,KAAK,EAAE;QACtB,OAAO,CAAC,CAAC,OAAO,CAAC;KAClB;IACD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,cAAc,CAAC;IAC7C,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE;QACf,QAAQ;KACT;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;QACxE,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,QAAQ,GAAG,SAAqC,CAAC;QACvD,IAAI;YACF,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC1D,4EAA4E;YAC5E,0CAA0C;YAC1C,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE;gBAC1D,MAAM;gBACN,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE;aAC5C,CAAC,CAAC;SACJ;QAAC,OAAO,CAAU,EAAE;YACnB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjB,2CAA2C;YAC3C,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE;gBAC1D,MAAM;gBACN,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE;aACrD,CAAC,CAAC;SACJ;IACH,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;IACzC,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,SAAS;KACV;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE;QAC5C,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACvC,SAAS;YACT,MAAM;YACN,OAAO,EAAE,WAAW,EAAE;SACvB,CAAC,CAAC;QACH,MAAM,YAAY,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACxC,CAAC;CACF,CAAC,CAAC;AAEH,6DAA6D;AAC7D,MAAM,OAAO,GAAG,wCAAwC,CAAC"}
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/component/worker.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAGtD,MAAM,CAAC,MAAM,kBAAkB,GAAG,gBAAgB,CAAC;IACjD,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE;QACf,QAAQ;QACR,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;KACpB;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE;QACnD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAsC,CAAC;QAC7D,IAAI;YACF,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACjE,+EAA+E;YAC/E,uFAAuF;YACvF,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC1D,IAAI,EAAE;oBACJ,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE;iBACjE;aACF,CAAC,CAAC;SACJ;QAAC,OAAO,CAAU,EAAE;YACnB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjB,MAAM,SAAS,GAAG,EAAE,IAAI,EAAE,QAAiB,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YACrE,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC1D,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;aACvC,CAAC,CAAC;SACJ;IACH,CAAC;CACF,CAAC,CAAC;AAEH,SAAS,WAAW,CAAC,CAAU;IAC7B,IAAI,CAAC,YAAY,KAAK,EAAE;QACtB,OAAO,CAAC,CAAC,OAAO,CAAC;KAClB;IACD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,cAAc,CAAC;IAC7C,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE;QACf,QAAQ;QACR,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;KACpB;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE;QACnD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAoC,CAAC;QAC3D,IAAI;YACF,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC/D,4EAA4E;YAC5E,0CAA0C;YAC1C,MAAM,SAAS,GAAc,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;YAC9D,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC1D,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;aACvC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAU,EAAE;YACnB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjB,2CAA2C;YAC3C,MAAM,SAAS,GAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC1D,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;aACvC,CAAC,CAAC;SACJ;IACH,CAAC;CACF,CAAC,CAAC;AAEH,6DAA6D;AAC7D,MAAM,OAAO,GAAG,wCAAwC,CAAC"}

package/package.json

@@ -7,7 +7,7 @@
     "email": "support@convex.dev",
     "url": "https://github.com/get-convex/workpool/issues"
   },
-  "version": "0.2.0-beta.0",
+  "version": "0.2.0",
   "license": "Apache-2.0",
   "keywords": [
     "convex",
@@ -24,14 +24,15 @@
     "build:cjs": "tsc --project ./commonjs.json && echo '{\\n  \"type\": \"commonjs\"\\n}' > dist/commonjs/package.json",
     "dev": "cd example; npm run dev",
     "typecheck": "tsc --noEmit",
-    "prepare": "npm run build",
-    "prepack": "node node10stubs.mjs",
-    "postpack": "node node10stubs.mjs --cleanup",
-    "test": "vitest",
     "lint": "tsc --noEmit && eslint . && prettier --check .",
     "format": "prettier --write .",
+    "test": "vitest run",
+    "test:watch": "vitest",
     "test:debug": "vitest --inspect-brk --no-file-parallelism",
-    "test:coverage": "vitest run --coverage --coverage.reporter=text"
+    "test:coverage": "vitest run --coverage --coverage.reporter=text",
+    "prepare": "npm run build",
+    "prepack": "node node10stubs.mjs",
+    "postpack": "node node10stubs.mjs --cleanup"
   },
   "files": [
     "dist",

package/src/client/index.ts

@@ -8,19 +8,22 @@ import {
 import { v, VString } from "convex/values";
 import { Mounts } from "../component/_generated/api.js";
 import {
+  DEFAULT_LOG_LEVEL,
+  type LogLevel,
+  logLevel,
+} from "../component/logging.js";
+import {
+  Config,
+  DEFAULT_MAX_PARALLELISM,
   OnComplete,
-  runResult as runResultValidator,
-  RunResult,
+  runResult as resultValidator,
   type RetryBehavior,
+  RunResult,
   OnCompleteArgs as SharedOnCompleteArgs,
   Status,
-  Config,
 } from "../component/shared.js";
-import { type LogLevel, logLevel } from "../component/logging.js";
 import { RunMutationCtx, RunQueryCtx, UseApi } from "./utils.js";
-import { DEFAULT_LOG_LEVEL } from "../component/logging.js";
-import { DEFAULT_MAX_PARALLELISM } from "../component/kick.js";
-export { runResultValidator, type RunResult };
+export { resultValidator, type RunResult };
 
 // Attempts will run with delay [0, 250, 500, 1000, 2000] (ms)
 export const DEFAULT_RETRY_BEHAVIOR: RetryBehavior = {
@@ -133,11 +136,18 @@ export class Workpool {
     fnArgs: Args,
     options?: CallbackOptions & SchedulerOptions
   ): Promise<WorkId> {
+    const onComplete: OnComplete | undefined = options?.onComplete
+      ? {
+          fnHandle: await createFunctionHandle(options.onComplete),
+          context: options.context,
+        }
+      : undefined;
     const id = await ctx.runMutation(this.component.lib.enqueue, {
       ...(await defaultEnqueueArgs(fn, this.options)),
       fnArgs,
       fnType: "mutation",
       runAt: getRunAt(options),
+      onComplete,
     });
     return id as WorkId;
   }
@@ -215,7 +225,7 @@ export type CallbackOptions = {
    *  args: {
    *    workId: workIdValidator,
    *    context: v.any(),
-   *    result: runResult,
+   *    result: resultValidator,
    *  },
    *  handler: async (ctx, args) => {
    *    console.log(args.result, "Got Context back -> ", args.context, Date.now() - args.context);

package/src/component/README.md

@@ -25,19 +25,19 @@ Concepts:
 flowchart LR
     Client -->|enqueue| pendingStart
     Client -->|cancel| pendingCancelation
-    Recovery-->|recover| pendingCancelation
-    Recovery-->|recover| pendingCompletion
-    Worker-->|"saveResult"| pendingCompletion
-    pendingStart -->|main| workerRunning["internalState.running"]
-    workerRunning-->|"main(pendingCompletion)"| Retry{"Needs retry?"}
-    Retry-->|"no / canceled"| complete
-    Retry-->|yes| pendingStart
-    pendingStart-->|"main(pendingCancelation)"| complete
+    complete --> |success or failure| pendingCompletion
+    pendingCompletion -->|retry| pendingStart
+    pendingStart --> workerRunning["worker running"]
+    workerRunning -->|worker finished| complete
+    workerRunning --> |recovery| complete
+    successfulCancel["AND"]@{shape: delay} --> |canceled| complete
+    pendingStart --> successfulCancel
+    pendingCancelation --> successfulCancel
 ```
 
 Notably:
 
-- The pending\* states are only written by other sources.
+- The pending\* states are written by outside sources.
 - The main loop federates changes to/from "running"
 - Canceling only impacts pending and retrying jobs.
 
@@ -53,12 +53,12 @@ flowchart TD
     running-->|"all done"| idle
 ```
 
-- While the loop is running, clients won't see database conflicts with the
-  state changing.
-- The "saturated" state is concretely "running" or "scheduled" with a boolean
-  set, to avoid clients from kicking the main loop on enqueueing, which is
-  unlikely to be productive, since the next action needs to be something
-  terminating.
+- While the loop is running, the runStatus doesn't change, making it safer to
+  read from clients without database conflicts.
+- The "saturated" state is concretely "running" or "scheduled" at max
+  parallelism. There is a boolean set on "scheduled" to avoid clients from
+  kicking the main loop on enqueueing, which is unlikely to be productive, since
+  the next action needs to be something terminating.
 
 ## Retention optimization strategy
 

package/src/component/_generated/api.d.ts

@@ -8,6 +8,7 @@
  * @module
  */
 
+import type * as complete from "../complete.js";
 import type * as kick from "../kick.js";
 import type * as lib from "../lib.js";
 import type * as logging from "../logging.js";
@@ -31,6 +32,7 @@ import type {
  * ```
  */
 declare const fullApi: ApiFromModules<{
+  complete: typeof complete;
   kick: typeof kick;
   lib: typeof lib;
   logging: typeof logging;
@@ -80,11 +82,14 @@ export type Mounts = {
       "query",
       "public",
       { id: string },
-      | { attempt: number; state: "pending" }
-      | { attempt: number; state: "running" }
+      | { previousAttempts: number; state: "pending" }
+      | { previousAttempts: number; state: "running" }
       | { state: "finished" }
     >;
   };
+  stats: {
+    queueLength: FunctionReference<"query", "public", {}, number>;
+  };
 };
 // For now fullApiWithMounts is only fullApi which provides
 // jump-to-definition in component client code.