@convex-dev/workpool 0.2.0-beta.0 → 0.2.0

package/src/component/schema.ts

@@ -51,18 +51,19 @@ export default defineSchema({
     ),
   }),
 
-  // Written on enqueue. Safe to read. Deleted by `complete`.
+  // Written on enqueue. Deleted by `complete` for success, failure, canceled.
   work: defineTable({
     fnType: v.union(v.literal("action"), v.literal("mutation")),
     fnHandle: v.string(),
     fnName: v.string(),
     fnArgs: v.any(),
-    attempts: v.number(),
+    attempts: v.number(), // number of completed attempts
     onComplete: v.optional(onComplete),
     retryBehavior: v.optional(retryBehavior),
+    canceled: v.optional(v.boolean()),
   }),
 
-  // Written on enqueue, read & deleted by `main`.
+  // Written on enqueue & rescheduled for retry, read & deleted by `main`.
   pendingStart: defineTable({
     workId: v.id("work"),
     segment,
@@ -70,11 +71,12 @@ export default defineSchema({
     .index("workId", ["workId"])
     .index("segment", ["segment"]),
 
-  // Written by job, read & deleted by `main`.
+  // Written by complete, read & deleted by `main`.
   pendingCompletion: defineTable({
     segment,
     runResult,
     workId: v.id("work"),
+    retry: v.boolean(),
   })
     .index("workId", ["workId"])
     .index("segment", ["segment"]),

package/src/component/shared.ts

@@ -3,7 +3,8 @@ import { Infer } from "convex/values";
 import { v } from "convex/values";
 import { Logger, logLevel } from "./logging.js";
 
-const SEGMENT_MS = 250;
+export const DEFAULT_MAX_PARALLELISM = 10;
+const SEGMENT_MS = 100;
 export const SECOND = 1000;
 export const MINUTE = 60 * SECOND;
 export const HOUR = 60 * MINUTE;
@@ -98,11 +99,11 @@ export const status = v.union(
   v.union(
     v.object({
       state: v.literal("pending"),
-      attempt: v.number(),
+      previousAttempts: v.number(),
     }),
     v.object({
       state: v.literal("running"),
-      attempt: v.number(),
+      previousAttempts: v.number(),
     }),
     v.object({
       state: v.literal("finished"),
@@ -113,15 +114,29 @@ export type Status = Infer<typeof status>;
 
 export function boundScheduledTime(ms: number, console: Logger): number {
   if (ms < Date.now() - YEAR) {
-    console.warn("runAt is too far in the past, defaulting to now", ms);
+    console.error("scheduled time is too old, defaulting to now", ms);
     return Date.now();
   }
   if (ms > Date.now() + 4 * YEAR) {
-    console.warn(
-      "runAt is too far in the future, defaulting to 1 year from now",
+    console.error(
+      "scheduled time is too far in the future, defaulting to 1 year from now",
       ms
     );
     return Date.now() + YEAR;
   }
   return ms;
 }
+
+/**
+ * Returns the smaller of two bigint values.
+ */
+export function min<T extends bigint>(a: T, b: T): T {
+  return a > b ? b : a;
+}
+
+/**
+ * Returns the larger of two bigint values.
+ */
+export function max<T extends bigint>(a: T, b: T): T {
+  return a < b ? b : a;
+}

package/src/component/stats.ts

@@ -1,6 +1,8 @@
 import { v } from "convex/values";
-import { Doc } from "./_generated/dataModel.js";
-import { internalQuery } from "./_generated/server.js";
+import { Doc, Id } from "./_generated/dataModel.js";
+import { internalQuery, query } from "./_generated/server.js";
+import { DEFAULT_MAX_PARALLELISM } from "./shared.js";
+import { Logger } from "./logging.js";
 
 /**
  * Record stats about work execution. Intended to be queried by Axiom or Datadog.
@@ -15,42 +17,58 @@ workpool
 	parse_json(trim("'", tostring(["data.message"]))),
 	parse_json('{}')
 )
-| extend lagSinceEnqueued = parsed_message["lagSinceEnqueued"]
+| extend startLag = parsed_message["startLag"]
 | extend fnName = parsed_message["fnName"]
-| summarize avg(todouble(lagSinceEnqueued)) by bin_auto(_time), tostring(fnName)
+| summarize avg(todouble(startLag)) by bin_auto(_time), tostring(fnName)
 
  */
 
-export function recordStarted(work: Doc<"work">): string {
-  return JSON.stringify({
+export function recordEnqueued(
+  console: Logger,
+  data: {
+    workId: Id<"work">;
+    fnName: string;
+    runAt: number;
+  }
+) {
+  console.event("enqueued", {
+    ...data,
+    enqueuedAt: Date.now(),
+  });
+}
+
+export function recordStarted(
+  console: Logger,
+  work: Doc<"work">,
+  lagMs: number
+) {
+  console.event("started", {
     workId: work._id,
-    event: "started",
     fnName: work.fnName,
     enqueuedAt: work._creationTime,
     startedAt: Date.now(),
-    lagSinceEnqueued: Date.now() - work._creationTime,
+    startLag: lagMs,
   });
 }
 
 export function recordCompleted(
+  console: Logger,
   work: Doc<"work">,
-  status: "success" | "failed" | "canceled"
-): string {
-  return JSON.stringify({
+  status: "success" | "failed" | "canceled" | "retrying"
+) {
+  console.event("completed", {
     workId: work._id,
-    event: "completed",
     fnName: work.fnName,
     completedAt: Date.now(),
+    attempts: work.attempts,
     status,
-    lagSinceEnqueued: Date.now() - work._creationTime,
   });
 }
 
-export function recordReport(state: Doc<"internalState">): string {
+export function recordReport(console: Logger, state: Doc<"internalState">) {
   const { completed, succeeded, failed, retries, canceled } = state.report;
   const withoutRetries = completed - retries;
-  return JSON.stringify({
-    event: "report",
+  console.event("report", {
     completed,
     succeeded,
     failed,
@@ -65,7 +83,7 @@ export function recordReport(state: Doc<"internalState">): string {
  * Warning: this should not be used from a mutation, as it will cause conflicts.
  * Use this to debug or diagnose your queue length when it's backed up.
  */
-export const queueLength = internalQuery({
+export const queueLength = query({
   args: {},
   returns: v.number(),
   handler: async (ctx) => {
@@ -78,12 +96,14 @@ export const queueLength = internalQuery({
  * Warning: this should not be used from a mutation, as it will cause conflicts.
  * Use this while developing to see the state of the queue.
  */
-export const debugCounts = internalQuery({
+export const diagnostic = internalQuery({
   args: {},
   returns: v.any(),
   handler: async (ctx) => {
-    const inProgressWork =
-      (await ctx.db.query("internalState").unique())?.running.length ?? 0;
+    const global = await ctx.db.query("globals").unique();
+    const internalState = await ctx.db.query("internalState").unique();
+    const inProgressWork = internalState?.running.length ?? 0;
+    const maxParallelism = global?.maxParallelism ?? DEFAULT_MAX_PARALLELISM;
     /* eslint-disable @typescript-eslint/no-explicit-any */
     const pendingStart = await (ctx.db.query("pendingStart") as any).count();
     const pendingCompletion = await (
@@ -92,13 +112,16 @@ export const debugCounts = internalQuery({
     const pendingCancelation = await (
       ctx.db.query("pendingCancelation") as any
     ).count();
+    const runStatus = await ctx.db.query("runStatus").unique();
     /* eslint-enable @typescript-eslint/no-explicit-any */
     return {
-      pendingStart,
-      inProgressWork,
-      pendingCompletion,
-      pendingCancelation,
-      active: inProgressWork - pendingCompletion,
+      canceling: pendingCancelation,
+      waiting: pendingStart,
+      running: inProgressWork - pendingCompletion,
+      completing: pendingCompletion,
+      spareCapacity: maxParallelism - inProgressWork,
+      runStatus: runStatus?.state.kind,
+      generation: internalState?.generation,
     };
   },
 });

package/src/component/worker.ts

@@ -3,13 +3,12 @@
  * Should not touch any of loop's tables other than writing to `pendingCompletion`.
  * It is not responsible for handling retries.
  */
-import { FunctionHandle } from "convex/server";
+import type { FunctionHandle } from "convex/server";
 import { v } from "convex/values";
 import { internal } from "./_generated/api.js";
 import { internalAction, internalMutation } from "./_generated/server.js";
-import { kickMainLoop } from "./kick.js";
 import { createLogger, logLevel } from "./logging.js";
-import { nextSegment, runResult } from "./shared.js";
+import type { RunResult } from "./shared.js";
 
 export const runMutationWrapper = internalMutation({
   args: {
@@ -17,23 +16,25 @@ export const runMutationWrapper = internalMutation({
     fnHandle: v.string(),
     fnArgs: v.any(),
     logLevel,
+    attempt: v.number(),
   },
-  handler: async (ctx, { workId, fnHandle: handleStr, fnArgs, logLevel }) => {
-    const console = createLogger(logLevel);
-    const fnHandle = handleStr as FunctionHandle<"mutation">;
+  handler: async (ctx, { workId, attempt, ...args }) => {
+    const console = createLogger(args.logLevel);
+    const fnHandle = args.fnHandle as FunctionHandle<"mutation">;
     try {
-      const returnValue = await ctx.runMutation(fnHandle, fnArgs);
+      const returnValue = await ctx.runMutation(fnHandle, args.fnArgs);
       // NOTE: we could run the `saveResult` handler here, or call `ctx.runMutation`,
       // but we want the mutation to be a separate transaction to reduce the window for OCCs.
-      await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-        workId,
-        runResult: { kind: "success", returnValue },
+      await ctx.scheduler.runAfter(0, internal.complete.complete, {
+        jobs: [
+          { workId, runResult: { kind: "success", returnValue }, attempt },
+        ],
       });
     } catch (e: unknown) {
       console.error(e);
-      await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-        workId,
-        runResult: { kind: "failed", error: formatError(e) },
+      const runResult = { kind: "failed" as const, error: formatError(e) };
+      await ctx.scheduler.runAfter(0, internal.complete.complete, {
+        jobs: [{ workId, runResult, attempt }],
       });
     }
   },
@@ -52,43 +53,29 @@ export const runActionWrapper = internalAction({
     fnHandle: v.string(),
     fnArgs: v.any(),
     logLevel,
+    attempt: v.number(),
   },
-  handler: async (ctx, { workId, fnHandle: handleStr, fnArgs, logLevel }) => {
-    const console = createLogger(logLevel);
-    const fnHandle = handleStr as FunctionHandle<"action">;
+  handler: async (ctx, { workId, attempt, ...args }) => {
+    const console = createLogger(args.logLevel);
+    const fnHandle = args.fnHandle as FunctionHandle<"action">;
     try {
-      const returnValue = await ctx.runAction(fnHandle, fnArgs);
+      const returnValue = await ctx.runAction(fnHandle, args.fnArgs);
       // NOTE: we could run `ctx.runMutation`, but we want to guarantee execution,
       // and `ctx.scheduler.runAfter` won't OCC.
-      await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-        workId,
-        runResult: { kind: "success", returnValue },
+      const runResult: RunResult = { kind: "success", returnValue };
+      await ctx.scheduler.runAfter(0, internal.complete.complete, {
+        jobs: [{ workId, runResult, attempt }],
       });
     } catch (e: unknown) {
       console.error(e);
       // We let the main loop handle the retries.
-      await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-        workId,
-        runResult: { kind: "failed", error: formatError(e) },
+      const runResult: RunResult = { kind: "failed", error: formatError(e) };
+      await ctx.scheduler.runAfter(0, internal.complete.complete, {
+        jobs: [{ workId, runResult, attempt }],
       });
     }
   },
 });
 
-export const saveResult = internalMutation({
-  args: {
-    workId: v.id("work"),
-    runResult,
-  },
-  handler: async (ctx, { workId, runResult }) => {
-    await ctx.db.insert("pendingCompletion", {
-      runResult,
-      workId,
-      segment: nextSegment(),
-    });
-    await kickMainLoop(ctx, "saveResult");
-  },
-});
-
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const console = "THIS IS A REMINDER TO USE createLogger";