@convex-dev/workpool 0.4.3-alpha.0 → 0.4.3

package/src/component/recovery.test.ts

@@ -354,6 +354,119 @@ describe("recovery", () => {
       });
     });
 
+    it("should handle pending scheduled mutations", async () => {
+      // Create work and scheduled function
+      let workId: Id<"work">;
+      let scheduledId: Id<"_scheduled_functions">;
+
+      await t.run(async (ctx) => {
+        workId = await makeDummyWork(ctx, { fnType: "mutation" });
+        scheduledId = await makeDummyScheduledFunction(ctx, workId);
+      });
+
+      // Run recovery with mocked system.get
+      await t.run(async (ctx) => {
+        // Mock the system.get to return a pending state
+        ctx.db.system.get = patchedSystemGet(ctx.db, {
+          [scheduledId]: {
+            _id: scheduledId,
+            _creationTime: Date.now(),
+            name: "internal/worker.runMutationWrapper",
+            args: [
+              {
+                workId,
+                fnHandle: "test_handle",
+                fnArgs: {},
+                logLevel: "WARN",
+                attempt: 0,
+              },
+            ],
+            scheduledTime: Date.now(),
+            state: {
+              kind: "pending",
+            },
+          },
+        });
+
+        await recoveryHandler(ctx, {
+          jobs: [
+            {
+              scheduledId,
+              workId,
+              attempt: 0,
+              started: Date.now(),
+            },
+          ],
+        });
+      });
+
+      // Verify pendingCompletion was created with stuckInScheduler
+      await t.run(async (ctx) => {
+        const pendingCompletions = await ctx.db
+          .query("pendingCompletion")
+          .withIndex("workId", (q) => q.eq("workId", workId))
+          .collect();
+        expect(pendingCompletions).toHaveLength(1);
+        expect(pendingCompletions[0].runResult.kind).toBe("stuckInScheduler");
+      });
+    });
+
+    it("should not process pending scheduled actions", async () => {
+      // Create work and scheduled function
+      let workId: Id<"work">;
+      let scheduledId: Id<"_scheduled_functions">;
+
+      await t.run(async (ctx) => {
+        workId = await makeDummyWork(ctx);
+        scheduledId = await makeDummyScheduledFunction(ctx, workId);
+      });
+
+      // Run recovery with mocked system.get
+      await t.run(async (ctx) => {
+        // Mock the system.get to return a pending state
+        ctx.db.system.get = patchedSystemGet(ctx.db, {
+          [scheduledId]: {
+            _id: scheduledId,
+            _creationTime: Date.now(),
+            name: "internal/worker.runActionWrapper",
+            args: [
+              {
+                workId,
+                fnHandle: "test_handle",
+                fnArgs: {},
+                logLevel: "WARN",
+                attempt: 0,
+              },
+            ],
+            scheduledTime: Date.now(),
+            state: {
+              kind: "pending",
+            },
+          },
+        });
+
+        await recoveryHandler(ctx, {
+          jobs: [
+            {
+              scheduledId,
+              workId,
+              attempt: 0,
+              started: Date.now(),
+            },
+          ],
+        });
+      });
+
+      // Verify no pendingCompletion was created
+      await t.run(async (ctx) => {
+        const pendingCompletions = await ctx.db
+          .query("pendingCompletion")
+          .withIndex("workId", (q) => q.eq("workId", workId))
+          .collect();
+        expect(pendingCompletions).toHaveLength(0);
+      });
+    });
+
     it("should handle multiple jobs in a single call", async () => {
       // Create multiple work items and scheduled functions
       let workId1: Id<"work">;
@@ -472,7 +585,7 @@ describe("recovery", () => {
 
       // Run recovery with mocked system.get
       await t.run(async (ctx) => {
-        // Mock the system.get to return a pending state
+        // Mock the system.get to return a inProgress state
         ctx.db.system.get = patchedSystemGet(ctx.db, {
           [scheduledId]: {
             _id: scheduledId,
@@ -489,7 +602,7 @@ describe("recovery", () => {
             ],
             scheduledTime: Date.now(),
             state: {
-              kind: "pending",
+              kind: "inProgress",
             },
           },
         });

package/src/component/recovery.ts

@@ -96,6 +96,23 @@ export async function recoveryHandler(
         });
         break;
       }
+      case "pending": {
+        if (work.fnType === "action") {
+          // We do not cancel and re-enqueue actions. If a scheduled action is still
+          // pending, the scheduler is likely backlogged.
+          break;
+        }
+        // It looks like the function has been retried by the scheduler several times.
+        // The scheduler backoff is too long, so cancel and re-enqueue the job to
+        // free up space for more work.
+        await ctx.scheduler.cancel(scheduled._id);
+        completionJobs.push({
+          workId: job.workId,
+          runResult: { kind: "stuckInScheduler" },
+          attempt: job.attempt,
+        });
+        break;
+      }
     }
   }
   if (completionJobs.length > 0) {

package/src/component/schema.ts

@@ -1,5 +1,5 @@
 import { defineSchema, defineTable } from "convex/server";
-import { v } from "convex/values";
+import { v, type Infer } from "convex/values";
 import {
   fnType,
   vConfig,
@@ -11,6 +11,14 @@ import {
 // Represents a slice of time to process work.
 const segment = v.int64();
 
+export const vResultInternal = v.union(
+  vResult,
+  v.object({
+    kind: v.literal("stuckInScheduler"),
+  }),
+);
+export type RunResultInternal = Infer<typeof vResultInternal>;
+
 export default defineSchema({
   // Written from kickLoop, read everywhere.
   globals: defineTable(vConfig),
@@ -30,6 +38,7 @@ export default defineSchema({
       failed: v.number(), // failed after all retries
       retries: v.number(), // failure that turned into a retry
       canceled: v.number(), // cancelations processed
+      conflicted: v.optional(v.number()), // mutations conflicted in the scheduler
       lastReportTs: v.number(),
     }),
     running: v.array(
@@ -83,7 +92,7 @@ export default defineSchema({
   // Written by complete, read & deleted by `main`.
   pendingCompletion: defineTable({
     segment,
-    runResult: vResult,
+    runResult: vResultInternal,
     workId: v.id("work"),
     retry: v.boolean(),
   })

package/src/component/stats.test.ts

@@ -64,6 +64,7 @@ describe("stats", () => {
             failed: 0,
             retries: 0,
             canceled: 0,
+            conflicted: 0,
             lastReportTs: 0,
           },
           running: [],
@@ -109,6 +110,7 @@ describe("stats", () => {
             failed: 2,
             retries: 2,
             canceled: 0,
+            conflicted: 0,
             lastReportTs: 0,
           },
           running: [],
@@ -160,6 +162,7 @@ describe("stats", () => {
         failed: 2,
         retries: 2,
         canceled: 0,
+        conflicted: 0,
         failureRate: 0.4, // (failed + retries) / completed = (2 + 2) / 10 = 0.4
         permanentFailureRate: 0.25, // failed / (completed - retries) = 2 / (10 - 2) = 2/8
         lastReportTs: expect.any(Number),
@@ -183,6 +186,7 @@ describe("stats", () => {
             failed: 1,
             retries: 1,
             canceled: 0,
+            conflicted: 0,
             lastReportTs: 0,
           },
           running: [],
@@ -267,6 +271,7 @@ describe("stats", () => {
             failed: 1,
             retries: 1,
             canceled: 0,
+            conflicted: 0,
             lastReportTs: 0,
           },
           running: [],

package/src/component/stats.ts

@@ -53,7 +53,12 @@ export function recordStarted(
 export function recordCompleted(
   console: Logger,
   work: Doc<"work">,
-  status: "success" | "failed" | "canceled" | "retrying",
+  status:
+    | "success"
+    | "failed"
+    | "canceled"
+    | "retrying"
+    | "retrying conflicted",
   onCompleteScheduledFunctionId: Id<"_scheduled_functions"> | undefined,
 ) {
   console.event("completed", {

package/src/component/worker.ts

@@ -91,6 +91,19 @@ export const runActionWrapper = internalAction({
       // NOTE: we could run `ctx.runMutation`, but we want to guarantee execution,
       // and `ctx.scheduler.runAfter` won't OCC.
       const runResult: RunResult = { kind: "success", returnValue };
+      try {
+        // Attempt to run complete inline and onComplete inline
+        await ctx.runMutation(internal.complete.complete, {
+          jobs: [{ workId, runResult, attempt, runOnCompleteInline: true }],
+        });
+        console.info("[runActionWrapper] onComplete succeeded");
+        return;
+      } catch (e) {
+        console.error(
+          `[runActionWrapper] caught error while attempting to run complete inline, scheduling instead: ${e}`,
+        );
+        // Fall through and schedule complete instead (without running onComplete inline)
+      }
       await ctx.scheduler.runAfter(0, internal.complete.complete, {
         jobs: [{ workId, runResult, attempt }],
       });