@convex-dev/workpool 0.2.11 → 0.2.13

package/src/component/loop.test.ts

@@ -17,8 +17,10 @@ import {
   DEFAULT_MAX_PARALLELISM,
   getCurrentSegment,
   getNextSegment,
+  HOUR,
   toSegment,
 } from "./shared";
+import { DEFAULT_LOG_LEVEL } from "./logging";
 
 const modules = import.meta.glob("./**/*.ts");
 
@@ -32,9 +34,17 @@ describe("loop", () => {
 
   async function setMaxParallelism(maxParallelism: number) {
     await t.run(async (ctx) => {
-      await ctx.db.patch((await ctx.db.query("globals").unique())!._id, {
-        maxParallelism,
-      });
+      const globals = await ctx.db.query("globals").unique();
+      if (!globals) {
+        await ctx.db.insert("globals", {
+          logLevel: DEFAULT_LOG_LEVEL,
+          maxParallelism,
+        });
+      } else {
+        await ctx.db.patch(globals._id, {
+          maxParallelism,
+        });
+      }
     });
   }
 
@@ -574,20 +584,7 @@ describe("loop", () => {
     it("should handle generation mismatch", async () => {
       // Setup state with different generation
       await t.run(async (ctx) => {
-        await ctx.db.insert("internalState", {
-          generation: 2n,
-          segmentCursors: { incoming: 0n, completion: 0n, cancelation: 0n },
-          lastRecovery: 0n,
-          report: {
-            completed: 0,
-            succeeded: 0,
-            failed: 0,
-            retries: 0,
-            canceled: 0,
-            lastReportTs: Date.now(),
-          },
-          running: [],
-        });
+        await insertInternalState(ctx, { generation: 2n });
       });
 
       // Call main with mismatched generation
@@ -606,25 +603,8 @@ describe("loop", () => {
         const scheduledId = await makeDummyScheduledFunction(ctx, workId);
 
         // Create internal state
-        await ctx.db.insert("internalState", {
-          generation: 1n,
-          segmentCursors: { incoming: 0n, completion: 0n, cancelation: 0n },
-          lastRecovery: 0n,
-          report: {
-            completed: 0,
-            succeeded: 0,
-            failed: 0,
-            retries: 0,
-            canceled: 0,
-            lastReportTs: Date.now(),
-          },
-          running: [
-            {
-              workId,
-              scheduledId,
-              started: 900000,
-            },
-          ],
+        await insertInternalState(ctx, {
+          running: [{ workId, scheduledId, started: 900000 }],
         });
 
         // Create pending completion
@@ -672,18 +652,7 @@ describe("loop", () => {
         const scheduledId = await makeDummyScheduledFunction(ctx, workId);
 
         // Create internal state
-        await ctx.db.insert("internalState", {
-          generation: 1n,
-          segmentCursors: { incoming: 0n, completion: 0n, cancelation: 0n },
-          lastRecovery: 0n,
-          report: {
-            completed: 0,
-            succeeded: 0,
-            failed: 0,
-            retries: 0,
-            canceled: 0,
-            lastReportTs: Date.now(),
-          },
+        await insertInternalState(ctx, {
           running: [
             {
               workId,
@@ -743,25 +712,8 @@ describe("loop", () => {
         );
 
         // Create internal state
-        await ctx.db.insert("internalState", {
-          generation: 1n,
-          segmentCursors: { incoming: 0n, completion: 0n, cancelation: 0n },
-          lastRecovery: 0n,
-          report: {
-            completed: 0,
-            succeeded: 0,
-            failed: 0,
-            retries: 0,
-            canceled: 0,
-            lastReportTs: Date.now(),
-          },
-          running: [
-            {
-              workId: runningWorkId,
-              scheduledId,
-              started: 900000,
-            },
-          ],
+        await insertInternalState(ctx, {
+          running: [{ workId: runningWorkId, scheduledId, started: 900000 }],
         });
 
         // Create work
@@ -816,20 +768,7 @@ describe("loop", () => {
       // Setup state with pending start items
       const workId = await t.run<Id<"work">>(async (ctx) => {
         // Create internal state
-        await ctx.db.insert("internalState", {
-          generation: 1n,
-          segmentCursors: { incoming: 0n, completion: 0n, cancelation: 0n },
-          lastRecovery: 0n,
-          report: {
-            completed: 0,
-            succeeded: 0,
-            failed: 0,
-            retries: 0,
-            canceled: 0,
-            lastReportTs: Date.now(),
-          },
-          running: [],
-        });
+        await insertInternalState(ctx);
 
         // Create work
         const workId = await makeDummyWork(ctx);
@@ -872,25 +811,9 @@ describe("loop", () => {
         const scheduledId = await makeDummyScheduledFunction(ctx, workId);
 
         // Create internal state with old job
-        await ctx.db.insert("internalState", {
-          generation: 1n,
-          segmentCursors: { incoming: 0n, completion: 0n, cancelation: 0n },
+        await insertInternalState(ctx, {
           lastRecovery: 0n,
-          report: {
-            completed: 0,
-            succeeded: 0,
-            failed: 0,
-            retries: 0,
-            canceled: 0,
-            lastReportTs: Date.now(),
-          },
-          running: [
-            {
-              workId,
-              scheduledId,
-              started: oldTime,
-            },
-          ],
+          running: [{ workId, scheduledId, started: oldTime }],
         });
       });
 
@@ -918,20 +841,7 @@ describe("loop", () => {
     it("should handle generation mismatch", async () => {
       // Setup state with different generation
       await t.run(async (ctx) => {
-        await ctx.db.insert("internalState", {
-          generation: 2n,
-          segmentCursors: { incoming: 0n, completion: 0n, cancelation: 0n },
-          lastRecovery: 0n,
-          report: {
-            completed: 0,
-            succeeded: 0,
-            failed: 0,
-            retries: 0,
-            canceled: 0,
-            lastReportTs: Date.now(),
-          },
-          running: [],
-        });
+        await insertInternalState(ctx, { generation: 2n });
       });
 
       // Call updateRunStatus with mismatched generation
@@ -950,20 +860,7 @@ describe("loop", () => {
         const workId = await makeDummyWork(ctx);
 
         // Create internal state
-        await ctx.db.insert("internalState", {
-          generation: 1n,
-          segmentCursors: { incoming: 0n, completion: 0n, cancelation: 0n },
-          lastRecovery: 0n,
-          report: {
-            completed: 0,
-            succeeded: 0,
-            failed: 0,
-            retries: 0,
-            canceled: 0,
-            lastReportTs: Date.now(),
-          },
-          running: [],
-        });
+        await insertInternalState(ctx, {});
 
         // Create run status
         await ctx.db.insert("runStatus", {
@@ -998,20 +895,7 @@ describe("loop", () => {
       // Setup state with no work
       await t.run(async (ctx) => {
         // Create internal state with no running jobs
-        await ctx.db.insert("internalState", {
-          generation: 1n,
-          segmentCursors: { incoming: 0n, completion: 0n, cancelation: 0n },
-          lastRecovery: 0n,
-          report: {
-            completed: 0,
-            succeeded: 0,
-            failed: 0,
-            retries: 0,
-            canceled: 0,
-            lastReportTs: Date.now(),
-          },
-          running: [],
-        });
+        await insertInternalState(ctx, {});
 
         // Create run status in running state
         await ctx.db.insert("runStatus", {
@@ -1056,18 +940,7 @@ describe("loop", () => {
         );
 
         // Create internal state with max running jobs
-        await ctx.db.insert("internalState", {
-          generation: 1n,
-          segmentCursors: { incoming: 0n, completion: 0n, cancelation: 0n },
-          lastRecovery: now,
-          report: {
-            completed: 0,
-            succeeded: 0,
-            failed: 0,
-            retries: 0,
-            canceled: 0,
-            lastReportTs: Date.now(),
-          },
+        await insertInternalState(ctx, {
           running: runningJobs,
         });
 
@@ -1100,6 +973,59 @@ describe("loop", () => {
         expect(runStatus!.state.saturated).toBe(true);
       });
     });
+
+    it("should reset cursors correctly when there's old work detected", async () => {
+      // Setup state with old work
+      const now = getCurrentSegment();
+      await t.run(async (ctx) => {
+        // Create internal state with old work
+        await insertInternalState(ctx, {
+          segmentCursors: {
+            incoming: now - 1n,
+            completion: now - 1n,
+            cancelation: now - 1n,
+          },
+        });
+      });
+
+      // Insert very old work
+      await t.run(async (ctx) => {
+        const workId = await makeDummyWork(ctx);
+        await ctx.db.insert("pendingStart", {
+          workId,
+          segment: 0n,
+        });
+      });
+
+      // Call updateRunStatus
+      await t.mutation(internal.loop.updateRunStatus, {
+        generation: 1n,
+        segment: now,
+      });
+
+      // Verify cursors were reset
+      await t.run(async (ctx) => {
+        const state = await ctx.db.query("internalState").unique();
+        expect(state).toBeDefined();
+        expect(state!.segmentCursors.incoming).toBe(0n);
+      });
+
+      // Set maxParallelism to 0 so it doesn't schedule anything / make progress
+      await setMaxParallelism(0);
+
+      // Run main
+      await t.mutation(internal.loop.main, {
+        generation: 1n,
+        segment: now,
+      });
+
+      // Verify start cursor weren't updated
+      await t.run(async (ctx) => {
+        const state = await ctx.db.query("internalState").unique();
+        expect(state).toBeDefined();
+        expect(state!.segmentCursors.incoming).toBe(0n);
+      });
+    });
   });
 
   describe("complete function", () => {

package/src/component/loop.ts

@@ -28,7 +28,7 @@ const SECOND = 1000;
 const MINUTE = 60 * SECOND;
 const RECOVERY_THRESHOLD_MS = 5 * MINUTE; // attempt to recover jobs this old.
 export const RECOVERY_PERIOD_SEGMENTS = toSegment(1 * MINUTE); // how often to check.
-const CURSOR_BUFFER_SEGMENTS = toSegment(2 * SECOND); // buffer for cursor updates.
+const CURSOR_BUFFER_SEGMENTS = toSegment(30 * SECOND); // buffer for cursor updates.
 export const INITIAL_STATE: WithoutSystemFields<Doc<"internalState">> = {
   generation: 0n,
   segmentCursors: { incoming: 0n, completion: 0n, cancelation: 0n },
@@ -521,7 +521,14 @@ async function handleStart(
           .take(toSchedule)
       : [];
 
-  state.segmentCursors.incoming = pending.at(-1)?.segment ?? segment;
+  if (pending) {
+    if (pending.length > 0) {
+      state.segmentCursors.incoming = pending.at(-1)!.segment;
+    } else if (toSchedule > 0) {
+      // We have no more pending work, update to now
+      state.segmentCursors.incoming = segment;
+    }
+  }
   console.debug(`[main] scheduling ${pending.length} pending work`);
   // Start new work.
   state.running.push(