@convex-dev/workpool 0.2.0-alpha.0 → 0.2.0

package/src/component/complete.test.ts

@@ -30,7 +30,7 @@ describe("complete", () => {
     await t.run(async (ctx) => {
       await ctx.db.insert("globals", {
         maxParallelism: 10,
-        logLevel: "INFO",
+        logLevel: "WARN",
       });
     });
   });
@@ -50,7 +50,7 @@ describe("complete", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
@@ -95,7 +95,7 @@ describe("complete", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
         retryBehavior: {
           maxAttempts: 3,
@@ -146,7 +146,7 @@ describe("complete", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
         retryBehavior: {
           maxAttempts: 2, // Only 1 retry allowed
@@ -204,7 +204,7 @@ describe("complete", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
@@ -250,7 +250,7 @@ describe("complete", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
         onComplete: {
           fnHandle: "testOnComplete",
@@ -298,7 +298,7 @@ describe("complete", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
@@ -310,7 +310,7 @@ describe("complete", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
         retryBehavior: {
           maxAttempts: 3,
@@ -366,7 +366,7 @@ describe("complete", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
@@ -418,7 +418,7 @@ describe("complete", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
         retryBehavior: {
           maxAttempts: 3,

package/src/component/complete.ts

@@ -72,7 +72,7 @@ export async function completeHandler(
             // TODO: store failures in a table for later debugging
           }
         }
-        console.info(recordCompleted(work, job.runResult.kind));
+        recordCompleted(console, work, job.runResult.kind);
         // This is the terminating state for work.
         await ctx.db.delete(job.workId);
       }

package/src/component/kick.test.ts

@@ -8,13 +8,14 @@ import {
   test,
   vi,
 } from "vitest";
-import schema from "./schema.js";
-import { modules } from "./setup.test.js";
-import { DEFAULT_MAX_PARALLELISM, kickMainLoop } from "./kick.js";
-import { DEFAULT_LOG_LEVEL } from "./logging.js";
 import { internal } from "./_generated/api";
-import { toSegment, fromSegment, nextSegment } from "./shared";
 import { Id } from "./_generated/dataModel.js";
+import { kickMainLoop } from "./kick.js";
+import { DEFAULT_LOG_LEVEL } from "./logging.js";
+import schema from "./schema.js";
+import { modules } from "./setup.test.js";
+import { fromSegment, nextSegment, toSegment } from "./shared";
+import { DEFAULT_MAX_PARALLELISM } from "./shared.js";
 
 describe("kickMainLoop", () => {
   beforeEach(() => {
@@ -53,13 +54,13 @@ describe("kickMainLoop", () => {
       expect(globals.logLevel).toBe(DEFAULT_LOG_LEVEL);
       await kickMainLoop(ctx, "enqueue", {
         maxParallelism: DEFAULT_MAX_PARALLELISM + 1,
-        logLevel: "DEBUG",
+        logLevel: "ERROR",
       });
       const after = await ctx.db.query("globals").unique();
       expect(after).not.toBeNull();
       assert(after);
       expect(after.maxParallelism).toBe(DEFAULT_MAX_PARALLELISM + 1);
-      expect(after.logLevel).toBe("DEBUG");
+      expect(after.logLevel).toBe("ERROR");
     });
   });
 
@@ -242,7 +243,7 @@ describe("kickMainLoop", () => {
       // Initial kick with custom config
       await kickMainLoop(ctx, "enqueue", {
         maxParallelism: 5,
-        logLevel: "DEBUG",
+        logLevel: "ERROR",
       });
 
       // Kick from different sources
@@ -254,7 +255,7 @@ describe("kickMainLoop", () => {
       expect(globals).not.toBeNull();
       assert(globals);
       expect(globals.maxParallelism).toBe(5);
-      expect(globals.logLevel).toBe("DEBUG");
+      expect(globals.logLevel).toBe("ERROR");
     });
   });
 

package/src/component/kick.ts

@@ -2,9 +2,14 @@ import { internal } from "./_generated/api.js";
 import { internalMutation, MutationCtx } from "./_generated/server.js";
 import { createLogger, DEFAULT_LOG_LEVEL } from "./logging.js";
 import { INITIAL_STATE } from "./loop.js";
-import { Config, nextSegment } from "./shared.js";
+import {
+  boundScheduledTime,
+  Config,
+  DEFAULT_MAX_PARALLELISM,
+  fromSegment,
+  nextSegment,
+} from "./shared.js";
 
-export const DEFAULT_MAX_PARALLELISM = 10;
 /**
  * Called from outside the loop:
  */
@@ -51,12 +56,12 @@ export async function kickMainLoop(
         `[${source}] main is marked as scheduled, but it's status is ${scheduled?.state.kind}`
       );
     }
+  } else if (runStatus.state.kind === "idle") {
+    console.debug(`[${source}] main was idle, so run it now`);
   }
-  console.debug(
-    `[${source}] main was scheduled later, so reschedule it to run now`
-  );
   await ctx.db.patch(runStatus._id, { state: { kind: "running" } });
-  await ctx.scheduler.runAfter(0, internal.loop.main, {
+  const scheduledTime = boundScheduledTime(fromSegment(segment), console);
+  await ctx.scheduler.runAt(scheduledTime, internal.loop.main, {
     generation: runStatus.state.generation,
     segment,
   });

package/src/component/lib.test.ts

@@ -44,7 +44,7 @@ describe("lib", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
@@ -63,7 +63,7 @@ describe("lib", () => {
           runAt: Date.now(),
           config: {
             maxParallelism: 101, // More than MAX_POSSIBLE_PARALLELISM
-            logLevel: "INFO",
+            logLevel: "WARN",
           },
         })
       ).rejects.toThrow("maxParallelism must be <= 100");
@@ -79,7 +79,7 @@ describe("lib", () => {
           runAt: Date.now(),
           config: {
             maxParallelism: 0, // Less than minimum
-            logLevel: "INFO",
+            logLevel: "WARN",
           },
         })
       ).rejects.toThrow("maxParallelism must be >= 1");
@@ -96,13 +96,13 @@ describe("lib", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
       await t.mutation(api.lib.cancel, {
         id,
-        logLevel: "INFO",
+        logLevel: "WARN",
       });
 
       // Verify a pending cancelation was created
@@ -124,20 +124,20 @@ describe("lib", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
       // Cancel the first time
       await t.mutation(api.lib.cancel, {
         id,
-        logLevel: "INFO",
+        logLevel: "WARN",
       });
 
       // Cancel the second time
       await t.mutation(api.lib.cancel, {
         id,
-        logLevel: "INFO",
+        logLevel: "WARN",
       });
 
       // Verify only one pending cancelation was created
@@ -159,7 +159,7 @@ describe("lib", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
@@ -171,7 +171,7 @@ describe("lib", () => {
       // Try to cancel the deleted work
       await t.mutation(api.lib.cancel, {
         id,
-        logLevel: "INFO",
+        logLevel: "WARN",
       });
 
       // Verify no pending cancelation was created
@@ -196,14 +196,14 @@ describe("lib", () => {
           runAt: Date.now() + 5 * 60 * 1000,
           config: {
             maxParallelism: 10,
-            logLevel: "INFO",
+            logLevel: "WARN",
           },
         });
         ids.push(id);
       }
 
       await t.mutation(api.lib.cancelAll, {
-        logLevel: "INFO",
+        logLevel: "WARN",
         before: Date.now() + 1000,
       });
 
@@ -231,13 +231,13 @@ describe("lib", () => {
           runAt: Date.now(),
           config: {
             maxParallelism: 10,
-            logLevel: "INFO",
+            logLevel: "WARN",
           },
         });
       }
 
       await t.mutation(api.lib.cancelAll, {
-        logLevel: "INFO",
+        logLevel: "WARN",
         before: Date.now() + 1000,
       });
 
@@ -277,7 +277,7 @@ describe("lib", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
       await t.run(async (ctx) => {
@@ -297,7 +297,7 @@ describe("lib", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
@@ -323,7 +323,7 @@ describe("lib", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
@@ -353,7 +353,7 @@ describe("lib", () => {
         },
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
@@ -386,7 +386,7 @@ describe("lib", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 
@@ -419,7 +419,7 @@ describe("lib", () => {
         runAt: Date.now(),
         config: {
           maxParallelism: 10,
-          logLevel: "INFO",
+          logLevel: "WARN",
         },
       });
 

package/src/component/lib.ts

@@ -8,12 +8,14 @@ import {
   status as statusValidator,
   toSegment,
   boundScheduledTime,
+  max,
 } from "./shared.js";
 import { LogLevel, logLevel } from "./logging.js";
 import { kickMainLoop } from "./kick.js";
 import { api } from "./_generated/api.js";
 import { createLogger } from "./logging.js";
 import { Id } from "./_generated/dataModel.js";
+import { recordEnqueued } from "./stats.js";
 
 const MAX_POSSIBLE_PARALLELISM = 100;
 
@@ -45,10 +47,10 @@ export const enqueue = mutation({
     });
     await ctx.db.insert("pendingStart", {
       workId,
-      segment: toSegment(runAt),
+      segment: max(toSegment(runAt), nextSegment()),
     });
     await kickMainLoop(ctx, "enqueue", config);
-    // TODO: stats event
+    recordEnqueued(console, { workId, fnName: workArgs.fnName, runAt });
     return workId;
   },
 });

package/src/component/logging.ts

@@ -53,9 +53,8 @@ export function createLogger(level?: LogLevel): Logger {
     event: (event: string, payload: Record<string, unknown>) => {
       if (levelIndex <= 1) {
         const fullPayload = {
-          system: "idempotent-workpool-component",
           event,
-          payload,
+          ...payload,
         };
         console.info(JSON.stringify(fullPayload));
       }