@convex-dev/workpool 0.2.20-alpha.0 → 0.3.1-alpha.0

package/src/component/_generated/{server.d.ts → server.ts}

@@ -8,9 +8,8 @@
  * @module
  */
 
-import {
+import type {
   ActionBuilder,
-  AnyComponents,
   HttpActionBuilder,
   MutationBuilder,
   QueryBuilder,
@@ -19,15 +18,18 @@ import {
   GenericQueryCtx,
   GenericDatabaseReader,
   GenericDatabaseWriter,
-  FunctionReference,
+} from "convex/server";
+import {
+  actionGeneric,
+  httpActionGeneric,
+  queryGeneric,
+  mutationGeneric,
+  internalActionGeneric,
+  internalMutationGeneric,
+  internalQueryGeneric,
 } from "convex/server";
 import type { DataModel } from "./dataModel.js";
 
-type GenericCtx =
-  | GenericActionCtx<DataModel>
-  | GenericMutationCtx<DataModel>
-  | GenericQueryCtx<DataModel>;
-
 /**
  * Define a query in this Convex app's public API.
  *
@@ -36,7 +38,7 @@ type GenericCtx =
  * @param func - The query function. It receives a {@link QueryCtx} as its first argument.
  * @returns The wrapped query. Include this as an `export` to name it and make it accessible.
  */
-export declare const query: QueryBuilder<DataModel, "public">;
+export const query: QueryBuilder<DataModel, "public"> = queryGeneric;
 
 /**
  * Define a query that is only accessible from other Convex functions (but not from the client).
@@ -46,7 +48,8 @@ export declare const query: QueryBuilder<DataModel, "public">;
  * @param func - The query function. It receives a {@link QueryCtx} as its first argument.
  * @returns The wrapped query. Include this as an `export` to name it and make it accessible.
  */
-export declare const internalQuery: QueryBuilder<DataModel, "internal">;
+export const internalQuery: QueryBuilder<DataModel, "internal"> =
+  internalQueryGeneric;
 
 /**
  * Define a mutation in this Convex app's public API.
@@ -56,7 +59,7 @@ export declare const internalQuery: QueryBuilder<DataModel, "internal">;
  * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument.
  * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible.
  */
-export declare const mutation: MutationBuilder<DataModel, "public">;
+export const mutation: MutationBuilder<DataModel, "public"> = mutationGeneric;
 
 /**
  * Define a mutation that is only accessible from other Convex functions (but not from the client).
@@ -66,7 +69,8 @@ export declare const mutation: MutationBuilder<DataModel, "public">;
  * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument.
  * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible.
  */
-export declare const internalMutation: MutationBuilder<DataModel, "internal">;
+export const internalMutation: MutationBuilder<DataModel, "internal"> =
+  internalMutationGeneric;
 
 /**
  * Define an action in this Convex app's public API.
@@ -79,7 +83,7 @@ export declare const internalMutation: MutationBuilder<DataModel, "internal">;
  * @param func - The action. It receives an {@link ActionCtx} as its first argument.
  * @returns The wrapped action. Include this as an `export` to name it and make it accessible.
  */
-export declare const action: ActionBuilder<DataModel, "public">;
+export const action: ActionBuilder<DataModel, "public"> = actionGeneric;
 
 /**
  * Define an action that is only accessible from other Convex functions (but not from the client).
@@ -87,19 +91,26 @@ export declare const action: ActionBuilder<DataModel, "public">;
  * @param func - The function. It receives an {@link ActionCtx} as its first argument.
  * @returns The wrapped function. Include this as an `export` to name it and make it accessible.
  */
-export declare const internalAction: ActionBuilder<DataModel, "internal">;
+export const internalAction: ActionBuilder<DataModel, "internal"> =
+  internalActionGeneric;
 
 /**
  * Define an HTTP action.
  *
- * This function will be used to respond to HTTP requests received by a Convex
- * deployment if the requests matches the path and method where this action
- * is routed. Be sure to route your action in `convex/http.js`.
+ * The wrapped function will be used to respond to HTTP requests received
+ * by a Convex deployment if the requests matches the path and method where
+ * this action is routed. Be sure to route your httpAction in `convex/http.js`.
  *
- * @param func - The function. It receives an {@link ActionCtx} as its first argument.
+ * @param func - The function. It receives an {@link ActionCtx} as its first argument
+ * and a Fetch API `Request` object as its second.
  * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up.
  */
-export declare const httpAction: HttpActionBuilder;
+export const httpAction: HttpActionBuilder = httpActionGeneric;
+
+type GenericCtx =
+  | GenericActionCtx<DataModel>
+  | GenericMutationCtx<DataModel>
+  | GenericQueryCtx<DataModel>;
 
 /**
  * A set of services for use within Convex query functions.
@@ -107,8 +118,7 @@ export declare const httpAction: HttpActionBuilder;
  * The query context is passed as the first argument to any Convex query
  * function run on the server.
  *
- * This differs from the {@link MutationCtx} because all of the services are
- * read-only.
+ * If you're using code generation, use the `QueryCtx` type in `convex/_generated/server.d.ts` instead.
  */
 export type QueryCtx = GenericQueryCtx<DataModel>;
 
@@ -117,6 +127,8 @@ export type QueryCtx = GenericQueryCtx<DataModel>;
  *
  * The mutation context is passed as the first argument to any Convex mutation
  * function run on the server.
+ *
+ * If you're using code generation, use the `MutationCtx` type in `convex/_generated/server.d.ts` instead.
  */
 export type MutationCtx = GenericMutationCtx<DataModel>;
 

package/src/component/complete.test.ts

@@ -283,7 +283,7 @@ describe("complete", () => {
             workId,
             context: { someContext: "value" },
             result: { kind: "success", returnValue: "test result" },
-          })
+          }),
         );
       });
     });

package/src/component/complete.ts

@@ -19,12 +19,12 @@ export const completeArgs = v.object({
       runResult: vResultValidator,
       workId: v.id("work"),
       attempt: v.number(),
-    })
+    }),
   ),
 });
 export async function completeHandler(
   ctx: MutationCtx,
-  args: Infer<typeof completeArgs>
+  args: Infer<typeof completeArgs>,
 ) {
   const globals = await ctx.db.query("globals").unique();
   const console = createLogger(globals?.logLevel);
@@ -76,7 +76,7 @@ export async function completeHandler(
           } catch (e) {
             console.error(
               `[complete] error running onComplete for ${job.workId}`,
-              e
+              e,
             );
             // TODO: store failures in a table for later debugging
           }
@@ -92,7 +92,7 @@ export async function completeHandler(
           retry,
         });
       }
-    })
+    }),
   );
   if (pendingCompletions.length > 0) {
     const segment = await kickMainLoop(ctx, "complete");
@@ -101,8 +101,8 @@ export async function completeHandler(
         ctx.db.insert("pendingCompletion", {
           ...completion,
           segment,
-        })
-      )
+        }),
+      ),
     );
   }
 }

package/src/component/danger.ts

@@ -29,7 +29,7 @@ export const clearPending = internalMutation({
         if (work) {
           await ctx.db.delete(work._id);
         }
-      })
+      }),
     );
     if (!entries.isDone) {
       await ctx.scheduler.runAfter(0, internal.danger.clearPending, {
@@ -87,10 +87,10 @@ export const clearOldWork = internalMutation({
           })
             .filter(([_, v]) => v !== null)
             .map(([name]) => name)
-            .join(", ")})`
+            .join(", ")})`,
         );
         await ctx.db.delete(entry._id);
-      })
+      }),
     );
     if (!entries.isDone) {
       await ctx.scheduler.runAfter(0, internal.danger.clearOldWork, {

package/src/component/kick.test.ts

@@ -107,7 +107,7 @@ describe("kickMainLoop", () => {
         {
           generation: 0n,
           segment: futureSegment,
-        }
+        },
       );
       await ctx.db.patch(runStatus._id, {
         state: {
@@ -149,7 +149,7 @@ describe("kickMainLoop", () => {
         {
           generation: 0n,
           segment: nearFutureSegment,
-        }
+        },
       );
       await ctx.db.patch(runStatus._id, {
         state: {
@@ -221,8 +221,8 @@ describe("kickMainLoop", () => {
         t.run(async (ctx) => {
           const segment = await kickMainLoop(ctx, "enqueue");
           return segment;
-        })
-      )
+        }),
+      ),
     );
     expect(segments.filter((s) => s === getCurrentSegment())).toHaveLength(1);
 
@@ -278,7 +278,7 @@ describe("kickMainLoop", () => {
       const scheduledId = await ctx.scheduler.runAfter(
         10_000,
         internal.loop.main,
-        { generation: 0n, segment }
+        { generation: 0n, segment },
       );
       await ctx.db.patch(runStatus._id, {
         state: {

package/src/component/kick.ts

@@ -20,7 +20,7 @@ import {
 export async function kickMainLoop(
   ctx: MutationCtx,
   source: "enqueue" | "cancel" | "complete" | "kick",
-  config?: Partial<Config>
+  config?: Partial<Config>,
 ): Promise<bigint> {
   const globals = await getOrUpdateGlobals(ctx, config);
   const console = createLogger(globals.logLevel);
@@ -30,7 +30,7 @@ export async function kickMainLoop(
   // Only kick to run now if we're scheduled or idle.
   if (runStatus.state.kind === "running") {
     console.debug(
-      `[${source}] main is actively running, so we don't need to kick it`
+      `[${source}] main is actively running, so we don't need to kick it`,
     );
     return next;
   }
@@ -38,25 +38,25 @@ export async function kickMainLoop(
   if (runStatus.state.kind === "scheduled") {
     if (source === "enqueue" && runStatus.state.saturated) {
       console.debug(
-        `[${source}] main is saturated, so we don't need to kick it`
+        `[${source}] main is saturated, so we don't need to kick it`,
       );
       return next;
     }
     if (runStatus.state.segment <= toSegment(Date.now() + SECOND)) {
       console.debug(
-        `[${source}] main is scheduled to run soon enough, so we don't need to kick it`
+        `[${source}] main is scheduled to run soon enough, so we don't need to kick it`,
       );
       return next;
     }
     console.debug(
-      `[${source}] main is scheduled to run later, so reschedule it to run now`
+      `[${source}] main is scheduled to run later, so reschedule it to run now`,
     );
     const scheduled = await ctx.db.system.get(runStatus.state.scheduledId);
     if (scheduled && scheduled.state.kind === "pending") {
       await ctx.scheduler.cancel(runStatus.state.scheduledId);
     } else {
       console.warn(
-        `[${source}] main is marked as scheduled, but it's status is ${scheduled?.state.kind}`
+        `[${source}] main is marked as scheduled, but it's status is ${scheduled?.state.kind}`,
       );
     }
   } else if (runStatus.state.kind === "idle") {

package/src/component/lib.test.ts

@@ -62,11 +62,11 @@ describe("lib", () => {
           fnType: "mutation",
           runAt: Date.now(),
           config: {
-            maxParallelism: 101, // More than MAX_POSSIBLE_PARALLELISM
+            maxParallelism: 500, // More than MAX_POSSIBLE_PARALLELISM
             logLevel: "WARN",
           },
-        })
-      ).rejects.toThrow("maxParallelism must be <= 50");
+        }),
+      ).rejects.toThrow("maxParallelism must be <= 100");
     });
 
     it("should throw error if maxParallelism is too low", async () => {
@@ -81,7 +81,7 @@ describe("lib", () => {
             maxParallelism: 0, // Less than minimum
             logLevel: "WARN",
           },
-        })
+        }),
       ).rejects.toThrow("maxParallelism must be >= 1");
     });
   });
@@ -249,7 +249,7 @@ describe("lib", () => {
         expect(scheduledFunctions.length).toBeGreaterThan(0);
         // check that one of the scheduled functions is cancelAll
         const cancelAllScheduledFunction = scheduledFunctions.find(
-          (sf) => sf.name === "lib:cancelAll"
+          (sf) => sf.name === "lib:cancelAll",
         );
         expect(cancelAllScheduledFunction).toBeDefined();
         assert(cancelAllScheduledFunction);

package/src/component/lib.ts

@@ -27,8 +27,8 @@ import {
 } from "./shared.js";
 import { recordEnqueued } from "./stats.js";
 
-const MAX_POSSIBLE_PARALLELISM = 100;
-const MAX_PARALLELISM_SOFT_LIMIT = 50;
+const MAX_POSSIBLE_PARALLELISM = 200;
+const MAX_PARALLELISM_SOFT_LIMIT = 100;
 
 const itemArgs = {
   fnHandle: v.string(),
@@ -58,7 +58,7 @@ async function enqueueHandler(
   ctx: MutationCtx,
   console: Logger,
   kickSegment: bigint,
-  { runAt, ...workArgs }: ObjectType<typeof itemArgs>
+  { runAt, ...workArgs }: ObjectType<typeof itemArgs>,
 ) {
   runAt = boundScheduledTime(runAt, console);
   const workId = await ctx.db.insert("work", {
@@ -79,7 +79,7 @@ function validateConfig(config: Config) {
     throw new Error(`maxParallelism must be <= ${MAX_PARALLELISM_SOFT_LIMIT}`);
   } else if (config.maxParallelism > MAX_PARALLELISM_SOFT_LIMIT) {
     createLogger(config.logLevel).warn(
-      `maxParallelism should be <= ${MAX_PARALLELISM_SOFT_LIMIT}, but is set to ${config.maxParallelism}. This will be an error in a future version.`
+      `maxParallelism should be <= ${MAX_PARALLELISM_SOFT_LIMIT}, but is set to ${config.maxParallelism}. This will be an error in a future version.`,
     );
   } else if (config.maxParallelism < 1) {
     throw new Error("maxParallelism must be >= 1");
@@ -97,7 +97,7 @@ export const enqueueBatch = mutation({
     const console = createLogger(config.logLevel);
     const kickSegment = await kickMainLoop(ctx, "enqueue", config);
     return Promise.all(
-      items.map((item) => enqueueHandler(ctx, console, kickSegment, item))
+      items.map((item) => enqueueHandler(ctx, console, kickSegment, item)),
     );
   },
 });
@@ -136,8 +136,8 @@ export const cancelAll = mutation({
       .take(pageSize);
     const shouldCancel = await Promise.all(
       pageOfWork.map(async ({ _id }) =>
-        shouldCancelWorkItem(ctx, _id, logLevel)
-      )
+        shouldCancelWorkItem(ctx, _id, logLevel),
+      ),
     );
     let segment = getNextSegment();
     if (shouldCancel.some((c) => c)) {
@@ -151,7 +151,7 @@ export const cancelAll = mutation({
             segment,
           });
         }
-      })
+      }),
     );
     if (pageOfWork.length === pageSize) {
       await ctx.scheduler.runAfter(0, api.lib.cancelAll, {
@@ -196,7 +196,7 @@ export const statusBatch = query({
   returns: v.array(statusValidator),
   handler: async (ctx, { ids }) => {
     return await Promise.all(
-      ids.map(async (id) => await statusHandler(ctx, { id }))
+      ids.map(async (id) => await statusHandler(ctx, { id })),
     );
   },
 });
@@ -204,7 +204,7 @@ export const statusBatch = query({
 async function shouldCancelWorkItem(
   ctx: MutationCtx,
   workId: Id<"work">,
-  logLevel: LogLevel
+  logLevel: LogLevel,
 ) {
   const console = createLogger(logLevel);
   // No-op if the work doesn't exist or has completed.

package/src/component/logging.ts

@@ -10,7 +10,7 @@ export const logLevel = v.union(
   v.literal("INFO"),
   v.literal("REPORT"),
   v.literal("WARN"),
-  v.literal("ERROR")
+  v.literal("ERROR"),
 );
 export type LogLevel = Infer<typeof logLevel>;
 
@@ -30,7 +30,7 @@ const logLevelByName = logLevelOrder.reduce(
     acc[l] = i;
     return acc;
   },
-  {} as Record<LogLevel, number>
+  {} as Record<LogLevel, number>,
 );
 export function shouldLog(config: LogLevel, level: LogLevel) {
   return logLevelByName[config] <= logLevelByName[level];

package/src/component/loop.test.ts

@@ -49,7 +49,7 @@ describe("loop", () => {
 
   async function makeDummyWork(
     ctx: MutationCtx,
-    overrides: Partial<WithoutSystemFields<Doc<"work">>> = {}
+    overrides: Partial<WithoutSystemFields<Doc<"work">>> = {},
   ) {
     return ctx.db.insert("work", {
       fnType: "action",
@@ -63,7 +63,7 @@ describe("loop", () => {
 
   async function makeDummyScheduledFunction(
     ctx: MutationCtx,
-    workId: Id<"work">
+    workId: Id<"work">,
   ) {
     return ctx.scheduler.runAfter(0, internal.worker.runActionWrapper, {
       workId,
@@ -76,7 +76,7 @@ describe("loop", () => {
 
   async function insertInternalState(
     ctx: MutationCtx,
-    overrides: Partial<WithoutSystemFields<Doc<"internalState">>> = {}
+    overrides: Partial<WithoutSystemFields<Doc<"internalState">>> = {},
   ) {
     await ctx.db.insert("internalState", {
       generation: 1n,
@@ -450,7 +450,7 @@ describe("loop", () => {
         const scheduledId = await ctx.scheduler.runAfter(
           1000,
           internal.loop.main,
-          { generation: 1n, segment: getNextSegment() + 10n }
+          { generation: 1n, segment: getNextSegment() + 10n },
         );
 
         // Create scheduled runStatus
@@ -553,7 +553,7 @@ describe("loop", () => {
         const scheduledId = await ctx.scheduler.runAfter(
           1000,
           internal.loop.main,
-          { generation: 1n, segment }
+          { generation: 1n, segment },
         );
 
         await ctx.db.insert("runStatus", {
@@ -588,7 +588,7 @@ describe("loop", () => {
 
       // Call main with mismatched generation
       await expect(
-        t.mutation(internal.loop.main, { generation: 1n, segment: 1n })
+        t.mutation(internal.loop.main, { generation: 1n, segment: 1n }),
       ).rejects.toThrow("generation mismatch");
     });
 
@@ -707,7 +707,7 @@ describe("loop", () => {
         // Schedule a function and get its ID
         const scheduledId = await makeDummyScheduledFunction(
           ctx,
-          runningWorkId
+          runningWorkId,
         );
 
         // Create internal state
@@ -848,7 +848,7 @@ describe("loop", () => {
         t.mutation(internal.loop.updateRunStatus, {
           generation: 1n,
           segment: 1n,
-        })
+        }),
       ).rejects.toThrow("generation mismatch");
     });
 
@@ -935,7 +935,7 @@ describe("loop", () => {
               const scheduledId = await makeDummyScheduledFunction(ctx, workId);
 
               return { workId, scheduledId, started: Date.now() };
-            })
+            }),
         );
 
         // Create internal state with max running jobs