@convex-dev/workpool 0.2.19 → 0.3.0

package/src/component/loop.ts

@@ -52,7 +52,7 @@ export const main = internalMutation({
     const state = await getOrCreateState(ctx);
     if (generation !== state.generation) {
       throw new Error(
-        `generation mismatch: ${generation} !== ${state.generation}`
+        `generation mismatch: ${generation} !== ${state.generation}`,
       );
     }
     state.generation++;
@@ -129,7 +129,7 @@ export const updateRunStatus = internalMutation({
     const state = await getOrCreateState(ctx);
     if (generation !== state.generation) {
       throw new Error(
-        `generation mismatch: ${generation} !== ${state.generation}`
+        `generation mismatch: ${generation} !== ${state.generation}`,
       );
     }
 
@@ -154,7 +154,7 @@ export const updateRunStatus = internalMutation({
       ctx,
       state,
       maxParallelism,
-      nextSegment
+      nextSegment,
     );
     console.timeEnd("[updateRunStatus] nextSegmentIsActionable");
 
@@ -165,7 +165,7 @@ export const updateRunStatus = internalMutation({
         {
           generation,
           segment: nextSegment,
-        }
+        },
       );
       return;
     }
@@ -174,7 +174,7 @@ export const updateRunStatus = internalMutation({
     const [oldIsActionable, cursors] = await oldSegmentIsActionable(
       ctx,
       state,
-      maxParallelism
+      maxParallelism,
     );
     console.timeEnd("[updateRunStatus] oldSegmentIsActionable");
 
@@ -204,8 +204,8 @@ export const updateRunStatus = internalMutation({
     }
     const docs = await Promise.all(
       actionableTables.map(async (tableName) =>
-        getNextUp(ctx, tableName, { start: nextSegment })
-      )
+        getNextUp(ctx, tableName, { start: nextSegment }),
+      ),
     );
     console.timeEnd("[updateRunStatus] findNextSegment");
     let targetSegment = docs.map((d) => d?.segment).sort()[0];
@@ -221,7 +221,7 @@ export const updateRunStatus = internalMutation({
       const scheduledId = await ctx.scheduler.runAt(
         boundScheduledTime(fromSegment(targetSegment), console),
         internal.loop.main,
-        { generation, segment: targetSegment }
+        { generation, segment: targetSegment },
       );
       if (targetSegment > getNextSegment()) {
         await ctx.db.patch(runStatus._id, {
@@ -235,7 +235,7 @@ export const updateRunStatus = internalMutation({
         });
       } else {
         console.debug(
-          `[updateRunStatus] staying running because it's the next segment`
+          `[updateRunStatus] staying running because it's the next segment`,
         );
       }
       return;
@@ -251,7 +251,7 @@ async function nextSegmentIsActionable(
   ctx: MutationCtx,
   state: Doc<"internalState">,
   maxParallelism: number,
-  end: bigint
+  end: bigint,
 ): Promise<boolean> {
   // First, try with our cursor range, up to end.
   if (
@@ -286,7 +286,7 @@ async function nextSegmentIsActionable(
 async function oldSegmentIsActionable(
   ctx: MutationCtx,
   state: Doc<"internalState">,
-  maxParallelism: number
+  maxParallelism: number,
 ): Promise<
   [boolean, { completion?: bigint; cancelation?: bigint; incoming?: bigint }]
 > {
@@ -318,7 +318,7 @@ async function oldSegmentIsActionable(
 async function getNextUp(
   ctx: MutationCtx,
   table: "pendingCompletion" | "pendingCancelation" | "pendingStart",
-  range: { start?: bigint; end?: bigint }
+  range: { start?: bigint; end?: bigint },
 ) {
   return ctx.db
     .query(table)
@@ -331,7 +331,7 @@ async function getNextUp(
           : q.gt("segment", range.start - CURSOR_BUFFER_SEGMENTS)
         : range.end !== undefined
           ? q.lt("segment", range.end)
-          : q
+          : q,
     )
     .first();
 }
@@ -344,7 +344,7 @@ async function handleCompletions(
   ctx: MutationCtx,
   state: Doc<"internalState">,
   segment: bigint,
-  console: Logger
+  console: Logger,
 ) {
   const startSegment = state.segmentCursors.completion - CURSOR_BUFFER_SEGMENTS;
   // This won't be too many because the jobs all correspond to being scheduled
@@ -352,7 +352,7 @@ async function handleCompletions(
   const completed = await ctx.db
     .query("pendingCompletion")
     .withIndex("segment", (q) =>
-      q.gte("segment", startSegment).lte("segment", segment)
+      q.gte("segment", startSegment).lte("segment", segment),
     )
     .collect();
   state.segmentCursors.completion = segment;
@@ -365,7 +365,7 @@ async function handleCompletions(
       const running = state.running.find((r) => r.workId === c.workId);
       if (!running) {
         console.error(
-          `[main] completing ${c.workId} but it's not in "running"`
+          `[main] completing ${c.workId} but it's not in "running"`,
         );
         return;
       }
@@ -396,12 +396,12 @@ async function handleCompletions(
           state.report.failed++;
         }
       }
-    })
+    }),
   );
   // We do this after so the stats above know if it was in progress.
   const before = state.running.length;
   state.running = state.running.filter(
-    (r) => !completed.some((c) => c.workId === r.workId)
+    (r) => !completed.some((c) => c.workId === r.workId),
   );
   const numCompleted = before - state.running.length;
   state.report.completed += numCompleted;
@@ -414,13 +414,13 @@ async function handleCancelation(
   state: Doc<"internalState">,
   segment: bigint,
   console: Logger,
-  toCancel: CompleteJob[]
+  toCancel: CompleteJob[],
 ) {
   const start = state.segmentCursors.cancelation - CURSOR_BUFFER_SEGMENTS;
   const canceled = await ctx.db
     .query("pendingCancelation")
     .withIndex("segment", (q) =>
-      q.gte("segment", start).lte("segment", segment)
+      q.gte("segment", start).lte("segment", segment),
     )
     .take(CANCELLATION_BATCH_SIZE);
   state.segmentCursors.cancelation = canceled.at(-1)?.segment ?? segment;
@@ -458,9 +458,9 @@ async function handleCancelation(
             return { workId, runResult, attempt: work.attempts };
           }
           return null;
-        })
+        }),
       )
-    ).flatMap((r) => (r ? [r] : []))
+    ).flatMap((r) => (r ? [r] : [])),
   );
   if (jobs.length) {
     await ctx.scheduler.runAfter(0, internal.complete.complete, { jobs });
@@ -470,7 +470,7 @@ async function handleCancelation(
 async function handleRecovery(
   ctx: MutationCtx,
   state: Doc<"internalState">,
-  console: Logger
+  console: Logger,
 ) {
   const missing = new Set<Id<"work">>();
   const oldEnoughToConsider = Date.now() - RECOVERY_THRESHOLD_MS;
@@ -487,7 +487,7 @@ async function handleRecovery(
           return null;
         }
         return { ...r, attempt: work.attempts };
-      })
+      }),
     )
   ).flatMap((r) => (r ? [r] : []));
   state.running = state.running.filter((r) => !missing.has(r.workId));
@@ -501,7 +501,7 @@ async function handleStart(
   state: Doc<"internalState">,
   segment: bigint,
   console: Logger,
-  { maxParallelism, logLevel }: Config
+  { maxParallelism, logLevel }: Config,
 ) {
   // Schedule as many as needed to reach maxParallelism.
   const toSchedule = maxParallelism - state.running.length;
@@ -514,9 +514,9 @@ async function handleStart(
             q
               .gte(
                 "segment",
-                state.segmentCursors.incoming - CURSOR_BUFFER_SEGMENTS
+                state.segmentCursors.incoming - CURSOR_BUFFER_SEGMENTS,
               )
-              .lte("segment", segment)
+              .lte("segment", segment),
           )
           .take(toSchedule)
       : [];
@@ -543,9 +543,9 @@ async function handleStart(
           const scheduledId = await beginWork(ctx, workId, logLevel, lagMs);
           await ctx.db.delete(_id);
           return { scheduledId, workId, started: Date.now() };
-        })
+        }),
       )
-    ).flatMap((r) => (r ? [r] : []))
+    ).flatMap((r) => (r ? [r] : [])),
   );
 }
 
@@ -553,7 +553,7 @@ async function beginWork(
   ctx: MutationCtx,
   workId: Id<"work">,
   logLevel: LogLevel,
-  lagMs: number
+  lagMs: number,
 ): Promise<Id<"_scheduled_functions">> {
   const console = createLogger(logLevel);
   const work = await ctx.db.get(workId);
@@ -583,7 +583,7 @@ async function beginWork(
 async function rescheduleJob(
   ctx: MutationCtx,
   work: Doc<"work">,
-  console: Logger
+  console: Logger,
 ): Promise<boolean> {
   const pendingCancelation = await ctx.db
     .query("pendingCancelation")
@@ -645,7 +645,7 @@ async function getOrCreateState(ctx: MutationCtx) {
   const console = createLogger(globals.logLevel);
   console.error("No internalState in running loop! Re-creating empty one...");
   return (await ctx.db.get(
-    await ctx.db.insert("internalState", INITIAL_STATE)
+    await ctx.db.insert("internalState", INITIAL_STATE),
   ))!;
 }
 
@@ -656,7 +656,7 @@ async function getOrCreateRunningStatus(ctx: MutationCtx) {
   const console = createLogger(globals.logLevel);
   console.error("No runStatus in running loop! Re-creating one...");
   return (await ctx.db.get(
-    await ctx.db.insert("runStatus", { state: { kind: "running" } })
+    await ctx.db.insert("runStatus", { state: { kind: "running" } }),
   ))!;
 }
 // eslint-disable-next-line @typescript-eslint/no-unused-vars

package/src/component/recovery.test.ts

@@ -28,7 +28,7 @@ describe("recovery", () => {
   // Helper function to create a work item
   async function makeDummyWork(
     ctx: MutationCtx,
-    overrides: Partial<WithoutSystemFields<Doc<"work">>> = {}
+    overrides: Partial<WithoutSystemFields<Doc<"work">>> = {},
   ) {
     return ctx.db.insert("work", {
       fnType: "action",
@@ -43,7 +43,7 @@ describe("recovery", () => {
   // Helper function to create a scheduled function
   async function makeDummyScheduledFunction(
     ctx: MutationCtx,
-    workId: Id<"work">
+    workId: Id<"work">,
   ) {
     return ctx.scheduler.runAfter(0, internal.worker.runActionWrapper, {
       workId,
@@ -226,7 +226,7 @@ describe("recovery", () => {
         expect(pendingCompletions[0].runResult.kind).toBe("failed");
         assert(pendingCompletions[0].runResult.kind === "failed");
         expect(pendingCompletions[0].runResult.error).toContain(
-          "Scheduled job not found"
+          "Scheduled job not found",
         );
       });
     });
@@ -292,7 +292,7 @@ describe("recovery", () => {
         expect(pendingCompletions[0].runResult.kind).toBe("failed");
         assert(pendingCompletions[0].runResult.kind === "failed");
         expect(pendingCompletions[0].runResult.error).toBe(
-          "Function execution failed"
+          "Function execution failed",
         );
       });
     });
@@ -357,7 +357,7 @@ describe("recovery", () => {
         expect(pendingCompletions[0].runResult.kind).toBe("failed");
         assert(pendingCompletions[0].runResult.kind === "failed");
         expect(pendingCompletions[0].runResult.error).toBe(
-          "Canceled via scheduler"
+          "Canceled via scheduler",
         );
       });
     });
@@ -451,10 +451,10 @@ describe("recovery", () => {
 
         // Find completions for each work ID
         const completion1 = pendingCompletions.find(
-          (pc) => pc.workId === workId1
+          (pc) => pc.workId === workId1,
         );
         const completion2 = pendingCompletions.find(
-          (pc) => pc.workId === workId2
+          (pc) => pc.workId === workId2,
         );
 
         expect(completion1).toBeDefined();

package/src/component/recovery.ts

@@ -10,7 +10,7 @@ const recoveryArgs = v.object({
       workId: v.id("work"),
       attempt: v.number(),
       started: v.number(),
-    })
+    }),
   ),
 });
 
@@ -37,7 +37,7 @@ export const recover = internalMutation({
 // only exported for testing
 export async function recoveryHandler(
   ctx: MutationCtx,
-  { jobs }: Infer<typeof recoveryArgs>
+  { jobs }: Infer<typeof recoveryArgs>,
 ) {
   const globals = await ctx.db.query("globals").unique();
   const console = createLogger(globals?.logLevel);

package/src/component/schema.ts

@@ -37,7 +37,7 @@ export default defineSchema({
         workId: v.id("work"),
         scheduledId: v.id("_scheduled_functions"),
         started: v.number(),
-      })
+      }),
     ),
   }),
 
@@ -53,7 +53,7 @@ export default defineSchema({
         saturated: v.boolean(),
         generation: v.int64(),
       }),
-      v.object({ kind: v.literal("idle"), generation: v.int64() })
+      v.object({ kind: v.literal("idle"), generation: v.int64() }),
     ),
   }),
 

package/src/component/shared.ts

@@ -6,7 +6,7 @@ import { type Logger, logLevel } from "./logging.js";
 export const fnType = v.union(
   v.literal("action"),
   v.literal("mutation"),
-  v.literal("query")
+  v.literal("query"),
 );
 
 export const DEFAULT_MAX_PARALLELISM = 10;
@@ -61,6 +61,12 @@ export type RetryBehavior = {
    */
   base: number;
 };
+// Attempts will run with delay [0, 250, 500, 1000, 2000] (ms)
+export const DEFAULT_RETRY_BEHAVIOR: RetryBehavior = {
+  maxAttempts: 5,
+  initialBackoffMs: 250,
+  base: 2,
+};
 // This ensures that the type satisfies the schema.
 const _ = {} as RetryBehavior satisfies Infer<typeof retryBehavior>;
 
@@ -75,7 +81,7 @@ export const vResultValidator = v.union(
   }),
   v.object({
     kind: v.literal("canceled"),
-  })
+  }),
 );
 export type RunResult = Infer<typeof vResultValidator>;
 
@@ -113,8 +119,8 @@ export const status = v.union(
     }),
     v.object({
       state: v.literal("finished"),
-    })
-  )
+    }),
+  ),
 );
 export type Status = Infer<typeof status>;
 
@@ -126,7 +132,7 @@ export function boundScheduledTime(ms: number, console: Logger): number {
   if (ms > Date.now() + 4 * YEAR) {
     console.error(
       "scheduled time is too far in the future, defaulting to 1 year from now",
-      ms
+      ms,
     );
     return Date.now() + YEAR;
   }

package/src/component/stats.test.ts

@@ -240,7 +240,7 @@ describe("stats", () => {
 
         // Check that one of the scheduled functions is calculateBacklogAndReport
         const calculateBacklogScheduled = scheduledFunctions.find(
-          (sf) => sf.name === "stats:calculateBacklogAndReport"
+          (sf) => sf.name === "stats:calculateBacklogAndReport",
         );
         expect(calculateBacklogScheduled).toBeDefined();
         assert(calculateBacklogScheduled);
@@ -306,7 +306,7 @@ describe("stats", () => {
         return await paginator(ctx.db, schema)
           .query("pendingStart")
           .withIndex("segment", (q) =>
-            q.gte("segment", 0n).lt("segment", currentSegment)
+            q.gte("segment", 0n).lt("segment", currentSegment),
           )
           .paginate({
             numItems: 1,
@@ -336,7 +336,7 @@ describe("stats", () => {
 
         // Since our backlog is small, no additional scheduled functions should be created
         const calculateBacklogScheduled = scheduledFunctions.find(
-          (sf) => sf.name === "stats:calculateBacklogAndReport"
+          (sf) => sf.name === "stats:calculateBacklogAndReport",
         );
         expect(calculateBacklogScheduled).toBeUndefined();
       });

package/src/component/stats.ts

@@ -26,7 +26,7 @@ export function recordEnqueued(
     workId: Id<"work">;
     fnName: string;
     runAt: number;
-  }
+  },
 ) {
   console.event("enqueued", {
     ...data,
@@ -37,7 +37,7 @@ export function recordEnqueued(
 export function recordStarted(
   console: Logger,
   work: Doc<"work">,
-  lagMs: number
+  lagMs: number,
 ) {
   console.event("started", {
     workId: work._id,
@@ -51,7 +51,7 @@ export function recordStarted(
 export function recordCompleted(
   console: Logger,
   work: Doc<"work">,
-  status: "success" | "failed" | "canceled" | "retrying"
+  status: "success" | "failed" | "canceled" | "retrying",
 ) {
   console.event("completed", {
     workId: work._id,
@@ -66,7 +66,7 @@ export async function generateReport(
   ctx: MutationCtx,
   console: Logger,
   state: Doc<"internalState">,
-  { maxParallelism, logLevel }: Config
+  { maxParallelism, logLevel }: Config,
 ) {
   if (!shouldLog(logLevel, "REPORT")) {
     // Don't waste time if we're not going to log.
@@ -78,7 +78,7 @@ export async function generateReport(
     .withIndex("segment", (q) =>
       q
         .gte("segment", state.segmentCursors.incoming)
-        .lt("segment", currentSegment)
+        .lt("segment", currentSegment),
     )
     .paginate({
       numItems: maxParallelism,
@@ -125,7 +125,7 @@ export const calculateBacklogAndReport = internalMutation({
 
 function recordReport(
   console: Logger,
-  report: Doc<"internalState">["report"] & { running: number; backlog: number }
+  report: Doc<"internalState">["report"] & { running: number; backlog: number },
 ) {
   const { completed, failed, retries } = report;
   const withoutRetries = completed - retries;

package/src/test.ts

@@ -1,11 +1,18 @@
+/// <reference types="vite/client" />
 import type { TestConvex } from "convex-test";
 import type { GenericSchema, SchemaDefinition } from "convex/server";
 import schema from "./component/schema.js";
 const modules = import.meta.glob("./component/**/*.ts");
-function register(
+
+/**
+ * Register the component with the test convex instance.
+ * @param t - The test convex instance, e.g. from calling `convexTest`.
+ * @param name - The name of the component, as registered in convex.config.ts.
+ */
+export function register(
   t: TestConvex<SchemaDefinition<GenericSchema, boolean>>,
-  name: string
+  name: string = "workpool",
 ) {
   t.registerComponent(name, schema, modules);
 }
-export default { schema, modules, register };
+export default { register, schema, modules };

package/src/component/_generated/api.d.ts

@@ -1,151 +0,0 @@
-/* eslint-disable */
-/**
- * Generated `api` utility.
- *
- * THIS CODE IS AUTOMATICALLY GENERATED.
- *
- * To regenerate, run `npx convex dev`.
- * @module
- */
-
-import type * as complete from "../complete.js";
-import type * as crons from "../crons.js";
-import type * as danger from "../danger.js";
-import type * as kick from "../kick.js";
-import type * as lib from "../lib.js";
-import type * as logging from "../logging.js";
-import type * as loop from "../loop.js";
-import type * as recovery from "../recovery.js";
-import type * as shared from "../shared.js";
-import type * as stats from "../stats.js";
-import type * as worker from "../worker.js";
-
-import type {
-  ApiFromModules,
-  FilterApi,
-  FunctionReference,
-} from "convex/server";
-
-/**
- * A utility for referencing Convex functions in your app's API.
- *
- * Usage:
- * ```js
- * const myFunctionReference = api.myModule.myFunction;
- * ```
- */
-declare const fullApi: ApiFromModules<{
-  complete: typeof complete;
-  crons: typeof crons;
-  danger: typeof danger;
-  kick: typeof kick;
-  lib: typeof lib;
-  logging: typeof logging;
-  loop: typeof loop;
-  recovery: typeof recovery;
-  shared: typeof shared;
-  stats: typeof stats;
-  worker: typeof worker;
-}>;
-export type Mounts = {
-  lib: {
-    cancel: FunctionReference<
-      "mutation",
-      "public",
-      {
-        id: string;
-        logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
-      },
-      any
-    >;
-    cancelAll: FunctionReference<
-      "mutation",
-      "public",
-      {
-        before?: number;
-        limit?: number;
-        logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
-      },
-      any
-    >;
-    enqueue: FunctionReference<
-      "mutation",
-      "public",
-      {
-        config: {
-          logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
-          maxParallelism: number;
-        };
-        fnArgs: any;
-        fnHandle: string;
-        fnName: string;
-        fnType: "action" | "mutation" | "query";
-        onComplete?: { context?: any; fnHandle: string };
-        retryBehavior?: {
-          base: number;
-          initialBackoffMs: number;
-          maxAttempts: number;
-        };
-        runAt: number;
-      },
-      string
-    >;
-    enqueueBatch: FunctionReference<
-      "mutation",
-      "public",
-      {
-        config: {
-          logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
-          maxParallelism: number;
-        };
-        items: Array<{
-          fnArgs: any;
-          fnHandle: string;
-          fnName: string;
-          fnType: "action" | "mutation" | "query";
-          onComplete?: { context?: any; fnHandle: string };
-          retryBehavior?: {
-            base: number;
-            initialBackoffMs: number;
-            maxAttempts: number;
-          };
-          runAt: number;
-        }>;
-      },
-      Array<string>
-    >;
-    status: FunctionReference<
-      "query",
-      "public",
-      { id: string },
-      | { previousAttempts: number; state: "pending" }
-      | { previousAttempts: number; state: "running" }
-      | { state: "finished" }
-    >;
-    statusBatch: FunctionReference<
-      "query",
-      "public",
-      { ids: Array<string> },
-      Array<
-        | { previousAttempts: number; state: "pending" }
-        | { previousAttempts: number; state: "running" }
-        | { state: "finished" }
-      >
-    >;
-  };
-};
-// For now fullApiWithMounts is only fullApi which provides
-// jump-to-definition in component client code.
-// Use Mounts for the same type without the inference.
-declare const fullApiWithMounts: typeof fullApi;
-
-export declare const api: FilterApi<
-  typeof fullApiWithMounts,
-  FunctionReference<any, "public">
->;
-export declare const internal: FilterApi<
-  typeof fullApiWithMounts,
-  FunctionReference<any, "internal">
->;
-
-export declare const components: {};

package/src/component/_generated/api.js

@@ -1,23 +0,0 @@
-/* eslint-disable */
-/**
- * Generated `api` utility.
- *
- * THIS CODE IS AUTOMATICALLY GENERATED.
- *
- * To regenerate, run `npx convex dev`.
- * @module
- */
-
-import { anyApi, componentsGeneric } from "convex/server";
-
-/**
- * A utility for referencing Convex functions in your app's API.
- *
- * Usage:
- * ```js
- * const myFunctionReference = api.myModule.myFunction;
- * ```
- */
-export const api = anyApi;
-export const internal = anyApi;
-export const components = componentsGeneric();