@convex-dev/workpool 0.2.0-beta.0 → 0.2.1

package/src/client/index.ts

@@ -8,19 +8,22 @@ import {
 import { v, VString } from "convex/values";
 import { Mounts } from "../component/_generated/api.js";
 import {
+  DEFAULT_LOG_LEVEL,
+  type LogLevel,
+  logLevel,
+} from "../component/logging.js";
+import {
+  Config,
+  DEFAULT_MAX_PARALLELISM,
   OnComplete,
-  runResult as runResultValidator,
-  RunResult,
+  runResult as resultValidator,
   type RetryBehavior,
+  RunResult,
   OnCompleteArgs as SharedOnCompleteArgs,
   Status,
-  Config,
 } from "../component/shared.js";
-import { type LogLevel, logLevel } from "../component/logging.js";
 import { RunMutationCtx, RunQueryCtx, UseApi } from "./utils.js";
-import { DEFAULT_LOG_LEVEL } from "../component/logging.js";
-import { DEFAULT_MAX_PARALLELISM } from "../component/kick.js";
-export { runResultValidator, type RunResult };
+export { resultValidator, type RunResult };
 
 // Attempts will run with delay [0, 250, 500, 1000, 2000] (ms)
 export const DEFAULT_RETRY_BEHAVIOR: RetryBehavior = {
@@ -52,17 +55,20 @@ export class Workpool {
       maxParallelism?: number;
       /** How much to log. This is updated on each call to `enqueue*`,
        * `status`, or `cancel*`.
-       * Default is WARN.
-       * With INFO, you can see events for started and completed work, which can
-       * be parsed by tools like [Axiom](https://axiom.co) for monitoring.
+       * Default is REPORT, which logs warnings, errors, and a periodic report.
+       * With INFO, you can also see events for started and completed work.
+       * Stats generated can be parsed by tools like
+       * [Axiom](https://axiom.co) for monitoring.
        * With DEBUG, you can see timers and internal events for work being
        * scheduled.
        */
       logLevel?: LogLevel;
-      /** Default retry behavior for enqueued actions. */
+      /** Default retry behavior for enqueued actions.
+       * See {@link RetryBehavior}.
+       */
       defaultRetryBehavior?: RetryBehavior;
       /** Whether to retry actions that fail by default. Default: false.
-       * NOTE: Only do this if your actions are idempotent.
+       * NOTE: Only enable this if your actions are idempotent.
        * See the docs (README.md) for more details.
        */
       retryActionsByDefault?: boolean;
@@ -133,60 +139,59 @@ export class Workpool {
     fnArgs: Args,
     options?: CallbackOptions & SchedulerOptions
   ): Promise<WorkId> {
+    const onComplete: OnComplete | undefined = options?.onComplete
+      ? {
+          fnHandle: await createFunctionHandle(options.onComplete),
+          context: options.context,
+        }
+      : undefined;
     const id = await ctx.runMutation(this.component.lib.enqueue, {
       ...(await defaultEnqueueArgs(fn, this.options)),
       fnArgs,
       fnType: "mutation",
       runAt: getRunAt(options),
+      onComplete,
     });
     return id as WorkId;
   }
+  /**
+   * Cancels a work item. If it's already started, it will be allowed to finish
+   * but will not be retried.
+   *
+   * @param ctx - The mutation or action context that can call ctx.runMutation.
+   * @param id - The ID of the work to cancel.
+   */
   async cancel(ctx: RunMutationCtx, id: WorkId): Promise<void> {
     await ctx.runMutation(this.component.lib.cancel, {
       id,
       logLevel: this.options.logLevel ?? getDefaultLogLevel(),
     });
   }
+  /**
+   * Cancels all pending work items. See {@link cancel}.
+   *
+   * @param ctx - The mutation or action context that can call ctx.runMutation.
+   */
   async cancelAll(ctx: RunMutationCtx): Promise<void> {
     await ctx.runMutation(this.component.lib.cancelAll, {
       logLevel: this.options.logLevel ?? getDefaultLogLevel(),
     });
   }
+  /**
+   * Gets the status of a work item.
+   *
+   * @param ctx - The query context that can call ctx.runQuery.
+   * @param id - The ID of the work to get the status of.
+   * @returns The status of the work item. One of:
+   * - `{ state: "pending", previousAttempts: number }`
+   * - `{ state: "running", previousAttempts: number }`
+   * - `{ state: "finished" }`
+   */
   async status(ctx: RunQueryCtx, id: WorkId): Promise<Status> {
     return ctx.runQuery(this.component.lib.status, { id });
   }
 }
 
-function getRetryBehavior(
-  defaultRetryBehavior: RetryBehavior | undefined,
-  retryActionsByDefault: boolean | undefined,
-  retryOverride: boolean | RetryBehavior | undefined
-): RetryBehavior | undefined {
-  const defaultRetry = defaultRetryBehavior ?? DEFAULT_RETRY_BEHAVIOR;
-  const retryByDefault = retryActionsByDefault ?? false;
-  if (retryOverride === true) {
-    return defaultRetry;
-  }
-  if (retryOverride === false) {
-    return undefined;
-  }
-  return retryOverride ?? (retryByDefault ? defaultRetry : undefined);
-}
-
-async function defaultEnqueueArgs(
-  fn: FunctionReference<"action" | "mutation", FunctionVisibility>,
-  { logLevel, maxParallelism }: Partial<Config>
-) {
-  return {
-    fnHandle: await createFunctionHandle(fn),
-    fnName: getFunctionName(fn),
-    config: {
-      logLevel: logLevel ?? getDefaultLogLevel(),
-      maxParallelism: maxParallelism ?? DEFAULT_MAX_PARALLELISM,
-    },
-  };
-}
-
 export type SchedulerOptions =
   | {
       /**
@@ -215,7 +220,7 @@ export type CallbackOptions = {
    *  args: {
    *    workId: workIdValidator,
    *    context: v.any(),
-   *    result: runResult,
+   *    result: resultValidator,
    *  },
    *  handler: async (ctx, args) => {
    *    console.log(args.result, "Got Context back -> ", args.context, Date.now() - args.context);
@@ -254,6 +259,40 @@ export type OnCompleteArgs = {
 // ensure OnCompleteArgs satisfies SharedOnCompleteArgs
 const _ = {} as OnCompleteArgs satisfies SharedOnCompleteArgs;
 
+//
+// Helper functions
+//
+
+function getRetryBehavior(
+  defaultRetryBehavior: RetryBehavior | undefined,
+  retryActionsByDefault: boolean | undefined,
+  retryOverride: boolean | RetryBehavior | undefined
+): RetryBehavior | undefined {
+  const defaultRetry = defaultRetryBehavior ?? DEFAULT_RETRY_BEHAVIOR;
+  const retryByDefault = retryActionsByDefault ?? false;
+  if (retryOverride === true) {
+    return defaultRetry;
+  }
+  if (retryOverride === false) {
+    return undefined;
+  }
+  return retryOverride ?? (retryByDefault ? defaultRetry : undefined);
+}
+
+async function defaultEnqueueArgs(
+  fn: FunctionReference<"action" | "mutation", FunctionVisibility>,
+  { logLevel, maxParallelism }: Partial<Config>
+) {
+  return {
+    fnHandle: await createFunctionHandle(fn),
+    fnName: getFunctionName(fn),
+    config: {
+      logLevel: logLevel ?? getDefaultLogLevel(),
+      maxParallelism: maxParallelism ?? DEFAULT_MAX_PARALLELISM,
+    },
+  };
+}
+
 function getRunAt(options?: SchedulerOptions): number {
   if (!options) {
     return Date.now();

package/src/component/README.md

@@ -25,19 +25,19 @@ Concepts:
 flowchart LR
     Client -->|enqueue| pendingStart
     Client -->|cancel| pendingCancelation
-    Recovery-->|recover| pendingCancelation
-    Recovery-->|recover| pendingCompletion
-    Worker-->|"saveResult"| pendingCompletion
-    pendingStart -->|main| workerRunning["internalState.running"]
-    workerRunning-->|"main(pendingCompletion)"| Retry{"Needs retry?"}
-    Retry-->|"no / canceled"| complete
-    Retry-->|yes| pendingStart
-    pendingStart-->|"main(pendingCancelation)"| complete
+    complete --> |success or failure| pendingCompletion
+    pendingCompletion -->|retry| pendingStart
+    pendingStart --> workerRunning["worker running"]
+    workerRunning -->|worker finished| complete
+    workerRunning --> |recovery| complete
+    successfulCancel["AND"]@{shape: delay} --> |canceled| complete
+    pendingStart --> successfulCancel
+    pendingCancelation --> successfulCancel
 ```
 
 Notably:
 
-- The pending\* states are only written by other sources.
+- The pending\* states are written by outside sources.
 - The main loop federates changes to/from "running"
 - Canceling only impacts pending and retrying jobs.
 
@@ -53,12 +53,12 @@ flowchart TD
     running-->|"all done"| idle
 ```
 
-- While the loop is running, clients won't see database conflicts with the
-  state changing.
-- The "saturated" state is concretely "running" or "scheduled" with a boolean
-  set, to avoid clients from kicking the main loop on enqueueing, which is
-  unlikely to be productive, since the next action needs to be something
-  terminating.
+- While the loop is running, the runStatus doesn't change, making it safer to
+  read from clients without database conflicts.
+- The "saturated" state is concretely "running" or "scheduled" at max
+  parallelism. There is a boolean set on "scheduled" to avoid clients from
+  kicking the main loop on enqueueing, which is unlikely to be productive, since
+  the next action needs to be something terminating.
 
 ## Retention optimization strategy
 

package/src/component/_generated/api.d.ts

@@ -8,6 +8,7 @@
  * @module
  */
 
+import type * as complete from "../complete.js";
 import type * as kick from "../kick.js";
 import type * as lib from "../lib.js";
 import type * as logging from "../logging.js";
@@ -31,6 +32,7 @@ import type {
  * ```
  */
 declare const fullApi: ApiFromModules<{
+  complete: typeof complete;
   kick: typeof kick;
   lib: typeof lib;
   logging: typeof logging;
@@ -45,13 +47,16 @@ export type Mounts = {
     cancel: FunctionReference<
       "mutation",
       "public",
-      { id: string; logLevel: "DEBUG" | "INFO" | "WARN" | "ERROR" },
+      { id: string; logLevel: "DEBUG" | "INFO" | "REPORT" | "WARN" | "ERROR" },
       any
     >;
     cancelAll: FunctionReference<
       "mutation",
       "public",
-      { before?: number; logLevel: "DEBUG" | "INFO" | "WARN" | "ERROR" },
+      {
+        before?: number;
+        logLevel: "DEBUG" | "INFO" | "REPORT" | "WARN" | "ERROR";
+      },
       any
     >;
     enqueue: FunctionReference<
@@ -59,7 +64,7 @@ export type Mounts = {
       "public",
       {
         config: {
-          logLevel: "DEBUG" | "INFO" | "WARN" | "ERROR";
+          logLevel: "DEBUG" | "INFO" | "REPORT" | "WARN" | "ERROR";
           maxParallelism: number;
         };
         fnArgs: any;
@@ -80,8 +85,8 @@ export type Mounts = {
       "query",
       "public",
       { id: string },
-      | { attempt: number; state: "pending" }
-      | { attempt: number; state: "running" }
+      | { previousAttempts: number; state: "pending" }
+      | { previousAttempts: number; state: "running" }
       | { state: "finished" }
     >;
   };