@convex-dev/workpool 0.2.0-beta.0 → 0.2.1

package/src/component/stats.ts

@@ -1,89 +1,179 @@
 import { v } from "convex/values";
-import { Doc } from "./_generated/dataModel.js";
-import { internalQuery } from "./_generated/server.js";
+import { Doc, Id } from "./_generated/dataModel.js";
+import {
+  internalMutation,
+  internalQuery,
+  MutationCtx,
+} from "./_generated/server.js";
+import {
+  Config,
+  DEFAULT_MAX_PARALLELISM,
+  getCurrentSegment,
+} from "./shared.js";
+import { createLogger, Logger, logLevel, shouldLog } from "./logging.js";
+import { internal } from "./_generated/api.js";
+import schema from "./schema.js";
+import { paginator } from "convex-helpers/server/pagination";
+
+const BACKLOG_BATCH_SIZE = 100;
 
 /**
  * Record stats about work execution. Intended to be queried by Axiom or Datadog.
+ * See the [README](https://github.com/get-convex/workpool) for example queries.
  */
 
-/**
- * Sample axiom dashboard query:
-
-workpool
-| extend parsed_message = iff(
-	isnotnull(parse_json(trim("'", tostring(["data.message"])))),
-	parse_json(trim("'", tostring(["data.message"]))),
-	parse_json('{}')
-)
-| extend lagSinceEnqueued = parsed_message["lagSinceEnqueued"]
-| extend fnName = parsed_message["fnName"]
-| summarize avg(todouble(lagSinceEnqueued)) by bin_auto(_time), tostring(fnName)
-
- */
+export function recordEnqueued(
+  console: Logger,
+  data: {
+    workId: Id<"work">;
+    fnName: string;
+    runAt: number;
+  }
+) {
+  console.event("enqueued", {
+    ...data,
+    enqueuedAt: Date.now(),
+  });
+}
 
-export function recordStarted(work: Doc<"work">): string {
-  return JSON.stringify({
+export function recordStarted(
+  console: Logger,
+  work: Doc<"work">,
+  lagMs: number
+) {
+  console.event("started", {
     workId: work._id,
-    event: "started",
     fnName: work.fnName,
     enqueuedAt: work._creationTime,
     startedAt: Date.now(),
-    lagSinceEnqueued: Date.now() - work._creationTime,
+    startLag: lagMs,
   });
 }
 
 export function recordCompleted(
+  console: Logger,
   work: Doc<"work">,
-  status: "success" | "failed" | "canceled"
-): string {
-  return JSON.stringify({
+  status: "success" | "failed" | "canceled" | "retrying"
+) {
+  console.event("completed", {
     workId: work._id,
-    event: "completed",
     fnName: work.fnName,
     completedAt: Date.now(),
+    attempts: work.attempts,
     status,
-    lagSinceEnqueued: Date.now() - work._creationTime,
   });
 }
 
-export function recordReport(state: Doc<"internalState">): string {
-  const { completed, succeeded, failed, retries, canceled } = state.report;
-  const withoutRetries = completed - retries;
-  return JSON.stringify({
-    event: "report",
-    completed,
-    succeeded,
-    failed,
-    retries,
-    canceled,
-    failureRate: completed ? (failed + retries) / completed : 0,
-    permanentFailureRate: withoutRetries ? failed / withoutRetries : 0,
-  });
+export async function generateReport(
+  ctx: MutationCtx,
+  console: Logger,
+  state: Doc<"internalState">,
+  { maxParallelism, logLevel }: Config
+) {
+  if (!shouldLog(logLevel, "REPORT")) {
+    // Don't waste time if we're not going to log.
+    return;
+  }
+  const currentSegment = getCurrentSegment();
+  const pendingStart = await paginator(ctx.db, schema)
+    .query("pendingStart")
+    .withIndex("segment", (q) =>
+      q
+        .gte("segment", state.segmentCursors.incoming)
+        .lt("segment", currentSegment)
+    )
+    .paginate({
+      numItems: maxParallelism,
+      cursor: null,
+    });
+  if (pendingStart.isDone) {
+    recordReport(console, {
+      ...state.report,
+      running: state.running.length,
+      backlog: pendingStart.page.length,
+    });
+  } else {
+    await ctx.scheduler.runAfter(0, internal.stats.calculateBacklogAndReport, {
+      startSegment: state.segmentCursors.incoming,
+      endSegment: currentSegment,
+      cursor: pendingStart.continueCursor,
+      report: state.report,
+      running: state.running.length,
+      logLevel,
+    });
+  }
 }
 
-/**
- * Warning: this should not be used from a mutation, as it will cause conflicts.
- * Use this to debug or diagnose your queue length when it's backed up.
- */
-export const queueLength = internalQuery({
-  args: {},
-  returns: v.number(),
-  handler: async (ctx) => {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    return (ctx.db.query("pendingStart") as any).count();
+export const calculateBacklogAndReport = internalMutation({
+  args: {
+    startSegment: v.int64(),
+    endSegment: v.int64(),
+    cursor: v.string(),
+    report: schema.tables.internalState.validator.fields.report,
+    running: v.number(),
+    logLevel,
+  },
+  handler: async (ctx, args) => {
+    const pendingStart = await paginator(ctx.db, schema)
+      .query("pendingStart")
+      .withIndex("segment", (q) =>
+        q.gte("segment", args.startSegment).lt("segment", args.endSegment)
+      )
+      .paginate({
+        numItems: BACKLOG_BATCH_SIZE,
+        cursor: args.cursor,
+      });
+    const console = createLogger(args.logLevel);
+    if (pendingStart.isDone) {
+      recordReport(console, {
+        ...args.report,
+        running: args.running,
+        backlog: pendingStart.page.length,
+      });
+    } else {
+      await ctx.scheduler.runAfter(
+        0,
+        internal.stats.calculateBacklogAndReport,
+        {
+          startSegment: args.startSegment,
+          endSegment: args.endSegment,
+          cursor: pendingStart.continueCursor,
+          report: args.report,
+          running: args.running,
+          logLevel: args.logLevel,
+        }
+      );
+    }
   },
 });
 
+function recordReport(
+  console: Logger,
+  report: Doc<"internalState">["report"] & { running: number; backlog: number }
+) {
+  const { completed, failed, retries } = report;
+  const withoutRetries = completed - retries;
+  const failureRate = completed ? (failed + retries) / completed : 0;
+  const permanentFailureRate = withoutRetries ? failed / withoutRetries : 0;
+  console.event("report", {
+    ...report,
+    failureRate: Number(failureRate.toFixed(4)),
+    permanentFailureRate: Number(permanentFailureRate.toFixed(4)),
+  });
+}
+
 /**
  * Warning: this should not be used from a mutation, as it will cause conflicts.
  * Use this while developing to see the state of the queue.
  */
-export const debugCounts = internalQuery({
+export const diagnostics = internalQuery({
   args: {},
   returns: v.any(),
   handler: async (ctx) => {
-    const inProgressWork =
-      (await ctx.db.query("internalState").unique())?.running.length ?? 0;
+    const global = await ctx.db.query("globals").unique();
+    const internalState = await ctx.db.query("internalState").unique();
+    const inProgressWork = internalState?.running.length ?? 0;
+    const maxParallelism = global?.maxParallelism ?? DEFAULT_MAX_PARALLELISM;
     /* eslint-disable @typescript-eslint/no-explicit-any */
     const pendingStart = await (ctx.db.query("pendingStart") as any).count();
     const pendingCompletion = await (
@@ -92,13 +182,16 @@ export const debugCounts = internalQuery({
     const pendingCancelation = await (
       ctx.db.query("pendingCancelation") as any
     ).count();
+    const runStatus = await ctx.db.query("runStatus").unique();
     /* eslint-enable @typescript-eslint/no-explicit-any */
     return {
-      pendingStart,
-      inProgressWork,
-      pendingCompletion,
-      pendingCancelation,
-      active: inProgressWork - pendingCompletion,
+      canceling: pendingCancelation,
+      waiting: pendingStart,
+      running: inProgressWork - pendingCompletion,
+      completing: pendingCompletion,
+      spareCapacity: maxParallelism - inProgressWork,
+      runStatus: runStatus?.state.kind,
+      generation: internalState?.generation,
     };
   },
 });

package/src/component/worker.ts

@@ -3,13 +3,12 @@
  * Should not touch any of loop's tables other than writing to `pendingCompletion`.
  * It is not responsible for handling retries.
  */
-import { FunctionHandle } from "convex/server";
+import type { FunctionHandle } from "convex/server";
 import { v } from "convex/values";
 import { internal } from "./_generated/api.js";
 import { internalAction, internalMutation } from "./_generated/server.js";
-import { kickMainLoop } from "./kick.js";
 import { createLogger, logLevel } from "./logging.js";
-import { nextSegment, runResult } from "./shared.js";
+import type { RunResult } from "./shared.js";
 
 export const runMutationWrapper = internalMutation({
   args: {
@@ -17,23 +16,25 @@ export const runMutationWrapper = internalMutation({
     fnHandle: v.string(),
     fnArgs: v.any(),
     logLevel,
+    attempt: v.number(),
   },
-  handler: async (ctx, { workId, fnHandle: handleStr, fnArgs, logLevel }) => {
-    const console = createLogger(logLevel);
-    const fnHandle = handleStr as FunctionHandle<"mutation">;
+  handler: async (ctx, { workId, attempt, ...args }) => {
+    const console = createLogger(args.logLevel);
+    const fnHandle = args.fnHandle as FunctionHandle<"mutation">;
     try {
-      const returnValue = await ctx.runMutation(fnHandle, fnArgs);
+      const returnValue = await ctx.runMutation(fnHandle, args.fnArgs);
       // NOTE: we could run the `saveResult` handler here, or call `ctx.runMutation`,
       // but we want the mutation to be a separate transaction to reduce the window for OCCs.
-      await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-        workId,
-        runResult: { kind: "success", returnValue },
+      await ctx.scheduler.runAfter(0, internal.complete.complete, {
+        jobs: [
+          { workId, runResult: { kind: "success", returnValue }, attempt },
+        ],
       });
     } catch (e: unknown) {
       console.error(e);
-      await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-        workId,
-        runResult: { kind: "failed", error: formatError(e) },
+      const runResult = { kind: "failed" as const, error: formatError(e) };
+      await ctx.scheduler.runAfter(0, internal.complete.complete, {
+        jobs: [{ workId, runResult, attempt }],
       });
     }
   },
@@ -52,43 +53,29 @@ export const runActionWrapper = internalAction({
     fnHandle: v.string(),
     fnArgs: v.any(),
     logLevel,
+    attempt: v.number(),
   },
-  handler: async (ctx, { workId, fnHandle: handleStr, fnArgs, logLevel }) => {
-    const console = createLogger(logLevel);
-    const fnHandle = handleStr as FunctionHandle<"action">;
+  handler: async (ctx, { workId, attempt, ...args }) => {
+    const console = createLogger(args.logLevel);
+    const fnHandle = args.fnHandle as FunctionHandle<"action">;
     try {
-      const returnValue = await ctx.runAction(fnHandle, fnArgs);
+      const returnValue = await ctx.runAction(fnHandle, args.fnArgs);
       // NOTE: we could run `ctx.runMutation`, but we want to guarantee execution,
       // and `ctx.scheduler.runAfter` won't OCC.
-      await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-        workId,
-        runResult: { kind: "success", returnValue },
+      const runResult: RunResult = { kind: "success", returnValue };
+      await ctx.scheduler.runAfter(0, internal.complete.complete, {
+        jobs: [{ workId, runResult, attempt }],
       });
     } catch (e: unknown) {
       console.error(e);
       // We let the main loop handle the retries.
-      await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-        workId,
-        runResult: { kind: "failed", error: formatError(e) },
+      const runResult: RunResult = { kind: "failed", error: formatError(e) };
+      await ctx.scheduler.runAfter(0, internal.complete.complete, {
+        jobs: [{ workId, runResult, attempt }],
       });
     }
   },
 });
 
-export const saveResult = internalMutation({
-  args: {
-    workId: v.id("work"),
-    runResult,
-  },
-  handler: async (ctx, { workId, runResult }) => {
-    await ctx.db.insert("pendingCompletion", {
-      runResult,
-      workId,
-      segment: nextSegment(),
-    });
-    await kickMainLoop(ctx, "saveResult");
-  },
-});
-
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const console = "THIS IS A REMINDER TO USE createLogger";