@convex-dev/workpool 0.2.0-beta.0 → 0.2.1

package/dist/commonjs/component/shared.js

@@ -1,6 +1,7 @@
 import { v } from "convex/values";
 import { logLevel } from "./logging.js";
-const SEGMENT_MS = 250;
+export const DEFAULT_MAX_PARALLELISM = 10;
+const SEGMENT_MS = 100;
 export const SECOND = 1000;
 export const MINUTE = 60 * SECOND;
 export const HOUR = 60 * MINUTE;
@@ -9,10 +10,10 @@ export const YEAR = 365 * DAY;
 export function toSegment(ms) {
     return BigInt(Math.floor(ms / SEGMENT_MS));
 }
-export function currentSegment() {
+export function getCurrentSegment() {
     return toSegment(Date.now());
 }
-export function nextSegment() {
+export function getNextSegment() {
     return toSegment(Date.now()) + 1n;
 }
 export function fromSegment(segment) {
@@ -44,22 +45,34 @@ export const onComplete = v.object({
 });
 export const status = v.union(v.union(v.object({
     state: v.literal("pending"),
-    attempt: v.number(),
+    previousAttempts: v.number(),
 }), v.object({
     state: v.literal("running"),
-    attempt: v.number(),
+    previousAttempts: v.number(),
 }), v.object({
     state: v.literal("finished"),
 })));
 export function boundScheduledTime(ms, console) {
     if (ms < Date.now() - YEAR) {
-        console.warn("runAt is too far in the past, defaulting to now", ms);
+        console.error("scheduled time is too old, defaulting to now", ms);
         return Date.now();
     }
     if (ms > Date.now() + 4 * YEAR) {
-        console.warn("runAt is too far in the future, defaulting to 1 year from now", ms);
+        console.error("scheduled time is too far in the future, defaulting to 1 year from now", ms);
         return Date.now() + YEAR;
     }
     return ms;
 }
+/**
+ * Returns the smaller of two bigint values.
+ */
+export function min(a, b) {
+    return a > b ? b : a;
+}
+/**
+ * Returns the larger of two bigint values.
+ */
+export function max(a, b) {
+    return a < b ? b : a;
+}
 //# sourceMappingURL=shared.js.map

package/dist/commonjs/component/shared.js.map

@@ -1 +1 @@
-{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/component/shared.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAU,QAAQ,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,CAAC;AAC3B,MAAM,CAAC,MAAM,MAAM,GAAG,EAAE,GAAG,MAAM,CAAC;AAClC,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE,GAAG,MAAM,CAAC;AAChC,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,GAAG,IAAI,CAAC;AAC7B,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,GAAG,GAAG,CAAC;AAE9B,MAAM,UAAU,SAAS,CAAC,EAAU;IAClC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,OAAO,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,UAAU,CAAC;AACtC,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;IAC1B,QAAQ;CACT,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE;IAC5B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;CACjB,CAAC,CAAC;AAkBH,mDAAmD;AACnD,MAAM,CAAC,GAAG,EAAyD,CAAC;AAEpE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAC9B,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IAC1B,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE;CACrB,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;IACzB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;CAClB,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;CAC5B,CAAC,CACH,CAAC;AAGF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;CAC7B,CAAC,CAAC;AAmBH,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAC3B,CAAC,CAAC,KAAK,CACL,CAAC,CAAC,MAAM,CAAC;IACP,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IAC3B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;CACpB,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;IACP,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IAC3B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;CACpB,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;IACP,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;CAC7B,CAAC,CACH,CACF,CAAC;AAGF,MAAM,UAAU,kBAAkB,CAAC,EAAU,EAAE,OAAe;IAC5D,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE;QAC1B,OAAO,CAAC,IAAI,CAAC,iDAAiD,EAAE,EAAE,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC;KACnB;IACD,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,EAAE;QAC9B,OAAO,CAAC,IAAI,CACV,+DAA+D,EAC/D,EAAE,CACH,CAAC;QACF,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;KAC1B;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}
+{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/component/shared.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAU,QAAQ,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAC1C,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,CAAC;AAC3B,MAAM,CAAC,MAAM,MAAM,GAAG,EAAE,GAAG,MAAM,CAAC;AAClC,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE,GAAG,MAAM,CAAC;AAChC,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,GAAG,IAAI,CAAC;AAC7B,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,GAAG,GAAG,CAAC;AAE9B,MAAM,UAAU,SAAS,CAAC,EAAU;IAClC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,UAAU,CAAC;AACtC,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;IAC1B,QAAQ;CACT,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE;IAC5B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;CACjB,CAAC,CAAC;AAkBH,mDAAmD;AACnD,MAAM,CAAC,GAAG,EAAyD,CAAC;AAEpE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAC9B,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IAC1B,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE;CACrB,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;IACzB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;CAClB,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;CAC5B,CAAC,CACH,CAAC;AAGF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;CAC7B,CAAC,CAAC;AAmBH,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAC3B,CAAC,CAAC,KAAK,CACL,CAAC,CAAC,MAAM,CAAC;IACP,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IAC3B,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE;CAC7B,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;IACP,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IAC3B,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE;CAC7B,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;IACP,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;CAC7B,CAAC,CACH,CACF,CAAC;AAGF,MAAM,UAAU,kBAAkB,CAAC,EAAU,EAAE,OAAe;IAC5D,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE;QAC1B,OAAO,CAAC,KAAK,CAAC,8CAA8C,EAAE,EAAE,CAAC,CAAC;QAClE,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC;KACnB;IACD,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,EAAE;QAC9B,OAAO,CAAC,KAAK,CACX,wEAAwE,EACxE,EAAE,CACH,CAAC;QACF,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;KAC1B;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,CAAmB,CAAI,EAAE,CAAI;IAC9C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,CAAmB,CAAI,EAAE,CAAI;IAC9C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACvB,CAAC"}

package/dist/commonjs/component/stats.d.ts

@@ -1,38 +1,45 @@
-import { Doc } from "./_generated/dataModel.js";
+import { Doc, Id } from "./_generated/dataModel.js";
+import { MutationCtx } from "./_generated/server.js";
+import { Config } from "./shared.js";
+import { Logger } from "./logging.js";
 /**
  * Record stats about work execution. Intended to be queried by Axiom or Datadog.
+ * See the [README](https://github.com/get-convex/workpool) for example queries.
  */
-/**
- * Sample axiom dashboard query:
-
-workpool
-| extend parsed_message = iff(
-    isnotnull(parse_json(trim("'", tostring(["data.message"])))),
-    parse_json(trim("'", tostring(["data.message"]))),
-    parse_json('{}')
-)
-| extend lagSinceEnqueued = parsed_message["lagSinceEnqueued"]
-| extend fnName = parsed_message["fnName"]
-| summarize avg(todouble(lagSinceEnqueued)) by bin_auto(_time), tostring(fnName)
-
- */
-export declare function recordStarted(work: Doc<"work">): string;
-export declare function recordCompleted(work: Doc<"work">, status: "success" | "failed" | "canceled"): string;
-export declare function recordReport(state: Doc<"internalState">): string;
-/**
- * Warning: this should not be used from a mutation, as it will cause conflicts.
- * Use this to debug or diagnose your queue length when it's backed up.
- */
-export declare const queueLength: import("convex/server").RegisteredQuery<"internal", {}, Promise<any>>;
+export declare function recordEnqueued(console: Logger, data: {
+    workId: Id<"work">;
+    fnName: string;
+    runAt: number;
+}): void;
+export declare function recordStarted(console: Logger, work: Doc<"work">, lagMs: number): void;
+export declare function recordCompleted(console: Logger, work: Doc<"work">, status: "success" | "failed" | "canceled" | "retrying"): void;
+export declare function generateReport(ctx: MutationCtx, console: Logger, state: Doc<"internalState">, { maxParallelism, logLevel }: Config): Promise<void>;
+export declare const calculateBacklogAndReport: import("convex/server").RegisteredMutation<"internal", {
+    report: {
+        failed: number;
+        canceled: number;
+        completed: number;
+        succeeded: number;
+        retries: number;
+        lastReportTs: number;
+    };
+    logLevel: "DEBUG" | "INFO" | "REPORT" | "WARN" | "ERROR";
+    running: number;
+    startSegment: bigint;
+    endSegment: bigint;
+    cursor: string;
+}, Promise<void>>;
 /**
  * Warning: this should not be used from a mutation, as it will cause conflicts.
  * Use this while developing to see the state of the queue.
  */
-export declare const debugCounts: import("convex/server").RegisteredQuery<"internal", {}, Promise<{
-    pendingStart: any;
-    inProgressWork: number;
-    pendingCompletion: any;
-    pendingCancelation: any;
-    active: number;
+export declare const diagnostics: import("convex/server").RegisteredQuery<"internal", {}, Promise<{
+    canceling: any;
+    waiting: any;
+    running: number;
+    completing: any;
+    spareCapacity: number;
+    runStatus: "running" | "scheduled" | "idle" | undefined;
+    generation: bigint | undefined;
 }>>;
 //# sourceMappingURL=stats.d.ts.map

package/dist/commonjs/component/stats.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../../src/component/stats.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAE,MAAM,2BAA2B,CAAC;AAGhD;;GAEG;AAEH;;;;;;;;;;;;;GAaG;AAEH,wBAAgB,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,CASvD;AAED,wBAAgB,eAAe,CAC7B,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,UAAU,GACxC,MAAM,CASR;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,eAAe,CAAC,GAAG,MAAM,CAahE;AAED;;;GAGG;AACH,eAAO,MAAM,WAAW,uEAOtB,CAAC;AAEH;;;GAGG;AACH,eAAO,MAAM,WAAW;;;;;;GAuBtB,CAAC"}
+{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../../src/component/stats.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,2BAA2B,CAAC;AACpD,OAAO,EAGL,WAAW,EACZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,MAAM,EAGP,MAAM,aAAa,CAAC;AACrB,OAAO,EAAgB,MAAM,EAAuB,MAAM,cAAc,CAAC;AAOzE;;;GAGG;AAEH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IACJ,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf,QAMF;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,KAAK,EAAE,MAAM,QASd;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,UAAU,GAAG,UAAU,QASvD;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,WAAW,EAChB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,GAAG,CAAC,eAAe,CAAC,EAC3B,EAAE,cAAc,EAAE,QAAQ,EAAE,EAAE,MAAM,iBAkCrC;AAED,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;iBAyCpC,CAAC;AAiBH;;;GAGG;AACH,eAAO,MAAM,WAAW;;;;;;;;GA4BtB,CAAC"}

package/dist/commonjs/component/stats.js

@@ -1,88 +1,146 @@
 import { v } from "convex/values";
-import { internalQuery } from "./_generated/server.js";
+import { internalMutation, internalQuery, } from "./_generated/server.js";
+import { DEFAULT_MAX_PARALLELISM, getCurrentSegment, } from "./shared.js";
+import { createLogger, logLevel, shouldLog } from "./logging.js";
+import { internal } from "./_generated/api.js";
+import schema from "./schema.js";
+import { paginator } from "convex-helpers/server/pagination";
+const BACKLOG_BATCH_SIZE = 100;
 /**
  * Record stats about work execution. Intended to be queried by Axiom or Datadog.
+ * See the [README](https://github.com/get-convex/workpool) for example queries.
  */
-/**
- * Sample axiom dashboard query:
-
-workpool
-| extend parsed_message = iff(
-    isnotnull(parse_json(trim("'", tostring(["data.message"])))),
-    parse_json(trim("'", tostring(["data.message"]))),
-    parse_json('{}')
-)
-| extend lagSinceEnqueued = parsed_message["lagSinceEnqueued"]
-| extend fnName = parsed_message["fnName"]
-| summarize avg(todouble(lagSinceEnqueued)) by bin_auto(_time), tostring(fnName)
-
- */
-export function recordStarted(work) {
-    return JSON.stringify({
+export function recordEnqueued(console, data) {
+    console.event("enqueued", {
+        ...data,
+        enqueuedAt: Date.now(),
+    });
+}
+export function recordStarted(console, work, lagMs) {
+    console.event("started", {
         workId: work._id,
-        event: "started",
         fnName: work.fnName,
         enqueuedAt: work._creationTime,
         startedAt: Date.now(),
-        lagSinceEnqueued: Date.now() - work._creationTime,
+        startLag: lagMs,
     });
 }
-export function recordCompleted(work, status) {
-    return JSON.stringify({
+export function recordCompleted(console, work, status) {
+    console.event("completed", {
         workId: work._id,
-        event: "completed",
         fnName: work.fnName,
         completedAt: Date.now(),
+        attempts: work.attempts,
         status,
-        lagSinceEnqueued: Date.now() - work._creationTime,
     });
 }
-export function recordReport(state) {
-    const { completed, succeeded, failed, retries, canceled } = state.report;
-    const withoutRetries = completed - retries;
-    return JSON.stringify({
-        event: "report",
-        completed,
-        succeeded,
-        failed,
-        retries,
-        canceled,
-        failureRate: completed ? (failed + retries) / completed : 0,
-        permanentFailureRate: withoutRetries ? failed / withoutRetries : 0,
+export async function generateReport(ctx, console, state, { maxParallelism, logLevel }) {
+    if (!shouldLog(logLevel, "REPORT")) {
+        // Don't waste time if we're not going to log.
+        return;
+    }
+    const currentSegment = getCurrentSegment();
+    const pendingStart = await paginator(ctx.db, schema)
+        .query("pendingStart")
+        .withIndex("segment", (q) => q
+        .gte("segment", state.segmentCursors.incoming)
+        .lt("segment", currentSegment))
+        .paginate({
+        numItems: maxParallelism,
+        cursor: null,
     });
+    if (pendingStart.isDone) {
+        recordReport(console, {
+            ...state.report,
+            running: state.running.length,
+            backlog: pendingStart.page.length,
+        });
+    }
+    else {
+        await ctx.scheduler.runAfter(0, internal.stats.calculateBacklogAndReport, {
+            startSegment: state.segmentCursors.incoming,
+            endSegment: currentSegment,
+            cursor: pendingStart.continueCursor,
+            report: state.report,
+            running: state.running.length,
+            logLevel,
+        });
+    }
 }
-/**
- * Warning: this should not be used from a mutation, as it will cause conflicts.
- * Use this to debug or diagnose your queue length when it's backed up.
- */
-export const queueLength = internalQuery({
-    args: {},
-    returns: v.number(),
-    handler: async (ctx) => {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        return ctx.db.query("pendingStart").count();
+export const calculateBacklogAndReport = internalMutation({
+    args: {
+        startSegment: v.int64(),
+        endSegment: v.int64(),
+        cursor: v.string(),
+        report: schema.tables.internalState.validator.fields.report,
+        running: v.number(),
+        logLevel,
+    },
+    handler: async (ctx, args) => {
+        const pendingStart = await paginator(ctx.db, schema)
+            .query("pendingStart")
+            .withIndex("segment", (q) => q.gte("segment", args.startSegment).lt("segment", args.endSegment))
+            .paginate({
+            numItems: BACKLOG_BATCH_SIZE,
+            cursor: args.cursor,
+        });
+        const console = createLogger(args.logLevel);
+        if (pendingStart.isDone) {
+            recordReport(console, {
+                ...args.report,
+                running: args.running,
+                backlog: pendingStart.page.length,
+            });
+        }
+        else {
+            await ctx.scheduler.runAfter(0, internal.stats.calculateBacklogAndReport, {
+                startSegment: args.startSegment,
+                endSegment: args.endSegment,
+                cursor: pendingStart.continueCursor,
+                report: args.report,
+                running: args.running,
+                logLevel: args.logLevel,
+            });
+        }
     },
 });
+function recordReport(console, report) {
+    const { completed, failed, retries } = report;
+    const withoutRetries = completed - retries;
+    const failureRate = completed ? (failed + retries) / completed : 0;
+    const permanentFailureRate = withoutRetries ? failed / withoutRetries : 0;
+    console.event("report", {
+        ...report,
+        failureRate: Number(failureRate.toFixed(4)),
+        permanentFailureRate: Number(permanentFailureRate.toFixed(4)),
+    });
+}
 /**
  * Warning: this should not be used from a mutation, as it will cause conflicts.
  * Use this while developing to see the state of the queue.
  */
-export const debugCounts = internalQuery({
+export const diagnostics = internalQuery({
     args: {},
     returns: v.any(),
     handler: async (ctx) => {
-        const inProgressWork = (await ctx.db.query("internalState").unique())?.running.length ?? 0;
+        const global = await ctx.db.query("globals").unique();
+        const internalState = await ctx.db.query("internalState").unique();
+        const inProgressWork = internalState?.running.length ?? 0;
+        const maxParallelism = global?.maxParallelism ?? DEFAULT_MAX_PARALLELISM;
         /* eslint-disable @typescript-eslint/no-explicit-any */
         const pendingStart = await ctx.db.query("pendingStart").count();
         const pendingCompletion = await ctx.db.query("pendingCompletion").count();
         const pendingCancelation = await ctx.db.query("pendingCancelation").count();
+        const runStatus = await ctx.db.query("runStatus").unique();
         /* eslint-enable @typescript-eslint/no-explicit-any */
         return {
-            pendingStart,
-            inProgressWork,
-            pendingCompletion,
-            pendingCancelation,
-            active: inProgressWork - pendingCompletion,
+            canceling: pendingCancelation,
+            waiting: pendingStart,
+            running: inProgressWork - pendingCompletion,
+            completing: pendingCompletion,
+            spareCapacity: maxParallelism - inProgressWork,
+            runStatus: runStatus?.state.kind,
+            generation: internalState?.generation,
         };
     },
 });

package/dist/commonjs/component/stats.js.map

@@ -1 +1 @@
-{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../../src/component/stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD;;GAEG;AAEH;;;;;;;;;;;;;GAaG;AAEH,MAAM,UAAU,aAAa,CAAC,IAAiB;IAC7C,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,KAAK,EAAE,SAAS;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,aAAa;QAC9B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,gBAAgB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa;KAClD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,IAAiB,EACjB,MAAyC;IAEzC,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,KAAK,EAAE,WAAW;QAClB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,MAAM;QACN,gBAAgB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa;KAClD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAA2B;IACtD,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC;IACzE,MAAM,cAAc,GAAG,SAAS,GAAG,OAAO,CAAC;IAC3C,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,KAAK,EAAE,QAAQ;QACf,SAAS;QACT,SAAS;QACT,MAAM;QACN,OAAO;QACP,QAAQ;QACR,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC3D,oBAAoB,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;KACnE,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;IACvC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,8DAA8D;QAC9D,OAAQ,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAS,CAAC,KAAK,EAAE,CAAC;IACvD,CAAC;CACF,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;IACvC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE;IAChB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,cAAc,GAClB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QACtE,uDAAuD;QACvD,MAAM,YAAY,GAAG,MAAO,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAS,CAAC,KAAK,EAAE,CAAC;QACzE,MAAM,iBAAiB,GAAG,MACxB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,mBAAmB,CACjC,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,kBAAkB,GAAG,MACzB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,oBAAoB,CAClC,CAAC,KAAK,EAAE,CAAC;QACV,sDAAsD;QACtD,OAAO;YACL,YAAY;YACZ,cAAc;YACd,iBAAiB;YACjB,kBAAkB;YAClB,MAAM,EAAE,cAAc,GAAG,iBAAiB;SAC3C,CAAC;IACJ,CAAC;CACF,CAAC,CAAC"}
+{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../../src/component/stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,OAAO,EACL,gBAAgB,EAChB,aAAa,GAEd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAU,QAAQ,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,kCAAkC,CAAC;AAE7D,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B;;;GAGG;AAEH,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,IAIC;IAED,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE;QACxB,GAAG,IAAI;QACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;KACvB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,OAAe,EACf,IAAiB,EACjB,KAAa;IAEb,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE;QACvB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,aAAa;QAC9B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,IAAiB,EACjB,MAAsD;IAEtD,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE;QACzB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM;KACP,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAgB,EAChB,OAAe,EACf,KAA2B,EAC3B,EAAE,cAAc,EAAE,QAAQ,EAAU;IAEpC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE;QAClC,8CAA8C;QAC9C,OAAO;KACR;IACD,MAAM,cAAc,GAAG,iBAAiB,EAAE,CAAC;IAC3C,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC;SACjD,KAAK,CAAC,cAAc,CAAC;SACrB,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC;SACE,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC;SAC7C,EAAE,CAAC,SAAS,EAAE,cAAc,CAAC,CACjC;SACA,QAAQ,CAAC;QACR,QAAQ,EAAE,cAAc;QACxB,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;IACL,IAAI,YAAY,CAAC,MAAM,EAAE;QACvB,YAAY,CAAC,OAAO,EAAE;YACpB,GAAG,KAAK,CAAC,MAAM;YACf,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YAC7B,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM;SAClC,CAAC,CAAC;KACJ;SAAM;QACL,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,yBAAyB,EAAE;YACxE,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ;YAC3C,UAAU,EAAE,cAAc;YAC1B,MAAM,EAAE,YAAY,CAAC,cAAc;YACnC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YAC7B,QAAQ;SACT,CAAC,CAAC;KACJ;AACH,CAAC;AAED,MAAM,CAAC,MAAM,yBAAyB,GAAG,gBAAgB,CAAC;IACxD,IAAI,EAAE;QACJ,YAAY,EAAE,CAAC,CAAC,KAAK,EAAE;QACvB,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;QAClB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM;QAC3D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,QAAQ;KACT;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC;aACjD,KAAK,CAAC,cAAc,CAAC;aACrB,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CACnE;aACA,QAAQ,CAAC;YACR,QAAQ,EAAE,kBAAkB;YAC5B,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QACL,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,YAAY,CAAC,MAAM,EAAE;YACvB,YAAY,CAAC,OAAO,EAAE;gBACpB,GAAG,IAAI,CAAC,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM;aAClC,CAAC,CAAC;SACJ;aAAM;YACL,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAC1B,CAAC,EACD,QAAQ,CAAC,KAAK,CAAC,yBAAyB,EACxC;gBACE,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,MAAM,EAAE,YAAY,CAAC,cAAc;gBACnC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CACF,CAAC;SACH;IACH,CAAC;CACF,CAAC,CAAC;AAEH,SAAS,YAAY,CACnB,OAAe,EACf,MAA6E;IAE7E,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAC9C,MAAM,cAAc,GAAG,SAAS,GAAG,OAAO,CAAC;IAC3C,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,MAAM,oBAAoB,GAAG,cAAc,CAAC,CAAC,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE;QACtB,GAAG,MAAM;QACT,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC3C,oBAAoB,EAAE,MAAM,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;KAC9D,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;IACvC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE;IAChB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;QACtD,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC;QACnE,MAAM,cAAc,GAAG,aAAa,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAC1D,MAAM,cAAc,GAAG,MAAM,EAAE,cAAc,IAAI,uBAAuB,CAAC;QACzE,uDAAuD;QACvD,MAAM,YAAY,GAAG,MAAO,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAS,CAAC,KAAK,EAAE,CAAC;QACzE,MAAM,iBAAiB,GAAG,MACxB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,mBAAmB,CACjC,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,kBAAkB,GAAG,MACzB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,oBAAoB,CAClC,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3D,sDAAsD;QACtD,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,OAAO,EAAE,YAAY;YACrB,OAAO,EAAE,cAAc,GAAG,iBAAiB;YAC3C,UAAU,EAAE,iBAAiB;YAC7B,aAAa,EAAE,cAAc,GAAG,cAAc;YAC9C,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI;YAChC,UAAU,EAAE,aAAa,EAAE,UAAU;SACtC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC"}

package/dist/commonjs/component/worker.d.ts

@@ -1,25 +1,15 @@
 export declare const runMutationWrapper: import("convex/server").RegisteredMutation<"internal", {
-    logLevel: "DEBUG" | "INFO" | "WARN" | "ERROR";
+    logLevel: "DEBUG" | "INFO" | "REPORT" | "WARN" | "ERROR";
     fnHandle: string;
     workId: import("convex/values").GenericId<"work">;
     fnArgs: any;
+    attempt: number;
 }, Promise<void>>;
 export declare const runActionWrapper: import("convex/server").RegisteredAction<"internal", {
-    logLevel: "DEBUG" | "INFO" | "WARN" | "ERROR";
+    logLevel: "DEBUG" | "INFO" | "REPORT" | "WARN" | "ERROR";
     fnHandle: string;
     workId: import("convex/values").GenericId<"work">;
     fnArgs: any;
-}, Promise<void>>;
-export declare const saveResult: import("convex/server").RegisteredMutation<"internal", {
-    workId: import("convex/values").GenericId<"work">;
-    runResult: {
-        kind: "success";
-        returnValue: any;
-    } | {
-        kind: "failed";
-        error: string;
-    } | {
-        kind: "canceled";
-    };
+    attempt: number;
 }, Promise<void>>;
 //# sourceMappingURL=worker.d.ts.map

package/dist/commonjs/component/worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../../src/component/worker.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,kBAAkB;;;;;iBA0B7B,CAAC;AASH,eAAO,MAAM,gBAAgB;;;;;iBA2B3B,CAAC;AAEH,eAAO,MAAM,UAAU;;;;;;;;;;;iBAarB,CAAC"}
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../../src/component/worker.ts"],"names":[],"mappings":"AAYA,eAAO,MAAM,kBAAkB;;;;;;iBA4B7B,CAAC;AASH,eAAO,MAAM,gBAAgB;;;;;;iBA4B3B,CAAC"}

package/dist/commonjs/component/worker.js

@@ -1,33 +1,33 @@
 import { v } from "convex/values";
 import { internal } from "./_generated/api.js";
 import { internalAction, internalMutation } from "./_generated/server.js";
-import { kickMainLoop } from "./kick.js";
 import { createLogger, logLevel } from "./logging.js";
-import { nextSegment, runResult } from "./shared.js";
 export const runMutationWrapper = internalMutation({
     args: {
         workId: v.id("work"),
         fnHandle: v.string(),
         fnArgs: v.any(),
         logLevel,
+        attempt: v.number(),
     },
-    handler: async (ctx, { workId, fnHandle: handleStr, fnArgs, logLevel }) => {
-        const console = createLogger(logLevel);
-        const fnHandle = handleStr;
+    handler: async (ctx, { workId, attempt, ...args }) => {
+        const console = createLogger(args.logLevel);
+        const fnHandle = args.fnHandle;
         try {
-            const returnValue = await ctx.runMutation(fnHandle, fnArgs);
+            const returnValue = await ctx.runMutation(fnHandle, args.fnArgs);
             // NOTE: we could run the `saveResult` handler here, or call `ctx.runMutation`,
             // but we want the mutation to be a separate transaction to reduce the window for OCCs.
-            await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-                workId,
-                runResult: { kind: "success", returnValue },
+            await ctx.scheduler.runAfter(0, internal.complete.complete, {
+                jobs: [
+                    { workId, runResult: { kind: "success", returnValue }, attempt },
+                ],
             });
         }
         catch (e) {
             console.error(e);
-            await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-                workId,
-                runResult: { kind: "failed", error: formatError(e) },
+            const runResult = { kind: "failed", error: formatError(e) };
+            await ctx.scheduler.runAfter(0, internal.complete.complete, {
+                jobs: [{ workId, runResult, attempt }],
             });
         }
     },
@@ -44,43 +44,30 @@ export const runActionWrapper = internalAction({
         fnHandle: v.string(),
         fnArgs: v.any(),
         logLevel,
+        attempt: v.number(),
     },
-    handler: async (ctx, { workId, fnHandle: handleStr, fnArgs, logLevel }) => {
-        const console = createLogger(logLevel);
-        const fnHandle = handleStr;
+    handler: async (ctx, { workId, attempt, ...args }) => {
+        const console = createLogger(args.logLevel);
+        const fnHandle = args.fnHandle;
         try {
-            const returnValue = await ctx.runAction(fnHandle, fnArgs);
+            const returnValue = await ctx.runAction(fnHandle, args.fnArgs);
             // NOTE: we could run `ctx.runMutation`, but we want to guarantee execution,
             // and `ctx.scheduler.runAfter` won't OCC.
-            await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-                workId,
-                runResult: { kind: "success", returnValue },
+            const runResult = { kind: "success", returnValue };
+            await ctx.scheduler.runAfter(0, internal.complete.complete, {
+                jobs: [{ workId, runResult, attempt }],
             });
         }
         catch (e) {
             console.error(e);
             // We let the main loop handle the retries.
-            await ctx.scheduler.runAfter(0, internal.worker.saveResult, {
-                workId,
-                runResult: { kind: "failed", error: formatError(e) },
+            const runResult = { kind: "failed", error: formatError(e) };
+            await ctx.scheduler.runAfter(0, internal.complete.complete, {
+                jobs: [{ workId, runResult, attempt }],
             });
         }
     },
 });
-export const saveResult = internalMutation({
-    args: {
-        workId: v.id("work"),
-        runResult,
-    },
-    handler: async (ctx, { workId, runResult }) => {
-        await ctx.db.insert("pendingCompletion", {
-            runResult,
-            workId,
-            segment: nextSegment(),
-        });
-        await kickMainLoop(ctx, "saveResult");
-    },
-});
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const console = "THIS IS A REMINDER TO USE createLogger";
 //# sourceMappingURL=worker.js.map

package/dist/commonjs/component/worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/component/worker.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,CAAC,MAAM,kBAAkB,GAAG,gBAAgB,CAAC;IACjD,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE;QACf,QAAQ;KACT;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;QACxE,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,QAAQ,GAAG,SAAuC,CAAC;QACzD,IAAI;YACF,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC5D,+EAA+E;YAC/E,uFAAuF;YACvF,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE;gBAC1D,MAAM;gBACN,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE;aAC5C,CAAC,CAAC;SACJ;QAAC,OAAO,CAAU,EAAE;YACnB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjB,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE;gBAC1D,MAAM;gBACN,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE;aACrD,CAAC,CAAC;SACJ;IACH,CAAC;CACF,CAAC,CAAC;AAEH,SAAS,WAAW,CAAC,CAAU;IAC7B,IAAI,CAAC,YAAY,KAAK,EAAE;QACtB,OAAO,CAAC,CAAC,OAAO,CAAC;KAClB;IACD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,cAAc,CAAC;IAC7C,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE;QACf,QAAQ;KACT;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;QACxE,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,QAAQ,GAAG,SAAqC,CAAC;QACvD,IAAI;YACF,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC1D,4EAA4E;YAC5E,0CAA0C;YAC1C,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE;gBAC1D,MAAM;gBACN,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE;aAC5C,CAAC,CAAC;SACJ;QAAC,OAAO,CAAU,EAAE;YACnB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjB,2CAA2C;YAC3C,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE;gBAC1D,MAAM;gBACN,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE;aACrD,CAAC,CAAC;SACJ;IACH,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;IACzC,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,SAAS;KACV;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE;QAC5C,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACvC,SAAS;YACT,MAAM;YACN,OAAO,EAAE,WAAW,EAAE;SACvB,CAAC,CAAC;QACH,MAAM,YAAY,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACxC,CAAC;CACF,CAAC,CAAC;AAEH,6DAA6D;AAC7D,MAAM,OAAO,GAAG,wCAAwC,CAAC"}
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/component/worker.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAGtD,MAAM,CAAC,MAAM,kBAAkB,GAAG,gBAAgB,CAAC;IACjD,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE;QACf,QAAQ;QACR,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;KACpB;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE;QACnD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAsC,CAAC;QAC7D,IAAI;YACF,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACjE,+EAA+E;YAC/E,uFAAuF;YACvF,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC1D,IAAI,EAAE;oBACJ,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE;iBACjE;aACF,CAAC,CAAC;SACJ;QAAC,OAAO,CAAU,EAAE;YACnB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjB,MAAM,SAAS,GAAG,EAAE,IAAI,EAAE,QAAiB,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YACrE,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC1D,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;aACvC,CAAC,CAAC;SACJ;IACH,CAAC;CACF,CAAC,CAAC;AAEH,SAAS,WAAW,CAAC,CAAU;IAC7B,IAAI,CAAC,YAAY,KAAK,EAAE;QACtB,OAAO,CAAC,CAAC,OAAO,CAAC;KAClB;IACD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,cAAc,CAAC;IAC7C,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE;QACf,QAAQ;QACR,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;KACpB;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE;QACnD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAoC,CAAC;QAC3D,IAAI;YACF,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC/D,4EAA4E;YAC5E,0CAA0C;YAC1C,MAAM,SAAS,GAAc,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;YAC9D,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC1D,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;aACvC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAU,EAAE;YACnB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjB,2CAA2C;YAC3C,MAAM,SAAS,GAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC1D,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;aACvC,CAAC,CAAC;SACJ;IACH,CAAC;CACF,CAAC,CAAC;AAEH,6DAA6D;AAC7D,MAAM,OAAO,GAAG,wCAAwC,CAAC"}

package/dist/esm/client/index.d.ts

@@ -1,10 +1,10 @@
 import { DefaultFunctionArgs, FunctionReference, FunctionVisibility } from "convex/server";
 import { VString } from "convex/values";
 import { Mounts } from "../component/_generated/api.js";
-import { runResult as runResultValidator, RunResult, type RetryBehavior, Status } from "../component/shared.js";
 import { type LogLevel } from "../component/logging.js";
+import { runResult as resultValidator, type RetryBehavior, RunResult, Status } from "../component/shared.js";
 import { RunMutationCtx, RunQueryCtx, UseApi } from "./utils.js";
-export { runResultValidator, type RunResult };
+export { resultValidator, type RunResult };
 export declare const DEFAULT_RETRY_BEHAVIOR: RetryBehavior;
 export type WorkId = string & {
     __isWorkId: true;
@@ -32,17 +32,20 @@ export declare class Workpool {
         maxParallelism?: number;
         /** How much to log. This is updated on each call to `enqueue*`,
          * `status`, or `cancel*`.
-         * Default is WARN.
-         * With INFO, you can see events for started and completed work, which can
-         * be parsed by tools like [Axiom](https://axiom.co) for monitoring.
+         * Default is REPORT, which logs warnings, errors, and a periodic report.
+         * With INFO, you can also see events for started and completed work.
+         * Stats generated can be parsed by tools like
+         * [Axiom](https://axiom.co) for monitoring.
          * With DEBUG, you can see timers and internal events for work being
          * scheduled.
          */
         logLevel?: LogLevel;
-        /** Default retry behavior for enqueued actions. */
+        /** Default retry behavior for enqueued actions.
+         * See {@link RetryBehavior}.
+         */
         defaultRetryBehavior?: RetryBehavior;
         /** Whether to retry actions that fail by default. Default: false.
-         * NOTE: Only do this if your actions are idempotent.
+         * NOTE: Only enable this if your actions are idempotent.
          * See the docs (README.md) for more details.
          */
         retryActionsByDefault?: boolean;
@@ -79,8 +82,30 @@ export declare class Workpool {
      *   and scheduling via `runAt` or `runAfter`.
      */
     enqueueMutation<Args extends DefaultFunctionArgs, ReturnType>(ctx: RunMutationCtx, fn: FunctionReference<"mutation", FunctionVisibility, Args, ReturnType>, fnArgs: Args, options?: CallbackOptions & SchedulerOptions): Promise<WorkId>;
+    /**
+     * Cancels a work item. If it's already started, it will be allowed to finish
+     * but will not be retried.
+     *
+     * @param ctx - The mutation or action context that can call ctx.runMutation.
+     * @param id - The ID of the work to cancel.
+     */
     cancel(ctx: RunMutationCtx, id: WorkId): Promise<void>;
+    /**
+     * Cancels all pending work items. See {@link cancel}.
+     *
+     * @param ctx - The mutation or action context that can call ctx.runMutation.
+     */
     cancelAll(ctx: RunMutationCtx): Promise<void>;
+    /**
+     * Gets the status of a work item.
+     *
+     * @param ctx - The query context that can call ctx.runQuery.
+     * @param id - The ID of the work to get the status of.
+     * @returns The status of the work item. One of:
+     * - `{ state: "pending", previousAttempts: number }`
+     * - `{ state: "running", previousAttempts: number }`
+     * - `{ state: "finished" }`
+     */
     status(ctx: RunQueryCtx, id: WorkId): Promise<Status>;
 }
 export type SchedulerOptions = {
@@ -108,7 +133,7 @@ export type CallbackOptions = {
      *  args: {
      *    workId: workIdValidator,
      *    context: v.any(),
-     *    result: runResult,
+     *    result: resultValidator,
      *  },
      *  handler: async (ctx, args) => {
      *    console.log(args.result, "Got Context back -> ", args.context, Date.now() - args.context);

package/dist/esm/client/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/client/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,EAEnB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAK,OAAO,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,gCAAgC,CAAC;AACxD,OAAO,EAEL,SAAS,IAAI,kBAAkB,EAC/B,SAAS,EACT,KAAK,aAAa,EAElB,MAAM,EAEP,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,KAAK,QAAQ,EAAY,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAGjE,OAAO,EAAE,kBAAkB,EAAE,KAAK,SAAS,EAAE,CAAC;AAG9C,eAAO,MAAM,sBAAsB,EAAE,aAIpC,CAAC;AACF,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG;IAAE,UAAU,EAAE,IAAI,CAAA;CAAE,CAAC;AACnD,eAAO,MAAM,eAAe,6BAAgC,CAAC;AAE7D,qBAAa,QAAQ;IAajB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,OAAO;IAbjB;;;;;;;;;;OAUG;gBAEO,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,qCAAqC;IAChE,OAAO,EAAE;QACf;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB;;;;;;;WAOG;QACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;QACpB,mDAAmD;QACnD,oBAAoB,CAAC,EAAE,aAAa,CAAC;QACrC;;;WAGG;QACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;KACjC;IAEH;;;;;;;;;OASG;IACG,aAAa,CAAC,IAAI,SAAS,mBAAmB,EAAE,UAAU,EAC9D,GAAG,EAAE,cAAc,EACnB,EAAE,EAAE,iBAAiB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,UAAU,CAAC,EACrE,MAAM,EAAE,IAAI,EACZ,OAAO,CAAC,EAAE;QACR;;;;WAIG;QACH,KAAK,CAAC,EAAE,OAAO,GAAG,aAAa,CAAC;KACjC,GAAG,eAAe,GACjB,gBAAgB,GACjB,OAAO,CAAC,MAAM,CAAC;IAuBlB;;;;;;;;;;;;OAYG;IACG,eAAe,CAAC,IAAI,SAAS,mBAAmB,EAAE,UAAU,EAChE,GAAG,EAAE,cAAc,EACnB,EAAE,EAAE,iBAAiB,CAAC,UAAU,EAAE,kBAAkB,EAAE,IAAI,EAAE,UAAU,CAAC,EACvE,MAAM,EAAE,IAAI,EACZ,OAAO,CAAC,EAAE,eAAe,GAAG,gBAAgB,GAC3C,OAAO,CAAC,MAAM,CAAC;IASZ,MAAM,CAAC,GAAG,EAAE,cAAc,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMtD,SAAS,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAK7C,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAG5D;AAgCD,MAAM,MAAM,gBAAgB,GACxB;IACE;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEN,MAAM,MAAM,eAAe,GAAG;IAC5B;;;;;;;;;;;;;;;;OAgBG;IACH,UAAU,CAAC,EAAE,iBAAiB,CAC5B,UAAU,EACV,kBAAkB,EAClB,cAAc,CACf,GAAG,IAAI,CAAC;IAET;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,MAAM,EAAE,SAAS,CAAC;CACnB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/client/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,EAEnB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAK,OAAO,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,gCAAgC,CAAC;AACxD,OAAO,EAEL,KAAK,QAAQ,EAEd,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAIL,SAAS,IAAI,eAAe,EAC5B,KAAK,aAAa,EAClB,SAAS,EAET,MAAM,EACP,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,KAAK,SAAS,EAAE,CAAC;AAG3C,eAAO,MAAM,sBAAsB,EAAE,aAIpC,CAAC;AACF,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG;IAAE,UAAU,EAAE,IAAI,CAAA;CAAE,CAAC;AACnD,eAAO,MAAM,eAAe,6BAAgC,CAAC;AAE7D,qBAAa,QAAQ;IAajB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,OAAO;IAbjB;;;;;;;;;;OAUG;gBAEO,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,qCAAqC;IAChE,OAAO,EAAE;QACf;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB;;;;;;;;WAQG;QACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;QACpB;;WAEG;QACH,oBAAoB,CAAC,EAAE,aAAa,CAAC;QACrC;;;WAGG;QACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;KACjC;IAEH;;;;;;;;;OASG;IACG,aAAa,CAAC,IAAI,SAAS,mBAAmB,EAAE,UAAU,EAC9D,GAAG,EAAE,cAAc,EACnB,EAAE,EAAE,iBAAiB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,UAAU,CAAC,EACrE,MAAM,EAAE,IAAI,EACZ,OAAO,CAAC,EAAE;QACR;;;;WAIG;QACH,KAAK,CAAC,EAAE,OAAO,GAAG,aAAa,CAAC;KACjC,GAAG,eAAe,GACjB,gBAAgB,GACjB,OAAO,CAAC,MAAM,CAAC;IAuBlB;;;;;;;;;;;;OAYG;IACG,eAAe,CAAC,IAAI,SAAS,mBAAmB,EAAE,UAAU,EAChE,GAAG,EAAE,cAAc,EACnB,EAAE,EAAE,iBAAiB,CAAC,UAAU,EAAE,kBAAkB,EAAE,IAAI,EAAE,UAAU,CAAC,EACvE,MAAM,EAAE,IAAI,EACZ,OAAO,CAAC,EAAE,eAAe,GAAG,gBAAgB,GAC3C,OAAO,CAAC,MAAM,CAAC;IAgBlB;;;;;;OAMG;IACG,MAAM,CAAC,GAAG,EAAE,cAAc,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAM5D;;;;OAIG;IACG,SAAS,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAKnD;;;;;;;;;OASG;IACG,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAG5D;AAED,MAAM,MAAM,gBAAgB,GACxB;IACE;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEN,MAAM,MAAM,eAAe,GAAG;IAC5B;;;;;;;;;;;;;;;;OAgBG;IACH,UAAU,CAAC,EAAE,iBAAiB,CAC5B,UAAU,EACV,kBAAkB,EAClB,cAAc,CACf,GAAG,IAAI,CAAC;IAET;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,MAAM,EAAE,SAAS,CAAC;CACnB,CAAC"}