@convex-dev/workpool 0.1.0

package/src/component/logging.ts

@@ -0,0 +1,59 @@
+import { v } from "convex/values";
+
+import { Infer } from "convex/values";
+
+export const logLevel = v.union(
+  v.literal("DEBUG"),
+  v.literal("INFO"),
+  v.literal("WARN"),
+  v.literal("ERROR")
+);
+export type LogLevel = Infer<typeof logLevel>;
+
+export type Logger = {
+  debug: (...args: unknown[]) => void;
+  info: (...args: unknown[]) => void;
+  warn: (...args: unknown[]) => void;
+  error: (...args: unknown[]) => void;
+  time: (label: string) => void;
+  timeEnd: (label: string) => void;
+};
+
+export function createLogger(level: LogLevel): Logger {
+  const levelIndex = ["DEBUG", "INFO", "WARN", "ERROR"].indexOf(level);
+  if (levelIndex === -1) {
+    throw new Error(`Invalid log level: ${level}`);
+  }
+  return {
+    debug: (...args: unknown[]) => {
+      if (levelIndex <= 0) {
+        console.debug(...args);
+      }
+    },
+    info: (...args: unknown[]) => {
+      if (levelIndex <= 1) {
+        console.info(...args);
+      }
+    },
+    warn: (...args: unknown[]) => {
+      if (levelIndex <= 2) {
+        console.warn(...args);
+      }
+    },
+    error: (...args: unknown[]) => {
+      if (levelIndex <= 3) {
+        console.error(...args);
+      }
+    },
+    time: (label: string) => {
+      if (levelIndex <= 0) {
+        console.time(label);
+      }
+    },
+    timeEnd: (label: string) => {
+      if (levelIndex <= 0) {
+        console.timeEnd(label);
+      }
+    },
+  };
+}

package/src/component/schema.ts

@@ -0,0 +1,103 @@
+import { defineSchema, defineTable } from "convex/server";
+import { Infer, v } from "convex/values";
+import { logLevel } from "./logging";
+
+export const completionStatus = v.union(
+  v.literal("success"),
+  v.literal("error"),
+  v.literal("canceled"),
+  v.literal("timeout")
+);
+export type CompletionStatus = Infer<typeof completionStatus>;
+
+/**
+Data flow:
+
+- The mutation `mainLoop` runs periodically and serially.
+- Several tables act as queues, with client-driven mutations enqueueing at high
+  timestamps and `mainLoop` popping at low timestamps:
+  pendingStart, pendingCompletion, and pendingCancelation.
+  - The `enqueue` mutation writes to pendingStart.
+  - The `cancel` mutation writes to pendingCancelation.
+  - The `saveResult` mutation, run as part of scheduled work, writes to pendingCompletion.
+- mainLoop processes the queues:
+  - pendingStart => inProgressWork.
+  - pendingCompletion and pendingCancelation => completedWork.
+  - inProgressWork that finishes uncleanly (timeout or system failure) => completedWork.
+- `mainLoop` schedules itself to run.
+- `enqueue`, `cancel`, and `saveResult` mutations check when `mainLoop` is scheduled to run,
+  and if it's too far in the future, they schedule it to run sooner.
+- `status` query reads from pendingWork and completedWork.
+- `cleanup` mutation deletes old rows from completedWork.
+
+To avoid OCCs, we restrict which mutations can read and write from each table:
+- pools: read by all, written only when static Workpool options change.
+- mainLoop (table): read by all, written mostly by `mainLoop`.
+  If `mainLoop` will not run for a while, mainLoop table is written by `enqueue`, `cancel`, or `saveResult`.
+- pendingWork: `enqueue` inserts at high timestamps, `mainLoop` pops at low timestamps. `status` query does point-reads.
+- pendingCompletion: `saveResult` inserts at high timestamps, `mainLoop` pops at low timestamps.
+- pendingCancelation: `cancel` inserts at high timestamps, `mainLoop` pops at low timestamps.
+- inProgressWork: `mainLoop` inserts, reads all, and deletes.
+- completedWork: `mainLoop` inserts at hight timestamps, `status` query reads, `cleanup` deletes at low timestamps.
+
+ */
+
+export default defineSchema({
+  // Statically configured, singleton.
+  pool: defineTable({
+    maxParallelism: v.number(),
+    statusTtl: v.number(),
+    logLevel,
+  }),
+
+  mainLoop: defineTable({
+    state: v.union(
+      v.object({ kind: v.literal("running") }),
+      v.object({
+        kind: v.literal("scheduled"),
+        runAtTime: v.number(),
+        fn: v.id("_scheduled_functions"),
+      }),
+      v.object({ kind: v.literal("idle") })
+    ),
+  }),
+
+  work: defineTable({
+    fnType: v.union(v.literal("action"), v.literal("mutation")),
+    fnHandle: v.string(),
+    fnName: v.string(),
+    fnArgs: v.any(),
+  }),
+
+  pendingStart: defineTable({
+    workId: v.id("work"),
+  }).index("workId", ["workId"]),
+  pendingCompletion: defineTable({
+    generation: v.number(),
+    completionStatus,
+    workId: v.id("work"),
+  })
+    .index("workId", ["workId"])
+    .index("generation", ["generation"]),
+  pendingCancelation: defineTable({
+    workId: v.id("work"),
+  }),
+
+  inProgressWork: defineTable({
+    running: v.id("_scheduled_functions"),
+    timeoutMs: v.union(v.number(), v.null()),
+    workId: v.id("work"),
+  }).index("workId", ["workId"]),
+  inProgressCount: defineTable({
+    count: v.number(),
+  }),
+
+  completedWork: defineTable({
+    completionStatus,
+    workId: v.id("work"),
+  }).index("workId", ["workId"]),
+
+  completionGeneration: defineTable({
+    generation: v.number(),
+  }),
+});

package/src/component/stats.ts

@@ -0,0 +1,90 @@
+import { v } from "convex/values";
+import { Doc } from "./_generated/dataModel";
+import { internalQuery } from "./_generated/server";
+
+/**
+ * Record stats about work execution. Intended to be queried by Axiom or Datadog.
+ */
+
+/**
+ * Sample axiom dashboard query:
+
+workpool
+| extend parsed_message = iff(
+	isnotnull(parse_json(trim("'", tostring(["data.message"])))),
+	parse_json(trim("'", tostring(["data.message"]))),
+	parse_json('{}')
+)
+| extend lagSinceEnqueued = parsed_message["lagSinceEnqueued"]
+| extend fnName = parsed_message["fnName"]
+| summarize avg(todouble(lagSinceEnqueued)) by bin_auto(_time), tostring(fnName)
+
+ */
+
+export function recordStarted(work: Doc<"work">): string {
+  return JSON.stringify({
+    workId: work._id,
+    event: "started",
+    fnName: work.fnName,
+    enqueuedAt: work._creationTime,
+    startedAt: Date.now(),
+    lagSinceEnqueued: Date.now() - work._creationTime,
+  });
+}
+
+export function recordCompleted(
+  work: Doc<"work">,
+  status: "success" | "error" | "canceled" | "timeout"
+): string {
+  return JSON.stringify({
+    workId: work._id,
+    event: "completed",
+    fnName: work.fnName,
+    completedAt: Date.now(),
+    status,
+    lagSinceEnqueued: Date.now() - work._creationTime,
+  });
+}
+
+/**
+ * Warning: this should not be used from a mutation, as it will cause conflicts.
+ * Use this to debug or diagnose your queue length when it's backed up.
+ */
+export const queueLength = internalQuery({
+  args: {},
+  returns: v.number(),
+  handler: async (ctx) => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return (ctx.db.query("pendingStart") as any).count();
+  },
+});
+
+/**
+ * Warning: this should not be used from a mutation, as it will cause conflicts.
+ * Use this while developing to see the state of the queue.
+ */
+export const debugCounts = internalQuery({
+  args: {},
+  returns: v.any(),
+  handler: async (ctx) => {
+    /* eslint-disable @typescript-eslint/no-explicit-any */
+    const inProgressWork = await (
+      ctx.db.query("inProgressWork") as any
+    ).count();
+    const pendingStart = await (ctx.db.query("pendingStart") as any).count();
+    const pendingCompletion = await (
+      ctx.db.query("pendingCompletion") as any
+    ).count();
+    const pendingCancelation = await (
+      ctx.db.query("pendingCancelation") as any
+    ).count();
+    return {
+      pendingStart,
+      inProgressWork,
+      pendingCompletion,
+      pendingCancelation,
+      active: inProgressWork - pendingCompletion - pendingCancelation,
+    };
+    /* eslint-enable @typescript-eslint/no-explicit-any */
+  },
+});