@convex-dev/workpool 0.2.0-beta.0 → 0.2.1

package/src/component/lib.ts

@@ -1,18 +1,20 @@
 import { v } from "convex/values";
-import { mutation, query } from "./_generated/server.js";
+import { api } from "./_generated/api.js";
+import { Id } from "./_generated/dataModel.js";
+import { mutation, MutationCtx, query } from "./_generated/server.js";
+import { kickMainLoop } from "./kick.js";
+import { createLogger, LogLevel, logLevel } from "./logging.js";
 import {
-  nextSegment,
+  boundScheduledTime,
+  config,
+  getNextSegment,
+  max,
   onComplete,
   retryBehavior,
-  config,
   status as statusValidator,
   toSegment,
-  boundScheduledTime,
 } from "./shared.js";
-import { logLevel } from "./logging.js";
-import { kickMainLoop } from "./kick.js";
-import { api } from "./_generated/api.js";
-import { createLogger } from "./logging.js";
+import { recordEnqueued } from "./stats.js";
 
 const MAX_POSSIBLE_PARALLELISM = 100;
 
@@ -42,12 +44,12 @@ export const enqueue = mutation({
       ...workArgs,
       attempts: 0,
     });
+    const limit = await kickMainLoop(ctx, "enqueue", config);
     await ctx.db.insert("pendingStart", {
       workId,
-      segment: toSegment(runAt),
+      segment: max(toSegment(runAt), limit),
     });
-    await kickMainLoop(ctx, "enqueue", config);
-    // TODO: stats event
+    recordEnqueued(console, { workId, fnName: workArgs.fnName, runAt });
     return workId;
   },
 });
@@ -58,12 +60,14 @@ export const cancel = mutation({
     logLevel,
   },
   handler: async (ctx, { id, logLevel }) => {
-    await ctx.db.insert("pendingCancelation", {
-      workId: id,
-      segment: nextSegment(),
-    });
-    await kickMainLoop(ctx, "cancel", { logLevel });
-    // TODO: stats event
+    const shouldCancel = await shouldCancelWorkItem(ctx, id, logLevel);
+    if (shouldCancel) {
+      const segment = await kickMainLoop(ctx, "cancel", { logLevel });
+      await ctx.db.insert("pendingCancelation", {
+        workId: id,
+        segment,
+      });
+    }
   },
 });
 
@@ -72,23 +76,28 @@ export const cancelAll = mutation({
   args: { logLevel, before: v.optional(v.number()) },
   handler: async (ctx, { logLevel, before }) => {
     const beforeTime = before ?? Date.now();
-    const segment = nextSegment();
     const pageOfWork = await ctx.db
       .query("work")
       .withIndex("by_creation_time", (q) => q.lte("_creationTime", beforeTime))
       .order("desc")
       .take(PAGE_SIZE);
+    const shouldCancel = await Promise.all(
+      pageOfWork.map(async ({ _id }) =>
+        shouldCancelWorkItem(ctx, _id, logLevel)
+      )
+    );
+    let segment = getNextSegment();
+    if (shouldCancel.some((c) => c)) {
+      segment = await kickMainLoop(ctx, "cancel", { logLevel });
+    }
     await Promise.all(
-      pageOfWork.map(async ({ _id }) => {
-        if (
-          await ctx.db
-            .query("pendingCancelation")
-            .withIndex("workId", (q) => q.eq("workId", _id))
-            .first()
-        ) {
-          return;
+      pageOfWork.map(({ _id }, index) => {
+        if (shouldCancel[index]) {
+          return ctx.db.insert("pendingCancelation", {
+            workId: _id,
+            segment,
+          });
         }
-        await ctx.db.insert("pendingCancelation", { workId: _id, segment });
       })
     );
     if (pageOfWork.length === PAGE_SIZE) {
@@ -97,7 +106,6 @@ export const cancelAll = mutation({
         before: pageOfWork[pageOfWork.length - 1]._creationTime,
       });
     }
-    await kickMainLoop(ctx, "cancel", { logLevel });
   },
 });
 
@@ -114,12 +122,42 @@ export const status = query({
       .withIndex("workId", (q) => q.eq("workId", id))
       .unique();
     if (pendingStart) {
-      return { state: "pending", attempt: work.attempts } as const;
+      return { state: "pending", previousAttempts: work.attempts } as const;
+    }
+    const pendingCompletion = await ctx.db
+      .query("pendingCompletion")
+      .withIndex("workId", (q) => q.eq("workId", id))
+      .unique();
+    if (pendingCompletion?.retry) {
+      return { state: "pending", previousAttempts: work.attempts } as const;
     }
     // Assume it's in progress. It could be pending cancelation
-    return { state: "running", attempt: work.attempts } as const;
+    return { state: "running", previousAttempts: work.attempts } as const;
   },
 });
 
+async function shouldCancelWorkItem(
+  ctx: MutationCtx,
+  workId: Id<"work">,
+  logLevel: LogLevel
+) {
+  const console = createLogger(logLevel);
+  // No-op if the work doesn't exist or has completed.
+  const work = await ctx.db.get(workId);
+  if (!work) {
+    console.warn(`[cancel] work ${workId} doesn't exist`);
+    return false;
+  }
+  const pendingCancelation = await ctx.db
+    .query("pendingCancelation")
+    .withIndex("workId", (q) => q.eq("workId", workId))
+    .unique();
+  if (pendingCancelation) {
+    console.warn(`[cancel] work ${workId} has already been canceled`);
+    return false;
+  }
+  return true;
+}
+
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const console = "THIS IS A REMINDER TO USE createLogger";

package/src/component/logging.test.ts

@@ -0,0 +1,16 @@
+import { describe, expect, it } from "vitest";
+import { shouldLog } from "./logging";
+
+describe("logging", () => {
+  describe("shouldLog", () => {
+    it("should return true if the log level is above the config level", () => {
+      expect(shouldLog("INFO", "DEBUG")).toBe(false);
+    });
+    it("should return false if the log level is below the config level", () => {
+      expect(shouldLog("INFO", "WARN")).toBe(true);
+    });
+    it("should return true if the log level is equal to the config level", () => {
+      expect(shouldLog("INFO", "INFO")).toBe(true);
+    });
+  });
+});

package/src/component/logging.ts

@@ -1,6 +1,17 @@
 import { v, Infer } from "convex/values";
 
-export const DEFAULT_LOG_LEVEL: LogLevel = "WARN";
+export const DEFAULT_LOG_LEVEL: LogLevel = "REPORT";
+
+// NOTE: the ordering here is important! A config level of "INFO" will log
+// "INFO", "REPORT", "WARN",and "ERROR" events.
+export const logLevel = v.union(
+  v.literal("DEBUG"),
+  v.literal("INFO"),
+  v.literal("REPORT"),
+  v.literal("WARN"),
+  v.literal("ERROR")
+);
+export type LogLevel = Infer<typeof logLevel>;
 
 export type Logger = {
   debug: (...args: unknown[]) => void;
@@ -12,60 +23,70 @@ export type Logger = {
   event: (event: string, payload: Record<string, unknown>) => void;
 };
 
+const logLevelOrder = logLevel.members.map((l) => l.value);
+const logLevelByName = logLevelOrder.reduce(
+  (acc, l, i) => {
+    acc[l] = i;
+    return acc;
+  },
+  {} as Record<LogLevel, number>
+);
+export function shouldLog(config: LogLevel, level: LogLevel) {
+  return logLevelByName[config] <= logLevelByName[level];
+}
+const DEBUG = logLevelByName["DEBUG"];
+const INFO = logLevelByName["INFO"];
+const REPORT = logLevelByName["REPORT"];
+const WARN = logLevelByName["WARN"];
+const ERROR = logLevelByName["ERROR"];
+
 export function createLogger(level?: LogLevel): Logger {
-  const levelIndex = ["DEBUG", "INFO", "WARN", "ERROR"].indexOf(
-    level ?? DEFAULT_LOG_LEVEL
-  );
-  if (levelIndex === -1) {
+  const levelIndex = logLevelByName[level ?? DEFAULT_LOG_LEVEL];
+  if (levelIndex === undefined) {
     throw new Error(`Invalid log level: ${level}`);
   }
   return {
     debug: (...args: unknown[]) => {
-      if (levelIndex <= 0) {
+      if (levelIndex <= DEBUG) {
         console.debug(...args);
       }
     },
     info: (...args: unknown[]) => {
-      if (levelIndex <= 1) {
+      if (levelIndex <= INFO) {
         console.info(...args);
       }
     },
     warn: (...args: unknown[]) => {
-      if (levelIndex <= 2) {
+      if (levelIndex <= WARN) {
         console.warn(...args);
       }
     },
     error: (...args: unknown[]) => {
-      if (levelIndex <= 3) {
+      if (levelIndex <= ERROR) {
         console.error(...args);
       }
     },
     time: (label: string) => {
-      if (levelIndex <= 0) {
+      if (levelIndex <= DEBUG) {
         console.time(label);
       }
     },
     timeEnd: (label: string) => {
-      if (levelIndex <= 0) {
+      if (levelIndex <= DEBUG) {
         console.timeEnd(label);
       }
     },
     event: (event: string, payload: Record<string, unknown>) => {
-      if (levelIndex <= 1) {
-        const fullPayload = {
-          system: "idempotent-workpool-component",
-          event,
-          payload,
-        };
+      const fullPayload = {
+        component: "workpool",
+        event,
+        ...payload,
+      };
+      if (levelIndex === REPORT && event === "report") {
+        console.info(JSON.stringify(fullPayload));
+      } else if (levelIndex <= INFO) {
         console.info(JSON.stringify(fullPayload));
       }
     },
   };
 }
-export const logLevel = v.union(
-  v.literal("DEBUG"),
-  v.literal("INFO"),
-  v.literal("WARN"),
-  v.literal("ERROR")
-);
-export type LogLevel = Infer<typeof logLevel>;