@convex-dev/workpool 0.3.1-alpha.1 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +9 -0
  2. package/dist/client/index.d.ts +9 -9
  3. package/dist/client/index.d.ts.map +1 -1
  4. package/dist/client/index.js +22 -22
  5. package/dist/client/index.js.map +1 -1
  6. package/dist/component/_generated/api.d.ts +2 -0
  7. package/dist/component/_generated/api.d.ts.map +1 -1
  8. package/dist/component/_generated/api.js.map +1 -1
  9. package/dist/component/_generated/component.d.ts +12 -6
  10. package/dist/component/_generated/component.d.ts.map +1 -1
  11. package/dist/component/_generated/dataModel.d.ts +1 -1
  12. package/dist/component/complete.d.ts.map +1 -1
  13. package/dist/component/complete.js +2 -2
  14. package/dist/component/complete.js.map +1 -1
  15. package/dist/component/config.d.ts +16 -0
  16. package/dist/component/config.d.ts.map +1 -0
  17. package/dist/component/config.js +63 -0
  18. package/dist/component/config.js.map +1 -0
  19. package/dist/component/kick.d.ts +1 -1
  20. package/dist/component/kick.d.ts.map +1 -1
  21. package/dist/component/kick.js +4 -29
  22. package/dist/component/kick.js.map +1 -1
  23. package/dist/component/lib.d.ts +6 -8
  24. package/dist/component/lib.d.ts.map +1 -1
  25. package/dist/component/lib.js +19 -29
  26. package/dist/component/lib.js.map +1 -1
  27. package/dist/component/loop.d.ts.map +1 -1
  28. package/dist/component/loop.js +4 -4
  29. package/dist/component/loop.js.map +1 -1
  30. package/dist/component/schema.d.ts +1 -3
  31. package/dist/component/schema.d.ts.map +1 -1
  32. package/dist/component/schema.js +4 -4
  33. package/dist/component/schema.js.map +1 -1
  34. package/dist/component/shared.d.ts +6 -9
  35. package/dist/component/shared.d.ts.map +1 -1
  36. package/dist/component/shared.js +3 -4
  37. package/dist/component/shared.js.map +1 -1
  38. package/package.json +22 -22
  39. package/src/client/index.ts +33 -25
  40. package/src/component/_generated/api.ts +2 -0
  41. package/src/component/_generated/component.ts +18 -6
  42. package/src/component/_generated/dataModel.ts +1 -1
  43. package/src/component/complete.ts +2 -6
  44. package/src/component/config.test.ts +31 -0
  45. package/src/component/config.ts +72 -0
  46. package/src/component/kick.test.ts +2 -23
  47. package/src/component/kick.ts +4 -32
  48. package/src/component/lib.test.ts +3 -3
  49. package/src/component/lib.ts +21 -34
  50. package/src/component/loop.ts +3 -4
  51. package/src/component/recovery.test.ts +122 -122
  52. package/src/component/schema.ts +6 -6
  53. package/src/component/shared.ts +5 -7
@@ -1,6 +1,7 @@
1
1
  import {
2
2
  createFunctionHandle,
3
3
  type DefaultFunctionArgs,
4
+ type FunctionArgs,
4
5
  type FunctionHandle,
5
6
  type FunctionReference,
6
7
  type FunctionType,
@@ -18,17 +19,15 @@ import {
18
19
  type VString,
19
20
  } from "convex/values";
20
21
  import type { ComponentApi } from "../component/_generated/component.js";
21
- import { DEFAULT_LOG_LEVEL, type LogLevel } from "../component/logging.js";
22
+ import { type LogLevel } from "../component/logging.js";
22
23
  import {
23
24
  type Config,
24
- DEFAULT_MAX_PARALLELISM,
25
25
  DEFAULT_RETRY_BEHAVIOR,
26
- type OnComplete,
27
26
  type RetryBehavior,
28
27
  type RunResult,
29
28
  type OnCompleteArgs as SharedOnCompleteArgs,
30
29
  type Status,
31
- vResultValidator,
30
+ vResult,
32
31
  } from "../component/shared.js";
33
32
  import {
34
33
  type RunMutationCtx,
@@ -37,16 +36,21 @@ import {
37
36
  } from "./utils.js";
38
37
  export { logLevel as vLogLevel, type LogLevel } from "../component/logging.js";
39
38
  export { retryBehavior as vRetryBehavior } from "../component/shared.js";
40
- export { vResultValidator, type RetryBehavior, type RunResult };
41
39
 
42
40
  export type WorkId = string & { __isWorkId: true };
43
- export const vWorkIdValidator = v.string() as VString<WorkId>;
41
+ export const vWorkId = v.string() as VString<WorkId>;
44
42
  export {
43
+ vResult,
45
44
  DEFAULT_RETRY_BEHAVIOR,
46
- /** @deprecated Use `vResultValidator` instead. */
47
- vResultValidator as resultValidator,
48
- /** @deprecated Use `vWorkIdValidator` instead. */
49
- vWorkIdValidator as workIdValidator,
45
+ /** @deprecated Use `vResult` instead. */
46
+ vResult as resultValidator,
47
+ /** @deprecated Use `vWorkId` instead. */
48
+ vWorkId as workIdValidator,
49
+ // Older name, not deprecated yet
50
+ vWorkId as vWorkIdValidator,
51
+ vResult as vResultValidator,
52
+ type RetryBehavior,
53
+ type RunResult,
50
54
  };
51
55
  /** Equivalent to `vOnCompleteArgs(<your-context-validator>)`. */
52
56
  export const vOnComplete = vOnCompleteArgs(v.any());
@@ -234,7 +238,7 @@ export class Workpool {
234
238
  async cancel(ctx: RunMutationCtx, id: WorkId): Promise<void> {
235
239
  await ctx.runMutation(this.component.lib.cancel, {
236
240
  id,
237
- logLevel: this.options.logLevel ?? DEFAULT_LOG_LEVEL,
241
+ logLevel: this.options.logLevel,
238
242
  });
239
243
  }
240
244
  /**
@@ -247,10 +251,11 @@ export class Workpool {
247
251
  options?: { limit?: number },
248
252
  ): Promise<void> {
249
253
  await ctx.runMutation(this.component.lib.cancelAll, {
250
- logLevel: this.options.logLevel ?? DEFAULT_LOG_LEVEL,
254
+ logLevel: this.options.logLevel,
251
255
  ...options,
252
256
  });
253
257
  }
258
+
254
259
  /**
255
260
  * Gets the status of a work item.
256
261
  *
@@ -337,9 +342,9 @@ export function vOnCompleteArgs<
337
342
  V extends Validator<any, "required", any> = VAny,
338
343
  >(context?: V) {
339
344
  return v.object({
340
- workId: vWorkIdValidator,
345
+ workId: vWorkId,
341
346
  context: (context ?? v.optional(v.any())) as V,
342
- result: vResultValidator,
347
+ result: vResult,
343
348
  });
344
349
  }
345
350
 
@@ -355,7 +360,8 @@ export type RetryOption = {
355
360
 
356
361
  export type WorkpoolOptions = {
357
362
  /** How many actions/mutations can be running at once within this pool.
358
- * Min 1, Suggested max: 100 on Pro, 20 on the free plan.
363
+ * Suggested max: 100 on Pro, 20 on free plan.
364
+ * If set to 0, no new work will be started.
359
365
  */
360
366
  maxParallelism?: number;
361
367
  /** How much to log. This is updated on each call to `enqueue*`,
@@ -495,23 +501,25 @@ async function enqueueArgs(
495
501
  typeof fn === "string" && fn.startsWith("function://")
496
502
  ? [fn, opts?.name ?? fn]
497
503
  : [await createFunctionHandle(fn), opts?.name ?? safeFunctionName(fn)];
498
- const onComplete: OnComplete | undefined = opts?.onComplete
499
- ? {
500
- fnHandle: await createFunctionHandle(opts.onComplete),
501
- context: opts.context,
502
- }
503
- : undefined;
504
504
  return {
505
505
  fnHandle,
506
506
  fnName,
507
- onComplete,
507
+ onComplete: opts?.onComplete
508
+ ? {
509
+ fnHandle: await createFunctionHandle(opts.onComplete),
510
+ context: opts.context,
511
+ }
512
+ : undefined,
508
513
  runAt: getRunAt(opts),
509
514
  retryBehavior: opts?.retryBehavior,
510
515
  config: {
511
- logLevel: opts?.logLevel ?? DEFAULT_LOG_LEVEL,
512
- maxParallelism: opts?.maxParallelism ?? DEFAULT_MAX_PARALLELISM,
516
+ logLevel: opts?.logLevel,
517
+ maxParallelism: opts?.maxParallelism,
513
518
  },
514
- };
519
+ } satisfies Omit<
520
+ FunctionArgs<ComponentApi["lib"]["enqueue"]>,
521
+ "fnArgs" | "fnType"
522
+ >;
515
523
  }
516
524
 
517
525
  function getRunAt(
@@ -9,6 +9,7 @@
9
9
  */
10
10
 
11
11
  import type * as complete from "../complete.js";
12
+ import type * as config from "../config.js";
12
13
  import type * as crons from "../crons.js";
13
14
  import type * as danger from "../danger.js";
14
15
  import type * as kick from "../kick.js";
@@ -29,6 +30,7 @@ import { anyApi, componentsGeneric } from "convex/server";
29
30
 
30
31
  const fullApi: ApiFromModules<{
31
32
  complete: typeof complete;
33
+ config: typeof config;
32
34
  crons: typeof crons;
33
35
  danger: typeof danger;
34
36
  kick: typeof kick;
@@ -23,13 +23,25 @@ import type { FunctionReference } from "convex/server";
23
23
  */
24
24
  export type ComponentApi<Name extends string | undefined = string | undefined> =
25
25
  {
26
+ config: {
27
+ update: FunctionReference<
28
+ "mutation",
29
+ "internal",
30
+ {
31
+ logLevel?: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
32
+ maxParallelism?: number;
33
+ },
34
+ any,
35
+ Name
36
+ >;
37
+ };
26
38
  lib: {
27
39
  cancel: FunctionReference<
28
40
  "mutation",
29
41
  "internal",
30
42
  {
31
43
  id: string;
32
- logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
44
+ logLevel?: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
33
45
  },
34
46
  any,
35
47
  Name
@@ -40,7 +52,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
40
52
  {
41
53
  before?: number;
42
54
  limit?: number;
43
- logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
55
+ logLevel?: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
44
56
  },
45
57
  any,
46
58
  Name
@@ -50,8 +62,8 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
50
62
  "internal",
51
63
  {
52
64
  config: {
53
- logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
54
- maxParallelism: number;
65
+ logLevel?: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
66
+ maxParallelism?: number;
55
67
  };
56
68
  fnArgs: any;
57
69
  fnHandle: string;
@@ -73,8 +85,8 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
73
85
  "internal",
74
86
  {
75
87
  config: {
76
- logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
77
- maxParallelism: number;
88
+ logLevel?: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
89
+ maxParallelism?: number;
78
90
  };
79
91
  items: Array<{
80
92
  fnArgs: any;
@@ -38,7 +38,7 @@ export type Doc<TableName extends TableNames> = DocumentByName<
38
38
  * Convex documents are uniquely identified by their `Id`, which is accessible
39
39
  * on the `_id` field. To learn more, see [Document IDs](https://docs.convex.dev/using/document-ids).
40
40
  *
41
- * Documents can be loaded using `db.get(id)` in query and mutation functions.
41
+ * Documents can be loaded using `db.get(tableName, id)` in query and mutation functions.
42
42
  *
43
43
  * IDs are just strings at runtime, but this type can be used to distinguish them from other
44
44
  * strings when type checking.
@@ -4,11 +4,7 @@ import type { Id } from "./_generated/dataModel.js";
4
4
  import { internalMutation, type MutationCtx } from "./_generated/server.js";
5
5
  import { kickMainLoop } from "./kick.js";
6
6
  import { createLogger } from "./logging.js";
7
- import {
8
- type OnCompleteArgs,
9
- type RunResult,
10
- vResultValidator,
11
- } from "./shared.js";
7
+ import { type OnCompleteArgs, type RunResult, vResult } from "./shared.js";
12
8
  import { recordCompleted } from "./stats.js";
13
9
 
14
10
  export type CompleteJob = Infer<typeof completeArgs.fields.jobs.element>;
@@ -16,7 +12,7 @@ export type CompleteJob = Infer<typeof completeArgs.fields.jobs.element>;
16
12
  export const completeArgs = v.object({
17
13
  jobs: v.array(
18
14
  v.object({
19
- runResult: vResultValidator,
15
+ runResult: vResult,
20
16
  workId: v.id("work"),
21
17
  attempt: v.number(),
22
18
  }),
@@ -0,0 +1,31 @@
1
+ import { convexTest } from "convex-test";
2
+ import { assert, expect, test } from "vitest";
3
+ import { DEFAULT_LOG_LEVEL } from "./logging.js";
4
+ import schema from "./schema.js";
5
+ import { modules } from "./setup.test.js";
6
+ import { DEFAULT_MAX_PARALLELISM } from "./shared.js";
7
+ import { getOrUpdateGlobals } from "./config.js";
8
+
9
+ test("it updates the globals when they change", async () => {
10
+ const t = convexTest(schema, modules);
11
+ await t.run(async (ctx) => {
12
+ await getOrUpdateGlobals(ctx, {
13
+ maxParallelism: DEFAULT_MAX_PARALLELISM,
14
+ logLevel: DEFAULT_LOG_LEVEL,
15
+ });
16
+ const globals = await ctx.db.query("globals").unique();
17
+ expect(globals).not.toBeNull();
18
+ assert(globals);
19
+ expect(globals.maxParallelism).toBe(DEFAULT_MAX_PARALLELISM);
20
+ expect(globals.logLevel).toBe(DEFAULT_LOG_LEVEL);
21
+ await getOrUpdateGlobals(ctx, {
22
+ maxParallelism: DEFAULT_MAX_PARALLELISM + 1,
23
+ logLevel: "ERROR",
24
+ });
25
+ const after = await ctx.db.query("globals").unique();
26
+ expect(after).not.toBeNull();
27
+ assert(after);
28
+ expect(after.maxParallelism).toBe(DEFAULT_MAX_PARALLELISM + 1);
29
+ expect(after.logLevel).toBe("ERROR");
30
+ });
31
+ });
@@ -0,0 +1,72 @@
1
+ import { mutation, type MutationCtx } from "./_generated/server.js";
2
+ import { vConfig, DEFAULT_MAX_PARALLELISM, type Config } from "./shared.js";
3
+ import { createLogger, DEFAULT_LOG_LEVEL } from "./logging.js";
4
+ import { kickMainLoop } from "./kick.js";
5
+
6
+ export const MAX_POSSIBLE_PARALLELISM = 200;
7
+ export const MAX_PARALLELISM_SOFT_LIMIT = 100;
8
+
9
+ export const update = mutation({
10
+ args: vConfig.partial(),
11
+ handler: async (ctx, args) => {
12
+ const { globals, previousValue } = await _getOrUpdateGlobals(ctx, args);
13
+ if (args.maxParallelism && args.maxParallelism > previousValue) {
14
+ await kickMainLoop(ctx, "kick", globals);
15
+ }
16
+ },
17
+ });
18
+
19
+ export function validateConfig(config: Partial<Config>) {
20
+ if (config.maxParallelism !== undefined) {
21
+ if (config.maxParallelism > MAX_POSSIBLE_PARALLELISM) {
22
+ throw new Error(`maxParallelism must be <= ${MAX_POSSIBLE_PARALLELISM}`);
23
+ } else if (config.maxParallelism > MAX_PARALLELISM_SOFT_LIMIT) {
24
+ createLogger(config.logLevel ?? DEFAULT_LOG_LEVEL).warn(
25
+ `maxParallelism should be <= ${MAX_PARALLELISM_SOFT_LIMIT}, but is set to ${config.maxParallelism}. This will be an error in a future version.`,
26
+ );
27
+ } else if (config.maxParallelism < 0) {
28
+ throw new Error("maxParallelism must be >= 0");
29
+ }
30
+ }
31
+ }
32
+ export async function getOrUpdateGlobals(
33
+ ctx: MutationCtx,
34
+ config?: Partial<Config>,
35
+ ) {
36
+ const { globals } = await _getOrUpdateGlobals(ctx, config);
37
+ return globals;
38
+ }
39
+ async function _getOrUpdateGlobals(
40
+ ctx: MutationCtx,
41
+ config?: Partial<Config>,
42
+ ) {
43
+ if (config) {
44
+ validateConfig(config);
45
+ }
46
+ const globals = await ctx.db.query("globals").unique();
47
+ const previousValue = globals?.maxParallelism ?? DEFAULT_MAX_PARALLELISM;
48
+ if (!globals) {
49
+ const id = await ctx.db.insert("globals", {
50
+ maxParallelism: config?.maxParallelism ?? DEFAULT_MAX_PARALLELISM,
51
+ logLevel: config?.logLevel ?? DEFAULT_LOG_LEVEL,
52
+ });
53
+ return { globals: (await ctx.db.get("globals", id))!, previousValue };
54
+ } else if (config) {
55
+ let updated = false;
56
+ if (
57
+ config.maxParallelism !== undefined &&
58
+ config.maxParallelism !== globals.maxParallelism
59
+ ) {
60
+ globals.maxParallelism = config.maxParallelism;
61
+ updated = true;
62
+ }
63
+ if (config.logLevel && config.logLevel !== globals.logLevel) {
64
+ globals.logLevel = config.logLevel;
65
+ updated = true;
66
+ }
67
+ if (updated) {
68
+ await ctx.db.replace("globals", globals._id, globals);
69
+ }
70
+ }
71
+ return { globals, previousValue };
72
+ }
@@ -20,6 +20,7 @@ import {
20
20
  getNextSegment,
21
21
  toSegment,
22
22
  } from "./shared.js";
23
+ import { getOrUpdateGlobals } from "./config.js";
23
24
 
24
25
  describe("kickMainLoop", () => {
25
26
  beforeEach(() => {
@@ -47,27 +48,6 @@ describe("kickMainLoop", () => {
47
48
  });
48
49
  });
49
50
 
50
- test("it updates the globals when they change", async () => {
51
- const t = convexTest(schema, modules);
52
- await t.run(async (ctx) => {
53
- await kickMainLoop(ctx, "enqueue");
54
- const globals = await ctx.db.query("globals").unique();
55
- expect(globals).not.toBeNull();
56
- assert(globals);
57
- expect(globals.maxParallelism).toBe(DEFAULT_MAX_PARALLELISM);
58
- expect(globals.logLevel).toBe(DEFAULT_LOG_LEVEL);
59
- await kickMainLoop(ctx, "enqueue", {
60
- maxParallelism: DEFAULT_MAX_PARALLELISM + 1,
61
- logLevel: "ERROR",
62
- });
63
- const after = await ctx.db.query("globals").unique();
64
- expect(after).not.toBeNull();
65
- assert(after);
66
- expect(after.maxParallelism).toBe(DEFAULT_MAX_PARALLELISM + 1);
67
- expect(after.logLevel).toBe("ERROR");
68
- });
69
- });
70
-
71
51
  test("does not kick when already running", async () => {
72
52
  const t = convexTest(schema, modules);
73
53
  await t.run(async (ctx) => {
@@ -249,8 +229,7 @@ describe("kickMainLoop", () => {
249
229
  test("preserves state between kicks with different sources", async () => {
250
230
  const t = convexTest(schema, modules);
251
231
  await t.run(async (ctx) => {
252
- // Initial kick with custom config
253
- await kickMainLoop(ctx, "enqueue", {
232
+ await getOrUpdateGlobals(ctx, {
254
233
  maxParallelism: 5,
255
234
  logLevel: "ERROR",
256
235
  });
@@ -1,11 +1,11 @@
1
1
  import { internal } from "./_generated/api.js";
2
2
  import { internalMutation, type MutationCtx } from "./_generated/server.js";
3
- import { createLogger, DEFAULT_LOG_LEVEL } from "./logging.js";
3
+ import { getOrUpdateGlobals } from "./config.js";
4
+ import { createLogger } from "./logging.js";
4
5
  import { INITIAL_STATE } from "./loop.js";
5
6
  import {
6
7
  boundScheduledTime,
7
8
  type Config,
8
- DEFAULT_MAX_PARALLELISM,
9
9
  fromSegment,
10
10
  getCurrentSegment,
11
11
  getNextSegment,
@@ -20,9 +20,9 @@ import {
20
20
  export async function kickMainLoop(
21
21
  ctx: MutationCtx,
22
22
  source: "enqueue" | "cancel" | "complete" | "kick",
23
- config?: Partial<Config>,
23
+ config?: Config,
24
24
  ): Promise<bigint> {
25
- const globals = await getOrUpdateGlobals(ctx, config);
25
+ const globals = config ?? (await getOrUpdateGlobals(ctx, config));
26
26
  const console = createLogger(globals.logLevel);
27
27
  const runStatus = await getOrCreateRunStatus(ctx);
28
28
  const next = getNextSegment();
@@ -98,31 +98,3 @@ async function getOrCreateRunStatus(ctx: MutationCtx) {
98
98
  }
99
99
  return runStatus;
100
100
  }
101
-
102
- async function getOrUpdateGlobals(ctx: MutationCtx, config?: Partial<Config>) {
103
- const globals = await ctx.db.query("globals").unique();
104
- if (!globals) {
105
- const id = await ctx.db.insert("globals", {
106
- maxParallelism: config?.maxParallelism ?? DEFAULT_MAX_PARALLELISM,
107
- logLevel: config?.logLevel ?? DEFAULT_LOG_LEVEL,
108
- });
109
- return (await ctx.db.get(id))!;
110
- } else if (config) {
111
- let updated = false;
112
- if (
113
- config.maxParallelism &&
114
- config.maxParallelism !== globals.maxParallelism
115
- ) {
116
- globals.maxParallelism = config.maxParallelism;
117
- updated = true;
118
- }
119
- if (config.logLevel && config.logLevel !== globals.logLevel) {
120
- globals.logLevel = config.logLevel;
121
- updated = true;
122
- }
123
- if (updated) {
124
- await ctx.db.replace(globals._id, globals);
125
- }
126
- }
127
- return globals;
128
- }
@@ -66,7 +66,7 @@ describe("lib", () => {
66
66
  logLevel: "WARN",
67
67
  },
68
68
  }),
69
- ).rejects.toThrow("maxParallelism must be <= 100");
69
+ ).rejects.toThrow("maxParallelism must be <= 200");
70
70
  });
71
71
 
72
72
  it("should throw error if maxParallelism is too low", async () => {
@@ -78,11 +78,11 @@ describe("lib", () => {
78
78
  fnType: "mutation",
79
79
  runAt: Date.now(),
80
80
  config: {
81
- maxParallelism: 0, // Less than minimum
81
+ maxParallelism: -1, // Less than minimum
82
82
  logLevel: "WARN",
83
83
  },
84
84
  }),
85
- ).rejects.toThrow("maxParallelism must be >= 1");
85
+ ).rejects.toThrow("maxParallelism must be >= 0");
86
86
  });
87
87
  });
88
88
 
@@ -1,4 +1,4 @@
1
- import { type Infer, type ObjectType, v } from "convex/values";
1
+ import { type ObjectType, v } from "convex/values";
2
2
  import { api } from "./_generated/api.js";
3
3
  import type { Id } from "./_generated/dataModel.js";
4
4
  import {
@@ -16,19 +16,17 @@ import {
16
16
  } from "./logging.js";
17
17
  import {
18
18
  boundScheduledTime,
19
- config,
19
+ vConfig,
20
20
  fnType,
21
21
  getNextSegment,
22
22
  max,
23
- onComplete,
23
+ vOnCompleteFnContext,
24
24
  retryBehavior,
25
25
  status as statusValidator,
26
26
  toSegment,
27
27
  } from "./shared.js";
28
28
  import { recordEnqueued } from "./stats.js";
29
-
30
- const MAX_POSSIBLE_PARALLELISM = 200;
31
- const MAX_PARALLELISM_SOFT_LIMIT = 100;
29
+ import { getOrUpdateGlobals } from "./config.js";
32
30
 
33
31
  const itemArgs = {
34
32
  fnHandle: v.string(),
@@ -37,20 +35,20 @@ const itemArgs = {
37
35
  fnType,
38
36
  runAt: v.number(),
39
37
  // TODO: annotation?
40
- onComplete: v.optional(onComplete),
38
+ onComplete: v.optional(vOnCompleteFnContext),
41
39
  retryBehavior: v.optional(retryBehavior),
42
40
  };
43
41
  const enqueueArgs = {
44
42
  ...itemArgs,
45
- config,
43
+ config: vConfig.partial(),
46
44
  };
47
45
  export const enqueue = mutation({
48
46
  args: enqueueArgs,
49
47
  returns: v.id("work"),
50
48
  handler: async (ctx, { config, ...itemArgs }) => {
51
- validateConfig(config);
52
- const console = createLogger(config.logLevel);
53
- const kickSegment = await kickMainLoop(ctx, "enqueue", config);
49
+ const globals = await getOrUpdateGlobals(ctx, config);
50
+ const console = createLogger(globals.logLevel);
51
+ const kickSegment = await kickMainLoop(ctx, "enqueue", globals);
54
52
  return await enqueueHandler(ctx, console, kickSegment, itemArgs);
55
53
  },
56
54
  });
@@ -73,29 +71,16 @@ async function enqueueHandler(
73
71
  return workId;
74
72
  }
75
73
 
76
- type Config = Infer<typeof config>;
77
- function validateConfig(config: Config) {
78
- if (config.maxParallelism > MAX_POSSIBLE_PARALLELISM) {
79
- throw new Error(`maxParallelism must be <= ${MAX_PARALLELISM_SOFT_LIMIT}`);
80
- } else if (config.maxParallelism > MAX_PARALLELISM_SOFT_LIMIT) {
81
- createLogger(config.logLevel).warn(
82
- `maxParallelism should be <= ${MAX_PARALLELISM_SOFT_LIMIT}, but is set to ${config.maxParallelism}. This will be an error in a future version.`,
83
- );
84
- } else if (config.maxParallelism < 1) {
85
- throw new Error("maxParallelism must be >= 1");
86
- }
87
- }
88
-
89
74
  export const enqueueBatch = mutation({
90
75
  args: {
91
76
  items: v.array(v.object(itemArgs)),
92
- config,
77
+ config: vConfig.partial(),
93
78
  },
94
79
  returns: v.array(v.id("work")),
95
80
  handler: async (ctx, { config, items }) => {
96
- validateConfig(config);
97
- const console = createLogger(config.logLevel);
98
- const kickSegment = await kickMainLoop(ctx, "enqueue", config);
81
+ const globals = await getOrUpdateGlobals(ctx, config);
82
+ const console = createLogger(globals.logLevel);
83
+ const kickSegment = await kickMainLoop(ctx, "enqueue", globals);
99
84
  return Promise.all(
100
85
  items.map((item) => enqueueHandler(ctx, console, kickSegment, item)),
101
86
  );
@@ -105,12 +90,13 @@ export const enqueueBatch = mutation({
105
90
  export const cancel = mutation({
106
91
  args: {
107
92
  id: v.id("work"),
108
- logLevel,
93
+ logLevel: v.optional(logLevel),
109
94
  },
110
95
  handler: async (ctx, { id, logLevel }) => {
111
- const shouldCancel = await shouldCancelWorkItem(ctx, id, logLevel);
96
+ const globals = await getOrUpdateGlobals(ctx, { logLevel });
97
+ const shouldCancel = await shouldCancelWorkItem(ctx, id, globals.logLevel);
112
98
  if (shouldCancel) {
113
- const segment = await kickMainLoop(ctx, "cancel", { logLevel });
99
+ const segment = await kickMainLoop(ctx, "cancel", globals);
114
100
  await ctx.db.insert("pendingCancelation", {
115
101
  workId: id,
116
102
  segment,
@@ -122,7 +108,7 @@ export const cancel = mutation({
122
108
  const PAGE_SIZE = 64;
123
109
  export const cancelAll = mutation({
124
110
  args: {
125
- logLevel,
111
+ logLevel: v.optional(logLevel),
126
112
  before: v.optional(v.number()),
127
113
  limit: v.optional(v.number()),
128
114
  },
@@ -134,14 +120,15 @@ export const cancelAll = mutation({
134
120
  .withIndex("by_creation_time", (q) => q.lte("_creationTime", beforeTime))
135
121
  .order("desc")
136
122
  .take(pageSize);
123
+ const globals = await getOrUpdateGlobals(ctx, { logLevel });
137
124
  const shouldCancel = await Promise.all(
138
125
  pageOfWork.map(async ({ _id }) =>
139
- shouldCancelWorkItem(ctx, _id, logLevel),
126
+ shouldCancelWorkItem(ctx, _id, globals.logLevel),
140
127
  ),
141
128
  );
142
129
  let segment = getNextSegment();
143
130
  if (shouldCancel.some((c) => c)) {
144
- segment = await kickMainLoop(ctx, "cancel", { logLevel });
131
+ segment = await kickMainLoop(ctx, "cancel", globals);
145
132
  }
146
133
  await Promise.all(
147
134
  pageOfWork.map(({ _id }, index) => {
@@ -23,7 +23,7 @@ import {
23
23
  } from "./shared.js";
24
24
  import { generateReport, recordCompleted, recordStarted } from "./stats.js";
25
25
 
26
- const CANCELATION_BATCH_SIZE = 64; // the only queue that can get unbounded.
26
+ const CANCELLATION_BATCH_SIZE = 64; // the only queue that can get unbounded.
27
27
  const SECOND = 1000;
28
28
  const MINUTE = 60 * SECOND;
29
29
  const RECOVERY_THRESHOLD_MS = 5 * MINUTE; // attempt to recover jobs this old.
@@ -75,7 +75,7 @@ export const main = internalMutation({
75
75
 
76
76
  // Read pendingCancelation, deleting from pendingStart. If it's still running, queue to cancel.
77
77
  console.time("[main] pendingCancelation");
78
- await handleCancelation(ctx, state, segment, console, toCancel, globals);
78
+ await handleCancelation(ctx, state, segment, console, toCancel);
79
79
  console.timeEnd("[main] pendingCancelation");
80
80
 
81
81
  if (state.running.length === 0) {
@@ -415,7 +415,6 @@ async function handleCancelation(
415
415
  segment: bigint,
416
416
  console: Logger,
417
417
  toCancel: CompleteJob[],
418
- { cancelationBatchSize }: Config,
419
418
  ) {
420
419
  const start = state.segmentCursors.cancelation - CURSOR_BUFFER_SEGMENTS;
421
420
  const canceled = await ctx.db
@@ -423,7 +422,7 @@ async function handleCancelation(
423
422
  .withIndex("segment", (q) =>
424
423
  q.gte("segment", start).lte("segment", segment),
425
424
  )
426
- .take(cancelationBatchSize ?? CANCELATION_BATCH_SIZE);
425
+ .take(CANCELLATION_BATCH_SIZE);
427
426
  state.segmentCursors.cancelation = canceled.at(-1)?.segment ?? segment;
428
427
  if (canceled.length) {
429
428
  console.debug(`[main] attempting to cancel ${canceled.length}`);