@convex-dev/workpool 0.3.1-alpha.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/client/index.d.ts +9 -9
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +22 -22
- package/dist/client/index.js.map +1 -1
- package/dist/component/_generated/api.d.ts +2 -0
- package/dist/component/_generated/api.d.ts.map +1 -1
- package/dist/component/_generated/api.js.map +1 -1
- package/dist/component/_generated/component.d.ts +12 -6
- package/dist/component/_generated/component.d.ts.map +1 -1
- package/dist/component/_generated/dataModel.d.ts +1 -1
- package/dist/component/complete.d.ts.map +1 -1
- package/dist/component/complete.js +2 -2
- package/dist/component/complete.js.map +1 -1
- package/dist/component/config.d.ts +16 -0
- package/dist/component/config.d.ts.map +1 -0
- package/dist/component/config.js +63 -0
- package/dist/component/config.js.map +1 -0
- package/dist/component/kick.d.ts +1 -1
- package/dist/component/kick.d.ts.map +1 -1
- package/dist/component/kick.js +4 -29
- package/dist/component/kick.js.map +1 -1
- package/dist/component/lib.d.ts +6 -8
- package/dist/component/lib.d.ts.map +1 -1
- package/dist/component/lib.js +19 -29
- package/dist/component/lib.js.map +1 -1
- package/dist/component/loop.d.ts.map +1 -1
- package/dist/component/loop.js +4 -4
- package/dist/component/loop.js.map +1 -1
- package/dist/component/schema.d.ts +1 -3
- package/dist/component/schema.d.ts.map +1 -1
- package/dist/component/schema.js +4 -4
- package/dist/component/schema.js.map +1 -1
- package/dist/component/shared.d.ts +6 -9
- package/dist/component/shared.d.ts.map +1 -1
- package/dist/component/shared.js +3 -4
- package/dist/component/shared.js.map +1 -1
- package/package.json +22 -22
- package/src/client/index.ts +33 -25
- package/src/component/_generated/api.ts +2 -0
- package/src/component/_generated/component.ts +18 -6
- package/src/component/_generated/dataModel.ts +1 -1
- package/src/component/complete.ts +2 -6
- package/src/component/config.test.ts +31 -0
- package/src/component/config.ts +72 -0
- package/src/component/kick.test.ts +2 -23
- package/src/component/kick.ts +4 -32
- package/src/component/lib.test.ts +3 -3
- package/src/component/lib.ts +21 -34
- package/src/component/loop.ts +3 -4
- package/src/component/recovery.test.ts +122 -122
- package/src/component/schema.ts +6 -6
- package/src/component/shared.ts +5 -7
package/src/client/index.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createFunctionHandle,
|
|
3
3
|
type DefaultFunctionArgs,
|
|
4
|
+
type FunctionArgs,
|
|
4
5
|
type FunctionHandle,
|
|
5
6
|
type FunctionReference,
|
|
6
7
|
type FunctionType,
|
|
@@ -18,17 +19,15 @@ import {
|
|
|
18
19
|
type VString,
|
|
19
20
|
} from "convex/values";
|
|
20
21
|
import type { ComponentApi } from "../component/_generated/component.js";
|
|
21
|
-
import {
|
|
22
|
+
import { type LogLevel } from "../component/logging.js";
|
|
22
23
|
import {
|
|
23
24
|
type Config,
|
|
24
|
-
DEFAULT_MAX_PARALLELISM,
|
|
25
25
|
DEFAULT_RETRY_BEHAVIOR,
|
|
26
|
-
type OnComplete,
|
|
27
26
|
type RetryBehavior,
|
|
28
27
|
type RunResult,
|
|
29
28
|
type OnCompleteArgs as SharedOnCompleteArgs,
|
|
30
29
|
type Status,
|
|
31
|
-
|
|
30
|
+
vResult,
|
|
32
31
|
} from "../component/shared.js";
|
|
33
32
|
import {
|
|
34
33
|
type RunMutationCtx,
|
|
@@ -37,16 +36,21 @@ import {
|
|
|
37
36
|
} from "./utils.js";
|
|
38
37
|
export { logLevel as vLogLevel, type LogLevel } from "../component/logging.js";
|
|
39
38
|
export { retryBehavior as vRetryBehavior } from "../component/shared.js";
|
|
40
|
-
export { vResultValidator, type RetryBehavior, type RunResult };
|
|
41
39
|
|
|
42
40
|
export type WorkId = string & { __isWorkId: true };
|
|
43
|
-
export const
|
|
41
|
+
export const vWorkId = v.string() as VString<WorkId>;
|
|
44
42
|
export {
|
|
43
|
+
vResult,
|
|
45
44
|
DEFAULT_RETRY_BEHAVIOR,
|
|
46
|
-
/** @deprecated Use `
|
|
47
|
-
|
|
48
|
-
/** @deprecated Use `
|
|
49
|
-
|
|
45
|
+
/** @deprecated Use `vResult` instead. */
|
|
46
|
+
vResult as resultValidator,
|
|
47
|
+
/** @deprecated Use `vWorkId` instead. */
|
|
48
|
+
vWorkId as workIdValidator,
|
|
49
|
+
// Older name, not deprecated yet
|
|
50
|
+
vWorkId as vWorkIdValidator,
|
|
51
|
+
vResult as vResultValidator,
|
|
52
|
+
type RetryBehavior,
|
|
53
|
+
type RunResult,
|
|
50
54
|
};
|
|
51
55
|
/** Equivalent to `vOnCompleteArgs(<your-context-validator>)`. */
|
|
52
56
|
export const vOnComplete = vOnCompleteArgs(v.any());
|
|
@@ -234,7 +238,7 @@ export class Workpool {
|
|
|
234
238
|
async cancel(ctx: RunMutationCtx, id: WorkId): Promise<void> {
|
|
235
239
|
await ctx.runMutation(this.component.lib.cancel, {
|
|
236
240
|
id,
|
|
237
|
-
logLevel: this.options.logLevel
|
|
241
|
+
logLevel: this.options.logLevel,
|
|
238
242
|
});
|
|
239
243
|
}
|
|
240
244
|
/**
|
|
@@ -247,10 +251,11 @@ export class Workpool {
|
|
|
247
251
|
options?: { limit?: number },
|
|
248
252
|
): Promise<void> {
|
|
249
253
|
await ctx.runMutation(this.component.lib.cancelAll, {
|
|
250
|
-
logLevel: this.options.logLevel
|
|
254
|
+
logLevel: this.options.logLevel,
|
|
251
255
|
...options,
|
|
252
256
|
});
|
|
253
257
|
}
|
|
258
|
+
|
|
254
259
|
/**
|
|
255
260
|
* Gets the status of a work item.
|
|
256
261
|
*
|
|
@@ -337,9 +342,9 @@ export function vOnCompleteArgs<
|
|
|
337
342
|
V extends Validator<any, "required", any> = VAny,
|
|
338
343
|
>(context?: V) {
|
|
339
344
|
return v.object({
|
|
340
|
-
workId:
|
|
345
|
+
workId: vWorkId,
|
|
341
346
|
context: (context ?? v.optional(v.any())) as V,
|
|
342
|
-
result:
|
|
347
|
+
result: vResult,
|
|
343
348
|
});
|
|
344
349
|
}
|
|
345
350
|
|
|
@@ -355,7 +360,8 @@ export type RetryOption = {
|
|
|
355
360
|
|
|
356
361
|
export type WorkpoolOptions = {
|
|
357
362
|
/** How many actions/mutations can be running at once within this pool.
|
|
358
|
-
*
|
|
363
|
+
* Suggested max: 100 on Pro, 20 on free plan.
|
|
364
|
+
* If set to 0, no new work will be started.
|
|
359
365
|
*/
|
|
360
366
|
maxParallelism?: number;
|
|
361
367
|
/** How much to log. This is updated on each call to `enqueue*`,
|
|
@@ -495,23 +501,25 @@ async function enqueueArgs(
|
|
|
495
501
|
typeof fn === "string" && fn.startsWith("function://")
|
|
496
502
|
? [fn, opts?.name ?? fn]
|
|
497
503
|
: [await createFunctionHandle(fn), opts?.name ?? safeFunctionName(fn)];
|
|
498
|
-
const onComplete: OnComplete | undefined = opts?.onComplete
|
|
499
|
-
? {
|
|
500
|
-
fnHandle: await createFunctionHandle(opts.onComplete),
|
|
501
|
-
context: opts.context,
|
|
502
|
-
}
|
|
503
|
-
: undefined;
|
|
504
504
|
return {
|
|
505
505
|
fnHandle,
|
|
506
506
|
fnName,
|
|
507
|
-
onComplete
|
|
507
|
+
onComplete: opts?.onComplete
|
|
508
|
+
? {
|
|
509
|
+
fnHandle: await createFunctionHandle(opts.onComplete),
|
|
510
|
+
context: opts.context,
|
|
511
|
+
}
|
|
512
|
+
: undefined,
|
|
508
513
|
runAt: getRunAt(opts),
|
|
509
514
|
retryBehavior: opts?.retryBehavior,
|
|
510
515
|
config: {
|
|
511
|
-
logLevel: opts?.logLevel
|
|
512
|
-
maxParallelism: opts?.maxParallelism
|
|
516
|
+
logLevel: opts?.logLevel,
|
|
517
|
+
maxParallelism: opts?.maxParallelism,
|
|
513
518
|
},
|
|
514
|
-
}
|
|
519
|
+
} satisfies Omit<
|
|
520
|
+
FunctionArgs<ComponentApi["lib"]["enqueue"]>,
|
|
521
|
+
"fnArgs" | "fnType"
|
|
522
|
+
>;
|
|
515
523
|
}
|
|
516
524
|
|
|
517
525
|
function getRunAt(
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import type * as complete from "../complete.js";
|
|
12
|
+
import type * as config from "../config.js";
|
|
12
13
|
import type * as crons from "../crons.js";
|
|
13
14
|
import type * as danger from "../danger.js";
|
|
14
15
|
import type * as kick from "../kick.js";
|
|
@@ -29,6 +30,7 @@ import { anyApi, componentsGeneric } from "convex/server";
|
|
|
29
30
|
|
|
30
31
|
const fullApi: ApiFromModules<{
|
|
31
32
|
complete: typeof complete;
|
|
33
|
+
config: typeof config;
|
|
32
34
|
crons: typeof crons;
|
|
33
35
|
danger: typeof danger;
|
|
34
36
|
kick: typeof kick;
|
|
@@ -23,13 +23,25 @@ import type { FunctionReference } from "convex/server";
|
|
|
23
23
|
*/
|
|
24
24
|
export type ComponentApi<Name extends string | undefined = string | undefined> =
|
|
25
25
|
{
|
|
26
|
+
config: {
|
|
27
|
+
update: FunctionReference<
|
|
28
|
+
"mutation",
|
|
29
|
+
"internal",
|
|
30
|
+
{
|
|
31
|
+
logLevel?: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
|
|
32
|
+
maxParallelism?: number;
|
|
33
|
+
},
|
|
34
|
+
any,
|
|
35
|
+
Name
|
|
36
|
+
>;
|
|
37
|
+
};
|
|
26
38
|
lib: {
|
|
27
39
|
cancel: FunctionReference<
|
|
28
40
|
"mutation",
|
|
29
41
|
"internal",
|
|
30
42
|
{
|
|
31
43
|
id: string;
|
|
32
|
-
logLevel
|
|
44
|
+
logLevel?: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
|
|
33
45
|
},
|
|
34
46
|
any,
|
|
35
47
|
Name
|
|
@@ -40,7 +52,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
|
|
|
40
52
|
{
|
|
41
53
|
before?: number;
|
|
42
54
|
limit?: number;
|
|
43
|
-
logLevel
|
|
55
|
+
logLevel?: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
|
|
44
56
|
},
|
|
45
57
|
any,
|
|
46
58
|
Name
|
|
@@ -50,8 +62,8 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
|
|
|
50
62
|
"internal",
|
|
51
63
|
{
|
|
52
64
|
config: {
|
|
53
|
-
logLevel
|
|
54
|
-
maxParallelism
|
|
65
|
+
logLevel?: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
|
|
66
|
+
maxParallelism?: number;
|
|
55
67
|
};
|
|
56
68
|
fnArgs: any;
|
|
57
69
|
fnHandle: string;
|
|
@@ -73,8 +85,8 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
|
|
|
73
85
|
"internal",
|
|
74
86
|
{
|
|
75
87
|
config: {
|
|
76
|
-
logLevel
|
|
77
|
-
maxParallelism
|
|
88
|
+
logLevel?: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
|
|
89
|
+
maxParallelism?: number;
|
|
78
90
|
};
|
|
79
91
|
items: Array<{
|
|
80
92
|
fnArgs: any;
|
|
@@ -38,7 +38,7 @@ export type Doc<TableName extends TableNames> = DocumentByName<
|
|
|
38
38
|
* Convex documents are uniquely identified by their `Id`, which is accessible
|
|
39
39
|
* on the `_id` field. To learn more, see [Document IDs](https://docs.convex.dev/using/document-ids).
|
|
40
40
|
*
|
|
41
|
-
* Documents can be loaded using `db.get(id)` in query and mutation functions.
|
|
41
|
+
* Documents can be loaded using `db.get(tableName, id)` in query and mutation functions.
|
|
42
42
|
*
|
|
43
43
|
* IDs are just strings at runtime, but this type can be used to distinguish them from other
|
|
44
44
|
* strings when type checking.
|
|
@@ -4,11 +4,7 @@ import type { Id } from "./_generated/dataModel.js";
|
|
|
4
4
|
import { internalMutation, type MutationCtx } from "./_generated/server.js";
|
|
5
5
|
import { kickMainLoop } from "./kick.js";
|
|
6
6
|
import { createLogger } from "./logging.js";
|
|
7
|
-
import {
|
|
8
|
-
type OnCompleteArgs,
|
|
9
|
-
type RunResult,
|
|
10
|
-
vResultValidator,
|
|
11
|
-
} from "./shared.js";
|
|
7
|
+
import { type OnCompleteArgs, type RunResult, vResult } from "./shared.js";
|
|
12
8
|
import { recordCompleted } from "./stats.js";
|
|
13
9
|
|
|
14
10
|
export type CompleteJob = Infer<typeof completeArgs.fields.jobs.element>;
|
|
@@ -16,7 +12,7 @@ export type CompleteJob = Infer<typeof completeArgs.fields.jobs.element>;
|
|
|
16
12
|
export const completeArgs = v.object({
|
|
17
13
|
jobs: v.array(
|
|
18
14
|
v.object({
|
|
19
|
-
runResult:
|
|
15
|
+
runResult: vResult,
|
|
20
16
|
workId: v.id("work"),
|
|
21
17
|
attempt: v.number(),
|
|
22
18
|
}),
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { convexTest } from "convex-test";
|
|
2
|
+
import { assert, expect, test } from "vitest";
|
|
3
|
+
import { DEFAULT_LOG_LEVEL } from "./logging.js";
|
|
4
|
+
import schema from "./schema.js";
|
|
5
|
+
import { modules } from "./setup.test.js";
|
|
6
|
+
import { DEFAULT_MAX_PARALLELISM } from "./shared.js";
|
|
7
|
+
import { getOrUpdateGlobals } from "./config.js";
|
|
8
|
+
|
|
9
|
+
test("it updates the globals when they change", async () => {
|
|
10
|
+
const t = convexTest(schema, modules);
|
|
11
|
+
await t.run(async (ctx) => {
|
|
12
|
+
await getOrUpdateGlobals(ctx, {
|
|
13
|
+
maxParallelism: DEFAULT_MAX_PARALLELISM,
|
|
14
|
+
logLevel: DEFAULT_LOG_LEVEL,
|
|
15
|
+
});
|
|
16
|
+
const globals = await ctx.db.query("globals").unique();
|
|
17
|
+
expect(globals).not.toBeNull();
|
|
18
|
+
assert(globals);
|
|
19
|
+
expect(globals.maxParallelism).toBe(DEFAULT_MAX_PARALLELISM);
|
|
20
|
+
expect(globals.logLevel).toBe(DEFAULT_LOG_LEVEL);
|
|
21
|
+
await getOrUpdateGlobals(ctx, {
|
|
22
|
+
maxParallelism: DEFAULT_MAX_PARALLELISM + 1,
|
|
23
|
+
logLevel: "ERROR",
|
|
24
|
+
});
|
|
25
|
+
const after = await ctx.db.query("globals").unique();
|
|
26
|
+
expect(after).not.toBeNull();
|
|
27
|
+
assert(after);
|
|
28
|
+
expect(after.maxParallelism).toBe(DEFAULT_MAX_PARALLELISM + 1);
|
|
29
|
+
expect(after.logLevel).toBe("ERROR");
|
|
30
|
+
});
|
|
31
|
+
});
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { mutation, type MutationCtx } from "./_generated/server.js";
|
|
2
|
+
import { vConfig, DEFAULT_MAX_PARALLELISM, type Config } from "./shared.js";
|
|
3
|
+
import { createLogger, DEFAULT_LOG_LEVEL } from "./logging.js";
|
|
4
|
+
import { kickMainLoop } from "./kick.js";
|
|
5
|
+
|
|
6
|
+
export const MAX_POSSIBLE_PARALLELISM = 200;
|
|
7
|
+
export const MAX_PARALLELISM_SOFT_LIMIT = 100;
|
|
8
|
+
|
|
9
|
+
export const update = mutation({
|
|
10
|
+
args: vConfig.partial(),
|
|
11
|
+
handler: async (ctx, args) => {
|
|
12
|
+
const { globals, previousValue } = await _getOrUpdateGlobals(ctx, args);
|
|
13
|
+
if (args.maxParallelism && args.maxParallelism > previousValue) {
|
|
14
|
+
await kickMainLoop(ctx, "kick", globals);
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
export function validateConfig(config: Partial<Config>) {
|
|
20
|
+
if (config.maxParallelism !== undefined) {
|
|
21
|
+
if (config.maxParallelism > MAX_POSSIBLE_PARALLELISM) {
|
|
22
|
+
throw new Error(`maxParallelism must be <= ${MAX_POSSIBLE_PARALLELISM}`);
|
|
23
|
+
} else if (config.maxParallelism > MAX_PARALLELISM_SOFT_LIMIT) {
|
|
24
|
+
createLogger(config.logLevel ?? DEFAULT_LOG_LEVEL).warn(
|
|
25
|
+
`maxParallelism should be <= ${MAX_PARALLELISM_SOFT_LIMIT}, but is set to ${config.maxParallelism}. This will be an error in a future version.`,
|
|
26
|
+
);
|
|
27
|
+
} else if (config.maxParallelism < 0) {
|
|
28
|
+
throw new Error("maxParallelism must be >= 0");
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
export async function getOrUpdateGlobals(
|
|
33
|
+
ctx: MutationCtx,
|
|
34
|
+
config?: Partial<Config>,
|
|
35
|
+
) {
|
|
36
|
+
const { globals } = await _getOrUpdateGlobals(ctx, config);
|
|
37
|
+
return globals;
|
|
38
|
+
}
|
|
39
|
+
async function _getOrUpdateGlobals(
|
|
40
|
+
ctx: MutationCtx,
|
|
41
|
+
config?: Partial<Config>,
|
|
42
|
+
) {
|
|
43
|
+
if (config) {
|
|
44
|
+
validateConfig(config);
|
|
45
|
+
}
|
|
46
|
+
const globals = await ctx.db.query("globals").unique();
|
|
47
|
+
const previousValue = globals?.maxParallelism ?? DEFAULT_MAX_PARALLELISM;
|
|
48
|
+
if (!globals) {
|
|
49
|
+
const id = await ctx.db.insert("globals", {
|
|
50
|
+
maxParallelism: config?.maxParallelism ?? DEFAULT_MAX_PARALLELISM,
|
|
51
|
+
logLevel: config?.logLevel ?? DEFAULT_LOG_LEVEL,
|
|
52
|
+
});
|
|
53
|
+
return { globals: (await ctx.db.get("globals", id))!, previousValue };
|
|
54
|
+
} else if (config) {
|
|
55
|
+
let updated = false;
|
|
56
|
+
if (
|
|
57
|
+
config.maxParallelism !== undefined &&
|
|
58
|
+
config.maxParallelism !== globals.maxParallelism
|
|
59
|
+
) {
|
|
60
|
+
globals.maxParallelism = config.maxParallelism;
|
|
61
|
+
updated = true;
|
|
62
|
+
}
|
|
63
|
+
if (config.logLevel && config.logLevel !== globals.logLevel) {
|
|
64
|
+
globals.logLevel = config.logLevel;
|
|
65
|
+
updated = true;
|
|
66
|
+
}
|
|
67
|
+
if (updated) {
|
|
68
|
+
await ctx.db.replace("globals", globals._id, globals);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return { globals, previousValue };
|
|
72
|
+
}
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
getNextSegment,
|
|
21
21
|
toSegment,
|
|
22
22
|
} from "./shared.js";
|
|
23
|
+
import { getOrUpdateGlobals } from "./config.js";
|
|
23
24
|
|
|
24
25
|
describe("kickMainLoop", () => {
|
|
25
26
|
beforeEach(() => {
|
|
@@ -47,27 +48,6 @@ describe("kickMainLoop", () => {
|
|
|
47
48
|
});
|
|
48
49
|
});
|
|
49
50
|
|
|
50
|
-
test("it updates the globals when they change", async () => {
|
|
51
|
-
const t = convexTest(schema, modules);
|
|
52
|
-
await t.run(async (ctx) => {
|
|
53
|
-
await kickMainLoop(ctx, "enqueue");
|
|
54
|
-
const globals = await ctx.db.query("globals").unique();
|
|
55
|
-
expect(globals).not.toBeNull();
|
|
56
|
-
assert(globals);
|
|
57
|
-
expect(globals.maxParallelism).toBe(DEFAULT_MAX_PARALLELISM);
|
|
58
|
-
expect(globals.logLevel).toBe(DEFAULT_LOG_LEVEL);
|
|
59
|
-
await kickMainLoop(ctx, "enqueue", {
|
|
60
|
-
maxParallelism: DEFAULT_MAX_PARALLELISM + 1,
|
|
61
|
-
logLevel: "ERROR",
|
|
62
|
-
});
|
|
63
|
-
const after = await ctx.db.query("globals").unique();
|
|
64
|
-
expect(after).not.toBeNull();
|
|
65
|
-
assert(after);
|
|
66
|
-
expect(after.maxParallelism).toBe(DEFAULT_MAX_PARALLELISM + 1);
|
|
67
|
-
expect(after.logLevel).toBe("ERROR");
|
|
68
|
-
});
|
|
69
|
-
});
|
|
70
|
-
|
|
71
51
|
test("does not kick when already running", async () => {
|
|
72
52
|
const t = convexTest(schema, modules);
|
|
73
53
|
await t.run(async (ctx) => {
|
|
@@ -249,8 +229,7 @@ describe("kickMainLoop", () => {
|
|
|
249
229
|
test("preserves state between kicks with different sources", async () => {
|
|
250
230
|
const t = convexTest(schema, modules);
|
|
251
231
|
await t.run(async (ctx) => {
|
|
252
|
-
|
|
253
|
-
await kickMainLoop(ctx, "enqueue", {
|
|
232
|
+
await getOrUpdateGlobals(ctx, {
|
|
254
233
|
maxParallelism: 5,
|
|
255
234
|
logLevel: "ERROR",
|
|
256
235
|
});
|
package/src/component/kick.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { internal } from "./_generated/api.js";
|
|
2
2
|
import { internalMutation, type MutationCtx } from "./_generated/server.js";
|
|
3
|
-
import {
|
|
3
|
+
import { getOrUpdateGlobals } from "./config.js";
|
|
4
|
+
import { createLogger } from "./logging.js";
|
|
4
5
|
import { INITIAL_STATE } from "./loop.js";
|
|
5
6
|
import {
|
|
6
7
|
boundScheduledTime,
|
|
7
8
|
type Config,
|
|
8
|
-
DEFAULT_MAX_PARALLELISM,
|
|
9
9
|
fromSegment,
|
|
10
10
|
getCurrentSegment,
|
|
11
11
|
getNextSegment,
|
|
@@ -20,9 +20,9 @@ import {
|
|
|
20
20
|
export async function kickMainLoop(
|
|
21
21
|
ctx: MutationCtx,
|
|
22
22
|
source: "enqueue" | "cancel" | "complete" | "kick",
|
|
23
|
-
config?:
|
|
23
|
+
config?: Config,
|
|
24
24
|
): Promise<bigint> {
|
|
25
|
-
const globals = await getOrUpdateGlobals(ctx, config);
|
|
25
|
+
const globals = config ?? (await getOrUpdateGlobals(ctx, config));
|
|
26
26
|
const console = createLogger(globals.logLevel);
|
|
27
27
|
const runStatus = await getOrCreateRunStatus(ctx);
|
|
28
28
|
const next = getNextSegment();
|
|
@@ -98,31 +98,3 @@ async function getOrCreateRunStatus(ctx: MutationCtx) {
|
|
|
98
98
|
}
|
|
99
99
|
return runStatus;
|
|
100
100
|
}
|
|
101
|
-
|
|
102
|
-
async function getOrUpdateGlobals(ctx: MutationCtx, config?: Partial<Config>) {
|
|
103
|
-
const globals = await ctx.db.query("globals").unique();
|
|
104
|
-
if (!globals) {
|
|
105
|
-
const id = await ctx.db.insert("globals", {
|
|
106
|
-
maxParallelism: config?.maxParallelism ?? DEFAULT_MAX_PARALLELISM,
|
|
107
|
-
logLevel: config?.logLevel ?? DEFAULT_LOG_LEVEL,
|
|
108
|
-
});
|
|
109
|
-
return (await ctx.db.get(id))!;
|
|
110
|
-
} else if (config) {
|
|
111
|
-
let updated = false;
|
|
112
|
-
if (
|
|
113
|
-
config.maxParallelism &&
|
|
114
|
-
config.maxParallelism !== globals.maxParallelism
|
|
115
|
-
) {
|
|
116
|
-
globals.maxParallelism = config.maxParallelism;
|
|
117
|
-
updated = true;
|
|
118
|
-
}
|
|
119
|
-
if (config.logLevel && config.logLevel !== globals.logLevel) {
|
|
120
|
-
globals.logLevel = config.logLevel;
|
|
121
|
-
updated = true;
|
|
122
|
-
}
|
|
123
|
-
if (updated) {
|
|
124
|
-
await ctx.db.replace(globals._id, globals);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
return globals;
|
|
128
|
-
}
|
|
@@ -66,7 +66,7 @@ describe("lib", () => {
|
|
|
66
66
|
logLevel: "WARN",
|
|
67
67
|
},
|
|
68
68
|
}),
|
|
69
|
-
).rejects.toThrow("maxParallelism must be <=
|
|
69
|
+
).rejects.toThrow("maxParallelism must be <= 200");
|
|
70
70
|
});
|
|
71
71
|
|
|
72
72
|
it("should throw error if maxParallelism is too low", async () => {
|
|
@@ -78,11 +78,11 @@ describe("lib", () => {
|
|
|
78
78
|
fnType: "mutation",
|
|
79
79
|
runAt: Date.now(),
|
|
80
80
|
config: {
|
|
81
|
-
maxParallelism:
|
|
81
|
+
maxParallelism: -1, // Less than minimum
|
|
82
82
|
logLevel: "WARN",
|
|
83
83
|
},
|
|
84
84
|
}),
|
|
85
|
-
).rejects.toThrow("maxParallelism must be >=
|
|
85
|
+
).rejects.toThrow("maxParallelism must be >= 0");
|
|
86
86
|
});
|
|
87
87
|
});
|
|
88
88
|
|
package/src/component/lib.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type
|
|
1
|
+
import { type ObjectType, v } from "convex/values";
|
|
2
2
|
import { api } from "./_generated/api.js";
|
|
3
3
|
import type { Id } from "./_generated/dataModel.js";
|
|
4
4
|
import {
|
|
@@ -16,19 +16,17 @@ import {
|
|
|
16
16
|
} from "./logging.js";
|
|
17
17
|
import {
|
|
18
18
|
boundScheduledTime,
|
|
19
|
-
|
|
19
|
+
vConfig,
|
|
20
20
|
fnType,
|
|
21
21
|
getNextSegment,
|
|
22
22
|
max,
|
|
23
|
-
|
|
23
|
+
vOnCompleteFnContext,
|
|
24
24
|
retryBehavior,
|
|
25
25
|
status as statusValidator,
|
|
26
26
|
toSegment,
|
|
27
27
|
} from "./shared.js";
|
|
28
28
|
import { recordEnqueued } from "./stats.js";
|
|
29
|
-
|
|
30
|
-
const MAX_POSSIBLE_PARALLELISM = 200;
|
|
31
|
-
const MAX_PARALLELISM_SOFT_LIMIT = 100;
|
|
29
|
+
import { getOrUpdateGlobals } from "./config.js";
|
|
32
30
|
|
|
33
31
|
const itemArgs = {
|
|
34
32
|
fnHandle: v.string(),
|
|
@@ -37,20 +35,20 @@ const itemArgs = {
|
|
|
37
35
|
fnType,
|
|
38
36
|
runAt: v.number(),
|
|
39
37
|
// TODO: annotation?
|
|
40
|
-
onComplete: v.optional(
|
|
38
|
+
onComplete: v.optional(vOnCompleteFnContext),
|
|
41
39
|
retryBehavior: v.optional(retryBehavior),
|
|
42
40
|
};
|
|
43
41
|
const enqueueArgs = {
|
|
44
42
|
...itemArgs,
|
|
45
|
-
config,
|
|
43
|
+
config: vConfig.partial(),
|
|
46
44
|
};
|
|
47
45
|
export const enqueue = mutation({
|
|
48
46
|
args: enqueueArgs,
|
|
49
47
|
returns: v.id("work"),
|
|
50
48
|
handler: async (ctx, { config, ...itemArgs }) => {
|
|
51
|
-
|
|
52
|
-
const console = createLogger(
|
|
53
|
-
const kickSegment = await kickMainLoop(ctx, "enqueue",
|
|
49
|
+
const globals = await getOrUpdateGlobals(ctx, config);
|
|
50
|
+
const console = createLogger(globals.logLevel);
|
|
51
|
+
const kickSegment = await kickMainLoop(ctx, "enqueue", globals);
|
|
54
52
|
return await enqueueHandler(ctx, console, kickSegment, itemArgs);
|
|
55
53
|
},
|
|
56
54
|
});
|
|
@@ -73,29 +71,16 @@ async function enqueueHandler(
|
|
|
73
71
|
return workId;
|
|
74
72
|
}
|
|
75
73
|
|
|
76
|
-
type Config = Infer<typeof config>;
|
|
77
|
-
function validateConfig(config: Config) {
|
|
78
|
-
if (config.maxParallelism > MAX_POSSIBLE_PARALLELISM) {
|
|
79
|
-
throw new Error(`maxParallelism must be <= ${MAX_PARALLELISM_SOFT_LIMIT}`);
|
|
80
|
-
} else if (config.maxParallelism > MAX_PARALLELISM_SOFT_LIMIT) {
|
|
81
|
-
createLogger(config.logLevel).warn(
|
|
82
|
-
`maxParallelism should be <= ${MAX_PARALLELISM_SOFT_LIMIT}, but is set to ${config.maxParallelism}. This will be an error in a future version.`,
|
|
83
|
-
);
|
|
84
|
-
} else if (config.maxParallelism < 1) {
|
|
85
|
-
throw new Error("maxParallelism must be >= 1");
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
74
|
export const enqueueBatch = mutation({
|
|
90
75
|
args: {
|
|
91
76
|
items: v.array(v.object(itemArgs)),
|
|
92
|
-
config,
|
|
77
|
+
config: vConfig.partial(),
|
|
93
78
|
},
|
|
94
79
|
returns: v.array(v.id("work")),
|
|
95
80
|
handler: async (ctx, { config, items }) => {
|
|
96
|
-
|
|
97
|
-
const console = createLogger(
|
|
98
|
-
const kickSegment = await kickMainLoop(ctx, "enqueue",
|
|
81
|
+
const globals = await getOrUpdateGlobals(ctx, config);
|
|
82
|
+
const console = createLogger(globals.logLevel);
|
|
83
|
+
const kickSegment = await kickMainLoop(ctx, "enqueue", globals);
|
|
99
84
|
return Promise.all(
|
|
100
85
|
items.map((item) => enqueueHandler(ctx, console, kickSegment, item)),
|
|
101
86
|
);
|
|
@@ -105,12 +90,13 @@ export const enqueueBatch = mutation({
|
|
|
105
90
|
export const cancel = mutation({
|
|
106
91
|
args: {
|
|
107
92
|
id: v.id("work"),
|
|
108
|
-
logLevel,
|
|
93
|
+
logLevel: v.optional(logLevel),
|
|
109
94
|
},
|
|
110
95
|
handler: async (ctx, { id, logLevel }) => {
|
|
111
|
-
const
|
|
96
|
+
const globals = await getOrUpdateGlobals(ctx, { logLevel });
|
|
97
|
+
const shouldCancel = await shouldCancelWorkItem(ctx, id, globals.logLevel);
|
|
112
98
|
if (shouldCancel) {
|
|
113
|
-
const segment = await kickMainLoop(ctx, "cancel",
|
|
99
|
+
const segment = await kickMainLoop(ctx, "cancel", globals);
|
|
114
100
|
await ctx.db.insert("pendingCancelation", {
|
|
115
101
|
workId: id,
|
|
116
102
|
segment,
|
|
@@ -122,7 +108,7 @@ export const cancel = mutation({
|
|
|
122
108
|
const PAGE_SIZE = 64;
|
|
123
109
|
export const cancelAll = mutation({
|
|
124
110
|
args: {
|
|
125
|
-
logLevel,
|
|
111
|
+
logLevel: v.optional(logLevel),
|
|
126
112
|
before: v.optional(v.number()),
|
|
127
113
|
limit: v.optional(v.number()),
|
|
128
114
|
},
|
|
@@ -134,14 +120,15 @@ export const cancelAll = mutation({
|
|
|
134
120
|
.withIndex("by_creation_time", (q) => q.lte("_creationTime", beforeTime))
|
|
135
121
|
.order("desc")
|
|
136
122
|
.take(pageSize);
|
|
123
|
+
const globals = await getOrUpdateGlobals(ctx, { logLevel });
|
|
137
124
|
const shouldCancel = await Promise.all(
|
|
138
125
|
pageOfWork.map(async ({ _id }) =>
|
|
139
|
-
shouldCancelWorkItem(ctx, _id, logLevel),
|
|
126
|
+
shouldCancelWorkItem(ctx, _id, globals.logLevel),
|
|
140
127
|
),
|
|
141
128
|
);
|
|
142
129
|
let segment = getNextSegment();
|
|
143
130
|
if (shouldCancel.some((c) => c)) {
|
|
144
|
-
segment = await kickMainLoop(ctx, "cancel",
|
|
131
|
+
segment = await kickMainLoop(ctx, "cancel", globals);
|
|
145
132
|
}
|
|
146
133
|
await Promise.all(
|
|
147
134
|
pageOfWork.map(({ _id }, index) => {
|
package/src/component/loop.ts
CHANGED
|
@@ -23,7 +23,7 @@ import {
|
|
|
23
23
|
} from "./shared.js";
|
|
24
24
|
import { generateReport, recordCompleted, recordStarted } from "./stats.js";
|
|
25
25
|
|
|
26
|
-
const
|
|
26
|
+
const CANCELLATION_BATCH_SIZE = 64; // the only queue that can get unbounded.
|
|
27
27
|
const SECOND = 1000;
|
|
28
28
|
const MINUTE = 60 * SECOND;
|
|
29
29
|
const RECOVERY_THRESHOLD_MS = 5 * MINUTE; // attempt to recover jobs this old.
|
|
@@ -75,7 +75,7 @@ export const main = internalMutation({
|
|
|
75
75
|
|
|
76
76
|
// Read pendingCancelation, deleting from pendingStart. If it's still running, queue to cancel.
|
|
77
77
|
console.time("[main] pendingCancelation");
|
|
78
|
-
await handleCancelation(ctx, state, segment, console, toCancel
|
|
78
|
+
await handleCancelation(ctx, state, segment, console, toCancel);
|
|
79
79
|
console.timeEnd("[main] pendingCancelation");
|
|
80
80
|
|
|
81
81
|
if (state.running.length === 0) {
|
|
@@ -415,7 +415,6 @@ async function handleCancelation(
|
|
|
415
415
|
segment: bigint,
|
|
416
416
|
console: Logger,
|
|
417
417
|
toCancel: CompleteJob[],
|
|
418
|
-
{ cancelationBatchSize }: Config,
|
|
419
418
|
) {
|
|
420
419
|
const start = state.segmentCursors.cancelation - CURSOR_BUFFER_SEGMENTS;
|
|
421
420
|
const canceled = await ctx.db
|
|
@@ -423,7 +422,7 @@ async function handleCancelation(
|
|
|
423
422
|
.withIndex("segment", (q) =>
|
|
424
423
|
q.gte("segment", start).lte("segment", segment),
|
|
425
424
|
)
|
|
426
|
-
.take(
|
|
425
|
+
.take(CANCELLATION_BATCH_SIZE);
|
|
427
426
|
state.segmentCursors.cancelation = canceled.at(-1)?.segment ?? segment;
|
|
428
427
|
if (canceled.length) {
|
|
429
428
|
console.debug(`[main] attempting to cancel ${canceled.length}`);
|