@convex-dev/workpool 0.1.2 → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +155 -17
- package/dist/commonjs/client/index.d.ts +123 -35
- package/dist/commonjs/client/index.d.ts.map +1 -1
- package/dist/commonjs/client/index.js +122 -15
- package/dist/commonjs/client/index.js.map +1 -1
- package/dist/commonjs/client/utils.d.ts +16 -0
- package/dist/commonjs/client/utils.d.ts.map +1 -0
- package/dist/commonjs/client/utils.js +2 -0
- package/dist/commonjs/client/utils.js.map +1 -0
- package/dist/commonjs/component/complete.d.ts +89 -0
- package/dist/commonjs/component/complete.d.ts.map +1 -0
- package/dist/commonjs/component/complete.js +80 -0
- package/dist/commonjs/component/complete.js.map +1 -0
- package/dist/commonjs/component/convex.config.d.ts.map +1 -1
- package/dist/commonjs/component/convex.config.js +0 -2
- package/dist/commonjs/component/convex.config.js.map +1 -1
- package/dist/commonjs/component/kick.d.ts +9 -0
- package/dist/commonjs/component/kick.d.ts.map +1 -0
- package/dist/commonjs/component/kick.js +97 -0
- package/dist/commonjs/component/kick.js.map +1 -0
- package/dist/commonjs/component/lib.d.ts +23 -32
- package/dist/commonjs/component/lib.d.ts.map +1 -1
- package/dist/commonjs/component/lib.js +91 -563
- package/dist/commonjs/component/lib.js.map +1 -1
- package/dist/commonjs/component/logging.d.ts +5 -3
- package/dist/commonjs/component/logging.d.ts.map +1 -1
- package/dist/commonjs/component/logging.js +13 -2
- package/dist/commonjs/component/logging.js.map +1 -1
- package/dist/commonjs/component/loop.d.ts +13 -0
- package/dist/commonjs/component/loop.d.ts.map +1 -0
- package/dist/commonjs/component/loop.js +482 -0
- package/dist/commonjs/component/loop.js.map +1 -0
- package/dist/commonjs/component/recovery.d.ts +24 -0
- package/dist/commonjs/component/recovery.d.ts.map +1 -0
- package/dist/commonjs/component/recovery.js +94 -0
- package/dist/commonjs/component/recovery.js.map +1 -0
- package/dist/commonjs/component/schema.d.ts +167 -93
- package/dist/commonjs/component/schema.d.ts.map +1 -1
- package/dist/commonjs/component/schema.js +56 -65
- package/dist/commonjs/component/schema.js.map +1 -1
- package/dist/commonjs/component/shared.d.ts +138 -0
- package/dist/commonjs/component/shared.d.ts.map +1 -0
- package/dist/commonjs/component/shared.js +77 -0
- package/dist/commonjs/component/shared.js.map +1 -0
- package/dist/commonjs/component/stats.d.ts +6 -3
- package/dist/commonjs/component/stats.d.ts.map +1 -1
- package/dist/commonjs/component/stats.js +23 -4
- package/dist/commonjs/component/stats.js.map +1 -1
- package/dist/commonjs/component/worker.d.ts +15 -0
- package/dist/commonjs/component/worker.d.ts.map +1 -0
- package/dist/commonjs/component/worker.js +73 -0
- package/dist/commonjs/component/worker.js.map +1 -0
- package/dist/esm/client/index.d.ts +123 -35
- package/dist/esm/client/index.d.ts.map +1 -1
- package/dist/esm/client/index.js +122 -15
- package/dist/esm/client/index.js.map +1 -1
- package/dist/esm/client/utils.d.ts +16 -0
- package/dist/esm/client/utils.d.ts.map +1 -0
- package/dist/esm/client/utils.js +2 -0
- package/dist/esm/client/utils.js.map +1 -0
- package/dist/esm/component/complete.d.ts +89 -0
- package/dist/esm/component/complete.d.ts.map +1 -0
- package/dist/esm/component/complete.js +80 -0
- package/dist/esm/component/complete.js.map +1 -0
- package/dist/esm/component/convex.config.d.ts.map +1 -1
- package/dist/esm/component/convex.config.js +0 -2
- package/dist/esm/component/convex.config.js.map +1 -1
- package/dist/esm/component/kick.d.ts +9 -0
- package/dist/esm/component/kick.d.ts.map +1 -0
- package/dist/esm/component/kick.js +97 -0
- package/dist/esm/component/kick.js.map +1 -0
- package/dist/esm/component/lib.d.ts +23 -32
- package/dist/esm/component/lib.d.ts.map +1 -1
- package/dist/esm/component/lib.js +91 -563
- package/dist/esm/component/lib.js.map +1 -1
- package/dist/esm/component/logging.d.ts +5 -3
- package/dist/esm/component/logging.d.ts.map +1 -1
- package/dist/esm/component/logging.js +13 -2
- package/dist/esm/component/logging.js.map +1 -1
- package/dist/esm/component/loop.d.ts +13 -0
- package/dist/esm/component/loop.d.ts.map +1 -0
- package/dist/esm/component/loop.js +482 -0
- package/dist/esm/component/loop.js.map +1 -0
- package/dist/esm/component/recovery.d.ts +24 -0
- package/dist/esm/component/recovery.d.ts.map +1 -0
- package/dist/esm/component/recovery.js +94 -0
- package/dist/esm/component/recovery.js.map +1 -0
- package/dist/esm/component/schema.d.ts +167 -93
- package/dist/esm/component/schema.d.ts.map +1 -1
- package/dist/esm/component/schema.js +56 -65
- package/dist/esm/component/schema.js.map +1 -1
- package/dist/esm/component/shared.d.ts +138 -0
- package/dist/esm/component/shared.d.ts.map +1 -0
- package/dist/esm/component/shared.js +77 -0
- package/dist/esm/component/shared.js.map +1 -0
- package/dist/esm/component/stats.d.ts +6 -3
- package/dist/esm/component/stats.d.ts.map +1 -1
- package/dist/esm/component/stats.js +23 -4
- package/dist/esm/component/stats.js.map +1 -1
- package/dist/esm/component/worker.d.ts +15 -0
- package/dist/esm/component/worker.d.ts.map +1 -0
- package/dist/esm/component/worker.js +73 -0
- package/dist/esm/component/worker.js.map +1 -0
- package/package.json +6 -5
- package/src/client/index.ts +232 -68
- package/src/client/utils.ts +45 -0
- package/src/component/README.md +73 -0
- package/src/component/_generated/api.d.ts +38 -66
- package/src/component/complete.test.ts +508 -0
- package/src/component/complete.ts +98 -0
- package/src/component/convex.config.ts +0 -3
- package/src/component/kick.test.ts +285 -0
- package/src/component/kick.ts +118 -0
- package/src/component/lib.test.ts +448 -0
- package/src/component/lib.ts +105 -667
- package/src/component/logging.ts +24 -12
- package/src/component/loop.test.ts +1204 -0
- package/src/component/loop.ts +637 -0
- package/src/component/recovery.test.ts +541 -0
- package/src/component/recovery.ts +96 -0
- package/src/component/schema.ts +61 -77
- package/src/component/setup.test.ts +5 -0
- package/src/component/shared.ts +141 -0
- package/src/component/stats.ts +26 -8
- package/src/component/worker.ts +81 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { Infer } from "convex/values";
|
|
2
|
+
import { MutationCtx } from "./_generated/server.js";
|
|
3
|
+
export type CompleteJob = Infer<typeof completeArgs.fields.jobs.element>;
|
|
4
|
+
export declare const completeArgs: import("convex/values").VObject<{
|
|
5
|
+
jobs: {
|
|
6
|
+
workId: import("convex/values").GenericId<"work">;
|
|
7
|
+
runResult: {
|
|
8
|
+
kind: "success";
|
|
9
|
+
returnValue: any;
|
|
10
|
+
} | {
|
|
11
|
+
kind: "failed";
|
|
12
|
+
error: string;
|
|
13
|
+
} | {
|
|
14
|
+
kind: "canceled";
|
|
15
|
+
};
|
|
16
|
+
attempt: number;
|
|
17
|
+
}[];
|
|
18
|
+
}, {
|
|
19
|
+
jobs: import("convex/values").VArray<{
|
|
20
|
+
workId: import("convex/values").GenericId<"work">;
|
|
21
|
+
runResult: {
|
|
22
|
+
kind: "success";
|
|
23
|
+
returnValue: any;
|
|
24
|
+
} | {
|
|
25
|
+
kind: "failed";
|
|
26
|
+
error: string;
|
|
27
|
+
} | {
|
|
28
|
+
kind: "canceled";
|
|
29
|
+
};
|
|
30
|
+
attempt: number;
|
|
31
|
+
}[], import("convex/values").VObject<{
|
|
32
|
+
workId: import("convex/values").GenericId<"work">;
|
|
33
|
+
runResult: {
|
|
34
|
+
kind: "success";
|
|
35
|
+
returnValue: any;
|
|
36
|
+
} | {
|
|
37
|
+
kind: "failed";
|
|
38
|
+
error: string;
|
|
39
|
+
} | {
|
|
40
|
+
kind: "canceled";
|
|
41
|
+
};
|
|
42
|
+
attempt: number;
|
|
43
|
+
}, {
|
|
44
|
+
runResult: import("convex/values").VUnion<{
|
|
45
|
+
kind: "success";
|
|
46
|
+
returnValue: any;
|
|
47
|
+
} | {
|
|
48
|
+
kind: "failed";
|
|
49
|
+
error: string;
|
|
50
|
+
} | {
|
|
51
|
+
kind: "canceled";
|
|
52
|
+
}, [import("convex/values").VObject<{
|
|
53
|
+
kind: "success";
|
|
54
|
+
returnValue: any;
|
|
55
|
+
}, {
|
|
56
|
+
kind: import("convex/values").VLiteral<"success", "required">;
|
|
57
|
+
returnValue: import("convex/values").VAny<any, "required", string>;
|
|
58
|
+
}, "required", "kind" | "returnValue" | `returnValue.${string}`>, import("convex/values").VObject<{
|
|
59
|
+
kind: "failed";
|
|
60
|
+
error: string;
|
|
61
|
+
}, {
|
|
62
|
+
kind: import("convex/values").VLiteral<"failed", "required">;
|
|
63
|
+
error: import("convex/values").VString<string, "required">;
|
|
64
|
+
}, "required", "kind" | "error">, import("convex/values").VObject<{
|
|
65
|
+
kind: "canceled";
|
|
66
|
+
}, {
|
|
67
|
+
kind: import("convex/values").VLiteral<"canceled", "required">;
|
|
68
|
+
}, "required", "kind">], "required", "kind" | "returnValue" | `returnValue.${string}` | "error">;
|
|
69
|
+
workId: import("convex/values").VId<import("convex/values").GenericId<"work">, "required">;
|
|
70
|
+
attempt: import("convex/values").VFloat64<number, "required">;
|
|
71
|
+
}, "required", "workId" | "runResult" | "runResult.kind" | "runResult.returnValue" | `runResult.returnValue.${string}` | "runResult.error" | "attempt">, "required">;
|
|
72
|
+
}, "required", "jobs">;
|
|
73
|
+
export declare function completeHandler(ctx: MutationCtx, args: Infer<typeof completeArgs>): Promise<void>;
|
|
74
|
+
export declare const complete: import("convex/server").RegisteredMutation<"internal", {
|
|
75
|
+
jobs: {
|
|
76
|
+
workId: import("convex/values").GenericId<"work">;
|
|
77
|
+
runResult: {
|
|
78
|
+
kind: "success";
|
|
79
|
+
returnValue: any;
|
|
80
|
+
} | {
|
|
81
|
+
kind: "failed";
|
|
82
|
+
error: string;
|
|
83
|
+
} | {
|
|
84
|
+
kind: "canceled";
|
|
85
|
+
};
|
|
86
|
+
attempt: number;
|
|
87
|
+
}[];
|
|
88
|
+
}, Promise<void>>;
|
|
89
|
+
//# sourceMappingURL=complete.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"complete.d.ts","sourceRoot":"","sources":["../../../src/component/complete.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAK,MAAM,eAAe,CAAC;AACzC,OAAO,EAAoB,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAMvE,MAAM,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAEzE,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAQvB,CAAC;AACH,wBAAsB,eAAe,CACnC,GAAG,EAAE,WAAW,EAChB,IAAI,EAAE,KAAK,CAAC,OAAO,YAAY,CAAC,iBAuEjC;AAED,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;iBAGnB,CAAC"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { v } from "convex/values";
|
|
2
|
+
import { internalMutation } from "./_generated/server.js";
|
|
3
|
+
import { kickMainLoop } from "./kick.js";
|
|
4
|
+
import { createLogger } from "./logging.js";
|
|
5
|
+
import { nextSegment, runResult } from "./shared.js";
|
|
6
|
+
import { recordCompleted } from "./stats.js";
|
|
7
|
+
export const completeArgs = v.object({
|
|
8
|
+
jobs: v.array(v.object({
|
|
9
|
+
runResult: runResult,
|
|
10
|
+
workId: v.id("work"),
|
|
11
|
+
attempt: v.number(),
|
|
12
|
+
})),
|
|
13
|
+
});
|
|
14
|
+
export async function completeHandler(ctx, args) {
|
|
15
|
+
const globals = await ctx.db.query("globals").unique();
|
|
16
|
+
const console = createLogger(globals?.logLevel);
|
|
17
|
+
let anyPendingCompletions = false;
|
|
18
|
+
await Promise.all(args.jobs.map(async (job) => {
|
|
19
|
+
const work = await ctx.db.get(job.workId);
|
|
20
|
+
if (!work) {
|
|
21
|
+
console.warn(`[complete] ${job.workId} is done, but its work is gone`);
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
if (work.attempts !== job.attempt) {
|
|
25
|
+
console.warn(`[complete] ${job.workId} mismatched attempt number`);
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
work.attempts++;
|
|
29
|
+
await ctx.db.patch(work._id, { attempts: work.attempts });
|
|
30
|
+
const pendingCompletion = await ctx.db
|
|
31
|
+
.query("pendingCompletion")
|
|
32
|
+
.withIndex("workId", (q) => q.eq("workId", job.workId))
|
|
33
|
+
.unique();
|
|
34
|
+
if (pendingCompletion) {
|
|
35
|
+
console.warn(`[complete] ${job.workId} already in pendingCompletion`);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const maxAttempts = work.retryBehavior?.maxAttempts;
|
|
39
|
+
const retry = job.runResult.kind === "failed" &&
|
|
40
|
+
!!maxAttempts &&
|
|
41
|
+
work.attempts < maxAttempts;
|
|
42
|
+
if (!retry) {
|
|
43
|
+
if (work.onComplete) {
|
|
44
|
+
try {
|
|
45
|
+
const handle = work.onComplete.fnHandle;
|
|
46
|
+
await ctx.runMutation(handle, {
|
|
47
|
+
workId: work._id,
|
|
48
|
+
context: work.onComplete.context,
|
|
49
|
+
result: job.runResult,
|
|
50
|
+
});
|
|
51
|
+
console.debug(`[complete] onComplete for ${job.workId} completed`);
|
|
52
|
+
}
|
|
53
|
+
catch (e) {
|
|
54
|
+
console.error(`[complete] error running onComplete for ${job.workId}`, e);
|
|
55
|
+
// TODO: store failures in a table for later debugging
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
console.info(recordCompleted(work, job.runResult.kind));
|
|
59
|
+
// This is the terminating state for work.
|
|
60
|
+
await ctx.db.delete(job.workId);
|
|
61
|
+
}
|
|
62
|
+
if (job.runResult.kind !== "canceled") {
|
|
63
|
+
await ctx.db.insert("pendingCompletion", {
|
|
64
|
+
runResult: job.runResult,
|
|
65
|
+
workId: job.workId,
|
|
66
|
+
segment: nextSegment(),
|
|
67
|
+
retry,
|
|
68
|
+
});
|
|
69
|
+
anyPendingCompletions = true;
|
|
70
|
+
}
|
|
71
|
+
}));
|
|
72
|
+
if (anyPendingCompletions) {
|
|
73
|
+
await kickMainLoop(ctx, "complete");
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
export const complete = internalMutation({
|
|
77
|
+
args: completeArgs,
|
|
78
|
+
handler: completeHandler,
|
|
79
|
+
});
|
|
80
|
+
//# sourceMappingURL=complete.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"complete.js","sourceRoot":"","sources":["../../../src/component/complete.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,CAAC,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAe,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAkB,SAAS,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAI7C,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,IAAI,EAAE,CAAC,CAAC,KAAK,CACX,CAAC,CAAC,MAAM,CAAC;QACP,SAAS,EAAE,SAAS;QACpB,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;KACpB,CAAC,CACH;CACF,CAAC,CAAC;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,GAAgB,EAChB,IAAgC;IAEhC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;IACvD,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAChD,IAAI,qBAAqB,GAAG,KAAK,CAAC;IAClC,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,IAAI,EAAE;YACT,OAAO,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,MAAM,gCAAgC,CAAC,CAAC;YACvE,OAAO;SACR;QACD,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,CAAC,OAAO,EAAE;YACjC,OAAO,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,MAAM,4BAA4B,CAAC,CAAC;YACnE,OAAO;SACR;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC1D,MAAM,iBAAiB,GAAG,MAAM,GAAG,CAAC,EAAE;aACnC,KAAK,CAAC,mBAAmB,CAAC;aAC1B,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;aACtD,MAAM,EAAE,CAAC;QACZ,IAAI,iBAAiB,EAAE;YACrB,OAAO,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,MAAM,+BAA+B,CAAC,CAAC;YACtE,OAAO;SACR;QACD,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC;QACpD,MAAM,KAAK,GACT,GAAG,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ;YAC/B,CAAC,CAAC,WAAW;YACb,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC;QAC9B,IAAI,CAAC,KAAK,EAAE;YACV,IAAI,IAAI,CAAC,UAAU,EAAE;gBACnB,IAAI;oBACF,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAI9B,CAAC;oBACF,MAAM,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE;wBAC5B,MAAM,EAAE,IAAI,CAAC,GAAG;wBAChB,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO;wBAChC,MAAM,EAAE,GAAG,CAAC,SAAS;qBACtB,CAAC,CAAC;oBACH,OAAO,CAAC,KAAK,CAAC,6BAA6B,GAAG,CAAC,MAAM,YAAY,CAAC,CAAC;iBACpE;gBAAC,OAAO,CAAC,EAAE;oBACV,OAAO,CAAC,KAAK,CACX,2CAA2C,GAAG,CAAC,MAAM,EAAE,EACvD,CAAC,CACF,CAAC;oBACF,sDAAsD;iBACvD;aACF;YACD,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;YACxD,0CAA0C;YAC1C,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SACjC;QACD,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,EAAE;YACrC,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,mBAAmB,EAAE;gBACvC,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,OAAO,EAAE,WAAW,EAAE;gBACtB,KAAK;aACN,CAAC,CAAC;YACH,qBAAqB,GAAG,IAAI,CAAC;SAC9B;IACH,CAAC,CAAC,CACH,CAAC;IACF,IAAI,qBAAqB,EAAE;QACzB,MAAM,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;KACrC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,QAAQ,GAAG,gBAAgB,CAAC;IACvC,IAAI,EAAE,YAAY;IAClB,OAAO,EAAE,eAAe;CACzB,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convex.config.d.ts","sourceRoot":"","sources":["../../../src/component/convex.config.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"convex.config.d.ts","sourceRoot":"","sources":["../../../src/component/convex.config.ts"],"names":[],"mappings":"AAEA,QAAA,MAAM,SAAS,kDAA8B,CAAC;AAE9C,eAAe,SAAS,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convex.config.js","sourceRoot":"","sources":["../../../src/component/convex.config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"convex.config.js","sourceRoot":"","sources":["../../../src/component/convex.config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;AAE9C,eAAe,SAAS,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { MutationCtx } from "./_generated/server.js";
|
|
2
|
+
import { Config } from "./shared.js";
|
|
3
|
+
export declare const DEFAULT_MAX_PARALLELISM = 10;
|
|
4
|
+
/**
|
|
5
|
+
* Called from outside the loop:
|
|
6
|
+
*/
|
|
7
|
+
export declare function kickMainLoop(ctx: MutationCtx, source: "enqueue" | "cancel" | "complete", config?: Partial<Config>): Promise<void>;
|
|
8
|
+
export declare const forceKick: import("convex/server").RegisteredMutation<"internal", {}, Promise<void>>;
|
|
9
|
+
//# sourceMappingURL=kick.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"kick.d.ts","sourceRoot":"","sources":["../../../src/component/kick.ts"],"names":[],"mappings":"AACA,OAAO,EAAoB,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAGvE,OAAO,EAAE,MAAM,EAAe,MAAM,aAAa,CAAC;AAElD,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAC1C;;GAEG;AAEH,wBAAsB,YAAY,CAChC,GAAG,EAAE,WAAW,EAChB,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,UAAU,EACzC,MAAM,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,GACvB,OAAO,CAAC,IAAI,CAAC,CA+Cf;AAED,eAAO,MAAM,SAAS,2EAOpB,CAAC"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { internal } from "./_generated/api.js";
|
|
2
|
+
import { internalMutation } from "./_generated/server.js";
|
|
3
|
+
import { createLogger, DEFAULT_LOG_LEVEL } from "./logging.js";
|
|
4
|
+
import { INITIAL_STATE } from "./loop.js";
|
|
5
|
+
import { nextSegment } from "./shared.js";
|
|
6
|
+
export const DEFAULT_MAX_PARALLELISM = 10;
|
|
7
|
+
/**
|
|
8
|
+
* Called from outside the loop:
|
|
9
|
+
*/
|
|
10
|
+
export async function kickMainLoop(ctx, source, config) {
|
|
11
|
+
const globals = await getOrUpdateGlobals(ctx, config);
|
|
12
|
+
const console = createLogger(globals.logLevel);
|
|
13
|
+
const runStatus = await getOrCreateRunStatus(ctx);
|
|
14
|
+
// Only kick to run now if we're scheduled or idle.
|
|
15
|
+
if (runStatus.state.kind === "running") {
|
|
16
|
+
console.debug(`[${source}] main is actively running, so we don't need to kick it`);
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
const segment = nextSegment();
|
|
20
|
+
// main is scheduled to run later, so we should cancel it and reschedule.
|
|
21
|
+
if (runStatus.state.kind === "scheduled") {
|
|
22
|
+
if (source === "enqueue" && runStatus.state.saturated) {
|
|
23
|
+
console.debug(`[${source}] main is saturated, so we don't need to kick it`);
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
if (runStatus.state.segment <= segment) {
|
|
27
|
+
console.debug(`[${source}] main is scheduled to run soon enough, so we don't need to kick it`);
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
console.debug(`[${source}] main is scheduled to run later, so reschedule it to run now`);
|
|
31
|
+
const scheduled = await ctx.db.system.get(runStatus.state.scheduledId);
|
|
32
|
+
if (scheduled && scheduled.state.kind === "pending") {
|
|
33
|
+
await ctx.scheduler.cancel(runStatus.state.scheduledId);
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
console.warn(`[${source}] main is marked as scheduled, but it's status is ${scheduled?.state.kind}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
console.debug(`[${source}] main was scheduled later, so reschedule it to run now`);
|
|
40
|
+
await ctx.db.patch(runStatus._id, { state: { kind: "running" } });
|
|
41
|
+
await ctx.scheduler.runAfter(0, internal.loop.main, {
|
|
42
|
+
generation: runStatus.state.generation,
|
|
43
|
+
segment,
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
export const forceKick = internalMutation({
|
|
47
|
+
args: {},
|
|
48
|
+
handler: async (ctx) => {
|
|
49
|
+
const runStatus = await getOrCreateRunStatus(ctx);
|
|
50
|
+
await ctx.db.delete(runStatus._id);
|
|
51
|
+
await kickMainLoop(ctx, "complete");
|
|
52
|
+
},
|
|
53
|
+
});
|
|
54
|
+
async function getOrCreateRunStatus(ctx) {
|
|
55
|
+
let runStatus = await ctx.db.query("runStatus").unique();
|
|
56
|
+
if (!runStatus) {
|
|
57
|
+
const state = await ctx.db.query("internalState").unique();
|
|
58
|
+
const id = await ctx.db.insert("runStatus", {
|
|
59
|
+
state: {
|
|
60
|
+
kind: "idle",
|
|
61
|
+
generation: state?.generation ?? INITIAL_STATE.generation,
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
runStatus = (await ctx.db.get(id));
|
|
65
|
+
if (!state) {
|
|
66
|
+
await ctx.db.insert("internalState", INITIAL_STATE);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return runStatus;
|
|
70
|
+
}
|
|
71
|
+
async function getOrUpdateGlobals(ctx, config) {
|
|
72
|
+
const globals = await ctx.db.query("globals").unique();
|
|
73
|
+
if (!globals) {
|
|
74
|
+
const id = await ctx.db.insert("globals", {
|
|
75
|
+
maxParallelism: config?.maxParallelism ?? DEFAULT_MAX_PARALLELISM,
|
|
76
|
+
logLevel: config?.logLevel ?? DEFAULT_LOG_LEVEL,
|
|
77
|
+
});
|
|
78
|
+
return (await ctx.db.get(id));
|
|
79
|
+
}
|
|
80
|
+
else if (config) {
|
|
81
|
+
let updated = false;
|
|
82
|
+
if (config.maxParallelism &&
|
|
83
|
+
config.maxParallelism !== globals.maxParallelism) {
|
|
84
|
+
globals.maxParallelism = config.maxParallelism;
|
|
85
|
+
updated = true;
|
|
86
|
+
}
|
|
87
|
+
if (config.logLevel && config.logLevel !== globals.logLevel) {
|
|
88
|
+
globals.logLevel = config.logLevel;
|
|
89
|
+
updated = true;
|
|
90
|
+
}
|
|
91
|
+
if (updated) {
|
|
92
|
+
await ctx.db.replace(globals._id, globals);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return globals;
|
|
96
|
+
}
|
|
97
|
+
//# sourceMappingURL=kick.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"kick.js","sourceRoot":"","sources":["../../../src/component/kick.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAe,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAU,WAAW,EAAE,MAAM,aAAa,CAAC;AAElD,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAC1C;;GAEG;AAEH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAgB,EAChB,MAAyC,EACzC,MAAwB;IAExB,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;IAElD,mDAAmD;IACnD,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE;QACtC,OAAO,CAAC,KAAK,CACX,IAAI,MAAM,yDAAyD,CACpE,CAAC;QACF,OAAO;KACR;IACD,MAAM,OAAO,GAAG,WAAW,EAAE,CAAC;IAC9B,yEAAyE;IACzE,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE;QACxC,IAAI,MAAM,KAAK,SAAS,IAAI,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE;YACrD,OAAO,CAAC,KAAK,CACX,IAAI,MAAM,kDAAkD,CAC7D,CAAC;YACF,OAAO;SACR;QACD,IAAI,SAAS,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,EAAE;YACtC,OAAO,CAAC,KAAK,CACX,IAAI,MAAM,qEAAqE,CAChF,CAAC;YACF,OAAO;SACR;QACD,OAAO,CAAC,KAAK,CACX,IAAI,MAAM,+DAA+D,CAC1E,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACvE,IAAI,SAAS,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE;YACnD,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;SACzD;aAAM;YACL,OAAO,CAAC,IAAI,CACV,IAAI,MAAM,qDAAqD,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,CACvF,CAAC;SACH;KACF;IACD,OAAO,CAAC,KAAK,CACX,IAAI,MAAM,yDAAyD,CACpE,CAAC;IACF,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;IAClE,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE;QAClD,UAAU,EAAE,SAAS,CAAC,KAAK,CAAC,UAAU;QACtC,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAC;IACxC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;QAClD,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACtC,CAAC;CACF,CAAC,CAAC;AAEH,KAAK,UAAU,oBAAoB,CAAC,GAAgB;IAClD,IAAI,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC;IACzD,IAAI,CAAC,SAAS,EAAE;QACd,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3D,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE;YAC1C,KAAK,EAAE;gBACL,IAAI,EAAE,MAAM;gBACZ,UAAU,EAAE,KAAK,EAAE,UAAU,IAAI,aAAa,CAAC,UAAU;aAC1D;SACF,CAAC,CAAC;QACH,SAAS,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAE,CAAC;QACpC,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;SACrD;KACF;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,GAAgB,EAAE,MAAwB;IAC1E,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;IACvD,IAAI,CAAC,OAAO,EAAE;QACZ,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE;YACxC,cAAc,EAAE,MAAM,EAAE,cAAc,IAAI,uBAAuB;YACjE,QAAQ,EAAE,MAAM,EAAE,QAAQ,IAAI,iBAAiB;SAChD,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAE,CAAC;KAChC;SAAM,IAAI,MAAM,EAAE;QACjB,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IACE,MAAM,CAAC,cAAc;YACrB,MAAM,CAAC,cAAc,KAAK,OAAO,CAAC,cAAc,EAChD;YACA,OAAO,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;YAC/C,OAAO,GAAG,IAAI,CAAC;SAChB;QACD,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,EAAE;YAC3D,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YACnC,OAAO,GAAG,IAAI,CAAC;SAChB;QACD,IAAI,OAAO,EAAE;YACX,MAAM,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAC5C;KACF;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -1,50 +1,41 @@
|
|
|
1
1
|
export declare const enqueue: import("convex/server").RegisteredMutation<"public", {
|
|
2
|
-
|
|
2
|
+
onComplete?: {
|
|
3
|
+
context?: any;
|
|
4
|
+
fnHandle: string;
|
|
5
|
+
} | undefined;
|
|
6
|
+
retryBehavior?: {
|
|
7
|
+
maxAttempts: number;
|
|
8
|
+
initialBackoffMs: number;
|
|
9
|
+
base: number;
|
|
10
|
+
} | undefined;
|
|
3
11
|
fnHandle: string;
|
|
12
|
+
fnType: "action" | "mutation";
|
|
4
13
|
fnName: string;
|
|
5
14
|
fnArgs: any;
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
logLevel?: "DEBUG" | "INFO" | "WARN" | "ERROR" | undefined;
|
|
15
|
+
runAt: number;
|
|
16
|
+
config: {
|
|
9
17
|
maxParallelism: number;
|
|
18
|
+
logLevel: "DEBUG" | "INFO" | "WARN" | "ERROR";
|
|
10
19
|
};
|
|
11
20
|
}, Promise<import("convex/values").GenericId<"work">>>;
|
|
12
21
|
export declare const cancel: import("convex/server").RegisteredMutation<"public", {
|
|
13
22
|
id: import("convex/values").GenericId<"work">;
|
|
23
|
+
logLevel: "DEBUG" | "INFO" | "WARN" | "ERROR";
|
|
14
24
|
}, Promise<void>>;
|
|
15
|
-
export declare const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
export declare const runActionWrapper: import("convex/server").RegisteredAction<"internal", {
|
|
19
|
-
fnHandle: string;
|
|
20
|
-
fnArgs: any;
|
|
21
|
-
workId: import("convex/values").GenericId<"work">;
|
|
22
|
-
}, Promise<void>>;
|
|
23
|
-
export declare const saveResult: import("convex/server").RegisteredMutation<"internal", {
|
|
24
|
-
workId: import("convex/values").GenericId<"work">;
|
|
25
|
-
completionStatus: "success" | "error" | "canceled" | "timeout";
|
|
25
|
+
export declare const cancelAll: import("convex/server").RegisteredMutation<"public", {
|
|
26
|
+
before?: number | undefined;
|
|
27
|
+
logLevel: "DEBUG" | "INFO" | "WARN" | "ERROR";
|
|
26
28
|
}, Promise<void>>;
|
|
27
|
-
export declare const bumpGeneration: import("convex/server").RegisteredMutation<"internal", {}, Promise<void>>;
|
|
28
|
-
export declare const runMutationWrapper: import("convex/server").RegisteredMutation<"internal", {
|
|
29
|
-
fnHandle: string;
|
|
30
|
-
fnArgs: any;
|
|
31
|
-
workId: import("convex/values").GenericId<"work">;
|
|
32
|
-
}, Promise<void>>;
|
|
33
|
-
export declare const stopCleanup: import("convex/server").RegisteredMutation<"public", {}, Promise<void>>;
|
|
34
29
|
export declare const status: import("convex/server").RegisteredQuery<"public", {
|
|
35
30
|
id: import("convex/values").GenericId<"work">;
|
|
36
31
|
}, Promise<{
|
|
37
|
-
readonly
|
|
38
|
-
readonly
|
|
32
|
+
readonly state: "finished";
|
|
33
|
+
readonly previousAttempts?: undefined;
|
|
39
34
|
} | {
|
|
40
|
-
readonly
|
|
41
|
-
readonly
|
|
35
|
+
readonly state: "pending";
|
|
36
|
+
readonly previousAttempts: number;
|
|
42
37
|
} | {
|
|
43
|
-
readonly
|
|
44
|
-
readonly
|
|
38
|
+
readonly state: "running";
|
|
39
|
+
readonly previousAttempts: number;
|
|
45
40
|
}>>;
|
|
46
|
-
export declare const MAX_CLEANUP_DOCS = 1000;
|
|
47
|
-
export declare const cleanup: import("convex/server").RegisteredMutation<"public", {
|
|
48
|
-
maxAgeMs: number;
|
|
49
|
-
}, Promise<void>>;
|
|
50
41
|
//# sourceMappingURL=lib.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lib.d.ts","sourceRoot":"","sources":["../../../src/component/lib.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"lib.d.ts","sourceRoot":"","sources":["../../../src/component/lib.ts"],"names":[],"mappings":"AAmBA,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;sDAkClB,CAAC;AAEH,eAAO,MAAM,MAAM;;;iBAYjB,CAAC;AAGH,eAAO,MAAM,SAAS;;;iBAyBpB,CAAC;AAEH,eAAO,MAAM,MAAM;;;;;;;;;;;GAyBjB,CAAC"}
|