@convex-dev/workpool 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component/complete.d.ts +0 -22
- package/dist/component/complete.d.ts.map +1 -1
- package/dist/component/complete.js +5 -7
- package/dist/component/complete.js.map +1 -1
- package/dist/component/loop.d.ts.map +1 -1
- package/dist/component/loop.js +10 -36
- package/dist/component/loop.js.map +1 -1
- package/dist/component/recovery.d.ts.map +1 -1
- package/dist/component/recovery.js +0 -17
- package/dist/component/recovery.js.map +1 -1
- package/dist/component/schema.d.ts +2 -62
- package/dist/component/schema.d.ts.map +1 -1
- package/dist/component/schema.js +1 -5
- package/dist/component/schema.js.map +1 -1
- package/dist/component/stats.d.ts +1 -2
- package/dist/component/stats.d.ts.map +1 -1
- package/dist/component/stats.js.map +1 -1
- package/package.json +1 -1
- package/src/component/complete.test.ts +0 -60
- package/src/component/complete.ts +8 -10
- package/src/component/loop.test.ts +0 -83
- package/src/component/loop.ts +10 -42
- package/src/component/recovery.test.ts +2 -115
- package/src/component/recovery.ts +0 -17
- package/src/component/schema.ts +2 -11
- package/src/component/stats.test.ts +0 -5
- package/src/component/stats.ts +1 -6
|
@@ -13,8 +13,6 @@ declare const completeArgs: import("convex/values").VObject<{
|
|
|
13
13
|
error: string;
|
|
14
14
|
} | {
|
|
15
15
|
kind: "canceled";
|
|
16
|
-
} | {
|
|
17
|
-
kind: "stuckInScheduler";
|
|
18
16
|
};
|
|
19
17
|
attempt: number;
|
|
20
18
|
}[];
|
|
@@ -30,8 +28,6 @@ declare const completeArgs: import("convex/values").VObject<{
|
|
|
30
28
|
error: string;
|
|
31
29
|
} | {
|
|
32
30
|
kind: "canceled";
|
|
33
|
-
} | {
|
|
34
|
-
kind: "stuckInScheduler";
|
|
35
31
|
};
|
|
36
32
|
attempt: number;
|
|
37
33
|
}[], import("convex/values").VObject<{
|
|
@@ -45,8 +41,6 @@ declare const completeArgs: import("convex/values").VObject<{
|
|
|
45
41
|
error: string;
|
|
46
42
|
} | {
|
|
47
43
|
kind: "canceled";
|
|
48
|
-
} | {
|
|
49
|
-
kind: "stuckInScheduler";
|
|
50
44
|
};
|
|
51
45
|
attempt: number;
|
|
52
46
|
}, {
|
|
@@ -58,16 +52,6 @@ declare const completeArgs: import("convex/values").VObject<{
|
|
|
58
52
|
error: string;
|
|
59
53
|
} | {
|
|
60
54
|
kind: "canceled";
|
|
61
|
-
} | {
|
|
62
|
-
kind: "stuckInScheduler";
|
|
63
|
-
}, [import("convex/values").VUnion<{
|
|
64
|
-
kind: "success";
|
|
65
|
-
returnValue: any;
|
|
66
|
-
} | {
|
|
67
|
-
kind: "failed";
|
|
68
|
-
error: string;
|
|
69
|
-
} | {
|
|
70
|
-
kind: "canceled";
|
|
71
55
|
}, [import("convex/values").VObject<{
|
|
72
56
|
kind: "success";
|
|
73
57
|
returnValue: any;
|
|
@@ -84,10 +68,6 @@ declare const completeArgs: import("convex/values").VObject<{
|
|
|
84
68
|
kind: "canceled";
|
|
85
69
|
}, {
|
|
86
70
|
kind: import("convex/values").VLiteral<"canceled", "required">;
|
|
87
|
-
}, "required", "kind">], "required", "kind" | "returnValue" | `returnValue.${string}` | "error">, import("convex/values").VObject<{
|
|
88
|
-
kind: "stuckInScheduler";
|
|
89
|
-
}, {
|
|
90
|
-
kind: import("convex/values").VLiteral<"stuckInScheduler", "required">;
|
|
91
71
|
}, "required", "kind">], "required", "kind" | "returnValue" | `returnValue.${string}` | "error">;
|
|
92
72
|
workId: import("convex/values").VId<import("convex/values").GenericId<"work">, "required">;
|
|
93
73
|
attempt: import("convex/values").VFloat64<number, "required">;
|
|
@@ -107,8 +87,6 @@ export declare const complete: import("convex/server").RegisteredMutation<"inter
|
|
|
107
87
|
error: string;
|
|
108
88
|
} | {
|
|
109
89
|
kind: "canceled";
|
|
110
|
-
} | {
|
|
111
|
-
kind: "stuckInScheduler";
|
|
112
90
|
};
|
|
113
91
|
attempt: number;
|
|
114
92
|
}[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"complete.d.ts","sourceRoot":"","sources":["../../src/component/complete.ts"],"names":[],"mappings":"AACA,OAAO,EAAiB,KAAK,KAAK,EAAK,MAAM,eAAe,CAAC;AAG7D,OAAO,EAAoB,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"complete.d.ts","sourceRoot":"","sources":["../../src/component/complete.ts"],"names":[],"mappings":"AACA,OAAO,EAAiB,KAAK,KAAK,EAAK,MAAM,eAAe,CAAC;AAG7D,OAAO,EAAoB,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAO5E,MAAM,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAEzE,QAAA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAUhB,CAAC;AACH,wBAAsB,eAAe,CACnC,GAAG,EAAE,WAAW,EAChB,IAAI,EAAE,KAAK,CAAC,OAAO,YAAY,CAAC,iBAkLjC;AASD,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;iBAGnB,CAAC"}
|
|
@@ -3,13 +3,12 @@ import { internal } from "./_generated/api.js";
|
|
|
3
3
|
import { internalMutation } from "./_generated/server.js";
|
|
4
4
|
import { kickMainLoop } from "./kick.js";
|
|
5
5
|
import { createLogger } from "./logging.js";
|
|
6
|
-
import {} from "./shared.js";
|
|
6
|
+
import { vResult } from "./shared.js";
|
|
7
7
|
import { recordCompleted } from "./stats.js";
|
|
8
8
|
import { assert } from "convex-helpers";
|
|
9
|
-
import { vResultInternal } from "./schema.js";
|
|
10
9
|
const completeArgs = v.object({
|
|
11
10
|
jobs: v.array(v.object({
|
|
12
|
-
runResult:
|
|
11
|
+
runResult: vResult,
|
|
13
12
|
workId: v.id("work"),
|
|
14
13
|
attempt: v.number(),
|
|
15
14
|
// TODO: need to be careful about removing this field later
|
|
@@ -82,11 +81,10 @@ export async function completeHandler(ctx, args) {
|
|
|
82
81
|
return;
|
|
83
82
|
}
|
|
84
83
|
const maxAttempts = work.retryBehavior?.maxAttempts;
|
|
85
|
-
const retry =
|
|
84
|
+
const retry = job.runResult.kind === "failed" &&
|
|
86
85
|
!!maxAttempts &&
|
|
87
|
-
work.attempts < maxAttempts
|
|
88
|
-
|
|
89
|
-
if (!retry && job.runResult.kind !== "stuckInScheduler") {
|
|
86
|
+
work.attempts < maxAttempts;
|
|
87
|
+
if (!retry) {
|
|
90
88
|
let scheduledId = undefined;
|
|
91
89
|
if (work.onComplete) {
|
|
92
90
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"complete.js","sourceRoot":"","sources":["../../src/component/complete.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAc,CAAC,EAAE,MAAM,eAAe,CAAC;AAE7D,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAoB,MAAM,wBAAwB,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,
|
|
1
|
+
{"version":3,"file":"complete.js","sourceRoot":"","sources":["../../src/component/complete.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAc,CAAC,EAAE,MAAM,eAAe,CAAC;AAE7D,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAoB,MAAM,wBAAwB,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAuC,OAAO,EAAE,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAIxC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,IAAI,EAAE,CAAC,CAAC,KAAK,CACX,CAAC,CAAC,MAAM,CAAC;QACP,SAAS,EAAE,OAAO;QAClB,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,2DAA2D;QAC3D,mBAAmB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;KAC7C,CAAC,CACH;CACF,CAAC,CAAC;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,GAAgB,EAChB,IAAgC;IAEhC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;IACvD,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAChD,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,OAAO;IACT,CAAC;IACD,MAAM,kBAAkB,GAIlB,EAAE,CAAC;IACT,MAAM,WAAW,GAAG,CAClB,MAAM,OAAO,CAAC,GAAG,CACf,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,CAAC,IAAI,CACV,cAAc,GAAG,CAAC,MAAM,gCAAgC,CACzD,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,MAAM,4BAA4B,CAAC,CAAC;YACnE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;IACvB,CAAC,CAAC,CACH,CACF,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAC5B,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO;IACT,CAAC;IACD,MAAM,cAAc,GAAG,SAAS,CAAC,CAAC,qCAAqC;IAEvE,+BAA+B;IAC/B,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,YAAY,GAAuB,EAAE,CAAC;IAC1C,IAAI,gBAAgB,GAAG,CAAC,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,QAAQ,GACZ,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC;YACvB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YACxB,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;QAE/B,gEAAgE;QAChE,IACE,YAAY,CAAC,MAAM,GAAG,CAAC;YACvB,gBAAgB,GAAG,QAAQ,GAAG,cAAc,EAC5C,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3B,YAAY,GAAG,EAAE,CAAC;YAClB,gBAAgB,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,gBAAgB,IAAI,QAAQ,CAAC;IAC/B,CAAC;IAED,qCAAqC;IACrC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC7B,CAAC;IAED,2CAA2C;IAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE;YAC1D,IAAI,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC;SACxC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEjB,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE;QACnC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC1D,MAAM,iBAAiB,GAAG,MAAM,GAAG,CAAC,EAAE;aACnC,KAAK,CAAC,mBAAmB,CAAC;aAC1B,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;aACtD,MAAM,EAAE,CAAC;QACZ,IAAI,iBAAiB,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,MAAM,+BAA+B,CAAC,CAAC;YACtE,OAAO;QACT,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC;QACpD,MAAM,KAAK,GACT,GAAG,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ;YAC/B,CAAC,CAAC,WAAW;YACb,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC;QAC9B,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,WAAW,GAAG,SAAS,CAAC;YAC5B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,IAAI,CAAC;oBACH,8CAA8C;oBAC9C,IAAI,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;oBACtC,IAAI,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;wBAC5C,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;wBACjD,IAAI,OAAO,EAAE,CAAC;4BACZ,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;wBAC5B,CAAC;oBACH,CAAC;oBAED,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAI9B,CAAC;oBACF,MAAM,cAAc,GAAG;wBACrB,MAAM,EAAE,IAAI,CAAC,GAAG;wBAChB,OAAO;wBACP,MAAM,EAAE,GAAG,CAAC,SAAS;qBACtB,CAAC;oBACF,IAAI,GAAG,CAAC,mBAAmB,EAAE,CAAC;wBAC5B,IAAI,CAAC;4BACH,MAAM,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;wBAChD,CAAC;wBAAC,OAAO,CAAC,EAAE,CAAC;4BACX,OAAO,CAAC,KAAK,CACX,+DAA+D,GAAG,CAAC,MAAM,yBAAyB,CAAC,EAAE,CACtG,CAAC;4BACF,WAAW,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CACxC,CAAC,EACD,MAAM,EACN,cAAc,CACf,CAAC;wBACJ,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,WAAW,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CACxC,CAAC,EACD,MAAM,EACN,cAAc,CACf,CAAC;wBACF,OAAO,CAAC,KAAK,CACX,6BAA6B,GAAG,CAAC,MAAM,YAAY,CACpD,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACX,2CAA2C,GAAG,CAAC,MAAM,EAAE,EACvD,CAAC,CACF,CAAC;oBACF,sDAAsD;gBACxD,CAAC;YACH,CAAC;YACD,eAAe,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAEhE,sDAAsD;YACtD,yEAAyE;YACzE,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,CAAC;YAED,0CAA0C;YAC1C,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACtC,kBAAkB,CAAC,IAAI,CAAC;gBACtB,SAAS,EAAE,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC;gBACrC,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IACF,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACpD,MAAM,OAAO,CAAC,GAAG,CACf,kBAAkB,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACpC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACjC,GAAG,UAAU;YACb,OAAO;SACR,CAAC,CACH,CACF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,MAAiB;IACpC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAChD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,QAAQ,GAAG,gBAAgB,CAAC;IACvC,IAAI,EAAE,YAAY;IAClB,OAAO,EAAE,eAAe;CACzB,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loop.d.ts","sourceRoot":"","sources":["../../src/component/loop.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAGzD,OAAO,KAAK,EAAE,GAAG,EAAM,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"loop.d.ts","sourceRoot":"","sources":["../../src/component/loop.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAGzD,OAAO,KAAK,EAAE,GAAG,EAAM,MAAM,2BAA2B,CAAC;AA4BzD,eAAO,MAAM,wBAAwB,QAAwB,CAAC;AAC9D,eAAO,MAAM,eAAe,QAAa,CAAC;AAC1C,eAAO,MAAM,uBAAuB,QAAW,CAAC;AAEhD,eAAO,MAAM,aAAa,EAAE,mBAAmB,CAAC,GAAG,CAAC,eAAe,CAAC,CAanE,CAAC;AAGF,eAAO,MAAM,IAAI;;;iBAyEf,CAAC;AAEH,eAAO,MAAM,eAAe;;;iBAiJ1B,CAAC;AAqYH,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,UAEvC"}
|
package/dist/component/loop.js
CHANGED
|
@@ -9,8 +9,7 @@ const RECOVERY_BATCH_SIZE = 32;
|
|
|
9
9
|
const MS = 1;
|
|
10
10
|
const SECOND = 1000 * MS;
|
|
11
11
|
const MINUTE = 60 * SECOND;
|
|
12
|
-
const
|
|
13
|
-
const MUTATION_RECOVERY_THRESHOLD_MS = 1 * MINUTE; // attempt to recover jobs this old.
|
|
12
|
+
const RECOVERY_THRESHOLD_MS = 5 * MINUTE; // attempt to recover jobs this old.
|
|
14
13
|
export const RECOVERY_PERIOD_SEGMENTS = toSegment(1 * MINUTE); // how often to check.
|
|
15
14
|
export const STATUS_COOLDOWN = 2 * SECOND;
|
|
16
15
|
export const COOLDOWN_CHECK_INTERVAL = 200 * MS;
|
|
@@ -25,7 +24,6 @@ export const INITIAL_STATE = {
|
|
|
25
24
|
failed: 0,
|
|
26
25
|
retries: 0,
|
|
27
26
|
canceled: 0,
|
|
28
|
-
conflicted: 0,
|
|
29
27
|
lastReportTs: 0,
|
|
30
28
|
},
|
|
31
29
|
running: [],
|
|
@@ -86,7 +84,6 @@ export const main = internalMutation({
|
|
|
86
84
|
failed: 0,
|
|
87
85
|
retries: 0,
|
|
88
86
|
canceled: 0,
|
|
89
|
-
conflicted: 0,
|
|
90
87
|
lastReportTs,
|
|
91
88
|
};
|
|
92
89
|
}
|
|
@@ -293,17 +290,10 @@ async function handleCompletions(ctx, state, segment, console) {
|
|
|
293
290
|
console.warn(`[main] ${c.workId} is gone, but trying to complete`);
|
|
294
291
|
return;
|
|
295
292
|
}
|
|
296
|
-
const
|
|
297
|
-
const retried = await rescheduleJob(ctx, work, console, wasStuckInScheduler);
|
|
293
|
+
const retried = await rescheduleJob(ctx, work, console);
|
|
298
294
|
if (retried) {
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
recordCompleted(console, work, "retrying conflicted", undefined);
|
|
302
|
-
}
|
|
303
|
-
else {
|
|
304
|
-
state.report.retries++;
|
|
305
|
-
recordCompleted(console, work, "retrying", undefined);
|
|
306
|
-
}
|
|
295
|
+
state.report.retries++;
|
|
296
|
+
recordCompleted(console, work, "retrying", undefined);
|
|
307
297
|
}
|
|
308
298
|
else {
|
|
309
299
|
// We don't retry if it's been canceled in the mean time.
|
|
@@ -377,12 +367,9 @@ async function handleCancelation(ctx, state, segment, console, toCancel) {
|
|
|
377
367
|
}
|
|
378
368
|
async function handleRecovery(ctx, state, console) {
|
|
379
369
|
const missing = new Set();
|
|
380
|
-
const
|
|
381
|
-
const mutationOldEnoughToConsider = Date.now() - MUTATION_RECOVERY_THRESHOLD_MS;
|
|
370
|
+
const oldEnoughToConsider = Date.now() - RECOVERY_THRESHOLD_MS;
|
|
382
371
|
const jobs = (await Promise.all(state.running.map(async (r) => {
|
|
383
|
-
if (r.started >=
|
|
384
|
-
Math.max(actionOldEnoughToConsider, mutationOldEnoughToConsider)) {
|
|
385
|
-
// Avoid getting the work if possible
|
|
372
|
+
if (r.started >= oldEnoughToConsider) {
|
|
386
373
|
return null;
|
|
387
374
|
}
|
|
388
375
|
const work = await ctx.db.get(r.workId);
|
|
@@ -391,12 +378,6 @@ async function handleRecovery(ctx, state, console) {
|
|
|
391
378
|
console.error(`[main] ${r.workId} already gone (skipping recovery)`);
|
|
392
379
|
return null;
|
|
393
380
|
}
|
|
394
|
-
const oldEnoughToConsider = work.fnType === "action"
|
|
395
|
-
? actionOldEnoughToConsider
|
|
396
|
-
: mutationOldEnoughToConsider;
|
|
397
|
-
if (r.started >= oldEnoughToConsider) {
|
|
398
|
-
return null;
|
|
399
|
-
}
|
|
400
381
|
return { ...r, attempt: work.attempts };
|
|
401
382
|
}))).flatMap((r) => (r ? [r] : []));
|
|
402
383
|
state.running = state.running.filter((r) => !missing.has(r.workId));
|
|
@@ -467,7 +448,7 @@ async function beginWork(ctx, workId, logLevel, lagMs) {
|
|
|
467
448
|
* If it's been canceled in the mean time, don't retry.
|
|
468
449
|
* @returns true if the job was rescheduled, false if it was not.
|
|
469
450
|
*/
|
|
470
|
-
async function rescheduleJob(ctx, work, console
|
|
451
|
+
async function rescheduleJob(ctx, work, console) {
|
|
471
452
|
const pendingCancelation = await ctx.db
|
|
472
453
|
.query("pendingCancelation")
|
|
473
454
|
.withIndex("workId", (q) => q.eq("workId", work._id))
|
|
@@ -480,16 +461,7 @@ async function rescheduleJob(ctx, work, console, wasStuckInScheduler) {
|
|
|
480
461
|
if (work.canceled) {
|
|
481
462
|
return false;
|
|
482
463
|
}
|
|
483
|
-
|
|
484
|
-
if (wasStuckInScheduler) {
|
|
485
|
-
backoffMs = 0;
|
|
486
|
-
}
|
|
487
|
-
else if (work.retryBehavior) {
|
|
488
|
-
backoffMs =
|
|
489
|
-
work.retryBehavior.initialBackoffMs *
|
|
490
|
-
Math.pow(work.retryBehavior.base, work.attempts - 1);
|
|
491
|
-
}
|
|
492
|
-
else {
|
|
464
|
+
if (!work.retryBehavior) {
|
|
493
465
|
console.warn(`[main] ${work._id} has no retryBehavior so not retrying`);
|
|
494
466
|
return false;
|
|
495
467
|
}
|
|
@@ -502,6 +474,8 @@ async function rescheduleJob(ctx, work, console, wasStuckInScheduler) {
|
|
|
502
474
|
console.error(`[main] ${work._id} already in pendingStart so not retrying`);
|
|
503
475
|
return false;
|
|
504
476
|
}
|
|
477
|
+
const backoffMs = work.retryBehavior.initialBackoffMs *
|
|
478
|
+
Math.pow(work.retryBehavior.base, work.attempts - 1);
|
|
505
479
|
const nextAttempt = withJitter(backoffMs);
|
|
506
480
|
const startTime = boundScheduledTime(Date.now() + nextAttempt, console);
|
|
507
481
|
const segment = toSegment(startTime);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loop.js","sourceRoot":"","sources":["../../src/component/loop.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAE/C,OAAO,EAAE,gBAAgB,EAAoB,MAAM,wBAAwB,CAAC;AAE5E,OAAO,EACL,YAAY,EACZ,iBAAiB,GAGlB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,kBAAkB,EAElB,uBAAuB,EACvB,WAAW,EACX,iBAAiB,EACjB,cAAc,EACd,GAAG,EAEH,SAAS,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE5E,MAAM,uBAAuB,GAAG,EAAE,CAAC,CAAC,yCAAyC;AAC7E,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAC/B,MAAM,EAAE,GAAG,CAAC,CAAC;AACb,MAAM,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;AACzB,MAAM,MAAM,GAAG,EAAE,GAAG,MAAM,CAAC;AAC3B,MAAM,4BAA4B,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,oCAAoC;AACrF,MAAM,8BAA8B,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,oCAAoC;AACvF,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,sBAAsB;AACrF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,GAAG,MAAM,CAAC;AAC1C,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,GAAG,EAAE,CAAC;AAChD,MAAM,sBAAsB,GAAG,SAAS,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,6BAA6B;AACpF,MAAM,CAAC,MAAM,aAAa,GAA8C;IACtE,UAAU,EAAE,EAAE;IACd,cAAc,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE;IACjE,YAAY,EAAE,EAAE;IAChB,MAAM,EAAE;QACN,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;QACT,OAAO,EAAE,CAAC;QACV,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,CAAC;KAChB;IACD,OAAO,EAAE,EAAE;CACZ,CAAC;AAEF,uEAAuE;AACvE,MAAM,CAAC,MAAM,IAAI,GAAG,gBAAgB,CAAC;IACnC,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE;IACnD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,EAAE;QAC9C,iEAAiE;QACjE,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,wBAAwB,UAAU,QAAQ,KAAK,CAAC,UAAU,EAAE,CAC7D,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,UAAU,EAAE,CAAC;QACnB,MAAM,SAAS,GAAG,MAAM,wBAAwB,CAAC,GAAG,CAAC,CAAC;QACtD,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE;gBAChC,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAClD,OAAO,CAAC,KAAK,CAAC,qBAAqB,UAAU,YAAY,OAAO,IAAI,CAAC,CAAC;QAEtE,qDAAqD;QACrD,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACvE,OAAO,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;QAE5C,+FAA+F;QAC/F,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAChE,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAE7C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,iDAAiD;YACjD,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC;QAC/B,CAAC;aAAM,IAAI,OAAO,GAAG,KAAK,CAAC,YAAY,IAAI,wBAAwB,EAAE,CAAC;YACpE,gDAAgD;YAChD,MAAM,cAAc,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YAC1C,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC;QAC/B,CAAC;QAED,sFAAsF;QACtF,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACpC,MAAM,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACzD,OAAO,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;QAEvC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,IAAI,MAAM,EAAE,CAAC;YACrD,oDAAoD;YACpD,mDAAmD;YACnD,IAAI,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC;YACtD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3C,wCAAwC;gBACxC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,CAAC;YACD,MAAM,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YACnD,KAAK,CAAC,MAAM,GAAG;gBACb,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC;gBACT,OAAO,EAAE,CAAC;gBACV,QAAQ,EAAE,CAAC;gBACX,UAAU,EAAE,CAAC;gBACb,YAAY;aACb,CAAC;QACJ,CAAC;QAED,MAAM,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACvC,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE;YAC7D,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO;SACR,CAAC,CAAC;QACH,kEAAkE;IACpE,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,gBAAgB,CAAC;IAC9C,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE;IACnD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,EAAE;QAC9C,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/C,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAC9C,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,wBAAwB,UAAU,QAAQ,KAAK,CAAC,UAAU,EAAE,CAC7D,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QAC1D,MAAM,uBAAuB,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,oBAAoB,EAAE;YACzE,KAAK,EAAE,KAAK,CAAC,cAAc,CAAC,WAAW;YACvC,GAAG,EAAE,OAAO;SACb,CAAC,CAAC;QACH,OAAO,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;QAC7D,IAAI,uBAAuB,EAAE,CAAC;YAC5B,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE;gBAClD,UAAU;gBACV,OAAO;aACR,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,uEAAuE;QACvE,OAAO,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,GAAG,EAAE,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAC3D,MAAM,gBAAgB,GAAG,MAAM,uBAAuB,CACpD,GAAG,EACH,KAAK,EACL,cAAc,EACd,WAAW,CACZ,CAAC;QACF,OAAO,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;QAE7D,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,GAAG,CAAC,SAAS,CAAC,KAAK,CACvB,kBAAkB,CAAC,WAAW,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,EACrD,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB;gBACE,UAAU;gBACV,OAAO,EAAE,WAAW;aACrB,CACF,CAAC;YACF,OAAO;QACT,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;QACzD,MAAM,CAAC,eAAe,EAAE,OAAO,CAAC,GAAG,MAAM,sBAAsB,CAC7D,GAAG,EACH,KAAK,EACL,cAAc,CACf,CAAC;QACF,OAAO,CAAC,OAAO,CAAC,0CAA0C,CAAC,CAAC;QAE5D,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;gBAC5B,cAAc,EAAE;oBACd,GAAG,KAAK,CAAC,cAAc;oBACvB,GAAG,OAAO;iBACX;aACF,CAAC,CAAC;YACH,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE;gBAClD,UAAU;gBACV,OAAO,EAAE,iBAAiB,EAAE;aAC7B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,qEAAqE;QACrE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,KAAK,CAAC,cAAc,CAAC;QACnE,MAAM,YAAY,GAAG,WAAW,CAC9B,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,CAC5C,CAAC;QACF,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,GAAG,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,eAAe,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC,CAAC;YAChE,OAAO,CAAC,KAAK,CACX,+BAA+B,SAAS,mCAAmC,uBAAuB,IAAI,CACvG,CAAC;YACF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,uBAAuB,CAAC;YACrD,MAAM,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;YACxC,MAAM,GAAG,CAAC,SAAS,CAAC,KAAK,CACvB,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,EACpC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAC7B,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,EAAE,CACtC,CAAC;YACF,OAAO;QACT,CAAC;QAED,oDAAoD;QACpD,OAAO,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QAClD,MAAM,gBAAgB,GAIhB,CAAC,mBAAmB,EAAE,oBAAoB,CAAC,CAAC;QAClD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;YAC1C,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACxC,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,gBAAgB,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CACvC,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAClD,CACF,CAAC;QACF,OAAO,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC;QACrD,IAAI,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,MAAM,wBAAwB,CAAC,GAAG,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC;QACzD,IAAI,aAAa,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5D,oEAAoE;YACpE,+CAA+C;YAC/C,MAAM,mBAAmB,GAAG,KAAK,CAAC,YAAY,GAAG,wBAAwB,CAAC;YAC1E,IAAI,CAAC,aAAa,IAAI,aAAa,GAAG,mBAAmB,EAAE,CAAC;gBAC1D,aAAa,GAAG,mBAAmB,CAAC;YACtC,CAAC;YACD,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,KAAK,CAC3C,kBAAkB,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC,EACvD,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CACvC,CAAC;YACF,IAAI,aAAa,GAAG,cAAc,EAAE,EAAE,CAAC;gBACrC,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE;oBAChC,KAAK,EAAE;wBACL,IAAI,EAAE,WAAW;wBACjB,WAAW;wBACX,SAAS;wBACT,UAAU;wBACV,OAAO,EAAE,aAAa;qBACvB;iBACF,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CACX,iEAAiE,CAClE,CAAC;YACJ,CAAC;YACD,OAAO;QACT,CAAC;QACD,6DAA6D;QAC7D,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE;YAChC,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE;SACpC,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,KAAK,UAAU,uBAAuB,CACpC,GAAgB,EAChB,KAA2B,EAC3B,cAAsB,EACtB,GAAW;IAEX,+CAA+C;IAC/C,IACE,MAAM,SAAS,CAAC,GAAG,EAAE,oBAAoB,EAAE;QACzC,KAAK,EAAE,KAAK,CAAC,cAAc,CAAC,WAAW;QACvC,GAAG;KACJ,CAAC,EACF,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IACE,MAAM,SAAS,CAAC,GAAG,EAAE,mBAAmB,EAAE;QACxC,KAAK,EAAE,KAAK,CAAC,cAAc,CAAC,UAAU;QACtC,GAAG;KACJ,CAAC,EACF,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QAC1C,IACE,MAAM,SAAS,CAAC,GAAG,EAAE,cAAc,EAAE;YACnC,KAAK,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ;YACpC,GAAG;SACJ,CAAC,EACF,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,GAAgB,EAChB,KAA2B,EAC3B,cAAsB;IAItB,+DAA+D;IAC/D,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,mBAAmB,EAAE;QAC9D,GAAG,EAAE,KAAK,CAAC,cAAc,CAAC,UAAU;KACrC,CAAC,CAAC;IACH,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,cAAc,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,oBAAoB,EAAE;QAChE,GAAG,EAAE,KAAK,CAAC,cAAc,CAAC,WAAW;KACtC,CAAC,CAAC;IACH,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,cAAc,EAAE;YACpD,GAAG,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ;SACnC,CAAC,CAAC;QACH,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IACD,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACrB,CAAC;AAED,gFAAgF;AAChF,KAAK,UAAU,SAAS,CACtB,GAAgB,EAChB,KAAkE,EAClE,KAAuC;IAEvC,OAAO,GAAG,CAAC,EAAE;SACV,KAAK,CAAC,KAAK,CAAC;SACZ,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,KAAK,CAAC,KAAK,KAAK,SAAS;QACvB,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,SAAS;YACvB,CAAC,CAAC,CAAC;iBACE,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,GAAG,sBAAsB,CAAC;iBACpD,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC;YAC9B,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,GAAG,sBAAsB,CAAC;QACzD,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,SAAS;YACvB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC;YAC5B,CAAC,CAAC,CAAC,CACR;SACA,KAAK,EAAE,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,iBAAiB,CAC9B,GAAgB,EAChB,KAA2B,EAC3B,OAAe,EACf,OAAe;IAEf,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,CAAC,UAAU,GAAG,sBAAsB,CAAC;IAC9E,4EAA4E;IAC5E,8EAA8E;IAC9E,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE;SAC3B,KAAK,CAAC,mBAAmB,CAAC;SAC1B,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CACvD;SACA,OAAO,EAAE,CAAC;IACb,KAAK,CAAC,cAAc,CAAC,UAAU,GAAG,OAAO,CAAC;IAC1C,0EAA0E;IAC1E,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,OAAO,CAAC,GAAG,CACf,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACxB,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAE3B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;QACjE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CACX,qBAAqB,CAAC,CAAC,MAAM,4BAA4B,CAC1D,CAAC;YACF,OAAO;QACT,CAAC;QACD,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YACZ,mDAAmD;YACnD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,kCAAkC,CAAC,CAAC;gBACnE,OAAO;YACT,CAAC;YACD,MAAM,mBAAmB,GAAG,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,kBAAkB,CAAC;YACpE,MAAM,OAAO,GAAG,MAAM,aAAa,CACjC,GAAG,EACH,IAAI,EACJ,OAAO,EACP,mBAAmB,CACpB,CAAC;YACF,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,mBAAmB,EAAE,CAAC;oBACxB,KAAK,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;oBAC7D,eAAe,CAAC,OAAO,EAAE,IAAI,EAAE,qBAAqB,EAAE,SAAS,CAAC,CAAC;gBACnE,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBACvB,eAAe,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,yDAAyD;gBACzD,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACxB,QAAQ,CAAC,IAAI,CAAC;oBACZ,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,SAAS,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE;oBAC/B,OAAO,EAAE,IAAI,CAAC,QAAQ;iBACvB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBACnC,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3B,CAAC;iBAAM,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACzC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IACF,kEAAkE;IAClE,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;IACpC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAClC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,CACrD,CAAC;IACF,MAAM,YAAY,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;IACnD,KAAK,CAAC,MAAM,CAAC,SAAS,IAAI,YAAY,CAAC;IACvC,OAAO,CAAC,KAAK,CAAC,oBAAoB,YAAY,OAAO,CAAC,CAAC;IACvD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,GAAgB,EAChB,KAA2B,EAC3B,OAAe,EACf,OAAe,EACf,QAAuB;IAEvB,MAAM,KAAK,GAAG,KAAK,CAAC,cAAc,CAAC,WAAW,GAAG,sBAAsB,CAAC;IACxE,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE;SAC1B,KAAK,CAAC,oBAAoB,CAAC;SAC3B,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAChD;SACA,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACjC,KAAK,CAAC,cAAc,CAAC,WAAW,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,OAAO,CAAC;IACvE,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,MAAM,YAAY,GAAoB,IAAI,GAAG,EAAE,CAAC;IAChD,MAAM,SAAS,GAAc,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAC1B,GAAG,CACD,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;QACpD,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,qEAAqE;YACrE,OAAO,CAAC,KAAK,CAAC,UAAU,MAAM,mBAAmB,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,CAAC,IAAI,CAAC,UAAU,MAAM,gCAAgC,CAAC,CAAC;YAC/D,OAAO,IAAI,CAAC;QACd,CAAC;QACD,2BAA2B;QAC3B,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,2BAA2B;QAC3B,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,EAAE;aAC9B,KAAK,CAAC,cAAc,CAAC;aACrB,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;aAClD,MAAM,EAAE,CAAC;QACZ,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9C,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACxB,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YACtC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CACH,CACF,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CACjC,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,GAAgB,EAChB,KAA2B,EAC3B,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI,GAAG,EAAc,CAAC;IACtC,MAAM,yBAAyB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,4BAA4B,CAAC;IAC5E,MAAM,2BAA2B,GAC/B,IAAI,CAAC,GAAG,EAAE,GAAG,8BAA8B,CAAC;IAC9C,MAAM,IAAI,GAAG,CACX,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QAC5B,IACE,CAAC,CAAC,OAAO;YACT,IAAI,CAAC,GAAG,CAAC,yBAAyB,EAAE,2BAA2B,CAAC,EAChE,CAAC;YACD,qCAAqC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,mCAAmC,CAAC,CAAC;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,mBAAmB,GACvB,IAAI,CAAC,MAAM,KAAK,QAAQ;YACtB,CAAC,CAAC,yBAAyB;YAC3B,CAAC,CAAC,2BAA2B,CAAC;QAClC,IAAI,CAAC,CAAC,OAAO,IAAI,mBAAmB,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;IAC1C,CAAC,CAAC,CACH,CACF,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACpE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,mBAAmB,EAAE,CAAC;QAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,mBAAmB,CAAC,CAAC;QACrD,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,GAAgB,EAChB,KAA2B,EAC3B,OAAe,EACf,OAAe,EACf,EAAE,cAAc,EAAE,QAAQ,EAAU;IAEpC,sDAAsD;IACtD,MAAM,UAAU,GAAG,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;IAEzD,MAAM,OAAO,GACX,UAAU,GAAG,CAAC;QACZ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;aACT,KAAK,CAAC,cAAc,CAAC;aACrB,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC;aACE,GAAG,CACF,SAAS,EACT,KAAK,CAAC,cAAc,CAAC,QAAQ,GAAG,sBAAsB,CACvD;aACA,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAC3B;aACA,IAAI,CAAC,UAAU,CAAC;QACrB,CAAC,CAAC,EAAE,CAAC;IAET,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,KAAK,CAAC,cAAc,CAAC,QAAQ,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC;QAC1D,CAAC;aAAM,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YAC1B,8CAA8C;YAC9C,KAAK,CAAC,cAAc,CAAC,QAAQ,GAAG,OAAO,CAAC;QAC1C,CAAC;IACH,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,qBAAqB,OAAO,CAAC,MAAM,eAAe,CAAC,CAAC;IAClE,kBAAkB;IAClB,KAAK,CAAC,OAAO,CAAC,IAAI,CAChB,GAAG,CACD,MAAM,OAAO,CAAC,GAAG,CACf,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE;QAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE,CAAC;YACnD,OAAO,CAAC,KAAK,CAAC,UAAU,MAAM,mCAAmC,CAAC,CAAC;YACnE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAClE,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IACtD,CAAC,CAAC,CACH,CACF,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CACjC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,GAAgB,EAChB,MAAkB,EAClB,QAAkB,EAClB,KAAa;IAEb,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAChE,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;IACxE,IAAI,UAAU,CAAC;IACf,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC7B,UAAU,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CACvC,CAAC,EACD,QAAQ,CAAC,MAAM,CAAC,gBAAgB,EAChC,IAAI,CACL,CAAC;IACJ,CAAC;SAAM,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACjE,UAAU,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CACvC,CAAC,EACD,QAAQ,CAAC,MAAM,CAAC,kBAAkB,EAClC;YACE,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CACF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAChD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,aAAa,CAC1B,GAAgB,EAChB,IAAiB,EACjB,OAAe,EACf,mBAA4B;IAE5B,MAAM,kBAAkB,GAAG,MAAM,GAAG,CAAC,EAAE;SACpC,KAAK,CAAC,oBAAoB,CAAC;SAC3B,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;SACpD,MAAM,EAAE,CAAC;IACZ,IAAI,kBAAkB,EAAE,CAAC;QACvB,+DAA+D;QAC/D,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,wCAAwC,CAAC,CAAC;QACzE,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,SAAiB,CAAC;IACtB,IAAI,mBAAmB,EAAE,CAAC;QACxB,SAAS,GAAG,CAAC,CAAC;IAChB,CAAC;SAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QAC9B,SAAS;YACP,IAAI,CAAC,aAAa,CAAC,gBAAgB;gBACnC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,uCAAuC,CAAC,CAAC;QACxE,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE;SAC1B,KAAK,CAAC,cAAc,CAAC;SACrB,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;SACpD,KAAK,EAAE,CAAC;IACX,IAAI,QAAQ,EAAE,CAAC;QACb,yEAAyE;QACzE,OAAO,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,GAAG,0CAA0C,CAAC,CAAC;QAC5E,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,WAAW,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,EAAE,OAAO,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE;QAClC,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,OAAO;KACR,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,OAAO,KAAK,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,GAAgB;IACxC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;IACvD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,cAAc,EAAE,uBAAuB;YACvC,QAAQ,EAAE,iBAAiB;SAC5B,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAgB;IAC9C,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC;IAC3D,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IACxB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAC5E,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CACtB,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,eAAe,EAAE,aAAa,CAAC,CACpD,CAAE,CAAC;AACN,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,GAAgB;IACtD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC;IAC3D,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAChC,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;IAClE,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CACtB,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC,CACjE,CAAE,CAAC;AACN,CAAC;AACD,6DAA6D;AAC7D,MAAM,OAAO,GAAG,wCAAwC,CAAC"}
|
|
1
|
+
{"version":3,"file":"loop.js","sourceRoot":"","sources":["../../src/component/loop.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAE/C,OAAO,EAAE,gBAAgB,EAAoB,MAAM,wBAAwB,CAAC;AAE5E,OAAO,EACL,YAAY,EACZ,iBAAiB,GAGlB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,kBAAkB,EAElB,uBAAuB,EACvB,WAAW,EACX,iBAAiB,EACjB,cAAc,EACd,GAAG,EAEH,SAAS,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE5E,MAAM,uBAAuB,GAAG,EAAE,CAAC,CAAC,yCAAyC;AAC7E,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAC/B,MAAM,EAAE,GAAG,CAAC,CAAC;AACb,MAAM,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;AACzB,MAAM,MAAM,GAAG,EAAE,GAAG,MAAM,CAAC;AAC3B,MAAM,qBAAqB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,oCAAoC;AAC9E,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,sBAAsB;AACrF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,GAAG,MAAM,CAAC;AAC1C,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,GAAG,EAAE,CAAC;AAChD,MAAM,sBAAsB,GAAG,SAAS,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,6BAA6B;AACpF,MAAM,CAAC,MAAM,aAAa,GAA8C;IACtE,UAAU,EAAE,EAAE;IACd,cAAc,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE;IACjE,YAAY,EAAE,EAAE;IAChB,MAAM,EAAE;QACN,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;QACT,OAAO,EAAE,CAAC;QACV,QAAQ,EAAE,CAAC;QACX,YAAY,EAAE,CAAC;KAChB;IACD,OAAO,EAAE,EAAE;CACZ,CAAC;AAEF,uEAAuE;AACvE,MAAM,CAAC,MAAM,IAAI,GAAG,gBAAgB,CAAC;IACnC,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE;IACnD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,EAAE;QAC9C,iEAAiE;QACjE,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,wBAAwB,UAAU,QAAQ,KAAK,CAAC,UAAU,EAAE,CAC7D,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,UAAU,EAAE,CAAC;QACnB,MAAM,SAAS,GAAG,MAAM,wBAAwB,CAAC,GAAG,CAAC,CAAC;QACtD,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE;gBAChC,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAClD,OAAO,CAAC,KAAK,CAAC,qBAAqB,UAAU,YAAY,OAAO,IAAI,CAAC,CAAC;QAEtE,qDAAqD;QACrD,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACvE,OAAO,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;QAE5C,+FAA+F;QAC/F,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAChE,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAE7C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,iDAAiD;YACjD,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC;QAC/B,CAAC;aAAM,IAAI,OAAO,GAAG,KAAK,CAAC,YAAY,IAAI,wBAAwB,EAAE,CAAC;YACpE,gDAAgD;YAChD,MAAM,cAAc,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YAC1C,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC;QAC/B,CAAC;QAED,sFAAsF;QACtF,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACpC,MAAM,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACzD,OAAO,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;QAEvC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,IAAI,MAAM,EAAE,CAAC;YACrD,oDAAoD;YACpD,mDAAmD;YACnD,IAAI,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC;YACtD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3C,wCAAwC;gBACxC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,CAAC;YACD,MAAM,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YACnD,KAAK,CAAC,MAAM,GAAG;gBACb,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC;gBACT,OAAO,EAAE,CAAC;gBACV,QAAQ,EAAE,CAAC;gBACX,YAAY;aACb,CAAC;QACJ,CAAC;QAED,MAAM,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACvC,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE;YAC7D,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO;SACR,CAAC,CAAC;QACH,kEAAkE;IACpE,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,gBAAgB,CAAC;IAC9C,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE;IACnD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,EAAE;QAC9C,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/C,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAC9C,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,wBAAwB,UAAU,QAAQ,KAAK,CAAC,UAAU,EAAE,CAC7D,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QAC1D,MAAM,uBAAuB,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,oBAAoB,EAAE;YACzE,KAAK,EAAE,KAAK,CAAC,cAAc,CAAC,WAAW;YACvC,GAAG,EAAE,OAAO;SACb,CAAC,CAAC;QACH,OAAO,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;QAC7D,IAAI,uBAAuB,EAAE,CAAC;YAC5B,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE;gBAClD,UAAU;gBACV,OAAO;aACR,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,uEAAuE;QACvE,OAAO,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,GAAG,EAAE,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAC3D,MAAM,gBAAgB,GAAG,MAAM,uBAAuB,CACpD,GAAG,EACH,KAAK,EACL,cAAc,EACd,WAAW,CACZ,CAAC;QACF,OAAO,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;QAE7D,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,GAAG,CAAC,SAAS,CAAC,KAAK,CACvB,kBAAkB,CAAC,WAAW,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,EACrD,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB;gBACE,UAAU;gBACV,OAAO,EAAE,WAAW;aACrB,CACF,CAAC;YACF,OAAO;QACT,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;QACzD,MAAM,CAAC,eAAe,EAAE,OAAO,CAAC,GAAG,MAAM,sBAAsB,CAC7D,GAAG,EACH,KAAK,EACL,cAAc,CACf,CAAC;QACF,OAAO,CAAC,OAAO,CAAC,0CAA0C,CAAC,CAAC;QAE5D,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;gBAC5B,cAAc,EAAE;oBACd,GAAG,KAAK,CAAC,cAAc;oBACvB,GAAG,OAAO;iBACX;aACF,CAAC,CAAC;YACH,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE;gBAClD,UAAU;gBACV,OAAO,EAAE,iBAAiB,EAAE;aAC7B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,qEAAqE;QACrE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,KAAK,CAAC,cAAc,CAAC;QACnE,MAAM,YAAY,GAAG,WAAW,CAC9B,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,CAC5C,CAAC;QACF,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,GAAG,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,eAAe,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC,CAAC;YAChE,OAAO,CAAC,KAAK,CACX,+BAA+B,SAAS,mCAAmC,uBAAuB,IAAI,CACvG,CAAC;YACF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,uBAAuB,CAAC;YACrD,MAAM,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;YACxC,MAAM,GAAG,CAAC,SAAS,CAAC,KAAK,CACvB,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,EACpC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAC7B,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,EAAE,CACtC,CAAC;YACF,OAAO;QACT,CAAC;QAED,oDAAoD;QACpD,OAAO,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QAClD,MAAM,gBAAgB,GAIhB,CAAC,mBAAmB,EAAE,oBAAoB,CAAC,CAAC;QAClD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;YAC1C,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACxC,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,gBAAgB,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CACvC,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAClD,CACF,CAAC;QACF,OAAO,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC;QACrD,IAAI,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,MAAM,wBAAwB,CAAC,GAAG,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC;QACzD,IAAI,aAAa,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5D,oEAAoE;YACpE,+CAA+C;YAC/C,MAAM,mBAAmB,GAAG,KAAK,CAAC,YAAY,GAAG,wBAAwB,CAAC;YAC1E,IAAI,CAAC,aAAa,IAAI,aAAa,GAAG,mBAAmB,EAAE,CAAC;gBAC1D,aAAa,GAAG,mBAAmB,CAAC;YACtC,CAAC;YACD,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,KAAK,CAC3C,kBAAkB,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC,EACvD,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CACvC,CAAC;YACF,IAAI,aAAa,GAAG,cAAc,EAAE,EAAE,CAAC;gBACrC,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE;oBAChC,KAAK,EAAE;wBACL,IAAI,EAAE,WAAW;wBACjB,WAAW;wBACX,SAAS;wBACT,UAAU;wBACV,OAAO,EAAE,aAAa;qBACvB;iBACF,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CACX,iEAAiE,CAClE,CAAC;YACJ,CAAC;YACD,OAAO;QACT,CAAC;QACD,6DAA6D;QAC7D,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE;YAChC,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE;SACpC,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,KAAK,UAAU,uBAAuB,CACpC,GAAgB,EAChB,KAA2B,EAC3B,cAAsB,EACtB,GAAW;IAEX,+CAA+C;IAC/C,IACE,MAAM,SAAS,CAAC,GAAG,EAAE,oBAAoB,EAAE;QACzC,KAAK,EAAE,KAAK,CAAC,cAAc,CAAC,WAAW;QACvC,GAAG;KACJ,CAAC,EACF,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IACE,MAAM,SAAS,CAAC,GAAG,EAAE,mBAAmB,EAAE;QACxC,KAAK,EAAE,KAAK,CAAC,cAAc,CAAC,UAAU;QACtC,GAAG;KACJ,CAAC,EACF,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QAC1C,IACE,MAAM,SAAS,CAAC,GAAG,EAAE,cAAc,EAAE;YACnC,KAAK,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ;YACpC,GAAG;SACJ,CAAC,EACF,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,GAAgB,EAChB,KAA2B,EAC3B,cAAsB;IAItB,+DAA+D;IAC/D,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,mBAAmB,EAAE;QAC9D,GAAG,EAAE,KAAK,CAAC,cAAc,CAAC,UAAU;KACrC,CAAC,CAAC;IACH,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,cAAc,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,oBAAoB,EAAE;QAChE,GAAG,EAAE,KAAK,CAAC,cAAc,CAAC,WAAW;KACtC,CAAC,CAAC;IACH,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,cAAc,EAAE;YACpD,GAAG,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ;SACnC,CAAC,CAAC;QACH,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IACD,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACrB,CAAC;AAED,gFAAgF;AAChF,KAAK,UAAU,SAAS,CACtB,GAAgB,EAChB,KAAkE,EAClE,KAAuC;IAEvC,OAAO,GAAG,CAAC,EAAE;SACV,KAAK,CAAC,KAAK,CAAC;SACZ,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,KAAK,CAAC,KAAK,KAAK,SAAS;QACvB,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,SAAS;YACvB,CAAC,CAAC,CAAC;iBACE,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,GAAG,sBAAsB,CAAC;iBACpD,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC;YAC9B,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,GAAG,sBAAsB,CAAC;QACzD,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,SAAS;YACvB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC;YAC5B,CAAC,CAAC,CAAC,CACR;SACA,KAAK,EAAE,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,iBAAiB,CAC9B,GAAgB,EAChB,KAA2B,EAC3B,OAAe,EACf,OAAe;IAEf,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,CAAC,UAAU,GAAG,sBAAsB,CAAC;IAC9E,4EAA4E;IAC5E,8EAA8E;IAC9E,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE;SAC3B,KAAK,CAAC,mBAAmB,CAAC;SAC1B,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CACvD;SACA,OAAO,EAAE,CAAC;IACb,KAAK,CAAC,cAAc,CAAC,UAAU,GAAG,OAAO,CAAC;IAC1C,0EAA0E;IAC1E,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,OAAO,CAAC,GAAG,CACf,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACxB,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAE3B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;QACjE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CACX,qBAAqB,CAAC,CAAC,MAAM,4BAA4B,CAC1D,CAAC;YACF,OAAO;QACT,CAAC;QACD,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YACZ,mDAAmD;YACnD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,kCAAkC,CAAC,CAAC;gBACnE,OAAO;YACT,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YACxD,IAAI,OAAO,EAAE,CAAC;gBACZ,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvB,eAAe,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YACxD,CAAC;iBAAM,CAAC;gBACN,yDAAyD;gBACzD,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACxB,QAAQ,CAAC,IAAI,CAAC;oBACZ,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,SAAS,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE;oBAC/B,OAAO,EAAE,IAAI,CAAC,QAAQ;iBACvB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBACnC,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3B,CAAC;iBAAM,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACzC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IACF,kEAAkE;IAClE,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;IACpC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAClC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,CACrD,CAAC;IACF,MAAM,YAAY,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;IACnD,KAAK,CAAC,MAAM,CAAC,SAAS,IAAI,YAAY,CAAC;IACvC,OAAO,CAAC,KAAK,CAAC,oBAAoB,YAAY,OAAO,CAAC,CAAC;IACvD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,GAAgB,EAChB,KAA2B,EAC3B,OAAe,EACf,OAAe,EACf,QAAuB;IAEvB,MAAM,KAAK,GAAG,KAAK,CAAC,cAAc,CAAC,WAAW,GAAG,sBAAsB,CAAC;IACxE,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE;SAC1B,KAAK,CAAC,oBAAoB,CAAC;SAC3B,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAChD;SACA,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACjC,KAAK,CAAC,cAAc,CAAC,WAAW,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,OAAO,CAAC;IACvE,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,MAAM,YAAY,GAAoB,IAAI,GAAG,EAAE,CAAC;IAChD,MAAM,SAAS,GAAc,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAC1B,GAAG,CACD,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;QACpD,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,qEAAqE;YACrE,OAAO,CAAC,KAAK,CAAC,UAAU,MAAM,mBAAmB,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,CAAC,IAAI,CAAC,UAAU,MAAM,gCAAgC,CAAC,CAAC;YAC/D,OAAO,IAAI,CAAC;QACd,CAAC;QACD,2BAA2B;QAC3B,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,2BAA2B;QAC3B,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,EAAE;aAC9B,KAAK,CAAC,cAAc,CAAC;aACrB,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;aAClD,MAAM,EAAE,CAAC;QACZ,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9C,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACxB,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YACtC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CACH,CACF,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CACjC,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,GAAgB,EAChB,KAA2B,EAC3B,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI,GAAG,EAAc,CAAC;IACtC,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,qBAAqB,CAAC;IAC/D,MAAM,IAAI,GAAG,CACX,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QAC5B,IAAI,CAAC,CAAC,OAAO,IAAI,mBAAmB,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,mCAAmC,CAAC,CAAC;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;IAC1C,CAAC,CAAC,CACH,CACF,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACpE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,mBAAmB,EAAE,CAAC;QAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,mBAAmB,CAAC,CAAC;QACrD,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,GAAgB,EAChB,KAA2B,EAC3B,OAAe,EACf,OAAe,EACf,EAAE,cAAc,EAAE,QAAQ,EAAU;IAEpC,sDAAsD;IACtD,MAAM,UAAU,GAAG,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;IAEzD,MAAM,OAAO,GACX,UAAU,GAAG,CAAC;QACZ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;aACT,KAAK,CAAC,cAAc,CAAC;aACrB,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC;aACE,GAAG,CACF,SAAS,EACT,KAAK,CAAC,cAAc,CAAC,QAAQ,GAAG,sBAAsB,CACvD;aACA,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAC3B;aACA,IAAI,CAAC,UAAU,CAAC;QACrB,CAAC,CAAC,EAAE,CAAC;IAET,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,KAAK,CAAC,cAAc,CAAC,QAAQ,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC;QAC1D,CAAC;aAAM,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YAC1B,8CAA8C;YAC9C,KAAK,CAAC,cAAc,CAAC,QAAQ,GAAG,OAAO,CAAC;QAC1C,CAAC;IACH,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,qBAAqB,OAAO,CAAC,MAAM,eAAe,CAAC,CAAC;IAClE,kBAAkB;IAClB,KAAK,CAAC,OAAO,CAAC,IAAI,CAChB,GAAG,CACD,MAAM,OAAO,CAAC,GAAG,CACf,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE;QAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE,CAAC;YACnD,OAAO,CAAC,KAAK,CAAC,UAAU,MAAM,mCAAmC,CAAC,CAAC;YACnE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAClE,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IACtD,CAAC,CAAC,CACH,CACF,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CACjC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,GAAgB,EAChB,MAAkB,EAClB,QAAkB,EAClB,KAAa;IAEb,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAChE,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;IACxE,IAAI,UAAU,CAAC;IACf,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC7B,UAAU,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CACvC,CAAC,EACD,QAAQ,CAAC,MAAM,CAAC,gBAAgB,EAChC,IAAI,CACL,CAAC;IACJ,CAAC;SAAM,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACjE,UAAU,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CACvC,CAAC,EACD,QAAQ,CAAC,MAAM,CAAC,kBAAkB,EAClC;YACE,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CACF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAChD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,aAAa,CAC1B,GAAgB,EAChB,IAAiB,EACjB,OAAe;IAEf,MAAM,kBAAkB,GAAG,MAAM,GAAG,CAAC,EAAE;SACpC,KAAK,CAAC,oBAAoB,CAAC;SAC3B,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;SACpD,MAAM,EAAE,CAAC;IACZ,IAAI,kBAAkB,EAAE,CAAC;QACvB,+DAA+D;QAC/D,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,wCAAwC,CAAC,CAAC;QACzE,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,uCAAuC,CAAC,CAAC;QACxE,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE;SAC1B,KAAK,CAAC,cAAc,CAAC;SACrB,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;SACpD,KAAK,EAAE,CAAC;IACX,IAAI,QAAQ,EAAE,CAAC;QACb,yEAAyE;QACzE,OAAO,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,GAAG,0CAA0C,CAAC,CAAC;QAC5E,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,SAAS,GACb,IAAI,CAAC,aAAa,CAAC,gBAAgB;QACnC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;IACvD,MAAM,WAAW,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,EAAE,OAAO,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE;QAClC,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,OAAO;KACR,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,OAAO,KAAK,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,GAAgB;IACxC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;IACvD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,cAAc,EAAE,uBAAuB;YACvC,QAAQ,EAAE,iBAAiB;SAC5B,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAgB;IAC9C,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC;IAC3D,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IACxB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAC5E,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CACtB,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,eAAe,EAAE,aAAa,CAAC,CACpD,CAAE,CAAC;AACN,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,GAAgB;IACtD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC;IAC3D,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAChC,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;IAClE,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CACtB,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC,CACjE,CAAE,CAAC;AACN,CAAC;AACD,6DAA6D;AAC7D,MAAM,OAAO,GAAG,wCAAwC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recovery.d.ts","sourceRoot":"","sources":["../../src/component/recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,KAAK,EAAK,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAoB,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAI5E,QAAA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;sBAShB,CAAC;AAEH;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,OAAO;;;;;;;iBAGlB,CAAC;AAGH,wBAAsB,eAAe,CACnC,GAAG,EAAE,WAAW,EAChB,EAAE,IAAI,EAAE,EAAE,KAAK,CAAC,OAAO,YAAY,CAAC,
|
|
1
|
+
{"version":3,"file":"recovery.d.ts","sourceRoot":"","sources":["../../src/component/recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,KAAK,EAAK,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAoB,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAI5E,QAAA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;sBAShB,CAAC;AAEH;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,OAAO;;;;;;;iBAGlB,CAAC;AAGH,wBAAsB,eAAe,CACnC,GAAG,EAAE,WAAW,EAChB,EAAE,IAAI,EAAE,EAAE,KAAK,CAAC,OAAO,YAAY,CAAC,iBAgErC"}
|
|
@@ -88,23 +88,6 @@ export async function recoveryHandler(ctx, { jobs }) {
|
|
|
88
88
|
});
|
|
89
89
|
break;
|
|
90
90
|
}
|
|
91
|
-
case "pending": {
|
|
92
|
-
if (work.fnType === "action") {
|
|
93
|
-
// We do not cancel and re-enqueue actions. If a scheduled action is still
|
|
94
|
-
// pending, the scheduler is likely backlogged.
|
|
95
|
-
break;
|
|
96
|
-
}
|
|
97
|
-
// It looks like the function has been retried by the scheduler several times.
|
|
98
|
-
// The scheduler backoff is too long, so cancel and re-enqueue the job to
|
|
99
|
-
// free up space for more work.
|
|
100
|
-
await ctx.scheduler.cancel(scheduled._id);
|
|
101
|
-
completionJobs.push({
|
|
102
|
-
workId: job.workId,
|
|
103
|
-
runResult: { kind: "stuckInScheduler" },
|
|
104
|
-
attempt: job.attempt,
|
|
105
|
-
});
|
|
106
|
-
break;
|
|
107
|
-
}
|
|
108
91
|
}
|
|
109
92
|
}
|
|
110
93
|
if (completionJobs.length > 0) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recovery.js","sourceRoot":"","sources":["../../src/component/recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,CAAC,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAoB,MAAM,wBAAwB,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAoB,eAAe,EAAE,MAAM,eAAe,CAAC;AAElE,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,IAAI,EAAE,CAAC,CAAC,KAAK,CACX,CAAC,CAAC,MAAM,CAAC;QACP,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,sBAAsB,CAAC;QACzC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;KACpB,CAAC,CACH;CACF,CAAC,CAAC;AAEH;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,gBAAgB,CAAC;IACtC,IAAI,EAAE,YAAY;IAClB,OAAO,EAAE,eAAe;CACzB,CAAC,CAAC;AAEH,4BAA4B;AAC5B,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,GAAgB,EAChB,EAAE,IAAI,EAA8B;IAEpC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;IACvD,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,cAAc,GAAkB,EAAE,CAAC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,QAAQ,GAAG,4BAA4B,GAAG,CAAC,WAAW,aAAa,GAAG,CAAC,MAAM,EAAE,CAAC;QACtF,MAAM,iBAAiB,GAAG,MAAM,GAAG,CAAC,EAAE;aACnC,KAAK,CAAC,mBAAmB,CAAC;aAC1B,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;aACtD,KAAK,EAAE,CAAC;QACX,IAAI,iBAAiB,EAAE,CAAC;YACtB,sDAAsD;YACtD,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,yCAAyC,CAAC,CAAC;YACpE,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,mEAAmE;YACnE,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,2BAA2B,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC;YAClC,iDAAiD;YACjD,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,8BAA8B,CAAC,CAAC;YACxD,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,oCAAoC,CAAC,CAAC;YAC9D,cAAc,CAAC,IAAI,CAAC;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,yBAAyB,EAAE;gBAC/D,OAAO,EAAE,GAAG,CAAC,OAAO;aACrB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,qEAAqE;QACrE,iDAAiD;QACjD,QAAQ,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7B,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,kCAAkC,CAAC,CAAC;gBAC7D,cAAc,CAAC,IAAI,CAAC;oBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,SAAS,EAAE,SAAS,CAAC,KAAK;oBAC1B,OAAO,EAAE,GAAG,CAAC,OAAO;iBACrB,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YACD,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,wCAAwC,CAAC,CAAC;gBACnE,cAAc,CAAC,IAAI,CAAC;oBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,wBAAwB,EAAE;oBAC9D,OAAO,EAAE,GAAG,CAAC,OAAO;iBACrB,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;
|
|
1
|
+
{"version":3,"file":"recovery.js","sourceRoot":"","sources":["../../src/component/recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,CAAC,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAoB,MAAM,wBAAwB,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAoB,eAAe,EAAE,MAAM,eAAe,CAAC;AAElE,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,IAAI,EAAE,CAAC,CAAC,KAAK,CACX,CAAC,CAAC,MAAM,CAAC;QACP,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,sBAAsB,CAAC;QACzC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;KACpB,CAAC,CACH;CACF,CAAC,CAAC;AAEH;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,gBAAgB,CAAC;IACtC,IAAI,EAAE,YAAY;IAClB,OAAO,EAAE,eAAe;CACzB,CAAC,CAAC;AAEH,4BAA4B;AAC5B,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,GAAgB,EAChB,EAAE,IAAI,EAA8B;IAEpC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;IACvD,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,cAAc,GAAkB,EAAE,CAAC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,QAAQ,GAAG,4BAA4B,GAAG,CAAC,WAAW,aAAa,GAAG,CAAC,MAAM,EAAE,CAAC;QACtF,MAAM,iBAAiB,GAAG,MAAM,GAAG,CAAC,EAAE;aACnC,KAAK,CAAC,mBAAmB,CAAC;aAC1B,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;aACtD,KAAK,EAAE,CAAC;QACX,IAAI,iBAAiB,EAAE,CAAC;YACtB,sDAAsD;YACtD,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,yCAAyC,CAAC,CAAC;YACpE,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,mEAAmE;YACnE,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,2BAA2B,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC;YAClC,iDAAiD;YACjD,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,8BAA8B,CAAC,CAAC;YACxD,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,oCAAoC,CAAC,CAAC;YAC9D,cAAc,CAAC,IAAI,CAAC;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,yBAAyB,EAAE;gBAC/D,OAAO,EAAE,GAAG,CAAC,OAAO;aACrB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,qEAAqE;QACrE,iDAAiD;QACjD,QAAQ,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7B,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,kCAAkC,CAAC,CAAC;gBAC7D,cAAc,CAAC,IAAI,CAAC;oBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,SAAS,EAAE,SAAS,CAAC,KAAK;oBAC1B,OAAO,EAAE,GAAG,CAAC,OAAO;iBACrB,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YACD,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,wCAAwC,CAAC,CAAC;gBACnE,cAAc,CAAC,IAAI,CAAC;oBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,wBAAwB,EAAE;oBAC9D,OAAO,EAAE,GAAG,CAAC,OAAO;iBACrB,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,eAAe,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;IACvD,CAAC;AACH,CAAC"}
|
|
@@ -1,44 +1,3 @@
|
|
|
1
|
-
import { type Infer } from "convex/values";
|
|
2
|
-
export declare const vResultInternal: import("convex/values").VUnion<{
|
|
3
|
-
kind: "success";
|
|
4
|
-
returnValue: any;
|
|
5
|
-
} | {
|
|
6
|
-
kind: "failed";
|
|
7
|
-
error: string;
|
|
8
|
-
} | {
|
|
9
|
-
kind: "canceled";
|
|
10
|
-
} | {
|
|
11
|
-
kind: "stuckInScheduler";
|
|
12
|
-
}, [import("convex/values").VUnion<{
|
|
13
|
-
kind: "success";
|
|
14
|
-
returnValue: any;
|
|
15
|
-
} | {
|
|
16
|
-
kind: "failed";
|
|
17
|
-
error: string;
|
|
18
|
-
} | {
|
|
19
|
-
kind: "canceled";
|
|
20
|
-
}, [import("convex/values").VObject<{
|
|
21
|
-
kind: "success";
|
|
22
|
-
returnValue: any;
|
|
23
|
-
}, {
|
|
24
|
-
kind: import("convex/values").VLiteral<"success", "required">;
|
|
25
|
-
returnValue: import("convex/values").VAny<any, "required", string>;
|
|
26
|
-
}, "required", "kind" | "returnValue" | `returnValue.${string}`>, import("convex/values").VObject<{
|
|
27
|
-
kind: "failed";
|
|
28
|
-
error: string;
|
|
29
|
-
}, {
|
|
30
|
-
kind: import("convex/values").VLiteral<"failed", "required">;
|
|
31
|
-
error: import("convex/values").VString<string, "required">;
|
|
32
|
-
}, "required", "kind" | "error">, import("convex/values").VObject<{
|
|
33
|
-
kind: "canceled";
|
|
34
|
-
}, {
|
|
35
|
-
kind: import("convex/values").VLiteral<"canceled", "required">;
|
|
36
|
-
}, "required", "kind">], "required", "kind" | "returnValue" | `returnValue.${string}` | "error">, import("convex/values").VObject<{
|
|
37
|
-
kind: "stuckInScheduler";
|
|
38
|
-
}, {
|
|
39
|
-
kind: import("convex/values").VLiteral<"stuckInScheduler", "required">;
|
|
40
|
-
}, "required", "kind">], "required", "kind" | "returnValue" | `returnValue.${string}` | "error">;
|
|
41
|
-
export type RunResultInternal = Infer<typeof vResultInternal>;
|
|
42
1
|
declare const _default: import("convex/server").SchemaDefinition<{
|
|
43
2
|
globals: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
44
3
|
logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
|
|
@@ -54,7 +13,6 @@ declare const _default: import("convex/server").SchemaDefinition<{
|
|
|
54
13
|
started: number;
|
|
55
14
|
}[];
|
|
56
15
|
report: {
|
|
57
|
-
conflicted?: number | undefined;
|
|
58
16
|
failed: number;
|
|
59
17
|
canceled: number;
|
|
60
18
|
completed: number;
|
|
@@ -82,7 +40,6 @@ declare const _default: import("convex/server").SchemaDefinition<{
|
|
|
82
40
|
}, "required", "incoming" | "completion" | "cancelation">;
|
|
83
41
|
lastRecovery: import("convex/values").VInt64<bigint, "required">;
|
|
84
42
|
report: import("convex/values").VObject<{
|
|
85
|
-
conflicted?: number | undefined;
|
|
86
43
|
failed: number;
|
|
87
44
|
canceled: number;
|
|
88
45
|
completed: number;
|
|
@@ -95,9 +52,8 @@ declare const _default: import("convex/server").SchemaDefinition<{
|
|
|
95
52
|
failed: import("convex/values").VFloat64<number, "required">;
|
|
96
53
|
retries: import("convex/values").VFloat64<number, "required">;
|
|
97
54
|
canceled: import("convex/values").VFloat64<number, "required">;
|
|
98
|
-
conflicted: import("convex/values").VFloat64<number | undefined, "optional">;
|
|
99
55
|
lastReportTs: import("convex/values").VFloat64<number, "required">;
|
|
100
|
-
}, "required", "failed" | "canceled" | "completed" | "succeeded" | "retries" | "
|
|
56
|
+
}, "required", "failed" | "canceled" | "completed" | "succeeded" | "retries" | "lastReportTs">;
|
|
101
57
|
running: import("convex/values").VArray<{
|
|
102
58
|
workId: import("convex/values").GenericId<"work">;
|
|
103
59
|
scheduledId: import("convex/values").GenericId<"_scheduled_functions">;
|
|
@@ -111,7 +67,7 @@ declare const _default: import("convex/server").SchemaDefinition<{
|
|
|
111
67
|
scheduledId: import("convex/values").VId<import("convex/values").GenericId<"_scheduled_functions">, "required">;
|
|
112
68
|
started: import("convex/values").VFloat64<number, "required">;
|
|
113
69
|
}, "required", "workId" | "scheduledId" | "started">, "required">;
|
|
114
|
-
}, "required", "running" | "report" | "lastRecovery" | "generation" | "segmentCursors" | "report.failed" | "report.canceled" | "report.completed" | "report.succeeded" | "report.retries" | "report.
|
|
70
|
+
}, "required", "running" | "report" | "lastRecovery" | "generation" | "segmentCursors" | "report.failed" | "report.canceled" | "report.completed" | "report.succeeded" | "report.retries" | "report.lastReportTs" | "segmentCursors.incoming" | "segmentCursors.completion" | "segmentCursors.cancelation">, {}, {}, {}>;
|
|
115
71
|
runStatus: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
116
72
|
state: {
|
|
117
73
|
kind: "running";
|
|
@@ -227,8 +183,6 @@ declare const _default: import("convex/server").SchemaDefinition<{
|
|
|
227
183
|
error: string;
|
|
228
184
|
} | {
|
|
229
185
|
kind: "canceled";
|
|
230
|
-
} | {
|
|
231
|
-
kind: "stuckInScheduler";
|
|
232
186
|
};
|
|
233
187
|
}, {
|
|
234
188
|
segment: import("convex/values").VInt64<bigint, "required">;
|
|
@@ -240,16 +194,6 @@ declare const _default: import("convex/server").SchemaDefinition<{
|
|
|
240
194
|
error: string;
|
|
241
195
|
} | {
|
|
242
196
|
kind: "canceled";
|
|
243
|
-
} | {
|
|
244
|
-
kind: "stuckInScheduler";
|
|
245
|
-
}, [import("convex/values").VUnion<{
|
|
246
|
-
kind: "success";
|
|
247
|
-
returnValue: any;
|
|
248
|
-
} | {
|
|
249
|
-
kind: "failed";
|
|
250
|
-
error: string;
|
|
251
|
-
} | {
|
|
252
|
-
kind: "canceled";
|
|
253
197
|
}, [import("convex/values").VObject<{
|
|
254
198
|
kind: "success";
|
|
255
199
|
returnValue: any;
|
|
@@ -266,10 +210,6 @@ declare const _default: import("convex/server").SchemaDefinition<{
|
|
|
266
210
|
kind: "canceled";
|
|
267
211
|
}, {
|
|
268
212
|
kind: import("convex/values").VLiteral<"canceled", "required">;
|
|
269
|
-
}, "required", "kind">], "required", "kind" | "returnValue" | `returnValue.${string}` | "error">, import("convex/values").VObject<{
|
|
270
|
-
kind: "stuckInScheduler";
|
|
271
|
-
}, {
|
|
272
|
-
kind: import("convex/values").VLiteral<"stuckInScheduler", "required">;
|
|
273
213
|
}, "required", "kind">], "required", "kind" | "returnValue" | `returnValue.${string}` | "error">;
|
|
274
214
|
workId: import("convex/values").VId<import("convex/values").GenericId<"work">, "required">;
|
|
275
215
|
retry: import("convex/values").VBoolean<boolean, "required">;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAaA,wBA4FG"}
|
package/dist/component/schema.js
CHANGED
|
@@ -3,9 +3,6 @@ import { v } from "convex/values";
|
|
|
3
3
|
import { fnType, vConfig, vOnCompleteFnContext, retryBehavior, vResult, } from "./shared.js";
|
|
4
4
|
// Represents a slice of time to process work.
|
|
5
5
|
const segment = v.int64();
|
|
6
|
-
export const vResultInternal = v.union(vResult, v.object({
|
|
7
|
-
kind: v.literal("stuckInScheduler"),
|
|
8
|
-
}));
|
|
9
6
|
export default defineSchema({
|
|
10
7
|
// Written from kickLoop, read everywhere.
|
|
11
8
|
globals: defineTable(vConfig),
|
|
@@ -25,7 +22,6 @@ export default defineSchema({
|
|
|
25
22
|
failed: v.number(), // failed after all retries
|
|
26
23
|
retries: v.number(), // failure that turned into a retry
|
|
27
24
|
canceled: v.number(), // cancelations processed
|
|
28
|
-
conflicted: v.optional(v.number()), // mutations conflicted in the scheduler
|
|
29
25
|
lastReportTs: v.number(),
|
|
30
26
|
}),
|
|
31
27
|
running: v.array(v.object({
|
|
@@ -69,7 +65,7 @@ export default defineSchema({
|
|
|
69
65
|
// Written by complete, read & deleted by `main`.
|
|
70
66
|
pendingCompletion: defineTable({
|
|
71
67
|
segment,
|
|
72
|
-
runResult:
|
|
68
|
+
runResult: vResult,
|
|
73
69
|
workId: v.id("work"),
|
|
74
70
|
retry: v.boolean(),
|
|
75
71
|
})
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EACL,MAAM,EACN,OAAO,EACP,oBAAoB,EACpB,aAAa,EACb,OAAO,GACR,MAAM,aAAa,CAAC;AAErB,8CAA8C;AAC9C,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;AAE1B,eAAe,YAAY,CAAC;IAC1B,0CAA0C;IAC1C,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;IAC7B,4CAA4C;IAC5C,aAAa,EAAE,WAAW,CAAC;QACzB,kDAAkD;QAClD,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE;QACrB,cAAc,EAAE,CAAC,CAAC,MAAM,CAAC;YACvB,QAAQ,EAAE,OAAO;YACjB,UAAU,EAAE,OAAO;YACnB,WAAW,EAAE,OAAO;SACrB,CAAC;QACF,YAAY,EAAE,OAAO;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,8CAA8C;YACrE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,+CAA+C;YACtE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,2BAA2B;YAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,mCAAmC;YACxD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,yBAAyB;YAC/C,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;SACzB,CAAC;QACF,OAAO,EAAE,CAAC,CAAC,KAAK,CACd,CAAC,CAAC,MAAM,CAAC;YACP,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;YACpB,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,sBAAsB,CAAC;YACzC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;SACpB,CAAC,CACH;KACF,CAAC;IAEF,uFAAuF;IACvF,mEAAmE;IACnE,SAAS,EAAE,WAAW,CAAC;QACrB,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,EACxC,CAAC,CAAC,MAAM,CAAC;YACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;YAC5B,OAAO;YACP,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,sBAAsB,CAAC;YACzC,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;YACtB,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE;SACtB,CAAC,EACF,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAC7D;KACF,CAAC;IAEF,4EAA4E;IAC5E,IAAI,EAAE,WAAW,CAAC;QAChB,MAAM;QACN,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;QAClB,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QAC3B,kEAAkE;QAClE,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QACtC,WAAW,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,+BAA+B;QACrD,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QAC5C,aAAa,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC;QACxC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;KAClC,CAAC;IAEF,wEAAwE;IACxE,YAAY,EAAE,WAAW,CAAC;QACxB,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,OAAO;KACR,CAAC;SACC,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;SAC3B,KAAK,CAAC,SAAS,EAAE,CAAC,SAAS,CAAC,CAAC;IAEhC,iDAAiD;IACjD,iBAAiB,EAAE,WAAW,CAAC;QAC7B,OAAO;QACP,SAAS,EAAE,OAAO;QAClB,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;QACpB,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE;KACnB,CAAC;SACC,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;SAC3B,KAAK,CAAC,SAAS,EAAE,CAAC,SAAS,CAAC,CAAC;IAEhC,oDAAoD;IACpD,kBAAkB,EAAE,WAAW,CAAC;QAC9B,OAAO;QACP,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC;KACrB,CAAC;SACC,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;SAC3B,KAAK,CAAC,SAAS,EAAE,CAAC,SAAS,CAAC,CAAC;IAEhC,4DAA4D;IAC5D,OAAO,EAAE,WAAW,CAAC;QACnB,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QAC/C,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;KAC7B,CAAC;CACH,CAAC,CAAC"}
|
|
@@ -12,13 +12,12 @@ export declare function recordEnqueued(console: Logger, data: {
|
|
|
12
12
|
runAt: number;
|
|
13
13
|
}): void;
|
|
14
14
|
export declare function recordStarted(console: Logger, work: Doc<"work">, lagMs: number, scheduledFunctionId: Id<"_scheduled_functions">): void;
|
|
15
|
-
export declare function recordCompleted(console: Logger, work: Doc<"work">, status: "success" | "failed" | "canceled" | "retrying"
|
|
15
|
+
export declare function recordCompleted(console: Logger, work: Doc<"work">, status: "success" | "failed" | "canceled" | "retrying", onCompleteScheduledFunctionId: Id<"_scheduled_functions"> | undefined): void;
|
|
16
16
|
export declare function generateReport(ctx: MutationCtx, console: Logger, state: Doc<"internalState">, { maxParallelism, logLevel }: Config): Promise<void>;
|
|
17
17
|
export declare const calculateBacklogAndReport: import("convex/server").RegisteredMutation<"internal", {
|
|
18
18
|
logLevel: "DEBUG" | "TRACE" | "INFO" | "REPORT" | "WARN" | "ERROR";
|
|
19
19
|
running: number;
|
|
20
20
|
report: {
|
|
21
|
-
conflicted?: number | undefined;
|
|
22
21
|
failed: number;
|
|
23
22
|
canceled: number;
|
|
24
23
|
completed: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/component/stats.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,KAAK,MAAM,EAGZ,MAAM,aAAa,CAAC;AACrB,OAAO,EAAgB,KAAK,MAAM,EAAuB,MAAM,cAAc,CAAC;AAK9E;;;GAGG;AAEH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IACJ,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf,QAMF;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,KAAK,EAAE,MAAM,EACb,mBAAmB,EAAE,EAAE,CAAC,sBAAsB,CAAC,QAUhD;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,MAAM,
|
|
1
|
+
{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/component/stats.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,KAAK,MAAM,EAGZ,MAAM,aAAa,CAAC;AACrB,OAAO,EAAgB,KAAK,MAAM,EAAuB,MAAM,cAAc,CAAC;AAK9E;;;GAGG;AAEH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IACJ,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf,QAMF;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,KAAK,EAAE,MAAM,EACb,mBAAmB,EAAE,EAAE,CAAC,sBAAsB,CAAC,QAUhD;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,UAAU,GAAG,UAAU,EACtD,6BAA6B,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,SAAS,QAUtE;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,WAAW,EAChB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,GAAG,CAAC,eAAe,CAAC,EAC3B,EAAE,cAAc,EAAE,QAAQ,EAAE,EAAE,MAAM,iBAkCrC;AAED,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;iBAmBpC,CAAC;AAiBH;;;GAGG;AACH,eAAO,MAAM,WAAW;;;;;;;;GA0BtB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../src/component/stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,OAAO,EACL,gBAAgB,EAChB,aAAa,GAEd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAe,QAAQ,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9E,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,kCAAkC,CAAC;AAE7D;;;GAGG;AAEH,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,IAIC;IAED,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE;QACxB,GAAG,IAAI;QACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;KACvB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,OAAe,EACf,IAAiB,EACjB,KAAa,EACb,mBAA+C;IAE/C,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE;QACvB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,aAAa;QAC9B,mBAAmB;QACnB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,IAAiB,EACjB,
|
|
1
|
+
{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../src/component/stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,OAAO,EACL,gBAAgB,EAChB,aAAa,GAEd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAe,QAAQ,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9E,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,kCAAkC,CAAC;AAE7D;;;GAGG;AAEH,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,IAIC;IAED,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE;QACxB,GAAG,IAAI;QACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;KACvB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,OAAe,EACf,IAAiB,EACjB,KAAa,EACb,mBAA+C;IAE/C,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE;QACvB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,aAAa;QAC9B,mBAAmB;QACnB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,IAAiB,EACjB,MAAsD,EACtD,6BAAqE;IAErE,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE;QACzB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,6BAA6B;QAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM;KACP,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAgB,EAChB,OAAe,EACf,KAA2B,EAC3B,EAAE,cAAc,EAAE,QAAQ,EAAU;IAEpC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QACnC,8CAA8C;QAC9C,OAAO;IACT,CAAC;IACD,MAAM,cAAc,GAAG,iBAAiB,EAAE,CAAC;IAC3C,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC;SACjD,KAAK,CAAC,cAAc,CAAC;SACrB,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC;SACE,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC;SAC7C,EAAE,CAAC,SAAS,EAAE,cAAc,CAAC,CACjC;SACA,QAAQ,CAAC;QACR,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;IACL,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;QACxB,YAAY,CAAC,OAAO,EAAE;YACpB,GAAG,KAAK,CAAC,MAAM;YACf,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YAC7B,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM;SAClC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,yBAAyB,EAAE;YACxE,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,cAAc;YAC1B,MAAM,EAAE,YAAY,CAAC,cAAc;YACnC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YAC7B,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,yBAAyB,GAAG,gBAAgB,CAAC;IACxD,IAAI,EAAE;QACJ,YAAY,EAAE,CAAC,CAAC,KAAK,EAAE;QACvB,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;QAClB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM;QAC3D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,QAAQ;KACT;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,MAAM,YAAY,GAAG,MAAO,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAS,CAAC,KAAK,EAAE,CAAC;QAEzE,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,YAAY,CAAC,OAAO,EAAE;YACpB,GAAG,IAAI,CAAC,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,YAAY;SACtB,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,SAAS,YAAY,CACnB,OAAe,EACf,MAA6E;IAE7E,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAC9C,MAAM,cAAc,GAAG,SAAS,GAAG,OAAO,CAAC;IAC3C,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,MAAM,oBAAoB,GAAG,cAAc,CAAC,CAAC,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE;QACtB,GAAG,MAAM;QACT,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC3C,oBAAoB,EAAE,MAAM,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;KAC9D,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;IACvC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE;IAChB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;QACtD,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC;QACnE,MAAM,cAAc,GAAG,aAAa,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAC1D,MAAM,cAAc,GAAG,MAAM,EAAE,cAAc,IAAI,uBAAuB,CAAC;QACzE,MAAM,YAAY,GAAG,MAAO,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAS,CAAC,KAAK,EAAE,CAAC;QACzE,MAAM,iBAAiB,GAAG,MACxB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,mBAAmB,CACjC,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,kBAAkB,GAAG,MACzB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,oBAAoB,CAClC,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3D,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,OAAO,EAAE,YAAY;YACrB,OAAO,EAAE,cAAc,GAAG,iBAAiB;YAC3C,UAAU,EAAE,iBAAiB;YAC7B,aAAa,EAAE,cAAc,GAAG,cAAc;YAC9C,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI;YAChC,UAAU,EAAE,aAAa,EAAE,UAAU;SACtC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -237,66 +237,6 @@ describe("complete", () => {
|
|
|
237
237
|
});
|
|
238
238
|
});
|
|
239
239
|
|
|
240
|
-
it("should process a stuckInScheduler job", async () => {
|
|
241
|
-
// Create a spy on runMutation
|
|
242
|
-
const runMutationSpy = vi.fn();
|
|
243
|
-
|
|
244
|
-
// Enqueue a work item
|
|
245
|
-
const workId = await t.mutation(api.lib.enqueue, {
|
|
246
|
-
fnHandle: "testHandle",
|
|
247
|
-
fnName: "testFunction",
|
|
248
|
-
fnArgs: { test: "data" },
|
|
249
|
-
fnType: "mutation",
|
|
250
|
-
runAt: Date.now(),
|
|
251
|
-
config: {
|
|
252
|
-
maxParallelism: 10,
|
|
253
|
-
logLevel: "WARN",
|
|
254
|
-
},
|
|
255
|
-
onComplete: {
|
|
256
|
-
fnHandle: "testOnComplete",
|
|
257
|
-
},
|
|
258
|
-
});
|
|
259
|
-
|
|
260
|
-
// Simulate a stuckInScheduler job completion
|
|
261
|
-
await t.run(async (ctx) => {
|
|
262
|
-
// Create a modified context with a spy on runMutation
|
|
263
|
-
const spyCtx = {
|
|
264
|
-
...ctx,
|
|
265
|
-
runMutation: runMutationSpy,
|
|
266
|
-
};
|
|
267
|
-
await completeHandler(spyCtx, {
|
|
268
|
-
jobs: [
|
|
269
|
-
{
|
|
270
|
-
workId,
|
|
271
|
-
runResult: { kind: "stuckInScheduler" },
|
|
272
|
-
attempt: 0,
|
|
273
|
-
},
|
|
274
|
-
],
|
|
275
|
-
});
|
|
276
|
-
// Verify onComplete was not called
|
|
277
|
-
expect(runMutationSpy).not.toHaveBeenCalled();
|
|
278
|
-
});
|
|
279
|
-
|
|
280
|
-
// Verify work was not deleted (since it should be retried)
|
|
281
|
-
await t.run(async (ctx) => {
|
|
282
|
-
const work = await ctx.db.get(workId);
|
|
283
|
-
expect(work).not.toBeNull();
|
|
284
|
-
// Verify attempts was incremented from 0
|
|
285
|
-
expect(work?.attempts).toBe(1);
|
|
286
|
-
});
|
|
287
|
-
|
|
288
|
-
// Verify pendingCompletion was created with retry=true
|
|
289
|
-
await t.run(async (ctx) => {
|
|
290
|
-
const pendingCompletions = await ctx.db
|
|
291
|
-
.query("pendingCompletion")
|
|
292
|
-
.withIndex("workId", (q) => q.eq("workId", workId))
|
|
293
|
-
.collect();
|
|
294
|
-
expect(pendingCompletions).toHaveLength(1);
|
|
295
|
-
expect(pendingCompletions[0].runResult.kind).toBe("stuckInScheduler");
|
|
296
|
-
expect(pendingCompletions[0].retry).toBe(true);
|
|
297
|
-
});
|
|
298
|
-
});
|
|
299
|
-
|
|
300
240
|
it("should call onComplete handler for successful jobs", async () => {
|
|
301
241
|
// Create a spy on runMutation
|
|
302
242
|
// const runMutationSpy = vi.fn();
|
|
@@ -5,17 +5,16 @@ import { internal } from "./_generated/api.js";
|
|
|
5
5
|
import { internalMutation, type MutationCtx } from "./_generated/server.js";
|
|
6
6
|
import { kickMainLoop } from "./kick.js";
|
|
7
7
|
import { createLogger } from "./logging.js";
|
|
8
|
-
import { type OnCompleteArgs } from "./shared.js";
|
|
8
|
+
import { type OnCompleteArgs, type RunResult, vResult } from "./shared.js";
|
|
9
9
|
import { recordCompleted } from "./stats.js";
|
|
10
10
|
import { assert } from "convex-helpers";
|
|
11
|
-
import { vResultInternal, type RunResultInternal } from "./schema.js";
|
|
12
11
|
|
|
13
12
|
export type CompleteJob = Infer<typeof completeArgs.fields.jobs.element>;
|
|
14
13
|
|
|
15
14
|
const completeArgs = v.object({
|
|
16
15
|
jobs: v.array(
|
|
17
16
|
v.object({
|
|
18
|
-
runResult:
|
|
17
|
+
runResult: vResult,
|
|
19
18
|
workId: v.id("work"),
|
|
20
19
|
attempt: v.number(),
|
|
21
20
|
// TODO: need to be careful about removing this field later
|
|
@@ -34,7 +33,7 @@ export async function completeHandler(
|
|
|
34
33
|
return;
|
|
35
34
|
}
|
|
36
35
|
const pendingCompletions: {
|
|
37
|
-
runResult:
|
|
36
|
+
runResult: RunResult;
|
|
38
37
|
workId: Id<"work">;
|
|
39
38
|
retry: boolean;
|
|
40
39
|
}[] = [];
|
|
@@ -115,11 +114,10 @@ export async function completeHandler(
|
|
|
115
114
|
}
|
|
116
115
|
const maxAttempts = work.retryBehavior?.maxAttempts;
|
|
117
116
|
const retry =
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
if (!retry && job.runResult.kind !== "stuckInScheduler") {
|
|
117
|
+
job.runResult.kind === "failed" &&
|
|
118
|
+
!!maxAttempts &&
|
|
119
|
+
work.attempts < maxAttempts;
|
|
120
|
+
if (!retry) {
|
|
123
121
|
let scheduledId = undefined;
|
|
124
122
|
if (work.onComplete) {
|
|
125
123
|
try {
|
|
@@ -206,7 +204,7 @@ export async function completeHandler(
|
|
|
206
204
|
}
|
|
207
205
|
}
|
|
208
206
|
|
|
209
|
-
function stripResult(result:
|
|
207
|
+
function stripResult(result: RunResult): RunResult {
|
|
210
208
|
if (result.kind === "success") {
|
|
211
209
|
return { kind: "success", returnValue: null };
|
|
212
210
|
}
|
|
@@ -89,7 +89,6 @@ describe("loop", () => {
|
|
|
89
89
|
failed: 0,
|
|
90
90
|
retries: 0,
|
|
91
91
|
canceled: 0,
|
|
92
|
-
conflicted: 0,
|
|
93
92
|
lastReportTs: Date.now(),
|
|
94
93
|
},
|
|
95
94
|
running: [],
|
|
@@ -314,88 +313,6 @@ describe("loop", () => {
|
|
|
314
313
|
expect(work!.attempts).toBe(1);
|
|
315
314
|
});
|
|
316
315
|
});
|
|
317
|
-
|
|
318
|
-
it("should follow the complete -> pendingCompletion -> pendingStart flow for mutations stuck in the scheduler", async () => {
|
|
319
|
-
// Setup initial state with a running job that gets stuck in the "pending" state in the scheduler
|
|
320
|
-
const workId = await t.run<Id<"work">>(async (ctx) => {
|
|
321
|
-
// Create internal state
|
|
322
|
-
await insertInternalState(ctx);
|
|
323
|
-
|
|
324
|
-
// Create running runStatus
|
|
325
|
-
await ctx.db.insert("runStatus", {
|
|
326
|
-
state: { kind: "running" },
|
|
327
|
-
});
|
|
328
|
-
|
|
329
|
-
// Create work
|
|
330
|
-
const workId = await makeDummyWork(ctx, { fnType: "mutation" });
|
|
331
|
-
|
|
332
|
-
// Schedule a function and get its ID
|
|
333
|
-
const scheduledId = await makeDummyScheduledFunction(ctx, workId);
|
|
334
|
-
|
|
335
|
-
// Add to running list
|
|
336
|
-
const state = await ctx.db.query("internalState").unique();
|
|
337
|
-
assert(state);
|
|
338
|
-
await ctx.db.patch(state._id, {
|
|
339
|
-
running: [{ workId, scheduledId, started: Date.now() }],
|
|
340
|
-
});
|
|
341
|
-
|
|
342
|
-
return workId;
|
|
343
|
-
});
|
|
344
|
-
|
|
345
|
-
// Complete the work with failure (workerRunning -> complete)
|
|
346
|
-
await t.mutation(internal.complete.complete, {
|
|
347
|
-
jobs: [
|
|
348
|
-
{
|
|
349
|
-
workId,
|
|
350
|
-
runResult: { kind: "stuckInScheduler" },
|
|
351
|
-
attempt: 0,
|
|
352
|
-
},
|
|
353
|
-
],
|
|
354
|
-
});
|
|
355
|
-
|
|
356
|
-
// Verify pendingCompletion was created with retry=true
|
|
357
|
-
await t.run(async (ctx) => {
|
|
358
|
-
const pendingCompletions = await ctx.db
|
|
359
|
-
.query("pendingCompletion")
|
|
360
|
-
.collect();
|
|
361
|
-
expect(pendingCompletions).toHaveLength(1);
|
|
362
|
-
expect(pendingCompletions[0].workId).toBe(workId);
|
|
363
|
-
expect(pendingCompletions[0].runResult.kind).toBe("stuckInScheduler");
|
|
364
|
-
expect(pendingCompletions[0].retry).toBe(true);
|
|
365
|
-
});
|
|
366
|
-
|
|
367
|
-
// Run main loop to process pendingCompletion -> pendingStart.
|
|
368
|
-
// Since stuckInScheduler retries have 0 backoff, the pendingStart
|
|
369
|
-
// is immediately picked up by handleStart in the same main call.
|
|
370
|
-
await t.mutation(internal.loop.main, {
|
|
371
|
-
generation: 1n,
|
|
372
|
-
segment: getNextSegment(),
|
|
373
|
-
});
|
|
374
|
-
|
|
375
|
-
// Verify the job was re-started immediately
|
|
376
|
-
await t.run(async (ctx) => {
|
|
377
|
-
// Check that pendingCompletion was deleted
|
|
378
|
-
const pendingCompletions = await ctx.db
|
|
379
|
-
.query("pendingCompletion")
|
|
380
|
-
.collect();
|
|
381
|
-
expect(pendingCompletions).toHaveLength(0);
|
|
382
|
-
|
|
383
|
-
// pendingStart was consumed by handleStart in the same main call
|
|
384
|
-
const pendingStarts = await ctx.db.query("pendingStart").collect();
|
|
385
|
-
expect(pendingStarts).toHaveLength(0);
|
|
386
|
-
|
|
387
|
-
// Check that work still exists and attempts was incremented
|
|
388
|
-
const work = await ctx.db.get(workId);
|
|
389
|
-
expect(work).not.toBeNull();
|
|
390
|
-
expect(work!.attempts).toBe(1);
|
|
391
|
-
|
|
392
|
-
// Check that the job is back in the running list
|
|
393
|
-
const state = await ctx.db.query("internalState").unique();
|
|
394
|
-
assert(state);
|
|
395
|
-
expect(state.running).toHaveLength(1);
|
|
396
|
-
expect(state.running[0].workId).toBe(workId);
|
|
397
|
-
});
|
|
398
|
-
});
|
|
399
316
|
});
|
|
400
317
|
|
|
401
318
|
describe("status transitions", () => {
|
package/src/component/loop.ts
CHANGED
|
@@ -28,8 +28,7 @@ const RECOVERY_BATCH_SIZE = 32;
|
|
|
28
28
|
const MS = 1;
|
|
29
29
|
const SECOND = 1000 * MS;
|
|
30
30
|
const MINUTE = 60 * SECOND;
|
|
31
|
-
const
|
|
32
|
-
const MUTATION_RECOVERY_THRESHOLD_MS = 1 * MINUTE; // attempt to recover jobs this old.
|
|
31
|
+
const RECOVERY_THRESHOLD_MS = 5 * MINUTE; // attempt to recover jobs this old.
|
|
33
32
|
export const RECOVERY_PERIOD_SEGMENTS = toSegment(1 * MINUTE); // how often to check.
|
|
34
33
|
export const STATUS_COOLDOWN = 2 * SECOND;
|
|
35
34
|
export const COOLDOWN_CHECK_INTERVAL = 200 * MS;
|
|
@@ -44,7 +43,6 @@ export const INITIAL_STATE: WithoutSystemFields<Doc<"internalState">> = {
|
|
|
44
43
|
failed: 0,
|
|
45
44
|
retries: 0,
|
|
46
45
|
canceled: 0,
|
|
47
|
-
conflicted: 0,
|
|
48
46
|
lastReportTs: 0,
|
|
49
47
|
},
|
|
50
48
|
running: [],
|
|
@@ -113,7 +111,6 @@ export const main = internalMutation({
|
|
|
113
111
|
failed: 0,
|
|
114
112
|
retries: 0,
|
|
115
113
|
canceled: 0,
|
|
116
|
-
conflicted: 0,
|
|
117
114
|
lastReportTs,
|
|
118
115
|
};
|
|
119
116
|
}
|
|
@@ -403,21 +400,10 @@ async function handleCompletions(
|
|
|
403
400
|
console.warn(`[main] ${c.workId} is gone, but trying to complete`);
|
|
404
401
|
return;
|
|
405
402
|
}
|
|
406
|
-
const
|
|
407
|
-
const retried = await rescheduleJob(
|
|
408
|
-
ctx,
|
|
409
|
-
work,
|
|
410
|
-
console,
|
|
411
|
-
wasStuckInScheduler,
|
|
412
|
-
);
|
|
403
|
+
const retried = await rescheduleJob(ctx, work, console);
|
|
413
404
|
if (retried) {
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
recordCompleted(console, work, "retrying conflicted", undefined);
|
|
417
|
-
} else {
|
|
418
|
-
state.report.retries++;
|
|
419
|
-
recordCompleted(console, work, "retrying", undefined);
|
|
420
|
-
}
|
|
405
|
+
state.report.retries++;
|
|
406
|
+
recordCompleted(console, work, "retrying", undefined);
|
|
421
407
|
} else {
|
|
422
408
|
// We don't retry if it's been canceled in the mean time.
|
|
423
409
|
state.report.canceled++;
|
|
@@ -511,17 +497,11 @@ async function handleRecovery(
|
|
|
511
497
|
console: Logger,
|
|
512
498
|
) {
|
|
513
499
|
const missing = new Set<Id<"work">>();
|
|
514
|
-
const
|
|
515
|
-
const mutationOldEnoughToConsider =
|
|
516
|
-
Date.now() - MUTATION_RECOVERY_THRESHOLD_MS;
|
|
500
|
+
const oldEnoughToConsider = Date.now() - RECOVERY_THRESHOLD_MS;
|
|
517
501
|
const jobs = (
|
|
518
502
|
await Promise.all(
|
|
519
503
|
state.running.map(async (r) => {
|
|
520
|
-
if (
|
|
521
|
-
r.started >=
|
|
522
|
-
Math.max(actionOldEnoughToConsider, mutationOldEnoughToConsider)
|
|
523
|
-
) {
|
|
524
|
-
// Avoid getting the work if possible
|
|
504
|
+
if (r.started >= oldEnoughToConsider) {
|
|
525
505
|
return null;
|
|
526
506
|
}
|
|
527
507
|
const work = await ctx.db.get(r.workId);
|
|
@@ -530,13 +510,6 @@ async function handleRecovery(
|
|
|
530
510
|
console.error(`[main] ${r.workId} already gone (skipping recovery)`);
|
|
531
511
|
return null;
|
|
532
512
|
}
|
|
533
|
-
const oldEnoughToConsider =
|
|
534
|
-
work.fnType === "action"
|
|
535
|
-
? actionOldEnoughToConsider
|
|
536
|
-
: mutationOldEnoughToConsider;
|
|
537
|
-
if (r.started >= oldEnoughToConsider) {
|
|
538
|
-
return null;
|
|
539
|
-
}
|
|
540
513
|
return { ...r, attempt: work.attempts };
|
|
541
514
|
}),
|
|
542
515
|
)
|
|
@@ -646,7 +619,6 @@ async function rescheduleJob(
|
|
|
646
619
|
ctx: MutationCtx,
|
|
647
620
|
work: Doc<"work">,
|
|
648
621
|
console: Logger,
|
|
649
|
-
wasStuckInScheduler: boolean,
|
|
650
622
|
): Promise<boolean> {
|
|
651
623
|
const pendingCancelation = await ctx.db
|
|
652
624
|
.query("pendingCancelation")
|
|
@@ -660,14 +632,7 @@ async function rescheduleJob(
|
|
|
660
632
|
if (work.canceled) {
|
|
661
633
|
return false;
|
|
662
634
|
}
|
|
663
|
-
|
|
664
|
-
if (wasStuckInScheduler) {
|
|
665
|
-
backoffMs = 0;
|
|
666
|
-
} else if (work.retryBehavior) {
|
|
667
|
-
backoffMs =
|
|
668
|
-
work.retryBehavior.initialBackoffMs *
|
|
669
|
-
Math.pow(work.retryBehavior.base, work.attempts - 1);
|
|
670
|
-
} else {
|
|
635
|
+
if (!work.retryBehavior) {
|
|
671
636
|
console.warn(`[main] ${work._id} has no retryBehavior so not retrying`);
|
|
672
637
|
return false;
|
|
673
638
|
}
|
|
@@ -680,6 +645,9 @@ async function rescheduleJob(
|
|
|
680
645
|
console.error(`[main] ${work._id} already in pendingStart so not retrying`);
|
|
681
646
|
return false;
|
|
682
647
|
}
|
|
648
|
+
const backoffMs =
|
|
649
|
+
work.retryBehavior.initialBackoffMs *
|
|
650
|
+
Math.pow(work.retryBehavior.base, work.attempts - 1);
|
|
683
651
|
const nextAttempt = withJitter(backoffMs);
|
|
684
652
|
const startTime = boundScheduledTime(Date.now() + nextAttempt, console);
|
|
685
653
|
const segment = toSegment(startTime);
|
|
@@ -354,119 +354,6 @@ describe("recovery", () => {
|
|
|
354
354
|
});
|
|
355
355
|
});
|
|
356
356
|
|
|
357
|
-
it("should handle pending scheduled mutations", async () => {
|
|
358
|
-
// Create work and scheduled function
|
|
359
|
-
let workId: Id<"work">;
|
|
360
|
-
let scheduledId: Id<"_scheduled_functions">;
|
|
361
|
-
|
|
362
|
-
await t.run(async (ctx) => {
|
|
363
|
-
workId = await makeDummyWork(ctx, { fnType: "mutation" });
|
|
364
|
-
scheduledId = await makeDummyScheduledFunction(ctx, workId);
|
|
365
|
-
});
|
|
366
|
-
|
|
367
|
-
// Run recovery with mocked system.get
|
|
368
|
-
await t.run(async (ctx) => {
|
|
369
|
-
// Mock the system.get to return a pending state
|
|
370
|
-
ctx.db.system.get = patchedSystemGet(ctx.db, {
|
|
371
|
-
[scheduledId]: {
|
|
372
|
-
_id: scheduledId,
|
|
373
|
-
_creationTime: Date.now(),
|
|
374
|
-
name: "internal/worker.runMutationWrapper",
|
|
375
|
-
args: [
|
|
376
|
-
{
|
|
377
|
-
workId,
|
|
378
|
-
fnHandle: "test_handle",
|
|
379
|
-
fnArgs: {},
|
|
380
|
-
logLevel: "WARN",
|
|
381
|
-
attempt: 0,
|
|
382
|
-
},
|
|
383
|
-
],
|
|
384
|
-
scheduledTime: Date.now(),
|
|
385
|
-
state: {
|
|
386
|
-
kind: "pending",
|
|
387
|
-
},
|
|
388
|
-
},
|
|
389
|
-
});
|
|
390
|
-
|
|
391
|
-
await recoveryHandler(ctx, {
|
|
392
|
-
jobs: [
|
|
393
|
-
{
|
|
394
|
-
scheduledId,
|
|
395
|
-
workId,
|
|
396
|
-
attempt: 0,
|
|
397
|
-
started: Date.now(),
|
|
398
|
-
},
|
|
399
|
-
],
|
|
400
|
-
});
|
|
401
|
-
});
|
|
402
|
-
|
|
403
|
-
// Verify pendingCompletion was created with stuckInScheduler
|
|
404
|
-
await t.run(async (ctx) => {
|
|
405
|
-
const pendingCompletions = await ctx.db
|
|
406
|
-
.query("pendingCompletion")
|
|
407
|
-
.withIndex("workId", (q) => q.eq("workId", workId))
|
|
408
|
-
.collect();
|
|
409
|
-
expect(pendingCompletions).toHaveLength(1);
|
|
410
|
-
expect(pendingCompletions[0].runResult.kind).toBe("stuckInScheduler");
|
|
411
|
-
});
|
|
412
|
-
});
|
|
413
|
-
|
|
414
|
-
it("should not process pending scheduled actions", async () => {
|
|
415
|
-
// Create work and scheduled function
|
|
416
|
-
let workId: Id<"work">;
|
|
417
|
-
let scheduledId: Id<"_scheduled_functions">;
|
|
418
|
-
|
|
419
|
-
await t.run(async (ctx) => {
|
|
420
|
-
workId = await makeDummyWork(ctx);
|
|
421
|
-
scheduledId = await makeDummyScheduledFunction(ctx, workId);
|
|
422
|
-
});
|
|
423
|
-
|
|
424
|
-
// Run recovery with mocked system.get
|
|
425
|
-
await t.run(async (ctx) => {
|
|
426
|
-
// Mock the system.get to return a pending state
|
|
427
|
-
ctx.db.system.get = patchedSystemGet(ctx.db, {
|
|
428
|
-
[scheduledId]: {
|
|
429
|
-
_id: scheduledId,
|
|
430
|
-
_creationTime: Date.now(),
|
|
431
|
-
name: "internal/worker.runActionWrapper",
|
|
432
|
-
args: [
|
|
433
|
-
{
|
|
434
|
-
workId,
|
|
435
|
-
fnHandle: "test_handle",
|
|
436
|
-
fnArgs: {},
|
|
437
|
-
logLevel: "WARN",
|
|
438
|
-
attempt: 0,
|
|
439
|
-
},
|
|
440
|
-
],
|
|
441
|
-
scheduledTime: Date.now(),
|
|
442
|
-
state: {
|
|
443
|
-
kind: "pending",
|
|
444
|
-
},
|
|
445
|
-
},
|
|
446
|
-
});
|
|
447
|
-
|
|
448
|
-
await recoveryHandler(ctx, {
|
|
449
|
-
jobs: [
|
|
450
|
-
{
|
|
451
|
-
scheduledId,
|
|
452
|
-
workId,
|
|
453
|
-
attempt: 0,
|
|
454
|
-
started: Date.now(),
|
|
455
|
-
},
|
|
456
|
-
],
|
|
457
|
-
});
|
|
458
|
-
});
|
|
459
|
-
|
|
460
|
-
// Verify no pendingCompletion was created
|
|
461
|
-
await t.run(async (ctx) => {
|
|
462
|
-
const pendingCompletions = await ctx.db
|
|
463
|
-
.query("pendingCompletion")
|
|
464
|
-
.withIndex("workId", (q) => q.eq("workId", workId))
|
|
465
|
-
.collect();
|
|
466
|
-
expect(pendingCompletions).toHaveLength(0);
|
|
467
|
-
});
|
|
468
|
-
});
|
|
469
|
-
|
|
470
357
|
it("should handle multiple jobs in a single call", async () => {
|
|
471
358
|
// Create multiple work items and scheduled functions
|
|
472
359
|
let workId1: Id<"work">;
|
|
@@ -585,7 +472,7 @@ describe("recovery", () => {
|
|
|
585
472
|
|
|
586
473
|
// Run recovery with mocked system.get
|
|
587
474
|
await t.run(async (ctx) => {
|
|
588
|
-
// Mock the system.get to return a
|
|
475
|
+
// Mock the system.get to return a pending state
|
|
589
476
|
ctx.db.system.get = patchedSystemGet(ctx.db, {
|
|
590
477
|
[scheduledId]: {
|
|
591
478
|
_id: scheduledId,
|
|
@@ -602,7 +489,7 @@ describe("recovery", () => {
|
|
|
602
489
|
],
|
|
603
490
|
scheduledTime: Date.now(),
|
|
604
491
|
state: {
|
|
605
|
-
kind: "
|
|
492
|
+
kind: "pending",
|
|
606
493
|
},
|
|
607
494
|
},
|
|
608
495
|
});
|
|
@@ -96,23 +96,6 @@ export async function recoveryHandler(
|
|
|
96
96
|
});
|
|
97
97
|
break;
|
|
98
98
|
}
|
|
99
|
-
case "pending": {
|
|
100
|
-
if (work.fnType === "action") {
|
|
101
|
-
// We do not cancel and re-enqueue actions. If a scheduled action is still
|
|
102
|
-
// pending, the scheduler is likely backlogged.
|
|
103
|
-
break;
|
|
104
|
-
}
|
|
105
|
-
// It looks like the function has been retried by the scheduler several times.
|
|
106
|
-
// The scheduler backoff is too long, so cancel and re-enqueue the job to
|
|
107
|
-
// free up space for more work.
|
|
108
|
-
await ctx.scheduler.cancel(scheduled._id);
|
|
109
|
-
completionJobs.push({
|
|
110
|
-
workId: job.workId,
|
|
111
|
-
runResult: { kind: "stuckInScheduler" },
|
|
112
|
-
attempt: job.attempt,
|
|
113
|
-
});
|
|
114
|
-
break;
|
|
115
|
-
}
|
|
116
99
|
}
|
|
117
100
|
}
|
|
118
101
|
if (completionJobs.length > 0) {
|
package/src/component/schema.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { defineSchema, defineTable } from "convex/server";
|
|
2
|
-
import { v
|
|
2
|
+
import { v } from "convex/values";
|
|
3
3
|
import {
|
|
4
4
|
fnType,
|
|
5
5
|
vConfig,
|
|
@@ -11,14 +11,6 @@ import {
|
|
|
11
11
|
// Represents a slice of time to process work.
|
|
12
12
|
const segment = v.int64();
|
|
13
13
|
|
|
14
|
-
export const vResultInternal = v.union(
|
|
15
|
-
vResult,
|
|
16
|
-
v.object({
|
|
17
|
-
kind: v.literal("stuckInScheduler"),
|
|
18
|
-
}),
|
|
19
|
-
);
|
|
20
|
-
export type RunResultInternal = Infer<typeof vResultInternal>;
|
|
21
|
-
|
|
22
14
|
export default defineSchema({
|
|
23
15
|
// Written from kickLoop, read everywhere.
|
|
24
16
|
globals: defineTable(vConfig),
|
|
@@ -38,7 +30,6 @@ export default defineSchema({
|
|
|
38
30
|
failed: v.number(), // failed after all retries
|
|
39
31
|
retries: v.number(), // failure that turned into a retry
|
|
40
32
|
canceled: v.number(), // cancelations processed
|
|
41
|
-
conflicted: v.optional(v.number()), // mutations conflicted in the scheduler
|
|
42
33
|
lastReportTs: v.number(),
|
|
43
34
|
}),
|
|
44
35
|
running: v.array(
|
|
@@ -92,7 +83,7 @@ export default defineSchema({
|
|
|
92
83
|
// Written by complete, read & deleted by `main`.
|
|
93
84
|
pendingCompletion: defineTable({
|
|
94
85
|
segment,
|
|
95
|
-
runResult:
|
|
86
|
+
runResult: vResult,
|
|
96
87
|
workId: v.id("work"),
|
|
97
88
|
retry: v.boolean(),
|
|
98
89
|
})
|
|
@@ -64,7 +64,6 @@ describe("stats", () => {
|
|
|
64
64
|
failed: 0,
|
|
65
65
|
retries: 0,
|
|
66
66
|
canceled: 0,
|
|
67
|
-
conflicted: 0,
|
|
68
67
|
lastReportTs: 0,
|
|
69
68
|
},
|
|
70
69
|
running: [],
|
|
@@ -110,7 +109,6 @@ describe("stats", () => {
|
|
|
110
109
|
failed: 2,
|
|
111
110
|
retries: 2,
|
|
112
111
|
canceled: 0,
|
|
113
|
-
conflicted: 0,
|
|
114
112
|
lastReportTs: 0,
|
|
115
113
|
},
|
|
116
114
|
running: [],
|
|
@@ -162,7 +160,6 @@ describe("stats", () => {
|
|
|
162
160
|
failed: 2,
|
|
163
161
|
retries: 2,
|
|
164
162
|
canceled: 0,
|
|
165
|
-
conflicted: 0,
|
|
166
163
|
failureRate: 0.4, // (failed + retries) / completed = (2 + 2) / 10 = 0.4
|
|
167
164
|
permanentFailureRate: 0.25, // failed / (completed - retries) = 2 / (10 - 2) = 2/8
|
|
168
165
|
lastReportTs: expect.any(Number),
|
|
@@ -186,7 +183,6 @@ describe("stats", () => {
|
|
|
186
183
|
failed: 1,
|
|
187
184
|
retries: 1,
|
|
188
185
|
canceled: 0,
|
|
189
|
-
conflicted: 0,
|
|
190
186
|
lastReportTs: 0,
|
|
191
187
|
},
|
|
192
188
|
running: [],
|
|
@@ -271,7 +267,6 @@ describe("stats", () => {
|
|
|
271
267
|
failed: 1,
|
|
272
268
|
retries: 1,
|
|
273
269
|
canceled: 0,
|
|
274
|
-
conflicted: 0,
|
|
275
270
|
lastReportTs: 0,
|
|
276
271
|
},
|
|
277
272
|
running: [],
|
package/src/component/stats.ts
CHANGED
|
@@ -53,12 +53,7 @@ export function recordStarted(
|
|
|
53
53
|
export function recordCompleted(
|
|
54
54
|
console: Logger,
|
|
55
55
|
work: Doc<"work">,
|
|
56
|
-
status:
|
|
57
|
-
| "success"
|
|
58
|
-
| "failed"
|
|
59
|
-
| "canceled"
|
|
60
|
-
| "retrying"
|
|
61
|
-
| "retrying conflicted",
|
|
56
|
+
status: "success" | "failed" | "canceled" | "retrying",
|
|
62
57
|
onCompleteScheduledFunctionId: Id<"_scheduled_functions"> | undefined,
|
|
63
58
|
) {
|
|
64
59
|
console.event("completed", {
|