@convex-dev/workpool 0.4.3-alpha.1 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component/complete.d.ts +24 -1
- package/dist/component/complete.d.ts.map +1 -1
- package/dist/component/complete.js +8 -6
- package/dist/component/complete.js.map +1 -1
- package/dist/component/loop.d.ts +1 -0
- package/dist/component/loop.d.ts.map +1 -1
- package/dist/component/loop.js +45 -14
- package/dist/component/loop.js.map +1 -1
- package/dist/component/recovery.d.ts.map +1 -1
- package/dist/component/recovery.js +17 -0
- package/dist/component/recovery.js.map +1 -1
- package/dist/component/schema.d.ts +62 -2
- package/dist/component/schema.d.ts.map +1 -1
- package/dist/component/schema.js +5 -1
- package/dist/component/schema.js.map +1 -1
- package/dist/component/stats.d.ts +2 -1
- package/dist/component/stats.d.ts.map +1 -1
- package/dist/component/stats.js.map +1 -1
- package/package.json +7 -8
- package/src/component/complete.test.ts +60 -0
- package/src/component/complete.ts +11 -9
- package/src/component/loop.test.ts +87 -3
- package/src/component/loop.ts +54 -15
- package/src/component/recovery.test.ts +115 -2
- package/src/component/recovery.ts +17 -0
- package/src/component/schema.ts +11 -2
- package/src/component/stats.test.ts +5 -0
- package/src/component/stats.ts +6 -1
- package/src/test.ts +3 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/component/stats.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,KAAK,MAAM,EAGZ,MAAM,aAAa,CAAC;AACrB,OAAO,EAAgB,KAAK,MAAM,EAAuB,MAAM,cAAc,CAAC;AAK9E;;;GAGG;AAEH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IACJ,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf,QAMF;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,KAAK,EAAE,MAAM,EACb,mBAAmB,EAAE,EAAE,CAAC,sBAAsB,CAAC,QAUhD;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,MAAM,
|
|
1
|
+
{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/component/stats.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,KAAK,MAAM,EAGZ,MAAM,aAAa,CAAC;AACrB,OAAO,EAAgB,KAAK,MAAM,EAAuB,MAAM,cAAc,CAAC;AAK9E;;;GAGG;AAEH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IACJ,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf,QAMF;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,KAAK,EAAE,MAAM,EACb,mBAAmB,EAAE,EAAE,CAAC,sBAAsB,CAAC,QAUhD;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,MAAM,EACF,SAAS,GACT,QAAQ,GACR,UAAU,GACV,UAAU,GACV,qBAAqB,EACzB,6BAA6B,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,SAAS,QAUtE;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,WAAW,EAChB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,GAAG,CAAC,eAAe,CAAC,EAC3B,EAAE,cAAc,EAAE,QAAQ,EAAE,EAAE,MAAM,iBAkCrC;AAED,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;iBAmBpC,CAAC;AAiBH;;;GAGG;AACH,eAAO,MAAM,WAAW;;;;;;;;GA0BtB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../src/component/stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,OAAO,EACL,gBAAgB,EAChB,aAAa,GAEd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAe,QAAQ,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9E,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,kCAAkC,CAAC;AAE7D;;;GAGG;AAEH,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,IAIC;IAED,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE;QACxB,GAAG,IAAI;QACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;KACvB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,OAAe,EACf,IAAiB,EACjB,KAAa,EACb,mBAA+C;IAE/C,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE;QACvB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,aAAa;QAC9B,mBAAmB;QACnB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,IAAiB,EACjB,
|
|
1
|
+
{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../src/component/stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,OAAO,EACL,gBAAgB,EAChB,aAAa,GAEd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAe,QAAQ,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9E,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,kCAAkC,CAAC;AAE7D;;;GAGG;AAEH,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,IAIC;IAED,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE;QACxB,GAAG,IAAI;QACP,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;KACvB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,OAAe,EACf,IAAiB,EACjB,KAAa,EACb,mBAA+C;IAE/C,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE;QACvB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,aAAa;QAC9B,mBAAmB;QACnB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,IAAiB,EACjB,MAKyB,EACzB,6BAAqE;IAErE,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE;QACzB,MAAM,EAAE,IAAI,CAAC,GAAG;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,6BAA6B;QAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM;KACP,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAgB,EAChB,OAAe,EACf,KAA2B,EAC3B,EAAE,cAAc,EAAE,QAAQ,EAAU;IAEpC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QACnC,8CAA8C;QAC9C,OAAO;IACT,CAAC;IACD,MAAM,cAAc,GAAG,iBAAiB,EAAE,CAAC;IAC3C,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC;SACjD,KAAK,CAAC,cAAc,CAAC;SACrB,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1B,CAAC;SACE,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC;SAC7C,EAAE,CAAC,SAAS,EAAE,cAAc,CAAC,CACjC;SACA,QAAQ,CAAC;QACR,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;IACL,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;QACxB,YAAY,CAAC,OAAO,EAAE;YACpB,GAAG,KAAK,CAAC,MAAM;YACf,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YAC7B,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM;SAClC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,yBAAyB,EAAE;YACxE,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,cAAc;YAC1B,MAAM,EAAE,YAAY,CAAC,cAAc;YACnC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YAC7B,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,yBAAyB,GAAG,gBAAgB,CAAC;IACxD,IAAI,EAAE;QACJ,YAAY,EAAE,CAAC,CAAC,KAAK,EAAE;QACvB,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;QAClB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM;QAC3D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,QAAQ;KACT;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,MAAM,YAAY,GAAG,MAAO,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAS,CAAC,KAAK,EAAE,CAAC;QAEzE,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,YAAY,CAAC,OAAO,EAAE;YACpB,GAAG,IAAI,CAAC,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,YAAY;SACtB,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,SAAS,YAAY,CACnB,OAAe,EACf,MAA6E;IAE7E,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAC9C,MAAM,cAAc,GAAG,SAAS,GAAG,OAAO,CAAC;IAC3C,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,MAAM,oBAAoB,GAAG,cAAc,CAAC,CAAC,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE;QACtB,GAAG,MAAM;QACT,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC3C,oBAAoB,EAAE,MAAM,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;KAC9D,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;IACvC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE;IAChB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;QACtD,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC;QACnE,MAAM,cAAc,GAAG,aAAa,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAC1D,MAAM,cAAc,GAAG,MAAM,EAAE,cAAc,IAAI,uBAAuB,CAAC;QACzE,MAAM,YAAY,GAAG,MAAO,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAS,CAAC,KAAK,EAAE,CAAC;QACzE,MAAM,iBAAiB,GAAG,MACxB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,mBAAmB,CACjC,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,kBAAkB,GAAG,MACzB,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,oBAAoB,CAClC,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3D,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,OAAO,EAAE,YAAY;YACrB,OAAO,EAAE,cAAc,GAAG,iBAAiB;YAC3C,UAAU,EAAE,iBAAiB;YAC7B,aAAa,EAAE,cAAc,GAAG,cAAc;YAC9C,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI;YAChC,UAAU,EAAE,aAAa,EAAE,UAAU;SACtC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
"email": "support@convex.dev",
|
|
8
8
|
"url": "https://github.com/get-convex/workpool/issues"
|
|
9
9
|
},
|
|
10
|
-
"version": "0.4.
|
|
10
|
+
"version": "0.4.4",
|
|
11
11
|
"license": "Apache-2.0",
|
|
12
12
|
"keywords": [
|
|
13
13
|
"convex",
|
|
@@ -37,10 +37,9 @@
|
|
|
37
37
|
"test:debug": "vitest --inspect-brk --no-file-parallelism",
|
|
38
38
|
"test:coverage": "vitest run --coverage --coverage.reporter=text",
|
|
39
39
|
"preversion": "npm ci && npm run build:clean && run-p test lint typecheck",
|
|
40
|
-
"prepublishOnly": "npm whoami || npm login",
|
|
41
40
|
"alpha": "npm version prerelease --preid alpha && npm publish --tag alpha && git push --follow-tags",
|
|
42
41
|
"release": "npm version patch && npm publish && git push --follow-tags",
|
|
43
|
-
"version": "vim -c 'normal o' -c 'normal o## '$npm_package_version CHANGELOG.md && prettier -w CHANGELOG.md && git add CHANGELOG.md"
|
|
42
|
+
"version": "(npm whoami || npm login) && vim -c 'normal o' -c 'normal o## '$npm_package_version CHANGELOG.md && prettier -w CHANGELOG.md && git add CHANGELOG.md"
|
|
44
43
|
},
|
|
45
44
|
"files": [
|
|
46
45
|
"dist",
|
|
@@ -72,19 +71,19 @@
|
|
|
72
71
|
"devDependencies": {
|
|
73
72
|
"@edge-runtime/vm": "5.0.0",
|
|
74
73
|
"@eslint/eslintrc": "3.3.3",
|
|
75
|
-
"@eslint/js": "
|
|
74
|
+
"@eslint/js": "10.0.1",
|
|
76
75
|
"@types/node": "24.10.11",
|
|
77
76
|
"@types/react": "19.2.13",
|
|
78
77
|
"@types/react-dom": "19.2.3",
|
|
79
78
|
"@vitejs/plugin-react": "5.1.3",
|
|
80
79
|
"@vitest/coverage-v8": "4.0.18",
|
|
81
80
|
"chokidar-cli": "3.0.0",
|
|
82
|
-
"convex": "1.
|
|
81
|
+
"convex": "1.34.0",
|
|
83
82
|
"convex-helpers": "0.1.111",
|
|
84
83
|
"convex-test": "0.0.41",
|
|
85
84
|
"cpy-cli": "7.0.0",
|
|
86
|
-
"eslint": "
|
|
87
|
-
"eslint-plugin-react-hooks": "7.0
|
|
85
|
+
"eslint": "10.0.2",
|
|
86
|
+
"eslint-plugin-react-hooks": "7.1.0-canary-3f0b9e61-20260317",
|
|
88
87
|
"eslint-plugin-react-refresh": "0.5.0",
|
|
89
88
|
"globals": "17.3.0",
|
|
90
89
|
"npm-run-all2": "8.0.4",
|
|
@@ -94,7 +93,7 @@
|
|
|
94
93
|
"react": "19.2.4",
|
|
95
94
|
"react-dom": "19.2.4",
|
|
96
95
|
"typescript": "5.9.3",
|
|
97
|
-
"typescript-eslint": "8.
|
|
96
|
+
"typescript-eslint": "8.56.1",
|
|
98
97
|
"vite": "7.3.1",
|
|
99
98
|
"vitest": "4.0.18"
|
|
100
99
|
},
|
|
@@ -237,6 +237,66 @@ describe("complete", () => {
|
|
|
237
237
|
});
|
|
238
238
|
});
|
|
239
239
|
|
|
240
|
+
it("should process a stuckInScheduler job", async () => {
|
|
241
|
+
// Create a spy on runMutation
|
|
242
|
+
const runMutationSpy = vi.fn();
|
|
243
|
+
|
|
244
|
+
// Enqueue a work item
|
|
245
|
+
const workId = await t.mutation(api.lib.enqueue, {
|
|
246
|
+
fnHandle: "testHandle",
|
|
247
|
+
fnName: "testFunction",
|
|
248
|
+
fnArgs: { test: "data" },
|
|
249
|
+
fnType: "mutation",
|
|
250
|
+
runAt: Date.now(),
|
|
251
|
+
config: {
|
|
252
|
+
maxParallelism: 10,
|
|
253
|
+
logLevel: "WARN",
|
|
254
|
+
},
|
|
255
|
+
onComplete: {
|
|
256
|
+
fnHandle: "testOnComplete",
|
|
257
|
+
},
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
// Simulate a stuckInScheduler job completion
|
|
261
|
+
await t.run(async (ctx) => {
|
|
262
|
+
// Create a modified context with a spy on runMutation
|
|
263
|
+
const spyCtx = {
|
|
264
|
+
...ctx,
|
|
265
|
+
runMutation: runMutationSpy,
|
|
266
|
+
};
|
|
267
|
+
await completeHandler(spyCtx, {
|
|
268
|
+
jobs: [
|
|
269
|
+
{
|
|
270
|
+
workId,
|
|
271
|
+
runResult: { kind: "stuckInScheduler" },
|
|
272
|
+
attempt: 0,
|
|
273
|
+
},
|
|
274
|
+
],
|
|
275
|
+
});
|
|
276
|
+
// Verify onComplete was not called
|
|
277
|
+
expect(runMutationSpy).not.toHaveBeenCalled();
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
// Verify work was not deleted (since it should be retried)
|
|
281
|
+
await t.run(async (ctx) => {
|
|
282
|
+
const work = await ctx.db.get(workId);
|
|
283
|
+
expect(work).not.toBeNull();
|
|
284
|
+
// Verify attempts was incremented from 0
|
|
285
|
+
expect(work?.attempts).toBe(1);
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
// Verify pendingCompletion was created with retry=true
|
|
289
|
+
await t.run(async (ctx) => {
|
|
290
|
+
const pendingCompletions = await ctx.db
|
|
291
|
+
.query("pendingCompletion")
|
|
292
|
+
.withIndex("workId", (q) => q.eq("workId", workId))
|
|
293
|
+
.collect();
|
|
294
|
+
expect(pendingCompletions).toHaveLength(1);
|
|
295
|
+
expect(pendingCompletions[0].runResult.kind).toBe("stuckInScheduler");
|
|
296
|
+
expect(pendingCompletions[0].retry).toBe(true);
|
|
297
|
+
});
|
|
298
|
+
});
|
|
299
|
+
|
|
240
300
|
it("should call onComplete handler for successful jobs", async () => {
|
|
241
301
|
// Create a spy on runMutation
|
|
242
302
|
// const runMutationSpy = vi.fn();
|
|
@@ -5,16 +5,17 @@ import { internal } from "./_generated/api.js";
|
|
|
5
5
|
import { internalMutation, type MutationCtx } from "./_generated/server.js";
|
|
6
6
|
import { kickMainLoop } from "./kick.js";
|
|
7
7
|
import { createLogger } from "./logging.js";
|
|
8
|
-
import { type OnCompleteArgs
|
|
8
|
+
import { type OnCompleteArgs } from "./shared.js";
|
|
9
9
|
import { recordCompleted } from "./stats.js";
|
|
10
10
|
import { assert } from "convex-helpers";
|
|
11
|
+
import { vResultInternal, type RunResultInternal } from "./schema.js";
|
|
11
12
|
|
|
12
13
|
export type CompleteJob = Infer<typeof completeArgs.fields.jobs.element>;
|
|
13
14
|
|
|
14
|
-
|
|
15
|
+
const completeArgs = v.object({
|
|
15
16
|
jobs: v.array(
|
|
16
17
|
v.object({
|
|
17
|
-
runResult:
|
|
18
|
+
runResult: vResultInternal,
|
|
18
19
|
workId: v.id("work"),
|
|
19
20
|
attempt: v.number(),
|
|
20
21
|
// TODO: need to be careful about removing this field later
|
|
@@ -33,7 +34,7 @@ export async function completeHandler(
|
|
|
33
34
|
return;
|
|
34
35
|
}
|
|
35
36
|
const pendingCompletions: {
|
|
36
|
-
runResult:
|
|
37
|
+
runResult: RunResultInternal;
|
|
37
38
|
workId: Id<"work">;
|
|
38
39
|
retry: boolean;
|
|
39
40
|
}[] = [];
|
|
@@ -114,10 +115,11 @@ export async function completeHandler(
|
|
|
114
115
|
}
|
|
115
116
|
const maxAttempts = work.retryBehavior?.maxAttempts;
|
|
116
117
|
const retry =
|
|
117
|
-
job.runResult.kind === "failed" &&
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
118
|
+
(job.runResult.kind === "failed" &&
|
|
119
|
+
!!maxAttempts &&
|
|
120
|
+
work.attempts < maxAttempts) ||
|
|
121
|
+
job.runResult.kind === "stuckInScheduler";
|
|
122
|
+
if (!retry && job.runResult.kind !== "stuckInScheduler") {
|
|
121
123
|
let scheduledId = undefined;
|
|
122
124
|
if (work.onComplete) {
|
|
123
125
|
try {
|
|
@@ -204,7 +206,7 @@ export async function completeHandler(
|
|
|
204
206
|
}
|
|
205
207
|
}
|
|
206
208
|
|
|
207
|
-
function stripResult(result:
|
|
209
|
+
function stripResult(result: RunResultInternal): RunResultInternal {
|
|
208
210
|
if (result.kind === "success") {
|
|
209
211
|
return { kind: "success", returnValue: null };
|
|
210
212
|
}
|
|
@@ -16,7 +16,6 @@ import { DEFAULT_LOG_LEVEL } from "./logging.js";
|
|
|
16
16
|
import schema from "./schema.js";
|
|
17
17
|
import {
|
|
18
18
|
DEFAULT_MAX_PARALLELISM,
|
|
19
|
-
fromSegment,
|
|
20
19
|
getCurrentSegment,
|
|
21
20
|
getNextSegment,
|
|
22
21
|
toSegment,
|
|
@@ -90,6 +89,7 @@ describe("loop", () => {
|
|
|
90
89
|
failed: 0,
|
|
91
90
|
retries: 0,
|
|
92
91
|
canceled: 0,
|
|
92
|
+
conflicted: 0,
|
|
93
93
|
lastReportTs: Date.now(),
|
|
94
94
|
},
|
|
95
95
|
running: [],
|
|
@@ -314,6 +314,88 @@ describe("loop", () => {
|
|
|
314
314
|
expect(work!.attempts).toBe(1);
|
|
315
315
|
});
|
|
316
316
|
});
|
|
317
|
+
|
|
318
|
+
it("should follow the complete -> pendingCompletion -> pendingStart flow for mutations stuck in the scheduler", async () => {
|
|
319
|
+
// Setup initial state with a running job that gets stuck in the "pending" state in the scheduler
|
|
320
|
+
const workId = await t.run<Id<"work">>(async (ctx) => {
|
|
321
|
+
// Create internal state
|
|
322
|
+
await insertInternalState(ctx);
|
|
323
|
+
|
|
324
|
+
// Create running runStatus
|
|
325
|
+
await ctx.db.insert("runStatus", {
|
|
326
|
+
state: { kind: "running" },
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
// Create work
|
|
330
|
+
const workId = await makeDummyWork(ctx, { fnType: "mutation" });
|
|
331
|
+
|
|
332
|
+
// Schedule a function and get its ID
|
|
333
|
+
const scheduledId = await makeDummyScheduledFunction(ctx, workId);
|
|
334
|
+
|
|
335
|
+
// Add to running list
|
|
336
|
+
const state = await ctx.db.query("internalState").unique();
|
|
337
|
+
assert(state);
|
|
338
|
+
await ctx.db.patch(state._id, {
|
|
339
|
+
running: [{ workId, scheduledId, started: Date.now() }],
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
return workId;
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
// Complete the work with failure (workerRunning -> complete)
|
|
346
|
+
await t.mutation(internal.complete.complete, {
|
|
347
|
+
jobs: [
|
|
348
|
+
{
|
|
349
|
+
workId,
|
|
350
|
+
runResult: { kind: "stuckInScheduler" },
|
|
351
|
+
attempt: 0,
|
|
352
|
+
},
|
|
353
|
+
],
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
// Verify pendingCompletion was created with retry=true
|
|
357
|
+
await t.run(async (ctx) => {
|
|
358
|
+
const pendingCompletions = await ctx.db
|
|
359
|
+
.query("pendingCompletion")
|
|
360
|
+
.collect();
|
|
361
|
+
expect(pendingCompletions).toHaveLength(1);
|
|
362
|
+
expect(pendingCompletions[0].workId).toBe(workId);
|
|
363
|
+
expect(pendingCompletions[0].runResult.kind).toBe("stuckInScheduler");
|
|
364
|
+
expect(pendingCompletions[0].retry).toBe(true);
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
// Run main loop to process pendingCompletion -> pendingStart.
|
|
368
|
+
// Since stuckInScheduler retries have 0 backoff, the pendingStart
|
|
369
|
+
// is immediately picked up by handleStart in the same main call.
|
|
370
|
+
await t.mutation(internal.loop.main, {
|
|
371
|
+
generation: 1n,
|
|
372
|
+
segment: getNextSegment(),
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
// Verify the job was re-started immediately
|
|
376
|
+
await t.run(async (ctx) => {
|
|
377
|
+
// Check that pendingCompletion was deleted
|
|
378
|
+
const pendingCompletions = await ctx.db
|
|
379
|
+
.query("pendingCompletion")
|
|
380
|
+
.collect();
|
|
381
|
+
expect(pendingCompletions).toHaveLength(0);
|
|
382
|
+
|
|
383
|
+
// pendingStart was consumed by handleStart in the same main call
|
|
384
|
+
const pendingStarts = await ctx.db.query("pendingStart").collect();
|
|
385
|
+
expect(pendingStarts).toHaveLength(0);
|
|
386
|
+
|
|
387
|
+
// Check that work still exists and attempts was incremented
|
|
388
|
+
const work = await ctx.db.get(workId);
|
|
389
|
+
expect(work).not.toBeNull();
|
|
390
|
+
expect(work!.attempts).toBe(1);
|
|
391
|
+
|
|
392
|
+
// Check that the job is back in the running list
|
|
393
|
+
const state = await ctx.db.query("internalState").unique();
|
|
394
|
+
assert(state);
|
|
395
|
+
expect(state.running).toHaveLength(1);
|
|
396
|
+
expect(state.running[0].workId).toBe(workId);
|
|
397
|
+
});
|
|
398
|
+
});
|
|
317
399
|
});
|
|
318
400
|
|
|
319
401
|
describe("status transitions", () => {
|
|
@@ -1176,7 +1258,10 @@ describe("loop", () => {
|
|
|
1176
1258
|
// Enqueue wave 2 while the loop is still warm
|
|
1177
1259
|
await t.run(async (ctx) => {
|
|
1178
1260
|
const workId2 = await makeDummyWork(ctx);
|
|
1179
|
-
await ctx.db.insert("pendingStart", {
|
|
1261
|
+
await ctx.db.insert("pendingStart", {
|
|
1262
|
+
workId: workId2,
|
|
1263
|
+
segment: segment2,
|
|
1264
|
+
});
|
|
1180
1265
|
});
|
|
1181
1266
|
|
|
1182
1267
|
// The scheduled main from cooldown should pick up wave 2
|
|
@@ -1201,7 +1286,6 @@ describe("loop", () => {
|
|
|
1201
1286
|
const TASKS_PER_WAVE = 3;
|
|
1202
1287
|
const WAVE_GAP_MS = 1000; // 1s between waves, well within 5s cooldown
|
|
1203
1288
|
|
|
1204
|
-
const segment = getNextSegment();
|
|
1205
1289
|
await t.run(async (ctx) => {
|
|
1206
1290
|
await insertInternalState(ctx);
|
|
1207
1291
|
await ctx.db.insert("runStatus", { state: { kind: "running" } });
|
package/src/component/loop.ts
CHANGED
|
@@ -25,11 +25,14 @@ import { generateReport, recordCompleted, recordStarted } from "./stats.js";
|
|
|
25
25
|
|
|
26
26
|
const CANCELLATION_BATCH_SIZE = 64; // the only queue that can get unbounded.
|
|
27
27
|
const RECOVERY_BATCH_SIZE = 32;
|
|
28
|
-
const
|
|
28
|
+
const MS = 1;
|
|
29
|
+
const SECOND = 1000 * MS;
|
|
29
30
|
const MINUTE = 60 * SECOND;
|
|
30
|
-
const
|
|
31
|
+
const ACTION_RECOVERY_THRESHOLD_MS = 5 * MINUTE; // attempt to recover jobs this old.
|
|
32
|
+
const MUTATION_RECOVERY_THRESHOLD_MS = 1 * MINUTE; // attempt to recover jobs this old.
|
|
31
33
|
export const RECOVERY_PERIOD_SEGMENTS = toSegment(1 * MINUTE); // how often to check.
|
|
32
|
-
export const STATUS_COOLDOWN =
|
|
34
|
+
export const STATUS_COOLDOWN = 2 * SECOND;
|
|
35
|
+
export const COOLDOWN_CHECK_INTERVAL = 200 * MS;
|
|
33
36
|
const CURSOR_BUFFER_SEGMENTS = toSegment(30 * SECOND); // buffer for cursor updates.
|
|
34
37
|
export const INITIAL_STATE: WithoutSystemFields<Doc<"internalState">> = {
|
|
35
38
|
generation: 0n,
|
|
@@ -41,6 +44,7 @@ export const INITIAL_STATE: WithoutSystemFields<Doc<"internalState">> = {
|
|
|
41
44
|
failed: 0,
|
|
42
45
|
retries: 0,
|
|
43
46
|
canceled: 0,
|
|
47
|
+
conflicted: 0,
|
|
44
48
|
lastReportTs: 0,
|
|
45
49
|
},
|
|
46
50
|
running: [],
|
|
@@ -109,6 +113,7 @@ export const main = internalMutation({
|
|
|
109
113
|
failed: 0,
|
|
110
114
|
retries: 0,
|
|
111
115
|
canceled: 0,
|
|
116
|
+
conflicted: 0,
|
|
112
117
|
lastReportTs,
|
|
113
118
|
};
|
|
114
119
|
}
|
|
@@ -200,11 +205,16 @@ export const updateRunStatus = internalMutation({
|
|
|
200
205
|
max(incoming, max(completion, cancelation)),
|
|
201
206
|
);
|
|
202
207
|
if (Date.now() - latestCursor < STATUS_COOLDOWN) {
|
|
203
|
-
const
|
|
208
|
+
const remaining = STATUS_COOLDOWN - (Date.now() - latestCursor);
|
|
209
|
+
console.debug(
|
|
210
|
+
`[updateRunStatus] cooldown: ${remaining}ms remaining, checking again in ${COOLDOWN_CHECK_INTERVAL}ms`,
|
|
211
|
+
);
|
|
212
|
+
const checkAt = Date.now() + COOLDOWN_CHECK_INTERVAL;
|
|
213
|
+
const checkSegment = toSegment(checkAt);
|
|
204
214
|
await ctx.scheduler.runAt(
|
|
205
|
-
boundScheduledTime(
|
|
215
|
+
boundScheduledTime(checkAt, console),
|
|
206
216
|
internal.loop.updateRunStatus,
|
|
207
|
-
{ generation, segment:
|
|
217
|
+
{ generation, segment: checkSegment },
|
|
208
218
|
);
|
|
209
219
|
return;
|
|
210
220
|
}
|
|
@@ -393,10 +403,21 @@ async function handleCompletions(
|
|
|
393
403
|
console.warn(`[main] ${c.workId} is gone, but trying to complete`);
|
|
394
404
|
return;
|
|
395
405
|
}
|
|
396
|
-
const
|
|
406
|
+
const wasStuckInScheduler = c.runResult.kind === "stuckInScheduler";
|
|
407
|
+
const retried = await rescheduleJob(
|
|
408
|
+
ctx,
|
|
409
|
+
work,
|
|
410
|
+
console,
|
|
411
|
+
wasStuckInScheduler,
|
|
412
|
+
);
|
|
397
413
|
if (retried) {
|
|
398
|
-
|
|
399
|
-
|
|
414
|
+
if (wasStuckInScheduler) {
|
|
415
|
+
state.report.conflicted = (state.report.conflicted ?? 0) + 1;
|
|
416
|
+
recordCompleted(console, work, "retrying conflicted", undefined);
|
|
417
|
+
} else {
|
|
418
|
+
state.report.retries++;
|
|
419
|
+
recordCompleted(console, work, "retrying", undefined);
|
|
420
|
+
}
|
|
400
421
|
} else {
|
|
401
422
|
// We don't retry if it's been canceled in the mean time.
|
|
402
423
|
state.report.canceled++;
|
|
@@ -490,11 +511,17 @@ async function handleRecovery(
|
|
|
490
511
|
console: Logger,
|
|
491
512
|
) {
|
|
492
513
|
const missing = new Set<Id<"work">>();
|
|
493
|
-
const
|
|
514
|
+
const actionOldEnoughToConsider = Date.now() - ACTION_RECOVERY_THRESHOLD_MS;
|
|
515
|
+
const mutationOldEnoughToConsider =
|
|
516
|
+
Date.now() - MUTATION_RECOVERY_THRESHOLD_MS;
|
|
494
517
|
const jobs = (
|
|
495
518
|
await Promise.all(
|
|
496
519
|
state.running.map(async (r) => {
|
|
497
|
-
if (
|
|
520
|
+
if (
|
|
521
|
+
r.started >=
|
|
522
|
+
Math.max(actionOldEnoughToConsider, mutationOldEnoughToConsider)
|
|
523
|
+
) {
|
|
524
|
+
// Avoid getting the work if possible
|
|
498
525
|
return null;
|
|
499
526
|
}
|
|
500
527
|
const work = await ctx.db.get(r.workId);
|
|
@@ -503,6 +530,13 @@ async function handleRecovery(
|
|
|
503
530
|
console.error(`[main] ${r.workId} already gone (skipping recovery)`);
|
|
504
531
|
return null;
|
|
505
532
|
}
|
|
533
|
+
const oldEnoughToConsider =
|
|
534
|
+
work.fnType === "action"
|
|
535
|
+
? actionOldEnoughToConsider
|
|
536
|
+
: mutationOldEnoughToConsider;
|
|
537
|
+
if (r.started >= oldEnoughToConsider) {
|
|
538
|
+
return null;
|
|
539
|
+
}
|
|
506
540
|
return { ...r, attempt: work.attempts };
|
|
507
541
|
}),
|
|
508
542
|
)
|
|
@@ -612,6 +646,7 @@ async function rescheduleJob(
|
|
|
612
646
|
ctx: MutationCtx,
|
|
613
647
|
work: Doc<"work">,
|
|
614
648
|
console: Logger,
|
|
649
|
+
wasStuckInScheduler: boolean,
|
|
615
650
|
): Promise<boolean> {
|
|
616
651
|
const pendingCancelation = await ctx.db
|
|
617
652
|
.query("pendingCancelation")
|
|
@@ -625,7 +660,14 @@ async function rescheduleJob(
|
|
|
625
660
|
if (work.canceled) {
|
|
626
661
|
return false;
|
|
627
662
|
}
|
|
628
|
-
|
|
663
|
+
let backoffMs: number;
|
|
664
|
+
if (wasStuckInScheduler) {
|
|
665
|
+
backoffMs = 0;
|
|
666
|
+
} else if (work.retryBehavior) {
|
|
667
|
+
backoffMs =
|
|
668
|
+
work.retryBehavior.initialBackoffMs *
|
|
669
|
+
Math.pow(work.retryBehavior.base, work.attempts - 1);
|
|
670
|
+
} else {
|
|
629
671
|
console.warn(`[main] ${work._id} has no retryBehavior so not retrying`);
|
|
630
672
|
return false;
|
|
631
673
|
}
|
|
@@ -638,9 +680,6 @@ async function rescheduleJob(
|
|
|
638
680
|
console.error(`[main] ${work._id} already in pendingStart so not retrying`);
|
|
639
681
|
return false;
|
|
640
682
|
}
|
|
641
|
-
const backoffMs =
|
|
642
|
-
work.retryBehavior.initialBackoffMs *
|
|
643
|
-
Math.pow(work.retryBehavior.base, work.attempts - 1);
|
|
644
683
|
const nextAttempt = withJitter(backoffMs);
|
|
645
684
|
const startTime = boundScheduledTime(Date.now() + nextAttempt, console);
|
|
646
685
|
const segment = toSegment(startTime);
|
|
@@ -354,6 +354,119 @@ describe("recovery", () => {
|
|
|
354
354
|
});
|
|
355
355
|
});
|
|
356
356
|
|
|
357
|
+
it("should handle pending scheduled mutations", async () => {
|
|
358
|
+
// Create work and scheduled function
|
|
359
|
+
let workId: Id<"work">;
|
|
360
|
+
let scheduledId: Id<"_scheduled_functions">;
|
|
361
|
+
|
|
362
|
+
await t.run(async (ctx) => {
|
|
363
|
+
workId = await makeDummyWork(ctx, { fnType: "mutation" });
|
|
364
|
+
scheduledId = await makeDummyScheduledFunction(ctx, workId);
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
// Run recovery with mocked system.get
|
|
368
|
+
await t.run(async (ctx) => {
|
|
369
|
+
// Mock the system.get to return a pending state
|
|
370
|
+
ctx.db.system.get = patchedSystemGet(ctx.db, {
|
|
371
|
+
[scheduledId]: {
|
|
372
|
+
_id: scheduledId,
|
|
373
|
+
_creationTime: Date.now(),
|
|
374
|
+
name: "internal/worker.runMutationWrapper",
|
|
375
|
+
args: [
|
|
376
|
+
{
|
|
377
|
+
workId,
|
|
378
|
+
fnHandle: "test_handle",
|
|
379
|
+
fnArgs: {},
|
|
380
|
+
logLevel: "WARN",
|
|
381
|
+
attempt: 0,
|
|
382
|
+
},
|
|
383
|
+
],
|
|
384
|
+
scheduledTime: Date.now(),
|
|
385
|
+
state: {
|
|
386
|
+
kind: "pending",
|
|
387
|
+
},
|
|
388
|
+
},
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
await recoveryHandler(ctx, {
|
|
392
|
+
jobs: [
|
|
393
|
+
{
|
|
394
|
+
scheduledId,
|
|
395
|
+
workId,
|
|
396
|
+
attempt: 0,
|
|
397
|
+
started: Date.now(),
|
|
398
|
+
},
|
|
399
|
+
],
|
|
400
|
+
});
|
|
401
|
+
});
|
|
402
|
+
|
|
403
|
+
// Verify pendingCompletion was created with stuckInScheduler
|
|
404
|
+
await t.run(async (ctx) => {
|
|
405
|
+
const pendingCompletions = await ctx.db
|
|
406
|
+
.query("pendingCompletion")
|
|
407
|
+
.withIndex("workId", (q) => q.eq("workId", workId))
|
|
408
|
+
.collect();
|
|
409
|
+
expect(pendingCompletions).toHaveLength(1);
|
|
410
|
+
expect(pendingCompletions[0].runResult.kind).toBe("stuckInScheduler");
|
|
411
|
+
});
|
|
412
|
+
});
|
|
413
|
+
|
|
414
|
+
it("should not process pending scheduled actions", async () => {
|
|
415
|
+
// Create work and scheduled function
|
|
416
|
+
let workId: Id<"work">;
|
|
417
|
+
let scheduledId: Id<"_scheduled_functions">;
|
|
418
|
+
|
|
419
|
+
await t.run(async (ctx) => {
|
|
420
|
+
workId = await makeDummyWork(ctx);
|
|
421
|
+
scheduledId = await makeDummyScheduledFunction(ctx, workId);
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
// Run recovery with mocked system.get
|
|
425
|
+
await t.run(async (ctx) => {
|
|
426
|
+
// Mock the system.get to return a pending state
|
|
427
|
+
ctx.db.system.get = patchedSystemGet(ctx.db, {
|
|
428
|
+
[scheduledId]: {
|
|
429
|
+
_id: scheduledId,
|
|
430
|
+
_creationTime: Date.now(),
|
|
431
|
+
name: "internal/worker.runActionWrapper",
|
|
432
|
+
args: [
|
|
433
|
+
{
|
|
434
|
+
workId,
|
|
435
|
+
fnHandle: "test_handle",
|
|
436
|
+
fnArgs: {},
|
|
437
|
+
logLevel: "WARN",
|
|
438
|
+
attempt: 0,
|
|
439
|
+
},
|
|
440
|
+
],
|
|
441
|
+
scheduledTime: Date.now(),
|
|
442
|
+
state: {
|
|
443
|
+
kind: "pending",
|
|
444
|
+
},
|
|
445
|
+
},
|
|
446
|
+
});
|
|
447
|
+
|
|
448
|
+
await recoveryHandler(ctx, {
|
|
449
|
+
jobs: [
|
|
450
|
+
{
|
|
451
|
+
scheduledId,
|
|
452
|
+
workId,
|
|
453
|
+
attempt: 0,
|
|
454
|
+
started: Date.now(),
|
|
455
|
+
},
|
|
456
|
+
],
|
|
457
|
+
});
|
|
458
|
+
});
|
|
459
|
+
|
|
460
|
+
// Verify no pendingCompletion was created
|
|
461
|
+
await t.run(async (ctx) => {
|
|
462
|
+
const pendingCompletions = await ctx.db
|
|
463
|
+
.query("pendingCompletion")
|
|
464
|
+
.withIndex("workId", (q) => q.eq("workId", workId))
|
|
465
|
+
.collect();
|
|
466
|
+
expect(pendingCompletions).toHaveLength(0);
|
|
467
|
+
});
|
|
468
|
+
});
|
|
469
|
+
|
|
357
470
|
it("should handle multiple jobs in a single call", async () => {
|
|
358
471
|
// Create multiple work items and scheduled functions
|
|
359
472
|
let workId1: Id<"work">;
|
|
@@ -472,7 +585,7 @@ describe("recovery", () => {
|
|
|
472
585
|
|
|
473
586
|
// Run recovery with mocked system.get
|
|
474
587
|
await t.run(async (ctx) => {
|
|
475
|
-
// Mock the system.get to return a
|
|
588
|
+
// Mock the system.get to return a inProgress state
|
|
476
589
|
ctx.db.system.get = patchedSystemGet(ctx.db, {
|
|
477
590
|
[scheduledId]: {
|
|
478
591
|
_id: scheduledId,
|
|
@@ -489,7 +602,7 @@ describe("recovery", () => {
|
|
|
489
602
|
],
|
|
490
603
|
scheduledTime: Date.now(),
|
|
491
604
|
state: {
|
|
492
|
-
kind: "
|
|
605
|
+
kind: "inProgress",
|
|
493
606
|
},
|
|
494
607
|
},
|
|
495
608
|
});
|
|
@@ -96,6 +96,23 @@ export async function recoveryHandler(
|
|
|
96
96
|
});
|
|
97
97
|
break;
|
|
98
98
|
}
|
|
99
|
+
case "pending": {
|
|
100
|
+
if (work.fnType === "action") {
|
|
101
|
+
// We do not cancel and re-enqueue actions. If a scheduled action is still
|
|
102
|
+
// pending, the scheduler is likely backlogged.
|
|
103
|
+
break;
|
|
104
|
+
}
|
|
105
|
+
// It looks like the function has been retried by the scheduler several times.
|
|
106
|
+
// The scheduler backoff is too long, so cancel and re-enqueue the job to
|
|
107
|
+
// free up space for more work.
|
|
108
|
+
await ctx.scheduler.cancel(scheduled._id);
|
|
109
|
+
completionJobs.push({
|
|
110
|
+
workId: job.workId,
|
|
111
|
+
runResult: { kind: "stuckInScheduler" },
|
|
112
|
+
attempt: job.attempt,
|
|
113
|
+
});
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
99
116
|
}
|
|
100
117
|
}
|
|
101
118
|
if (completionJobs.length > 0) {
|
package/src/component/schema.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { defineSchema, defineTable } from "convex/server";
|
|
2
|
-
import { v } from "convex/values";
|
|
2
|
+
import { v, type Infer } from "convex/values";
|
|
3
3
|
import {
|
|
4
4
|
fnType,
|
|
5
5
|
vConfig,
|
|
@@ -11,6 +11,14 @@ import {
|
|
|
11
11
|
// Represents a slice of time to process work.
|
|
12
12
|
const segment = v.int64();
|
|
13
13
|
|
|
14
|
+
export const vResultInternal = v.union(
|
|
15
|
+
vResult,
|
|
16
|
+
v.object({
|
|
17
|
+
kind: v.literal("stuckInScheduler"),
|
|
18
|
+
}),
|
|
19
|
+
);
|
|
20
|
+
export type RunResultInternal = Infer<typeof vResultInternal>;
|
|
21
|
+
|
|
14
22
|
export default defineSchema({
|
|
15
23
|
// Written from kickLoop, read everywhere.
|
|
16
24
|
globals: defineTable(vConfig),
|
|
@@ -30,6 +38,7 @@ export default defineSchema({
|
|
|
30
38
|
failed: v.number(), // failed after all retries
|
|
31
39
|
retries: v.number(), // failure that turned into a retry
|
|
32
40
|
canceled: v.number(), // cancelations processed
|
|
41
|
+
conflicted: v.optional(v.number()), // mutations conflicted in the scheduler
|
|
33
42
|
lastReportTs: v.number(),
|
|
34
43
|
}),
|
|
35
44
|
running: v.array(
|
|
@@ -83,7 +92,7 @@ export default defineSchema({
|
|
|
83
92
|
// Written by complete, read & deleted by `main`.
|
|
84
93
|
pendingCompletion: defineTable({
|
|
85
94
|
segment,
|
|
86
|
-
runResult:
|
|
95
|
+
runResult: vResultInternal,
|
|
87
96
|
workId: v.id("work"),
|
|
88
97
|
retry: v.boolean(),
|
|
89
98
|
})
|