@whyour/qinglong 2.21.0-8 → 2.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (265) hide show
  1. package/.dockerignore +13 -0
  2. package/.env.example +7 -0
  3. package/AGENTS.md +2 -2
  4. package/README-en.md +24 -0
  5. package/README.md +24 -0
  6. package/back/protos/api.proto +6 -1
  7. package/back/protos/cron.proto +4 -1
  8. package/docker/Dockerfile +31 -12
  9. package/docker/Dockerfile.310 +31 -12
  10. package/docker/Dockerfile.debian +22 -14
  11. package/docker/Dockerfile.debian310 +24 -14
  12. package/docker/build-manifest.cjs +28 -0
  13. package/docker/verify-build.cjs +49 -0
  14. package/docs/releases/2.22.0-validation.md +49 -0
  15. package/docs/releases/2.22.0.md +54 -0
  16. package/ecosystem.config.js +10 -1
  17. package/package.json +11 -6
  18. package/sample/config.sample.sh +10 -0
  19. package/sample/notify.js +54 -0
  20. package/sample/notify.py +37 -0
  21. package/scripts/benchmark-execution.cjs +96 -0
  22. package/scripts/write-build-info.cjs +27 -0
  23. package/shell/api.sh +34 -13
  24. package/shell/check.sh +59 -19
  25. package/shell/lang/en.sh +6 -0
  26. package/shell/lang/zh.sh +6 -0
  27. package/shell/node_path_cache.sh +128 -0
  28. package/shell/otask.sh +7 -1
  29. package/shell/preload/sitecustomize.js +5 -1
  30. package/shell/share.sh +25 -8
  31. package/shell/start.sh +6 -3
  32. package/shell/task.sh +13 -3
  33. package/shell/update.sh +65 -4
  34. package/static/build/api/clientIp.js +42 -0
  35. package/static/build/api/config.js +8 -16
  36. package/static/build/api/cron.js +38 -2
  37. package/static/build/api/dashboard.js +37 -0
  38. package/static/build/api/health.js +2 -2
  39. package/static/build/api/index.js +4 -0
  40. package/static/build/api/log.js +30 -19
  41. package/static/build/api/retention.js +63 -0
  42. package/static/build/api/script.js +23 -22
  43. package/static/build/api/subscription.js +16 -2
  44. package/static/build/api/system.js +25 -2
  45. package/static/build/api/user.js +75 -15
  46. package/static/build/app.js +54 -45
  47. package/static/build/bootstrap/database.js +51 -0
  48. package/static/build/config/const.js +1 -0
  49. package/static/build/config/share.js +5 -4
  50. package/static/build/config/util.js +133 -38
  51. package/static/build/data/cron.js +1 -0
  52. package/static/build/data/notify.js +11 -1
  53. package/static/build/loaders/app.js +2 -0
  54. package/static/build/loaders/db.js +3 -33
  55. package/static/build/loaders/deps.js +29 -0
  56. package/static/build/loaders/express.js +30 -21
  57. package/static/build/loaders/initData.js +10 -7
  58. package/static/build/loaders/sock.js +72 -6
  59. package/static/build/middlewares/monitoring.js +12 -33
  60. package/static/build/middlewares/protectedPathCase.js +17 -0
  61. package/static/build/protos/api.js +56 -2
  62. package/static/build/protos/cron.js +21 -4
  63. package/static/build/protos/health.js +1 -1
  64. package/static/build/schedule/addCron.js +9 -0
  65. package/static/build/schedule/api.js +5 -2
  66. package/static/build/schedule/client.js +38 -6
  67. package/static/build/schedule/health.js +72 -7
  68. package/static/build/services/config.js +5 -42
  69. package/static/build/services/cron.js +338 -199
  70. package/static/build/services/dependence.js +47 -0
  71. package/static/build/services/grpc.js +1 -1
  72. package/static/build/services/health.js +4 -7
  73. package/static/build/services/http.js +8 -2
  74. package/static/build/services/log.js +2 -3
  75. package/static/build/services/metrics.js +46 -14
  76. package/static/build/services/notify.js +26 -0
  77. package/static/build/services/retention.js +123 -0
  78. package/static/build/services/schedule.js +57 -33
  79. package/static/build/services/script.js +4 -5
  80. package/static/build/services/subscription.js +55 -34
  81. package/static/build/services/system.js +37 -9
  82. package/static/build/services/user.js +213 -25
  83. package/static/build/shared/auth.js +18 -5
  84. package/static/build/shared/authMutation.js +17 -0
  85. package/static/build/shared/childProcess.js +82 -0
  86. package/static/build/shared/clientIp.js +27 -0
  87. package/static/build/shared/errors.js +9 -0
  88. package/static/build/shared/fileAccess.js +60 -0
  89. package/static/build/shared/i18n.js +12 -5
  90. package/static/build/shared/logReader.js +85 -0
  91. package/static/build/shared/logStreamManager.js +80 -60
  92. package/static/build/shared/password.js +30 -0
  93. package/static/build/shared/retention.js +58 -0
  94. package/static/build/shared/runCron.js +28 -15
  95. package/static/build/shared/schedulerMutationLock.js +44 -0
  96. package/static/build/shared/schedulerReadiness.js +82 -0
  97. package/static/build/shared/schemaMigrations.js +58 -0
  98. package/static/build/shared/startupProcess.js +48 -0
  99. package/static/build/shared/store.js +7 -0
  100. package/static/build/shared/trustProxy.js +119 -0
  101. package/static/build/validation/schedule.js +3 -3
  102. package/static/dist/{1885.e1ea09f6.async.js → 1885.b8f427bf.async.js} +1 -1
  103. package/static/dist/2031.6af40025.async.js +1 -0
  104. package/static/dist/2693.51c2636b.async.js +1 -0
  105. package/static/dist/{2890.9e373f5b.async.js → 2890.add55e85.async.js} +1 -1
  106. package/static/dist/2901.2ca95295.async.js +1 -0
  107. package/static/dist/445.a65ff63b.async.js +1 -0
  108. package/static/dist/4736.1d2b5cb6.async.js +1 -0
  109. package/static/dist/{4859.6592cebb.async.js → 4859.97d6653c.async.js} +1 -1
  110. package/static/dist/{6013.4e38de36.async.js → 6013.db535339.async.js} +1 -1
  111. package/static/dist/6212.9e6d9913.async.js +1 -0
  112. package/static/dist/{6339.ab305e96.async.js → 6431.4603ad0e.async.js} +1 -1
  113. package/static/dist/{6872.42f939f1.async.js → 6872.a503ef55.async.js} +1 -1
  114. package/static/dist/7231.f7e5b668.async.js +1 -0
  115. package/static/dist/{7355.f9f263d8.async.js → 7355.fca746f5.async.js} +1 -1
  116. package/static/dist/{7508.7dcee91c.async.js → 7508.d90eac63.async.js} +1 -1
  117. package/static/dist/{7984.594296a5.async.js → 7984.eaac7402.async.js} +1 -1
  118. package/static/dist/8844.a0fef08e.async.js +1 -0
  119. package/static/dist/index.html +2 -2
  120. package/static/dist/layouts__index.633d3553.async.js +1 -0
  121. package/static/dist/preload_helper.cf7de3fa.js +1 -0
  122. package/static/dist/{src__pages__config__index.a22ff7dc.async.js → src__pages__config__index.9aba4054.async.js} +1 -1
  123. package/static/dist/src__pages__crontab__detail.86b2eebb.async.js +1 -0
  124. package/static/dist/src__pages__crontab__index.ae4e06f2.async.js +1 -0
  125. package/static/dist/{src__pages__crontab__logModal.831d5e4b.async.js → src__pages__crontab__logModal.930535e2.async.js} +1 -1
  126. package/static/dist/src__pages__crontab__modal.a2c8fdcc.async.js +1 -0
  127. package/static/dist/src__pages__crontab__viewCreateModal.7b4f9cbe.async.js +1 -0
  128. package/static/dist/src__pages__crontab__viewManageModal.cd454ea0.async.js +1 -0
  129. package/static/dist/src__pages__dashboard__index.243c5a35.async.js +1 -0
  130. package/static/dist/src__pages__dashboard__taskResultModal.7795d6e1.async.js +1 -0
  131. package/static/dist/src__pages__dependence__index.874d501f.async.js +1 -0
  132. package/static/dist/src__pages__dependence__logModal.154efe7a.async.js +1 -0
  133. package/static/dist/src__pages__dependence__modal.a29c1a0f.async.js +1 -0
  134. package/static/dist/{src__pages__diff__index.013b7c75.async.js → src__pages__diff__index.1714b941.async.js} +1 -1
  135. package/static/dist/src__pages__env__editNameModal.c13d47d2.async.js +1 -0
  136. package/static/dist/src__pages__env__index.fd6fd2b0.async.js +1 -0
  137. package/static/dist/src__pages__env__modal.4d71adb3.async.js +1 -0
  138. package/static/dist/src__pages__error__index.aae50c94.async.js +1 -0
  139. package/static/dist/src__pages__initialization__index.82a2e967.async.js +1 -0
  140. package/static/dist/src__pages__log__index.71250e0d.async.js +1 -0
  141. package/static/dist/{src__pages__login__index.8c813eb1.async.js → src__pages__login__index.a0fd5ed8.async.js} +1 -1
  142. package/static/dist/src__pages__script__editModal.22acdf7c.async.js +1 -0
  143. package/static/dist/src__pages__script__editNameModal.1a521cc0.async.js +1 -0
  144. package/static/dist/src__pages__script__index.a18b2692.async.js +1 -0
  145. package/static/dist/src__pages__script__renameModal.0ab029e7.async.js +1 -0
  146. package/static/dist/src__pages__script__saveModal.8965b74c.async.js +1 -0
  147. package/static/dist/src__pages__script__setting.35ad4f02.async.js +1 -0
  148. package/static/dist/src__pages__setting__appModal.7eaeb9d3.async.js +1 -0
  149. package/static/dist/{src__pages__setting__checkUpdate.34d773f2.async.js → src__pages__setting__checkUpdate.d175b154.async.js} +1 -1
  150. package/static/dist/src__pages__setting__clientIp.3a8711ed.async.js +1 -0
  151. package/static/dist/src__pages__setting__dependence.9c0f749b.async.js +1 -0
  152. package/static/dist/src__pages__setting__index.cb397741.async.js +1 -0
  153. package/static/dist/src__pages__setting__loginLog.e0a1f54d.async.js +1 -0
  154. package/static/dist/src__pages__setting__notification.2ffe4e5a.async.js +1 -0
  155. package/static/dist/src__pages__setting__other.3d0b8492.async.js +1 -0
  156. package/static/dist/{src__pages__setting__progress.8823ac90.async.js → src__pages__setting__progress.2ccae3d7.async.js} +1 -1
  157. package/static/dist/src__pages__setting__security.e2d71ebb.async.js +1 -0
  158. package/static/dist/src__pages__setting__systemLog.b3046955.async.js +1 -0
  159. package/static/dist/src__pages__subscription__index.43086a06.async.js +1 -0
  160. package/static/dist/src__pages__subscription__logModal.cdba5139.async.js +1 -0
  161. package/static/dist/src__pages__subscription__modal.38991e1e.async.js +1 -0
  162. package/static/dist/{umi.a54a59f6.js → umi.fc019050.js} +1 -1
  163. package/test/back/auth-security.test.cjs +74 -0
  164. package/test/back/build-provenance.test.cjs +150 -0
  165. package/test/back/cron-log-name.test.cjs +110 -0
  166. package/test/back/cron-validation.test.cjs +45 -0
  167. package/test/back/dashboard-failures.test.cjs +173 -0
  168. package/test/back/database-bootstrap.test.cjs +193 -0
  169. package/test/back/dependence-cache-status.test.cjs +125 -0
  170. package/test/back/deprecated-file-routes.test.cjs +82 -0
  171. package/test/back/ecosystem.test.cjs +28 -0
  172. package/test/back/env-name-parsing.test.cjs +96 -0
  173. package/test/back/execution-lifecycle.test.cjs +236 -0
  174. package/test/back/file-access-security.test.cjs +47 -0
  175. package/test/back/fileTreeListing.test.cjs +57 -0
  176. package/test/back/grpc-cron-response.test.cjs +60 -0
  177. package/test/back/http-exclusive-listen.test.cjs +31 -0
  178. package/test/back/http-security.test.cjs +213 -0
  179. package/test/back/listen-security.test.cjs +67 -0
  180. package/test/back/log-path-security.test.cjs +158 -0
  181. package/test/back/logReader.test.cjs +72 -0
  182. package/test/back/manual-execution.test.cjs +85 -0
  183. package/test/back/manual-stop-claim.test.cjs +220 -0
  184. package/test/back/metrics.test.cjs +34 -0
  185. package/test/back/monitoring.test.cjs +73 -0
  186. package/test/back/node-path-cache.test.cjs +173 -0
  187. package/test/back/node-path-lock.test.cjs +178 -0
  188. package/test/back/npm-publication.test.cjs +70 -0
  189. package/test/back/primary-apm.test.cjs +24 -0
  190. package/test/back/process-exit-state.test.cjs +81 -0
  191. package/test/back/process-stop-signal.test.cjs +77 -0
  192. package/test/back/protected-path-case.test.cjs +58 -0
  193. package/test/back/qlServiceLogCommand.test.cjs +128 -0
  194. package/test/back/retention.test.cjs +131 -0
  195. package/test/back/role-loading.test.cjs +118 -0
  196. package/test/back/schedule-health.test.cjs +86 -0
  197. package/test/back/scheduler-file-lock.test.cjs +51 -0
  198. package/test/back/scheduler-mutation.test.cjs +312 -0
  199. package/test/back/scheduler-readiness.test.cjs +85 -0
  200. package/test/back/scheduler-reconciliation.test.cjs +152 -0
  201. package/test/back/scheduler-rpc-timeout.test.cjs +49 -0
  202. package/test/back/schema-migrations.test.cjs +127 -0
  203. package/test/back/shell-api-parsing.test.cjs +184 -0
  204. package/test/back/sock-polling.test.cjs +297 -0
  205. package/test/back/sock-security.test.cjs +72 -0
  206. package/test/back/start-env-preservation.test.cjs +53 -0
  207. package/test/back/stop-failure-feedback.test.cjs +158 -0
  208. package/test/back/stop-race.test.cjs +167 -0
  209. package/test/back/subscription-cleanup.test.cjs +53 -0
  210. package/test/back/subscription-stop-state.test.cjs +66 -0
  211. package/test/back/system-dependence-cache.test.cjs +126 -0
  212. package/test/back/task-time.test.cjs +116 -0
  213. package/test/back/user-api.test.cjs +99 -0
  214. package/test/back/user-security.test.cjs +248 -0
  215. package/test/back/worker-apm.test.cjs +84 -0
  216. package/test/back/wpush-grpc.test.cjs +115 -0
  217. package/test/front/cron-log-pagination.test.cjs +214 -0
  218. package/test/front/http-auth.test.cjs +152 -0
  219. package/test/front/http-error.test.cjs +29 -0
  220. package/test/helpers/load-security-module.cjs +42 -0
  221. package/version.yaml +19 -34
  222. package/CLAUDE.md +0 -43
  223. package/static/dist/1147.62793604.async.js +0 -1
  224. package/static/dist/134.54df9e38.async.js +0 -1
  225. package/static/dist/2208.85285476.async.js +0 -1
  226. package/static/dist/2286.f371c743.async.js +0 -1
  227. package/static/dist/3191.545ab873.async.js +0 -1
  228. package/static/dist/4902.555f92ab.async.js +0 -1
  229. package/static/dist/5520.ff2cfd6b.async.js +0 -1
  230. package/static/dist/6431.c50479be.async.js +0 -1
  231. package/static/dist/8826.db34ad8d.async.js +0 -1
  232. package/static/dist/layouts__index.a7a2bfe0.async.js +0 -1
  233. package/static/dist/preload_helper.3048fe5b.js +0 -1
  234. package/static/dist/src__pages__crontab__detail.b98f0084.async.js +0 -1
  235. package/static/dist/src__pages__crontab__index.68e78a33.async.js +0 -1
  236. package/static/dist/src__pages__crontab__modal.f7041c7c.async.js +0 -1
  237. package/static/dist/src__pages__crontab__viewCreateModal.4d589f66.async.js +0 -1
  238. package/static/dist/src__pages__crontab__viewManageModal.0e317746.async.js +0 -1
  239. package/static/dist/src__pages__dashboard__index.58177357.async.js +0 -1
  240. package/static/dist/src__pages__dependence__index.b7bbc795.async.js +0 -1
  241. package/static/dist/src__pages__dependence__logModal.0681830b.async.js +0 -1
  242. package/static/dist/src__pages__dependence__modal.f7bb47ba.async.js +0 -1
  243. package/static/dist/src__pages__env__editNameModal.48715576.async.js +0 -1
  244. package/static/dist/src__pages__env__index.452cdc47.async.js +0 -1
  245. package/static/dist/src__pages__env__modal.e1234834.async.js +0 -1
  246. package/static/dist/src__pages__error__index.dedb2b90.async.js +0 -1
  247. package/static/dist/src__pages__initialization__index.22620a77.async.js +0 -1
  248. package/static/dist/src__pages__log__index.9c66639f.async.js +0 -1
  249. package/static/dist/src__pages__script__editModal.cbf4ec0e.async.js +0 -1
  250. package/static/dist/src__pages__script__editNameModal.532a3044.async.js +0 -1
  251. package/static/dist/src__pages__script__index.1fc20f22.async.js +0 -1
  252. package/static/dist/src__pages__script__renameModal.5cdde919.async.js +0 -1
  253. package/static/dist/src__pages__script__saveModal.5ffed35f.async.js +0 -1
  254. package/static/dist/src__pages__script__setting.1a82c6d7.async.js +0 -1
  255. package/static/dist/src__pages__setting__appModal.43b7758a.async.js +0 -1
  256. package/static/dist/src__pages__setting__dependence.9c04c79a.async.js +0 -1
  257. package/static/dist/src__pages__setting__index.7e070838.async.js +0 -1
  258. package/static/dist/src__pages__setting__loginLog.cea6056c.async.js +0 -1
  259. package/static/dist/src__pages__setting__notification.bac481c2.async.js +0 -1
  260. package/static/dist/src__pages__setting__other.707cfafe.async.js +0 -1
  261. package/static/dist/src__pages__setting__security.b7235753.async.js +0 -1
  262. package/static/dist/src__pages__setting__systemLog.b81a8c1d.async.js +0 -1
  263. package/static/dist/src__pages__subscription__index.d0ab2abc.async.js +0 -1
  264. package/static/dist/src__pages__subscription__logModal.3864b37f.async.js +0 -1
  265. package/static/dist/src__pages__subscription__modal.562448e2.async.js +0 -1
@@ -0,0 +1,312 @@
1
+ const test = require('node:test');
2
+ const assert = require('node:assert/strict');
3
+ const fs = require('node:fs/promises');
4
+ const os = require('node:os');
5
+ const path = require('node:path');
6
+ const { spawn } = require('node:child_process');
7
+ const { once } = require('node:events');
8
+ const { setTimeout: delay } = require('node:timers/promises');
9
+ const { Sequelize, DataTypes } = require('sequelize');
10
+ const { status } = require('@grpc/grpc-js');
11
+ const load = require('../helpers/load-security-module.cjs');
12
+
13
+ function gate() {
14
+ let release;
15
+ const promise = new Promise((resolve) => {
16
+ release = resolve;
17
+ });
18
+ return { promise, release };
19
+ }
20
+
21
+ async function fixture(t) {
22
+ const root = await fs.mkdtemp(
23
+ path.join(os.tmpdir(), 'ql-scheduler-mutation-'),
24
+ );
25
+ const db = new Sequelize({
26
+ dialect: 'sqlite',
27
+ storage: ':memory:',
28
+ logging: false,
29
+ });
30
+ const crons = db.define('Cron', {
31
+ name: DataTypes.STRING,
32
+ command: DataTypes.STRING,
33
+ schedule: DataTypes.STRING,
34
+ isDisabled: DataTypes.INTEGER,
35
+ saved: DataTypes.BOOLEAN,
36
+ queued_token: DataTypes.STRING,
37
+ });
38
+ await db.sync();
39
+ t.after(async () => {
40
+ await db.close();
41
+ await fs.rm(root, { recursive: true, force: true });
42
+ });
43
+ const jobs = new Map();
44
+ const client = {
45
+ addCron: async (items, replace) => {
46
+ if (replace) jobs.clear();
47
+ items.forEach((item) => jobs.set(item.id, item.command));
48
+ },
49
+ delCron: async (ids) => ids.forEach((id) => jobs.delete(id)),
50
+ };
51
+ const CronService = load('back/services/cron.ts', {
52
+ '../config': {
53
+ crontabFile: path.join(root, 'crontab.list'),
54
+ logPath: root,
55
+ },
56
+ '../data/cron': {
57
+ CrontabModel: crons,
58
+ Crontab: class {
59
+ constructor(options) {
60
+ Object.assign(this, { isDisabled: 0 }, options);
61
+ }
62
+ },
63
+ },
64
+ '../data/runningInstance': {},
65
+ '../config/util': { isDemoEnv: () => false },
66
+ '../config/const': {},
67
+ '../schedule/client': client,
68
+ '../shared/pLimit': {},
69
+ '../shared/utils': {},
70
+ '../shared/i18n': { t: (s) => s },
71
+ '../shared/logReader': {},
72
+ '../shared/logStreamManager': {},
73
+ }).default;
74
+ const service = new CronService({ error() {}, warn() {}, info() {} });
75
+ service.getLogName = async () => 'task';
76
+ service.shouldUseCronClient = () => true;
77
+ service.makeCommand = (doc) => doc.command;
78
+ service.crontabs = async () => ({ data: await crons.findAll({ raw: true }) });
79
+ service.setCrontab = async () => {};
80
+ return { service, crons, jobs, client, root };
81
+ }
82
+
83
+ for (const operation of ['create', 'update', 'remove', 'disabled', 'enabled']) {
84
+ test(`recovery snapshot cannot overwrite concurrent ${operation}`, async (t) => {
85
+ const { service, crons, jobs } = await fixture(t);
86
+ await crons.create({
87
+ id: 1,
88
+ command: 'old',
89
+ schedule: '* * * * *',
90
+ isDisabled: operation === 'enabled' ? 1 : 0,
91
+ });
92
+ const entered = gate(),
93
+ release = gate();
94
+ let first = true;
95
+ service.setCrontab = async () => {
96
+ if (first) {
97
+ first = false;
98
+ entered.release();
99
+ await release.promise;
100
+ }
101
+ };
102
+ const recovery = service.autosave_crontab(true);
103
+ await entered.promise;
104
+ const before = await crons.findAll({ raw: true });
105
+ const mutation =
106
+ operation === 'create'
107
+ ? service.create({ command: 'new', schedule: '* * * * *' })
108
+ : operation === 'update'
109
+ ? service.update({ id: 1, command: 'new' })
110
+ : service[operation]([1]);
111
+ await delay(50);
112
+ assert.deepEqual(
113
+ await crons.findAll({ raw: true }),
114
+ before,
115
+ 'DB writes must also wait for recovery',
116
+ );
117
+ release.release();
118
+ await Promise.all([recovery, mutation]);
119
+ const rows = await crons.findAll({ raw: true });
120
+ assert.deepEqual(
121
+ [...jobs.entries()].sort(),
122
+ rows
123
+ .filter((row) => !row.isDisabled)
124
+ .map((row) => [String(row.id), row.command])
125
+ .sort(),
126
+ );
127
+ });
128
+ }
129
+
130
+ test('recovery waits for an admitted mutation to finish registration before reading the DB', async (t) => {
131
+ const { service, client, jobs } = await fixture(t);
132
+ const entered = gate(),
133
+ release = gate();
134
+ const add = client.addCron;
135
+ client.addCron = async (items, replace) => {
136
+ if (!replace) {
137
+ entered.release();
138
+ await release.promise;
139
+ }
140
+ return add(items, replace);
141
+ };
142
+ const mutation = service.create({ command: 'new', schedule: '* * * * *' });
143
+ await entered.promise;
144
+ let reads = 0;
145
+ const read = service.crontabs;
146
+ service.crontabs = async () => {
147
+ reads++;
148
+ return read();
149
+ };
150
+ const recovery = service.autosave_crontab(true);
151
+ await delay(50);
152
+ assert.equal(reads, 0);
153
+ release.release();
154
+ await Promise.all([mutation, recovery]);
155
+ assert.deepEqual([...jobs.values()], ['new']);
156
+ });
157
+
158
+ for (const operation of ['create', 'update', 'enabled']) {
159
+ test(`${operation} preserves scheduler 503 after rollback and releases its lock`, async (t) => {
160
+ const { service, crons, client } = await fixture(t);
161
+ if (operation !== 'create')
162
+ await crons.create({
163
+ id: 1,
164
+ command: 'old',
165
+ schedule: '* * * * *',
166
+ isDisabled: operation === 'enabled' ? 1 : 0,
167
+ });
168
+ const cause = Object.assign(Error('unavailable'), { status: 503 });
169
+ client.addCron = async () => {
170
+ throw cause;
171
+ };
172
+ const mutation =
173
+ operation === 'create'
174
+ ? service.create({ command: 'new', schedule: '* * * * *' })
175
+ : operation === 'update'
176
+ ? service.update({ id: 1, command: 'new' })
177
+ : service.enabled([1]);
178
+ await assert.rejects(
179
+ mutation,
180
+ (err) =>
181
+ err.status === 503 && err.cause === cause && /回滚/.test(err.message),
182
+ );
183
+ if (operation === 'create') assert.equal(await crons.count(), 0);
184
+ else {
185
+ const row = await crons.findByPk(1);
186
+ assert.equal(row.command, 'old');
187
+ assert.equal(row.isDisabled, operation === 'enabled' ? 1 : 0);
188
+ }
189
+ client.addCron = async () => {};
190
+ await service.autosave_crontab(true);
191
+ });
192
+ }
193
+
194
+ test(
195
+ 'scheduler lock excludes a second OS process and leaves no lock after completion',
196
+ { timeout: 10000 },
197
+ async (t) => {
198
+ const root = await fs.mkdtemp(
199
+ path.join(os.tmpdir(), 'ql-scheduler-process-'),
200
+ );
201
+ t.after(() => fs.rm(root, { recursive: true, force: true }));
202
+ const config = { crontabFile: path.join(root, 'crontab.list') };
203
+ const { withSchedulerMutation } = load(
204
+ 'back/shared/schedulerMutationLock.ts',
205
+ { '../config': config },
206
+ );
207
+ const entered = gate(),
208
+ release = gate();
209
+ const owner = withSchedulerMutation(async () => {
210
+ entered.release();
211
+ await release.promise;
212
+ });
213
+ await entered.promise;
214
+ const child = spawn(
215
+ process.execPath,
216
+ [
217
+ '-e',
218
+ `
219
+ const load = require('./test/helpers/load-security-module.cjs');
220
+ const { withSchedulerMutation } = load('back/shared/schedulerMutationLock.ts', {'../config': JSON.parse(process.argv[1])});
221
+ process.send('attempting');
222
+ withSchedulerMutation(async () => process.send('entered')).then(() => process.disconnect()).catch(err => { console.error(err); process.exit(1); });
223
+ `,
224
+ JSON.stringify(config),
225
+ ],
226
+ { stdio: ['ignore', 'pipe', 'pipe', 'ipc'] },
227
+ );
228
+ t.after(() => {
229
+ if (child.exitCode === null) child.kill('SIGKILL');
230
+ release.release();
231
+ });
232
+ const closed = once(child, 'close');
233
+ assert.equal((await once(child, 'message'))[0], 'attempting');
234
+ let acquired = false;
235
+ const acquisition = once(child, 'message').then(([message]) => {
236
+ acquired = true;
237
+ assert.equal(message, 'entered');
238
+ });
239
+ await delay(150);
240
+ assert.equal(acquired, false);
241
+ release.release();
242
+ await owner;
243
+ await acquisition;
244
+ assert.equal((await closed)[0], 0);
245
+ await assert.rejects(fs.stat(`${config.crontabFile}.scheduler.lock`), {
246
+ code: 'ENOENT',
247
+ });
248
+ },
249
+ );
250
+
251
+ for (const operation of ['update', 'remove', 'disabled']) {
252
+ for (const code of [status.DEADLINE_EXCEEDED, status.UNAVAILABLE]) {
253
+ for (const applied of [false, true]) {
254
+ test(`${operation} aborts on RPC ${code} with deletion applied=${applied} and reconciles the original DB`, async (t) => {
255
+ const { service, crons, client, jobs } = await fixture(t);
256
+ await crons.create({
257
+ id: 1, command: 'old', schedule: '* * * * *', isDisabled: 0,
258
+ });
259
+ jobs.set('1', 'old');
260
+ const before = await crons.findAll({ raw: true });
261
+ const failure = Object.assign(Error('uncertain deletion'), { code, status: 503 });
262
+ let deletes = 0;
263
+ client.delCron = async (ids) => {
264
+ deletes++;
265
+ if (applied) ids.forEach((id) => jobs.delete(id));
266
+ throw failure;
267
+ };
268
+ const add = client.addCron;
269
+ client.addCron = async () => assert.fail('must not register after failed deletion');
270
+ service.setCrontab = async () => assert.fail('must not publish after failed deletion');
271
+ await assert.rejects(
272
+ operation === 'update'
273
+ ? service.update({ id: 1, command: 'new' })
274
+ : service[operation]([1]),
275
+ (error) => error === failure && error.status === 503,
276
+ );
277
+ assert.equal(deletes, 1, 'must not replay an uncertain deletion');
278
+ assert.deepEqual(await crons.findAll({ raw: true }), before);
279
+ client.addCron = add;
280
+ service.setCrontab = async () => {};
281
+ await service.autosave_crontab(true);
282
+ assert.deepEqual([...jobs.entries()], [['1', 'old']]);
283
+ });
284
+ }
285
+ }
286
+ }
287
+
288
+ test('configuration rollback cannot restore an obsolete manual queue token', async (t) => {
289
+ const { service, crons, client } = await fixture(t);
290
+ await crons.create({
291
+ id: 1,
292
+ command: 'old',
293
+ schedule: '* * * * *',
294
+ isDisabled: 0,
295
+ queued_token: 'old-token',
296
+ });
297
+ let first = true;
298
+ client.addCron = async () => {
299
+ if (first) {
300
+ first = false;
301
+ await crons.update({ queued_token: 'new-token' }, { where: { id: 1 } });
302
+ throw Object.assign(Error('unavailable'), { status: 503 });
303
+ }
304
+ };
305
+ await assert.rejects(
306
+ service.update({ id: 1, command: 'new' }),
307
+ (err) => err.status === 503,
308
+ );
309
+ const row = await crons.findByPk(1);
310
+ assert.equal(row.command, 'old');
311
+ assert.equal(row.queued_token, 'new-token');
312
+ });
@@ -0,0 +1,85 @@
1
+ const test=require('node:test');
2
+ const assert=require('node:assert/strict');
3
+ const {setTimeout:sleep}=require('node:timers/promises');
4
+ const {SchedulerReadiness}=require('../../back/shared/schedulerReadiness');
5
+ const load=require('../helpers/load-security-module.cjs');
6
+ const fs=require('node:fs');
7
+ const ts=require('typescript');
8
+
9
+ test('readiness stays false through failed restoration and retries without idle polling',async()=>{
10
+ let probes=0,restores=0,fail=true;
11
+ const state=new SchedulerReadiness(async()=>{probes++;},10);
12
+ state.configure(async()=>{restores++;if(fail)throw Error('registration failed');});
13
+ assert.equal(await state.recover(),false);assert.equal(await state.check(),false);
14
+ fail=false;await sleep(40);assert.equal(await state.check(),true);assert.ok(restores>=2);
15
+ const before=probes;await sleep(40);assert.equal(probes,before);
16
+ });
17
+ test('single-flight recovery cannot mark an invalidated generation ready',async()=>{
18
+ let release,restores=0;
19
+ const state=new SchedulerReadiness(async()=>{},10);
20
+ state.configure(async()=>{if(++restores===1)await new Promise(r=>release=r);});
21
+ const a=state.recover();assert.equal(state.recover(),a);await sleep(1);
22
+ state.invalidate();release();assert.equal(await a,false);assert.equal(await state.check(),false);
23
+ await sleep(30);assert.equal(await state.check(),true);assert.equal(restores,2);
24
+ });
25
+ test('mutation waiting is bounded, a later recovery can succeed',async()=>{
26
+ let release;
27
+ const state=new SchedulerReadiness(async()=>{},10);state.configure(()=>new Promise(r=>release=r));
28
+ await assert.rejects(state.ensureReady(15),e=>e.status===503);
29
+ release();await sleep(1);assert.equal(await state.check(),true);
30
+ });
31
+ test('probe failure immediately invalidates a previously ready scheduler',async()=>{
32
+ let fail=false;
33
+ const state=new SchedulerReadiness(async()=>{if(fail)throw Error('unavailable');},10);
34
+ state.configure(async()=>{});assert.equal(await state.recover(),true);
35
+ fail=true;assert.equal(await state.check(),false);
36
+ fail=false;await sleep(40);assert.equal(await state.check(),true);
37
+ });
38
+ test('health uses actual readiness and returns HTTP 503 until recovery',async(t)=>{
39
+ let ready=false;const express=require('express');
40
+ const {HealthService}=load('back/services/health.ts',{
41
+ typedi:{Service:()=>x=>x},'../loaders/logger':{error(){}},'./http':{},
42
+ '../schedule/client':{readiness:{check:async()=>ready}},
43
+ });
44
+ const service=new HealthService({getServer:()=>({})});
45
+ const app=express();load('back/api/health.ts',{
46
+ typedi:{get:()=>service},'../services/health':{HealthService},'../loaders/logger':{error(){}},
47
+ }).default(app);
48
+ const server=app.listen(0,'127.0.0.1');await new Promise(r=>server.once('listening',r));
49
+ t.after(()=>new Promise(r=>{server.close(r);server.closeAllConnections();}));
50
+ for(const expected of [503,200,503]){
51
+ ready=expected===200;const r=await fetch(`http://127.0.0.1:${server.address().port}/health`);const b=await r.json();
52
+ assert.equal(r.status,expected);assert.equal(b.code,expected);assert.equal(b.data.services.grpc,ready);
53
+ }
54
+ });
55
+ test('recovery registration errors propagate while ordinary autosave retains file synchronization',async()=>{
56
+ const source=fs.readFileSync('back/services/cron.ts','utf8');const a=source.indexOf(' public async autosave_crontab('),z=source.indexOf(' public async bootTask',a);
57
+ const js=ts.transpileModule('class Fixture {\n'+source.slice(a,z)+'}\nmodule.exports=Fixture;', {compilerOptions:{target:ts.ScriptTarget.ES2020}}).outputText;
58
+ const module={exports:{}};let files=0;
59
+ new Function('module','isDemoEnv','cronClient','withSchedulerMutation',js)(module,()=>false,{addCron:async()=>{throw Error('registration unavailable');}},fn=>fn());
60
+ const fixture=new module.exports();fixture.crontabs=async()=>({data:[]});fixture.setCrontab=async()=>{files++;};fixture.logger={warn(){}};
61
+ await fixture.autosave_crontab();assert.equal(files,1);
62
+ await assert.rejects(fixture.autosave_crontab(true),/registration unavailable/);assert.equal(files,2);
63
+ });
64
+ test('scheduler probe uses the cron channel and failed writes are not replayed',async()=>{
65
+ const calls=[];const fake={
66
+ waitForReady(deadline,cb){calls.push(['wait',deadline]);cb();},
67
+ makeUnaryRequest(path,serialize,deserialize,request,options,cb){calls.push(['probe',path,request,options]);cb(null,{status:1});},
68
+ addCron(request,metadata,options,cb){calls.push(['add',request,options]);cb(Object.assign(Error('invalid'),{code:3}));},
69
+ };
70
+ const client=load('back/schedule/client.ts',{
71
+ '../protos/cron':{CronClient:class{constructor(){return fake;}}},
72
+ '../config':{grpcPort:5500},'../config/grpcCerts':{getGrpcCerts:()=>({caCert:'ca',clientKey:'key',clientCert:'cert'})},
73
+ '@grpc/grpc-js':{...require('@grpc/grpc-js'),credentials:{createSsl:()=>({})},status:{UNAVAILABLE:14},Metadata:class{}},
74
+ }).default;
75
+ client.readiness.configure(async()=>{});assert.equal(await client.readiness.recover(),true);
76
+ assert.ok(calls.filter(x=>x[0]==='probe').every(x=>x[1]==='/com.ql.health.Health/Check'&&x[2].service==='scheduler'&&x[3].deadline>Date.now()));
77
+ await assert.rejects(client.addCron([]),/invalid/);assert.equal(calls.filter(x=>x[0]==='add').length,1);
78
+ });
79
+ test('scheduler health probe never calls back into HTTP health',async()=>{
80
+ const {check}=load('back/schedule/health.ts',{
81
+ '../config':{},undici:{request:()=>{throw Error('recursive HTTP health call');}},
82
+ });
83
+ const result=await new Promise(resolve=>check({request:{service:'scheduler'}},(error,response)=>resolve({error,response})));
84
+ assert.equal(result.error,null);assert.deepEqual(result.response,{status:1});
85
+ });
@@ -0,0 +1,152 @@
1
+ const test = require('node:test');
2
+ const assert = require('node:assert/strict');
3
+ const fs = require('node:fs');
4
+ const ts = require('typescript');
5
+ const load = require('../helpers/load-security-module.cjs');
6
+ const { SchedulerReadiness } = require('../../back/shared/schedulerReadiness');
7
+ const { AddCronRequest } = require('../../back/protos/cron');
8
+
9
+ function fixture(client) {
10
+ const source = fs.readFileSync('back/services/cron.ts', 'utf8');
11
+ const a = source.indexOf(' public async autosave_crontab(');
12
+ const z = source.indexOf(' public async bootTask', a);
13
+ const js = ts.transpileModule(
14
+ 'class Fixture {\n' + source.slice(a, z) + '}\nmodule.exports=Fixture;',
15
+ { compilerOptions: { target: ts.ScriptTarget.ES2020 } },
16
+ ).outputText;
17
+ const module = { exports: {} };
18
+ new Function('module', 'isDemoEnv', 'cronClient', 'withSchedulerMutation', js)(
19
+ module,
20
+ () => false,
21
+ client,
22
+ (fn) => fn(),
23
+ );
24
+ const service = new module.exports();
25
+ service.setCrontab = async () => {};
26
+ service.logger = { warn() {} };
27
+ service.shouldUseCronClient = () => true;
28
+ service.makeCommand = () => 'true';
29
+ return service;
30
+ }
31
+
32
+ test('recovery reconciles a surviving scheduler after missed deletes and disables, including an empty DB', async () => {
33
+ const cancelled = [];
34
+ const stacks = new Map(
35
+ ['removed', 'disabled', 'kept'].map((id) => [
36
+ id,
37
+ [{ cancel: () => cancelled.push(id) }],
38
+ ]),
39
+ );
40
+ const { addCron } = load('back/schedule/addCron.ts', {
41
+ './data': { scheduleStacks: stacks },
42
+ 'node-schedule': {
43
+ scheduleJob: (id) => ({ cancel: () => cancelled.push(id) }),
44
+ },
45
+ '../shared/runCron': {},
46
+ '../loaders/logger': { info() {}, warn() {} },
47
+ '../shared/i18n': { tf: (s) => s },
48
+ });
49
+ const client = {
50
+ addCron: async (crons, replace) => {
51
+ // Exercise the real protobuf field, not just a JavaScript-only flag.
52
+ const request = AddCronRequest.decode(
53
+ AddCronRequest.encode({ crons, replace }).finish(),
54
+ );
55
+ await new Promise((resolve, reject) =>
56
+ addCron({ request }, (err) => (err ? reject(err) : resolve())),
57
+ );
58
+ },
59
+ };
60
+ const service = fixture(client);
61
+ let rows = [
62
+ { id: 'kept', schedule: '* * * * *', isDisabled: 0 },
63
+ { id: 'disabled', schedule: '* * * * *', isDisabled: 1 },
64
+ ];
65
+ service.crontabs = async () => ({ data: rows });
66
+ const state = new SchedulerReadiness(async () => {});
67
+ state.configure(() => service.autosave_crontab(true));
68
+ assert.equal(await state.recover(), true);
69
+ assert.equal(await state.check(), true);
70
+ assert.deepEqual([...stacks.keys()], ['kept']);
71
+ assert.deepEqual(cancelled.sort(), ['disabled', 'kept', 'removed']);
72
+ rows = [];
73
+ state.invalidate();
74
+ assert.equal(await state.recover(), true);
75
+ assert.equal(stacks.size, 0);
76
+ assert.equal(await state.check(), true);
77
+ });
78
+
79
+ test('invalid replacement leaves the previous schedule intact', async () => {
80
+ const stacks = new Map([
81
+ [
82
+ 'old',
83
+ [
84
+ {
85
+ cancel: () => {
86
+ throw Error('must not cancel');
87
+ },
88
+ },
89
+ ],
90
+ ],
91
+ ]);
92
+ const { addCron } = load('back/schedule/addCron.ts', {
93
+ './data': { scheduleStacks: stacks },
94
+ '../shared/runCron': {},
95
+ '../loaders/logger': {},
96
+ '../shared/i18n': { tf: (s) => s },
97
+ });
98
+ await assert.rejects(
99
+ new Promise((resolve, reject) =>
100
+ addCron(
101
+ {
102
+ request: {
103
+ replace: true,
104
+ crons: [{ id: 'bad', schedule: '?', extra_schedules: [] }],
105
+ },
106
+ },
107
+ (err) => (err ? reject(err) : resolve()),
108
+ ),
109
+ ),
110
+ );
111
+ assert.deepEqual([...stacks.keys()], ['old']);
112
+ });
113
+
114
+ test('pre-RPC channel failures invalidate readiness and return 503 without executing or replaying writes', async () => {
115
+ for (const method of ['addCron', 'delCron']) {
116
+ let invalidations = 0,
117
+ writes = 0;
118
+ const fake = {
119
+ waitForReady: (_deadline, cb) => cb(Error('channel unavailable')),
120
+ addCron: () => writes++,
121
+ delCron: () => writes++,
122
+ };
123
+ const client = load('back/schedule/client.ts', {
124
+ '../protos/cron': {
125
+ CronClient: class {
126
+ constructor() {
127
+ return fake;
128
+ }
129
+ },
130
+ },
131
+ '../config': { grpcPort: 5500 },
132
+ '../config/grpcCerts': {
133
+ getGrpcCerts: () => ({
134
+ caCert: 'ca',
135
+ clientKey: 'key',
136
+ clientCert: 'cert',
137
+ }),
138
+ },
139
+ '@grpc/grpc-js': {
140
+ ...require('@grpc/grpc-js'),
141
+ credentials: { createSsl: () => ({}) },
142
+ },
143
+ }).default;
144
+ client.readiness.invalidate = () => invalidations++;
145
+ await assert.rejects(
146
+ client[method]([]),
147
+ (err) => err.status === 503 && /channel unavailable/.test(err.message),
148
+ );
149
+ assert.equal(invalidations, 1);
150
+ assert.equal(writes, 0);
151
+ }
152
+ });
@@ -0,0 +1,49 @@
1
+ const test = require('node:test');
2
+ const assert = require('node:assert/strict');
3
+ const grpc = require('@grpc/grpc-js');
4
+ const load = require('../helpers/load-security-module.cjs');
5
+
6
+ for (const method of ['addCron', 'delCron']) {
7
+ test(`${method} invalidates an uncertain timed-out write without replaying it`, async () => {
8
+ let writes = 0;
9
+ let invalidations = 0;
10
+ const timeout = Object.assign(new Error('response deadline exceeded'), {
11
+ code: grpc.status.DEADLINE_EXCEEDED,
12
+ });
13
+ const fake = {
14
+ waitForReady: (_deadline, callback) => callback(),
15
+ [method]: (_request, _metadata, _options, callback) => {
16
+ // The server may have applied the write before the response was lost.
17
+ writes++;
18
+ callback(timeout);
19
+ },
20
+ };
21
+ const client = load('back/schedule/client.ts', {
22
+ '../protos/cron': {
23
+ CronClient: class {
24
+ constructor() {
25
+ return fake;
26
+ }
27
+ },
28
+ },
29
+ '../config': { grpcPort: 5500 },
30
+ '../config/grpcCerts': {
31
+ getGrpcCerts: () => ({
32
+ caCert: 'ca',
33
+ clientKey: 'key',
34
+ clientCert: 'cert',
35
+ }),
36
+ },
37
+ '@grpc/grpc-js': { ...grpc, credentials: { createSsl: () => ({}) } },
38
+ }).default;
39
+ client.readiness.invalidate = () => invalidations++;
40
+ await assert.rejects(client[method]([]), (error) => error === timeout);
41
+ assert.equal(
42
+ invalidations,
43
+ 1,
44
+ 'reconcile from the DB after an uncertain RPC result',
45
+ );
46
+ assert.equal(timeout.status, 503);
47
+ assert.equal(writes, 1, 'do not replay an uncertain mutation');
48
+ });
49
+ }