@whyour/qinglong 2.22.0 → 2.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/back/protos/cron.proto +4 -0
- package/docs/development/node-cron-migration.md +49 -0
- package/docs/development/task-startup.md +78 -0
- package/docs/releases/2.22.0.md +8 -0
- package/docs/releases/2.23.0.md +33 -0
- package/docs/script-history.md +53 -0
- package/package.json +9 -8
- package/scripts/benchmark-script-history.cjs +39 -0
- package/scripts/publish-static.sh +34 -0
- package/scripts/sync-static.sh +20 -0
- package/scripts/wait-for-mirror.sh +214 -0
- package/shell/api.sh +56 -70
- package/shell/node_path_cache.sh +10 -2
- package/shell/otask.sh +7 -9
- package/shell/preload/sitecustomize.js +6 -2
- package/shell/preload/sitecustomize.py +4 -2
- package/shell/share.sh +1 -1
- package/shell/task-timeout.sh +63 -0
- package/shell/task.sh +8 -6
- package/shell/update.sh +40 -95
- package/shell/upgrade.cjs +8 -0
- package/static/build/api/config.js +2 -6
- package/static/build/api/env.js +15 -14
- package/static/build/api/open.js +2 -1
- package/static/build/api/script.js +155 -9
- package/static/build/api/subscription.js +3 -6
- package/static/build/api/system.js +35 -3
- package/static/build/config/grpcCerts.js +89 -40
- package/static/build/config/subscription.js +19 -11
- package/static/build/config/util.js +6 -4
- package/static/build/loaders/express.js +2 -0
- package/static/build/loaders/initData.js +20 -30
- package/static/build/loaders/initTask.js +11 -1
- package/static/build/middlewares/uniqueConstraintError.js +15 -0
- package/static/build/protos/cron.js +63 -2
- package/static/build/schedule/addCron.js +45 -63
- package/static/build/schedule/client.js +19 -1
- package/static/build/schedule/delCron.js +1 -1
- package/static/build/schedule/setConcurrency.js +27 -0
- package/static/build/services/config.js +2 -3
- package/static/build/services/cron.js +36 -10
- package/static/build/services/env.js +1 -1
- package/static/build/services/grpc.js +2 -1
- package/static/build/services/log.js +1 -1
- package/static/build/services/open.js +106 -31
- package/static/build/services/schedule.js +20 -10
- package/static/build/services/script.js +1 -1
- package/static/build/services/scriptHistory.js +30 -0
- package/static/build/services/sshKey.js +39 -14
- package/static/build/services/subscription.js +104 -53
- package/static/build/services/system.js +67 -32
- package/static/build/services/user.js +15 -10
- package/static/build/shared/cronCommand.js +17 -0
- package/static/build/shared/cronSchedule.js +101 -0
- package/static/build/shared/cronScheduler.js +73 -0
- package/static/build/shared/i18n.js +3 -2
- package/static/build/shared/logPath.js +15 -0
- package/static/build/shared/loginThrottle.js +45 -0
- package/static/build/shared/pLimit.js +19 -6
- package/static/build/shared/runCron.js +13 -4
- package/static/build/shared/scriptHistory.js +445 -0
- package/static/build/shared/shellQuote.js +11 -0
- package/static/build/shared/subscriptionMutationLock.js +34 -0
- package/static/build/shared/subscriptionPath.js +41 -0
- package/static/build/shared/subscriptionStorage.js +98 -0
- package/static/build/shared/utils.js +2 -1
- package/static/build/token.js +3 -14
- package/static/build/validation/schedule.js +3 -18
- package/static/dist/292.264757f0.async.js +1 -0
- package/static/dist/3512.95e41255.async.js +1 -0
- package/static/dist/index.html +2 -2
- package/static/dist/preload_helper.01fcd534.js +1 -0
- package/static/dist/src__pages__dependence__modal.0fa8b421.async.js +1 -0
- package/static/dist/src__pages__diff__index.b94e5dc5.async.js +1 -0
- package/static/dist/src__pages__script__editModal.afabf073.async.js +1 -0
- package/static/dist/src__pages__script__editNameModal.789ad87b.async.js +1 -0
- package/static/dist/src__pages__script__historyDrawer.8c58d958.async.js +1 -0
- package/static/dist/src__pages__script__historyDrawer.a449193a.chunk.css +1 -0
- package/static/dist/src__pages__script__index.ab360776.chunk.css +1 -0
- package/static/dist/src__pages__script__index.fc575945.async.js +1 -0
- package/static/dist/src__pages__script__saveModal.8fca1a22.async.js +1 -0
- package/static/dist/src__pages__script__saveWithHistory.99ece830.async.js +1 -0
- package/static/dist/src__pages__subscription__modal.30868ba8.async.js +1 -0
- package/static/dist/{umi.fc019050.js → umi.3a24a9fa.js} +1 -1
- package/test/back/advisory-input-security.test.cjs +416 -0
- package/test/back/code-scanning-security.test.cjs +189 -0
- package/test/back/cron-macros.test.cjs +51 -0
- package/test/back/cron-schedule-compatibility.test.cjs +69 -0
- package/test/back/cron-scheduler-routing.test.cjs +59 -0
- package/test/back/draft-path-security.test.cjs +235 -0
- package/test/back/env-import-security.test.cjs +91 -0
- package/test/back/execution-lifecycle.test.cjs +1 -1
- package/test/back/file-access-scope.test.cjs +180 -0
- package/test/back/grpc-mtls-security.test.cjs +136 -0
- package/test/back/http-security.test.cjs +105 -5
- package/test/back/legacy-cron-recovery.test.cjs +76 -0
- package/test/back/linux-mirror-result.test.cjs +176 -0
- package/test/back/listen-security.test.cjs +1 -0
- package/test/back/mirror-status.test.cjs +257 -0
- package/test/back/node-cron-migration.test.cjs +65 -0
- package/test/back/node-path-cache.test.cjs +29 -0
- package/test/back/node-path-lock.test.cjs +6 -2
- package/test/back/preload-lazy-notify.test.cjs +168 -0
- package/test/back/queued-cron-invalidation.test.cjs +98 -0
- package/test/back/restart-instance-recovery.test.cjs +116 -0
- package/test/back/scheduler-bulk-recovery.test.cjs +40 -0
- package/test/back/scheduler-concurrency.test.cjs +411 -0
- package/test/back/scheduler-reconciliation.test.cjs +58 -4
- package/test/back/script-file-access.test.cjs +210 -0
- package/test/back/script-history-api.test.cjs +551 -0
- package/test/back/script-history.test.cjs +383 -0
- package/test/back/shell-api-parsing.test.cjs +4 -4
- package/test/back/shell-translation-output.test.cjs +16 -0
- package/test/back/shell-upgrade.test.cjs +312 -0
- package/test/back/shellApiToken.test.cjs +151 -0
- package/test/back/ssh-config-home.test.cjs +113 -0
- package/test/back/static-mirror-trigger.test.cjs +441 -0
- package/test/back/static-mirror.test.cjs +179 -0
- package/test/back/stop-race.test.cjs +5 -1
- package/test/back/subscription-mutation-lock.test.cjs +83 -0
- package/test/back/subscription-review-regression.test.cjs +607 -0
- package/test/back/subscription-storage-regression.test.cjs +553 -0
- package/test/back/system-dependence-cache.test.cjs +1 -0
- package/test/back/system-export-history.test.cjs +119 -0
- package/test/back/systemTokenRenewal.test.cjs +497 -0
- package/test/back/task-shell-lifecycle.test.cjs +194 -0
- package/test/back/unique-constraint-error.test.cjs +72 -0
- package/test/back/upgrade-guards.test.cjs +285 -0
- package/test/back/upgrade-lifecycle.test.cjs +288 -0
- package/test/back/upgrade-transaction.test.cjs +133 -0
- package/test/back/user-security.test.cjs +102 -1
- package/test/back/wpush-grpc.test.cjs +1 -0
- package/test/fixtures/legacy-cron.json +1371 -0
- package/test/front/diff-file-routing.test.cjs +126 -0
- package/test/front/script-history-save.test.cjs +121 -0
- package/test/helpers/upgrade-fixture.cjs +30 -0
- package/version.yaml +14 -19
- package/static/dist/2096.6d98fd12.async.js +0 -1
- package/static/dist/4859.97d6653c.async.js +0 -1
- package/static/dist/6707.801b886b.async.js +0 -1
- package/static/dist/preload_helper.cf7de3fa.js +0 -1
- package/static/dist/src__pages__dependence__modal.a29c1a0f.async.js +0 -1
- package/static/dist/src__pages__diff__index.1714b941.async.js +0 -1
- package/static/dist/src__pages__script__editModal.22acdf7c.async.js +0 -1
- package/static/dist/src__pages__script__editNameModal.1a521cc0.async.js +0 -1
- package/static/dist/src__pages__script__index.60b6f827.chunk.css +0 -1
- package/static/dist/src__pages__script__index.a18b2692.async.js +0 -1
- package/static/dist/src__pages__script__saveModal.8965b74c.async.js +0 -1
- package/static/dist/src__pages__subscription__modal.38991e1e.async.js +0 -1
- /package/static/dist/{4859.1b962b29.chunk.css → 292.1b962b29.chunk.css} +0 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
const assert = require('node:assert/strict');
|
|
2
|
+
const test = require('node:test');
|
|
3
|
+
const { Sequelize, DataTypes } = require('sequelize');
|
|
4
|
+
const load = require('../helpers/load-security-module.cjs');
|
|
5
|
+
|
|
6
|
+
test('startup closes interrupted instances without inventing exit codes or rewriting history', async (t) => {
|
|
7
|
+
const db = new Sequelize({
|
|
8
|
+
dialect: 'sqlite',
|
|
9
|
+
storage: ':memory:',
|
|
10
|
+
logging: false,
|
|
11
|
+
});
|
|
12
|
+
t.after(() => db.close());
|
|
13
|
+
const instances = db.define('Instance', {
|
|
14
|
+
status: DataTypes.INTEGER,
|
|
15
|
+
started_at: DataTypes.INTEGER,
|
|
16
|
+
finished_at: DataTypes.INTEGER,
|
|
17
|
+
exit_code: DataTypes.INTEGER,
|
|
18
|
+
});
|
|
19
|
+
await db.sync();
|
|
20
|
+
await instances.bulkCreate([
|
|
21
|
+
{ id: 1, status: 0, started_at: 100 },
|
|
22
|
+
{ id: 2, status: 1, started_at: 100, finished_at: 110, exit_code: 0 },
|
|
23
|
+
{ id: 3, status: 2, started_at: 100, finished_at: 120, exit_code: 143 },
|
|
24
|
+
{ id: 4, status: 3, started_at: 100, finished_at: 130, exit_code: 7 },
|
|
25
|
+
]);
|
|
26
|
+
const systemConfig = { info: { lang: 'en-US' } };
|
|
27
|
+
const service = {
|
|
28
|
+
set_envs: async () => {},
|
|
29
|
+
getAuthInfo: async () => ({}),
|
|
30
|
+
initializeSystemApp: async () => {},
|
|
31
|
+
refreshApps: async () => {},
|
|
32
|
+
};
|
|
33
|
+
const initData = load('back/loaders/initData.ts', {
|
|
34
|
+
typedi: { Container: { get: () => service } },
|
|
35
|
+
'../services/dependence': class {},
|
|
36
|
+
'../services/cron': class {},
|
|
37
|
+
'../services/env': class {},
|
|
38
|
+
'../services/system': class {},
|
|
39
|
+
'../services/user': class {},
|
|
40
|
+
'../services/open': class {},
|
|
41
|
+
'../data/cron': {
|
|
42
|
+
CrontabModel: { update: async () => {}, findAll: async () => [] },
|
|
43
|
+
CrontabStatus: { idle: 1 },
|
|
44
|
+
},
|
|
45
|
+
'../data/dependence': {
|
|
46
|
+
DependenceModel: { findAll: async () => [], update: async () => {} },
|
|
47
|
+
DependenceStatus: { queued: 0 },
|
|
48
|
+
},
|
|
49
|
+
'../data/cronView': {
|
|
50
|
+
CrontabViewModel: { findAll: async () => [{}] },
|
|
51
|
+
CronViewType: { 系统: 0 },
|
|
52
|
+
},
|
|
53
|
+
'../data/env': {},
|
|
54
|
+
'../data/system': {
|
|
55
|
+
SystemModel: { findOrCreate: async () => [systemConfig] },
|
|
56
|
+
AuthDataType: {},
|
|
57
|
+
},
|
|
58
|
+
'../data/open': {
|
|
59
|
+
AppModel: {
|
|
60
|
+
findOne: async () => ({ get: () => ({ scopes: ['dashboard'] }) }),
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
'../data/runningInstance': {
|
|
64
|
+
RunningInstanceModel: instances,
|
|
65
|
+
InstanceStatus: { running: 0, stopped: 2 },
|
|
66
|
+
},
|
|
67
|
+
'../config': {},
|
|
68
|
+
'../config/util': { fileExist: async () => true, isDemoEnv: () => false },
|
|
69
|
+
'../shared/store': { shareStore: { updateAuthInfo: async () => {} } },
|
|
70
|
+
'../shared/i18n': { setLang() {} },
|
|
71
|
+
'./logger': {},
|
|
72
|
+
'../schedule/client': {
|
|
73
|
+
readiness: { configure() {}, recover: async () => {} },
|
|
74
|
+
},
|
|
75
|
+
}).default;
|
|
76
|
+
// Dependency installation is unrelated to startup reconciliation.
|
|
77
|
+
const originalTimeout = global.setTimeout;
|
|
78
|
+
global.setTimeout = () => 0;
|
|
79
|
+
const before = Math.floor(Date.now() / 1000);
|
|
80
|
+
try {
|
|
81
|
+
await initData();
|
|
82
|
+
} finally {
|
|
83
|
+
global.setTimeout = originalTimeout;
|
|
84
|
+
}
|
|
85
|
+
const rows = await instances.findAll({ order: [['id', 'ASC']], raw: true });
|
|
86
|
+
assert.equal(rows[0].status, 2);
|
|
87
|
+
assert.ok(
|
|
88
|
+
rows[0].finished_at >= before &&
|
|
89
|
+
rows[0].finished_at <= Math.floor(Date.now() / 1000),
|
|
90
|
+
);
|
|
91
|
+
assert.equal(rows[0].exit_code, null);
|
|
92
|
+
assert.deepEqual(
|
|
93
|
+
rows
|
|
94
|
+
.slice(1)
|
|
95
|
+
.map(({ status, finished_at, exit_code }) => [
|
|
96
|
+
status,
|
|
97
|
+
finished_at,
|
|
98
|
+
exit_code,
|
|
99
|
+
]),
|
|
100
|
+
[
|
|
101
|
+
[1, 110, 0],
|
|
102
|
+
[2, 120, 143],
|
|
103
|
+
[3, 130, 7],
|
|
104
|
+
],
|
|
105
|
+
);
|
|
106
|
+
global.setTimeout = () => 0;
|
|
107
|
+
try {
|
|
108
|
+
await initData();
|
|
109
|
+
} finally {
|
|
110
|
+
global.setTimeout = originalTimeout;
|
|
111
|
+
}
|
|
112
|
+
assert.deepEqual(
|
|
113
|
+
await instances.findAll({ order: [['id', 'ASC']], raw: true }),
|
|
114
|
+
rows,
|
|
115
|
+
);
|
|
116
|
+
});
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
const test = require('node:test');
|
|
2
|
+
const assert = require('node:assert/strict');
|
|
3
|
+
const grpc = require('@grpc/grpc-js');
|
|
4
|
+
const load = require('../helpers/load-security-module.cjs');
|
|
5
|
+
const { SchedulerReadiness } = require('../../back/shared/schedulerReadiness');
|
|
6
|
+
|
|
7
|
+
for (const [name, rows, delay, succeeds] of [
|
|
8
|
+
['large distinct snapshot', Array.from({length:10000}, (_, i) => ({id:String(i), schedule:`${i%60} ${Math.floor(i/60)%60} ${Math.floor(i/3600)} * * *`})), 10000, true],
|
|
9
|
+
['extra schedules', [{id:'extra', schedule:'* * * * *', extra_schedules:Array.from({length:2000},()=>({schedule:'0 * * * *'}))}], 10000, true],
|
|
10
|
+
['unresponsive small write', [{id:'small', schedule:'* * * * *'}], 30000, false],
|
|
11
|
+
['unresponsive large write remains bounded', Array.from({length:30000},()=>({schedule:'* * * * *'})), 180000, false],
|
|
12
|
+
]) {
|
|
13
|
+
test(`recovery deadline: ${name}`, async (t) => {
|
|
14
|
+
t.mock.timers.enable({apis:['Date','setTimeout'], now:0});
|
|
15
|
+
let calls=0, deadline;
|
|
16
|
+
const fake={waitForReady:(_,cb)=>cb(), addCron:(_req,_meta,options,cb)=>{
|
|
17
|
+
calls++; deadline=options.deadline;
|
|
18
|
+
const budget=deadline-Date.now();
|
|
19
|
+
setTimeout(()=>cb(delay<=budget ? null : Object.assign(Error('deadline'),{code:grpc.status.DEADLINE_EXCEEDED}), {}), Math.min(delay,budget));
|
|
20
|
+
}};
|
|
21
|
+
const client=load('back/schedule/client.ts', {
|
|
22
|
+
'../protos/cron':{CronClient:class{constructor(){return fake}}},
|
|
23
|
+
'../config':{grpcPort:5500},
|
|
24
|
+
'../config/grpcCerts':{getGrpcCerts:()=>({caCert:'ca',clientKey:'key',clientCert:'cert'})},
|
|
25
|
+
'@grpc/grpc-js':{...grpc,credentials:{createSsl:()=>({})}},
|
|
26
|
+
}).default;
|
|
27
|
+
const readiness=new SchedulerReadiness(async()=>{}, 60000);
|
|
28
|
+
t.after(()=>clearTimeout(readiness.retry));
|
|
29
|
+
client.readiness=readiness;
|
|
30
|
+
readiness.configure(()=>client.addCron(rows,true));
|
|
31
|
+
const recovering=readiness.recover();
|
|
32
|
+
await new Promise(setImmediate);
|
|
33
|
+
assert.equal(calls,1);
|
|
34
|
+
assert.ok(deadline<=120000);
|
|
35
|
+
t.mock.timers.tick(Math.min(delay,deadline));
|
|
36
|
+
assert.equal(await recovering,succeeds);
|
|
37
|
+
assert.equal(calls,1,'uncertain writes are not replayed inline');
|
|
38
|
+
assert.equal(await readiness.check(),succeeds);
|
|
39
|
+
});
|
|
40
|
+
}
|
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
const test = require('node:test');
|
|
2
|
+
const assert = require('node:assert/strict');
|
|
3
|
+
const fs = require('node:fs');
|
|
4
|
+
const os = require('node:os');
|
|
5
|
+
const path = require('node:path');
|
|
6
|
+
const grpc = require('@grpc/grpc-js');
|
|
7
|
+
const load = require('../helpers/load-security-module.cjs');
|
|
8
|
+
const logger = { info() {}, debug() {}, warn() {}, error() {} };
|
|
9
|
+
const tick = () => new Promise(setImmediate);
|
|
10
|
+
function createLimit() {
|
|
11
|
+
return load('back/shared/pLimit.ts', {
|
|
12
|
+
'../data/system': {
|
|
13
|
+
AuthDataType: { systemConfig: 'systemConfig' },
|
|
14
|
+
SystemModel: {
|
|
15
|
+
sync: async () => {},
|
|
16
|
+
findOne: async () => ({ info: { cronConcurrency: 1 } }),
|
|
17
|
+
},
|
|
18
|
+
},
|
|
19
|
+
'../loaders/logger': logger,
|
|
20
|
+
'../services/notify': class {},
|
|
21
|
+
'../shared/i18n': { t: (x) => x, tf: (x) => x },
|
|
22
|
+
'../config': {},
|
|
23
|
+
'../config/grpcCerts': { getGrpcCerts: () => null },
|
|
24
|
+
'../protos/api': {},
|
|
25
|
+
}).default;
|
|
26
|
+
}
|
|
27
|
+
function createSystem(client, limit, options = {}) {
|
|
28
|
+
const store = options.store || {
|
|
29
|
+
state: {
|
|
30
|
+
id: 1,
|
|
31
|
+
type: 'systemConfig',
|
|
32
|
+
info: { cronConcurrency: 1, timezone: 'Asia/Shanghai' },
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
const writes = [];
|
|
36
|
+
const Service = load('back/services/system.ts', {
|
|
37
|
+
'../config': options.config || {},
|
|
38
|
+
'../shared/schedulerMutationLock': options.config
|
|
39
|
+
? load('back/shared/schedulerMutationLock.ts', {
|
|
40
|
+
'../config': options.config,
|
|
41
|
+
})
|
|
42
|
+
: { withSchedulerMutation: (operation) => operation() },
|
|
43
|
+
'../config/const': {},
|
|
44
|
+
'../config/util': {},
|
|
45
|
+
'../data/dependence': {},
|
|
46
|
+
'../data/notify': {},
|
|
47
|
+
'../data/system': {
|
|
48
|
+
AuthDataType: { systemConfig: 'systemConfig' },
|
|
49
|
+
SystemModel: {
|
|
50
|
+
findOne: async () => ({
|
|
51
|
+
...store.state,
|
|
52
|
+
get: () => ({ ...store.state }),
|
|
53
|
+
}),
|
|
54
|
+
update: async (values) => {
|
|
55
|
+
writes.push(values);
|
|
56
|
+
store.state = { ...store.state, ...values };
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
'../shared/pLimit': limit,
|
|
61
|
+
'../schedule/client': client,
|
|
62
|
+
'../shared/i18n': { t: (x) => x },
|
|
63
|
+
'./notify': class {},
|
|
64
|
+
'./schedule': class {},
|
|
65
|
+
'./sock': class {},
|
|
66
|
+
}).default;
|
|
67
|
+
return {
|
|
68
|
+
service: new Service(logger, {}, {}),
|
|
69
|
+
writes,
|
|
70
|
+
state: () => store.state,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
async function rpcFixture(t) {
|
|
74
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-concurrency-rpc-'));
|
|
75
|
+
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
|
76
|
+
const certs = await load('back/config/grpcCerts.ts', {
|
|
77
|
+
'./index': { configPath: root },
|
|
78
|
+
'./util': { fileExist: async (p) => fs.existsSync(p) },
|
|
79
|
+
'../loaders/logger': logger,
|
|
80
|
+
}).initGrpcCerts();
|
|
81
|
+
const schedulerLimit = createLimit(),
|
|
82
|
+
apiLimit = createLimit();
|
|
83
|
+
await tick();
|
|
84
|
+
const handler = load('back/schedule/setConcurrency.ts', {
|
|
85
|
+
'../shared/pLimit': schedulerLimit,
|
|
86
|
+
});
|
|
87
|
+
let port;
|
|
88
|
+
const Grpc = load('back/services/grpc.ts', {
|
|
89
|
+
'../config': { bindHostGrpc: '127.0.0.1', grpcPort: 0 },
|
|
90
|
+
'../config/grpcCerts': { initGrpcCerts: async () => certs },
|
|
91
|
+
'../loaders/logger': logger,
|
|
92
|
+
'./metrics': { metricsService: { record() {} } },
|
|
93
|
+
'../schedule/addCron': {},
|
|
94
|
+
'../schedule/delCron': {},
|
|
95
|
+
'../schedule/api': {},
|
|
96
|
+
'../schedule/health': { check: (_call, cb) => cb(null, { status: 1 }) },
|
|
97
|
+
'../schedule/setConcurrency': handler,
|
|
98
|
+
'@grpc/grpc-js': {
|
|
99
|
+
...grpc,
|
|
100
|
+
Server: class extends grpc.Server {
|
|
101
|
+
bindAsync(address, creds, cb) {
|
|
102
|
+
super.bindAsync(address, creds, (error, boundPort) => {
|
|
103
|
+
port = boundPort;
|
|
104
|
+
cb(error, boundPort);
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
},
|
|
108
|
+
},
|
|
109
|
+
}).GrpcServerService;
|
|
110
|
+
const server = new Grpc();
|
|
111
|
+
await server.initialize();
|
|
112
|
+
const client = load('back/schedule/client.ts', {
|
|
113
|
+
'../config': { grpcPort: port },
|
|
114
|
+
'../config/grpcCerts': { getGrpcCerts: () => certs },
|
|
115
|
+
}).default;
|
|
116
|
+
t.after(async () => {
|
|
117
|
+
client._client?.close();
|
|
118
|
+
await server.shutdown();
|
|
119
|
+
});
|
|
120
|
+
return {
|
|
121
|
+
...createSystem(client, apiLimit),
|
|
122
|
+
schedulerLimit,
|
|
123
|
+
apiLimit,
|
|
124
|
+
client,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
function enqueue(t, limit, count) {
|
|
128
|
+
const releases = [],
|
|
129
|
+
runs = [];
|
|
130
|
+
for (let i = 0; i < count; i++) {
|
|
131
|
+
let release;
|
|
132
|
+
const gate = new Promise((resolve) => {
|
|
133
|
+
release = resolve;
|
|
134
|
+
});
|
|
135
|
+
releases.push(release);
|
|
136
|
+
runs.push(limit.runWithCronLimit({ id: String(i) }, () => gate));
|
|
137
|
+
}
|
|
138
|
+
t.after(async () => {
|
|
139
|
+
releases.forEach((release) => release());
|
|
140
|
+
await Promise.all(runs);
|
|
141
|
+
});
|
|
142
|
+
return { releases, runs };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
test('concurrency changes reach the independent scheduler queue over mTLS immediately and preserve running jobs', async (t) => {
|
|
146
|
+
const f = await rpcFixture(t);
|
|
147
|
+
const { releases, runs } = enqueue(t, f.schedulerLimit, 3);
|
|
148
|
+
await tick();
|
|
149
|
+
assert.equal(f.schedulerLimit.cronLimitActiveCount, 1);
|
|
150
|
+
assert.equal(f.schedulerLimit.cronLimitPendingCount, 2);
|
|
151
|
+
assert.equal(
|
|
152
|
+
(await f.service.updateCronConcurrency({ cronConcurrency: 2 })).code,
|
|
153
|
+
200,
|
|
154
|
+
);
|
|
155
|
+
await tick();
|
|
156
|
+
assert.equal(f.schedulerLimit.cronLimitActiveCount, 2);
|
|
157
|
+
assert.equal(f.schedulerLimit.cronLimitPendingCount, 1);
|
|
158
|
+
await f.service.updateCronConcurrency({ cronConcurrency: 1 });
|
|
159
|
+
assert.equal(
|
|
160
|
+
f.schedulerLimit.cronLimitActiveCount,
|
|
161
|
+
2,
|
|
162
|
+
'lowering the limit must not stop active jobs',
|
|
163
|
+
);
|
|
164
|
+
releases[0]();
|
|
165
|
+
await runs[0];
|
|
166
|
+
await tick();
|
|
167
|
+
assert.equal(f.schedulerLimit.cronLimitActiveCount, 1);
|
|
168
|
+
assert.equal(f.schedulerLimit.cronLimitPendingCount, 1);
|
|
169
|
+
releases[1]();
|
|
170
|
+
await runs[1];
|
|
171
|
+
await tick();
|
|
172
|
+
assert.equal(f.schedulerLimit.cronLimitActiveCount, 1);
|
|
173
|
+
assert.equal(f.schedulerLimit.cronLimitPendingCount, 0);
|
|
174
|
+
releases[2]();
|
|
175
|
+
await runs[2];
|
|
176
|
+
assert.equal(f.state().info.cronConcurrency, 1);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
test('zero, null and omitted concurrency restore the default in both API and scheduler queues without a restart', async (t) => {
|
|
180
|
+
const f = await rpcFixture(t);
|
|
181
|
+
for (const value of [0, null, undefined]) {
|
|
182
|
+
await f.service.updateCronConcurrency({ cronConcurrency: 1 });
|
|
183
|
+
const scheduler = enqueue(t, f.schedulerLimit, 4),
|
|
184
|
+
manual = enqueue(t, f.apiLimit, 4);
|
|
185
|
+
await tick();
|
|
186
|
+
assert.equal(f.schedulerLimit.cronLimitActiveCount, 1);
|
|
187
|
+
assert.equal(f.apiLimit.cronLimitActiveCount, 1);
|
|
188
|
+
await f.service.updateCronConcurrency(
|
|
189
|
+
value === undefined ? {} : { cronConcurrency: value },
|
|
190
|
+
);
|
|
191
|
+
await tick();
|
|
192
|
+
assert.equal(
|
|
193
|
+
f.state().info.cronConcurrency,
|
|
194
|
+
0,
|
|
195
|
+
'persist the same default that both queues apply',
|
|
196
|
+
);
|
|
197
|
+
assert.equal(f.schedulerLimit.cronLimitActiveCount, 4);
|
|
198
|
+
assert.equal(f.apiLimit.cronLimitActiveCount, 4);
|
|
199
|
+
[...scheduler.releases, ...manual.releases].forEach((release) => release());
|
|
200
|
+
await Promise.all([...scheduler.runs, ...manual.runs]);
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
test('invalid concurrency is rejected before configuration writes or queue changes', async () => {
|
|
205
|
+
const mutations = [];
|
|
206
|
+
const f = createSystem(
|
|
207
|
+
{ setConcurrency: async (value) => mutations.push(value) },
|
|
208
|
+
{ setCustomLimit: async (value) => mutations.push(value) },
|
|
209
|
+
);
|
|
210
|
+
for (const value of [-1, 1.5, NaN, Infinity, 2147483648])
|
|
211
|
+
await assert.rejects(
|
|
212
|
+
f.service.updateCronConcurrency({ cronConcurrency: value }),
|
|
213
|
+
{ status: 400 },
|
|
214
|
+
);
|
|
215
|
+
assert.deepEqual(f.writes, []);
|
|
216
|
+
assert.deepEqual(mutations, []);
|
|
217
|
+
const handler = load('back/schedule/setConcurrency.ts', {
|
|
218
|
+
'../shared/pLimit': {
|
|
219
|
+
setCustomLimit: async (value) => mutations.push(value),
|
|
220
|
+
},
|
|
221
|
+
}).setConcurrency;
|
|
222
|
+
for (const value of [-1, 0.5, 2147483648])
|
|
223
|
+
await handler({ request: { concurrency: value } }, (error) =>
|
|
224
|
+
assert.equal(error.code, grpc.status.INVALID_ARGUMENT),
|
|
225
|
+
);
|
|
226
|
+
assert.deepEqual(mutations, []);
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
for (const code of [grpc.status.UNAVAILABLE, grpc.status.DEADLINE_EXCEEDED]) {
|
|
230
|
+
test(`concurrency RPC failure ${code} invalidates readiness and is not reported as success or retried`, async () => {
|
|
231
|
+
let writes = 0,
|
|
232
|
+
invalidations = 0;
|
|
233
|
+
const fake = {
|
|
234
|
+
waitForReady: (_deadline, callback) => callback(),
|
|
235
|
+
setConcurrency: (_request, _metadata, options, callback) => {
|
|
236
|
+
assert.ok(options.deadline > Date.now());
|
|
237
|
+
writes++;
|
|
238
|
+
callback(Object.assign(new Error('scheduler unavailable'), { code }));
|
|
239
|
+
},
|
|
240
|
+
};
|
|
241
|
+
const client = load('back/schedule/client.ts', {
|
|
242
|
+
'../protos/cron': {
|
|
243
|
+
CronClient: class {
|
|
244
|
+
constructor() {
|
|
245
|
+
return fake;
|
|
246
|
+
}
|
|
247
|
+
},
|
|
248
|
+
},
|
|
249
|
+
'../config': { grpcPort: 1 },
|
|
250
|
+
'../config/grpcCerts': {
|
|
251
|
+
getGrpcCerts: () => ({
|
|
252
|
+
caCert: 'ca',
|
|
253
|
+
clientKey: 'key',
|
|
254
|
+
clientCert: 'cert',
|
|
255
|
+
}),
|
|
256
|
+
},
|
|
257
|
+
'@grpc/grpc-js': { ...grpc, credentials: { createSsl: () => ({}) } },
|
|
258
|
+
}).default;
|
|
259
|
+
client.readiness.invalidate = () => invalidations++;
|
|
260
|
+
const f = createSystem(client, { setCustomLimit: async () => {} });
|
|
261
|
+
await assert.rejects(
|
|
262
|
+
f.service.updateCronConcurrency({ cronConcurrency: 2 }),
|
|
263
|
+
{ status: 503 },
|
|
264
|
+
);
|
|
265
|
+
assert.equal(writes, 1);
|
|
266
|
+
assert.equal(invalidations, 1);
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function mutationConfig(t) {
|
|
271
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-concurrency-lock-'));
|
|
272
|
+
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
|
273
|
+
return { crontabFile: path.join(root, 'crontab.list') };
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
test('independent API service instances serialize database and scheduler updates with the shared filesystem lock', async (t) => {
|
|
277
|
+
const config = mutationConfig(t);
|
|
278
|
+
const store = { state: { id: 1, info: { cronConcurrency: 1 } } };
|
|
279
|
+
let release, entered;
|
|
280
|
+
const firstEntered = new Promise((resolve) => (entered = resolve));
|
|
281
|
+
const gate = new Promise((resolve) => (release = resolve));
|
|
282
|
+
t.after(() => release());
|
|
283
|
+
let scheduler = 1,
|
|
284
|
+
api = 1;
|
|
285
|
+
const client = {
|
|
286
|
+
setConcurrency: async (value) => {
|
|
287
|
+
if (value === 10) {
|
|
288
|
+
entered();
|
|
289
|
+
await gate;
|
|
290
|
+
}
|
|
291
|
+
scheduler = value;
|
|
292
|
+
},
|
|
293
|
+
};
|
|
294
|
+
const limit = { setCustomLimit: async (value) => (api = value) };
|
|
295
|
+
const a = createSystem(client, limit, { config, store });
|
|
296
|
+
const b = createSystem(client, limit, { config, store });
|
|
297
|
+
const first = a.service.updateCronConcurrency({ cronConcurrency: 10 });
|
|
298
|
+
await firstEntered;
|
|
299
|
+
const second = b.service.updateCronConcurrency({ cronConcurrency: 1 });
|
|
300
|
+
await new Promise((resolve) => setTimeout(resolve, 150));
|
|
301
|
+
assert.equal(
|
|
302
|
+
b.writes.length,
|
|
303
|
+
0,
|
|
304
|
+
'a second writer cannot persist ahead of the first RPC',
|
|
305
|
+
);
|
|
306
|
+
release();
|
|
307
|
+
await Promise.all([first, second]);
|
|
308
|
+
assert.deepEqual(
|
|
309
|
+
[store.state.info.cronConcurrency, api, scheduler],
|
|
310
|
+
[1, 1, 1],
|
|
311
|
+
);
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
for (const newerLimit of [undefined, 2]) {
|
|
315
|
+
test(`production recovery restores persisted concurrency before health becomes ready (newer limit: ${newerLimit})`, async (t) => {
|
|
316
|
+
const config = mutationConfig(t);
|
|
317
|
+
const store = { state: { id: 1, info: { cronConcurrency: 1 } } };
|
|
318
|
+
let scheduler = 1,
|
|
319
|
+
api = 1,
|
|
320
|
+
fail = false,
|
|
321
|
+
restoreJobs = 0;
|
|
322
|
+
let unblockRecovery, recoveryEntered;
|
|
323
|
+
const entered = new Promise((resolve) => (recoveryEntered = resolve));
|
|
324
|
+
const gate = new Promise((resolve) => (unblockRecovery = resolve));
|
|
325
|
+
t.after(() => unblockRecovery());
|
|
326
|
+
const fake = {
|
|
327
|
+
waitForReady: (_deadline, callback) => callback(),
|
|
328
|
+
makeUnaryRequest: (...args) => args.at(-1)(null, { status: 1 }),
|
|
329
|
+
setConcurrency: ({ concurrency }, _metadata, _options, callback) => {
|
|
330
|
+
if (fail) {
|
|
331
|
+
fail = false;
|
|
332
|
+
callback(
|
|
333
|
+
Object.assign(new Error('lost RPC'), {
|
|
334
|
+
code: grpc.status.UNAVAILABLE,
|
|
335
|
+
}),
|
|
336
|
+
);
|
|
337
|
+
} else {
|
|
338
|
+
scheduler = concurrency;
|
|
339
|
+
callback(null, {});
|
|
340
|
+
}
|
|
341
|
+
},
|
|
342
|
+
};
|
|
343
|
+
const client = load('back/schedule/client.ts', {
|
|
344
|
+
'../protos/cron': {
|
|
345
|
+
CronClient: class {
|
|
346
|
+
constructor() {
|
|
347
|
+
return fake;
|
|
348
|
+
}
|
|
349
|
+
},
|
|
350
|
+
},
|
|
351
|
+
'../config': { grpcPort: 1 },
|
|
352
|
+
'../config/grpcCerts': {
|
|
353
|
+
getGrpcCerts: () => ({
|
|
354
|
+
caCert: 'ca',
|
|
355
|
+
clientKey: 'key',
|
|
356
|
+
clientCert: 'cert',
|
|
357
|
+
}),
|
|
358
|
+
},
|
|
359
|
+
'@grpc/grpc-js': { ...grpc, credentials: { createSsl: () => ({}) } },
|
|
360
|
+
}).default;
|
|
361
|
+
const f = createSystem(
|
|
362
|
+
client,
|
|
363
|
+
{ setCustomLimit: async (value) => (api = value) },
|
|
364
|
+
{ config, store },
|
|
365
|
+
);
|
|
366
|
+
const source = fs.readFileSync('back/loaders/initData.ts', 'utf8');
|
|
367
|
+
const callback = source.match(
|
|
368
|
+
/cronClient\.readiness\.configure\((async \(\) => \{[\s\S]*?\n \})\);/,
|
|
369
|
+
)[1];
|
|
370
|
+
const restore = new Function(
|
|
371
|
+
'cronService',
|
|
372
|
+
'systemService',
|
|
373
|
+
`return ${callback}`,
|
|
374
|
+
)(
|
|
375
|
+
{
|
|
376
|
+
autosave_crontab: async (strict) => {
|
|
377
|
+
assert.equal(strict, true);
|
|
378
|
+
restoreJobs++;
|
|
379
|
+
if (restoreJobs > 1) {
|
|
380
|
+
recoveryEntered();
|
|
381
|
+
await gate;
|
|
382
|
+
}
|
|
383
|
+
},
|
|
384
|
+
},
|
|
385
|
+
f.service,
|
|
386
|
+
);
|
|
387
|
+
client.readiness.configure(restore);
|
|
388
|
+
assert.equal(await client.readiness.recover(), true);
|
|
389
|
+
fail = true;
|
|
390
|
+
await assert.rejects(
|
|
391
|
+
f.service.updateCronConcurrency({ cronConcurrency: 10 }),
|
|
392
|
+
{ status: 503 },
|
|
393
|
+
);
|
|
394
|
+
await entered;
|
|
395
|
+
assert.equal(await client.readiness.check(), false);
|
|
396
|
+
assert.equal(scheduler, 1);
|
|
397
|
+
// A newer successful request must win over the failed request being recovered.
|
|
398
|
+
if (newerLimit !== undefined)
|
|
399
|
+
await f.service.updateCronConcurrency({ cronConcurrency: newerLimit });
|
|
400
|
+
scheduler = 1; // Simulate the scheduler restarting while recovery is in progress.
|
|
401
|
+
const recovered = client.readiness.recover();
|
|
402
|
+
unblockRecovery();
|
|
403
|
+
assert.equal(await recovered, true);
|
|
404
|
+
const expected = newerLimit ?? 10;
|
|
405
|
+
assert.deepEqual(
|
|
406
|
+
[store.state.info.cronConcurrency, api, scheduler],
|
|
407
|
+
[expected, expected, expected],
|
|
408
|
+
);
|
|
409
|
+
assert.equal(await client.readiness.check(), true);
|
|
410
|
+
});
|
|
411
|
+
}
|
|
@@ -5,6 +5,7 @@ const ts = require('typescript');
|
|
|
5
5
|
const load = require('../helpers/load-security-module.cjs');
|
|
6
6
|
const { SchedulerReadiness } = require('../../back/shared/schedulerReadiness');
|
|
7
7
|
const { AddCronRequest } = require('../../back/protos/cron');
|
|
8
|
+
const { getInvalidCronSchedules } = require('../../back/shared/cronSchedule');
|
|
8
9
|
|
|
9
10
|
function fixture(client) {
|
|
10
11
|
const source = fs.readFileSync('back/services/cron.ts', 'utf8');
|
|
@@ -15,11 +16,12 @@ function fixture(client) {
|
|
|
15
16
|
{ compilerOptions: { target: ts.ScriptTarget.ES2020 } },
|
|
16
17
|
).outputText;
|
|
17
18
|
const module = { exports: {} };
|
|
18
|
-
new Function('module', 'isDemoEnv', 'cronClient', 'withSchedulerMutation', js)(
|
|
19
|
+
new Function('module', 'isDemoEnv', 'cronClient', 'withSchedulerMutation', 'getInvalidCronSchedules', js)(
|
|
19
20
|
module,
|
|
20
21
|
() => false,
|
|
21
22
|
client,
|
|
22
23
|
(fn) => fn(),
|
|
24
|
+
getInvalidCronSchedules,
|
|
23
25
|
);
|
|
24
26
|
const service = new module.exports();
|
|
25
27
|
service.setCrontab = async () => {};
|
|
@@ -39,8 +41,10 @@ test('recovery reconciles a surviving scheduler after missed deletes and disable
|
|
|
39
41
|
);
|
|
40
42
|
const { addCron } = load('back/schedule/addCron.ts', {
|
|
41
43
|
'./data': { scheduleStacks: stacks },
|
|
42
|
-
'
|
|
43
|
-
|
|
44
|
+
'../shared/cronScheduler': {
|
|
45
|
+
createCronJob: (_schedule, _callback, options) => ({
|
|
46
|
+
start() {}, cancel: () => cancelled.push(options.name.split(':')[0]),
|
|
47
|
+
}),
|
|
44
48
|
},
|
|
45
49
|
'../shared/runCron': {},
|
|
46
50
|
'../loaders/logger': { info() {}, warn() {} },
|
|
@@ -101,7 +105,7 @@ test('invalid replacement leaves the previous schedule intact', async () => {
|
|
|
101
105
|
{
|
|
102
106
|
request: {
|
|
103
107
|
replace: true,
|
|
104
|
-
crons: [{ id: 'bad', schedule: '
|
|
108
|
+
crons: [{ id: 'bad', schedule: 'not a cron', extra_schedules: [] }],
|
|
105
109
|
},
|
|
106
110
|
},
|
|
107
111
|
(err) => (err ? reject(err) : resolve()),
|
|
@@ -111,6 +115,56 @@ test('invalid replacement leaves the previous schedule intact', async () => {
|
|
|
111
115
|
assert.deepEqual([...stacks.keys()], ['old']);
|
|
112
116
|
});
|
|
113
117
|
|
|
118
|
+
test('invalid persisted schedules cannot block recovery, valid jobs or HTTP health', async (t) => {
|
|
119
|
+
const stacks = new Map();
|
|
120
|
+
const warnings = [];
|
|
121
|
+
const { addCron } = load('back/schedule/addCron.ts', {
|
|
122
|
+
'./data': { scheduleStacks: stacks },
|
|
123
|
+
'../shared/runCron': {},
|
|
124
|
+
'../loaders/logger': { info() {}, warn() {} },
|
|
125
|
+
'../shared/i18n': { tf: require('node:util').format },
|
|
126
|
+
});
|
|
127
|
+
const client = { addCron: (crons, replace) => new Promise((resolve, reject) => {
|
|
128
|
+
addCron({ request: { crons, replace } }, (error) => error ? reject(error) : resolve());
|
|
129
|
+
}) };
|
|
130
|
+
const service = fixture(client);
|
|
131
|
+
let snapshot;
|
|
132
|
+
service.setCrontab = async (tabs) => { snapshot = tabs.data; };
|
|
133
|
+
service.logger.warn = (...args) => warnings.push(require('node:util').format(...args));
|
|
134
|
+
let rows = [
|
|
135
|
+
{ id: 'good', name: 'valid', schedule: '0 0/30 * * * ?', isDisabled: 0 },
|
|
136
|
+
{ id: 'bad-main', name: 'bad main', schedule: 'not a cron', isDisabled: 0 },
|
|
137
|
+
{ id: 'bad-extra', name: 'bad extra', schedule: '* * * * *', extra_schedules: [{ schedule: '0 70 * * * ?' }], isDisabled: 0 },
|
|
138
|
+
{ id: 'unsupported', name: 'unsupported', schedule: 'H * * * *', isDisabled: 0 },
|
|
139
|
+
{ id: 'disabled', name: 'disabled', schedule: 'bad', isDisabled: 1 },
|
|
140
|
+
];
|
|
141
|
+
service.crontabs = async () => ({ data: rows });
|
|
142
|
+
const state = new SchedulerReadiness(async () => {}, 60000);
|
|
143
|
+
t.after(() => {
|
|
144
|
+
clearTimeout(state.retry);
|
|
145
|
+
for (const jobs of stacks.values()) for (const job of jobs) job.cancel();
|
|
146
|
+
});
|
|
147
|
+
state.configure(() => service.autosave_crontab(true));
|
|
148
|
+
assert.equal(await state.recover(), true);
|
|
149
|
+
assert.deepEqual([...stacks.keys()], ['good']);
|
|
150
|
+
assert.deepEqual(snapshot.map((x) => x.id), ['good', 'disabled']);
|
|
151
|
+
assert.equal(rows.length, 5, 'persisted rows are preserved for editing');
|
|
152
|
+
assert.equal(rows[1].isDisabled, 0);
|
|
153
|
+
assert.equal(warnings.length, 3);
|
|
154
|
+
assert.match(warnings.join('\n'), /bad-main.*bad main.*not a cron/);
|
|
155
|
+
assert.match(warnings.join('\n'), /bad-extra.*0 70/);
|
|
156
|
+
const { HealthService } = load('back/services/health.ts', {
|
|
157
|
+
typedi: { Service: () => (x) => x }, './http': {},
|
|
158
|
+
'../schedule/client': { readiness: state }, '../loaders/logger': { error() {} },
|
|
159
|
+
});
|
|
160
|
+
const health = new HealthService({ getServer: () => ({}) });
|
|
161
|
+
assert.equal((await health.check()).status, 'ok');
|
|
162
|
+
rows = rows.filter((x) => x.id !== 'good');
|
|
163
|
+
assert.equal(await state.recover(), true, 'even all-invalid snapshots can recover');
|
|
164
|
+
assert.equal(stacks.size, 0);
|
|
165
|
+
assert.equal((await health.check()).status, 'ok');
|
|
166
|
+
});
|
|
167
|
+
|
|
114
168
|
test('pre-RPC channel failures invalidate readiness and return 503 without executing or replaying writes', async () => {
|
|
115
169
|
for (const method of ['addCron', 'delCron']) {
|
|
116
170
|
let invalidations = 0,
|