@goodandready/dsh-cron 0.3.41 → 0.3.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/scheduler/scheduler.js +11 -0
- package/lib/store/store.js +149 -19
- package/package.json +1 -1
- package/lib/api-actions.js +0 -1
- package/lib/api-item.js +0 -1
- package/lib/artifacts.js +0 -1
- package/lib/routes/index.js +0 -11
- package/lib/runtimes/index.js +0 -4
- package/lib/scheduler/index.js +0 -6
- package/lib/scheduler-assists.js +0 -1
- package/lib/scheduler-completion.js +0 -1
- package/lib/scheduler-lifecycle.js +0 -1
- package/lib/scheduler-preflight.js +0 -1
- package/lib/store/index.js +0 -6
|
@@ -448,6 +448,13 @@ export class TaskScheduler {
|
|
|
448
448
|
this.isStopped = false;
|
|
449
449
|
this.generation = (this.generation || 0) + 1;
|
|
450
450
|
|
|
451
|
+
if (this.store && (this.store.isReadOnly?.() || this.store.hasLoadFailed?.())) {
|
|
452
|
+
const accessCode = this.store.getAccessErrorCode?.();
|
|
453
|
+
const codeInfo = accessCode ? ` (code: ${accessCode})` : '';
|
|
454
|
+
logger.warn(`[dsh-cron] scheduler started in read-only recovery mode${codeInfo}: tasks store could not be loaded. Task scheduling and automatic executions are suspended.`);
|
|
455
|
+
return;
|
|
456
|
+
}
|
|
457
|
+
|
|
451
458
|
// Recovery & cleanup on restart (#317, #321)
|
|
452
459
|
if (this.store && this.store.durableJournal) {
|
|
453
460
|
try {
|
|
@@ -769,6 +776,10 @@ export class TaskScheduler {
|
|
|
769
776
|
|
|
770
777
|
async runTask(taskId, options = {}) {
|
|
771
778
|
if (this.isStopped) return null;
|
|
779
|
+
if (this.store && (this.store.isReadOnly?.() || this.store.hasLoadFailed?.())) {
|
|
780
|
+
logger.warn(`[dsh-cron] skipping task run ${taskId}: store is in read-only recovery mode`);
|
|
781
|
+
return null;
|
|
782
|
+
}
|
|
772
783
|
return executeTask(this, taskId, options);
|
|
773
784
|
}
|
|
774
785
|
|
package/lib/store/store.js
CHANGED
|
@@ -64,6 +64,13 @@ function maskSecretValue(value) {
|
|
|
64
64
|
return s.slice(0, 4) + '••••••••' + s.slice(-3);
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
|
|
68
|
+
export function isAccessOrIoError(err) {
|
|
69
|
+
if (!err) return false;
|
|
70
|
+
const codes = ['EACCES', 'EPERM', 'EIO', 'EBUSY', 'EROFS', 'EMFILE', 'ENFILE'];
|
|
71
|
+
return Boolean(err.code && codes.includes(err.code));
|
|
72
|
+
}
|
|
73
|
+
|
|
67
74
|
export class TaskStore {
|
|
68
75
|
constructor(filePath) {
|
|
69
76
|
this.filePath = filePath || getDefaultStorePath();
|
|
@@ -82,6 +89,10 @@ export class TaskStore {
|
|
|
82
89
|
this._saveVersion = 0;
|
|
83
90
|
this._diskVersion = 0;
|
|
84
91
|
this._lastWrittenVersion = 0;
|
|
92
|
+
this._readOnly = false;
|
|
93
|
+
this._loadFailed = false;
|
|
94
|
+
this._accessErrorCode = null;
|
|
95
|
+
this._loadError = null;
|
|
85
96
|
this.init();
|
|
86
97
|
}
|
|
87
98
|
|
|
@@ -148,11 +159,18 @@ export class TaskStore {
|
|
|
148
159
|
}
|
|
149
160
|
};
|
|
150
161
|
|
|
162
|
+
let loaded = false;
|
|
151
163
|
if (fs.existsSync(this.filePath)) {
|
|
152
164
|
try {
|
|
153
165
|
bestEffort('chmod-store', () => fs.chmodSync(this.filePath, 0o600));
|
|
154
166
|
const raw = fs.readFileSync(this.filePath, 'utf-8');
|
|
155
167
|
loadData(raw);
|
|
168
|
+
loaded = true;
|
|
169
|
+
this._readOnly = false;
|
|
170
|
+
this._loadFailed = false;
|
|
171
|
+
this._accessErrorCode = null;
|
|
172
|
+
this._loadError = null;
|
|
173
|
+
|
|
156
174
|
if (!fs.existsSync(bakPath)) {
|
|
157
175
|
bestEffort('backup-initial', () => {
|
|
158
176
|
fs.copyFileSync(this.filePath, bakPath);
|
|
@@ -162,23 +180,41 @@ export class TaskStore {
|
|
|
162
180
|
bestEffort('chmod-bak', () => fs.chmodSync(bakPath, 0o600));
|
|
163
181
|
}
|
|
164
182
|
} catch (err) {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
logger.
|
|
171
|
-
} catch (snapErr) {
|
|
172
|
-
logger.error('[dsh-cron] failed to snapshot corrupted store:', snapErr.message);
|
|
173
|
-
}
|
|
174
|
-
if (fs.existsSync(bakPath)) {
|
|
183
|
+
if (isAccessOrIoError(err)) {
|
|
184
|
+
// File exists on disk but is inaccessible due to OS permissions/IO error (#435)
|
|
185
|
+
this._enterReadOnlyMode(err, 'read', this.filePath);
|
|
186
|
+
} else {
|
|
187
|
+
// Genuine corruption or invalid data structure
|
|
188
|
+
logger.error(`[dsh-cron] store read error (corrupted file): ${err.message}`);
|
|
175
189
|
try {
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
logger.warn(`[dsh-cron]
|
|
180
|
-
} catch (
|
|
181
|
-
logger.error('[dsh-cron] failed to
|
|
190
|
+
const corruptedPath = `${this.filePath}.corrupted.${Date.now()}`;
|
|
191
|
+
fs.copyFileSync(this.filePath, corruptedPath);
|
|
192
|
+
bestEffort('chmod-corrupted', () => fs.chmodSync(corruptedPath, 0o600));
|
|
193
|
+
logger.warn(`[dsh-cron] corrupted store backed up to ${corruptedPath}`);
|
|
194
|
+
} catch (snapErr) {
|
|
195
|
+
logger.error('[dsh-cron] failed to snapshot corrupted store:', snapErr.message);
|
|
196
|
+
}
|
|
197
|
+
if (fs.existsSync(bakPath)) {
|
|
198
|
+
try {
|
|
199
|
+
bestEffort('chmod-bak', () => fs.chmodSync(bakPath, 0o600));
|
|
200
|
+
const rawBak = fs.readFileSync(bakPath, 'utf-8');
|
|
201
|
+
loadData(rawBak);
|
|
202
|
+
loaded = true;
|
|
203
|
+
this._readOnly = false;
|
|
204
|
+
this._loadFailed = false;
|
|
205
|
+
this._accessErrorCode = null;
|
|
206
|
+
this._loadError = null;
|
|
207
|
+
logger.warn(`[dsh-cron] successfully recovered tasks store from ${bakPath}`);
|
|
208
|
+
} catch (bakErr) {
|
|
209
|
+
if (isAccessOrIoError(bakErr)) {
|
|
210
|
+
this._enterReadOnlyMode(bakErr, 'read', bakPath);
|
|
211
|
+
} else {
|
|
212
|
+
logger.error('[dsh-cron] failed to recover tasks store from .bak:', bakErr.message);
|
|
213
|
+
this._enterReadOnlyMode(bakErr, 'parse', bakPath);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
} else {
|
|
217
|
+
this._enterReadOnlyMode(err, 'parse', this.filePath);
|
|
182
218
|
}
|
|
183
219
|
}
|
|
184
220
|
}
|
|
@@ -187,27 +223,91 @@ export class TaskStore {
|
|
|
187
223
|
bestEffort('chmod-bak', () => fs.chmodSync(bakPath, 0o600));
|
|
188
224
|
const rawBak = fs.readFileSync(bakPath, 'utf-8');
|
|
189
225
|
loadData(rawBak);
|
|
226
|
+
loaded = true;
|
|
227
|
+
this._readOnly = false;
|
|
228
|
+
this._loadFailed = false;
|
|
229
|
+
this._accessErrorCode = null;
|
|
230
|
+
this._loadError = null;
|
|
190
231
|
bestEffort('restore-from-bak', () => {
|
|
191
232
|
fs.copyFileSync(bakPath, this.filePath);
|
|
192
233
|
fs.chmodSync(this.filePath, 0o600);
|
|
193
234
|
});
|
|
194
235
|
logger.warn(`[dsh-cron] restored missing tasks store from ${bakPath}`);
|
|
195
236
|
} catch (bakErr) {
|
|
196
|
-
|
|
237
|
+
if (isAccessOrIoError(bakErr)) {
|
|
238
|
+
this._enterReadOnlyMode(bakErr, 'read', bakPath);
|
|
239
|
+
} else {
|
|
240
|
+
logger.error('[dsh-cron] failed to restore tasks store from .bak:', bakErr.message);
|
|
241
|
+
this._enterReadOnlyMode(bakErr, 'parse', bakPath);
|
|
242
|
+
}
|
|
197
243
|
}
|
|
244
|
+
} else {
|
|
245
|
+
// Clean initial start: neither primary nor backup exists yet (#435)
|
|
246
|
+
this._readOnly = false;
|
|
247
|
+
this._loadFailed = false;
|
|
248
|
+
this._accessErrorCode = null;
|
|
249
|
+
this._loadError = null;
|
|
198
250
|
}
|
|
199
|
-
|
|
251
|
+
|
|
252
|
+
if (this._needsMigrationFlush && !this._readOnly && !this._loadFailed) {
|
|
200
253
|
this._needsMigrationFlush = false;
|
|
201
254
|
this.scheduleSave(100);
|
|
202
255
|
}
|
|
203
|
-
|
|
256
|
+
if (!this._readOnly && !this._loadFailed) {
|
|
257
|
+
bestEffort('prune-old-archives', () => this._pruneOldMonthlyArchives(dir, 6));
|
|
258
|
+
}
|
|
204
259
|
}
|
|
205
260
|
|
|
206
261
|
/**
|
|
207
262
|
* Schedule a debounced save using the asynchronous write queue to avoid
|
|
208
263
|
* blocking the event loop on large task stores (#244).
|
|
209
264
|
*/
|
|
265
|
+
_enterReadOnlyMode(err, operation = 'read', filePath = this.filePath) {
|
|
266
|
+
this._readOnly = true;
|
|
267
|
+
this._loadFailed = true;
|
|
268
|
+
this._loadError = err;
|
|
269
|
+
if (isAccessOrIoError(err)) {
|
|
270
|
+
this._accessErrorCode = err.code;
|
|
271
|
+
logger.error(`[dsh-cron] store access error (operation: ${operation}, code: ${err.code}): cannot access "${filePath}". Check file permissions and ownership.`);
|
|
272
|
+
} else {
|
|
273
|
+
this._accessErrorCode = null;
|
|
274
|
+
logger.error(`[dsh-cron] store read error (operation: ${operation}): cannot load tasks from "${filePath}": ${err?.message || err}. Entering read-only recovery mode to protect existing store.`);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
isReadOnly() {
|
|
279
|
+
return Boolean(this._readOnly);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
hasLoadFailed() {
|
|
283
|
+
return Boolean(this._loadFailed);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
isDegraded() {
|
|
287
|
+
return Boolean(this._readOnly || this._loadFailed);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
getAccessErrorCode() {
|
|
291
|
+
return this._accessErrorCode;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
getLoadError() {
|
|
295
|
+
return this._loadError;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
reload() {
|
|
299
|
+
this._readOnly = false;
|
|
300
|
+
this._loadFailed = false;
|
|
301
|
+
this._accessErrorCode = null;
|
|
302
|
+
this._loadError = null;
|
|
303
|
+
this.init();
|
|
304
|
+
return !this._readOnly && !this._loadFailed;
|
|
305
|
+
}
|
|
306
|
+
|
|
210
307
|
scheduleSave(delay = 50) {
|
|
308
|
+
if (this._readOnly || this._loadFailed) {
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
211
311
|
if (this._saveTimer) return;
|
|
212
312
|
this._saveTimer = setTimeout(() => {
|
|
213
313
|
this._saveTimer = null;
|
|
@@ -228,6 +328,9 @@ export class TaskStore {
|
|
|
228
328
|
clearTimeout(this._saveTimer);
|
|
229
329
|
this._saveTimer = null;
|
|
230
330
|
}
|
|
331
|
+
if (this._readOnly || this._loadFailed) {
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
231
334
|
this.save();
|
|
232
335
|
}
|
|
233
336
|
|
|
@@ -294,6 +397,13 @@ export class TaskStore {
|
|
|
294
397
|
}
|
|
295
398
|
|
|
296
399
|
async saveAsync() {
|
|
400
|
+
if (this._readOnly || this._loadFailed) {
|
|
401
|
+
const reason = this._accessErrorCode
|
|
402
|
+
? `access error (${this._accessErrorCode})`
|
|
403
|
+
: (this._loadError ? this._loadError.message : 'unrecovered read failure');
|
|
404
|
+
logger.error(`[dsh-cron] saveAsync blocked: store is in read-only recovery mode due to ${reason} on "${this.filePath}". File on disk was preserved.`);
|
|
405
|
+
return Promise.reject(new Error(`Store write blocked: read-only recovery mode (${this.filePath})`));
|
|
406
|
+
}
|
|
297
407
|
if (this._saveTimer) {
|
|
298
408
|
clearTimeout(this._saveTimer);
|
|
299
409
|
this._saveTimer = null;
|
|
@@ -473,6 +583,13 @@ export class TaskStore {
|
|
|
473
583
|
}
|
|
474
584
|
|
|
475
585
|
save() {
|
|
586
|
+
if (this._readOnly || this._loadFailed) {
|
|
587
|
+
const reason = this._accessErrorCode
|
|
588
|
+
? `access error (${this._accessErrorCode})`
|
|
589
|
+
: (this._loadError ? this._loadError.message : 'unrecovered read failure');
|
|
590
|
+
logger.error(`[dsh-cron] save blocked: store is in read-only recovery mode due to ${reason} on "${this.filePath}". File on disk was preserved.`);
|
|
591
|
+
throw new Error(`Store write blocked: read-only recovery mode (${this.filePath})`);
|
|
592
|
+
}
|
|
476
593
|
if (this._saveTimer) {
|
|
477
594
|
clearTimeout(this._saveTimer);
|
|
478
595
|
this._saveTimer = null;
|
|
@@ -653,6 +770,10 @@ export class TaskStore {
|
|
|
653
770
|
showChatHeaderButton: patch.showChatHeaderButton !== undefined ? Boolean(patch.showChatHeaderButton) : (this.settings.showChatHeaderButton !== undefined ? Boolean(this.settings.showChatHeaderButton) : true),
|
|
654
771
|
showChatHeaderActions: patch.showChatHeaderActions !== undefined ? Boolean(patch.showChatHeaderActions) : (this.settings.showChatHeaderActions !== undefined ? Boolean(this.settings.showChatHeaderActions) : true),
|
|
655
772
|
};
|
|
773
|
+
if (this._readOnly || this._loadFailed) {
|
|
774
|
+
logger.warn(`[dsh-cron] store is in read-only recovery mode; updated in-memory settings but skipped writing to "${this.filePath}".`);
|
|
775
|
+
return this.getSettings();
|
|
776
|
+
}
|
|
656
777
|
try {
|
|
657
778
|
this.save();
|
|
658
779
|
} catch (err) {
|
|
@@ -683,6 +804,9 @@ export class TaskStore {
|
|
|
683
804
|
}
|
|
684
805
|
|
|
685
806
|
set(task) {
|
|
807
|
+
if (this._readOnly || this._loadFailed) {
|
|
808
|
+
throw new Error(`Store write blocked: read-only recovery mode (${this.filePath})`);
|
|
809
|
+
}
|
|
686
810
|
const now = Date.now();
|
|
687
811
|
const id = task.id || ('cron_' + Math.random().toString(36).slice(2, 9));
|
|
688
812
|
const existing = this.tasks.get(id);
|
|
@@ -766,6 +890,9 @@ export class TaskStore {
|
|
|
766
890
|
recordHeartbeat(id) {
|
|
767
891
|
const task = this.tasks.get(id);
|
|
768
892
|
if (!task) return null;
|
|
893
|
+
if (this._readOnly || this._loadFailed) {
|
|
894
|
+
return null;
|
|
895
|
+
}
|
|
769
896
|
const now = Date.now();
|
|
770
897
|
const prevPing = task.lastPingAt;
|
|
771
898
|
const prevAlerted = task.heartbeatAlerted;
|
|
@@ -797,6 +924,9 @@ export class TaskStore {
|
|
|
797
924
|
}
|
|
798
925
|
|
|
799
926
|
delete(id) {
|
|
927
|
+
if (this._readOnly || this._loadFailed) {
|
|
928
|
+
throw new Error(`Store write blocked: read-only recovery mode (${this.filePath})`);
|
|
929
|
+
}
|
|
800
930
|
const existing = this.tasks.get(id);
|
|
801
931
|
if (!existing) return false;
|
|
802
932
|
const existingHistory = this.history.get(id);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-cron",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.43",
|
|
4
4
|
"description": "Background automation runner for DSH: isolated agent runs, script/HTTP/SSH/Docker runtimes, cost guard, notifications, heartbeats.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
package/lib/api-actions.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from './routes/api-actions.js';
|
package/lib/api-item.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from './routes/api-item.js';
|
package/lib/artifacts.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from './store/artifacts.js';
|
package/lib/routes/index.js
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
export * from './api-collection.js';
|
|
2
|
-
export * from './api-item.js';
|
|
3
|
-
export * from './api-actions.js';
|
|
4
|
-
export * from './api-import-export.js';
|
|
5
|
-
export * from './api-task-webhook.js';
|
|
6
|
-
export * from './api-webhook.js';
|
|
7
|
-
export * from './api-helpers.js';
|
|
8
|
-
export * from './metrics.js';
|
|
9
|
-
export * from './external-api.js';
|
|
10
|
-
export * from './routes.js';
|
|
11
|
-
export * from './api.js';
|
package/lib/runtimes/index.js
DELETED
package/lib/scheduler/index.js
DELETED
package/lib/scheduler-assists.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from './scheduler/scheduler-assists.js';
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from './scheduler/scheduler-completion.js';
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from './scheduler/scheduler-lifecycle.js';
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from './scheduler/scheduler-preflight.js';
|