bunsane 0.2.10 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +266 -0
- package/config/cache.config.ts +12 -2
- package/core/App.ts +296 -69
- package/core/ApplicationLifecycle.ts +68 -4
- package/core/Entity.ts +407 -256
- package/core/EntityHookManager.ts +88 -21
- package/core/EntityManager.ts +12 -3
- package/core/Logger.ts +4 -0
- package/core/RequestContext.ts +4 -1
- package/core/SchedulerManager.ts +92 -9
- package/core/cache/CacheFactory.ts +3 -1
- package/core/cache/CacheManager.ts +54 -17
- package/core/cache/RedisCache.ts +38 -3
- package/core/decorators/EntityHooks.ts +24 -12
- package/core/middleware/RateLimit.ts +105 -0
- package/core/middleware/index.ts +1 -0
- package/core/remote/OutboxWorker.ts +42 -35
- package/core/scheduler/DistributedLock.ts +22 -7
- package/gql/builders/ResolverBuilder.ts +4 -4
- package/gql/complexityLimit.ts +95 -0
- package/gql/index.ts +15 -3
- package/gql/visitors/ResolverGeneratorVisitor.ts +16 -2
- package/package.json +1 -1
- package/query/ComponentInclusionNode.ts +13 -6
- package/query/OrNode.ts +2 -4
- package/query/Query.ts +30 -3
- package/query/SqlIdentifier.ts +105 -0
- package/query/builders/FullTextSearchBuilder.ts +19 -6
- package/service/ServiceRegistry.ts +21 -8
- package/storage/LocalStorageProvider.ts +12 -3
- package/storage/S3StorageProvider.ts +6 -6
- package/tests/e2e/http.test.ts +6 -2
- package/tests/integration/entity/Entity.saveTimeout.test.ts +110 -0
- package/tests/unit/storage/S3StorageProvider.test.ts +6 -10
- package/upload/FileValidator.ts +9 -6
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
type LifecycleEvent
|
|
16
16
|
} from "./events/EntityLifecycleEvents";
|
|
17
17
|
import { logger as MainLogger } from "./Logger";
|
|
18
|
-
import ApplicationLifecycle, { ApplicationPhase } from "./ApplicationLifecycle";
|
|
18
|
+
import ApplicationLifecycle, { ApplicationPhase, type PhaseChangeEvent } from "./ApplicationLifecycle";
|
|
19
19
|
|
|
20
20
|
const logger = MainLogger.child({ scope: "EntityHookManager" });
|
|
21
21
|
|
|
@@ -99,6 +99,7 @@ class EntityHookManager {
|
|
|
99
99
|
private hooks: Map<string, RegisteredHook[]> = new Map();
|
|
100
100
|
private hookCounter: number = 0;
|
|
101
101
|
private metrics: Map<string, HookMetrics> = new Map();
|
|
102
|
+
private phaseListener: ((event: PhaseChangeEvent) => void) | null = null;
|
|
102
103
|
private globalMetrics: HookMetrics = {
|
|
103
104
|
totalExecutions: 0,
|
|
104
105
|
totalExecutionTime: 0,
|
|
@@ -117,7 +118,7 @@ class EntityHookManager {
|
|
|
117
118
|
*/
|
|
118
119
|
private initializeLifecycleIntegration(): void {
|
|
119
120
|
// Wait for components to be ready before allowing hook registration
|
|
120
|
-
|
|
121
|
+
this.phaseListener = (event: PhaseChangeEvent) => {
|
|
121
122
|
const phase = event.detail;
|
|
122
123
|
switch (phase) {
|
|
123
124
|
case ApplicationPhase.COMPONENTS_READY:
|
|
@@ -127,7 +128,15 @@ class EntityHookManager {
|
|
|
127
128
|
logger.info("EntityHookManager fully operational");
|
|
128
129
|
break;
|
|
129
130
|
}
|
|
130
|
-
}
|
|
131
|
+
};
|
|
132
|
+
ApplicationLifecycle.addPhaseListener(this.phaseListener);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
public dispose(): void {
|
|
136
|
+
if (this.phaseListener) {
|
|
137
|
+
ApplicationLifecycle.removePhaseListener(this.phaseListener);
|
|
138
|
+
this.phaseListener = null;
|
|
139
|
+
}
|
|
131
140
|
}
|
|
132
141
|
|
|
133
142
|
/**
|
|
@@ -293,14 +302,34 @@ class EntityHookManager {
|
|
|
293
302
|
|
|
294
303
|
try {
|
|
295
304
|
if (hook.options.timeout && hook.options.timeout > 0) {
|
|
296
|
-
// Execute with timeout
|
|
305
|
+
// Execute with timeout. Timer handle is stored so the
|
|
306
|
+
// normal-completion path clears it (no leaked pending
|
|
307
|
+
// timers per successful hook). The underlying callback
|
|
308
|
+
// promise is attached with a detached .catch so a late
|
|
309
|
+
// rejection after timeout does not escape as unhandled
|
|
310
|
+
// (H-HOOK-2 / H-MEM-2).
|
|
311
|
+
let timerHandle: ReturnType<typeof setTimeout> | null = null;
|
|
297
312
|
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
298
|
-
|
|
313
|
+
timerHandle = setTimeout(
|
|
314
|
+
() => reject(new Error(`Hook ${hook.id} timed out after ${hook.options.timeout}ms`)),
|
|
315
|
+
hook.options.timeout
|
|
316
|
+
);
|
|
299
317
|
});
|
|
300
|
-
|
|
318
|
+
const hookPromise = Promise.resolve().then(() => hook.callback(event));
|
|
319
|
+
hookPromise.catch((err) => {
|
|
320
|
+
logger.warn({ hookId: hook.id, err }, `Late rejection from hook after timeout`);
|
|
321
|
+
});
|
|
322
|
+
try {
|
|
323
|
+
await Promise.race([hookPromise, timeoutPromise]);
|
|
324
|
+
} finally {
|
|
325
|
+
if (timerHandle) clearTimeout(timerHandle);
|
|
326
|
+
}
|
|
301
327
|
} else {
|
|
302
|
-
//
|
|
303
|
-
|
|
328
|
+
// Always await — callback may be an async function declared
|
|
329
|
+
// with async:false by mistake. Without await, a rejection
|
|
330
|
+
// from such a callback escapes as an unhandled rejection
|
|
331
|
+
// and crashes the process under strict mode (C13).
|
|
332
|
+
await hook.callback(event);
|
|
304
333
|
}
|
|
305
334
|
} catch (error) {
|
|
306
335
|
logger.error(`Error executing sync hook ${hook.id} for event ${eventType}: ${error}`);
|
|
@@ -324,12 +353,26 @@ class EntityHookManager {
|
|
|
324
353
|
|
|
325
354
|
try {
|
|
326
355
|
if (hook.options.timeout && hook.options.timeout > 0) {
|
|
327
|
-
// Execute with timeout
|
|
328
|
-
|
|
356
|
+
// Execute with timeout. See sync path for rationale —
|
|
357
|
+
// clear the timer on normal completion and detach a
|
|
358
|
+
// .catch on the hook promise so late rejections do
|
|
359
|
+
// not escape (H-HOOK-2 / H-MEM-2).
|
|
360
|
+
let timerHandle: ReturnType<typeof setTimeout> | null = null;
|
|
361
|
+
const hookPromise = Promise.resolve().then(() => hook.callback(event));
|
|
362
|
+
hookPromise.catch((err) => {
|
|
363
|
+
logger.warn({ hookId: hook.id, err }, `Late rejection from hook after timeout`);
|
|
364
|
+
});
|
|
329
365
|
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
330
|
-
|
|
366
|
+
timerHandle = setTimeout(
|
|
367
|
+
() => reject(new Error(`Hook ${hook.id} timed out after ${hook.options.timeout}ms`)),
|
|
368
|
+
hook.options.timeout
|
|
369
|
+
);
|
|
331
370
|
});
|
|
332
|
-
|
|
371
|
+
try {
|
|
372
|
+
await Promise.race([hookPromise, timeoutPromise]);
|
|
373
|
+
} finally {
|
|
374
|
+
if (timerHandle) clearTimeout(timerHandle);
|
|
375
|
+
}
|
|
333
376
|
} else {
|
|
334
377
|
// Execute normally
|
|
335
378
|
await hook.callback(event);
|
|
@@ -467,14 +510,27 @@ class EntityHookManager {
|
|
|
467
510
|
|
|
468
511
|
try {
|
|
469
512
|
if (hook.options.timeout && hook.options.timeout > 0) {
|
|
470
|
-
//
|
|
513
|
+
// Same cleanup pattern as single-event path (H-HOOK-2 / H-MEM-2).
|
|
514
|
+
let timerHandle: ReturnType<typeof setTimeout> | null = null;
|
|
515
|
+
const hookPromise = Promise.resolve().then(() => hook.callback(event));
|
|
516
|
+
hookPromise.catch((err) => {
|
|
517
|
+
logger.warn({ hookId: hook.id, err }, `Late rejection from hook after timeout`);
|
|
518
|
+
});
|
|
471
519
|
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
472
|
-
|
|
520
|
+
timerHandle = setTimeout(
|
|
521
|
+
() => reject(new Error(`Hook ${hook.id} timed out after ${hook.options.timeout}ms`)),
|
|
522
|
+
hook.options.timeout
|
|
523
|
+
);
|
|
473
524
|
});
|
|
474
|
-
|
|
525
|
+
try {
|
|
526
|
+
await Promise.race([hookPromise, timeoutPromise]);
|
|
527
|
+
} finally {
|
|
528
|
+
if (timerHandle) clearTimeout(timerHandle);
|
|
529
|
+
}
|
|
475
530
|
} else {
|
|
476
|
-
//
|
|
477
|
-
|
|
531
|
+
// Await so async callbacks do not escape as unhandled
|
|
532
|
+
// rejections (C13 parity).
|
|
533
|
+
await hook.callback(event);
|
|
478
534
|
}
|
|
479
535
|
} catch (error) {
|
|
480
536
|
logger.error(`Error executing sync hook ${hook.id} for event ${eventType}: ${error}`);
|
|
@@ -518,12 +574,23 @@ class EntityHookManager {
|
|
|
518
574
|
(async () => {
|
|
519
575
|
try {
|
|
520
576
|
if (hook.options.timeout && hook.options.timeout > 0) {
|
|
521
|
-
//
|
|
522
|
-
|
|
577
|
+
// Same cleanup pattern (H-HOOK-2 / H-MEM-2).
|
|
578
|
+
let timerHandle: ReturnType<typeof setTimeout> | null = null;
|
|
579
|
+
const hookPromise = Promise.resolve().then(() => hook.callback(event));
|
|
580
|
+
hookPromise.catch((err) => {
|
|
581
|
+
logger.warn({ hookId: hook.id, err }, `Late rejection from hook after timeout`);
|
|
582
|
+
});
|
|
523
583
|
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
524
|
-
|
|
584
|
+
timerHandle = setTimeout(
|
|
585
|
+
() => reject(new Error(`Hook ${hook.id} timed out after ${hook.options.timeout}ms`)),
|
|
586
|
+
hook.options.timeout
|
|
587
|
+
);
|
|
525
588
|
});
|
|
526
|
-
|
|
589
|
+
try {
|
|
590
|
+
await Promise.race([hookPromise, timeoutPromise]);
|
|
591
|
+
} finally {
|
|
592
|
+
if (timerHandle) clearTimeout(timerHandle);
|
|
593
|
+
}
|
|
527
594
|
} else {
|
|
528
595
|
// Execute normally
|
|
529
596
|
await hook.callback(event);
|
package/core/EntityManager.ts
CHANGED
|
@@ -1,18 +1,27 @@
|
|
|
1
|
-
import ApplicationLifecycle, { ApplicationPhase } from "./ApplicationLifecycle";
|
|
1
|
+
import ApplicationLifecycle, { ApplicationPhase, type PhaseChangeEvent } from "./ApplicationLifecycle";
|
|
2
2
|
import type { IEntity } from "./EntityInterface";
|
|
3
3
|
|
|
4
4
|
class EntityManager {
|
|
5
5
|
static #instance: EntityManager;
|
|
6
6
|
private dbReady = false;
|
|
7
7
|
private entityQueue: IEntity[] = [];
|
|
8
|
+
private phaseListener: ((event: PhaseChangeEvent) => void) | null = null;
|
|
8
9
|
|
|
9
10
|
constructor() {
|
|
10
|
-
|
|
11
|
+
this.phaseListener = async (event: PhaseChangeEvent) => {
|
|
11
12
|
if (event.detail === ApplicationPhase.DATABASE_READY) {
|
|
12
13
|
this.dbReady = true;
|
|
13
14
|
await this.savePendingEntities();
|
|
14
15
|
}
|
|
15
|
-
}
|
|
16
|
+
};
|
|
17
|
+
ApplicationLifecycle.addPhaseListener(this.phaseListener);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
public dispose(): void {
|
|
21
|
+
if (this.phaseListener) {
|
|
22
|
+
ApplicationLifecycle.removePhaseListener(this.phaseListener);
|
|
23
|
+
this.phaseListener = null;
|
|
24
|
+
}
|
|
16
25
|
}
|
|
17
26
|
|
|
18
27
|
public deleteEntity(entity: IEntity, force: boolean = false) {
|
package/core/Logger.ts
CHANGED
|
@@ -9,6 +9,10 @@ export const logger = pino({
|
|
|
9
9
|
'config.password', '*.password', '*.secret', '*.token',
|
|
10
10
|
'*.accessKeyId', '*.secretAccessKey', '*.sessionToken',
|
|
11
11
|
'accessKeyId', 'secretAccessKey', 'sessionToken',
|
|
12
|
+
'headers.authorization', 'headers.cookie', 'headers["set-cookie"]',
|
|
13
|
+
'req.headers.authorization', 'req.headers.cookie',
|
|
14
|
+
'*.headers.authorization', '*.headers.cookie',
|
|
15
|
+
'redisUrl', '*.redisUrl', 'connectionString', '*.connectionString',
|
|
12
16
|
],
|
|
13
17
|
censor: '[REDACTED]',
|
|
14
18
|
},
|
package/core/RequestContext.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { createRequestLoaders } from './RequestLoaders';
|
|
|
3
3
|
import type { RequestLoaders } from './RequestLoaders';
|
|
4
4
|
import db from '../database';
|
|
5
5
|
import { CacheManager } from './cache/CacheManager';
|
|
6
|
+
import { getRequestId } from './middleware/RequestId';
|
|
6
7
|
|
|
7
8
|
declare module 'graphql-yoga' {
|
|
8
9
|
interface Context {
|
|
@@ -26,7 +27,9 @@ export function createRequestContextPlugin(): Plugin {
|
|
|
26
27
|
const cacheManager = CacheManager.getInstance();
|
|
27
28
|
// Mount loaders at context.loaders to match ArcheType.ts resolver access pattern
|
|
28
29
|
(args as any).contextValue.loaders = createRequestLoaders(db, cacheManager);
|
|
29
|
-
|
|
30
|
+
// Prefer the HTTP-layer request id (from requestId() middleware's
|
|
31
|
+
// AsyncLocalStorage) so access log + GraphQL logs share the same id.
|
|
32
|
+
(args as any).contextValue.requestId = getRequestId() ?? crypto.randomUUID();
|
|
30
33
|
(args as any).contextValue.cacheManager = cacheManager;
|
|
31
34
|
},
|
|
32
35
|
};
|
package/core/SchedulerManager.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { logger } from "./Logger";
|
|
2
|
-
import ApplicationLifecycle, { ApplicationPhase } from "./ApplicationLifecycle";
|
|
2
|
+
import ApplicationLifecycle, { ApplicationPhase, type PhaseChangeEvent } from "./ApplicationLifecycle";
|
|
3
3
|
import {
|
|
4
4
|
ScheduleInterval
|
|
5
5
|
} from "../types/scheduler.types";
|
|
@@ -30,6 +30,8 @@ export class SchedulerManager {
|
|
|
30
30
|
private eventListeners: SchedulerEventCallback[] = [];
|
|
31
31
|
public config: SchedulerConfig;
|
|
32
32
|
private distributedLock: DistributedLock;
|
|
33
|
+
private phaseListener: ((event: PhaseChangeEvent) => void) | null = null;
|
|
34
|
+
private inflightTasks: Set<Promise<any>> = new Set();
|
|
33
35
|
private metrics: SchedulerMetrics = {
|
|
34
36
|
totalTasks: 0,
|
|
35
37
|
runningTasks: 0,
|
|
@@ -76,14 +78,22 @@ export class SchedulerManager {
|
|
|
76
78
|
}
|
|
77
79
|
|
|
78
80
|
private initializeLifecycleIntegration(): void {
|
|
79
|
-
|
|
81
|
+
this.phaseListener = (event) => {
|
|
80
82
|
const phase = event.detail;
|
|
81
83
|
if (phase === ApplicationPhase.APPLICATION_READY) {
|
|
82
84
|
if (this.config.runOnStart) {
|
|
83
85
|
this.start();
|
|
84
86
|
}
|
|
85
87
|
}
|
|
86
|
-
}
|
|
88
|
+
};
|
|
89
|
+
ApplicationLifecycle.addPhaseListener(this.phaseListener);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
public disposeLifecycleIntegration(): void {
|
|
93
|
+
if (this.phaseListener) {
|
|
94
|
+
ApplicationLifecycle.removePhaseListener(this.phaseListener);
|
|
95
|
+
this.phaseListener = null;
|
|
96
|
+
}
|
|
87
97
|
}
|
|
88
98
|
|
|
89
99
|
public registerTask(taskInfo: ScheduledTaskInfo): void {
|
|
@@ -308,11 +318,34 @@ export class SchedulerManager {
|
|
|
308
318
|
}
|
|
309
319
|
|
|
310
320
|
private async executeTask(taskId: string): Promise<void> {
|
|
321
|
+
// Track this execution so stop() can await in-flight work before
|
|
322
|
+
// resources (DB pool, cache) are torn down. Without this, a task mid-
|
|
323
|
+
// write during SIGTERM hits a closed DB pool and silently corrupts
|
|
324
|
+
// or loses data.
|
|
325
|
+
const p = this.doExecuteTask(taskId);
|
|
326
|
+
this.inflightTasks.add(p);
|
|
327
|
+
p.finally(() => this.inflightTasks.delete(p));
|
|
328
|
+
return p;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
private async doExecuteTask(taskId: string): Promise<void> {
|
|
311
332
|
const taskInfo = this.tasks.get(taskId);
|
|
312
333
|
if (!taskInfo || !taskInfo.enabled) {
|
|
313
334
|
return;
|
|
314
335
|
}
|
|
315
336
|
|
|
337
|
+
// Skip if the previous tick is still executing. Without this guard
|
|
338
|
+
// a slow task with interval < execution-time burns a lock-acquire
|
|
339
|
+
// round-trip every tick and floods the skipped-executions metric
|
|
340
|
+
// (H-SCHED-1). Cheap in-process check before reaching out to PG.
|
|
341
|
+
if (taskInfo.isRunning) {
|
|
342
|
+
this.metrics.skippedExecutions++;
|
|
343
|
+
if (this.config.enableLogging) {
|
|
344
|
+
loggerInstance.debug(`Task ${taskInfo.name} skipped - previous execution still running`);
|
|
345
|
+
}
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
|
|
316
349
|
if (this.metrics.runningTasks >= this.config.maxConcurrentTasks) {
|
|
317
350
|
if (this.config.enableLogging) {
|
|
318
351
|
loggerInstance.warn(`Maximum concurrent tasks reached. Skipping execution of ${taskInfo.name}`);
|
|
@@ -492,7 +525,7 @@ export class SchedulerManager {
|
|
|
492
525
|
});
|
|
493
526
|
}
|
|
494
527
|
|
|
495
|
-
public async stop(): Promise<void> {
|
|
528
|
+
public async stop(drainTimeoutMs: number = 15_000): Promise<void> {
|
|
496
529
|
if (!this.isRunning) {
|
|
497
530
|
loggerInstance.warn("Scheduler is not running");
|
|
498
531
|
return;
|
|
@@ -500,16 +533,39 @@ export class SchedulerManager {
|
|
|
500
533
|
|
|
501
534
|
this.isRunning = false;
|
|
502
535
|
|
|
503
|
-
// Clear all intervals and timeouts
|
|
536
|
+
// Clear all intervals and timeouts so no new executions start.
|
|
504
537
|
for (const intervalId of this.intervals.values()) {
|
|
505
538
|
clearInterval(intervalId);
|
|
506
539
|
clearTimeout(intervalId as any);
|
|
507
540
|
}
|
|
508
541
|
this.intervals.clear();
|
|
509
542
|
|
|
543
|
+
// Drain in-flight tasks before releasing locks + returning control to
|
|
544
|
+
// App.shutdown (which will close the DB pool). Bounded by drainTimeoutMs
|
|
545
|
+
// so shutdown cannot hang forever on a stuck task.
|
|
546
|
+
if (this.inflightTasks.size > 0) {
|
|
547
|
+
const inflightSnapshot = [...this.inflightTasks];
|
|
548
|
+
if (this.config.enableLogging) {
|
|
549
|
+
loggerInstance.info(`Draining ${inflightSnapshot.length} in-flight scheduled task(s), timeout=${drainTimeoutMs}ms`);
|
|
550
|
+
}
|
|
551
|
+
const drainTimer = new Promise<'timeout'>((resolve) => {
|
|
552
|
+
const t = setTimeout(() => resolve('timeout'), drainTimeoutMs);
|
|
553
|
+
t.unref?.();
|
|
554
|
+
});
|
|
555
|
+
const result = await Promise.race([
|
|
556
|
+
Promise.allSettled(inflightSnapshot).then(() => 'drained' as const),
|
|
557
|
+
drainTimer,
|
|
558
|
+
]);
|
|
559
|
+
if (result === 'timeout') {
|
|
560
|
+
loggerInstance.warn(`Scheduler drain timed out after ${drainTimeoutMs}ms with ${this.inflightTasks.size} task(s) still running`);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
510
564
|
// Release all distributed locks held by this instance
|
|
511
565
|
await this.distributedLock.releaseAll();
|
|
512
566
|
|
|
567
|
+
this.disposeLifecycleIntegration();
|
|
568
|
+
|
|
513
569
|
if (this.config.enableLogging) {
|
|
514
570
|
loggerInstance.info("Scheduler stopped");
|
|
515
571
|
}
|
|
@@ -621,12 +677,20 @@ export class SchedulerManager {
|
|
|
621
677
|
}
|
|
622
678
|
|
|
623
679
|
/**
|
|
624
|
-
* Execute a task with timeout enforcement
|
|
680
|
+
* Execute a task with timeout enforcement.
|
|
681
|
+
*
|
|
682
|
+
* Note: JS has no way to cancel an arbitrary Promise — on timeout the
|
|
683
|
+
* wrapper rejects but the underlying task continues. The second .catch
|
|
684
|
+
* below captures a late rejection after the wrapper already rejected,
|
|
685
|
+
* preventing an unhandled-rejection process crash (H-SCHED-5). The
|
|
686
|
+
* `settled` flag guards against double-settle.
|
|
625
687
|
*/
|
|
626
688
|
private async executeWithTimeout<T>(task: Promise<T>, timeoutMs: number, taskInfo: ScheduledTaskInfo): Promise<T> {
|
|
627
689
|
return new Promise((resolve, reject) => {
|
|
690
|
+
let settled = false;
|
|
628
691
|
const timeoutId = setTimeout(() => {
|
|
629
|
-
|
|
692
|
+
if (settled) return;
|
|
693
|
+
settled = true;
|
|
630
694
|
this.metrics.timedOutTasks++;
|
|
631
695
|
this.updateTaskMetrics(taskInfo.id, {
|
|
632
696
|
timeoutCount: (this.metrics.taskMetrics[taskInfo.id]?.timeoutCount || 0) + 1
|
|
@@ -643,10 +707,20 @@ export class SchedulerManager {
|
|
|
643
707
|
|
|
644
708
|
task
|
|
645
709
|
.then((result) => {
|
|
710
|
+
if (settled) return;
|
|
711
|
+
settled = true;
|
|
646
712
|
clearTimeout(timeoutId);
|
|
647
713
|
resolve(result);
|
|
648
714
|
})
|
|
649
715
|
.catch((error) => {
|
|
716
|
+
if (settled) {
|
|
717
|
+
// Late rejection after timeout. Log only — the wrapper
|
|
718
|
+
// promise is already settled, so re-rejecting would be
|
|
719
|
+
// a no-op but the rejection would escape as unhandled.
|
|
720
|
+
loggerInstance.warn({ taskId: taskInfo.id, err: error }, 'Late rejection from scheduled task after timeout');
|
|
721
|
+
return;
|
|
722
|
+
}
|
|
723
|
+
settled = true;
|
|
650
724
|
clearTimeout(timeoutId);
|
|
651
725
|
reject(error);
|
|
652
726
|
});
|
|
@@ -678,10 +752,19 @@ export class SchedulerManager {
|
|
|
678
752
|
loggerInstance.warn(`Task ${taskInfo.name} failed (attempt ${taskInfo.retryCount}/${maxRetries}), retrying in ${retryDelay}ms: ${error.message}`);
|
|
679
753
|
}
|
|
680
754
|
|
|
681
|
-
// Schedule retry
|
|
682
|
-
|
|
755
|
+
// Schedule retry. Track the timer handle in `intervals` under a
|
|
756
|
+
// unique key so `stop()` can clear it (H-SCHED-3); without this
|
|
757
|
+
// the retry fires post-shutdown against a closed DB pool. Also
|
|
758
|
+
// skip the retry if the scheduler is no longer running by the
|
|
759
|
+
// time the timer fires, and rely on the new isRunning guard in
|
|
760
|
+
// doExecuteTask (H-SCHED-1) to prevent retry/tick overlap.
|
|
761
|
+
const retryKey = `${taskInfo.id}:retry:${taskInfo.retryCount}`;
|
|
762
|
+
const retryHandle = setTimeout(async () => {
|
|
763
|
+
this.intervals.delete(retryKey);
|
|
764
|
+
if (!this.isRunning) return;
|
|
683
765
|
await this.executeTask(taskInfo.id);
|
|
684
766
|
}, retryDelay);
|
|
767
|
+
this.intervals.set(retryKey, retryHandle as any);
|
|
685
768
|
|
|
686
769
|
this.emitEvent({
|
|
687
770
|
type: 'task.retry',
|
|
@@ -68,7 +68,9 @@ export class CacheFactory {
|
|
|
68
68
|
lazyConnect: false,
|
|
69
69
|
enableReadyCheck: true,
|
|
70
70
|
connectTimeout: config.redis.connectTimeout,
|
|
71
|
-
commandTimeout: config.redis.commandTimeout
|
|
71
|
+
commandTimeout: config.redis.commandTimeout,
|
|
72
|
+
maxReconnectAttempts: config.redis.maxReconnectAttempts,
|
|
73
|
+
enableOfflineQueue: config.redis.enableOfflineQueue
|
|
72
74
|
};
|
|
73
75
|
|
|
74
76
|
const { password: _pw, ...safeConfig } = redisConfig;
|
|
@@ -201,27 +201,45 @@ export class CacheManager {
|
|
|
201
201
|
|
|
202
202
|
try {
|
|
203
203
|
const effectiveTTL = ttl ?? this.config.component.ttl;
|
|
204
|
-
|
|
205
|
-
// Convert BaseComponent to ComponentData format for cache
|
|
204
|
+
|
|
205
|
+
// Convert BaseComponent to ComponentData format for cache
|
|
206
|
+
// compatibility with DataLoader. BaseComponent does not track
|
|
207
|
+
// createdAt/updatedAt today (data-model gap), but we preserve an
|
|
208
|
+
// existing cache entry's createdAt when available and stamp
|
|
209
|
+
// updatedAt=now, so consumers see monotonic update times rather
|
|
210
|
+
// than a reset on every write-through (H-CACHE-3 — full fix
|
|
211
|
+
// requires BaseComponent timestamp tracking).
|
|
206
212
|
for (const component of components) {
|
|
207
213
|
const typeId = componentType || component.getTypeID();
|
|
208
214
|
const key = `component:${entityId}:${typeId}`;
|
|
209
|
-
|
|
210
|
-
|
|
215
|
+
|
|
216
|
+
const now = new Date();
|
|
217
|
+
let createdAt: Date = now;
|
|
218
|
+
try {
|
|
219
|
+
const existing = await this.provider.get<ComponentData>(key);
|
|
220
|
+
if (existing && existing.createdAt) {
|
|
221
|
+
createdAt = existing.createdAt instanceof Date
|
|
222
|
+
? existing.createdAt
|
|
223
|
+
: new Date(existing.createdAt);
|
|
224
|
+
}
|
|
225
|
+
} catch {
|
|
226
|
+
// Cache miss or provider error — fall through to now.
|
|
227
|
+
}
|
|
228
|
+
|
|
211
229
|
const componentData: ComponentData = {
|
|
212
230
|
id: component.id,
|
|
213
231
|
entityId: entityId,
|
|
214
232
|
typeId: typeId,
|
|
215
233
|
data: component.data(),
|
|
216
|
-
createdAt
|
|
217
|
-
updatedAt:
|
|
234
|
+
createdAt,
|
|
235
|
+
updatedAt: now,
|
|
218
236
|
deletedAt: null
|
|
219
237
|
};
|
|
220
|
-
|
|
238
|
+
|
|
221
239
|
await this.provider.set(key, componentData, effectiveTTL);
|
|
222
240
|
}
|
|
223
241
|
} catch (error) {
|
|
224
|
-
logger.error({ scope: 'cache', component: 'CacheManager', msg: 'Error setting components in cache', error });
|
|
242
|
+
logger.error({ scope: 'cache', component: 'CacheManager', msg: 'Error setting components in cache', err: error });
|
|
225
243
|
}
|
|
226
244
|
}
|
|
227
245
|
|
|
@@ -489,21 +507,40 @@ export class CacheManager {
|
|
|
489
507
|
}
|
|
490
508
|
|
|
491
509
|
/**
|
|
492
|
-
* Shutdown the current provider (disconnect Redis, stop Memory cleanup
|
|
510
|
+
* Shutdown the current provider (disconnect Redis, stop Memory cleanup
|
|
511
|
+
* timer). For `MultiLevelCache`, descends into both L1 and L2 layers —
|
|
512
|
+
* previously the method only dispatched on the top-level provider, so a
|
|
513
|
+
* MultiLevelCache left its inner `MemoryCache` cleanup timer and Redis
|
|
514
|
+
* connection alive (H-CACHE-2).
|
|
493
515
|
*/
|
|
494
516
|
private async shutdownProvider(): Promise<void> {
|
|
517
|
+
const shutdownOne = async (p: any) => {
|
|
518
|
+
try {
|
|
519
|
+
if (p && typeof p.disconnect === 'function') {
|
|
520
|
+
await p.disconnect();
|
|
521
|
+
}
|
|
522
|
+
if (p && typeof p.stopCleanup === 'function') {
|
|
523
|
+
p.stopCleanup();
|
|
524
|
+
}
|
|
525
|
+
} catch (error) {
|
|
526
|
+
logger.error({ scope: 'cache', component: 'CacheManager', msg: 'Error shutting down layer', err: error });
|
|
527
|
+
}
|
|
528
|
+
};
|
|
529
|
+
|
|
495
530
|
try {
|
|
496
531
|
const provider = this.provider as any;
|
|
497
|
-
//
|
|
498
|
-
if (typeof provider.
|
|
499
|
-
await provider.
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
532
|
+
// MultiLevelCache exposes getL1Cache / getL2Cache.
|
|
533
|
+
if (provider && typeof provider.getL1Cache === 'function') {
|
|
534
|
+
await shutdownOne(provider.getL1Cache());
|
|
535
|
+
if (typeof provider.getL2Cache === 'function') {
|
|
536
|
+
await shutdownOne(provider.getL2Cache());
|
|
537
|
+
}
|
|
538
|
+
return;
|
|
504
539
|
}
|
|
540
|
+
// Single-layer providers.
|
|
541
|
+
await shutdownOne(provider);
|
|
505
542
|
} catch (error) {
|
|
506
|
-
logger.error({ scope: 'cache', component: 'CacheManager', msg: 'Error shutting down provider', error });
|
|
543
|
+
logger.error({ scope: 'cache', component: 'CacheManager', msg: 'Error shutting down provider', err: error });
|
|
507
544
|
}
|
|
508
545
|
}
|
|
509
546
|
|
package/core/cache/RedisCache.ts
CHANGED
|
@@ -35,12 +35,25 @@ export interface RedisCacheConfig {
|
|
|
35
35
|
password?: string;
|
|
36
36
|
db?: number;
|
|
37
37
|
keyPrefix?: string;
|
|
38
|
-
retryStrategy?: (times: number) => number | void;
|
|
38
|
+
retryStrategy?: (times: number) => number | null | void;
|
|
39
39
|
maxRetriesPerRequest?: number;
|
|
40
40
|
lazyConnect?: boolean;
|
|
41
41
|
enableReadyCheck?: boolean;
|
|
42
42
|
connectTimeout?: number;
|
|
43
43
|
commandTimeout?: number;
|
|
44
|
+
/**
|
|
45
|
+
* When true (default false), ioredis queues commands while offline. This
|
|
46
|
+
* can grow unboundedly during a Redis outage and exhaust heap. Keep false
|
|
47
|
+
* so cache operations fail fast and the caller's try/catch treats it as
|
|
48
|
+
* a cache miss instead of a hang.
|
|
49
|
+
*/
|
|
50
|
+
enableOfflineQueue?: boolean;
|
|
51
|
+
/**
|
|
52
|
+
* Maximum reconnect attempts before the retry strategy returns null and
|
|
53
|
+
* ioredis gives up. Prevents infinite reconnect storms when Redis is
|
|
54
|
+
* permanently unreachable. Default 20 attempts (~ 40s at 2s cap).
|
|
55
|
+
*/
|
|
56
|
+
maxReconnectAttempts?: number;
|
|
44
57
|
}
|
|
45
58
|
|
|
46
59
|
/**
|
|
@@ -65,18 +78,40 @@ export class RedisCache implements CacheProvider {
|
|
|
65
78
|
this.config = config;
|
|
66
79
|
this.keyPrefix = config.keyPrefix || 'bunsane:';
|
|
67
80
|
|
|
81
|
+
const maxReconnectAttempts = config.maxReconnectAttempts ?? 20;
|
|
82
|
+
const userRetryStrategy = config.retryStrategy;
|
|
83
|
+
|
|
84
|
+
// Wrap caller's retry strategy (or default) with a hard attempt cap so
|
|
85
|
+
// a permanently unreachable Redis cannot spin forever (C03).
|
|
86
|
+
const retryStrategy = (times: number): number | null => {
|
|
87
|
+
if (times > maxReconnectAttempts) {
|
|
88
|
+
logger.error({ scope: 'cache', provider: 'redis', attempts: times, msg: 'Redis retry cap reached — giving up reconnect attempts' });
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
if (userRetryStrategy) {
|
|
92
|
+
const result = userRetryStrategy(times);
|
|
93
|
+
if (result === null || result === undefined) return null;
|
|
94
|
+
return result as number;
|
|
95
|
+
}
|
|
96
|
+
return Math.min(times * 200, 2000);
|
|
97
|
+
};
|
|
98
|
+
|
|
68
99
|
const redisOptions: RedisOptions = {
|
|
69
100
|
host: config.host,
|
|
70
101
|
port: config.port,
|
|
71
102
|
password: config.password,
|
|
72
103
|
db: config.db || 0,
|
|
73
|
-
retryStrategy
|
|
104
|
+
retryStrategy,
|
|
74
105
|
maxRetriesPerRequest: config.maxRetriesPerRequest || 3,
|
|
75
106
|
lazyConnect: config.lazyConnect || false,
|
|
76
107
|
enableReadyCheck: config.enableReadyCheck || false,
|
|
77
108
|
connectTimeout: config.connectTimeout ?? 5000,
|
|
78
109
|
commandTimeout: config.commandTimeout ?? 3000,
|
|
79
|
-
|
|
110
|
+
// Fail-fast when Redis is down. Unbounded offline queue → heap
|
|
111
|
+
// exhaustion under sustained load during outage (C02). Callers
|
|
112
|
+
// already wrap cache ops in try/catch and treat failures as
|
|
113
|
+
// cache miss; bounded failure is better than buildup.
|
|
114
|
+
enableOfflineQueue: config.enableOfflineQueue ?? false,
|
|
80
115
|
};
|
|
81
116
|
|
|
82
117
|
this.client = new Redis(redisOptions);
|