instar 1.3.894 → 1.3.896
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/FailureEpisodeLatch.d.ts +8 -0
- package/dist/core/FailureEpisodeLatch.d.ts.map +1 -1
- package/dist/core/FailureEpisodeLatch.js +14 -0
- package/dist/core/FailureEpisodeLatch.js.map +1 -1
- package/dist/core/SelfHealGate.d.ts +148 -0
- package/dist/core/SelfHealGate.d.ts.map +1 -0
- package/dist/core/SelfHealGate.js +391 -0
- package/dist/core/SelfHealGate.js.map +1 -0
- package/dist/feedback-factory/drain/FeedbackFactoryDefaultsSelfHeal.d.ts +19 -0
- package/dist/feedback-factory/drain/FeedbackFactoryDefaultsSelfHeal.d.ts.map +1 -0
- package/dist/feedback-factory/drain/FeedbackFactoryDefaultsSelfHeal.js +118 -0
- package/dist/feedback-factory/drain/FeedbackFactoryDefaultsSelfHeal.js.map +1 -0
- package/dist/feedback-factory/drain/FeedbackFactoryGeneratedDefaults.d.ts +11 -1
- package/dist/feedback-factory/drain/FeedbackFactoryGeneratedDefaults.d.ts.map +1 -1
- package/dist/feedback-factory/drain/FeedbackFactoryGeneratedDefaults.js +88 -25
- package/dist/feedback-factory/drain/FeedbackFactoryGeneratedDefaults.js.map +1 -1
- package/dist/messaging/slack/SlackAdapter.js +15 -14
- package/dist/messaging/slack/SlackAdapter.js.map +1 -1
- package/dist/messaging/slack/types.d.ts +7 -4
- package/dist/messaging/slack/types.d.ts.map +1 -1
- package/dist/messaging/slack/types.js.map +1 -1
- package/dist/monitoring/selfaction/policies.d.ts.map +1 -1
- package/dist/monitoring/selfaction/policies.js +14 -0
- package/dist/monitoring/selfaction/policies.js.map +1 -1
- package/dist/permissions/AmbientContributionGate.d.ts +14 -17
- package/dist/permissions/AmbientContributionGate.d.ts.map +1 -1
- package/dist/permissions/AmbientContributionGate.js +50 -56
- package/dist/permissions/AmbientContributionGate.js.map +1 -1
- package/dist/server/AgentServer.d.ts.map +1 -1
- package/dist/server/AgentServer.js +24 -6
- package/dist/server/AgentServer.js.map +1 -1
- package/dist/testing/selfActionRegistry.d.ts.map +1 -1
- package/dist/testing/selfActionRegistry.js +25 -0
- package/dist/testing/selfActionRegistry.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +3 -3
- package/src/data/state-coherence-registry.json +17 -0
- package/upgrades/1.3.895.md +23 -0
- package/upgrades/1.3.896.md +25 -0
- package/upgrades/side-effects/self-heal-gate.md +87 -0
- package/upgrades/side-effects/slack-considered-acknowledgment-v1.md +80 -0
|
@@ -33,6 +33,12 @@ export interface FailureRecord {
|
|
|
33
33
|
/** Consecutive failures including this one. */
|
|
34
34
|
failures: number;
|
|
35
35
|
}
|
|
36
|
+
export interface FailureEpisodeSnapshot {
|
|
37
|
+
schemaVersion: 1;
|
|
38
|
+
failingSince: number | null;
|
|
39
|
+
failures: number;
|
|
40
|
+
signaledFor: number | null;
|
|
41
|
+
}
|
|
36
42
|
export declare class FailureEpisodeLatch {
|
|
37
43
|
private readonly signalAfterMs;
|
|
38
44
|
private readonly now;
|
|
@@ -46,5 +52,7 @@ export declare class FailureEpisodeLatch {
|
|
|
46
52
|
recovered: boolean;
|
|
47
53
|
failures: number;
|
|
48
54
|
};
|
|
55
|
+
snapshot(): FailureEpisodeSnapshot;
|
|
56
|
+
restore(snapshot: FailureEpisodeSnapshot): void;
|
|
49
57
|
}
|
|
50
58
|
//# sourceMappingURL=FailureEpisodeLatch.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FailureEpisodeLatch.d.ts","sourceRoot":"","sources":["../../src/core/FailureEpisodeLatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,uBAAuB;IACtC,qEAAqE;IACrE,aAAa,EAAE,MAAM,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,iEAAiE;IACjE,cAAc,EAAE,OAAO,CAAC;IACxB,uFAAuF;IACvF,YAAY,EAAE,OAAO,CAAC;IACtB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IAKnC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,QAAQ,CAAK;IACrB,oFAAoF;IACpF,OAAO,CAAC,WAAW,CAAuB;gBAE9B,IAAI,EAAE,uBAAuB;IAKzC,aAAa,IAAI,aAAa;IAc9B,aAAa,IAAI;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;
|
|
1
|
+
{"version":3,"file":"FailureEpisodeLatch.d.ts","sourceRoot":"","sources":["../../src/core/FailureEpisodeLatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,uBAAuB;IACtC,qEAAqE;IACrE,aAAa,EAAE,MAAM,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,iEAAiE;IACjE,cAAc,EAAE,OAAO,CAAC;IACxB,uFAAuF;IACvF,YAAY,EAAE,OAAO,CAAC;IACtB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,CAAC,CAAC;IACjB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IAKnC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,QAAQ,CAAK;IACrB,oFAAoF;IACpF,OAAO,CAAC,WAAW,CAAuB;gBAE9B,IAAI,EAAE,uBAAuB;IAKzC,aAAa,IAAI,aAAa;IAc9B,aAAa,IAAI;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;IASzD,QAAQ,IAAI,sBAAsB;IAIlC,OAAO,CAAC,QAAQ,EAAE,sBAAsB,GAAG,IAAI;CAWhD"}
|
|
@@ -55,5 +55,19 @@ export class FailureEpisodeLatch {
|
|
|
55
55
|
this.signaledFor = null;
|
|
56
56
|
return { recovered, failures };
|
|
57
57
|
}
|
|
58
|
+
snapshot() {
|
|
59
|
+
return { schemaVersion: 1, failingSince: this.failingSince, failures: this.failures, signaledFor: this.signaledFor };
|
|
60
|
+
}
|
|
61
|
+
restore(snapshot) {
|
|
62
|
+
const finiteNullable = (value) => value === null || (typeof value === 'number' && Number.isFinite(value) && value >= 0);
|
|
63
|
+
if (snapshot?.schemaVersion !== 1 || !finiteNullable(snapshot.failingSince) || !Number.isInteger(snapshot.failures) || snapshot.failures < 0 ||
|
|
64
|
+
!finiteNullable(snapshot.signaledFor) || (snapshot.signaledFor !== null && snapshot.signaledFor !== snapshot.failingSince) ||
|
|
65
|
+
(snapshot.failingSince === null && snapshot.failures !== 0) || (snapshot.failingSince !== null && snapshot.failures === 0)) {
|
|
66
|
+
throw new Error('invalid FailureEpisodeLatch snapshot');
|
|
67
|
+
}
|
|
68
|
+
this.failingSince = snapshot.failingSince;
|
|
69
|
+
this.failures = snapshot.failures;
|
|
70
|
+
this.signaledFor = snapshot.signaledFor;
|
|
71
|
+
}
|
|
58
72
|
}
|
|
59
73
|
//# sourceMappingURL=FailureEpisodeLatch.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FailureEpisodeLatch.js","sourceRoot":"","sources":["../../src/core/FailureEpisodeLatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;
|
|
1
|
+
{"version":3,"file":"FailureEpisodeLatch.js","sourceRoot":"","sources":["../../src/core/FailureEpisodeLatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AA0BH,MAAM,OAAO,mBAAmB;IACb,aAAa,CAAS;IACtB,GAAG,CAAe;IACnC,4EAA4E;IAC5E,uEAAuE;IACvE,0EAA0E;IAC1E,oDAAoD;IAC5C,YAAY,GAAkB,IAAI,CAAC;IACnC,QAAQ,GAAG,CAAC,CAAC;IACrB,oFAAoF;IAC5E,WAAW,GAAkB,IAAI,CAAC;IAE1C,YAAY,IAA6B;QACvC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;QACxC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IAClC,CAAC;IAED,aAAa;QACX,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC;QAClD,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI;YAAE,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,MAAM,YAAY,GAAG,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC;QAC7C,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,YAAY,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;YACjF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC;YACrC,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;IACjF,CAAC;IAED,aAAa;QACX,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,MAAM,SAAS,GAAG,QAAQ,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;IACjC,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;IACvH,CAAC;IAED,OAAO,CAAC,QAAgC;QACtC,MAAM,cAAc,GAAG,CAAC,KAAc,EAA0B,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;QACzJ,IAAI,QAAQ,EAAE,aAAa,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,GAAG,CAAC;YAC1I,CAAC,cAAc,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,KAAK,IAAI,IAAI,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,YAAY,CAAC;YAC1H,CAAC,QAAQ,CAAC,YAAY,KAAK,IAAI,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,KAAK,IAAI,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;YAC7H,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC;QAC1C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;IAC1C,CAAC;CACF"}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { type FailureEpisodeSnapshot } from './FailureEpisodeLatch.js';
|
|
2
|
+
import type { Admission, AdmissionToken, AdmitOpts, DerivedTarget } from '../monitoring/selfaction/types.js';
|
|
3
|
+
export type SelfHealSeverity = 'recoverable' | 'irreversible' | 'data-loss' | 'security' | 'unknown';
|
|
4
|
+
export type SelfHealEvidenceCode = 'source-absent' | 'schema-stale' | 'repaired' | 'already-healthy' | 'malformed-json' | 'access-denied' | 'io-error' | 'symlink-refused' | 'non-regular-refused' | 'sqlite-busy' | 'fence-stale' | 'verification-failed';
|
|
5
|
+
export type SelfHealRemediationResult = {
|
|
6
|
+
outcome: 'healed' | 'pending-restart' | 'not-healed';
|
|
7
|
+
evidence: SelfHealEvidenceCode;
|
|
8
|
+
};
|
|
9
|
+
type NoticeReason = 'unknown-severity' | 'irreversible' | 'data-loss' | 'security' | 'latency' | 'max-attempts' | 'wall-clock' | 'flap' | 'state-failure';
|
|
10
|
+
type NoticeState = {
|
|
11
|
+
state: 'pending' | 'enqueued';
|
|
12
|
+
id: string;
|
|
13
|
+
};
|
|
14
|
+
type RestartState = {
|
|
15
|
+
state: 'required' | 'requested';
|
|
16
|
+
requestingBootId: string;
|
|
17
|
+
stableRequestId: string;
|
|
18
|
+
};
|
|
19
|
+
export interface SelfHealNotice {
|
|
20
|
+
id: string;
|
|
21
|
+
controllerId: string;
|
|
22
|
+
reason: NoticeReason;
|
|
23
|
+
priority: 'HIGH' | 'URGENT';
|
|
24
|
+
}
|
|
25
|
+
export interface SelfHealAuditEvent {
|
|
26
|
+
at: number;
|
|
27
|
+
controllerId: string;
|
|
28
|
+
event: string;
|
|
29
|
+
reason?: string;
|
|
30
|
+
attempt?: number;
|
|
31
|
+
elapsedMs?: number;
|
|
32
|
+
}
|
|
33
|
+
export interface SelfHealSpec<Ctx> {
|
|
34
|
+
id: string;
|
|
35
|
+
controllerResource: 'hardware-bound' | 'pool-shared';
|
|
36
|
+
episodeAuthority: 'process-local' | 'durable-machine-local';
|
|
37
|
+
classId: string;
|
|
38
|
+
severity(ctx: Ctx): SelfHealSeverity;
|
|
39
|
+
dedupeKey(ctx: Ctx): string;
|
|
40
|
+
eligible(ctx: Ctx): {
|
|
41
|
+
eligible: boolean;
|
|
42
|
+
fence: string | null;
|
|
43
|
+
};
|
|
44
|
+
remediation(ctx: Ctx): SelfHealRemediationResult;
|
|
45
|
+
maxAttempts: number;
|
|
46
|
+
maxWallClockMs: number;
|
|
47
|
+
backoffMs(attempt: number): number;
|
|
48
|
+
notificationLatencyCeilingMs: number;
|
|
49
|
+
flap: {
|
|
50
|
+
maxRecoveries: number;
|
|
51
|
+
windowMs: number;
|
|
52
|
+
};
|
|
53
|
+
remediationActions: {
|
|
54
|
+
operation: string;
|
|
55
|
+
idempotencyGuard: string;
|
|
56
|
+
compensation: string;
|
|
57
|
+
};
|
|
58
|
+
restartVerified?(ctx: Ctx): boolean;
|
|
59
|
+
}
|
|
60
|
+
export type SelfHealResult = {
|
|
61
|
+
outcome: 'healed' | 'attempted' | 'backoff' | 'busy' | 'governed' | 'exhausted' | 'invalid-severity' | 'state-failure';
|
|
62
|
+
reason: string;
|
|
63
|
+
noticeAttempted?: boolean;
|
|
64
|
+
noticeReason?: NoticeReason;
|
|
65
|
+
tokenValid?: boolean;
|
|
66
|
+
};
|
|
67
|
+
export interface SelfHealEpisodeRecord {
|
|
68
|
+
schemaVersion: 1;
|
|
69
|
+
key: string;
|
|
70
|
+
revision: number;
|
|
71
|
+
status: 'active' | 'recovered' | 'exhausted';
|
|
72
|
+
startedAt: number;
|
|
73
|
+
attempts: number;
|
|
74
|
+
nextEligibleAt: number;
|
|
75
|
+
latch: FailureEpisodeSnapshot;
|
|
76
|
+
notices: Partial<Record<NoticeReason, NoticeState>>;
|
|
77
|
+
recoveries: number[];
|
|
78
|
+
restart: RestartState | null;
|
|
79
|
+
updatedAt: number;
|
|
80
|
+
}
|
|
81
|
+
export interface SelfHealEpisodeStore {
|
|
82
|
+
load(key: string): SelfHealEpisodeRecord | null;
|
|
83
|
+
create(record: SelfHealEpisodeRecord): SelfHealEpisodeRecord | null;
|
|
84
|
+
mutate(key: string, expectedRevision: number, update: (record: SelfHealEpisodeRecord) => SelfHealEpisodeRecord, guard?: () => boolean): SelfHealEpisodeRecord | null;
|
|
85
|
+
appendAudit?(event: SelfHealAuditEvent): void;
|
|
86
|
+
close?(): void;
|
|
87
|
+
}
|
|
88
|
+
export declare class InMemorySelfHealEpisodeStore implements SelfHealEpisodeStore {
|
|
89
|
+
private readonly rows;
|
|
90
|
+
readonly audits: SelfHealAuditEvent[];
|
|
91
|
+
load(key: string): SelfHealEpisodeRecord | null;
|
|
92
|
+
create(record: SelfHealEpisodeRecord): SelfHealEpisodeRecord | null;
|
|
93
|
+
mutate(key: string, expectedRevision: number, update: (record: SelfHealEpisodeRecord) => SelfHealEpisodeRecord, guard?: () => boolean): SelfHealEpisodeRecord | null;
|
|
94
|
+
appendAudit(event: SelfHealAuditEvent): void;
|
|
95
|
+
}
|
|
96
|
+
export declare class SqliteSelfHealEpisodeStore implements SelfHealEpisodeStore {
|
|
97
|
+
private readonly db;
|
|
98
|
+
private readonly unregisterSqlite;
|
|
99
|
+
constructor(dbPath: string);
|
|
100
|
+
load(key: string): SelfHealEpisodeRecord | null;
|
|
101
|
+
create(record: SelfHealEpisodeRecord): SelfHealEpisodeRecord | null;
|
|
102
|
+
mutate(key: string, expectedRevision: number, update: (record: SelfHealEpisodeRecord) => SelfHealEpisodeRecord, guard?: () => boolean): SelfHealEpisodeRecord | null;
|
|
103
|
+
appendAudit(event: SelfHealAuditEvent): void;
|
|
104
|
+
close(): void;
|
|
105
|
+
}
|
|
106
|
+
export interface SelfHealGateDeps {
|
|
107
|
+
admit(target: DerivedTarget, opts: AdmitOpts): Promise<Admission>;
|
|
108
|
+
consumeToken(token: AdmissionToken, controllerId: string, opts: {
|
|
109
|
+
targetKey: string;
|
|
110
|
+
nowMs: number;
|
|
111
|
+
}): {
|
|
112
|
+
proceed: boolean;
|
|
113
|
+
valid: boolean;
|
|
114
|
+
reason?: string;
|
|
115
|
+
};
|
|
116
|
+
notify(notice: SelfHealNotice): void | Promise<void>;
|
|
117
|
+
audit(event: SelfHealAuditEvent): void;
|
|
118
|
+
now?: () => number;
|
|
119
|
+
episodeStore?: SelfHealEpisodeStore;
|
|
120
|
+
bootId?: string;
|
|
121
|
+
requestRestart?: (stableRequestId: string) => boolean;
|
|
122
|
+
}
|
|
123
|
+
export declare class SelfHealGate<Ctx> {
|
|
124
|
+
private readonly spec;
|
|
125
|
+
private readonly deps;
|
|
126
|
+
private readonly now;
|
|
127
|
+
private readonly store;
|
|
128
|
+
private readonly bootId;
|
|
129
|
+
private readonly stateFailureNotices;
|
|
130
|
+
constructor(spec: SelfHealSpec<Ctx>, deps: SelfHealGateDeps);
|
|
131
|
+
needsRestartVerification(ctx: Ctx): boolean;
|
|
132
|
+
attempt(ctx: Ctx): Promise<SelfHealResult>;
|
|
133
|
+
recordHealthy(ctx: Ctx): SelfHealResult;
|
|
134
|
+
private attemptInner;
|
|
135
|
+
private runAdmitted;
|
|
136
|
+
private requestPendingRestart;
|
|
137
|
+
private ensureNotice;
|
|
138
|
+
private noticeAndReturn;
|
|
139
|
+
private exhaust;
|
|
140
|
+
private stateFailure;
|
|
141
|
+
private stateFailureWithNotice;
|
|
142
|
+
private key;
|
|
143
|
+
private newEpisode;
|
|
144
|
+
private cas;
|
|
145
|
+
private audit;
|
|
146
|
+
}
|
|
147
|
+
export {};
|
|
148
|
+
//# sourceMappingURL=SelfHealGate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SelfHealGate.d.ts","sourceRoot":"","sources":["../../src/core/SelfHealGate.ts"],"names":[],"mappings":"AAIA,OAAO,EAAuB,KAAK,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAE5F,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAE7G,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,CAAC;AACrG,MAAM,MAAM,oBAAoB,GAAG,eAAe,GAAG,cAAc,GAAG,UAAU,GAAG,iBAAiB,GAAG,gBAAgB,GAAG,eAAe,GAAG,UAAU,GAAG,iBAAiB,GAAG,qBAAqB,GAAG,aAAa,GAAG,aAAa,GAAG,qBAAqB,CAAC;AAC3P,MAAM,MAAM,yBAAyB,GAAG;IAAE,OAAO,EAAE,QAAQ,GAAG,iBAAiB,GAAG,YAAY,CAAC;IAAC,QAAQ,EAAE,oBAAoB,CAAA;CAAE,CAAC;AACjI,KAAK,YAAY,GAAG,kBAAkB,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,GAAG,cAAc,GAAG,YAAY,GAAG,MAAM,GAAG,eAAe,CAAC;AAC1J,KAAK,WAAW,GAAG;IAAE,KAAK,EAAE,SAAS,GAAG,UAAU,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC;AACjE,KAAK,YAAY,GAAG;IAAE,KAAK,EAAE,UAAU,GAAG,WAAW,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3G,MAAM,WAAW,cAAc;IAAG,EAAE,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAA;CAAE;AACvH,MAAM,WAAW,kBAAkB;IAAG,EAAE,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE;AAC9I,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,EAAE,EAAE,MAAM,CAAC;IAAC,kBAAkB,EAAE,gBAAgB,GAAG,aAAa,CAAC;IAAC,gBAAgB,EAAE,eAAe,GAAG,uBAAuB,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAC/I,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,gBAAgB,CAAC;IAAC,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IACnI,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,yBAAyB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IAClI,4BAA4B,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,kBAAkB,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1F,eAAe,CAAC,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC;CACrC;AACD,MAAM,MAAM,cAAc,GAAG;IAAE,OAAO,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,MAAM,GAAG,UAAU,GAAG,WAAW,GAAG,kBAAkB,GAAG,eAAe,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,CAAC,EAAE,YAAY,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAEtP,MAAM,WAAW,qBAAqB;IACpC,aAAa,EAAE,CAAC,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,WAAW,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IACnI,cAAc,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,sBAAsB,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC;IAAC,UAAU,EAAE,MAAM,EAAE,CAAC;IACjI,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;CACjD;AACD,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,qBAAqB,GAAG,IAAI,CAAC;IAChD,MAAM,CAAC,MAAM,EAAE,qBAAqB,GAAG,qBAAqB,GAAG,IAAI,CAAC;IACpE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,qBAAqB,KAAK,qBAAqB,EAAE,KAAK,CAAC,EAAE,MAAM,OAAO,GAAG,qBAAqB,GAAG,IAAI,CAAC;IACrK,WAAW,CAAC,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAAC,KAAK,CAAC,IAAI,IAAI,CAAC;CAC/D;AAcD,qBAAa,4BAA6B,YAAW,oBAAoB;IACvE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA4C;IAAC,QAAQ,CAAC,MAAM,EAAE,kBAAkB,EAAE,CAAM;IAC7G,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,qBAAqB,GAAG,IAAI;IAC/C,MAAM,CAAC,MAAM,EAAE,qBAAqB,GAAG,qBAAqB,GAAG,IAAI;IACnE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,qBAAqB,KAAK,qBAAqB,EAAE,KAAK,CAAC,EAAE,MAAM,OAAO,GAAG,qBAAqB,GAAG,IAAI;IAQpK,WAAW,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;CAC7C;AAED,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IACvC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAa;gBAClC,MAAM,EAAE,MAAM;IAS1B,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,qBAAqB,GAAG,IAAI;IAC/C,MAAM,CAAC,MAAM,EAAE,qBAAqB,GAAG,qBAAqB,GAAG,IAAI;IAOnE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,qBAAqB,KAAK,qBAAqB,EAAE,KAAK,CAAC,EAAE,MAAM,OAAO,GAAG,qBAAqB,GAAG,IAAI;IASpK,WAAW,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IAC5C,KAAK,IAAI,IAAI;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAAC,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAChO,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAAC,KAAK,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,oBAAoB,CAAC;IACtJ,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,CAAC,eAAe,EAAE,MAAM,KAAK,OAAO,CAAC;CACxE;AAED,qBAAa,YAAY,CAAC,GAAG;IAGf,OAAO,CAAC,QAAQ,CAAC,IAAI;IAAqB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAF3E,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuB;IAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAClH,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAqB;gBAC5B,IAAI,EAAE,YAAY,CAAC,GAAG,CAAC,EAAmB,IAAI,EAAE,gBAAgB;IAM7F,wBAAwB,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO;IACrC,OAAO,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,cAAc,CAAC;IAChD,aAAa,CAAC,GAAG,EAAE,GAAG,GAAG,cAAc;YAMzB,YAAY;YAkCZ,WAAW;IAyBzB,OAAO,CAAC,qBAAqB;YAIf,YAAY;YAMZ,eAAe;YACf,OAAO;IACrB,OAAO,CAAC,YAAY;YACN,sBAAsB;IASpC,OAAO,CAAC,GAAG;IACX,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,GAAG;IACX,OAAO,CAAC,KAAK;CACd"}
|
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import Database from 'better-sqlite3';
|
|
5
|
+
import { FailureEpisodeLatch } from './FailureEpisodeLatch.js';
|
|
6
|
+
import { registerSqliteHandle } from './SqliteRegistry.js';
|
|
7
|
+
function cloneRecord(record) { return structuredClone(record); }
|
|
8
|
+
function validateRecord(value) {
|
|
9
|
+
if (!value || typeof value !== 'object')
|
|
10
|
+
throw new Error('invalid self-heal episode');
|
|
11
|
+
const r = value;
|
|
12
|
+
if (r.schemaVersion !== 1 || typeof r.key !== 'string' || r.key.length < 1 || r.key.length > 240)
|
|
13
|
+
throw new Error('invalid self-heal episode');
|
|
14
|
+
for (const n of [r.revision, r.startedAt, r.attempts, r.nextEligibleAt, r.updatedAt])
|
|
15
|
+
if (!Number.isFinite(n) || n < 0)
|
|
16
|
+
throw new Error('invalid self-heal episode');
|
|
17
|
+
if (!Number.isInteger(r.revision) || !Number.isInteger(r.attempts) || !['active', 'recovered', 'exhausted'].includes(r.status))
|
|
18
|
+
throw new Error('invalid self-heal episode');
|
|
19
|
+
if (!Array.isArray(r.recoveries) || r.recoveries.length > 64 || r.recoveries.some((n) => !Number.isFinite(n) || n < 0))
|
|
20
|
+
throw new Error('invalid self-heal episode');
|
|
21
|
+
const latch = new FailureEpisodeLatch({ signalAfterMs: 1 });
|
|
22
|
+
latch.restore(r.latch);
|
|
23
|
+
return r;
|
|
24
|
+
}
|
|
25
|
+
export class InMemorySelfHealEpisodeStore {
|
|
26
|
+
rows = new Map();
|
|
27
|
+
audits = [];
|
|
28
|
+
load(key) { const row = this.rows.get(key); return row ? cloneRecord(row) : null; }
|
|
29
|
+
create(record) { if (this.rows.has(record.key))
|
|
30
|
+
return null; const row = validateRecord(cloneRecord(record)); this.rows.set(row.key, row); return cloneRecord(row); }
|
|
31
|
+
mutate(key, expectedRevision, update, guard) {
|
|
32
|
+
const current = this.rows.get(key);
|
|
33
|
+
if (!current || current.revision !== expectedRevision)
|
|
34
|
+
return null;
|
|
35
|
+
if (guard && !guard())
|
|
36
|
+
return null;
|
|
37
|
+
const next = validateRecord(update(cloneRecord(current)));
|
|
38
|
+
if (next.key !== key || next.revision !== expectedRevision + 1)
|
|
39
|
+
throw new Error('self-heal mutation must increment revision once');
|
|
40
|
+
if (current.status === 'exhausted' && next.status !== 'exhausted')
|
|
41
|
+
throw new Error('self-heal exhausted state is monotonic');
|
|
42
|
+
this.rows.set(key, cloneRecord(next));
|
|
43
|
+
return cloneRecord(next);
|
|
44
|
+
}
|
|
45
|
+
appendAudit(event) { this.audits.push(structuredClone(event)); }
|
|
46
|
+
}
|
|
47
|
+
export class SqliteSelfHealEpisodeStore {
|
|
48
|
+
db;
|
|
49
|
+
unregisterSqlite;
|
|
50
|
+
constructor(dbPath) {
|
|
51
|
+
const parent = path.dirname(dbPath);
|
|
52
|
+
fs.mkdirSync(parent, { recursive: true, mode: 0o700 });
|
|
53
|
+
const ps = fs.lstatSync(parent);
|
|
54
|
+
if (!ps.isDirectory() || ps.isSymbolicLink())
|
|
55
|
+
throw new Error('unsafe self-heal store parent');
|
|
56
|
+
if (fs.existsSync(dbPath)) {
|
|
57
|
+
const st = fs.lstatSync(dbPath);
|
|
58
|
+
if (!st.isFile() || st.isSymbolicLink())
|
|
59
|
+
throw new Error('unsafe self-heal store file');
|
|
60
|
+
}
|
|
61
|
+
this.db = new Database(dbPath, { timeout: 100 });
|
|
62
|
+
this.db.pragma('journal_mode = WAL');
|
|
63
|
+
this.db.pragma('busy_timeout = 100');
|
|
64
|
+
this.db.exec('CREATE TABLE IF NOT EXISTS self_heal_episodes (episode_key TEXT PRIMARY KEY, revision INTEGER NOT NULL, payload TEXT NOT NULL, updated_at INTEGER NOT NULL); CREATE TABLE IF NOT EXISTS self_heal_audit (id INTEGER PRIMARY KEY, at_ms INTEGER NOT NULL, controller_id TEXT NOT NULL, event TEXT NOT NULL, reason TEXT, attempt INTEGER, elapsed_ms INTEGER);');
|
|
65
|
+
try {
|
|
66
|
+
fs.chmodSync(dbPath, 0o600);
|
|
67
|
+
}
|
|
68
|
+
catch { /* non-posix */ }
|
|
69
|
+
this.unregisterSqlite = registerSqliteHandle(() => { try {
|
|
70
|
+
this.db.close();
|
|
71
|
+
}
|
|
72
|
+
catch { /* already closed */ } });
|
|
73
|
+
}
|
|
74
|
+
load(key) { const row = this.db.prepare('SELECT payload FROM self_heal_episodes WHERE episode_key=?').get(key); return row ? validateRecord(JSON.parse(row.payload)) : null; }
|
|
75
|
+
create(record) {
|
|
76
|
+
const row = validateRecord(cloneRecord(record));
|
|
77
|
+
const tx = this.db.transaction(() => {
|
|
78
|
+
const count = this.db.prepare('SELECT COUNT(*) AS n FROM self_heal_episodes').get().n;
|
|
79
|
+
if (count >= 256)
|
|
80
|
+
return false;
|
|
81
|
+
return this.db.prepare('INSERT OR IGNORE INTO self_heal_episodes(episode_key,revision,payload,updated_at) VALUES(?,?,?,?)').run(row.key, row.revision, JSON.stringify(row), row.updatedAt).changes === 1;
|
|
82
|
+
});
|
|
83
|
+
try {
|
|
84
|
+
return tx() ? cloneRecord(row) : null;
|
|
85
|
+
}
|
|
86
|
+
catch (error) {
|
|
87
|
+
if (/busy|locked/i.test(String(error)))
|
|
88
|
+
return null;
|
|
89
|
+
throw error;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
mutate(key, expectedRevision, update, guard) {
|
|
93
|
+
const tx = this.db.transaction(() => {
|
|
94
|
+
const current = this.load(key);
|
|
95
|
+
if (!current || current.revision !== expectedRevision)
|
|
96
|
+
return null;
|
|
97
|
+
if (guard && !guard())
|
|
98
|
+
return null;
|
|
99
|
+
const next = validateRecord(update(cloneRecord(current)));
|
|
100
|
+
if (next.key !== key || next.revision !== expectedRevision + 1)
|
|
101
|
+
throw new Error('self-heal mutation must increment revision once');
|
|
102
|
+
if (current.status === 'exhausted' && next.status !== 'exhausted')
|
|
103
|
+
throw new Error('self-heal exhausted state is monotonic');
|
|
104
|
+
const changed = this.db.prepare('UPDATE self_heal_episodes SET revision=?,payload=?,updated_at=? WHERE episode_key=? AND revision=?').run(next.revision, JSON.stringify(next), next.updatedAt, key, expectedRevision).changes;
|
|
105
|
+
return changed === 1 ? next : null;
|
|
106
|
+
});
|
|
107
|
+
try {
|
|
108
|
+
const row = tx();
|
|
109
|
+
return row ? cloneRecord(row) : null;
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
if (/busy|locked/i.test(String(error)))
|
|
113
|
+
return null;
|
|
114
|
+
throw error;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
appendAudit(event) { this.db.prepare('INSERT OR REPLACE INTO self_heal_audit(id,at_ms,controller_id,event,reason,attempt,elapsed_ms) VALUES(?,?,?,?,?,?,?)').run(event.at % 1024, event.at, event.controllerId, event.event, event.reason ?? null, event.attempt ?? null, event.elapsedMs ?? null); }
|
|
118
|
+
close() { this.unregisterSqlite(); this.db.close(); }
|
|
119
|
+
}
|
|
120
|
+
export class SelfHealGate {
|
|
121
|
+
spec;
|
|
122
|
+
deps;
|
|
123
|
+
now;
|
|
124
|
+
store;
|
|
125
|
+
bootId;
|
|
126
|
+
stateFailureNotices = new Set();
|
|
127
|
+
constructor(spec, deps) {
|
|
128
|
+
this.spec = spec;
|
|
129
|
+
this.deps = deps;
|
|
130
|
+
if (spec.controllerResource !== 'hardware-bound')
|
|
131
|
+
throw new Error('v1 SelfHealGate refuses pool-shared controllers');
|
|
132
|
+
if (!Number.isInteger(spec.maxAttempts) || spec.maxAttempts < 1 || spec.maxAttempts > 32)
|
|
133
|
+
throw new Error('invalid maxAttempts');
|
|
134
|
+
for (const value of [spec.maxWallClockMs, spec.notificationLatencyCeilingMs, spec.flap.maxRecoveries, spec.flap.windowMs])
|
|
135
|
+
if (!Number.isFinite(value) || value <= 0)
|
|
136
|
+
throw new Error('invalid SelfHealGate bound');
|
|
137
|
+
this.now = deps.now ?? Date.now;
|
|
138
|
+
this.store = deps.episodeStore ?? new InMemorySelfHealEpisodeStore();
|
|
139
|
+
this.bootId = deps.bootId ?? crypto.randomUUID();
|
|
140
|
+
}
|
|
141
|
+
needsRestartVerification(ctx) { return this.store.load(this.key(ctx))?.restart !== null; }
|
|
142
|
+
async attempt(ctx) { const began = this.now(); try {
|
|
143
|
+
return await this.attemptInner(ctx);
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
return this.stateFailureWithNotice('store-operation-failed');
|
|
147
|
+
}
|
|
148
|
+
finally {
|
|
149
|
+
this.audit({ event: 'attempt-finished', elapsedMs: Math.max(0, this.now() - began) });
|
|
150
|
+
} }
|
|
151
|
+
recordHealthy(ctx) {
|
|
152
|
+
const current = this.store.load(this.key(ctx));
|
|
153
|
+
if (!current)
|
|
154
|
+
return { outcome: 'healed', reason: 'already-healthy' };
|
|
155
|
+
if (current.status === 'exhausted')
|
|
156
|
+
return { outcome: 'exhausted', reason: 'terminal' };
|
|
157
|
+
if (current.restart)
|
|
158
|
+
return { outcome: 'governed', reason: 'restart-verification-required' };
|
|
159
|
+
const next = this.cas(current, (row) => ({ ...row, status: 'recovered', recoveries: [...row.recoveries, this.now()].slice(-64), restart: null }));
|
|
160
|
+
return next ? { outcome: 'healed', reason: 'recorded-healthy' } : { outcome: 'busy', reason: 'revision-conflict' };
|
|
161
|
+
}
|
|
162
|
+
async attemptInner(ctx) {
|
|
163
|
+
const key = this.key(ctx);
|
|
164
|
+
let row = this.store.load(key);
|
|
165
|
+
if (!row) {
|
|
166
|
+
const fresh = this.newEpisode(key);
|
|
167
|
+
row = this.store.create(fresh) ?? this.store.load(key);
|
|
168
|
+
}
|
|
169
|
+
if (!row)
|
|
170
|
+
return this.stateFailureWithNotice('create-failed');
|
|
171
|
+
if (row.status === 'active') {
|
|
172
|
+
const now = this.now();
|
|
173
|
+
const recent = row.recoveries.filter((at) => now - at <= this.spec.flap.windowMs);
|
|
174
|
+
if (now - row.startedAt >= this.spec.maxWallClockMs)
|
|
175
|
+
return this.exhaust(row, 'wall-clock');
|
|
176
|
+
if (recent.length >= this.spec.flap.maxRecoveries)
|
|
177
|
+
return this.exhaust(row, 'flap');
|
|
178
|
+
if (row.attempts >= this.spec.maxAttempts && !row.restart)
|
|
179
|
+
return this.exhaust(row, 'max-attempts');
|
|
180
|
+
}
|
|
181
|
+
if (row.restart) {
|
|
182
|
+
if (row.restart.state === 'required')
|
|
183
|
+
return this.requestPendingRestart(row);
|
|
184
|
+
if (row.restart.requestingBootId === this.bootId)
|
|
185
|
+
return { outcome: 'attempted', reason: 'restart-awaiting-new-boot' };
|
|
186
|
+
let verified = false;
|
|
187
|
+
try {
|
|
188
|
+
verified = this.spec.restartVerified?.(ctx) === true;
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
verified = false;
|
|
192
|
+
}
|
|
193
|
+
if (!verified) {
|
|
194
|
+
let severity;
|
|
195
|
+
try {
|
|
196
|
+
severity = this.spec.severity(ctx);
|
|
197
|
+
}
|
|
198
|
+
catch {
|
|
199
|
+
severity = 'unknown';
|
|
200
|
+
}
|
|
201
|
+
if (severity === 'unknown')
|
|
202
|
+
return this.noticeAndReturn(row, 'unknown-severity', 'invalid-severity');
|
|
203
|
+
return { outcome: 'attempted', reason: 'restart-verification-not-healthy' };
|
|
204
|
+
}
|
|
205
|
+
const closed = this.cas(row, (r) => ({ ...r, status: 'recovered', restart: null, recoveries: [...r.recoveries, this.now()].slice(-64) }));
|
|
206
|
+
return closed ? { outcome: 'healed', reason: 'restart-verified' } : { outcome: 'busy', reason: 'revision-conflict' };
|
|
207
|
+
}
|
|
208
|
+
let severity;
|
|
209
|
+
try {
|
|
210
|
+
severity = this.spec.severity(ctx);
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
severity = 'unknown';
|
|
214
|
+
}
|
|
215
|
+
if (!['recoverable', 'irreversible', 'data-loss', 'security'].includes(severity))
|
|
216
|
+
return this.noticeAndReturn(row, 'unknown-severity', 'invalid-severity');
|
|
217
|
+
if (severity !== 'recoverable') {
|
|
218
|
+
const notice = await this.ensureNotice(row, severity);
|
|
219
|
+
row = notice.row;
|
|
220
|
+
}
|
|
221
|
+
if (row.status === 'recovered') {
|
|
222
|
+
const now = this.now();
|
|
223
|
+
const reset = this.cas(row, (r) => ({ ...r, status: 'active', startedAt: now, attempts: 0, nextEligibleAt: 0, latch: emptyLatch(), notices: {}, restart: null }));
|
|
224
|
+
if (!reset)
|
|
225
|
+
return { outcome: 'busy', reason: 'revision-conflict' };
|
|
226
|
+
row = reset;
|
|
227
|
+
}
|
|
228
|
+
if (row.status === 'exhausted')
|
|
229
|
+
return { outcome: 'exhausted', reason: 'terminal' };
|
|
230
|
+
const latch = new FailureEpisodeLatch({ signalAfterMs: this.spec.notificationLatencyCeilingMs, now: this.now });
|
|
231
|
+
latch.restore(row.latch);
|
|
232
|
+
const failure = latch.recordFailure();
|
|
233
|
+
const detected = this.cas(row, (r) => ({ ...r, latch: latch.snapshot() }));
|
|
234
|
+
if (!detected)
|
|
235
|
+
return { outcome: 'busy', reason: 'revision-conflict' };
|
|
236
|
+
row = detected;
|
|
237
|
+
if (failure.shouldSignal) {
|
|
238
|
+
const notice = await this.ensureNotice(row, 'latency');
|
|
239
|
+
row = notice.row;
|
|
240
|
+
}
|
|
241
|
+
const now = this.now();
|
|
242
|
+
const recent = row.recoveries.filter((at) => now - at <= this.spec.flap.windowMs);
|
|
243
|
+
if (now - row.startedAt >= this.spec.maxWallClockMs)
|
|
244
|
+
return this.exhaust(row, 'wall-clock');
|
|
245
|
+
if (recent.length >= this.spec.flap.maxRecoveries)
|
|
246
|
+
return this.exhaust(row, 'flap');
|
|
247
|
+
if (row.attempts >= this.spec.maxAttempts)
|
|
248
|
+
return this.exhaust(row, 'max-attempts');
|
|
249
|
+
if (row.nextEligibleAt > now)
|
|
250
|
+
return { outcome: 'backoff', reason: 'not-eligible-yet' };
|
|
251
|
+
const eligibility = safeEligible(this.spec, ctx);
|
|
252
|
+
if (!eligibility.eligible || !eligibility.fence)
|
|
253
|
+
return { outcome: 'governed', reason: 'fence-stale' };
|
|
254
|
+
const target = { key: keyHash(key), classId: this.spec.classId, keyIsVolatile: false };
|
|
255
|
+
const admission = await this.deps.admit(target, { incarnation: eligibility.fence, eligible: () => safeEligible(this.spec, ctx).fence === eligibility.fence, onAdmitted: (token) => { void this.runAdmitted(ctx, target, token).catch(() => { void this.stateFailureWithNotice('queued-attempt-failed'); }); }, lane: 'job', nowMs: now });
|
|
256
|
+
if (admission.outcome !== 'allow')
|
|
257
|
+
return { outcome: 'governed', reason: admission.outcome };
|
|
258
|
+
return this.runAdmitted(ctx, target, admission.token);
|
|
259
|
+
}
|
|
260
|
+
async runAdmitted(ctx, target, token) {
|
|
261
|
+
let row = this.store.load(this.key(ctx));
|
|
262
|
+
if (!row || row.status !== 'active')
|
|
263
|
+
return { outcome: 'governed', reason: 'episode-not-active' };
|
|
264
|
+
const latch = new FailureEpisodeLatch({ signalAfterMs: this.spec.notificationLatencyCeilingMs, now: this.now });
|
|
265
|
+
latch.restore(row.latch);
|
|
266
|
+
const failure = latch.recordFailure();
|
|
267
|
+
const observed = this.cas(row, (r) => ({ ...r, latch: latch.snapshot() }));
|
|
268
|
+
if (!observed)
|
|
269
|
+
return { outcome: 'busy', reason: 'revision-conflict' };
|
|
270
|
+
row = observed;
|
|
271
|
+
if (failure.shouldSignal) {
|
|
272
|
+
const notice = await this.ensureNotice(row, 'latency');
|
|
273
|
+
row = notice.row;
|
|
274
|
+
}
|
|
275
|
+
const now = this.now();
|
|
276
|
+
const recent = row.recoveries.filter((at) => now - at <= this.spec.flap.windowMs);
|
|
277
|
+
if (now - row.startedAt >= this.spec.maxWallClockMs)
|
|
278
|
+
return this.exhaust(row, 'wall-clock');
|
|
279
|
+
if (recent.length >= this.spec.flap.maxRecoveries)
|
|
280
|
+
return this.exhaust(row, 'flap');
|
|
281
|
+
if (row.nextEligibleAt > now)
|
|
282
|
+
return { outcome: 'backoff', reason: 'not-eligible-yet' };
|
|
283
|
+
const eligibility = safeEligible(this.spec, ctx);
|
|
284
|
+
if (!eligibility.eligible || !eligibility.fence)
|
|
285
|
+
return { outcome: 'governed', reason: 'fence-stale' };
|
|
286
|
+
const consumed = this.deps.consumeToken(token, this.spec.id, { targetKey: target.key, nowMs: this.now() });
|
|
287
|
+
if (!consumed.proceed)
|
|
288
|
+
return { outcome: 'governed', reason: consumed.reason ?? 'token-refused', tokenValid: consumed.valid };
|
|
289
|
+
const again = safeEligible(this.spec, ctx);
|
|
290
|
+
if (!again.eligible || again.fence !== eligibility.fence)
|
|
291
|
+
return { outcome: 'governed', reason: 'fence-stale', tokenValid: consumed.valid };
|
|
292
|
+
if (row.attempts >= this.spec.maxAttempts)
|
|
293
|
+
return this.exhaust(row, 'max-attempts');
|
|
294
|
+
const claim = this.cas(row, (r) => ({ ...r, attempts: r.attempts + 1 }), () => {
|
|
295
|
+
const live = safeEligible(this.spec, ctx);
|
|
296
|
+
return live.eligible && live.fence === eligibility.fence;
|
|
297
|
+
});
|
|
298
|
+
if (!claim)
|
|
299
|
+
return { outcome: 'governed', reason: 'fence-stale-or-revision-conflict' };
|
|
300
|
+
row = claim;
|
|
301
|
+
let result;
|
|
302
|
+
try {
|
|
303
|
+
result = this.spec.remediation(ctx);
|
|
304
|
+
if (result && typeof result.then === 'function')
|
|
305
|
+
throw new Error('async remediation forbidden');
|
|
306
|
+
}
|
|
307
|
+
catch {
|
|
308
|
+
result = { outcome: 'not-healed', evidence: 'io-error' };
|
|
309
|
+
}
|
|
310
|
+
if (result.outcome === 'healed') {
|
|
311
|
+
const closed = this.cas(row, (r) => ({ ...r, status: 'recovered', recoveries: [...r.recoveries, this.now()].slice(-64) }));
|
|
312
|
+
return closed ? { outcome: 'healed', reason: result.evidence, tokenValid: consumed.valid } : { outcome: 'busy', reason: 'revision-conflict' };
|
|
313
|
+
}
|
|
314
|
+
if (result.outcome === 'pending-restart') {
|
|
315
|
+
const stableRequestId = `self-heal:${keyHash(row.key)}:${row.startedAt}`;
|
|
316
|
+
const pending = this.cas(row, (r) => ({ ...r, restart: { state: 'required', requestingBootId: this.bootId, stableRequestId } }));
|
|
317
|
+
return pending ? this.requestPendingRestart(pending) : { outcome: 'busy', reason: 'revision-conflict' };
|
|
318
|
+
}
|
|
319
|
+
const failed = this.cas(row, (r) => ({ ...r, nextEligibleAt: this.now() + Math.max(0, this.spec.backoffMs(row.attempts)) }));
|
|
320
|
+
return failed ? { outcome: 'attempted', reason: result.evidence, tokenValid: consumed.valid } : { outcome: 'busy', reason: 'revision-conflict' };
|
|
321
|
+
}
|
|
322
|
+
requestPendingRestart(row) {
|
|
323
|
+
if (!row.restart)
|
|
324
|
+
return { outcome: 'state-failure', reason: 'restart-state-missing' };
|
|
325
|
+
const ok = this.deps.requestRestart?.(row.restart.stableRequestId) ?? false;
|
|
326
|
+
if (!ok)
|
|
327
|
+
return { outcome: 'attempted', reason: 'restart-request-failed' };
|
|
328
|
+
if (row.restart.state === 'requested')
|
|
329
|
+
return { outcome: 'attempted', reason: 'restart-requested' };
|
|
330
|
+
const requested = this.cas(row, (r) => ({ ...r, restart: r.restart ? { ...r.restart, state: 'requested' } : null }));
|
|
331
|
+
return requested ? { outcome: 'attempted', reason: 'restart-requested' } : { outcome: 'busy', reason: 'revision-conflict' };
|
|
332
|
+
}
|
|
333
|
+
async ensureNotice(row, reason) {
|
|
334
|
+
const existing = row.notices[reason];
|
|
335
|
+
if (existing?.state === 'enqueued')
|
|
336
|
+
return { row, attempted: false };
|
|
337
|
+
const id = existing?.id ?? `self-heal:${keyHash(row.key)}:${reason}:${row.startedAt}`;
|
|
338
|
+
if (!existing) {
|
|
339
|
+
const pending = this.cas(row, (r) => ({ ...r, notices: { ...r.notices, [reason]: { state: 'pending', id } } }));
|
|
340
|
+
if (!pending)
|
|
341
|
+
return { row: this.store.load(row.key) ?? row, attempted: false };
|
|
342
|
+
row = pending;
|
|
343
|
+
}
|
|
344
|
+
try {
|
|
345
|
+
await this.deps.notify({ id, controllerId: this.spec.id, reason, priority: ['irreversible', 'data-loss', 'security'].includes(reason) ? 'URGENT' : 'HIGH' });
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
this.audit({ event: 'notice-enqueue-failed', reason });
|
|
349
|
+
return { row, attempted: true };
|
|
350
|
+
}
|
|
351
|
+
const enqueued = this.cas(row, (r) => ({ ...r, notices: { ...r.notices, [reason]: { state: 'enqueued', id } } }));
|
|
352
|
+
return { row: enqueued ?? row, attempted: true };
|
|
353
|
+
}
|
|
354
|
+
async noticeAndReturn(row, reason, outcome) { const notice = await this.ensureNotice(row, reason); return { outcome, reason, noticeAttempted: notice.attempted, noticeReason: reason }; }
|
|
355
|
+
async exhaust(row, reason) { const notice = await this.ensureNotice(row, reason); row = notice.row; const terminal = this.cas(row, (r) => ({ ...r, status: 'exhausted' })); return terminal ? { outcome: 'exhausted', reason, noticeAttempted: notice.attempted, noticeReason: reason } : { outcome: 'busy', reason: 'revision-conflict' }; }
|
|
356
|
+
stateFailure(reason) { this.audit({ event: 'state-failure', reason }); return { outcome: 'state-failure', reason }; }
|
|
357
|
+
async stateFailureWithNotice(reason) {
|
|
358
|
+
this.audit({ event: 'state-failure', reason });
|
|
359
|
+
const id = `self-heal:${this.spec.id}:state-failure:${this.bootId}`;
|
|
360
|
+
if (!this.stateFailureNotices.has(id)) {
|
|
361
|
+
try {
|
|
362
|
+
await this.deps.notify({ id, controllerId: this.spec.id, reason: 'state-failure', priority: 'HIGH' });
|
|
363
|
+
this.stateFailureNotices.add(id);
|
|
364
|
+
}
|
|
365
|
+
catch {
|
|
366
|
+
this.audit({ event: 'notice-enqueue-failed', reason: 'state-failure' });
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
return { outcome: 'state-failure', reason, noticeAttempted: true, noticeReason: 'state-failure' };
|
|
370
|
+
}
|
|
371
|
+
key(ctx) { const d = this.spec.dedupeKey(ctx); if (!/^[a-z0-9][a-z0-9._:-]{0,119}$/i.test(d))
|
|
372
|
+
throw new Error('invalid self-heal dedupe key'); return `${this.spec.id}:${d}`; }
|
|
373
|
+
newEpisode(key) { const now = this.now(); return { schemaVersion: 1, key, revision: 0, status: 'active', startedAt: now, attempts: 0, nextEligibleAt: 0, latch: emptyLatch(), notices: {}, recoveries: [], restart: null, updatedAt: now }; }
|
|
374
|
+
cas(row, update, guard) { return this.store.mutate(row.key, row.revision, (r) => ({ ...update(r), revision: r.revision + 1, updatedAt: this.now() }), guard); }
|
|
375
|
+
audit(input) { const event = { ...input, at: this.now(), controllerId: this.spec.id }; try {
|
|
376
|
+
this.store.appendAudit?.(event);
|
|
377
|
+
}
|
|
378
|
+
catch { /* secondary sink below */ } try {
|
|
379
|
+
this.deps.audit(event);
|
|
380
|
+
}
|
|
381
|
+
catch { /* metadata audit best effort */ } }
|
|
382
|
+
}
|
|
383
|
+
function safeEligible(spec, ctx) { try {
|
|
384
|
+
return spec.eligible(ctx);
|
|
385
|
+
}
|
|
386
|
+
catch {
|
|
387
|
+
return { eligible: false, fence: null };
|
|
388
|
+
} }
|
|
389
|
+
function emptyLatch() { return { schemaVersion: 1, failingSince: null, failures: 0, signaledFor: null }; }
|
|
390
|
+
function keyHash(value) { return crypto.createHash('sha256').update(value).digest('hex').slice(0, 24); }
|
|
391
|
+
//# sourceMappingURL=SelfHealGate.js.map
|