@claude-flow/plugin-agent-federation 1.0.0-alpha.5 → 1.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/federation-breaker-service.d.ts +182 -0
- package/dist/application/federation-breaker-service.d.ts.map +1 -0
- package/dist/application/federation-breaker-service.js +251 -0
- package/dist/application/federation-breaker-service.js.map +1 -0
- package/dist/application/federation-coordinator.d.ts +35 -1
- package/dist/application/federation-coordinator.d.ts.map +1 -1
- package/dist/application/federation-coordinator.js +72 -2
- package/dist/application/federation-coordinator.js.map +1 -1
- package/dist/application/policy-engine.d.ts.map +1 -1
- package/dist/application/policy-engine.js +9 -0
- package/dist/application/policy-engine.js.map +1 -1
- package/dist/domain/entities/federation-envelope.d.ts +1 -1
- package/dist/domain/entities/federation-envelope.d.ts.map +1 -1
- package/dist/domain/entities/federation-envelope.js +3 -0
- package/dist/domain/entities/federation-envelope.js.map +1 -1
- package/dist/domain/entities/federation-node.d.ts +55 -0
- package/dist/domain/entities/federation-node.d.ts.map +1 -1
- package/dist/domain/entities/federation-node.js +86 -0
- package/dist/domain/entities/federation-node.js.map +1 -1
- package/dist/domain/value-objects/federation-budget.d.ts +89 -0
- package/dist/domain/value-objects/federation-budget.d.ts.map +1 -0
- package/dist/domain/value-objects/federation-budget.js +165 -0
- package/dist/domain/value-objects/federation-budget.js.map +1 -0
- package/dist/domain/value-objects/federation-node-state.d.ts +92 -0
- package/dist/domain/value-objects/federation-node-state.d.ts.map +1 -0
- package/dist/domain/value-objects/federation-node-state.js +100 -0
- package/dist/domain/value-objects/federation-node-state.js.map +1 -0
- package/dist/mcp-tools.d.ts.map +1 -1
- package/dist/mcp-tools.js +34 -2
- package/dist/mcp-tools.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Federation circuit breaker service (ADR-097 Phase 2.b).
|
|
3
|
+
*
|
|
4
|
+
* Phase 2.a shipped the FederationNode state machine + canTransition table.
|
|
5
|
+
* This service is what drives transitions: it records per-peer send outcomes
|
|
6
|
+
* (success/failure + spend), evaluates thresholds against the configured
|
|
7
|
+
* policy, and calls suspend()/evict()/reactivate() on the entity.
|
|
8
|
+
*
|
|
9
|
+
* Architecture is two-layer to keep the policy testable in isolation:
|
|
10
|
+
*
|
|
11
|
+
* 1. **evaluatePolicy** (pure function) — given rolling samples for a peer
|
|
12
|
+
* + the policy + now, returns a BreakerDecision describing what should
|
|
13
|
+
* happen. No I/O, no clock, no entity mutation.
|
|
14
|
+
* 2. **FederationBreakerService** — owns the rolling-sample buffer,
|
|
15
|
+
* wraps evaluatePolicy with the entity mutation step, and exposes
|
|
16
|
+
* `recordOutcome` (called by the coordinator on each send completion)
|
|
17
|
+
* + `tick` (called periodically to drive cooldown/auto-evict).
|
|
18
|
+
*
|
|
19
|
+
* The cost-tracker integration (Phase 3 upstream — emits federation_spend
|
|
20
|
+
* events when sends complete) is *not* wired here; this service holds its
|
|
21
|
+
* own in-memory rolling buffer. When Phase 3 upstream lands, the consumer
|
|
22
|
+
* can call `recordOutcome` from the bus subscriber and the buffer becomes
|
|
23
|
+
* a transparent cache. This keeps Phase 2 deliverable without a hard
|
|
24
|
+
* cost-tracker dependency.
|
|
25
|
+
*
|
|
26
|
+
* Security invariants:
|
|
27
|
+
*
|
|
28
|
+
* 1. Negative tokens/usd are clamped to 0 — a malicious receive-side
|
|
29
|
+
* reporter cannot drive an "unsuspend" through credit inflation.
|
|
30
|
+
* 2. Sample buffer is bounded per-peer (DEFAULT_MAX_SAMPLES_PER_PEER) so
|
|
31
|
+
* a high-throughput peer cannot exhaust memory.
|
|
32
|
+
* 3. Policy evaluation is order-independent for SUSPEND triggers —
|
|
33
|
+
* cost OR failure-ratio fires the same SUSPEND. The reason field
|
|
34
|
+
* records which one tripped.
|
|
35
|
+
* 4. evaluatePolicy is pure & deterministic given (samples, policy, now)
|
|
36
|
+
* — no Date.now() calls inside, no random IDs, no hidden state.
|
|
37
|
+
* 5. Auto-eviction only runs from SUSPENDED state and only after the
|
|
38
|
+
* grace age. Cannot skip SUSPENDED for an ACTIVE → EVICTED jump.
|
|
39
|
+
*/
|
|
40
|
+
import { FederationNode } from '../domain/entities/federation-node.js';
|
|
41
|
+
import { SuspensionReason } from '../domain/value-objects/federation-node-state.js';
|
|
42
|
+
/** A single send outcome recorded by the coordinator. */
|
|
43
|
+
export interface SendOutcome {
|
|
44
|
+
readonly nodeId: string;
|
|
45
|
+
readonly success: boolean;
|
|
46
|
+
/** Tokens consumed by this leg. Negative values are treated as 0. */
|
|
47
|
+
readonly tokensUsed?: number;
|
|
48
|
+
/** USD spent on this leg. Negative values are treated as 0. */
|
|
49
|
+
readonly usdSpent?: number;
|
|
50
|
+
/** Caller-supplied timestamp (for testability). Defaults to recordOutcome's `now`. */
|
|
51
|
+
readonly at?: Date;
|
|
52
|
+
}
|
|
53
|
+
/** Configurable thresholds. All defaults come from ADR-097 Part 2. */
|
|
54
|
+
export interface BreakerPolicy {
|
|
55
|
+
/** 24h spend cap per peer. Default $5.00. */
|
|
56
|
+
readonly costSuspensionUsd: number;
|
|
57
|
+
/** Window for the cost cap. Default 24h. */
|
|
58
|
+
readonly costWindowMs: number;
|
|
59
|
+
/** Failure ratio that trips the breaker. Default 0.5 (50%). */
|
|
60
|
+
readonly failureRatioThreshold: number;
|
|
61
|
+
/** Window for failure-ratio computation. Default 1h. */
|
|
62
|
+
readonly failureWindowMs: number;
|
|
63
|
+
/** Minimum samples in the failure window before evaluating (anti-noise). Default 10. */
|
|
64
|
+
readonly failureMinSamples: number;
|
|
65
|
+
/** Cooldown before SUSPENDED → reactivate-eligible. Default 30min. */
|
|
66
|
+
readonly cooldownMs: number;
|
|
67
|
+
/** Continuous suspension age before auto-evict. Default 24h. */
|
|
68
|
+
readonly autoEvictionAgeMs: number;
|
|
69
|
+
}
|
|
70
|
+
export declare const DEFAULT_BREAKER_POLICY: BreakerPolicy;
|
|
71
|
+
/** Per-peer sample buffer cap. ~24h of 1-Hz traffic. */
|
|
72
|
+
export declare const DEFAULT_MAX_SAMPLES_PER_PEER = 100000;
|
|
73
|
+
/** A normalized, time-stamped outcome (negatives clamped, `at` resolved). */
|
|
74
|
+
interface NormalizedSample {
|
|
75
|
+
readonly success: boolean;
|
|
76
|
+
readonly tokensUsed: number;
|
|
77
|
+
readonly usdSpent: number;
|
|
78
|
+
readonly at: Date;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* What the breaker decided for a peer at evaluation time. The service then
|
|
82
|
+
* applies the action (calls node.suspend()/evict()/reactivate()) but the
|
|
83
|
+
* decision itself is a pure return so tests can assert it without checking
|
|
84
|
+
* mutated state.
|
|
85
|
+
*/
|
|
86
|
+
export type BreakerDecision = {
|
|
87
|
+
readonly action: 'NONE';
|
|
88
|
+
readonly nodeId: string;
|
|
89
|
+
} | {
|
|
90
|
+
readonly action: 'SUSPEND';
|
|
91
|
+
readonly nodeId: string;
|
|
92
|
+
readonly reason: SuspensionReason;
|
|
93
|
+
} | {
|
|
94
|
+
readonly action: 'EVICT';
|
|
95
|
+
readonly nodeId: string;
|
|
96
|
+
readonly reason: SuspensionReason;
|
|
97
|
+
} | {
|
|
98
|
+
readonly action: 'REACTIVATE_ELIGIBLE';
|
|
99
|
+
readonly nodeId: string;
|
|
100
|
+
};
|
|
101
|
+
/**
|
|
102
|
+
* Pure policy evaluator. Returns the decision the breaker should apply.
|
|
103
|
+
* No I/O, no clock, no entity mutation — given the same inputs, returns the
|
|
104
|
+
* same output. The service layer is responsible for calling node.suspend()
|
|
105
|
+
* / evict() / reactivate() based on this.
|
|
106
|
+
*
|
|
107
|
+
* Decision rules:
|
|
108
|
+
*
|
|
109
|
+
* For ACTIVE peers:
|
|
110
|
+
* - 24h cost > policy.costSuspensionUsd → SUSPEND/COST_THRESHOLD_EXCEEDED
|
|
111
|
+
* - 1h failure ratio > policy.failureRatioThreshold (≥ failureMinSamples
|
|
112
|
+
* in window) → SUSPEND/FAILURE_RATIO_EXCEEDED
|
|
113
|
+
* - else NONE
|
|
114
|
+
*
|
|
115
|
+
* For SUSPENDED peers:
|
|
116
|
+
* - suspended ≥ autoEvictionAgeMs → EVICT/GRACE_PERIOD_EXPIRED
|
|
117
|
+
* - cooldown elapsed → REACTIVATE_ELIGIBLE (caller probes + reactivates)
|
|
118
|
+
* - else NONE
|
|
119
|
+
*
|
|
120
|
+
* For EVICTED peers: always NONE (terminal under breaker flow).
|
|
121
|
+
*
|
|
122
|
+
* Cost takes priority over failure-ratio when both trip, because cost is
|
|
123
|
+
* the more conservative signal (a malicious peer can manufacture failures
|
|
124
|
+
* by trickling errors but cannot manufacture spend without actually
|
|
125
|
+
* burning budget).
|
|
126
|
+
*/
|
|
127
|
+
export declare function evaluatePolicy(samples: readonly NormalizedSample[], node: FederationNode, policy: BreakerPolicy, now: Date): BreakerDecision;
|
|
128
|
+
/**
|
|
129
|
+
* Stateful service: owns the per-peer sample buffer, applies decisions to
|
|
130
|
+
* entities, and exposes the breaker control surface to the coordinator.
|
|
131
|
+
*
|
|
132
|
+
* Construct once per federation coordinator. Inject the policy if you need
|
|
133
|
+
* non-default thresholds (Phase 4 doctor surface will let users tune them).
|
|
134
|
+
*/
|
|
135
|
+
export declare class FederationBreakerService {
|
|
136
|
+
private readonly policy;
|
|
137
|
+
private readonly samples;
|
|
138
|
+
private readonly maxSamplesPerPeer;
|
|
139
|
+
constructor(policy?: BreakerPolicy, maxSamplesPerPeer?: number);
|
|
140
|
+
/** Read the active policy (handy for tests + doctor surface). */
|
|
141
|
+
getPolicy(): BreakerPolicy;
|
|
142
|
+
/**
|
|
143
|
+
* Record a send outcome for a peer. Called by the coordinator after each
|
|
144
|
+
* `routing.send` completion. Returns a BreakerDecision describing what,
|
|
145
|
+
* if anything, the breaker would do for this peer right now — but does
|
|
146
|
+
* NOT apply it (the caller asks `evaluate(node)` if they want the action
|
|
147
|
+
* to actually happen). This split lets the coordinator decide whether to
|
|
148
|
+
* apply on every outcome or batch via tick.
|
|
149
|
+
*
|
|
150
|
+
* Negative tokens/usd are clamped to 0. A peer cannot inflate its own
|
|
151
|
+
* remaining cost ceiling by reporting credits.
|
|
152
|
+
*/
|
|
153
|
+
recordOutcome(outcome: SendOutcome, now?: Date): void;
|
|
154
|
+
/**
|
|
155
|
+
* Evaluate the breaker for one peer and apply the resulting transition
|
|
156
|
+
* to the entity. Returns the decision so callers can observe the outcome.
|
|
157
|
+
*
|
|
158
|
+
* `apply` defaults to true — set to false to get the dry-run decision
|
|
159
|
+
* without mutating the entity (useful for the doctor surface).
|
|
160
|
+
*/
|
|
161
|
+
evaluate(node: FederationNode, now?: Date, apply?: boolean): BreakerDecision;
|
|
162
|
+
/**
|
|
163
|
+
* Periodic tick — evaluate all known peers. Intended to be called from a
|
|
164
|
+
* timer or after each batch of outcomes. Returns the decisions for
|
|
165
|
+
* observability.
|
|
166
|
+
*/
|
|
167
|
+
tick(nodes: readonly FederationNode[], now?: Date): BreakerDecision[];
|
|
168
|
+
/** Drop a peer's sample buffer (e.g. after reactivate or evict). */
|
|
169
|
+
forget(nodeId: string): void;
|
|
170
|
+
/**
|
|
171
|
+
* Snapshot of per-peer aggregates for the doctor surface. Pure read —
|
|
172
|
+
* does not mutate the buffer.
|
|
173
|
+
*/
|
|
174
|
+
snapshot(nodeId: string, now?: Date): {
|
|
175
|
+
readonly nodeId: string;
|
|
176
|
+
readonly sampleCount: number;
|
|
177
|
+
readonly cumUsdInWindow: number;
|
|
178
|
+
readonly failureRatioInWindow: number | null;
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
export {};
|
|
182
|
+
//# sourceMappingURL=federation-breaker-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"federation-breaker-service.d.ts","sourceRoot":"","sources":["../../src/application/federation-breaker-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAEL,gBAAgB,EAMjB,MAAM,kDAAkD,CAAC;AAE1D,yDAAyD;AACzD,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,qEAAqE;IACrE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,+DAA+D;IAC/D,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,sFAAsF;IACtF,QAAQ,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC;CACpB;AAED,sEAAsE;AACtE,MAAM,WAAW,aAAa;IAC5B,6CAA6C;IAC7C,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,4CAA4C;IAC5C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,+DAA+D;IAC/D,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,wDAAwD;IACxD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,wFAAwF;IACxF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,sEAAsE;IACtE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,gEAAgE;IAChE,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACpC;AAED,eAAO,MAAM,sBAAsB,EAAE,aAQnC,CAAC;AAEH,wDAAwD;AACxD,eAAO,MAAM,4BAA4B,SAAU,CAAC;AAEpD,6EAA6E;AAC7E,UAAU,gBAAgB;IACxB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GACvB;IAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACpD;IACE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;CACnC,GACD;IACE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;CACnC,GACD;IACE,QAAQ,CAAC,MAAM,EAAE,qBAAqB,CAAC;IACvC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEN;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,SAAS,gBAAgB,EAAE,EACpC,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,aAAa,EACrB,GAAG,EAAE,IAAI,GACR,eAAe,CAqDjB;AAED;;;;;;GAMG;AACH,qBAAa,wBAAwB;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;gBAGzC,MAAM,GAAE,aAAsC,EAC9C,iBAAiB,GAAE,MAAqC;IAO1D,iEAAiE;IACjE,SAAS,IAAI,aAAa;IAI1B;;;;;;;;;;OAUG;IACH,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,GAAE,IAAiB,GAAG,IAAI;IAkBjE;;;;;;OAMG;IACH,QAAQ,CACN,IAAI,EAAE,cAAc,EACpB,GAAG,GAAE,IAAiB,EACtB,KAAK,GAAE,OAAc,GACpB,eAAe;IA6BlB;;;;OAIG;IACH,IAAI,CAAC,KAAK,EAAE,SAAS,cAAc,EAAE,EAAE,GAAG,GAAE,IAAiB,GAAG,eAAe,EAAE;IAIjF,oEAAoE;IACpE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAI5B;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,GAAE,IAAiB,GAAG;QAChD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;QAChC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9C;CAqBF"}
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Federation circuit breaker service (ADR-097 Phase 2.b).
|
|
3
|
+
*
|
|
4
|
+
* Phase 2.a shipped the FederationNode state machine + canTransition table.
|
|
5
|
+
* This service is what drives transitions: it records per-peer send outcomes
|
|
6
|
+
* (success/failure + spend), evaluates thresholds against the configured
|
|
7
|
+
* policy, and calls suspend()/evict()/reactivate() on the entity.
|
|
8
|
+
*
|
|
9
|
+
* Architecture is two-layer to keep the policy testable in isolation:
|
|
10
|
+
*
|
|
11
|
+
* 1. **evaluatePolicy** (pure function) — given rolling samples for a peer
|
|
12
|
+
* + the policy + now, returns a BreakerDecision describing what should
|
|
13
|
+
* happen. No I/O, no clock, no entity mutation.
|
|
14
|
+
* 2. **FederationBreakerService** — owns the rolling-sample buffer,
|
|
15
|
+
* wraps evaluatePolicy with the entity mutation step, and exposes
|
|
16
|
+
* `recordOutcome` (called by the coordinator on each send completion)
|
|
17
|
+
* + `tick` (called periodically to drive cooldown/auto-evict).
|
|
18
|
+
*
|
|
19
|
+
* The cost-tracker integration (Phase 3 upstream — emits federation_spend
|
|
20
|
+
* events when sends complete) is *not* wired here; this service holds its
|
|
21
|
+
* own in-memory rolling buffer. When Phase 3 upstream lands, the consumer
|
|
22
|
+
* can call `recordOutcome` from the bus subscriber and the buffer becomes
|
|
23
|
+
* a transparent cache. This keeps Phase 2 deliverable without a hard
|
|
24
|
+
* cost-tracker dependency.
|
|
25
|
+
*
|
|
26
|
+
* Security invariants:
|
|
27
|
+
*
|
|
28
|
+
* 1. Negative tokens/usd are clamped to 0 — a malicious receive-side
|
|
29
|
+
* reporter cannot drive an "unsuspend" through credit inflation.
|
|
30
|
+
* 2. Sample buffer is bounded per-peer (DEFAULT_MAX_SAMPLES_PER_PEER) so
|
|
31
|
+
* a high-throughput peer cannot exhaust memory.
|
|
32
|
+
* 3. Policy evaluation is order-independent for SUSPEND triggers —
|
|
33
|
+
* cost OR failure-ratio fires the same SUSPEND. The reason field
|
|
34
|
+
* records which one tripped.
|
|
35
|
+
* 4. evaluatePolicy is pure & deterministic given (samples, policy, now)
|
|
36
|
+
* — no Date.now() calls inside, no random IDs, no hidden state.
|
|
37
|
+
* 5. Auto-eviction only runs from SUSPENDED state and only after the
|
|
38
|
+
* grace age. Cannot skip SUSPENDED for an ACTIVE → EVICTED jump.
|
|
39
|
+
*/
|
|
40
|
+
import { FederationNodeState, isCooldownElapsed, shouldAutoEvict, DEFAULT_SUSPENSION_COOLDOWN_MS, DEFAULT_AUTO_EVICTION_AGE_MS, } from '../domain/value-objects/federation-node-state.js';
|
|
41
|
+
export const DEFAULT_BREAKER_POLICY = Object.freeze({
|
|
42
|
+
costSuspensionUsd: 5.0,
|
|
43
|
+
costWindowMs: 24 * 60 * 60 * 1000,
|
|
44
|
+
failureRatioThreshold: 0.5,
|
|
45
|
+
failureWindowMs: 60 * 60 * 1000,
|
|
46
|
+
failureMinSamples: 10,
|
|
47
|
+
cooldownMs: DEFAULT_SUSPENSION_COOLDOWN_MS,
|
|
48
|
+
autoEvictionAgeMs: DEFAULT_AUTO_EVICTION_AGE_MS,
|
|
49
|
+
});
|
|
50
|
+
/** Per-peer sample buffer cap. ~24h of 1-Hz traffic. */
|
|
51
|
+
export const DEFAULT_MAX_SAMPLES_PER_PEER = 100_000;
|
|
52
|
+
/**
|
|
53
|
+
* Pure policy evaluator. Returns the decision the breaker should apply.
|
|
54
|
+
* No I/O, no clock, no entity mutation — given the same inputs, returns the
|
|
55
|
+
* same output. The service layer is responsible for calling node.suspend()
|
|
56
|
+
* / evict() / reactivate() based on this.
|
|
57
|
+
*
|
|
58
|
+
* Decision rules:
|
|
59
|
+
*
|
|
60
|
+
* For ACTIVE peers:
|
|
61
|
+
* - 24h cost > policy.costSuspensionUsd → SUSPEND/COST_THRESHOLD_EXCEEDED
|
|
62
|
+
* - 1h failure ratio > policy.failureRatioThreshold (≥ failureMinSamples
|
|
63
|
+
* in window) → SUSPEND/FAILURE_RATIO_EXCEEDED
|
|
64
|
+
* - else NONE
|
|
65
|
+
*
|
|
66
|
+
* For SUSPENDED peers:
|
|
67
|
+
* - suspended ≥ autoEvictionAgeMs → EVICT/GRACE_PERIOD_EXPIRED
|
|
68
|
+
* - cooldown elapsed → REACTIVATE_ELIGIBLE (caller probes + reactivates)
|
|
69
|
+
* - else NONE
|
|
70
|
+
*
|
|
71
|
+
* For EVICTED peers: always NONE (terminal under breaker flow).
|
|
72
|
+
*
|
|
73
|
+
* Cost takes priority over failure-ratio when both trip, because cost is
|
|
74
|
+
* the more conservative signal (a malicious peer can manufacture failures
|
|
75
|
+
* by trickling errors but cannot manufacture spend without actually
|
|
76
|
+
* burning budget).
|
|
77
|
+
*/
|
|
78
|
+
export function evaluatePolicy(samples, node, policy, now) {
|
|
79
|
+
if (node.state === FederationNodeState.EVICTED) {
|
|
80
|
+
return { action: 'NONE', nodeId: node.nodeId };
|
|
81
|
+
}
|
|
82
|
+
if (node.state === FederationNodeState.SUSPENDED) {
|
|
83
|
+
if (shouldAutoEvict(node.stateChangedAt, now, policy.autoEvictionAgeMs)) {
|
|
84
|
+
return {
|
|
85
|
+
action: 'EVICT',
|
|
86
|
+
nodeId: node.nodeId,
|
|
87
|
+
reason: 'GRACE_PERIOD_EXPIRED',
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
if (isCooldownElapsed(node.stateChangedAt, now, policy.cooldownMs)) {
|
|
91
|
+
return { action: 'REACTIVATE_ELIGIBLE', nodeId: node.nodeId };
|
|
92
|
+
}
|
|
93
|
+
return { action: 'NONE', nodeId: node.nodeId };
|
|
94
|
+
}
|
|
95
|
+
// ACTIVE: check cost first, then failure ratio. Cost takes priority.
|
|
96
|
+
const costCutoff = now.getTime() - policy.costWindowMs;
|
|
97
|
+
let cumUsd = 0;
|
|
98
|
+
for (const s of samples) {
|
|
99
|
+
if (s.at.getTime() >= costCutoff)
|
|
100
|
+
cumUsd += s.usdSpent;
|
|
101
|
+
}
|
|
102
|
+
if (cumUsd > policy.costSuspensionUsd) {
|
|
103
|
+
return {
|
|
104
|
+
action: 'SUSPEND',
|
|
105
|
+
nodeId: node.nodeId,
|
|
106
|
+
reason: 'COST_THRESHOLD_EXCEEDED',
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
const failCutoff = now.getTime() - policy.failureWindowMs;
|
|
110
|
+
let total = 0;
|
|
111
|
+
let failures = 0;
|
|
112
|
+
for (const s of samples) {
|
|
113
|
+
if (s.at.getTime() < failCutoff)
|
|
114
|
+
continue;
|
|
115
|
+
total++;
|
|
116
|
+
if (!s.success)
|
|
117
|
+
failures++;
|
|
118
|
+
}
|
|
119
|
+
if (total >= policy.failureMinSamples) {
|
|
120
|
+
const ratio = failures / total;
|
|
121
|
+
if (ratio > policy.failureRatioThreshold) {
|
|
122
|
+
return {
|
|
123
|
+
action: 'SUSPEND',
|
|
124
|
+
nodeId: node.nodeId,
|
|
125
|
+
reason: 'FAILURE_RATIO_EXCEEDED',
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return { action: 'NONE', nodeId: node.nodeId };
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Stateful service: owns the per-peer sample buffer, applies decisions to
|
|
133
|
+
* entities, and exposes the breaker control surface to the coordinator.
|
|
134
|
+
*
|
|
135
|
+
* Construct once per federation coordinator. Inject the policy if you need
|
|
136
|
+
* non-default thresholds (Phase 4 doctor surface will let users tune them).
|
|
137
|
+
*/
|
|
138
|
+
export class FederationBreakerService {
|
|
139
|
+
policy;
|
|
140
|
+
samples;
|
|
141
|
+
maxSamplesPerPeer;
|
|
142
|
+
constructor(policy = DEFAULT_BREAKER_POLICY, maxSamplesPerPeer = DEFAULT_MAX_SAMPLES_PER_PEER) {
|
|
143
|
+
this.policy = policy;
|
|
144
|
+
this.samples = new Map();
|
|
145
|
+
this.maxSamplesPerPeer = maxSamplesPerPeer;
|
|
146
|
+
}
|
|
147
|
+
/** Read the active policy (handy for tests + doctor surface). */
|
|
148
|
+
getPolicy() {
|
|
149
|
+
return this.policy;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Record a send outcome for a peer. Called by the coordinator after each
|
|
153
|
+
* `routing.send` completion. Returns a BreakerDecision describing what,
|
|
154
|
+
* if anything, the breaker would do for this peer right now — but does
|
|
155
|
+
* NOT apply it (the caller asks `evaluate(node)` if they want the action
|
|
156
|
+
* to actually happen). This split lets the coordinator decide whether to
|
|
157
|
+
* apply on every outcome or batch via tick.
|
|
158
|
+
*
|
|
159
|
+
* Negative tokens/usd are clamped to 0. A peer cannot inflate its own
|
|
160
|
+
* remaining cost ceiling by reporting credits.
|
|
161
|
+
*/
|
|
162
|
+
recordOutcome(outcome, now = new Date()) {
|
|
163
|
+
const normalized = {
|
|
164
|
+
success: outcome.success,
|
|
165
|
+
tokensUsed: Math.max(0, outcome.tokensUsed ?? 0),
|
|
166
|
+
usdSpent: Math.max(0, outcome.usdSpent ?? 0),
|
|
167
|
+
at: outcome.at ?? now,
|
|
168
|
+
};
|
|
169
|
+
const buf = this.samples.get(outcome.nodeId) ?? [];
|
|
170
|
+
buf.push(normalized);
|
|
171
|
+
// Bounded buffer: drop oldest if over cap. Keeps memory bounded under
|
|
172
|
+
// sustained high traffic while preserving the most-recent window.
|
|
173
|
+
if (buf.length > this.maxSamplesPerPeer) {
|
|
174
|
+
buf.splice(0, buf.length - this.maxSamplesPerPeer);
|
|
175
|
+
}
|
|
176
|
+
this.samples.set(outcome.nodeId, buf);
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Evaluate the breaker for one peer and apply the resulting transition
|
|
180
|
+
* to the entity. Returns the decision so callers can observe the outcome.
|
|
181
|
+
*
|
|
182
|
+
* `apply` defaults to true — set to false to get the dry-run decision
|
|
183
|
+
* without mutating the entity (useful for the doctor surface).
|
|
184
|
+
*/
|
|
185
|
+
evaluate(node, now = new Date(), apply = true) {
|
|
186
|
+
const buf = this.samples.get(node.nodeId) ?? [];
|
|
187
|
+
const decision = evaluatePolicy(buf, node, this.policy, now);
|
|
188
|
+
if (!apply)
|
|
189
|
+
return decision;
|
|
190
|
+
switch (decision.action) {
|
|
191
|
+
case 'SUSPEND': {
|
|
192
|
+
const reason = { reason: decision.reason };
|
|
193
|
+
node.suspend(reason, now);
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
case 'EVICT': {
|
|
197
|
+
const reason = { reason: decision.reason };
|
|
198
|
+
node.evict(reason, now);
|
|
199
|
+
break;
|
|
200
|
+
}
|
|
201
|
+
case 'REACTIVATE_ELIGIBLE':
|
|
202
|
+
// The breaker does NOT auto-reactivate. The integrator's health
|
|
203
|
+
// probe is responsible for confirming the peer is healthy and
|
|
204
|
+
// calling node.reactivate() itself. We only signal eligibility.
|
|
205
|
+
break;
|
|
206
|
+
case 'NONE':
|
|
207
|
+
break;
|
|
208
|
+
}
|
|
209
|
+
return decision;
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Periodic tick — evaluate all known peers. Intended to be called from a
|
|
213
|
+
* timer or after each batch of outcomes. Returns the decisions for
|
|
214
|
+
* observability.
|
|
215
|
+
*/
|
|
216
|
+
tick(nodes, now = new Date()) {
|
|
217
|
+
return nodes.map((n) => this.evaluate(n, now));
|
|
218
|
+
}
|
|
219
|
+
/** Drop a peer's sample buffer (e.g. after reactivate or evict). */
|
|
220
|
+
forget(nodeId) {
|
|
221
|
+
this.samples.delete(nodeId);
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Snapshot of per-peer aggregates for the doctor surface. Pure read —
|
|
225
|
+
* does not mutate the buffer.
|
|
226
|
+
*/
|
|
227
|
+
snapshot(nodeId, now = new Date()) {
|
|
228
|
+
const buf = this.samples.get(nodeId) ?? [];
|
|
229
|
+
const costCutoff = now.getTime() - this.policy.costWindowMs;
|
|
230
|
+
const failCutoff = now.getTime() - this.policy.failureWindowMs;
|
|
231
|
+
let cumUsd = 0;
|
|
232
|
+
let total = 0;
|
|
233
|
+
let failures = 0;
|
|
234
|
+
for (const s of buf) {
|
|
235
|
+
if (s.at.getTime() >= costCutoff)
|
|
236
|
+
cumUsd += s.usdSpent;
|
|
237
|
+
if (s.at.getTime() >= failCutoff) {
|
|
238
|
+
total++;
|
|
239
|
+
if (!s.success)
|
|
240
|
+
failures++;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return {
|
|
244
|
+
nodeId,
|
|
245
|
+
sampleCount: buf.length,
|
|
246
|
+
cumUsdInWindow: cumUsd,
|
|
247
|
+
failureRatioInWindow: total > 0 ? failures / total : null,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
//# sourceMappingURL=federation-breaker-service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"federation-breaker-service.js","sourceRoot":"","sources":["../../src/application/federation-breaker-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAGH,OAAO,EACL,mBAAmB,EAGnB,iBAAiB,EACjB,eAAe,EACf,8BAA8B,EAC9B,4BAA4B,GAC7B,MAAM,kDAAkD,CAAC;AAgC1D,MAAM,CAAC,MAAM,sBAAsB,GAAkB,MAAM,CAAC,MAAM,CAAC;IACjE,iBAAiB,EAAE,GAAG;IACtB,YAAY,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;IACjC,qBAAqB,EAAE,GAAG;IAC1B,eAAe,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI;IAC/B,iBAAiB,EAAE,EAAE;IACrB,UAAU,EAAE,8BAA8B;IAC1C,iBAAiB,EAAE,4BAA4B;CAChD,CAAC,CAAC;AAEH,wDAAwD;AACxD,MAAM,CAAC,MAAM,4BAA4B,GAAG,OAAO,CAAC;AAiCpD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,cAAc,CAC5B,OAAoC,EACpC,IAAoB,EACpB,MAAqB,EACrB,GAAS;IAET,IAAI,IAAI,CAAC,KAAK,KAAK,mBAAmB,CAAC,OAAO,EAAE,CAAC;QAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IACjD,CAAC;IAED,IAAI,IAAI,CAAC,KAAK,KAAK,mBAAmB,CAAC,SAAS,EAAE,CAAC;QACjD,IAAI,eAAe,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,EAAE,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACxE,OAAO;gBACL,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,sBAAsB;aAC/B,CAAC;QACJ,CAAC;QACD,IAAI,iBAAiB,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;YACnE,OAAO,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QAChE,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IACjD,CAAC;IAED,qEAAqE;IACrE,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC;IACvD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,UAAU;YAAE,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC;IACzD,CAAC;IACD,IAAI,MAAM,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC;QACtC,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,yBAAyB;SAClC,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,eAAe,CAAC;IAC1D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,UAAU;YAAE,SAAS;QAC1C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,CAAC,CAAC,OAAO;YAAE,QAAQ,EAAE,CAAC;IAC7B,CAAC;IACD,IAAI,KAAK,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,QAAQ,GAAG,KAAK,CAAC;QAC/B,IAAI,KAAK,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;YACzC,OAAO;gBACL,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,wBAAwB;aACjC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;AACjD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,OAAO,wBAAwB;IAClB,MAAM,CAAgB;IACtB,OAAO,CAAkC;IACzC,iBAAiB,CAAS;IAE3C,YACE,SAAwB,sBAAsB,EAC9C,oBAA4B,4BAA4B;QAExD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;IAC7C,CAAC;IAED,iEAAiE;IACjE,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;;;;;;;;OAUG;IACH,aAAa,CAAC,OAAoB,EAAE,MAAY,IAAI,IAAI,EAAE;QACxD,MAAM,UAAU,GAAqB;YACnC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;YAChD,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;YAC5C,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,GAAG;SACtB,CAAC;QAEF,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACnD,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACrB,sEAAsE;QACtE,kEAAkE;QAClE,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACxC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACxC,CAAC;IAED;;;;;;OAMG;IACH,QAAQ,CACN,IAAoB,EACpB,MAAY,IAAI,IAAI,EAAE,EACtB,QAAiB,IAAI;QAErB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAE7D,IAAI,CAAC,KAAK;YAAE,OAAO,QAAQ,CAAC;QAE5B,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC;YACxB,KAAK,SAAS,CAAC,CAAC,CAAC;gBACf,MAAM,MAAM,GAAqB,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAC7D,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC1B,MAAM;YACR,CAAC;YACD,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,MAAM,GAAqB,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAC7D,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,KAAK,qBAAqB;gBACxB,gEAAgE;gBAChE,8DAA8D;gBAC9D,gEAAgE;gBAChE,MAAM;YACR,KAAK,MAAM;gBACT,MAAM;QACV,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;;OAIG;IACH,IAAI,CAAC,KAAgC,EAAE,MAAY,IAAI,IAAI,EAAE;QAC3D,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,oEAAoE;IACpE,MAAM,CAAC,MAAc;QACnB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,MAAc,EAAE,MAAY,IAAI,IAAI,EAAE;QAM7C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3C,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;QAC5D,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC;QAC/D,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;YACpB,IAAI,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,UAAU;gBAAE,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC;YACvD,IAAI,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,UAAU,EAAE,CAAC;gBACjC,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,CAAC,CAAC,OAAO;oBAAE,QAAQ,EAAE,CAAC;YAC7B,CAAC;QACH,CAAC;QACD,OAAO;YACL,MAAM;YACN,WAAW,EAAE,GAAG,CAAC,MAAM;YACvB,cAAc,EAAE,MAAM;YACtB,oBAAoB,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI;SAC1D,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -8,6 +8,31 @@ import { AuditService } from '../domain/services/audit-service.js';
|
|
|
8
8
|
import { PIIPipelineService } from '../domain/services/pii-pipeline-service.js';
|
|
9
9
|
import { TrustEvaluator } from './trust-evaluator.js';
|
|
10
10
|
import { PolicyEngine } from './policy-engine.js';
|
|
11
|
+
import { enforceBudget, validateBudget } from '../domain/value-objects/federation-budget.js';
|
|
12
|
+
/**
|
|
13
|
+
* Optional per-call budget controls (ADR-097 Phase 1). All fields are
|
|
14
|
+
* optional; omitted means "no limit on that axis" (the default unbounded
|
|
15
|
+
* budget still has maxHops = 8 to defang recursive delegation loops).
|
|
16
|
+
*/
|
|
17
|
+
export interface SendOptions {
|
|
18
|
+
readonly budget?: {
|
|
19
|
+
maxTokens?: number;
|
|
20
|
+
maxUsd?: number;
|
|
21
|
+
};
|
|
22
|
+
readonly maxHops?: number;
|
|
23
|
+
/**
|
|
24
|
+
* How many hops this message has already traveled (0 on the originator).
|
|
25
|
+
* Phase 1 enforces at the send side because there is no inbound dispatcher
|
|
26
|
+
* yet; callers re-forwarding a received message should pass the original
|
|
27
|
+
* envelope's hopCount here.
|
|
28
|
+
*/
|
|
29
|
+
readonly hopCount?: number;
|
|
30
|
+
/** Caller-reported usage from the previous leg, for cumulative checks. */
|
|
31
|
+
readonly spent?: {
|
|
32
|
+
tokens?: number;
|
|
33
|
+
usd?: number;
|
|
34
|
+
};
|
|
35
|
+
}
|
|
11
36
|
export interface FederationCoordinatorConfig {
|
|
12
37
|
readonly nodeId: string;
|
|
13
38
|
readonly publicKey: string;
|
|
@@ -37,7 +62,16 @@ export declare class FederationCoordinator {
|
|
|
37
62
|
shutdown(): Promise<void>;
|
|
38
63
|
joinPeer(endpoint: string): Promise<FederationSession>;
|
|
39
64
|
leavePeer(nodeId: string): Promise<void>;
|
|
40
|
-
sendMessage<T>(targetNodeId: string, messageType: FederationMessageType, payload: T): Promise<RoutingResult>;
|
|
65
|
+
sendMessage<T>(targetNodeId: string, messageType: FederationMessageType, payload: T, options?: SendOptions): Promise<RoutingResult>;
|
|
66
|
+
/**
|
|
67
|
+
* Re-export the budget primitives so external callers (e.g. follow-up
|
|
68
|
+
* iterations adding receive-side decrement) can reuse them without
|
|
69
|
+
* dipping into domain/value-objects directly.
|
|
70
|
+
*/
|
|
71
|
+
static readonly budget: {
|
|
72
|
+
validate: typeof validateBudget;
|
|
73
|
+
enforce: typeof enforceBudget;
|
|
74
|
+
};
|
|
41
75
|
broadcastMessage<T>(messageType: FederationMessageType, payload: T): Promise<RoutingResult[]>;
|
|
42
76
|
handleThreatDetection(nodeId: string): void;
|
|
43
77
|
handleHmacFailure(nodeId: string): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"federation-coordinator.d.ts","sourceRoot":"","sources":["../../src/application/federation-coordinator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAE,KAAK,qBAAqB,EAAE,MAAM,2CAA2C,CAAC;AACvF,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,yCAAyC,CAAC;AACpG,OAAO,EAAE,gBAAgB,EAAE,MAAM,yCAAyC,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAC3F,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4CAA4C,CAAC;AAChF,OAAO,EAAE,cAAc,EAAiC,MAAM,sBAAsB,CAAC;AACrF,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"federation-coordinator.d.ts","sourceRoot":"","sources":["../../src/application/federation-coordinator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAE,KAAK,qBAAqB,EAAE,MAAM,2CAA2C,CAAC;AACvF,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,yCAAyC,CAAC;AACpG,OAAO,EAAE,gBAAgB,EAAE,MAAM,yCAAyC,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAC3F,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4CAA4C,CAAC;AAChF,OAAO,EAAE,cAAc,EAAiC,MAAM,sBAAsB,CAAC;AACrF,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAGL,aAAa,EACb,cAAc,EACf,MAAM,8CAA8C,CAAC;AAEtD;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1D,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;OAKG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,0EAA0E;IAC1E,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACpD;AAED,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1C;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACjD,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA8B;IACrD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiC;IAC1D,OAAO,CAAC,WAAW,CAAU;gBAG3B,MAAM,EAAE,2BAA2B,EACnC,SAAS,EAAE,gBAAgB,EAC3B,SAAS,EAAE,gBAAgB,EAC3B,OAAO,EAAE,cAAc,EACvB,KAAK,EAAE,YAAY,EACnB,WAAW,EAAE,kBAAkB,EAC/B,cAAc,EAAE,cAAc,EAC9B,YAAY,EAAE,YAAY;IActB,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAU1E,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAatD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBxC,WAAW,CAAC,CAAC,EACjB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,qBAAqB,EAClC,OAAO,EAAE,CAAC,EACV,OAAO,GAAE,WAAgB,GACxB,OAAO,CAAC,aAAa,CAAC;IAqHzB;;;;OAIG;IACH,MAAM,CAAC,QAAQ,CAAC,MAAM;;;MAAwD;IAExE,gBAAgB,CAAC,CAAC,EACtB,WAAW,EAAE,qBAAqB,EAClC,OAAO,EAAE,CAAC,GACT,OAAO,CAAC,aAAa,EAAE,CAAC;IAK3B,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAqB3C,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAkBvC,SAAS,IAAI,gBAAgB;IAgB7B,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS;IAI5D,iBAAiB,IAAI,iBAAiB,EAAE;YAI1B,gBAAgB;IAgC9B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,iBAAiB;CAK1B"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { TrustLevel } from '../domain/entities/trust-level.js';
|
|
2
|
+
import { enforceBudget, validateBudget, } from '../domain/value-objects/federation-budget.js';
|
|
2
3
|
export class FederationCoordinator {
|
|
3
4
|
config;
|
|
4
5
|
discovery;
|
|
@@ -65,8 +66,29 @@ export class FederationCoordinator {
|
|
|
65
66
|
}
|
|
66
67
|
this.discovery.removePeer(nodeId);
|
|
67
68
|
}
|
|
68
|
-
async sendMessage(targetNodeId, messageType, payload) {
|
|
69
|
+
async sendMessage(targetNodeId, messageType, payload, options = {}) {
|
|
69
70
|
this.ensureInitialized();
|
|
71
|
+
// ADR-097 Phase 2.b: outbound short-circuit on tripped breaker. If the
|
|
72
|
+
// peer is SUSPENDED or EVICTED we refuse the send before doing any work,
|
|
73
|
+
// and emit a constant-string error (no remaining-budget echo per the
|
|
74
|
+
// anti-oracle posture inherited from Phase 1). The check fires *before*
|
|
75
|
+
// session lookup so an evicted peer cannot consume cycles even on a
|
|
76
|
+
// dangling session reference.
|
|
77
|
+
const peer = this.discovery.getPeer(targetNodeId);
|
|
78
|
+
if (peer && !peer.isActive) {
|
|
79
|
+
const reason = peer.isEvicted ? 'PEER_EVICTED' : 'PEER_SUSPENDED';
|
|
80
|
+
await this.audit.log('message_rejected', {
|
|
81
|
+
targetNodeId,
|
|
82
|
+
metadata: { reason },
|
|
83
|
+
});
|
|
84
|
+
return {
|
|
85
|
+
success: false,
|
|
86
|
+
mode: 'direct',
|
|
87
|
+
envelopeId: '',
|
|
88
|
+
targetNodeIds: [targetNodeId],
|
|
89
|
+
error: reason,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
70
92
|
const session = this.findSessionByNodeId(targetNodeId);
|
|
71
93
|
if (!session) {
|
|
72
94
|
return {
|
|
@@ -92,15 +114,63 @@ export class FederationCoordinator {
|
|
|
92
114
|
error: policyResult.reason,
|
|
93
115
|
};
|
|
94
116
|
}
|
|
117
|
+
// ADR-097 Phase 1: validate caller-supplied budget + enforce hop limit
|
|
118
|
+
// and cumulative spend BEFORE outbound dispatch. Synchronous & atomic —
|
|
119
|
+
// no awaits between validate and enforce, so concurrent sends cannot
|
|
120
|
+
// both pass a single-hop budget. Errors are constant strings so a
|
|
121
|
+
// malicious caller cannot use the response as a budget oracle.
|
|
122
|
+
const budgetResult = validateBudget(options.budget, options.maxHops);
|
|
123
|
+
if (!budgetResult.ok) {
|
|
124
|
+
await this.audit.log('message_rejected', {
|
|
125
|
+
targetNodeId,
|
|
126
|
+
sessionId: session.sessionId,
|
|
127
|
+
metadata: { reason: 'INVALID_BUDGET', detail: budgetResult.error },
|
|
128
|
+
});
|
|
129
|
+
return {
|
|
130
|
+
success: false,
|
|
131
|
+
mode: 'direct',
|
|
132
|
+
envelopeId: '',
|
|
133
|
+
targetNodeIds: [targetNodeId],
|
|
134
|
+
error: 'INVALID_BUDGET',
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
const enforcement = enforceBudget(budgetResult.budget, options.hopCount ?? 0, {
|
|
138
|
+
tokens: options.spent?.tokens ?? 0,
|
|
139
|
+
usd: options.spent?.usd ?? 0,
|
|
140
|
+
});
|
|
141
|
+
if (!enforcement.ok) {
|
|
142
|
+
await this.audit.log('message_rejected', {
|
|
143
|
+
targetNodeId,
|
|
144
|
+
sessionId: session.sessionId,
|
|
145
|
+
metadata: { reason: enforcement.reason },
|
|
146
|
+
});
|
|
147
|
+
return {
|
|
148
|
+
success: false,
|
|
149
|
+
mode: 'direct',
|
|
150
|
+
envelopeId: '',
|
|
151
|
+
targetNodeIds: [targetNodeId],
|
|
152
|
+
error: enforcement.reason, // constant string — no oracle leak
|
|
153
|
+
};
|
|
154
|
+
}
|
|
95
155
|
const result = await this.routing.send(session, messageType, payload);
|
|
96
156
|
await this.audit.log(result.success ? 'message_sent' : 'message_rejected', {
|
|
97
157
|
targetNodeId,
|
|
98
158
|
sessionId: session.sessionId,
|
|
99
159
|
latencyMs: result.latencyMs,
|
|
100
|
-
metadata: {
|
|
160
|
+
metadata: {
|
|
161
|
+
messageType,
|
|
162
|
+
mode: result.mode,
|
|
163
|
+
hopCount: enforcement.nextHopCount,
|
|
164
|
+
},
|
|
101
165
|
});
|
|
102
166
|
return result;
|
|
103
167
|
}
|
|
168
|
+
/**
|
|
169
|
+
* Re-export the budget primitives so external callers (e.g. follow-up
|
|
170
|
+
* iterations adding receive-side decrement) can reuse them without
|
|
171
|
+
* dipping into domain/value-objects directly.
|
|
172
|
+
*/
|
|
173
|
+
static budget = { validate: validateBudget, enforce: enforceBudget };
|
|
104
174
|
async broadcastMessage(messageType, payload) {
|
|
105
175
|
this.ensureInitialized();
|
|
106
176
|
return this.routing.broadcast(messageType, payload);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"federation-coordinator.js","sourceRoot":"","sources":["../../src/application/federation-coordinator.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"federation-coordinator.js","sourceRoot":"","sources":["../../src/application/federation-coordinator.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAQ/D,OAAO,EAGL,aAAa,EACb,cAAc,GACf,MAAM,8CAA8C,CAAC;AAoCtD,MAAM,OAAO,qBAAqB;IACf,MAAM,CAA8B;IACpC,SAAS,CAAmB;IAC5B,SAAS,CAAmB;IAC5B,OAAO,CAAiB;IACxB,KAAK,CAAe;IACpB,WAAW,CAAqB;IAChC,cAAc,CAAiB;IAC/B,YAAY,CAAe;IAC3B,QAAQ,CAAiC;IAClD,WAAW,CAAU;IAE7B,YACE,MAAmC,EACnC,SAA2B,EAC3B,SAA2B,EAC3B,OAAuB,EACvB,KAAmB,EACnB,WAA+B,EAC/B,cAA8B,EAC9B,YAA0B;QAE1B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,QAA+C;QAC9D,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,CAAC,SAAS,CAAC,sBAAsB,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,yBAAyB,EAAE;YAC9C,QAAQ,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,SAAS,CAAC,qBAAqB,EAAE,CAAC;QAEvC,KAAK,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjD,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,EAAE;gBACzC,SAAS;gBACT,YAAY,EAAE,OAAO,CAAC,YAAY;aACnC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,QAAgB;QAC7B,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAE1D,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,EAAE;YACtC,YAAY,EAAE,IAAI,CAAC,MAAM;YACzB,QAAQ,EAAE,EAAE,QAAQ,EAAE;SACvB,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAc;QAC5B,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACjD,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAExC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,EAAE;gBACzC,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,YAAY,EAAE,MAAM;aACrB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,WAAW,CACf,YAAoB,EACpB,WAAkC,EAClC,OAAU,EACV,UAAuB,EAAE;QAEzB,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,uEAAuE;QACvE,yEAAyE;QACzE,qEAAqE;QACrE,wEAAwE;QACxE,oEAAoE;QACpE,8BAA8B;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAClD,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,gBAAgB,CAAC;YAClE,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,EAAE;gBACvC,YAAY;gBACZ,QAAQ,EAAE,EAAE,MAAM,EAAE;aACrB,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,CAAC,YAAY,CAAC;gBAC7B,KAAK,EAAE,MAAM;aACd,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACvD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,CAAC,YAAY,CAAC;gBAC7B,KAAK,EAAE,+BAA+B,YAAY,EAAE;aACrD,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,eAAe,CACpD,WAAW,EACX,OAAO,CAAC,UAAU,EAClB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,EAC9B,IAAI,CAAC,MAAM,CAAC,MAAM,CACnB,CAAC;QAEF,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,EAAE;gBACvC,YAAY;gBACZ,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,QAAQ,EAAE,EAAE,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE;aAC1C,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,CAAC,YAAY,CAAC;gBAC7B,KAAK,EAAE,YAAY,CAAC,MAAM;aAC3B,CAAC;QACJ,CAAC;QAED,uEAAuE;QACvE,wEAAwE;QACxE,qEAAqE;QACrE,kEAAkE;QAClE,+DAA+D;QAC/D,MAAM,YAAY,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QACrE,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,EAAE;gBACvC,YAAY;gBACZ,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,QAAQ,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,YAAY,CAAC,KAAK,EAAE;aACnE,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,CAAC,YAAY,CAAC;gBAC7B,KAAK,EAAE,gBAAgB;aACxB,CAAC;QACJ,CAAC;QACD,MAAM,WAAW,GAAsB,aAAa,CAClD,YAAY,CAAC,MAAM,EACnB,OAAO,CAAC,QAAQ,IAAI,CAAC,EACrB;YACE,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC;YAClC,GAAG,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC;SAC7B,CACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,EAAE;gBACvC,YAAY;gBACZ,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,QAAQ,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE;aACzC,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,CAAC,YAAY,CAAC;gBAC7B,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,mCAAmC;aAC/D,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;QAEtE,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,kBAAkB,EAAE;YACzE,YAAY;YACZ,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,QAAQ,EAAE;gBACR,WAAW;gBACX,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,QAAQ,EAAE,WAAW,CAAC,YAAY;aACnC;SACF,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAU,MAAM,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IAE9E,KAAK,CAAC,gBAAgB,CACpB,WAAkC,EAClC,OAAU;QAEV,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACtD,CAAC;IAED,qBAAqB,CAAC,MAAc;QAClC,MAAM,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAE5C,IAAI,IAAI,IAAI,gBAAgB,EAAE,CAAC;YAC7B,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,2BAA2B,CAAC,CAAC;YACjE,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,SAAS,EAAE,CAAC;gBACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC1C,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,EAAE;gBAC/B,YAAY,EAAE,MAAM;gBACpB,UAAU,EAAE,UAAU,CAAC,SAAS;gBAChC,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,EAAE,MAAM,EAAE,2BAA2B,EAAE;aAClD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,MAAc;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,2BAA2B,CAAC,CAAC;YACjE,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,SAAS,EAAE,CAAC;gBACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC1C,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,EAAE;gBAC/B,YAAY,EAAE,MAAM;gBACpB,UAAU,EAAE,UAAU,CAAC,SAAS;gBAChC,QAAQ,EAAE,EAAE,MAAM,EAAE,2BAA2B,EAAE;aAClD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,SAAS;QACP,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;QACzC,MAAM,WAAW,GAA+B,EAAE,CAAC;QACnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;QAC7C,CAAC;QAED,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;YAC1B,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM;YAC/E,UAAU,EAAE,KAAK,CAAC,MAAM;YACxB,WAAW;YACX,OAAO,EAAE,IAAI,CAAC,WAAW;SAC1B,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,SAAiB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IAED,iBAAiB;QACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IACpF,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAoB;QACjD,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,qBAAqB,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3E,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEpE,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACvC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,EAAE;gBACvC,YAAY,EAAE,IAAI,CAAC,MAAM;gBACzB,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE;aAClC,CAAC,CAAC;YACH,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACvD,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAE5D,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,qBAAqB,EAAE;YAC1C,YAAY,EAAE,IAAI,CAAC,MAAM;YACzB,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;YACnC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,UAAU;SACtC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,EAAE;YACtC,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;YACnC,YAAY,EAAE,IAAI,CAAC,MAAM;YACzB,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,UAAU;SACtC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAEO,mBAAmB,CAAC,MAAc;QACxC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAC7C,IAAI,OAAO,CAAC,YAAY,KAAK,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACtD,OAAO,OAAO,CAAC;YACjB,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,iBAAiB;QACvB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC"}
|