@cat-factory/orchestration 0.190.0 → 0.192.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/container/dependencies.d.ts +15 -2
- package/dist/container/dependencies.d.ts.map +1 -1
- package/dist/container/engine-dependent-modules.d.ts.map +1 -1
- package/dist/container/engine-dependent-modules.js +1 -1
- package/dist/container/engine-dependent-modules.js.map +1 -1
- package/dist/container/modules.d.ts +1 -1
- package/dist/container/modules.d.ts.map +1 -1
- package/dist/container/modules.js +26 -1
- package/dist/container/modules.js.map +1 -1
- package/dist/container/platform-modules.d.ts.map +1 -1
- package/dist/container/platform-modules.js +1 -0
- package/dist/container/platform-modules.js.map +1 -1
- package/dist/container.d.ts +1 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -3
- package/dist/index.js.map +1 -1
- package/dist/modules/execution/ExecutionServiceDependencies.d.ts +8 -1
- package/dist/modules/execution/ExecutionServiceDependencies.d.ts.map +1 -1
- package/dist/modules/execution/GateHelperDispatcher.js +1 -1
- package/dist/modules/execution/GateHelperDispatcher.js.map +1 -1
- package/dist/modules/execution/GateStepController.d.ts +15 -0
- package/dist/modules/execution/GateStepController.d.ts.map +1 -1
- package/dist/modules/execution/GateStepController.js +19 -0
- package/dist/modules/execution/GateStepController.js.map +1 -1
- package/dist/modules/execution/HumanTestController.js +1 -1
- package/dist/modules/execution/HumanTestController.js.map +1 -1
- package/dist/modules/execution/PollRunningController.d.ts +8 -0
- package/dist/modules/execution/PollRunningController.d.ts.map +1 -1
- package/dist/modules/execution/PollRunningController.js +14 -0
- package/dist/modules/execution/PollRunningController.js.map +1 -1
- package/dist/modules/execution/RalphController.js +1 -1
- package/dist/modules/execution/RalphController.js.map +1 -1
- package/dist/modules/execution/RunDispatcher.d.ts +6 -20
- package/dist/modules/execution/RunDispatcher.d.ts.map +1 -1
- package/dist/modules/execution/RunDispatcher.js +30 -7
- package/dist/modules/execution/RunDispatcher.js.map +1 -1
- package/dist/modules/execution/RunStateMachine.d.ts.map +1 -1
- package/dist/modules/execution/RunStateMachine.js +12 -0
- package/dist/modules/execution/RunStateMachine.js.map +1 -1
- package/dist/modules/execution/StepGraph.d.ts.map +1 -1
- package/dist/modules/execution/StepGraph.js +13 -1
- package/dist/modules/execution/StepGraph.js.map +1 -1
- package/dist/modules/execution/TesterController.js +2 -2
- package/dist/modules/execution/TesterController.js.map +1 -1
- package/dist/modules/execution/VisualConfirmationController.js +1 -1
- package/dist/modules/execution/VisualConfirmationController.js.map +1 -1
- package/dist/modules/execution/step-fold.logic.d.ts +11 -2
- package/dist/modules/execution/step-fold.logic.d.ts.map +1 -1
- package/dist/modules/execution/step-fold.logic.js +18 -2
- package/dist/modules/execution/step-fold.logic.js.map +1 -1
- package/dist/modules/observability/GateOutcomeRecorder.d.ts +33 -0
- package/dist/modules/observability/GateOutcomeRecorder.d.ts.map +1 -0
- package/dist/modules/observability/GateOutcomeRecorder.js +45 -0
- package/dist/modules/observability/GateOutcomeRecorder.js.map +1 -0
- package/dist/modules/observability/LlmObservabilityService.d.ts +22 -1
- package/dist/modules/observability/LlmObservabilityService.d.ts.map +1 -1
- package/dist/modules/observability/LlmObservabilityService.js +30 -0
- package/dist/modules/observability/LlmObservabilityService.js.map +1 -1
- package/dist/modules/observability/PlatformObservabilityService.d.ts +32 -1
- package/dist/modules/observability/PlatformObservabilityService.d.ts.map +1 -1
- package/dist/modules/observability/PlatformObservabilityService.js +71 -10
- package/dist/modules/observability/PlatformObservabilityService.js.map +1 -1
- package/dist/modules/observability/platform-health.logic.d.ts +35 -4
- package/dist/modules/observability/platform-health.logic.d.ts.map +1 -1
- package/dist/modules/observability/platform-health.logic.js +45 -0
- package/dist/modules/observability/platform-health.logic.js.map +1 -1
- package/dist/modules/observability/platform-observability.logic.d.ts +41 -4
- package/dist/modules/observability/platform-observability.logic.d.ts.map +1 -1
- package/dist/modules/observability/platform-observability.logic.js +88 -6
- package/dist/modules/observability/platform-observability.logic.js.map +1 -1
- package/dist/modules/observability/retentionPass.d.ts +22 -0
- package/dist/modules/observability/retentionPass.d.ts.map +1 -1
- package/dist/modules/observability/retentionPass.js +12 -0
- package/dist/modules/observability/retentionPass.js.map +1 -1
- package/dist/modules/observability/runTraceSpans.logic.d.ts +20 -0
- package/dist/modules/observability/runTraceSpans.logic.d.ts.map +1 -0
- package/dist/modules/observability/runTraceSpans.logic.js +127 -0
- package/dist/modules/observability/runTraceSpans.logic.js.map +1 -0
- package/dist/modules/recurring/RecurringPipelineService.d.ts +79 -14
- package/dist/modules/recurring/RecurringPipelineService.d.ts.map +1 -1
- package/dist/modules/recurring/RecurringPipelineService.js +157 -24
- package/dist/modules/recurring/RecurringPipelineService.js.map +1 -1
- package/dist/modules/recurring/issueIntake.logic.d.ts +43 -0
- package/dist/modules/recurring/issueIntake.logic.d.ts.map +1 -0
- package/dist/modules/recurring/issueIntake.logic.js +69 -0
- package/dist/modules/recurring/issueIntake.logic.js.map +1 -0
- package/package.json +11 -11
|
@@ -10,7 +10,8 @@ import { shouldPersistActivity } from './job.logic.js';
|
|
|
10
10
|
// it and it is the exact counterpart the poll site reads back.
|
|
11
11
|
/**
|
|
12
12
|
* Persist the attribution a DISPATCH knows and the poll site cannot re-derive: the resolved
|
|
13
|
-
* model, plus (for a subscription-harness job) the leased pool row and the run's initiator
|
|
13
|
+
* model, plus (for a subscription-harness job) the leased pool row and the run's initiator,
|
|
14
|
+
* plus the agent kind the job actually ran AS.
|
|
14
15
|
*
|
|
15
16
|
* An async container job settles on the durable poll path, which rebuilds the job handle from
|
|
16
17
|
* the step alone — so anything not recorded here is lost by the time the usage lands. Dropping
|
|
@@ -18,14 +19,29 @@ import { shouldPersistActivity } from './job.logic.js';
|
|
|
18
19
|
* the other two silently skips the pooled-token usage feedback (usage-aware rotation) and leaves
|
|
19
20
|
* the quota-cycle counters with no target. Each field is written only when the handle carries it,
|
|
20
21
|
* so a re-dispatch that resolves less never erases what an earlier one knew.
|
|
22
|
+
*
|
|
23
|
+
* `dispatchedKind` is a REQUIRED parameter rather than something read off the step, because
|
|
24
|
+
* `step.agentKind` is routinely not what ran: a gate escalates to its helper, a Tester hands off
|
|
25
|
+
* to the fixer, a two-phase coder proposes forks first. Every telemetry row that job produces is
|
|
26
|
+
* tagged with the dispatched kind, so a consumer grouping by kind (the external trace's step
|
|
27
|
+
* spans) has nothing to attach to unless the run records it. Being a parameter is the point: a
|
|
28
|
+
* new dispatch site cannot compile without answering the question, which is the same reason this
|
|
29
|
+
* function exists at all.
|
|
21
30
|
*/
|
|
22
|
-
export function recordDispatchAttribution(step, handle) {
|
|
31
|
+
export function recordDispatchAttribution(step, handle, dispatchedKind) {
|
|
23
32
|
if (handle.model)
|
|
24
33
|
step.model = handle.model;
|
|
25
34
|
if (handle.subscriptionTokenId)
|
|
26
35
|
step.subscriptionTokenId = handle.subscriptionTokenId;
|
|
27
36
|
if (handle.initiatedByUserId)
|
|
28
37
|
step.initiatedByUserId = handle.initiatedByUserId;
|
|
38
|
+
// Order-preserving by FIRST dispatch, counting every one after it: the count is what makes a
|
|
39
|
+
// gate's fourth fixer round visible, so a re-dispatch increments rather than deduplicating.
|
|
40
|
+
const dispatches = step.dispatches ?? [];
|
|
41
|
+
const existing = dispatches.find((d) => d.agentKind === dispatchedKind);
|
|
42
|
+
step.dispatches = existing
|
|
43
|
+
? dispatches.map((d) => (d === existing ? { ...d, count: d.count + 1 } : d))
|
|
44
|
+
: [...dispatches, { agentKind: dispatchedKind, count: 1 }];
|
|
29
45
|
}
|
|
30
46
|
export function applyContainerRunning(step, update) {
|
|
31
47
|
const prev = step.container ?? undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"step-fold.logic.js","sourceRoot":"","sources":["../../../src/modules/execution/step-fold.logic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAA0C,MAAM,qBAAqB,CAAA;AAC1F,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAA;AAEtD,8FAA8F;AAC9F,kGAAkG;AAClG,gGAAgG;AAChG,kGAAkG;AAClG,iGAAiG;AACjG,gGAAgG;AAChG,iGAAiG;AACjG,+DAA+D;AAE/D
|
|
1
|
+
{"version":3,"file":"step-fold.logic.js","sourceRoot":"","sources":["../../../src/modules/execution/step-fold.logic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAA0C,MAAM,qBAAqB,CAAA;AAC1F,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAA;AAEtD,8FAA8F;AAC9F,kGAAkG;AAClG,gGAAgG;AAChG,kGAAkG;AAClG,iGAAiG;AACjG,gGAAgG;AAChG,iGAAiG;AACjG,+DAA+D;AAE/D;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,yBAAyB,CACvC,IAAkB,EAClB,MAAsB,EACtB,cAAsB;IAEtB,IAAI,MAAM,CAAC,KAAK;QAAE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;IAC3C,IAAI,MAAM,CAAC,mBAAmB;QAAE,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,CAAA;IACrF,IAAI,MAAM,CAAC,iBAAiB;QAAE,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAA;IAC/E,6FAA6F;IAC7F,4FAA4F;IAC5F,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAA;IACxC,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,cAAc,CAAC,CAAA;IACvE,IAAI,CAAC,UAAU,GAAG,QAAQ;QACxB,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC,GAAG,UAAU,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;AAC9D,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,IAAkB,EAClB,MAAqE;IAErE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,SAAS,CAAA;IACxC,MAAM,IAAI,GAAG;QACX,MAAM,EAAE,IAAa;QACrB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI;QAC1C,EAAE,EAAE,MAAM,CAAC,SAAS,EAAE,EAAE,IAAI,IAAI,EAAE,EAAE,IAAI,IAAI;QAC5C,GAAG,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,IAAI,IAAI,EAAE,GAAG,IAAI,IAAI;KAChD,CAAA;IACD,IACE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC,MAAM;QAC5B,CAAC,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK;QACpC,CAAC,IAAI,EAAE,EAAE,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,EAAE;QAC9B,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,EAChC,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;IACrB,OAAO,IAAI,CAAA;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAClC,IAAkB,EAClB,MAAgC;IAEhC,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAE,OAAO,KAAK,CAAA;IAChE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAA;IACtB,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IACtE,OAAO,IAAI,CAAA;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAkB,EAAE,QAA4B;IAChF,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAA;IACvE,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAA;IAC9B,OAAO,IAAI,CAAA;AACb,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { ExecutionInstance, GateOutcomeKind, GateOutcomeRepository, Logger, PipelineStep } from '@cat-factory/kernel';
|
|
2
|
+
/** What the caller knows at the moment a gate settles. */
|
|
3
|
+
export interface SettledGate {
|
|
4
|
+
workspaceId: string;
|
|
5
|
+
instance: ExecutionInstance;
|
|
6
|
+
step: PipelineStep;
|
|
7
|
+
/** The gate step's index, which (with the run) gives the row its replay-stable identity. */
|
|
8
|
+
stepIndex: number;
|
|
9
|
+
/** The helper agent kind this gate escalates to, or null when it has none. */
|
|
10
|
+
helperKind: string | null;
|
|
11
|
+
outcome: GateOutcomeKind;
|
|
12
|
+
}
|
|
13
|
+
export interface GateOutcomeRecorderDeps {
|
|
14
|
+
gateOutcomeRepository: GateOutcomeRepository;
|
|
15
|
+
now: () => number;
|
|
16
|
+
logger?: Logger;
|
|
17
|
+
}
|
|
18
|
+
export declare class GateOutcomeRecorder {
|
|
19
|
+
private readonly deps;
|
|
20
|
+
constructor(deps: GateOutcomeRecorderDeps);
|
|
21
|
+
/**
|
|
22
|
+
* Record a gate's terminal verdict. Never throws.
|
|
23
|
+
*
|
|
24
|
+
* The row id is DERIVED (`<runId>:<stepIndex>:<outcome>`) rather than minted, because the
|
|
25
|
+
* durable drivers replay: a minted id would let one settle become two rows and inflate every
|
|
26
|
+
* statistic the projection exists to report. Deriving it makes the write idempotent under
|
|
27
|
+
* replay while still letting a step that is RE-RUN and ends DIFFERENTLY record that second,
|
|
28
|
+
* genuinely different verdict. A re-run that ends the same way collapses onto the first row,
|
|
29
|
+
* which is the honest reading: the gate settled that way, once, on this step.
|
|
30
|
+
*/
|
|
31
|
+
record(settled: SettledGate): Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=GateOutcomeRecorder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"GateOutcomeRecorder.d.ts","sourceRoot":"","sources":["../../../src/modules/observability/GateOutcomeRecorder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,qBAAqB,EACrB,MAAM,EACN,YAAY,EACb,MAAM,qBAAqB,CAAA;AAY5B,0DAA0D;AAC1D,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,iBAAiB,CAAA;IAC3B,IAAI,EAAE,YAAY,CAAA;IAClB,4FAA4F;IAC5F,SAAS,EAAE,MAAM,CAAA;IACjB,8EAA8E;IAC9E,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,OAAO,EAAE,eAAe,CAAA;CACzB;AAED,MAAM,WAAW,uBAAuB;IACtC,qBAAqB,EAAE,qBAAqB,CAAA;IAC5C,GAAG,EAAE,MAAM,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,qBAAa,mBAAmB;IAClB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAAjC,YAA6B,IAAI,EAAE,uBAAuB,EAAI;IAE9D;;;;;;;;;OASG;IACG,MAAM,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CA+BhD;CACF"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { noopLogger, runBestEffort } from '@cat-factory/kernel';
|
|
2
|
+
export class GateOutcomeRecorder {
|
|
3
|
+
deps;
|
|
4
|
+
constructor(deps) {
|
|
5
|
+
this.deps = deps;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Record a gate's terminal verdict. Never throws.
|
|
9
|
+
*
|
|
10
|
+
* The row id is DERIVED (`<runId>:<stepIndex>:<outcome>`) rather than minted, because the
|
|
11
|
+
* durable drivers replay: a minted id would let one settle become two rows and inflate every
|
|
12
|
+
* statistic the projection exists to report. Deriving it makes the write idempotent under
|
|
13
|
+
* replay while still letting a step that is RE-RUN and ends DIFFERENTLY record that second,
|
|
14
|
+
* genuinely different verdict. A re-run that ends the same way collapses onto the first row,
|
|
15
|
+
* which is the honest reading: the gate settled that way, once, on this step.
|
|
16
|
+
*/
|
|
17
|
+
async record(settled) {
|
|
18
|
+
const { workspaceId, instance, step, stepIndex, helperKind, outcome } = settled;
|
|
19
|
+
const gate = step.gate;
|
|
20
|
+
if (!gate)
|
|
21
|
+
return;
|
|
22
|
+
const now = this.deps.now();
|
|
23
|
+
// A gate that never dispatched a helper has no attempt log; the count of FAILED entries is
|
|
24
|
+
// what separates "the fixer keeps crashing" from "the fixer cannot fix it".
|
|
25
|
+
const helperFailures = (gate.attemptLog ?? []).filter((a) => a.outcome === 'failed').length;
|
|
26
|
+
await runBestEffort(this.deps.logger ?? noopLogger, 'gate-outcome: record settled gate', () => this.deps.gateOutcomeRepository.record({
|
|
27
|
+
id: `${instance.id}:${stepIndex}:${outcome}`,
|
|
28
|
+
workspaceId,
|
|
29
|
+
executionId: instance.id,
|
|
30
|
+
blockId: instance.blockId,
|
|
31
|
+
gateKind: step.agentKind,
|
|
32
|
+
helperKind,
|
|
33
|
+
outcome,
|
|
34
|
+
attempts: gate.attempts,
|
|
35
|
+
maxAttempts: gate.maxAttempts,
|
|
36
|
+
helperFailures,
|
|
37
|
+
// `watchSince` is stamped on every gate's first entry (not only the time-windowed
|
|
38
|
+
// ones), so it is the gate's own start. Null rather than 0 when it is somehow unset:
|
|
39
|
+
// a duration of zero would read as an instant gate, which is a different claim.
|
|
40
|
+
durationMs: gate.watchSince != null ? Math.max(0, now - gate.watchSince) : null,
|
|
41
|
+
createdAt: now,
|
|
42
|
+
}), { workspaceId, executionId: instance.id, gateKind: step.agentKind, outcome });
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=GateOutcomeRecorder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"GateOutcomeRecorder.js","sourceRoot":"","sources":["../../../src/modules/observability/GateOutcomeRecorder.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AA6B/D,MAAM,OAAO,mBAAmB;IACD,IAAI;IAAjC,YAA6B,IAA6B;oBAA7B,IAAI;IAA4B,CAAC;IAE9D;;;;;;;;;OASG;IACH,KAAK,CAAC,MAAM,CAAC,OAAoB;QAC/B,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAA;QAC/E,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;QACtB,IAAI,CAAC,IAAI;YAAE,OAAM;QACjB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAA;QAC3B,2FAA2F;QAC3F,4EAA4E;QAC5E,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;QAC3F,MAAM,aAAa,CACjB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,UAAU,EAC9B,mCAAmC,EACnC,GAAG,EAAE,CACH,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC;YACrC,EAAE,EAAE,GAAG,QAAQ,CAAC,EAAE,IAAI,SAAS,IAAI,OAAO,EAAE;YAC5C,WAAW;YACX,WAAW,EAAE,QAAQ,CAAC,EAAE;YACxB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,UAAU;YACV,OAAO;YACP,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,cAAc;YACd,kFAAkF;YAClF,qFAAqF;YACrF,gFAAgF;YAChF,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI;YAC/E,SAAS,EAAE,GAAG;SACf,CAAC,EACJ,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,CAC7E,CAAA;IACH,CAAC;CACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Clock, type IdGenerator } from '@cat-factory/kernel';
|
|
2
2
|
import type { GroupCacheHandle, InlineLlmCall, Logger, HarnessCallMetric, LlmCallMetric, LlmCallMetricRepository, LlmCallMetricSummary, LlmTraceSink, WorkspaceSettingsCacheValue, WorkspaceSettingsRepository } from '@cat-factory/kernel';
|
|
3
|
-
import type { LlmMetricsExport } from '@cat-factory/contracts';
|
|
3
|
+
import type { ExecutionInstance, LlmMetricsExport } from '@cat-factory/contracts';
|
|
4
4
|
export interface LlmObservabilityServiceDependencies {
|
|
5
5
|
llmCallMetricRepository: LlmCallMetricRepository;
|
|
6
6
|
idGenerator: IdGenerator;
|
|
@@ -152,6 +152,27 @@ export declare class LlmObservabilityService {
|
|
|
152
152
|
* still recorded.
|
|
153
153
|
*/
|
|
154
154
|
record(rawInput: RecordLlmCallInput): Promise<void>;
|
|
155
|
+
/**
|
|
156
|
+
* Close a settled run's external trace by emitting the PARENTS its generations and tool
|
|
157
|
+
* spans have been naming all along: the run's root span and one span per agent kind that
|
|
158
|
+
* ran. Called from the engine's single terminal hook, so a run reaching `done`/`failed` by
|
|
159
|
+
* any of its four routes lands here exactly the same way.
|
|
160
|
+
*
|
|
161
|
+
* Best-effort and never throwing, like every other fan-out from this service: a trace whose
|
|
162
|
+
* root is missing is a degraded trace, and must never be a failed run. A sink that groups by
|
|
163
|
+
* something other than span parentage (Langfuse) simply omits the method and nothing here
|
|
164
|
+
* changes for it.
|
|
165
|
+
*
|
|
166
|
+
* AWAITED, where the per-call `recordGeneration` fan-out above is deliberately not. The two
|
|
167
|
+
* are on opposite sides of the same trade: a generation is one span among thousands on the
|
|
168
|
+
* metering hot path, so its round trip must never extend that path, and losing one costs one
|
|
169
|
+
* span. These are the PARENTS every other span of the run already named, they are emitted
|
|
170
|
+
* once per run on a path that has already committed the run's state, and losing them orphans
|
|
171
|
+
* the whole trace rather than thinning it. The wait is bounded by the sink's own per-request
|
|
172
|
+
* timeout, and on the Worker it is also what keeps the export from being cut off when the
|
|
173
|
+
* isolate finishes.
|
|
174
|
+
*/
|
|
175
|
+
recordRunTrace(workspaceId: string, instance: ExecutionInstance): Promise<void>;
|
|
155
176
|
/**
|
|
156
177
|
* Resolve this call's prompt to a delta against the chain tip of its
|
|
157
178
|
* `(workspace, execution, agentKind)` conversation (or the full array when it can't
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LlmObservabilityService.d.ts","sourceRoot":"","sources":["../../../src/modules/observability/LlmObservabilityService.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,KAAK,EACV,KAAK,WAAW,EAOjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,KAAK,EACV,gBAAgB,EAChB,aAAa,EACb,MAAM,EACN,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,oBAAoB,EACpB,YAAY,EACZ,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,qBAAqB,CAAA;AAC5B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAA;
|
|
1
|
+
{"version":3,"file":"LlmObservabilityService.d.ts","sourceRoot":"","sources":["../../../src/modules/observability/LlmObservabilityService.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,KAAK,EACV,KAAK,WAAW,EAOjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,KAAK,EACV,gBAAgB,EAChB,aAAa,EACb,MAAM,EACN,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,oBAAoB,EACpB,YAAY,EACZ,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,qBAAqB,CAAA;AAC5B,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAA;AAKjF,MAAM,WAAW,mCAAmC;IAClD,uBAAuB,EAAE,uBAAuB,CAAA;IAChD,WAAW,EAAE,WAAW,CAAA;IACxB,KAAK,EAAE,KAAK,CAAA;IACZ;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,YAAY,CAAA;IACxB;;;;;;OAMG;IACH,2BAA2B,CAAC,EAAE,2BAA2B,CAAA;IACzD;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,gBAAgB,CAAC,2BAA2B,CAAC,CAAA;IACtE;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,QAAa,CAAA;AAcxC;;;;;;;;;;GAUG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,CAAA;AAOjD;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,OAAO,CAAA;IAClB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAA;IAC/B,6EAA6E;IAC7E,YAAY,EAAE,MAAM,CAAA;IACpB,4DAA4D;IAC5D,eAAe,EAAE,MAAM,CAAA;IACvB,sDAAsD;IACtD,gBAAgB,EAAE,MAAM,CAAA;IACxB,gBAAgB,EAAE,MAAM,CAAA;IACxB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,wDAAwD;IACxD,OAAO,EAAE,MAAM,CAAA;IACf,qDAAqD;IACrD,UAAU,EAAE,MAAM,CAAA;IAClB,EAAE,EAAE,OAAO,CAAA;IACX,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,UAAU,EAAE,WAAW,CAAA;IACvB,YAAY,EAAE,WAAW,CAAA;IACzB,0FAA0F;IAC1F,aAAa,EAAE,WAAW,CAAA;CAC3B;AAED;;;;;;;;GAQG;AACH,qBAAa,uBAAuB;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAyB;IACpD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAa;IACzC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAO;IAC7B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAc;IACzC;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAuB;IACrD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAQ;IAE5B,YAAY,EACV,uBAAuB,EACvB,WAAW,EACX,KAAK,EACL,aAAoB,EACpB,SAAS,EACT,2BAA2B,EAC3B,sBAAsB,EACtB,MAAM,GACP,EAAE,mCAAmC,EAWrC;IAED;;;;;;;;;;;OAWG;IACG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CA4FxD;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACG,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAQpF;IAED;;;;OAIG;YACW,2BAA2B;IAkBzC;;;OAGG;IACH,eAAe,CACb,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,EACnB,KAAK,GAAE,MAA2B,GACjC,OAAO,CAAC,aAAa,EAAE,CAAC,CAE1B;IAED,uEAAuE;IACvE,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAE9F;IAED;;;;OAIG;IACG,kBAAkB,CAAC,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAG5F;CACF;AAED,kGAAkG;AAClG,MAAM,WAAW,uBAAuB;IACtC,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,SAAS,EAAE,MAAM,CAAA;IACjB,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAA;IAChB,wFAAwF;IACxF,KAAK,EAAE,MAAM,CAAA;IACb;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,iBAAiB,EAAE,CAAA;CAC3B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,uBAAuB,GAC/B,CAAC,KAAK,EAAE,uBAAuB,KAAK,OAAO,CAAC,IAAI,CAAC,CAuCnD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AACH,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,uBAAuB,GAC/B,CAAC,IAAI,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,CA4BxC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createStoreAgentContextGate, noopLogger, normalizeCallPhase, redactSecrets, runBestEffort, } from '@cat-factory/kernel';
|
|
2
2
|
import { buildLlmMetricsExport, computeStoredPrompt } from './observability.logic.js';
|
|
3
|
+
import { buildRunTraceSpans } from './runTraceSpans.logic.js';
|
|
3
4
|
/**
|
|
4
5
|
* Defensive upper bound on a stored prompt/response body (characters). Real agent
|
|
5
6
|
* prompts sit far below this; the cap exists only so a pathological body can't blow
|
|
@@ -156,6 +157,35 @@ export class LlmObservabilityService {
|
|
|
156
157
|
})));
|
|
157
158
|
}
|
|
158
159
|
}
|
|
160
|
+
/**
|
|
161
|
+
* Close a settled run's external trace by emitting the PARENTS its generations and tool
|
|
162
|
+
* spans have been naming all along: the run's root span and one span per agent kind that
|
|
163
|
+
* ran. Called from the engine's single terminal hook, so a run reaching `done`/`failed` by
|
|
164
|
+
* any of its four routes lands here exactly the same way.
|
|
165
|
+
*
|
|
166
|
+
* Best-effort and never throwing, like every other fan-out from this service: a trace whose
|
|
167
|
+
* root is missing is a degraded trace, and must never be a failed run. A sink that groups by
|
|
168
|
+
* something other than span parentage (Langfuse) simply omits the method and nothing here
|
|
169
|
+
* changes for it.
|
|
170
|
+
*
|
|
171
|
+
* AWAITED, where the per-call `recordGeneration` fan-out above is deliberately not. The two
|
|
172
|
+
* are on opposite sides of the same trade: a generation is one span among thousands on the
|
|
173
|
+
* metering hot path, so its round trip must never extend that path, and losing one costs one
|
|
174
|
+
* span. These are the PARENTS every other span of the run already named, they are emitted
|
|
175
|
+
* once per run on a path that has already committed the run's state, and losing them orphans
|
|
176
|
+
* the whole trace rather than thinning it. The wait is bounded by the sink's own per-request
|
|
177
|
+
* timeout, and on the Worker it is also what keeps the export from being cut off when the
|
|
178
|
+
* isolate finishes.
|
|
179
|
+
*/
|
|
180
|
+
async recordRunTrace(workspaceId, instance) {
|
|
181
|
+
const traceSink = this.traceSink;
|
|
182
|
+
if (!traceSink?.recordRunSpans)
|
|
183
|
+
return;
|
|
184
|
+
const spans = buildRunTraceSpans(workspaceId, instance);
|
|
185
|
+
if (!spans)
|
|
186
|
+
return;
|
|
187
|
+
await runBestEffort(this.log, 'traceSink.recordRunSpans', () => Promise.resolve(traceSink.recordRunSpans?.(spans.run, spans.steps)));
|
|
188
|
+
}
|
|
159
189
|
/**
|
|
160
190
|
* Resolve this call's prompt to a delta against the chain tip of its
|
|
161
191
|
* `(workspace, execution, agentKind)` conversation (or the full array when it can't
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LlmObservabilityService.js","sourceRoot":"","sources":["../../../src/modules/observability/LlmObservabilityService.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,2BAA2B,EAC3B,UAAU,EACV,kBAAkB,EAClB,aAAa,EACb,aAAa,GACd,MAAM,qBAAqB,CAAA;AAe5B,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAA;
|
|
1
|
+
{"version":3,"file":"LlmObservabilityService.js","sourceRoot":"","sources":["../../../src/modules/observability/LlmObservabilityService.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,2BAA2B,EAC3B,UAAU,EACV,kBAAkB,EAClB,aAAa,EACb,aAAa,GACd,MAAM,qBAAqB,CAAA;AAe5B,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAA;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AA2C7D;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,GAAG,IAAI,CAAA;AAExC,sEAAsE;AACtE,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,IAAI,CAAC,MAAM,IAAI,cAAc;QAAE,OAAO,IAAI,CAAA;IAC9C,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,iBAAiB,IAAI,CAAC,MAAM,GAAG,cAAc,SAAS,CAAA;AAC/F,CAAC;AAED,oEAAoE;AACpE,MAAM,kBAAkB,GAAG,IAAI,CAAA;AAE/B,sFAAsF;AACtF,MAAM,mBAAmB,GAAiB,EAAE,UAAU,EAAE,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAA;AAelG,yFAAyF;AACzF,SAAS,WAAW,CAAC,IAAiB;IACpC,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;AACjD,CAAC;AA0DD;;;;;;;;GAQG;AACH,MAAM,OAAO,uBAAuB;IACjB,UAAU,CAAyB;IACnC,WAAW,CAAa;IACxB,KAAK,CAAO;IACZ,aAAa,CAAS;IACtB,SAAS,CAAe;IACzC;;;;OAIG;IACc,aAAa,CAAuB;IACpC,GAAG,CAAQ;IAE5B,YAAY,EACV,uBAAuB,EACvB,WAAW,EACX,KAAK,EACL,aAAa,GAAG,IAAI,EACpB,SAAS,EACT,2BAA2B,EAC3B,sBAAsB,EACtB,MAAM,GAC8B;QACpC,IAAI,CAAC,UAAU,GAAG,uBAAuB,CAAA;QACzC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAA;QAC9B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAA;QAClC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,aAAa,GAAG,2BAA2B,CAAC;YAC/C,UAAU,EAAE,2BAA2B;YACvC,KAAK,EAAE,sBAAsB;SAC9B,CAAC,CAAA;QACF,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,UAAU,CAAC,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC,CAAA;IAC1E,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,MAAM,CAAC,QAA4B;QACvC,wFAAwF;QACxF,qFAAqF;QACrF,kFAAkF;QAClF,8BAA8B;QAC9B,EAAE;QACF,kFAAkF;QAClF,wFAAwF;QACxF,mFAAmF;QACnF,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAA;QAC3F,mFAAmF;QACnF,qFAAqF;QACrF,iFAAiF;QACjF,qFAAqF;QACrF,gFAAgF;QAChF,8DAA8D;QAC9D,MAAM,KAAK,GAAG,CAAC,IAAiB,EAAU,EAAE,CAC1C,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAC9D,MAAM,KAAK,GAAG;YACZ,GAAG,QAAQ;YACX,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;YACtC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC1C,aAAa,EAAE,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC5C,qFAAqF;YACrF,sFAAsF;YACtF,kFAAkF;YAClF,+EAA+E;YAC/E,iFAAiF;YACjF,YAAY,EAAE,aAAa,CAAC,QAAQ,CAAC,YAAY,CAAC;SACnD,CAAA;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,CAAA;QAChE,MAAM,MAAM,GAAG,YAAY;YACzB,CAAC,CAAC,MAAM,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC;YAC/C,CAAC,CAAC,mBAAmB,CAAA;QACvB,MAAM,MAAM,GAAkB;YAC5B,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE;YAC3B,GAAG,KAAK;YACR,4EAA4E;YAC5E,gFAAgF;YAChF,8CAA8C;YAC9C,EAAE,EAAE,KAAK,CAAC,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAC5C,UAAU;YACV,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,KAAK,CAAC;YACtC,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;YAClC,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC;YACxC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,iFAAiF;YACjF,oFAAoF;YACpF,qDAAqD;YACrD,YAAY,EAAE,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE;YAC/D,aAAa,EAAE,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE;SAClE,CAAA;QACD,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QACpC,mFAAmF;QACnF,4EAA4E;QAC5E,mFAAmF;QACnF,mFAAmF;QACnF,mFAAmF;QACnF,+EAA+E;QAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAA;QAChC,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAA;YAChC,+EAA+E;YAC/E,iFAAiF;YACjF,sDAAsD;YACtD,KAAK,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,4BAA4B,EAAE,GAAG,EAAE,CAC9D,OAAO,CAAC,OAAO,CACb,SAAS,CAAC,gBAAgB,CAAC;gBACzB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC;gBAClD,OAAO;gBACP,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,eAAe,EAAE,KAAK,CAAC,eAAe;gBACtC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;gBACxC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;gBACxC,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;gBAC3C,2EAA2E;gBAC3E,+EAA+E;gBAC/E,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE;aACtE,CAAC,CACH,CACF,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,cAAc,CAAC,WAAmB,EAAE,QAA2B;QACnE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAA;QAChC,IAAI,CAAC,SAAS,EAAE,cAAc;YAAE,OAAM;QACtC,MAAM,KAAK,GAAG,kBAAkB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;QACvD,IAAI,CAAC,KAAK;YAAE,OAAM;QAClB,MAAM,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,0BAA0B,EAAE,GAAG,EAAE,CAC7D,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CACpE,CAAA;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,2BAA2B,CAAC,KAMzC;QACC,MAAM,IAAI,GACR,KAAK,CAAC,WAAW,IAAI,IAAI;YACvB,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,CAClC,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,SAAS,CAChB;YACH,CAAC,CAAC,IAAI,CAAA;QACV,OAAO,mBAAmB,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;IACpD,CAAC;IAED;;;OAGG;IACH,eAAe,CACb,WAAmB,EACnB,WAAmB,EACnB,KAAK,GAAW,kBAAkB;QAElC,OAAO,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,WAAW,EAAE,WAAW,EAAE,KAAK,CAAC,CAAA;IACzE,CAAC;IAED,uEAAuE;IACvE,oBAAoB,CAAC,WAAmB,EAAE,WAAmB;QAC3D,OAAO,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAA;IACvE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kBAAkB,CAAC,WAAmB,EAAE,WAAmB;QAC/D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,WAAW,CAAC,CAAA;QAClE,OAAO,qBAAqB,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAA;IACpE,CAAC;CACF;AAuBD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,uBAAuB,CACrC,OAAgC;IAEhC,OAAO,KAAK,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE;QACtF,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5C,mFAAmF;YACnF,oFAAoF;YACpF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,IAAI,KAAK,CAAA;YACnC,MAAM,OAAO,CAAC,MAAM,CAAC;gBACnB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,OAAO,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,WAAW;gBACX,WAAW;gBACX,SAAS;gBACT,QAAQ;gBACR,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,KAAK;gBAC1B,SAAS,EAAE,IAAI;gBACf,qFAAqF;gBACrF,uEAAuE;gBACvE,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1D,SAAS;gBACT,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS,EAAE,CAAC;gBACZ,gBAAgB,EAAE,IAAI;gBACtB,YAAY,EAAE,IAAI,CAAC,WAAW;gBAC9B,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;gBACvC,gBAAgB,EAAE,IAAI,CAAC,YAAY;gBACnC,WAAW,EACT,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,YAAY;gBACrF,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,OAAO,EAAE,CAAC;gBACV,UAAU,EAAE,CAAC;gBACb,EAAE,EAAE,IAAI;gBACR,UAAU,EAAE,IAAI;gBAChB,YAAY,EAAE,IAAI;gBAClB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,aAAa,EAAE,IAAI,CAAC,aAAa;aAClC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAA;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AACH,MAAM,UAAU,sBAAsB,CACpC,OAAgC;IAEhC,OAAO,CAAC,IAAI,EAAE,EAAE,CACd,OAAO,CAAC,MAAM,CAAC;QACb,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,SAAS,EAAE,KAAK;QAChB,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;QACjC,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;QACvC,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;QACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;QACvC,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,OAAO,EAAE,IAAI,CAAC,UAAU;QACxB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,UAAU,EAAE,IAAI;QAChB,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,aAAa,EAAE,IAAI,CAAC,aAAa;KAClC,CAAC,CAAA;AACN,CAAC"}
|
|
@@ -1,7 +1,14 @@
|
|
|
1
|
-
import type { Clock, PlatformMetricsRepository } from '@cat-factory/kernel';
|
|
1
|
+
import type { Clock, GateOutcomeRepository, PlatformMetricsRepository } from '@cat-factory/kernel';
|
|
2
2
|
import type { PlatformObservability, PlatformObservabilityWindow } from '@cat-factory/contracts';
|
|
3
3
|
export interface PlatformObservabilityServiceDependencies {
|
|
4
4
|
platformMetricsRepository: PlatformMetricsRepository;
|
|
5
|
+
/**
|
|
6
|
+
* The settled-gate projection behind the attempt statistics. Optional so a facade that has
|
|
7
|
+
* not wired it (or a test) degrades to an EMPTY gate list rather than failing the whole
|
|
8
|
+
* dashboard read: the projection is the newest of the sinks and the only one whose absence
|
|
9
|
+
* costs a section rather than the page.
|
|
10
|
+
*/
|
|
11
|
+
gateOutcomeRepository?: GateOutcomeRepository;
|
|
5
12
|
clock: Clock;
|
|
6
13
|
}
|
|
7
14
|
/**
|
|
@@ -9,10 +16,34 @@ export interface PlatformObservabilityServiceDependencies {
|
|
|
9
16
|
* behind {@link PlatformMetricsRepository} into the dashboard's windowed projection. Each
|
|
10
17
|
* rollup is one SQL GROUP BY, run in parallel (independent aggregates, NOT an N+1); the
|
|
11
18
|
* reshaping into totals / trend is the pure logic in `platform-observability.logic.ts`.
|
|
19
|
+
*
|
|
20
|
+
* Two windows' worth of routing lives here: `1h`/`24h`/`7d` scan `agent_runs` live, while
|
|
21
|
+
* `30d`/`90d` read the daily rollup the retention sweep materialises. The projection SAYS
|
|
22
|
+
* which answered (`source`) and, on the rollup path, how far the rollup actually reaches
|
|
23
|
+
* (`rolledUpThrough`), because an un-materialised rollup and an idle quarter produce the same
|
|
24
|
+
* empty series and are opposite facts.
|
|
12
25
|
*/
|
|
13
26
|
export declare class PlatformObservabilityService {
|
|
14
27
|
private readonly deps;
|
|
15
28
|
constructor(deps: PlatformObservabilityServiceDependencies);
|
|
16
29
|
summarize(accountId: string, window: PlatformObservabilityWindow): Promise<PlatformObservability>;
|
|
30
|
+
/**
|
|
31
|
+
* A bounded, per-workspace-capped sample of the account's recently FAILED runs since
|
|
32
|
+
* `sinceEpochMs`: the evidence a `platform_health` alert card deep-links to.
|
|
33
|
+
*
|
|
34
|
+
* On the service rather than read straight off the port because the alert sweep is a
|
|
35
|
+
* consumer of this module, not of the store: everything else it needs already arrives
|
|
36
|
+
* through `summarize`, and reaching past the service for one query is how a second,
|
|
37
|
+
* differently-scoped copy of the account filter gets written.
|
|
38
|
+
*/
|
|
39
|
+
failingRuns(accountId: string, sinceEpochMs: number, perWorkspaceLimit: number): Promise<import("@cat-factory/kernel").PlatformFailedRunRef[]>;
|
|
40
|
+
/** The live path: three independent aggregates over `agent_runs`. */
|
|
41
|
+
private summarizeFromRuns;
|
|
42
|
+
/**
|
|
43
|
+
* The long-window path: ONE read of the daily rollup serves the totals, the trend and the
|
|
44
|
+
* failure taxonomy alike (each rollup row carries the day, the status and, for a failure,
|
|
45
|
+
* its kind), plus the watermark read that says how far the rollup reaches.
|
|
46
|
+
*/
|
|
47
|
+
private summarizeFromRollup;
|
|
17
48
|
}
|
|
18
49
|
//# sourceMappingURL=PlatformObservabilityService.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PlatformObservabilityService.d.ts","sourceRoot":"","sources":["../../../src/modules/observability/PlatformObservabilityService.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAA;
|
|
1
|
+
{"version":3,"file":"PlatformObservabilityService.d.ts","sourceRoot":"","sources":["../../../src/modules/observability/PlatformObservabilityService.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,qBAAqB,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAA;AAClG,OAAO,KAAK,EAAE,qBAAqB,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAA;AAUhG,MAAM,WAAW,wCAAwC;IACvD,yBAAyB,EAAE,yBAAyB,CAAA;IACpD;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAA;IAC7C,KAAK,EAAE,KAAK,CAAA;CACb;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,4BAA4B;IAC3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IAAjC,YAA6B,IAAI,EAAE,wCAAwC,EAAI;IAEzE,SAAS,CACb,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,2BAA2B,GAClC,OAAO,CAAC,qBAAqB,CAAC,CA6BhC;IAED;;;;;;;;OAQG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,iEAMnF;IAED,qEAAqE;YACvD,iBAAiB;IAuB/B;;;;OAIG;YACW,mBAAmB;CAkBlC"}
|
|
@@ -1,9 +1,15 @@
|
|
|
1
|
-
import { PLATFORM_WINDOWS, buildTrend, summarizeOutcomes } from './platform-observability.logic.js';
|
|
1
|
+
import { PLATFORM_WINDOWS, buildTrend, dailyFailureSlices, dailyTrendRows, summarizeGateOutcomes, summarizeOutcomes, } from './platform-observability.logic.js';
|
|
2
2
|
/**
|
|
3
3
|
* Deployment-level (platform-operator) observability read: composes the aggregate rollups
|
|
4
4
|
* behind {@link PlatformMetricsRepository} into the dashboard's windowed projection. Each
|
|
5
5
|
* rollup is one SQL GROUP BY, run in parallel (independent aggregates, NOT an N+1); the
|
|
6
6
|
* reshaping into totals / trend is the pure logic in `platform-observability.logic.ts`.
|
|
7
|
+
*
|
|
8
|
+
* Two windows' worth of routing lives here: `1h`/`24h`/`7d` scan `agent_runs` live, while
|
|
9
|
+
* `30d`/`90d` read the daily rollup the retention sweep materialises. The projection SAYS
|
|
10
|
+
* which answered (`source`) and, on the rollup path, how far the rollup actually reaches
|
|
11
|
+
* (`rolledUpThrough`), because an un-materialised rollup and an idle quarter produce the same
|
|
12
|
+
* empty series and are opposite facts.
|
|
7
13
|
*/
|
|
8
14
|
export class PlatformObservabilityService {
|
|
9
15
|
deps;
|
|
@@ -11,26 +17,81 @@ export class PlatformObservabilityService {
|
|
|
11
17
|
this.deps = deps;
|
|
12
18
|
}
|
|
13
19
|
async summarize(accountId, window) {
|
|
14
|
-
const { windowMs, bucketMs } = PLATFORM_WINDOWS[window];
|
|
20
|
+
const { windowMs, bucketMs, source } = PLATFORM_WINDOWS[window];
|
|
15
21
|
const now = this.deps.clock.now();
|
|
16
22
|
const since = now - windowMs;
|
|
17
23
|
const repo = this.deps.platformMetricsRepository;
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
+
// The live-depth snapshot, the duration percentiles and the gate statistics are the same
|
|
25
|
+
// read on every window: the first two are not bucketed (so there is nothing to roll up),
|
|
26
|
+
// and the gate projection is small enough to scan directly at any window.
|
|
27
|
+
const [shared, windowed] = await Promise.all([
|
|
28
|
+
Promise.all([
|
|
29
|
+
repo.activeAndParkedCounts(accountId),
|
|
30
|
+
repo.durationStatsSince(accountId, since),
|
|
31
|
+
this.deps.gateOutcomeRepository?.statsSince(accountId, since) ?? Promise.resolve([]),
|
|
32
|
+
]),
|
|
33
|
+
source === 'daily-rollup'
|
|
34
|
+
? this.summarizeFromRollup(accountId, since, now, bucketMs)
|
|
35
|
+
: this.summarizeFromRuns(accountId, since, now, bucketMs),
|
|
24
36
|
]);
|
|
37
|
+
const [live, durations, gateRows] = shared;
|
|
25
38
|
return {
|
|
26
39
|
window,
|
|
27
40
|
generatedAt: now,
|
|
28
41
|
since,
|
|
42
|
+
source,
|
|
43
|
+
...windowed,
|
|
44
|
+
live,
|
|
45
|
+
durations,
|
|
46
|
+
gates: summarizeGateOutcomes(gateRows),
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* A bounded, per-workspace-capped sample of the account's recently FAILED runs since
|
|
51
|
+
* `sinceEpochMs`: the evidence a `platform_health` alert card deep-links to.
|
|
52
|
+
*
|
|
53
|
+
* On the service rather than read straight off the port because the alert sweep is a
|
|
54
|
+
* consumer of this module, not of the store: everything else it needs already arrives
|
|
55
|
+
* through `summarize`, and reaching past the service for one query is how a second,
|
|
56
|
+
* differently-scoped copy of the account filter gets written.
|
|
57
|
+
*/
|
|
58
|
+
async failingRuns(accountId, sinceEpochMs, perWorkspaceLimit) {
|
|
59
|
+
return this.deps.platformMetricsRepository.recentFailedRuns(accountId, sinceEpochMs, perWorkspaceLimit);
|
|
60
|
+
}
|
|
61
|
+
/** The live path: three independent aggregates over `agent_runs`. */
|
|
62
|
+
async summarizeFromRuns(accountId, since, now, bucketMs) {
|
|
63
|
+
const repo = this.deps.platformMetricsRepository;
|
|
64
|
+
const [outcomeRows, trendRows, failures] = await Promise.all([
|
|
65
|
+
repo.runOutcomesSince(accountId, since),
|
|
66
|
+
repo.runOutcomeTrend(accountId, since, bucketMs),
|
|
67
|
+
repo.failureKindBreakdown(accountId, since),
|
|
68
|
+
]);
|
|
69
|
+
return {
|
|
29
70
|
outcomes: summarizeOutcomes(outcomeRows),
|
|
71
|
+
// Null, not a stale watermark: there is no rollup in this path, so there is nothing
|
|
72
|
+
// about it that could be behind, and reporting one would invite the reader to believe
|
|
73
|
+
// the live numbers were bounded by it.
|
|
74
|
+
rolledUpThrough: null,
|
|
30
75
|
trend: { bucketMs, points: buildTrend(trendRows, since, now, bucketMs) },
|
|
31
76
|
failures: failures.map((f) => ({ kind: f.failureKind, count: f.count })),
|
|
32
|
-
|
|
33
|
-
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* The long-window path: ONE read of the daily rollup serves the totals, the trend and the
|
|
81
|
+
* failure taxonomy alike (each rollup row carries the day, the status and, for a failure,
|
|
82
|
+
* its kind), plus the watermark read that says how far the rollup reaches.
|
|
83
|
+
*/
|
|
84
|
+
async summarizeFromRollup(accountId, since, now, bucketMs) {
|
|
85
|
+
const repo = this.deps.platformMetricsRepository;
|
|
86
|
+
const [rows, rolledUpThrough] = await Promise.all([
|
|
87
|
+
repo.dailyRunTotalsSince(accountId, since),
|
|
88
|
+
repo.dailyRollupWatermark(),
|
|
89
|
+
]);
|
|
90
|
+
return {
|
|
91
|
+
outcomes: summarizeOutcomes(rows),
|
|
92
|
+
rolledUpThrough,
|
|
93
|
+
trend: { bucketMs, points: buildTrend(dailyTrendRows(rows), since, now, bucketMs) },
|
|
94
|
+
failures: dailyFailureSlices(rows),
|
|
34
95
|
};
|
|
35
96
|
}
|
|
36
97
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PlatformObservabilityService.js","sourceRoot":"","sources":["../../../src/modules/observability/PlatformObservabilityService.ts"],"names":[],"mappings":"AAEA,OAAO,
|
|
1
|
+
{"version":3,"file":"PlatformObservabilityService.js","sourceRoot":"","sources":["../../../src/modules/observability/PlatformObservabilityService.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,kBAAkB,EAClB,cAAc,EACd,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,mCAAmC,CAAA;AAc1C;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,4BAA4B;IACV,IAAI;IAAjC,YAA6B,IAA8C;oBAA9C,IAAI;IAA6C,CAAC;IAE/E,KAAK,CAAC,SAAS,CACb,SAAiB,EACjB,MAAmC;QAEnC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAA;QAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;QACjC,MAAM,KAAK,GAAG,GAAG,GAAG,QAAQ,CAAA;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAChD,yFAAyF;QACzF,yFAAyF;QACzF,0EAA0E;QAC1E,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC;gBACV,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC;gBACrC,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC;gBACzC,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aACrF,CAAC;YACF,MAAM,KAAK,cAAc;gBACvB,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC;gBAC3D,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC;SAC5D,CAAC,CAAA;QACF,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,MAAM,CAAA;QAC1C,OAAO;YACL,MAAM;YACN,WAAW,EAAE,GAAG;YAChB,KAAK;YACL,MAAM;YACN,GAAG,QAAQ;YACX,IAAI;YACJ,SAAS;YACT,KAAK,EAAE,qBAAqB,CAAC,QAAQ,CAAC;SACvC,CAAA;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,WAAW,CAAC,SAAiB,EAAE,YAAoB,EAAE,iBAAyB;QAClF,OAAO,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,gBAAgB,CACzD,SAAS,EACT,YAAY,EACZ,iBAAiB,CAClB,CAAA;IACH,CAAC;IAED,qEAAqE;IAC7D,KAAK,CAAC,iBAAiB,CAC7B,SAAiB,EACjB,KAAa,EACb,GAAW,EACX,QAAgB;QAEhB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAChD,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC;YACvC,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC;YAChD,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC;SAC5C,CAAC,CAAA;QACF,OAAO;YACL,QAAQ,EAAE,iBAAiB,CAAC,WAAW,CAAC;YACxC,oFAAoF;YACpF,sFAAsF;YACtF,uCAAuC;YACvC,eAAe,EAAE,IAAI;YACrB,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE;YACxE,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;SACzE,CAAA;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,mBAAmB,CAC/B,SAAiB,EACjB,KAAa,EACb,GAAW,EACX,QAAgB;QAEhB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAChD,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChD,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC;YAC1C,IAAI,CAAC,oBAAoB,EAAE;SAC5B,CAAC,CAAA;QACF,OAAO;YACL,QAAQ,EAAE,iBAAiB,CAAC,IAAI,CAAC;YACjC,eAAe;YACf,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE;YACnF,QAAQ,EAAE,kBAAkB,CAAC,IAAI,CAAC;SACnC,CAAA;IACH,CAAC;CACF"}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import type { PlatformAlert, PlatformAlertReason,
|
|
1
|
+
import type { PlatformAlert, PlatformAlertReason, PlatformAlertSettings, PlatformAlertWindow, PlatformObservability } from '@cat-factory/contracts';
|
|
2
2
|
/**
|
|
3
3
|
* The operator-configured ceilings a deployment's aggregate health is checked against. Each
|
|
4
|
-
* maps to one {@link PlatformAlertReason}. Deployment-level config
|
|
5
|
-
*
|
|
4
|
+
* maps to one {@link PlatformAlertReason}. Deployment-level config: the env vars set the
|
|
5
|
+
* defaults at boot and an account's stored settings may override any of them per field
|
|
6
|
+
* (see {@link resolveAccountAlertConfig}), NOT per-workspace.
|
|
6
7
|
*/
|
|
7
8
|
export interface PlatformAlertThresholds {
|
|
8
9
|
/**
|
|
@@ -45,6 +46,36 @@ export interface PlatformAlertThresholds {
|
|
|
45
46
|
}
|
|
46
47
|
/** Conservative defaults: quiet unless the deployment is genuinely unhealthy. */
|
|
47
48
|
export declare const DEFAULT_PLATFORM_ALERT_THRESHOLDS: PlatformAlertThresholds;
|
|
49
|
+
/**
|
|
50
|
+
* The alert configuration one account is actually evaluated under: the deployment's
|
|
51
|
+
* env-derived defaults with that account's stored settings layered on top.
|
|
52
|
+
*/
|
|
53
|
+
export interface ResolvedAccountAlertConfig {
|
|
54
|
+
/** Whether this account is alerted on at all. */
|
|
55
|
+
enabled: boolean;
|
|
56
|
+
window: PlatformAlertWindow;
|
|
57
|
+
thresholds: PlatformAlertThresholds;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Layer an account's stored alert settings over the deployment defaults, field by field.
|
|
61
|
+
*
|
|
62
|
+
* ABSENT INHERITS; it never means zero. That is the whole contract of this function and the
|
|
63
|
+
* reason it exists rather than a spread: a zero is a live, meaningful value in three of these
|
|
64
|
+
* thresholds (`minStalledPriorRuns: 0` says "page even on an idle window"), so a merge that
|
|
65
|
+
* treated a missing key and a stored `0` alike would turn "I did not set this" into the most
|
|
66
|
+
* aggressive setting available, on every account that ever saved any unrelated field.
|
|
67
|
+
*
|
|
68
|
+
* `enabled` is deliberately a ONE-WAY switch: an account may mute itself, but cannot turn
|
|
69
|
+
* alerting on where the deployment never started the sweep: there would be no timer to
|
|
70
|
+
* honour it, and a stored `true` that does nothing is worse than an absent one.
|
|
71
|
+
*/
|
|
72
|
+
export declare function resolveAccountAlertConfig(deployment: {
|
|
73
|
+
enabled: boolean;
|
|
74
|
+
window: PlatformAlertWindow;
|
|
75
|
+
thresholds: PlatformAlertThresholds;
|
|
76
|
+
}, stored: PlatformAlertSettings | undefined): ResolvedAccountAlertConfig;
|
|
77
|
+
/** Whether a fired reason set has failing runs behind it worth linking to. */
|
|
78
|
+
export declare function alertsHaveRunEvidence(reasons: readonly PlatformAlertReason[]): boolean;
|
|
48
79
|
/**
|
|
49
80
|
* Evaluate one account's windowed projection against the thresholds, returning every fired
|
|
50
81
|
* alert (empty when healthy). Each alert carries its observed value + the threshold it crossed
|
|
@@ -78,7 +109,7 @@ export declare function platformAlertReasons(alerts: PlatformAlert[]): PlatformA
|
|
|
78
109
|
* `payload.platformAlerts` as the dedup identity and the seam for a future localized rendering
|
|
79
110
|
* (the `usePipelineErrorToast` mapping pattern), not yet wired in the SPA.
|
|
80
111
|
*/
|
|
81
|
-
export declare function platformHealthCardContent(reasons: PlatformAlertReason[], window:
|
|
112
|
+
export declare function platformHealthCardContent(reasons: PlatformAlertReason[], window: PlatformAlertWindow): {
|
|
82
113
|
title: string;
|
|
83
114
|
body: string;
|
|
84
115
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"platform-health.logic.d.ts","sourceRoot":"","sources":["../../../src/modules/observability/platform-health.logic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EACb,mBAAmB,EACnB,qBAAqB,EACrB,
|
|
1
|
+
{"version":3,"file":"platform-health.logic.d.ts","sourceRoot":"","sources":["../../../src/modules/observability/platform-health.logic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EACb,mBAAmB,EACnB,qBAAqB,EACrB,mBAAmB,EACnB,qBAAqB,EACtB,MAAM,wBAAwB,CAAA;AAc/B;;;;;GAKG;AACH,MAAM,WAAW,uBAAuB;IACtC;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAA;IACf,2EAA2E;IAC3E,cAAc,EAAE,MAAM,CAAA;IACtB,oFAAoF;IACpF,gBAAgB,EAAE,MAAM,CAAA;IACxB,wFAAwF;IACxF,UAAU,EAAE,MAAM,CAAA;IAClB;;;;;OAKG;IACH,cAAc,EAAE,MAAM,CAAA;IACtB;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAA;IAC3B;;;;OAIG;IACH,mBAAmB,EAAE,MAAM,CAAA;IAC3B;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAA;CACzB;AAED,iFAAiF;AACjF,eAAO,MAAM,iCAAiC,EAAE,uBAa/C,CAAA;AAED;;;GAGG;AACH,MAAM,WAAW,0BAA0B;IACzC,iDAAiD;IACjD,OAAO,EAAE,OAAO,CAAA;IAChB,MAAM,EAAE,mBAAmB,CAAA;IAC3B,UAAU,EAAE,uBAAuB,CAAA;CACpC;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,yBAAyB,CACvC,UAAU,EAAE;IACV,OAAO,EAAE,OAAO,CAAA;IAChB,MAAM,EAAE,mBAAmB,CAAA;IAC3B,UAAU,EAAE,uBAAuB,CAAA;CACpC,EACD,MAAM,EAAE,qBAAqB,GAAG,SAAS,GACxC,0BAA0B,CAgB5B;AAcD,8EAA8E;AAC9E,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,SAAS,mBAAmB,EAAE,GAAG,OAAO,CAEtF;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,qBAAqB,EAC/B,UAAU,EAAE,uBAAuB;AACnC;;;;;GAKG;AACH,aAAa,CAAC,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GACrD,aAAa,EAAE,CAyEjB;AA6CD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,mBAAmB,EAAE,CAEnF;AAkBD;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,mBAAmB,EAAE,EAC9B,MAAM,EAAE,mBAAmB,GAC1B;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAUjC"}
|
|
@@ -13,6 +13,51 @@ export const DEFAULT_PLATFORM_ALERT_THRESHOLDS = {
|
|
|
13
13
|
maxFailureKindShare: 0.8,
|
|
14
14
|
maxSweepFailures: 3,
|
|
15
15
|
};
|
|
16
|
+
/**
|
|
17
|
+
* Layer an account's stored alert settings over the deployment defaults, field by field.
|
|
18
|
+
*
|
|
19
|
+
* ABSENT INHERITS; it never means zero. That is the whole contract of this function and the
|
|
20
|
+
* reason it exists rather than a spread: a zero is a live, meaningful value in three of these
|
|
21
|
+
* thresholds (`minStalledPriorRuns: 0` says "page even on an idle window"), so a merge that
|
|
22
|
+
* treated a missing key and a stored `0` alike would turn "I did not set this" into the most
|
|
23
|
+
* aggressive setting available, on every account that ever saved any unrelated field.
|
|
24
|
+
*
|
|
25
|
+
* `enabled` is deliberately a ONE-WAY switch: an account may mute itself, but cannot turn
|
|
26
|
+
* alerting on where the deployment never started the sweep: there would be no timer to
|
|
27
|
+
* honour it, and a stored `true` that does nothing is worse than an absent one.
|
|
28
|
+
*/
|
|
29
|
+
export function resolveAccountAlertConfig(deployment, stored) {
|
|
30
|
+
const t = stored?.thresholds;
|
|
31
|
+
return {
|
|
32
|
+
enabled: deployment.enabled && stored?.enabled !== false,
|
|
33
|
+
window: stored?.window ?? deployment.window,
|
|
34
|
+
thresholds: {
|
|
35
|
+
minRuns: t?.minRuns ?? deployment.thresholds.minRuns,
|
|
36
|
+
maxFailureRate: t?.maxFailureRate ?? deployment.thresholds.maxFailureRate,
|
|
37
|
+
maxP99DurationMs: t?.maxP99DurationMs ?? deployment.thresholds.maxP99DurationMs,
|
|
38
|
+
maxBacklog: t?.maxBacklog ?? deployment.thresholds.maxBacklog,
|
|
39
|
+
stalledBuckets: t?.stalledBuckets ?? deployment.thresholds.stalledBuckets,
|
|
40
|
+
minStalledPriorRuns: t?.minStalledPriorRuns ?? deployment.thresholds.minStalledPriorRuns,
|
|
41
|
+
maxFailureKindShare: t?.maxFailureKindShare ?? deployment.thresholds.maxFailureKindShare,
|
|
42
|
+
maxSweepFailures: t?.maxSweepFailures ?? deployment.thresholds.maxSweepFailures,
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The alert reasons whose evidence is a set of FAILED RUNS, so a card firing any of them can
|
|
48
|
+
* deep-link to the runs it is aggregating. The other conditions are about depth, latency or
|
|
49
|
+
* the absence of work, and have no failing run to point at. A link that resolved to nothing
|
|
50
|
+
* would read as "there are no failures", which is the opposite of what a backlog or stall
|
|
51
|
+
* alert is saying.
|
|
52
|
+
*/
|
|
53
|
+
const RUN_EVIDENCED_REASONS = new Set([
|
|
54
|
+
'failure_rate_high',
|
|
55
|
+
'failure_kind_dominant',
|
|
56
|
+
]);
|
|
57
|
+
/** Whether a fired reason set has failing runs behind it worth linking to. */
|
|
58
|
+
export function alertsHaveRunEvidence(reasons) {
|
|
59
|
+
return reasons.some((r) => RUN_EVIDENCED_REASONS.has(r));
|
|
60
|
+
}
|
|
16
61
|
/**
|
|
17
62
|
* Evaluate one account's windowed projection against the thresholds, returning every fired
|
|
18
63
|
* alert (empty when healthy). Each alert carries its observed value + the threshold it crossed
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"platform-health.logic.js","sourceRoot":"","sources":["../../../src/modules/observability/platform-health.logic.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"platform-health.logic.js","sourceRoot":"","sources":["../../../src/modules/observability/platform-health.logic.ts"],"names":[],"mappings":"AAkEA,iFAAiF;AACjF,MAAM,CAAC,MAAM,iCAAiC,GAA4B;IACxE,OAAO,EAAE,CAAC;IACV,cAAc,EAAE,GAAG;IACnB,gBAAgB,EAAE,EAAE,GAAG,MAAM,EAAE,aAAa;IAC5C,UAAU,EAAE,EAAE;IACd,uFAAuF;IACvF,4CAA4C;IAC5C,cAAc,EAAE,CAAC;IACjB,mBAAmB,EAAE,CAAC;IACtB,0FAA0F;IAC1F,gFAAgF;IAChF,mBAAmB,EAAE,GAAG;IACxB,gBAAgB,EAAE,CAAC;CACpB,CAAA;AAaD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,yBAAyB,CACvC,UAIC,EACD,MAAyC;IAEzC,MAAM,CAAC,GAAG,MAAM,EAAE,UAAU,CAAA;IAC5B,OAAO;QACL,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,MAAM,EAAE,OAAO,KAAK,KAAK;QACxD,MAAM,EAAE,MAAM,EAAE,MAAM,IAAI,UAAU,CAAC,MAAM;QAC3C,UAAU,EAAE;YACV,OAAO,EAAE,CAAC,EAAE,OAAO,IAAI,UAAU,CAAC,UAAU,CAAC,OAAO;YACpD,cAAc,EAAE,CAAC,EAAE,cAAc,IAAI,UAAU,CAAC,UAAU,CAAC,cAAc;YACzE,gBAAgB,EAAE,CAAC,EAAE,gBAAgB,IAAI,UAAU,CAAC,UAAU,CAAC,gBAAgB;YAC/E,UAAU,EAAE,CAAC,EAAE,UAAU,IAAI,UAAU,CAAC,UAAU,CAAC,UAAU;YAC7D,cAAc,EAAE,CAAC,EAAE,cAAc,IAAI,UAAU,CAAC,UAAU,CAAC,cAAc;YACzE,mBAAmB,EAAE,CAAC,EAAE,mBAAmB,IAAI,UAAU,CAAC,UAAU,CAAC,mBAAmB;YACxF,mBAAmB,EAAE,CAAC,EAAE,mBAAmB,IAAI,UAAU,CAAC,UAAU,CAAC,mBAAmB;YACxF,gBAAgB,EAAE,CAAC,EAAE,gBAAgB,IAAI,UAAU,CAAC,UAAU,CAAC,gBAAgB;SAChF;KACF,CAAA;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,qBAAqB,GAAqC,IAAI,GAAG,CAAsB;IAC3F,mBAAmB;IACnB,uBAAuB;CACxB,CAAC,CAAA;AAEF,8EAA8E;AAC9E,MAAM,UAAU,qBAAqB,CAAC,OAAuC;IAC3E,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CACpC,QAA+B,EAC/B,UAAmC;AACnC;;;;;GAKG;AACH,aAAsD;IAEtD,MAAM,MAAM,GAAoB,EAAE,CAAA;IAClC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,QAAQ,CAAA;IAE9C,4FAA4F;IAC5F,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAA;IAChD,IAAI,QAAQ,IAAI,UAAU,CAAC,OAAO,IAAI,QAAQ,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;QACpE,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAA;QAC5C,IAAI,WAAW,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC;gBACV,MAAM,EAAE,mBAAmB;gBAC3B,KAAK,EAAE,WAAW;gBAClB,SAAS,EAAE,UAAU,CAAC,cAAc;aACrC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,2FAA2F;IAC3F,IAAI,SAAS,CAAC,KAAK,KAAK,IAAI,IAAI,SAAS,CAAC,KAAK,IAAI,UAAU,CAAC,gBAAgB,EAAE,CAAC;QAC/E,MAAM,CAAC,IAAI,CAAC;YACV,MAAM,EAAE,mBAAmB;YAC3B,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,SAAS,EAAE,UAAU,CAAC,gBAAgB;SACvC,CAAC,CAAA;IACJ,CAAC;IAED,yFAAyF;IACzF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAA;IACxE,IAAI,OAAO,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;IAC3F,CAAC;IAED,2FAA2F;IAC3F,4FAA4F;IAC5F,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IACnD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC;YACV,MAAM,EAAE,oBAAoB;YAC5B,KAAK,EAAE,OAAO;YACd,SAAS,EAAE,UAAU,CAAC,cAAc;SACrC,CAAC,CAAA;IACJ,CAAC;IAED,oFAAoF;IACpF,6FAA6F;IAC7F,gFAAgF;IAChF,MAAM,aAAa,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IACpF,IAAI,QAAQ,IAAI,UAAU,CAAC,OAAO,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;QAC3E,MAAM,KAAK,GAAG,QAAQ,GAAG,aAAa,CAAA;QACtC,IAAI,KAAK,IAAI,UAAU,CAAC,mBAAmB,EAAE,CAAC;YAC5C,MAAM,CAAC,IAAI,CAAC;gBACV,MAAM,EAAE,uBAAuB;gBAC/B,KAAK,EAAE,KAAK;gBACZ,SAAS,EAAE,UAAU,CAAC,mBAAmB;aAC1C,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,0FAA0F;IAC1F,0FAA0F;IAC1F,uFAAuF;IACvF,IAAI,aAAa,IAAI,aAAa,CAAC,WAAW,IAAI,UAAU,CAAC,gBAAgB,EAAE,CAAC;QAC9E,MAAM,CAAC,IAAI,CAAC;YACV,MAAM,EAAE,gBAAgB;YACxB,KAAK,EAAE,aAAa,CAAC,WAAW;YAChC,SAAS,EAAE,UAAU,CAAC,gBAAgB;SACvC,CAAC,CAAA;IACJ,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,SAAS,aAAa,CACpB,QAA+B,EAC/B,UAAmC;IAEnC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAA;IACpC,MAAM,IAAI,GAAG,UAAU,CAAC,cAAc,CAAA;IACtC,0FAA0F;IAC1F,yFAAyF;IACzF,6FAA6F;IAC7F,+FAA+F;IAC/F,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,CAAC;QAAE,OAAO,IAAI,CAAA;IAC3F,IAAI,IAAI,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI;QAAE,OAAO,IAAI,CAAA;IACnD,MAAM,MAAM,GAAG,CAAC,KAA8B,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAA;IAC1F,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAA;IACtC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;IACnF,IAAI,KAAK,GAAG,UAAU,CAAC,mBAAmB;QAAE,OAAO,IAAI,CAAA;IACvD,6FAA6F;IAC7F,8FAA8F;IAC9F,mFAAmF;IACnF,IAAI,KAAK,GAAG,IAAI,CAAA;IAChB,OAAO,KAAK,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,KAAK,CAAE,CAAC,KAAK,CAAC;QAAE,KAAK,EAAE,CAAA;IACzF,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAuB;IAC1D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAA;AAC3C,CAAC;AAED,yFAAyF;AACzF,MAAM,aAAa,GAAwC;IACzD,iBAAiB,EAAE,8BAA8B;IACjD,iBAAiB,EAAE,0BAA0B;IAC7C,YAAY,EAAE,sCAAsC;IACpD,kBAAkB,EAAE,yCAAyC;IAC7D,qBAAqB,EAAE,wCAAwC;IAC/D,cAAc,EAAE,mDAAmD;CACpE,CAAA;AAED,MAAM,aAAa,GAAwC;IACzD,IAAI,EAAE,eAAe;IACrB,KAAK,EAAE,mBAAmB;IAC1B,IAAI,EAAE,iBAAiB;CACxB,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,yBAAyB,CACvC,OAA8B,EAC9B,MAA2B;IAE3B,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;IACpD,MAAM,IAAI,GACR,OAAO,CAAC,MAAM,IAAI,CAAC;QACjB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,gCAAgC,CAAC;QAClD,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAA;IAC7E,OAAO;QACL,KAAK,EAAE,uBAAuB;QAC9B,IAAI,EAAE,wBAAwB,IAAI,SAAS,aAAa,CAAC,MAAM,CAAC,2CAA2C;KAC5G,CAAA;AACH,CAAC"}
|