@telora/daemon 0.22.85 → 0.22.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build-info.json +4 -3
- package/dist/backends/index.d.ts +1 -1
- package/dist/backends/index.d.ts.map +1 -1
- package/dist/backends/index.js +1 -1
- package/dist/backends/index.js.map +1 -1
- package/dist/backends/resolve.d.ts +37 -8
- package/dist/backends/resolve.d.ts.map +1 -1
- package/dist/backends/resolve.js +62 -28
- package/dist/backends/resolve.js.map +1 -1
- package/dist/ci/ci-escalation.d.ts +17 -1
- package/dist/ci/ci-escalation.d.ts.map +1 -1
- package/dist/ci/ci-escalation.js +51 -3
- package/dist/ci/ci-escalation.js.map +1 -1
- package/dist/ci/ci-red-attribution.d.ts +85 -0
- package/dist/ci/ci-red-attribution.d.ts.map +1 -0
- package/dist/ci/ci-red-attribution.js +108 -0
- package/dist/ci/ci-red-attribution.js.map +1 -0
- package/dist/ci/ci-sensor-loop.d.ts +84 -80
- package/dist/ci/ci-sensor-loop.d.ts.map +1 -1
- package/dist/ci/ci-sensor-loop.js +132 -129
- package/dist/ci/ci-sensor-loop.js.map +1 -1
- package/dist/cli/connect/config-write.d.ts +7 -4
- package/dist/cli/connect/config-write.d.ts.map +1 -1
- package/dist/cli/connect/config-write.js +13 -7
- package/dist/cli/connect/config-write.js.map +1 -1
- package/dist/cli/connect.d.ts.map +1 -1
- package/dist/cli/connect.js +11 -1
- package/dist/cli/connect.js.map +1 -1
- package/dist/cli/update-commands.d.ts.map +1 -1
- package/dist/cli/update-commands.js +2 -1
- package/dist/cli/update-commands.js.map +1 -1
- package/dist/completion/coverage-stabilized.d.ts.map +1 -1
- package/dist/completion/coverage-stabilized.js +1 -1
- package/dist/completion/coverage-stabilized.js.map +1 -1
- package/dist/completion/escalations.d.ts +40 -5
- package/dist/completion/escalations.d.ts.map +1 -1
- package/dist/completion/escalations.js +19 -11
- package/dist/completion/escalations.js.map +1 -1
- package/dist/completion/event-escalations.d.ts +12 -1
- package/dist/completion/event-escalations.d.ts.map +1 -1
- package/dist/completion/event-escalations.js +73 -37
- package/dist/completion/event-escalations.js.map +1 -1
- package/dist/completion/event.d.ts +4 -1
- package/dist/completion/event.d.ts.map +1 -1
- package/dist/completion/event.js +1 -1
- package/dist/completion/event.js.map +1 -1
- package/dist/completion/human-only-act.d.ts +1 -1
- package/dist/completion/human-only-act.js +1 -1
- package/dist/completion/machine-remediable-demotion.d.ts +28 -1
- package/dist/completion/machine-remediable-demotion.d.ts.map +1 -1
- package/dist/completion/machine-remediable-demotion.js +9 -6
- package/dist/completion/machine-remediable-demotion.js.map +1 -1
- package/dist/completion/read-only-pristine.d.ts.map +1 -1
- package/dist/completion/read-only-pristine.js +4 -0
- package/dist/completion/read-only-pristine.js.map +1 -1
- package/dist/completion/review-exit-deps.js +1 -1
- package/dist/completion/review-exit-deps.js.map +1 -1
- package/dist/completion/review-exit-escalations.d.ts +10 -0
- package/dist/completion/review-exit-escalations.d.ts.map +1 -1
- package/dist/completion/review-exit-escalations.js +38 -3
- package/dist/completion/review-exit-escalations.js.map +1 -1
- package/dist/completion/review-exit-phase.d.ts.map +1 -1
- package/dist/completion/review-exit-phase.js +8 -7
- package/dist/completion/review-exit-phase.js.map +1 -1
- package/dist/completion/review-exit-shortfall.d.ts +5 -1
- package/dist/completion/review-exit-shortfall.d.ts.map +1 -1
- package/dist/completion/review-exit-shortfall.js +8 -3
- package/dist/completion/review-exit-shortfall.js.map +1 -1
- package/dist/completion/status-advance-phase.d.ts +2 -0
- package/dist/completion/status-advance-phase.d.ts.map +1 -1
- package/dist/completion/status-advance-phase.js +2 -1
- package/dist/completion/status-advance-phase.js.map +1 -1
- package/dist/completion/terminated-work-reconcile.js +2 -2
- package/dist/completion/terminated-work-reconcile.js.map +1 -1
- package/dist/delivery/delivery-advance-block.d.ts +3 -3
- package/dist/delivery/delivery-advance-block.d.ts.map +1 -1
- package/dist/delivery/delivery-advance-block.js +13 -8
- package/dist/delivery/delivery-advance-block.js.map +1 -1
- package/dist/delivery/delivery-lifecycle.d.ts +10 -0
- package/dist/delivery/delivery-lifecycle.d.ts.map +1 -1
- package/dist/delivery/delivery-lifecycle.js +12 -6
- package/dist/delivery/delivery-lifecycle.js.map +1 -1
- package/dist/delivery/delivery-merge-ops.d.ts.map +1 -1
- package/dist/delivery/delivery-merge-ops.js +39 -21
- package/dist/delivery/delivery-merge-ops.js.map +1 -1
- package/dist/delivery/external-hold.d.ts +8 -2
- package/dist/delivery/external-hold.d.ts.map +1 -1
- package/dist/delivery/external-hold.js +12 -5
- package/dist/delivery/external-hold.js.map +1 -1
- package/dist/delivery/operator-hold.d.ts +6 -1
- package/dist/delivery/operator-hold.d.ts.map +1 -1
- package/dist/delivery/operator-hold.js +31 -10
- package/dist/delivery/operator-hold.js.map +1 -1
- package/dist/delivery/paused-revival.d.ts +6 -0
- package/dist/delivery/paused-revival.d.ts.map +1 -1
- package/dist/delivery/paused-revival.js +3 -1
- package/dist/delivery/paused-revival.js.map +1 -1
- package/dist/directive/directive-dispatch.d.ts.map +1 -1
- package/dist/directive/directive-dispatch.js +6 -3
- package/dist/directive/directive-dispatch.js.map +1 -1
- package/dist/directive/directive-queue.d.ts.map +1 -1
- package/dist/directive/directive-queue.js +4 -1
- package/dist/directive/directive-queue.js.map +1 -1
- package/dist/drift/staleness-sensor.d.ts +20 -0
- package/dist/drift/staleness-sensor.d.ts.map +1 -1
- package/dist/drift/staleness-sensor.js +54 -21
- package/dist/drift/staleness-sensor.js.map +1 -1
- package/dist/drift/staleness.d.ts +18 -5
- package/dist/drift/staleness.d.ts.map +1 -1
- package/dist/drift/staleness.js +28 -15
- package/dist/drift/staleness.js.map +1 -1
- package/dist/focus/focus-executor.d.ts.map +1 -1
- package/dist/focus/focus-executor.js +6 -8
- package/dist/focus/focus-executor.js.map +1 -1
- package/dist/focus/focus-loop-registry.d.ts.map +1 -1
- package/dist/focus/focus-loop-registry.js +23 -13
- package/dist/focus/focus-loop-registry.js.map +1 -1
- package/dist/focus/focus-respawn-guard.d.ts.map +1 -1
- package/dist/focus/focus-respawn-guard.js +21 -4
- package/dist/focus/focus-respawn-guard.js.map +1 -1
- package/dist/focus/listener-auto-advance.d.ts.map +1 -1
- package/dist/focus/listener-auto-advance.js +10 -6
- package/dist/focus/listener-auto-advance.js.map +1 -1
- package/dist/focus/verify-open-issues.d.ts.map +1 -1
- package/dist/focus/verify-open-issues.js +21 -1
- package/dist/focus/verify-open-issues.js.map +1 -1
- package/dist/focus-stage-lifecycle/wedge-watchdog.d.ts.map +1 -1
- package/dist/focus-stage-lifecycle/wedge-watchdog.js +35 -14
- package/dist/focus-stage-lifecycle/wedge-watchdog.js.map +1 -1
- package/dist/loops/index.d.ts.map +1 -1
- package/dist/loops/index.js +2 -1
- package/dist/loops/index.js.map +1 -1
- package/dist/merge/focus-merge.d.ts +1 -1
- package/dist/merge/focus-merge.d.ts.map +1 -1
- package/dist/merge/focus-merge.js +5 -1
- package/dist/merge/focus-merge.js.map +1 -1
- package/dist/merge/merge-back-loop.d.ts +2 -1
- package/dist/merge/merge-back-loop.d.ts.map +1 -1
- package/dist/merge/merge-back-loop.js +15 -12
- package/dist/merge/merge-back-loop.js.map +1 -1
- package/dist/merge/merge-conflict-escalation.d.ts.map +1 -1
- package/dist/merge/merge-conflict-escalation.js +4 -0
- package/dist/merge/merge-conflict-escalation.js.map +1 -1
- package/dist/nexus/return.d.ts +14 -15
- package/dist/nexus/return.d.ts.map +1 -1
- package/dist/nexus/return.js +41 -31
- package/dist/nexus/return.js.map +1 -1
- package/dist/prompt-sections/execution-sections.d.ts +13 -5
- package/dist/prompt-sections/execution-sections.d.ts.map +1 -1
- package/dist/prompt-sections/execution-sections.js +78 -18
- package/dist/prompt-sections/execution-sections.js.map +1 -1
- package/dist/prompt-sections/routing-decider-sections.d.ts +60 -0
- package/dist/prompt-sections/routing-decider-sections.d.ts.map +1 -0
- package/dist/prompt-sections/routing-decider-sections.js +197 -0
- package/dist/prompt-sections/routing-decider-sections.js.map +1 -0
- package/dist/queries/agent-resolvable-aging.d.ts +4 -3
- package/dist/queries/agent-resolvable-aging.d.ts.map +1 -1
- package/dist/queries/agent-resolvable-aging.js +5 -4
- package/dist/queries/agent-resolvable-aging.js.map +1 -1
- package/dist/queries/agent-resolvable-promotion.d.ts +172 -124
- package/dist/queries/agent-resolvable-promotion.d.ts.map +1 -1
- package/dist/queries/agent-resolvable-promotion.js +332 -307
- package/dist/queries/agent-resolvable-promotion.js.map +1 -1
- package/dist/queries/agent-resolvable-retirement.d.ts +105 -0
- package/dist/queries/agent-resolvable-retirement.d.ts.map +1 -0
- package/dist/queries/agent-resolvable-retirement.js +227 -0
- package/dist/queries/agent-resolvable-retirement.js.map +1 -0
- package/dist/queries/cause-key.d.ts +109 -0
- package/dist/queries/cause-key.d.ts.map +1 -1
- package/dist/queries/cause-key.js +93 -1
- package/dist/queries/cause-key.js.map +1 -1
- package/dist/queries/deliveries.d.ts +14 -0
- package/dist/queries/deliveries.d.ts.map +1 -1
- package/dist/queries/deliveries.js +20 -2
- package/dist/queries/deliveries.js.map +1 -1
- package/dist/queries/delivery-gate-state.d.ts +130 -5
- package/dist/queries/delivery-gate-state.d.ts.map +1 -1
- package/dist/queries/delivery-gate-state.js +111 -2
- package/dist/queries/delivery-gate-state.js.map +1 -1
- package/dist/queries/escalation-consumption.d.ts +95 -15
- package/dist/queries/escalation-consumption.d.ts.map +1 -1
- package/dist/queries/escalation-consumption.js +162 -24
- package/dist/queries/escalation-consumption.js.map +1 -1
- package/dist/queries/escalation-dispositions.d.ts +19 -7
- package/dist/queries/escalation-dispositions.d.ts.map +1 -1
- package/dist/queries/escalation-dispositions.js +7 -0
- package/dist/queries/escalation-dispositions.js.map +1 -1
- package/dist/queries/escalation-suppression.d.ts +63 -19
- package/dist/queries/escalation-suppression.d.ts.map +1 -1
- package/dist/queries/escalation-suppression.js +64 -35
- package/dist/queries/escalation-suppression.js.map +1 -1
- package/dist/queries/escalations.d.ts +127 -70
- package/dist/queries/escalations.d.ts.map +1 -1
- package/dist/queries/escalations.js +170 -188
- package/dist/queries/escalations.js.map +1 -1
- package/dist/queries/filing-observation.d.ts +82 -0
- package/dist/queries/filing-observation.d.ts.map +1 -0
- package/dist/queries/filing-observation.js +157 -0
- package/dist/queries/filing-observation.js.map +1 -0
- package/dist/queries/holds.d.ts +69 -2
- package/dist/queries/holds.d.ts.map +1 -1
- package/dist/queries/holds.js +106 -11
- package/dist/queries/holds.js.map +1 -1
- package/dist/queries/incident-correlation.d.ts +291 -0
- package/dist/queries/incident-correlation.d.ts.map +1 -0
- package/dist/queries/incident-correlation.js +408 -0
- package/dist/queries/incident-correlation.js.map +1 -0
- package/dist/queries/routing-measurement.d.ts +48 -1
- package/dist/queries/routing-measurement.d.ts.map +1 -1
- package/dist/queries/routing-measurement.js +64 -2
- package/dist/queries/routing-measurement.js.map +1 -1
- package/dist/queries/routing-quality.d.ts +8 -0
- package/dist/queries/routing-quality.d.ts.map +1 -1
- package/dist/queries/routing-quality.js +12 -3
- package/dist/queries/routing-quality.js.map +1 -1
- package/dist/queries/shared.js +1 -1
- package/dist/shell/crash-recovery.d.ts.map +1 -1
- package/dist/shell/crash-recovery.js +6 -6
- package/dist/shell/crash-recovery.js.map +1 -1
- package/dist/shell/heartbeat-metadata.d.ts +18 -0
- package/dist/shell/heartbeat-metadata.d.ts.map +1 -1
- package/dist/shell/heartbeat-metadata.js +24 -0
- package/dist/shell/heartbeat-metadata.js.map +1 -1
- package/dist/shell/heartbeat.d.ts.map +1 -1
- package/dist/shell/heartbeat.js +6 -17
- package/dist/shell/heartbeat.js.map +1 -1
- package/dist/shell/install-probe-tick.d.ts.map +1 -1
- package/dist/shell/install-probe-tick.js +3 -1
- package/dist/shell/install-probe-tick.js.map +1 -1
- package/dist/shell/model-resolution.d.ts +9 -4
- package/dist/shell/model-resolution.d.ts.map +1 -1
- package/dist/shell/model-resolution.js +46 -7
- package/dist/shell/model-resolution.js.map +1 -1
- package/dist/shell/unified-init.d.ts +24 -0
- package/dist/shell/unified-init.d.ts.map +1 -1
- package/dist/shell/unified-init.js +22 -17
- package/dist/shell/unified-init.js.map +1 -1
- package/dist/spawn/model-id-guards.d.ts +50 -0
- package/dist/spawn/model-id-guards.d.ts.map +1 -0
- package/dist/spawn/model-id-guards.js +110 -0
- package/dist/spawn/model-id-guards.js.map +1 -0
- package/dist/spawn/model-routing.d.ts +1 -43
- package/dist/spawn/model-routing.d.ts.map +1 -1
- package/dist/spawn/model-routing.js +3 -94
- package/dist/spawn/model-routing.js.map +1 -1
- package/dist/spawn/respawn-cycle-escalation.d.ts +10 -2
- package/dist/spawn/respawn-cycle-escalation.d.ts.map +1 -1
- package/dist/spawn/respawn-cycle-escalation.js +16 -2
- package/dist/spawn/respawn-cycle-escalation.js.map +1 -1
- package/dist/spawn/routing-decider-spawn.d.ts +93 -0
- package/dist/spawn/routing-decider-spawn.d.ts.map +1 -0
- package/dist/spawn/routing-decider-spawn.js +185 -0
- package/dist/spawn/routing-decider-spawn.js.map +1 -0
- package/dist/spawn/routing-decider.d.ts +42 -0
- package/dist/spawn/routing-decider.d.ts.map +1 -0
- package/dist/spawn/routing-decider.js +95 -0
- package/dist/spawn/routing-decider.js.map +1 -0
- package/dist/spawn/shared.d.ts +8 -0
- package/dist/spawn/shared.d.ts.map +1 -1
- package/dist/spawn/shared.js +1 -1
- package/dist/spawn/shared.js.map +1 -1
- package/dist/spawn/spawn-decision.d.ts +24 -16
- package/dist/spawn/spawn-decision.d.ts.map +1 -1
- package/dist/spawn/spawn-decision.js +28 -11
- package/dist/spawn/spawn-decision.js.map +1 -1
- package/dist/spawn/worktree-session.d.ts +23 -7
- package/dist/spawn/worktree-session.d.ts.map +1 -1
- package/dist/spawn/worktree-session.js +54 -22
- package/dist/spawn/worktree-session.js.map +1 -1
- package/dist/testing/fixtures/index.js +1 -1
- package/dist/types/index.js +1 -1
- package/dist/update/auto-update.d.ts +20 -3
- package/dist/update/auto-update.d.ts.map +1 -1
- package/dist/update/auto-update.js +8 -24
- package/dist/update/auto-update.js.map +1 -1
- package/dist/update/self-update.d.ts +43 -4
- package/dist/update/self-update.d.ts.map +1 -1
- package/dist/update/self-update.js +61 -38
- package/dist/update/self-update.js.map +1 -1
- package/dist/update/version-check.d.ts +29 -45
- package/dist/update/version-check.d.ts.map +1 -1
- package/dist/update/version-check.js +48 -79
- package/dist/update/version-check.js.map +1 -1
- package/dist/verify/failure-signature.d.ts +91 -0
- package/dist/verify/failure-signature.d.ts.map +1 -0
- package/dist/verify/failure-signature.js +233 -0
- package/dist/verify/failure-signature.js.map +1 -0
- package/dist/verify/shortfall-repeat.d.ts +12 -9
- package/dist/verify/shortfall-repeat.d.ts.map +1 -1
- package/dist/verify/shortfall-repeat.js +38 -14
- package/dist/verify/shortfall-repeat.js.map +1 -1
- package/dist/verify/stall-detectors.d.ts +19 -43
- package/dist/verify/stall-detectors.d.ts.map +1 -1
- package/dist/verify/stall-detectors.js +26 -101
- package/dist/verify/stall-detectors.js.map +1 -1
- package/dist/verify/unreviewed-settle-detector.d.ts +8 -0
- package/dist/verify/unreviewed-settle-detector.d.ts.map +1 -1
- package/dist/verify/unreviewed-settle-detector.js +15 -1
- package/dist/verify/unreviewed-settle-detector.js.map +1 -1
- package/dist/verify/verification-deps-default.d.ts.map +1 -1
- package/dist/verify/verification-deps-default.js +14 -2
- package/dist/verify/verification-deps-default.js.map +1 -1
- package/dist/verify/verification-evidence-unavailable-feedback.d.ts.map +1 -1
- package/dist/verify/verification-evidence-unavailable-feedback.js +8 -2
- package/dist/verify/verification-evidence-unavailable-feedback.js.map +1 -1
- package/dist/verify/verification-feedback.d.ts +62 -14
- package/dist/verify/verification-feedback.d.ts.map +1 -1
- package/dist/verify/verification-feedback.js +101 -28
- package/dist/verify/verification-feedback.js.map +1 -1
- package/dist/verify/verify-gate-stall.d.ts +116 -0
- package/dist/verify/verify-gate-stall.d.ts.map +1 -0
- package/dist/verify/verify-gate-stall.js +169 -0
- package/dist/verify/verify-gate-stall.js.map +1 -0
- package/dist/verify/verify-restatement.d.ts.map +1 -1
- package/dist/verify/verify-restatement.js +2 -1
- package/dist/verify/verify-restatement.js.map +1 -1
- package/package.json +3 -3
|
@@ -11,25 +11,28 @@
|
|
|
11
11
|
* so the loop is unit-testable without a live remote or DB.
|
|
12
12
|
*
|
|
13
13
|
* The filing is a MACHINE receiver, not a human inbox line. Fixing a failing
|
|
14
|
-
* lint/typecheck/test/build gate is CODING work, so
|
|
15
|
-
* disposition `agent_resolvable` (hidden from the operator inbox)
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* to
|
|
19
|
-
*
|
|
20
|
-
*
|
|
14
|
+
* lint/typecheck/test/build gate is CODING work, so a focus-attributed filing
|
|
15
|
+
* carries disposition `agent_resolvable` (hidden from the operator inbox), and
|
|
16
|
+
* the daemon's focus-scoped pickup (reconcileAgentResolvableForFocus) rides it
|
|
17
|
+
* into that focus's next CODING pass. A BRANCH-ONLY filing (below) has no spawn
|
|
18
|
+
* to ride, so it is filed as a detector OBSERVATION (proposed needs_operator, no
|
|
19
|
+
* human act -- the fix is still coding work): with no focus anchor, the org-wide
|
|
20
|
+
* hand-off sweep (queries/agent-resolvable-promotion.ts) gives it to the routing
|
|
21
|
+
* decider agent on its next pass, and only that agent can make it human-facing.
|
|
22
|
+
* Neither path files a direct operator line (see ciRunRedEscalationParams).
|
|
21
23
|
*
|
|
22
|
-
* Anchor choice (decided, reversible): the
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* the
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
24
|
+
* Anchor choice (decided, reversible): the focus -- and, when it is the only one,
|
|
25
|
+
* the delivery -- whose merge INTRODUCED the red tip (ci/ci-red-attribution.ts:
|
|
26
|
+
* the tip's own `Telora-Delivery` trailer, else the deliveries of the commits the
|
|
27
|
+
* merge brought onto integration, read against its integration-side parent). When
|
|
28
|
+
* that is not determinable (no attributable commit, or work from several foci)
|
|
29
|
+
* the filing anchors BRANCH-ONLY, with no focusId -- a wrong focus is worse than
|
|
30
|
+
* none, since it rides the fix into an unrelated team and correlates the red with
|
|
31
|
+
* that focus's unrelated symptoms. The former "any currently-active focus"
|
|
32
|
+
* fallback was that guess and is gone. A branch-only filing has no team to ride
|
|
33
|
+
* into, so the held merge gate is ALSO surfaced as a health event (TEL-6); the
|
|
34
|
+
* fail-closed integration->main merge gate (resolveCiMergeGate, which reads the
|
|
35
|
+
* git ref independently of this escalation) still holds main out on the red.
|
|
33
36
|
*
|
|
34
37
|
* The cause key is BRANCH-scoped (queries/cause-key.ciRunRedCauseKey): successive
|
|
35
38
|
* red tips carrying the same unfixed breakage collapse onto ONE open filing
|
|
@@ -42,6 +45,9 @@
|
|
|
42
45
|
*/
|
|
43
46
|
import type { DaemonConfig } from '../types/index.js';
|
|
44
47
|
import { type CiStatusReader } from './ci-status.js';
|
|
48
|
+
import { type CiRedRunAnchor } from './ci-red-attribution.js';
|
|
49
|
+
import { type CreateEscalationParams } from '../queries/escalations.js';
|
|
50
|
+
import { type CallApiDeps } from '../queries/shared.js';
|
|
45
51
|
/** Health-event code for a red integration run that has no team to route to. */
|
|
46
52
|
export declare const MERGE_HELD_NO_RECEIVER_CODE = "merge.held_no_receiver";
|
|
47
53
|
/** Test-only: clear the held-gate dedup state. */
|
|
@@ -93,18 +99,25 @@ export declare function ciRunRedEscalationDescription(params: {
|
|
|
93
99
|
sha: string;
|
|
94
100
|
branch: string;
|
|
95
101
|
}): string;
|
|
102
|
+
/** Params a ci_run_red filing is built from. focusId/deliveryId null = branch-only. */
|
|
103
|
+
export interface CiRedFilingParams {
|
|
104
|
+
organizationId: string;
|
|
105
|
+
productId: string;
|
|
106
|
+
focusId: string | null;
|
|
107
|
+
deliveryId: string | null;
|
|
108
|
+
sha: string;
|
|
109
|
+
branch: string;
|
|
110
|
+
}
|
|
96
111
|
/**
|
|
97
112
|
* Structured metadata for a ci_run_red filing: the cause-key routing fields, the
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
113
|
+
* attributed anchor (focus_id / delivery_id, null when branch-only), the posture
|
|
114
|
+
* it sets and that posture's termination condition, PLUS the self-contained
|
|
115
|
+
* `diagnosis` route (see ciLogDiagnosisRoute) so a team inheriting the escalation
|
|
116
|
+
* later reads the same log route the original filing carried. Pure and exported
|
|
117
|
+
* so the metadata contract is unit-testable without mocking createEscalation.
|
|
102
118
|
*/
|
|
103
|
-
export declare function ciRunRedEscalationMetadata(params: {
|
|
104
|
-
|
|
105
|
-
branch: string;
|
|
106
|
-
productId: string;
|
|
107
|
-
focusId: string;
|
|
119
|
+
export declare function ciRunRedEscalationMetadata(params: Omit<CiRedFilingParams, 'organizationId' | 'deliveryId'> & {
|
|
120
|
+
deliveryId?: string | null;
|
|
108
121
|
}): Record<string, unknown>;
|
|
109
122
|
/** Minimal open-escalation row shape the sensor's dedup / resolve reads. */
|
|
110
123
|
interface CiEscalationRow {
|
|
@@ -115,23 +128,31 @@ interface CiEscalationRow {
|
|
|
115
128
|
metadata?: Record<string, unknown> | null;
|
|
116
129
|
}
|
|
117
130
|
/**
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
-
*
|
|
128
|
-
*
|
|
131
|
+
* An OPEN ci_run_red row for this product, including the two HAND-OFF rows that
|
|
132
|
+
* stand in for one:
|
|
133
|
+
* - a routing decider's human-facing filing (MCP escalate with sourceCauseKey):
|
|
134
|
+
* kind agent_filed, carrying metadata.original_cause_key `ci_run_red::...` and
|
|
135
|
+
* metadata.originalEscalationId but NO originalKind;
|
|
136
|
+
* - a historic promotion-backstop refile (the daemon no longer files these):
|
|
137
|
+
* re-kinded agent_resolvable_promoted, carrying metadata.originalKind.
|
|
138
|
+
* A row's originalKind, when it carries one, is authoritative for its origin; the
|
|
139
|
+
* original_cause_key prefix decides only for a row without it (the decider's).
|
|
140
|
+
* Covering both lets a later green resolve the human-facing row too (no stale
|
|
141
|
+
* inbox item once the red cleared), and keeps an open one suppressing a fresh
|
|
142
|
+
* branch-only observation. Exported for direct unit testing.
|
|
129
143
|
*/
|
|
144
|
+
export declare function isOpenCiRedForProduct(e: CiEscalationRow, productId: string): boolean;
|
|
130
145
|
/**
|
|
131
146
|
* Pure suppression predicate over already-fetched rows. A re-file for this red
|
|
132
|
-
* sha is suppressed when an
|
|
133
|
-
* (dismissal memory
|
|
134
|
-
*
|
|
147
|
+
* sha is suppressed when an OPEN or DISMISSED row carries the same cause key
|
|
148
|
+
* (dismissal memory: an operator-dismissed branch stays silent until a new
|
|
149
|
+
* breakage resolves and re-reds it), an OPEN row carries it as a live absorbed
|
|
150
|
+
* symptom (the filing would only be "already carried" by the incident reducer --
|
|
151
|
+
* and a green tick must still see it open, so resolve-on-green retires it), or a
|
|
152
|
+
* legacy OPEN product+kind row with no cause_key exists (deployment-window
|
|
153
|
+
* back-compat). A later green run resolves the open row via
|
|
154
|
+
* resolveCiRedEscalations. Exported for direct unit testing (no callApi mocking
|
|
155
|
+
* needed).
|
|
135
156
|
*/
|
|
136
157
|
export declare function isCiRedRefileSuppressed(rows: readonly CiEscalationRow[], params: {
|
|
137
158
|
productId: string;
|
|
@@ -139,57 +160,40 @@ export declare function isCiRedRefileSuppressed(rows: readonly CiEscalationRow[]
|
|
|
139
160
|
branch: string;
|
|
140
161
|
}): boolean;
|
|
141
162
|
/**
|
|
142
|
-
* Resolve the
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
* product (a CI-gate fix is cross-cutting, so any armed team can take it). Returns
|
|
149
|
-
* null when nothing resolves -- there is no coding pass to route the fix to, so
|
|
150
|
-
* the caller files nothing and retries next tick (the fail-closed merge gate
|
|
151
|
-
* still holds main). Best-effort: any git/DB failure degrades to null, never throws.
|
|
163
|
+
* Resolve-on-green (the production resolveCiRedEscalations): resolve the product's
|
|
164
|
+
* open ci_run_red rows (and the hand-off rows standing in for one: a routing decider's
|
|
165
|
+
* human-facing filing, a historic promoted refile) auto_swept, then retire a ci_run_red
|
|
166
|
+
* absorbed as a symptom onto another open row of the product (incident contract
|
|
167
|
+
* rule 6), over the same read. Best-effort per row. Exported with a callApi seam
|
|
168
|
+
* for tests.
|
|
152
169
|
*/
|
|
170
|
+
export declare function defaultResolveCiRedEscalations(productId: string, deps?: CallApiDeps): Promise<number>;
|
|
153
171
|
/**
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
}
|
|
166
|
-
/**
|
|
167
|
-
* Resolve the focus a red run's fix routes to. Order: (1) the tip's own
|
|
168
|
-
* `Telora-Delivery` trailer; (2) when the tip is a MERGE (no own trailer), the
|
|
169
|
-
* owning focus of the first trailered commit the merge introduced; (3) fallback
|
|
170
|
-
* to any active focus. The arbitrary-active-focus fallback is reached ONLY when
|
|
171
|
-
* no attributable commit exists at all. Best-effort: any git/DB failure degrades
|
|
172
|
-
* to the fallback / null, never throws.
|
|
172
|
+
* The ci_run_red filing. Pure and exported so its routing is unit-testable (the
|
|
173
|
+
* poll-loop tests inject a mock fileCiRedEscalation).
|
|
174
|
+
*
|
|
175
|
+
* - Focus-attributed: agent_resolvable. Fixing a failing CI gate is CODING work;
|
|
176
|
+
* the focus whose merge turned integration red carries it into its next coding
|
|
177
|
+
* pass (the focus-scoped agent_resolvable pickup).
|
|
178
|
+
* - Branch-only: a plain agent_resolvable row reaches no spawn and is never
|
|
179
|
+
* promoted, so the red would sit invisible while the merge gate holds main. It
|
|
180
|
+
* is proposed needs_operator with an explicit NO_HUMAN_ACT (the fix is still
|
|
181
|
+
* coding work, not a human act), so the filing chokepoint files it as a
|
|
182
|
+
* detector OBSERVATION, which the anchorless promotion sweep surfaces next pass.
|
|
173
183
|
*/
|
|
174
|
-
export declare function
|
|
184
|
+
export declare function ciRunRedEscalationParams(params: CiRedFilingParams): CreateEscalationParams;
|
|
175
185
|
/** Injected IO for the sensor tick. Defaults reach for the real implementations. */
|
|
176
186
|
export interface CiSensorDeps {
|
|
177
187
|
getCiStatus: CiStatusReader;
|
|
178
188
|
resolveIntegrationTip: (config: DaemonConfig) => string | null;
|
|
179
|
-
/**
|
|
180
|
-
|
|
189
|
+
/** Attribute a red run to the focus/delivery whose merge introduced it (branch-only when not determinable). */
|
|
190
|
+
resolveRedRunAnchor: (config: DaemonConfig, sha: string) => Promise<CiRedRunAnchor>;
|
|
181
191
|
hasOpenCiRedEscalation: (params: {
|
|
182
192
|
productId: string;
|
|
183
193
|
sha: string;
|
|
184
194
|
branch: string;
|
|
185
195
|
}) => Promise<boolean>;
|
|
186
|
-
fileCiRedEscalation: (params:
|
|
187
|
-
organizationId: string;
|
|
188
|
-
productId: string;
|
|
189
|
-
focusId: string;
|
|
190
|
-
sha: string;
|
|
191
|
-
branch: string;
|
|
192
|
-
}) => Promise<void>;
|
|
196
|
+
fileCiRedEscalation: (params: CiRedFilingParams) => Promise<void>;
|
|
193
197
|
resolveCiRedEscalations: (productId: string) => Promise<number>;
|
|
194
198
|
}
|
|
195
199
|
export interface CiSensorTickCounts {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ci-sensor-loop.d.ts","sourceRoot":"","sources":["../../src/ci/ci-sensor-loop.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ci-sensor-loop.d.ts","sourceRoot":"","sources":["../../src/ci/ci-sensor-loop.ts"],"names":[],"mappings":"AAMA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAuD,KAAK,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAE1G,OAAO,EAAuB,KAAK,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACnF,OAAO,EAAoE,KAAK,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAG1I,OAAO,EAAW,KAAK,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAOjE,gFAAgF;AAChF,eAAO,MAAM,2BAA2B,2BAA2B,CAAC;AAUpE,kDAAkD;AAClD,wBAAgB,kCAAkC,IAAI,IAAI,CAEzD;AAGD,uDAAuD;AACvD,eAAO,MAAM,0BAA0B,eAAe,CAAC;AAEvD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,kBAAkB,sBAAsB,CAAC;AACtD,eAAO,MAAM,6BAA6B,wBAAwB,CAAC;AAEnE,qEAAqE;AACrE,eAAO,MAAM,kBAAkB,qBAAqB,CAAC;AAErD,sEAAsE;AACtE,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,mBAAmB,CAQhF;AAED;;;;;;GAMG;AACH,wBAAgB,6BAA6B,CAAC,MAAM,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAiB7F;AAED,uFAAuF;AACvF,MAAM,WAAW,iBAAiB;IAChC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,gBAAgB,GAAG,YAAY,CAAC,GAAG;IAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAChG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAYzB;AAED,4EAA4E;AAC5E,UAAU,eAAe;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC3C;AASD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAUpF;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,SAAS,eAAe,EAAE,EAChC,MAAM,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACzD,OAAO,CAkBT;AAaD;;;;;;;GAOG;AACH,wBAAsB,8BAA8B,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAkB/G;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,iBAAiB,GAAG,sBAAsB,CAqB1F;AAMD,oFAAoF;AACpF,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,cAAc,CAAC;IAC5B,qBAAqB,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/D,+GAA+G;IAC/G,mBAAmB,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IACpF,sBAAsB,EAAE,CAAC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACzG,mBAAmB,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,uBAAuB,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACjE;AAWD,MAAM,WAAW,kBAAkB;IACjC,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iFAAiF;IACjF,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAwBD;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,YAAY,EACpB,IAAI,GAAE,YAA0B,GAC/B,OAAO,CAAC,kBAAkB,CAAC,CAkD7B"}
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
// PINNED-BY-BEGIN (generated by gen:pinned-by -- do not hand-edit; checked by lint:pinned-by)
|
|
2
|
-
// Pinned by: packages/daemon/src/ci/ci-
|
|
3
|
-
// packages/daemon/src/
|
|
2
|
+
// Pinned by: packages/daemon/src/ci/ci-sensor-loop.test.ts,
|
|
3
|
+
// packages/daemon/src/queries/incident-correlation.test.ts,
|
|
4
|
+
// packages/daemon/src/queries/incident-stamping.test.ts
|
|
4
5
|
// PINNED-BY-END
|
|
5
6
|
import { configForProduct } from '../shell/config.js';
|
|
6
7
|
import { decideCiSensorAction, createGitNativeCiStatusReader } from './ci-status.js';
|
|
7
8
|
import { listCiStatusRefs, resolveRemoteIntegrationTip } from './ci-status-git.js';
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
9
|
+
import { resolveRedRunAnchor } from './ci-red-attribution.js';
|
|
10
|
+
import { createEscalation, markEscalationResolved, retireAbsorbedSymptoms } from '../queries/escalations.js';
|
|
11
|
+
import { NO_HUMAN_ACT } from '../queries/filing-observation.js';
|
|
12
|
+
import { openRowSymptomCauseKeys } from '../queries/incident-correlation.js';
|
|
12
13
|
import { callApi } from '../queries/shared.js';
|
|
13
14
|
import { ciRunRedCauseKey } from '../queries/cause-key.js';
|
|
14
15
|
import { ESCALATION_REASONS, ESCALATION_DISPOSITIONS, healthEvents } from '@telora/daemon-core';
|
|
@@ -81,15 +82,16 @@ export function ciRunRedEscalationDescription(params) {
|
|
|
81
82
|
`Do not hand-roll the Gitea API chain or wait for a human to paste the log. Full ` +
|
|
82
83
|
`reference: ${route.runbook}.\n\n` +
|
|
83
84
|
`This filing auto-resolves when a later integration run goes green; if ` +
|
|
84
|
-
`
|
|
85
|
+
`no agent clears it, the agent_resolvable promotion backstop surfaces ` +
|
|
85
86
|
`it to an operator.`);
|
|
86
87
|
}
|
|
87
88
|
/**
|
|
88
89
|
* Structured metadata for a ci_run_red filing: the cause-key routing fields, the
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
90
|
+
* attributed anchor (focus_id / delivery_id, null when branch-only), the posture
|
|
91
|
+
* it sets and that posture's termination condition, PLUS the self-contained
|
|
92
|
+
* `diagnosis` route (see ciLogDiagnosisRoute) so a team inheriting the escalation
|
|
93
|
+
* later reads the same log route the original filing carried. Pure and exported
|
|
94
|
+
* so the metadata contract is unit-testable without mocking createEscalation.
|
|
93
95
|
*/
|
|
94
96
|
export function ciRunRedEscalationMetadata(params) {
|
|
95
97
|
return {
|
|
@@ -97,6 +99,7 @@ export function ciRunRedEscalationMetadata(params) {
|
|
|
97
99
|
branch: params.branch,
|
|
98
100
|
product_id: params.productId,
|
|
99
101
|
focus_id: params.focusId,
|
|
102
|
+
delivery_id: params.deliveryId ?? null,
|
|
100
103
|
cause_key: ciRunRedCauseKey(params.branch),
|
|
101
104
|
posture: CI_RUN_RED_POSTURE,
|
|
102
105
|
terminates_when: CI_RUN_RED_POSTURE_TERMINATES,
|
|
@@ -106,32 +109,46 @@ export function ciRunRedEscalationMetadata(params) {
|
|
|
106
109
|
const OPEN_STATUSES = new Set(['pending', 'in_review']);
|
|
107
110
|
/** Statuses whose cause_key suppresses a re-file: open (idempotency) + dismissed (memory). */
|
|
108
111
|
const SUPPRESSING_STATUSES = new Set(['pending', 'in_review', 'dismissed']);
|
|
109
|
-
|
|
112
|
+
/** The prefix every ci_run_red cause key carries (queries/cause-key.ciRunRedCauseKey). */
|
|
113
|
+
const CI_RUN_RED_CAUSE_KEY_PREFIX = `${CI_RUN_RED_ESCALATION_KIND}::`;
|
|
114
|
+
/**
|
|
115
|
+
* An OPEN ci_run_red row for this product, including the two HAND-OFF rows that
|
|
116
|
+
* stand in for one:
|
|
117
|
+
* - a routing decider's human-facing filing (MCP escalate with sourceCauseKey):
|
|
118
|
+
* kind agent_filed, carrying metadata.original_cause_key `ci_run_red::...` and
|
|
119
|
+
* metadata.originalEscalationId but NO originalKind;
|
|
120
|
+
* - a historic promotion-backstop refile (the daemon no longer files these):
|
|
121
|
+
* re-kinded agent_resolvable_promoted, carrying metadata.originalKind.
|
|
122
|
+
* A row's originalKind, when it carries one, is authoritative for its origin; the
|
|
123
|
+
* original_cause_key prefix decides only for a row without it (the decider's).
|
|
124
|
+
* Covering both lets a later green resolve the human-facing row too (no stale
|
|
125
|
+
* inbox item once the red cleared), and keeps an open one suppressing a fresh
|
|
126
|
+
* branch-only observation. Exported for direct unit testing.
|
|
127
|
+
*/
|
|
128
|
+
export function isOpenCiRedForProduct(e, productId) {
|
|
110
129
|
if (e.status && !OPEN_STATUSES.has(e.status))
|
|
111
130
|
return false;
|
|
112
|
-
if (e.escalationKind !== CI_RUN_RED_ESCALATION_KIND)
|
|
113
|
-
return false;
|
|
114
131
|
const meta = e.metadata ?? null;
|
|
115
|
-
|
|
132
|
+
const originalKind = meta?.originalKind;
|
|
133
|
+
const originalCauseKey = meta?.original_cause_key;
|
|
134
|
+
const handedOffCiRed = typeof originalKind === 'string'
|
|
135
|
+
? originalKind === CI_RUN_RED_ESCALATION_KIND
|
|
136
|
+
: typeof originalCauseKey === 'string' && originalCauseKey.startsWith(CI_RUN_RED_CAUSE_KEY_PREFIX);
|
|
137
|
+
if (e.escalationKind !== CI_RUN_RED_ESCALATION_KIND && !handedOffCiRed)
|
|
138
|
+
return false;
|
|
139
|
+
return e.productId === productId || meta?.productId === productId || meta?.product_id === productId;
|
|
116
140
|
}
|
|
117
|
-
/**
|
|
118
|
-
* Whether a re-file for THIS red sha must be suppressed. Two ways it is:
|
|
119
|
-
* - an OPEN or DISMISSED row already carries the same cause key (sha+branch) --
|
|
120
|
-
* the cause-keyed path with DISMISSAL MEMORY. This is what fixes the incident
|
|
121
|
-
* where an operator dismissed a ci_run_red for a sha and the daemon re-filed
|
|
122
|
-
* the SAME sha on the next scan: a dismissed cause_key now keeps it silent
|
|
123
|
-
* until a NEW sha (a different cause key) goes red.
|
|
124
|
-
* - a legacy OPEN product+kind row with no cause_key exists (deployment-window
|
|
125
|
-
* back-compat for rows filed before this change).
|
|
126
|
-
*
|
|
127
|
-
* A new sha yields a different cause key with no matching open/dismissed row and
|
|
128
|
-
* re-files; a later green run resolves the open row via resolveCiRedEscalations.
|
|
129
|
-
*/
|
|
130
141
|
/**
|
|
131
142
|
* Pure suppression predicate over already-fetched rows. A re-file for this red
|
|
132
|
-
* sha is suppressed when an
|
|
133
|
-
* (dismissal memory
|
|
134
|
-
*
|
|
143
|
+
* sha is suppressed when an OPEN or DISMISSED row carries the same cause key
|
|
144
|
+
* (dismissal memory: an operator-dismissed branch stays silent until a new
|
|
145
|
+
* breakage resolves and re-reds it), an OPEN row carries it as a live absorbed
|
|
146
|
+
* symptom (the filing would only be "already carried" by the incident reducer --
|
|
147
|
+
* and a green tick must still see it open, so resolve-on-green retires it), or a
|
|
148
|
+
* legacy OPEN product+kind row with no cause_key exists (deployment-window
|
|
149
|
+
* back-compat). A later green run resolves the open row via
|
|
150
|
+
* resolveCiRedEscalations. Exported for direct unit testing (no callApi mocking
|
|
151
|
+
* needed).
|
|
135
152
|
*/
|
|
136
153
|
export function isCiRedRefileSuppressed(rows, params) {
|
|
137
154
|
// Branch-scoped: one open ci_run_red per red integration branch. Successive
|
|
@@ -140,11 +157,16 @@ export function isCiRedRefileSuppressed(rows, params) {
|
|
|
140
157
|
return rows.some((e) => {
|
|
141
158
|
// A cause-key match suppresses only for OPEN or DISMISSED rows -- 'resolved'
|
|
142
159
|
// is excluded so a recurrence of the same sha after a green resolve re-files.
|
|
160
|
+
// original_cause_key marks a hand-off row for a branch-only observation (a
|
|
161
|
+
// routing decider's filing with sourceCauseKey, or a historic promotion
|
|
162
|
+
// refile): it must keep suppressing, or the sensor would re-file while it is open.
|
|
143
163
|
if (SUPPRESSING_STATUSES.has(e.status ?? '')) {
|
|
144
164
|
const meta = e.metadata ?? null;
|
|
145
|
-
if (
|
|
165
|
+
if (meta?.cause_key === causeKey || meta?.original_cause_key === causeKey)
|
|
146
166
|
return true;
|
|
147
167
|
}
|
|
168
|
+
if (openRowSymptomCauseKeys(e).includes(causeKey))
|
|
169
|
+
return true;
|
|
148
170
|
// Legacy back-compat: an OPEN product+kind row with no cause_key still suppresses.
|
|
149
171
|
return isOpenCiRedForProduct(e, params.productId);
|
|
150
172
|
});
|
|
@@ -157,8 +179,16 @@ async function defaultHasOpenCiRedEscalation(params) {
|
|
|
157
179
|
});
|
|
158
180
|
return isCiRedRefileSuppressed(result.escalations ?? [], params);
|
|
159
181
|
}
|
|
160
|
-
|
|
161
|
-
|
|
182
|
+
/**
|
|
183
|
+
* Resolve-on-green (the production resolveCiRedEscalations): resolve the product's
|
|
184
|
+
* open ci_run_red rows (and the hand-off rows standing in for one: a routing decider's
|
|
185
|
+
* human-facing filing, a historic promoted refile) auto_swept, then retire a ci_run_red
|
|
186
|
+
* absorbed as a symptom onto another open row of the product (incident contract
|
|
187
|
+
* rule 6), over the same read. Best-effort per row. Exported with a callApi seam
|
|
188
|
+
* for tests.
|
|
189
|
+
*/
|
|
190
|
+
export async function defaultResolveCiRedEscalations(productId, deps = {}) {
|
|
191
|
+
const result = await (deps.callApi ?? callApi)('escalation_list', {
|
|
162
192
|
fields: { statuses: ['pending', 'in_review'] },
|
|
163
193
|
limit: 500,
|
|
164
194
|
});
|
|
@@ -166,28 +196,43 @@ async function defaultResolveCiRedEscalations(productId) {
|
|
|
166
196
|
let resolved = 0;
|
|
167
197
|
for (const e of open) {
|
|
168
198
|
try {
|
|
169
|
-
await markEscalationResolved(e.id, 'Auto-resolved by the daemon: a later CI run on integration went green.');
|
|
199
|
+
await markEscalationResolved(e.id, 'auto_swept', 'Auto-resolved by the daemon: a later CI run on integration went green.', deps);
|
|
170
200
|
resolved++;
|
|
171
201
|
}
|
|
172
202
|
catch (err) {
|
|
173
203
|
log.warn(`Failed to resolve ci_run_red escalation ${e.id}: ${err.message}`);
|
|
174
204
|
}
|
|
175
205
|
}
|
|
206
|
+
const clearing = { kinds: [CI_RUN_RED_ESCALATION_KIND], scope: { productId }, resolutionClass: 'auto_swept' };
|
|
207
|
+
await retireAbsorbedSymptoms(clearing, { rows: result.escalations ?? [], skipIds: open.map((e) => e.id) }, deps);
|
|
176
208
|
return resolved;
|
|
177
209
|
}
|
|
178
|
-
|
|
179
|
-
|
|
210
|
+
/**
|
|
211
|
+
* The ci_run_red filing. Pure and exported so its routing is unit-testable (the
|
|
212
|
+
* poll-loop tests inject a mock fileCiRedEscalation).
|
|
213
|
+
*
|
|
214
|
+
* - Focus-attributed: agent_resolvable. Fixing a failing CI gate is CODING work;
|
|
215
|
+
* the focus whose merge turned integration red carries it into its next coding
|
|
216
|
+
* pass (the focus-scoped agent_resolvable pickup).
|
|
217
|
+
* - Branch-only: a plain agent_resolvable row reaches no spawn and is never
|
|
218
|
+
* promoted, so the red would sit invisible while the merge gate holds main. It
|
|
219
|
+
* is proposed needs_operator with an explicit NO_HUMAN_ACT (the fix is still
|
|
220
|
+
* coding work, not a human act), so the filing chokepoint files it as a
|
|
221
|
+
* detector OBSERVATION, which the anchorless promotion sweep surfaces next pass.
|
|
222
|
+
*/
|
|
223
|
+
export function ciRunRedEscalationParams(params) {
|
|
224
|
+
return {
|
|
180
225
|
organizationId: params.organizationId,
|
|
181
226
|
issueId: null,
|
|
182
227
|
reasonType: ESCALATION_REASONS.ERROR_ENCOUNTERED,
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
// promotion backstop, after the machine has failed to clear it.
|
|
188
|
-
disposition: ESCALATION_DISPOSITIONS.AGENT_RESOLVABLE,
|
|
228
|
+
disposition: params.focusId
|
|
229
|
+
? ESCALATION_DISPOSITIONS.AGENT_RESOLVABLE
|
|
230
|
+
: ESCALATION_DISPOSITIONS.NEEDS_OPERATOR,
|
|
231
|
+
humanAct: NO_HUMAN_ACT,
|
|
189
232
|
productId: params.productId,
|
|
190
|
-
|
|
233
|
+
// Branch-only when unattributable: no focus/delivery is guessed.
|
|
234
|
+
...(params.focusId ? { focusId: params.focusId } : {}),
|
|
235
|
+
...(params.deliveryId ? { deliveryId: params.deliveryId } : {}),
|
|
191
236
|
escalationKind: CI_RUN_RED_ESCALATION_KIND,
|
|
192
237
|
description: ciRunRedEscalationDescription(params),
|
|
193
238
|
// cause_key (branch) collapses successive red tips onto one open filing and
|
|
@@ -195,68 +240,40 @@ async function defaultFileCiRedEscalation(params) {
|
|
|
195
240
|
// The posture + terminates_when keys make the obligation's exit legible from
|
|
196
241
|
// the escalation itself (see ciRunRedEscalationMetadata).
|
|
197
242
|
metadata: ciRunRedEscalationMetadata(params),
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
const defaultOwningFocusResolverDeps = {
|
|
201
|
-
readTrailer: readDeliveryTrailer,
|
|
202
|
-
resolveMergeRange: resolveRecordedMergeRange,
|
|
203
|
-
listCommits: listRangeCommits,
|
|
204
|
-
focusForDelivery: async (deliveryId) => (await getDeliveryContext(deliveryId))?.focus?.id ?? null,
|
|
205
|
-
firstActiveFocus: async (config) => (await getActiveFocuses(config.organizationId, config.productId))[0]?.focus_id ?? null,
|
|
206
|
-
};
|
|
207
|
-
/**
|
|
208
|
-
* Resolve the focus a red run's fix routes to. Order: (1) the tip's own
|
|
209
|
-
* `Telora-Delivery` trailer; (2) when the tip is a MERGE (no own trailer), the
|
|
210
|
-
* owning focus of the first trailered commit the merge introduced; (3) fallback
|
|
211
|
-
* to any active focus. The arbitrary-active-focus fallback is reached ONLY when
|
|
212
|
-
* no attributable commit exists at all. Best-effort: any git/DB failure degrades
|
|
213
|
-
* to the fallback / null, never throws.
|
|
214
|
-
*/
|
|
215
|
-
export async function resolveOwningFocus(config, sha, deps = defaultOwningFocusResolverDeps) {
|
|
216
|
-
try {
|
|
217
|
-
const deliveryId = deps.readTrailer(config.repoPath, sha);
|
|
218
|
-
if (deliveryId) {
|
|
219
|
-
const focusId = await deps.focusForDelivery(deliveryId);
|
|
220
|
-
if (focusId)
|
|
221
|
-
return focusId;
|
|
222
|
-
}
|
|
223
|
-
// Tip has no own trailer (the merge-commit case that made this path degrade
|
|
224
|
-
// to the arbitrary-active-focus fallback and route red runs to the wrong
|
|
225
|
-
// focus). Walk the merge's own commits and route to their owning focus.
|
|
226
|
-
const mergeRange = deps.resolveMergeRange(config.repoPath, sha);
|
|
227
|
-
if (mergeRange) {
|
|
228
|
-
for (const commit of deps.listCommits(config.repoPath, mergeRange)) {
|
|
229
|
-
const did = deps.readTrailer(config.repoPath, commit);
|
|
230
|
-
if (!did)
|
|
231
|
-
continue;
|
|
232
|
-
const focusId = await deps.focusForDelivery(did);
|
|
233
|
-
if (focusId)
|
|
234
|
-
return focusId;
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
catch (err) {
|
|
239
|
-
log.debug(`owning-focus trailer resolution failed for ${sha.slice(0, 12)}: ${err.message}`);
|
|
240
|
-
}
|
|
241
|
-
try {
|
|
242
|
-
return await deps.firstActiveFocus(config);
|
|
243
|
-
}
|
|
244
|
-
catch (err) {
|
|
245
|
-
log.debug(`active-focus fallback failed for product ${config.productId}: ${err.message}`);
|
|
246
|
-
return null;
|
|
247
|
-
}
|
|
243
|
+
};
|
|
248
244
|
}
|
|
249
|
-
function
|
|
250
|
-
|
|
245
|
+
async function defaultFileCiRedEscalation(params) {
|
|
246
|
+
await createEscalation(ciRunRedEscalationParams(params));
|
|
251
247
|
}
|
|
252
248
|
const defaultDeps = {
|
|
253
249
|
getCiStatus: createGitNativeCiStatusReader(listCiStatusRefs),
|
|
254
250
|
resolveIntegrationTip: resolveRemoteIntegrationTip,
|
|
255
|
-
|
|
251
|
+
resolveRedRunAnchor: (config, sha) => resolveRedRunAnchor(config, sha),
|
|
256
252
|
hasOpenCiRedEscalation: defaultHasOpenCiRedEscalation,
|
|
257
253
|
fileCiRedEscalation: defaultFileCiRedEscalation,
|
|
258
254
|
resolveCiRedEscalations: defaultResolveCiRedEscalations,
|
|
259
255
|
};
|
|
256
|
+
/**
|
|
257
|
+
* Surface a red run that has no team to route to as durable operator-visible
|
|
258
|
+
* STATE (TEL-6): a health event, not an escalation -- the hold clears itself when
|
|
259
|
+
* the run goes green. Emitted once per red episode (cleared on green in the tick).
|
|
260
|
+
* Returns whether this call emitted.
|
|
261
|
+
*/
|
|
262
|
+
function surfaceHeldGate(productId, branch, sha) {
|
|
263
|
+
const heldKey = `${productId}:${sha}`;
|
|
264
|
+
if (surfacedHeldGates.has(heldKey))
|
|
265
|
+
return false;
|
|
266
|
+
surfacedHeldGates.add(heldKey);
|
|
267
|
+
healthEvents.emit({
|
|
268
|
+
severity: 'error',
|
|
269
|
+
source: `ci:${productId}`,
|
|
270
|
+
code: MERGE_HELD_NO_RECEIVER_CODE,
|
|
271
|
+
message: `Integration '${branch}' is RED at ${sha.slice(0, 12)} and the merge gate is holding ` +
|
|
272
|
+
'main out, but the red could not be attributed to one focus -- there is no team to route ' +
|
|
273
|
+
'the fix to (filed branch-only). Push a green fix, or pick up the branch-scoped ci_run_red.',
|
|
274
|
+
});
|
|
275
|
+
return true;
|
|
276
|
+
}
|
|
260
277
|
/**
|
|
261
278
|
* One CI-result sensor tick over every configured product. Per product: resolve
|
|
262
279
|
* the remote integration tip, read its CI status (fail-closed), and file/resolve
|
|
@@ -276,40 +293,26 @@ export async function runCiSensorTick(config, deps = defaultDeps) {
|
|
|
276
293
|
const hasOpen = await deps.hasOpenCiRedEscalation({ productId: pc.productId, sha, branch: pc.integrationBranch });
|
|
277
294
|
const action = decideCiSensorAction(status, hasOpen);
|
|
278
295
|
if (action === 'file') {
|
|
279
|
-
const
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
//
|
|
294
|
-
|
|
295
|
-
if (
|
|
296
|
-
surfacedHeldGates.add(heldKey);
|
|
296
|
+
const anchor = await deps.resolveRedRunAnchor(pc, sha);
|
|
297
|
+
await deps.fileCiRedEscalation({
|
|
298
|
+
organizationId: pc.organizationId,
|
|
299
|
+
productId: pc.productId,
|
|
300
|
+
focusId: anchor.focusId,
|
|
301
|
+
deliveryId: anchor.deliveryId,
|
|
302
|
+
sha,
|
|
303
|
+
branch: pc.integrationBranch,
|
|
304
|
+
});
|
|
305
|
+
counts.escalationsFiled++;
|
|
306
|
+
if (anchor.focusId) {
|
|
307
|
+
log.warn(`ci_run_red (agent_resolvable) filed for product ${pc.productId}, focus ${anchor.focusId.slice(0, 8)} (integration @ ${sha.slice(0, 12)}).`);
|
|
308
|
+
}
|
|
309
|
+
else {
|
|
310
|
+
// Not determinable: filed branch-only (never a guessed focus). No team
|
|
311
|
+
// rides it, so the held gate must not be SILENT either.
|
|
312
|
+
if (surfaceHeldGate(pc.productId, pc.integrationBranch, sha))
|
|
297
313
|
counts.heldGatesSurfaced++;
|
|
298
|
-
|
|
299
|
-
severity: 'error',
|
|
300
|
-
source: `ci:${pc.productId}`,
|
|
301
|
-
code: MERGE_HELD_NO_RECEIVER_CODE,
|
|
302
|
-
message: `Integration '${pc.integrationBranch}' is RED at ${sha.slice(0, 12)} and the ` +
|
|
303
|
-
'merge gate is holding main out, but no armed focus owns the fix -- there is no ' +
|
|
304
|
-
'team to route it to. Arm a focus on this product (or push a green fix) to clear the hold.',
|
|
305
|
-
});
|
|
306
|
-
}
|
|
307
|
-
log.warn(`CI red on ${pc.integrationBranch} @ ${sha.slice(0, 12)} but no armed focus to route the fix to; holding (merge gate still fail-closed, surfaced as ${MERGE_HELD_NO_RECEIVER_CODE}), will retry.`);
|
|
308
|
-
continue;
|
|
314
|
+
log.warn(`ci_run_red filed BRANCH-ONLY as an observation for product ${pc.productId} (integration @ ${sha.slice(0, 12)}): the red could not be attributed to one focus, so the promotion sweep surfaces it; merge gate still fail-closed, surfaced as ${MERGE_HELD_NO_RECEIVER_CODE}.`);
|
|
309
315
|
}
|
|
310
|
-
await deps.fileCiRedEscalation({ organizationId: pc.organizationId, productId: pc.productId, focusId, sha, branch: pc.integrationBranch });
|
|
311
|
-
counts.escalationsFiled++;
|
|
312
|
-
log.warn(`ci_run_red (agent_resolvable) filed for product ${pc.productId}, focus ${focusId.slice(0, 8)} (integration @ ${sha.slice(0, 12)}).`);
|
|
313
316
|
}
|
|
314
317
|
else if (action === 'resolve') {
|
|
315
318
|
counts.escalationsResolved += await deps.resolveCiRedEscalations(pc.productId);
|