in2-agent-server 0.20260821.185429
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +369 -0
- package/agents/plan-requirement.md +32 -0
- package/dist/campfire/configItem.js +201 -0
- package/dist/campfire/configItemDescription.js +379 -0
- package/dist/campfire/deriveFromBundle.js +1013 -0
- package/dist/cli.js +465 -0
- package/dist/core/betaFeedback.js +493 -0
- package/dist/core/chapterInstructions.js +473 -0
- package/dist/core/claudeAssets.js +61 -0
- package/dist/core/gate.js +201 -0
- package/dist/core/ids.js +9 -0
- package/dist/core/install.js +60 -0
- package/dist/core/operatorQa.js +462 -0
- package/dist/core/orgCallMeter.js +178 -0
- package/dist/core/parcels.js +1235 -0
- package/dist/core/paths.js +105 -0
- package/dist/core/phases.js +225 -0
- package/dist/core/planFile.js +251 -0
- package/dist/core/planFragments.js +119 -0
- package/dist/core/readsLog.js +398 -0
- package/dist/core/registration.js +24 -0
- package/dist/core/result.js +38 -0
- package/dist/core/resume.js +233 -0
- package/dist/core/revisionChain.js +351 -0
- package/dist/core/richText.js +61 -0
- package/dist/core/semanticHash.js +89 -0
- package/dist/core/sessionOrg.js +60 -0
- package/dist/core/sfQuery.js +19 -0
- package/dist/core/spawn.js +182 -0
- package/dist/core/supervisorContract.js +230 -0
- package/dist/core/telemetry.js +1297 -0
- package/dist/core/telemetryContext.js +154 -0
- package/dist/core/telemetrySpool.js +244 -0
- package/dist/core/transcriptUpload.js +516 -0
- package/dist/core/update.js +363 -0
- package/dist/core/viewer.js +117 -0
- package/dist/core/workspace.js +92 -0
- package/dist/core/writeGrants.js +93 -0
- package/dist/core/writebackManifest.js +92 -0
- package/dist/direct/applyLoop.js +2516 -0
- package/dist/direct/batchReport.js +510 -0
- package/dist/direct/comparatorNoise.js +75 -0
- package/dist/direct/containers.js +85 -0
- package/dist/direct/contamination.js +221 -0
- package/dist/direct/crud.js +56 -0
- package/dist/direct/destructive.js +168 -0
- package/dist/direct/knownTokens.js +94 -0
- package/dist/direct/lint.js +382 -0
- package/dist/direct/metadataSchema.generated.json +1 -0
- package/dist/direct/metadataSchema.js +179 -0
- package/dist/direct/operatorPage.js +349 -0
- package/dist/direct/orgRepo.js +1531 -0
- package/dist/direct/orgRepoSpawnCount.js +21 -0
- package/dist/direct/plannedScope.js +119 -0
- package/dist/direct/preImageTargets.js +63 -0
- package/dist/direct/refVerify.js +191 -0
- package/dist/direct/registry.js +129 -0
- package/dist/direct/removals.js +262 -0
- package/dist/direct/rollback.js +1513 -0
- package/dist/direct/rollbackRun.js +1360 -0
- package/dist/direct/sessionProjection.js +126 -0
- package/dist/direct/snapshotComparability.js +165 -0
- package/dist/direct/sourceTree.js +113 -0
- package/dist/direct/unstatedChanges.js +106 -0
- package/dist/direct/waveAssembly.js +556 -0
- package/dist/direct/waveCounts.js +19 -0
- package/dist/direct/waveDerivation.js +152 -0
- package/dist/direct/waveGrade.js +185 -0
- package/dist/index.js +90 -0
- package/dist/orgread/customFieldSnapshot.js +48 -0
- package/dist/orgread/fieldMetadata.js +178 -0
- package/dist/orgread/genericSnapshot.js +61 -0
- package/dist/orgread/layoutSnapshot.js +151 -0
- package/dist/orgread/salesforceOrgReader.js +184 -0
- package/dist/orgread/snapshotTypes.js +9 -0
- package/dist/projection/bridge.js +26 -0
- package/dist/projection/diff.js +141 -0
- package/dist/projection/mapmd.js +213 -0
- package/dist/projection/nameAmbiguity.js +106 -0
- package/dist/projection/precedent.js +136 -0
- package/dist/projection/project.js +214 -0
- package/dist/projection/scope.js +110 -0
- package/dist/projection/types.js +5 -0
- package/dist/tools/analyze.js +322 -0
- package/dist/tools/auth.js +915 -0
- package/dist/tools/campfire.js +1742 -0
- package/dist/tools/closeBatch.js +154 -0
- package/dist/tools/closeText.js +35 -0
- package/dist/tools/direct.js +1613 -0
- package/dist/tools/feedback.js +318 -0
- package/dist/tools/operator.js +287 -0
- package/dist/tools/org.js +739 -0
- package/dist/tools/phase.js +249 -0
- package/dist/tools/projection.js +172 -0
- package/dist/tools/session.js +313 -0
- package/dist/tools/update.js +82 -0
- package/dist/tools/waves.js +793 -0
- package/package.json +48 -0
- package/reference/campfire-configuration-items-guide.md +710 -0
- package/reference/deployability-cribs.md +119 -0
- package/reference/operator-voice.md +141 -0
- package/reference/org-read-cribs.md +34 -0
- package/reference/scoped-dml-interim.md +68 -0
- package/reference/supervisor-playbook.md +313 -0
- package/reference/templates.md +233 -0
- package/skills/in2-delivery/SKILL.md +322 -0
- package/skills/in2-delivery/references/campfire-configuration-items-guide.md +710 -0
- package/skills/in2-delivery/references/deployability-cribs.md +119 -0
- package/skills/in2-delivery/references/operator-voice.md +141 -0
- package/skills/in2-delivery/references/org-read-cribs.md +34 -0
- package/skills/in2-delivery/references/scoped-dml-interim.md +68 -0
- package/skills/in2-delivery/references/templates.md +233 -0
- package/skills/plan-requirement/SKILL.md +188 -0
- package/viewer/index.html +899 -0
- package/viewer/server.mjs +433 -0
|
@@ -0,0 +1,1297 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
3
|
+
import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { REPO_ROOT } from "./paths.js";
|
|
5
|
+
import { sessionDir } from "./workspace.js";
|
|
6
|
+
import { installedVersion } from "./install.js";
|
|
7
|
+
import { harnessVersion } from "./paths.js";
|
|
8
|
+
import { batchTelemetryClosed, bindBatchContext, closeBatchTelemetry, onBatchContext, processTelemetryIdentity, emitDomainTelemetryEvent, onDomainTelemetryEvent, registerBatchCloser, registerBoundaryFlusher, setProcessTelemetryIdentity } from "./telemetryContext.js";
|
|
9
|
+
import { TelemetryGrantStore, TelemetrySpool } from "./telemetrySpool.js";
|
|
10
|
+
import { announce, findTranscript, findTranscriptsForSession, shipSessionTranscripts, transcriptEndpointFrom, transcriptUploadEnabled } from "./transcriptUpload.js";
|
|
11
|
+
import { announceParcelUpload, flushParcels } from "./parcels.js";
|
|
12
|
+
import { announceOperatorQa, operatorQaEndpointFrom, shipOperatorQa } from "./operatorQa.js";
|
|
13
|
+
import { announceBeta, flushFeedback, redactSecrets } from "./betaFeedback.js";
|
|
14
|
+
// The ONE batch grading rule, shared with `composeBatchReport`. A dependency-free leaf (node:fs and
|
|
15
|
+
// node:path only) so this static import pulls nothing else across the core/direct line — see
|
|
16
|
+
// `writeReport` for why the report itself is imported lazily instead.
|
|
17
|
+
import { gradeWaves, readWaveBundles } from "../direct/waveGrade.js";
|
|
18
|
+
// PIPELINE TELEMETRY — where does a run's wall-clock actually go?
|
|
19
|
+
//
|
|
20
|
+
// A config run takes ~20-30 min against ~5 min for the old manual VFP process. Some of that is
|
|
21
|
+
// expected (interactive, verified, far more complete), but "expected" is not a number. Wall-clock
|
|
22
|
+
// splits three ways and only ONE of them is a performance problem:
|
|
23
|
+
//
|
|
24
|
+
// tool time — org round-trips (checkOnly, deploy, projections). Real cost, optimizable.
|
|
25
|
+
// agent time — the model reading, planning, authoring XML between tool calls.
|
|
26
|
+
// operator time — the human answering a question card. NOT a system cost, and must never be
|
|
27
|
+
// counted against the run; a run that waited 8 minutes for an answer is not slow.
|
|
28
|
+
//
|
|
29
|
+
// Every record carries the gap SINCE THE PREVIOUS TOOL CALL and classifies it, so the three are
|
|
30
|
+
// separable after the fact. The classifier is the one load-bearing rule: a gap that follows
|
|
31
|
+
// `ask_operator` is operator wait, because that tool exists precisely to hand control to the human.
|
|
32
|
+
//
|
|
33
|
+
// Bodies are never logged — only SHAPES (counts, flags, ids). Metadata XML, content bodies and
|
|
34
|
+
// note text stay out of the log: they are large, they are the client's, and they answer no timing
|
|
35
|
+
// question.
|
|
36
|
+
const STAGE_BY_TOOL = {
|
|
37
|
+
create_session: "intake",
|
|
38
|
+
import_requirements: "intake",
|
|
39
|
+
org_login: "auth",
|
|
40
|
+
activate_session: "intake",
|
|
41
|
+
project_org: "grounding",
|
|
42
|
+
list_components: "probe",
|
|
43
|
+
describe_object: "probe",
|
|
44
|
+
sf_read: "probe",
|
|
45
|
+
read_campfire_requirements: "probe",
|
|
46
|
+
direct_lint: "lint",
|
|
47
|
+
direct_apply: "deploy",
|
|
48
|
+
// Reading the org mirror. Its own stage rather than "probe": it is a LOCAL read of what this
|
|
49
|
+
// install already did, so counting it as org round-trip time would misattribute the wall-clock
|
|
50
|
+
// split this table exists to measure.
|
|
51
|
+
direct_history: "history",
|
|
52
|
+
ask_operator: "operator-gate",
|
|
53
|
+
// The other half of the gate. NOT a `classifyGap` case: the gap AFTER this call is the agent
|
|
54
|
+
// working with the answer, whereas the gap after `ask_operator` is the human deciding.
|
|
55
|
+
record_operator_answers: "operator-gate",
|
|
56
|
+
// Its own stage rather than folded into the chapter it enters. A boundary is not work — it is the
|
|
57
|
+
// moment between two stretches of work — so counting it as `grounding` or `deploy` would inflate
|
|
58
|
+
// whichever chapter happened to follow it with the cost of a flush.
|
|
59
|
+
advance_phase: "boundary",
|
|
60
|
+
resume_session: "boundary",
|
|
61
|
+
campfire_config_item_plan: "writeback-plan",
|
|
62
|
+
campfire_write_config_items: "writeback",
|
|
63
|
+
create_campfire_deployment_tasks: "writeback",
|
|
64
|
+
update_campfire_requirements: "writeback",
|
|
65
|
+
close_batch: "settle",
|
|
66
|
+
};
|
|
67
|
+
export function stageForTool(tool) {
|
|
68
|
+
return STAGE_BY_TOOL[tool] ?? "other";
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* THE ONE PHASE VOCABULARY THIS SYSTEM SPEAKS — derived from the table above, never restated.
|
|
72
|
+
*
|
|
73
|
+
* The beta feedback channel's `stage` argument enumerates exactly this (tools/feedback.ts), so a
|
|
74
|
+
* friction report joins the timing data instead of inventing a second phase language for the same
|
|
75
|
+
* run. Derived rather than listed so a tool added to `STAGE_BY_TOOL` with a new stage extends the
|
|
76
|
+
* feedback vocabulary in the same edit — a hand-copied enum is how the two would come to disagree.
|
|
77
|
+
*
|
|
78
|
+
* `other` is included because `stageForTool` returns it for anything unmapped, so it is a stage the
|
|
79
|
+
* timing data genuinely contains rather than a placeholder.
|
|
80
|
+
*/
|
|
81
|
+
export const TELEMETRY_STAGES = [...new Set([...Object.values(STAGE_BY_TOOL), "other"])].sort();
|
|
82
|
+
/** Batch close reports every step it could not complete, so failures need a printable form. */
|
|
83
|
+
function describeError(error) {
|
|
84
|
+
return error instanceof Error ? error.message : String(error);
|
|
85
|
+
}
|
|
86
|
+
/** Exit backstops are process-wide, so they are installed once however many Telemetry objects exist. */
|
|
87
|
+
let exitBackstopsInstalled = false;
|
|
88
|
+
// Shape-only input summary. Anything not named here is dropped, so a new tool can never
|
|
89
|
+
// accidentally start logging client metadata.
|
|
90
|
+
export function summarizeInput(input) {
|
|
91
|
+
if (!input || typeof input !== "object")
|
|
92
|
+
return {};
|
|
93
|
+
const i = input;
|
|
94
|
+
const out = {};
|
|
95
|
+
const passthrough = ["targetOrg", "consultantOrg", "sessionId", "label", "requirement", "extId", "typeName", "metadataType", "alias", "mode", "dryRunOnly"];
|
|
96
|
+
for (const k of passthrough)
|
|
97
|
+
if (i[k] !== undefined)
|
|
98
|
+
out[k] = i[k];
|
|
99
|
+
const arrays = ["components", "deletes", "objectFacetUpdates", "bundleDirs", "requirements", "names", "args", "ackLints"];
|
|
100
|
+
for (const k of arrays)
|
|
101
|
+
if (Array.isArray(i[k]))
|
|
102
|
+
out[`${k}Count`] = i[k].length;
|
|
103
|
+
const flags = ["sourceDir", "projectionDir", "sessionDir", "corpusDir", "compareTo", "content"];
|
|
104
|
+
for (const k of flags)
|
|
105
|
+
if (i[k] !== undefined)
|
|
106
|
+
out[`has_${k}`] = true;
|
|
107
|
+
if (i.campfire && typeof i.campfire === "object") {
|
|
108
|
+
const c = i.campfire;
|
|
109
|
+
out.campfireNames = Array.isArray(c.names) ? c.names.length : undefined;
|
|
110
|
+
}
|
|
111
|
+
if (i.scope && typeof i.scope === "object") {
|
|
112
|
+
const s = i.scope;
|
|
113
|
+
out.scope = {
|
|
114
|
+
objects: Array.isArray(s.objects) ? s.objects.length : undefined,
|
|
115
|
+
components: Array.isArray(s.components) ? s.components.length : undefined,
|
|
116
|
+
prefix: s.prefix ? true : undefined
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
return out;
|
|
120
|
+
}
|
|
121
|
+
// Shape-only output summary: the few fields that explain WHAT happened, never the payload.
|
|
122
|
+
export function summarizeOutput(result) {
|
|
123
|
+
if (!result || typeof result !== "object")
|
|
124
|
+
return {};
|
|
125
|
+
const r = result;
|
|
126
|
+
const out = { ok: !r.isError };
|
|
127
|
+
const s = r.structuredContent;
|
|
128
|
+
if (!s)
|
|
129
|
+
return out;
|
|
130
|
+
const err = s.error;
|
|
131
|
+
if (err?.code)
|
|
132
|
+
out.errorCode = err.code;
|
|
133
|
+
for (const k of ["verdict", "action", "id", "componentsLinted", "blocked", "referentialTier", "truncated"]) {
|
|
134
|
+
if (s[k] !== undefined)
|
|
135
|
+
out[k] = s[k];
|
|
136
|
+
}
|
|
137
|
+
// The self-update lifecycle, so the ledger can answer "which runs checked, which applied, and did
|
|
138
|
+
// the session need a restart afterwards" — the question QA round 1 left unanswerable when two
|
|
139
|
+
// stale builds each cost a run. Flags and version strings only; the changed-tool NAMES stay in
|
|
140
|
+
// the tool result (they are our own vocabulary, but the ledger's discipline is shape, not lists).
|
|
141
|
+
for (const k of [
|
|
142
|
+
"updated", "applied", "swappedLive", "launcherUpdated", "instructionsChanged",
|
|
143
|
+
"updateAvailable", "staleness", "behindDays", "currentVersion", "latestVersion", "previousVersion"
|
|
144
|
+
]) {
|
|
145
|
+
if (s[k] !== undefined)
|
|
146
|
+
out[k] = s[k];
|
|
147
|
+
}
|
|
148
|
+
for (const [k, source] of [["addedCount", "added"], ["removedCount", "removed"], ["changedCount", "changed"]]) {
|
|
149
|
+
if (Array.isArray(s[source]))
|
|
150
|
+
out[k] = s[source].length;
|
|
151
|
+
}
|
|
152
|
+
if (Array.isArray(s.rows))
|
|
153
|
+
out.rowCount = s.rows.length;
|
|
154
|
+
if (Array.isArray(s.findings))
|
|
155
|
+
out.findingCount = s.findings.length;
|
|
156
|
+
if (Array.isArray(s.imported))
|
|
157
|
+
out.importedCount = s.imported.length;
|
|
158
|
+
const counts = s.counts;
|
|
159
|
+
if (counts)
|
|
160
|
+
out.counts = counts;
|
|
161
|
+
const writebackCounts = {};
|
|
162
|
+
for (const key of ["created", "updated", "skipped", "failed", "total", "readbackConfirmed"]) {
|
|
163
|
+
if (s[key] !== undefined)
|
|
164
|
+
writebackCounts[key] = s[key];
|
|
165
|
+
}
|
|
166
|
+
if (Object.keys(writebackCounts).length > 0)
|
|
167
|
+
out.writebackCounts = writebackCounts;
|
|
168
|
+
if (Array.isArray(s.failures)) {
|
|
169
|
+
// SLICED TO THE VALIDATOR'S CAP. The ingest check refuses `failedRows` past 100 outright, and
|
|
170
|
+
// the tool result this reads carries the FULL failure list, so a writeback with 101+ failures
|
|
171
|
+
// used to reject its own `tool.completed` permanently — losing the tool call from the ledger
|
|
172
|
+
// beside the domain event it belongs to. The sibling emitter for `writeback.completed` already
|
|
173
|
+
// slices; this path was simply missed. Truncating loses nothing the payload does not still
|
|
174
|
+
// state: the count rides on `writebackCounts.failed` regardless.
|
|
175
|
+
out.failedRows = s.failures.slice(0, 100).map((failure) => ({
|
|
176
|
+
...(failure.metadataKey !== undefined ? { metadataKey: failure.metadataKey } : {}),
|
|
177
|
+
...(failure.requirement !== undefined ? { requirement: failure.requirement } : {}),
|
|
178
|
+
...(failure.externalId !== undefined ? { externalId: failure.externalId } : {})
|
|
179
|
+
}));
|
|
180
|
+
}
|
|
181
|
+
return out;
|
|
182
|
+
}
|
|
183
|
+
export function classifyGap(previousTool) {
|
|
184
|
+
if (previousTool === null)
|
|
185
|
+
return "first";
|
|
186
|
+
return previousTool === "ask_operator" ? "operator" : "agent";
|
|
187
|
+
}
|
|
188
|
+
// REMOTE TELEMETRY — the same shape-only records, shipped to the launch service so runs across
|
|
189
|
+
// every install are visible in one place (which tools fail, which versions are live, how many
|
|
190
|
+
// installs are active). ON BY DEFAULT; the same records already written locally, nothing more.
|
|
191
|
+
// Opt-outs: CC_MCP_TELEMETRY=0 kills all telemetry, CC_MCP_TELEMETRY_REMOTE=0 kills just the
|
|
192
|
+
// remote ship. The authenticated endpoint is bound by activate_session after it redeems the
|
|
193
|
+
// batch-scoped grant, including when the launch URL was passed as a tool argument.
|
|
194
|
+
//
|
|
195
|
+
// FAIL-OPEN IS LOAD-BEARING: telemetry must never slow, block, or break a tool call. Batches
|
|
196
|
+
// ship on a timer (unref'd, so the process never stays alive for it); a failed POST re-queues
|
|
197
|
+
// up to a hard cap and is retried next tick; past the cap, events are dropped and counted.
|
|
198
|
+
/** Stable anonymous id for THIS install — lets the collector tell five consultants apart. */
|
|
199
|
+
export function installId() {
|
|
200
|
+
const file = path.join(REPO_ROOT, ".local", "cc-mcp", "install-id");
|
|
201
|
+
try {
|
|
202
|
+
return readFileSync(file, "utf8").trim();
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
const id = `ins_${randomBytes(8).toString("hex")}`;
|
|
206
|
+
try {
|
|
207
|
+
mkdirSync(path.dirname(file), { recursive: true });
|
|
208
|
+
writeFileSync(file, `${id}\n`, "utf8");
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
/* unwritable disk — a per-process id is still useful */
|
|
212
|
+
}
|
|
213
|
+
return id;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
/** Longest gap between delivery-health reports while nothing about delivery has changed. */
|
|
217
|
+
const HEALTH_REPORT_FLOOR_MS = 5 * 60_000;
|
|
218
|
+
/** Where this install keeps its outbox and its grants. One place, so a reader cannot pick a different one. */
|
|
219
|
+
function localTelemetryRoot() {
|
|
220
|
+
return path.join(REPO_ROOT, ".local", "cc-mcp");
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Re-bind a batch this MACHINE already has a grant for, so a resumed run can ship again.
|
|
224
|
+
*
|
|
225
|
+
* WHY THIS IS NEEDED AT ALL. `bindBatchContext` is called by `activate_session`, and activation
|
|
226
|
+
* happens once — it redeems a single-use code. A fresh server process picking up a dead run has
|
|
227
|
+
* therefore never bound a batch, so `close_batch` reports "no batch bound to this server process",
|
|
228
|
+
* the boundary flush does the same, and the resumed run's events and transcript never leave the
|
|
229
|
+
* machine EVEN THOUGH the original run's grant is sitting in the grant store, valid, keyed by the
|
|
230
|
+
* batchId the session file records.
|
|
231
|
+
*
|
|
232
|
+
* The grant is the authority, exactly as everywhere else: this mints nothing and asserts nothing.
|
|
233
|
+
* It reads a credential this install was already issued for this batch, and binds it back. An
|
|
234
|
+
* expired or absent grant returns false, which the caller reports as a fact rather than working
|
|
235
|
+
* around — a resumed run whose grant has gone still DELIVERS, it just cannot ship its evidence, and
|
|
236
|
+
* saying so is the whole point.
|
|
237
|
+
*/
|
|
238
|
+
export function rebindStoredBatch(input) {
|
|
239
|
+
try {
|
|
240
|
+
const grant = new TelemetryGrantStore(path.join(localTelemetryRoot(), "telemetry-grants")).get(input.batchId);
|
|
241
|
+
if (!grant)
|
|
242
|
+
return false;
|
|
243
|
+
const identity = processTelemetryIdentity();
|
|
244
|
+
bindBatchContext({
|
|
245
|
+
batchId: input.batchId,
|
|
246
|
+
// The RESUMING process's identity, not the dead run's: install stays the same machine, but
|
|
247
|
+
// runId names which process produced the events, and claiming the old one would merge two
|
|
248
|
+
// runs into one in the ledger.
|
|
249
|
+
installId: identity?.installId ?? grant.installId,
|
|
250
|
+
runId: identity?.runId ?? grant.runId,
|
|
251
|
+
sessionId: input.sessionId,
|
|
252
|
+
serverVersion: identity?.serverVersion ?? grant.serverVersion,
|
|
253
|
+
credential: grant.credential,
|
|
254
|
+
expiresAt: grant.expiresAt,
|
|
255
|
+
endpoint: grant.endpoint
|
|
256
|
+
});
|
|
257
|
+
return true;
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
// Telemetry is fail-open everywhere else and is fail-open here: a resume must never break on it.
|
|
261
|
+
return false;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* The one event type that flushes the moment it is emitted.
|
|
266
|
+
*
|
|
267
|
+
* Named as a constant rather than inlined so the cadence is one decision in one place: the next
|
|
268
|
+
* domain event added is NOT a boundary unless somebody puts it here on purpose.
|
|
269
|
+
*/
|
|
270
|
+
const WAVE_BOUNDARY_EVENT = "deployment.wave_completed";
|
|
271
|
+
/**
|
|
272
|
+
* Upper bound on startup drain passes — 200 events each, so 40,000 events, far beyond what any run
|
|
273
|
+
* strands and well under the outbox's own 10 MB cap. A backstop against a non-shrinking outbox,
|
|
274
|
+
* not a real limit; the loop normally exits on "no progress" or "nothing pending".
|
|
275
|
+
*/
|
|
276
|
+
const MAX_DRAIN_PASSES = 200;
|
|
277
|
+
export function cloudEvent(record) {
|
|
278
|
+
if (record.event === "server_start") {
|
|
279
|
+
return {
|
|
280
|
+
eventId: record.eventId ?? `evt_${randomBytes(16).toString("hex")}`,
|
|
281
|
+
eventType: "server.started",
|
|
282
|
+
occurredAt: record.ts,
|
|
283
|
+
outcome: "started",
|
|
284
|
+
payload: { ...(record.pid !== undefined ? { pid: record.pid } : {}) }
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
const output = record.output ?? {};
|
|
288
|
+
const errorCode = typeof output.errorCode === "string"
|
|
289
|
+
? output.errorCode
|
|
290
|
+
: record.threw
|
|
291
|
+
? "TOOL_THREW"
|
|
292
|
+
: undefined;
|
|
293
|
+
return {
|
|
294
|
+
eventId: record.eventId ?? `evt_${randomBytes(16).toString("hex")}`,
|
|
295
|
+
eventType: record.threw ? "tool.threw" : "tool.completed",
|
|
296
|
+
occurredAt: record.ts,
|
|
297
|
+
...(record.stage ? { stage: record.stage } : {}),
|
|
298
|
+
...(record.ms !== undefined ? { durationMs: Math.max(0, Math.round(record.ms)) } : {}),
|
|
299
|
+
...(record.gapMs !== undefined ? { gapMs: Math.max(0, Math.round(record.gapMs)) } : {}),
|
|
300
|
+
...(record.gapClass ? { gapClass: record.gapClass } : {}),
|
|
301
|
+
outcome: record.threw ? "threw" : output.ok === false ? String(errorCode ?? "error") : "ok",
|
|
302
|
+
payload: record.threw
|
|
303
|
+
? { tool: record.tool ?? "unknown", ...(errorCode ? { errorCode } : {}) }
|
|
304
|
+
: { tool: record.tool ?? "unknown", input: record.input ?? {}, output }
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
export class DurableRemoteSink {
|
|
308
|
+
options;
|
|
309
|
+
/**
|
|
310
|
+
* Where a full transcript would go, and the credential to send it with — the SAME batch-scoped
|
|
311
|
+
* grant the events use, so a transcript can only be attributed to the batch that uploaded it.
|
|
312
|
+
* Null until a grant exists, which is also the point before which there is nothing to attribute.
|
|
313
|
+
*/
|
|
314
|
+
transcriptEndpoint() {
|
|
315
|
+
const batchId = this.context?.batchId;
|
|
316
|
+
if (!batchId)
|
|
317
|
+
return null;
|
|
318
|
+
const grant = this.options.grants.get(batchId);
|
|
319
|
+
return grant ? transcriptEndpointFrom(grant.endpoint) : null;
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* The batch-scoped grant, which every content route uses — transcript, parcel and operator Q&A.
|
|
323
|
+
* One credential rather than three: the grant is what BINDS an upload to a batch, so a route with
|
|
324
|
+
* its own would be a second, weaker claim about whose run this is.
|
|
325
|
+
*/
|
|
326
|
+
batchCredential() {
|
|
327
|
+
const batchId = this.context?.batchId;
|
|
328
|
+
if (!batchId)
|
|
329
|
+
return null;
|
|
330
|
+
return this.options.grants.get(batchId)?.credential ?? null;
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Where the question cards and their answers go. Same host, same grant, different route — and
|
|
334
|
+
* resolved from the same grant store for the same reason: a Q&A record can only ever be
|
|
335
|
+
* attributed to the batch whose credential sent it.
|
|
336
|
+
*/
|
|
337
|
+
operatorQaEndpoint() {
|
|
338
|
+
const batchId = this.context?.batchId;
|
|
339
|
+
if (!batchId)
|
|
340
|
+
return null;
|
|
341
|
+
const grant = this.options.grants.get(batchId);
|
|
342
|
+
return grant ? operatorQaEndpointFrom(grant.endpoint) : null;
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* Strings that identify this IN2 session inside a Claude Code transcript.
|
|
346
|
+
*
|
|
347
|
+
* The session id is the reliable one: every org-touching tool takes it as an argument, so the
|
|
348
|
+
* client records it as tool input on essentially every call. batchId is included because a
|
|
349
|
+
* session that ended before its first org call still names its batch. The activation code is
|
|
350
|
+
* deliberately NOT used — it is a credential, and matching on it would mean reading it back out
|
|
351
|
+
* of state it has no reason to leave.
|
|
352
|
+
*/
|
|
353
|
+
/** The IN2 session this batch is running, once activation has bound one. */
|
|
354
|
+
batchSessionId() {
|
|
355
|
+
return this.context?.sessionId ?? null;
|
|
356
|
+
}
|
|
357
|
+
transcriptMarkers() {
|
|
358
|
+
return [this.context?.sessionId, this.context?.batchId].filter((v) => typeof v === "string" && v.length >= 8);
|
|
359
|
+
}
|
|
360
|
+
context = null;
|
|
361
|
+
beforeContext = [];
|
|
362
|
+
fetchFn;
|
|
363
|
+
maxPreContext;
|
|
364
|
+
/** The flush currently on the wire, or null. Both the concurrency guard and the thing to await. */
|
|
365
|
+
active = null;
|
|
366
|
+
/**
|
|
367
|
+
* The flush currently in flight, so a caller can await a shipment it did not start.
|
|
368
|
+
*
|
|
369
|
+
* `flush()` is deliberately fire-and-forget at every call site — telemetry may never delay a tool
|
|
370
|
+
* — which leaves no handle on "has the wave's event actually gone". This is that handle, and it
|
|
371
|
+
* exists for the same reason the cadence does: the interesting question about a wave-boundary
|
|
372
|
+
* flush is whether it COMPLETED, and a test that polls a timer is measuring the timer.
|
|
373
|
+
*/
|
|
374
|
+
pending = Promise.resolve();
|
|
375
|
+
warned = false;
|
|
376
|
+
retries = 0;
|
|
377
|
+
rejectedSchemaEventIds = new Set();
|
|
378
|
+
/** Distinct rejection REASONS already printed this process — see reportRejection. */
|
|
379
|
+
reportedRejectionReasons = new Set();
|
|
380
|
+
/** Events the ledger has ACCEPTED on this sink — what `drainOutbox` reports as recovered. */
|
|
381
|
+
acceptedEvents = 0;
|
|
382
|
+
lastAcknowledgedEventId;
|
|
383
|
+
lastHealthSignature = "";
|
|
384
|
+
lastHealthAt = 0;
|
|
385
|
+
constructor(options) {
|
|
386
|
+
this.options = options;
|
|
387
|
+
this.fetchFn = options.fetchFn ?? fetch;
|
|
388
|
+
this.maxPreContext = options.maxPreContext ?? 500;
|
|
389
|
+
onBatchContext((context) => this.bind(context));
|
|
390
|
+
}
|
|
391
|
+
bind(context) {
|
|
392
|
+
this.context = context;
|
|
393
|
+
try {
|
|
394
|
+
if (context.credential)
|
|
395
|
+
this.options.grants.put(context);
|
|
396
|
+
for (const record of this.beforeContext.splice(0)) {
|
|
397
|
+
if (!this.options.spool.append({ batchId: context.batchId, event: cloudEvent(record) })) {
|
|
398
|
+
this.warnOnce(new Error("local telemetry outbox is unavailable"));
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
void this.flush();
|
|
402
|
+
}
|
|
403
|
+
catch (error) {
|
|
404
|
+
this.warnOnce(error);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
push(record) {
|
|
408
|
+
if (!this.context) {
|
|
409
|
+
this.beforeContext.push(record);
|
|
410
|
+
if (this.beforeContext.length > this.maxPreContext)
|
|
411
|
+
this.beforeContext.shift();
|
|
412
|
+
return;
|
|
413
|
+
}
|
|
414
|
+
if (!this.options.spool.append({ batchId: this.context.batchId, event: cloudEvent(record) })) {
|
|
415
|
+
this.warnOnce(new Error("local telemetry outbox is unavailable"));
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
pushDomain(event) {
|
|
419
|
+
if (!this.context)
|
|
420
|
+
return;
|
|
421
|
+
if (!this.options.spool.append({
|
|
422
|
+
batchId: this.context.batchId,
|
|
423
|
+
event: {
|
|
424
|
+
eventId: event.eventId,
|
|
425
|
+
eventType: event.eventType,
|
|
426
|
+
occurredAt: event.occurredAt,
|
|
427
|
+
...(event.stage ? { stage: event.stage } : {}),
|
|
428
|
+
...(event.outcome ? { outcome: event.outcome } : {}),
|
|
429
|
+
payload: event.payload
|
|
430
|
+
}
|
|
431
|
+
})) {
|
|
432
|
+
this.warnOnce(new Error("local telemetry outbox is unavailable"));
|
|
433
|
+
}
|
|
434
|
+
// A WAVE BOUNDARY SHIPS IMMEDIATELY.
|
|
435
|
+
//
|
|
436
|
+
// Everything else waits for the 20-second timer or for `close_batch`, and both are bets on the
|
|
437
|
+
// process surviving. A Windows run whose waves took ~8 minutes each died before either, and its
|
|
438
|
+
// whole ledger stayed on disk — the events were never lost, they were never SENT, and no later
|
|
439
|
+
// run flushed them because the drain did not exist yet (it does now; see drainOutbox).
|
|
440
|
+
//
|
|
441
|
+
// A wave boundary is the right cadence and not merely a convenient one: it is RARE (a handful
|
|
442
|
+
// per batch, against a health tick every 20s and a tool event per call), its payload is a few
|
|
443
|
+
// hundred bytes, and it lands at the one moment the run has just finished the longest and
|
|
444
|
+
// riskiest thing it does. Flushing on every emit would put a network hop behind the health
|
|
445
|
+
// heartbeat; flushing here costs one POST per delivered wave.
|
|
446
|
+
//
|
|
447
|
+
// Unawaited, like every other flush: the contract is that telemetry never delays a tool. The
|
|
448
|
+
// event is already durable on disk (appendFileSync), so the only thing at risk is the hop.
|
|
449
|
+
if (event.eventType === WAVE_BOUNDARY_EVENT)
|
|
450
|
+
this.flushSoon();
|
|
451
|
+
}
|
|
452
|
+
/**
|
|
453
|
+
* Start a flush without waiting for it, and keep a handle on the one in flight.
|
|
454
|
+
*
|
|
455
|
+
* CHAINED ONTO WHATEVER IS RUNNING, not fired blind. `flush()` returns immediately when another
|
|
456
|
+
* flush is in flight, and that guard is a silent no-op — so a wave boundary landing during the
|
|
457
|
+
* 20-second timer's flush would ship NOTHING, and the running flush cannot carry the event either
|
|
458
|
+
* because it read its batch off the outbox before the event was appended. The boundary therefore
|
|
459
|
+
* has to WAIT for the in-flight flush and then start a fresh one. (Caught by test, and it is the
|
|
460
|
+
* exact failure this cadence exists to prevent: the wave that most needs shipping, on the run
|
|
461
|
+
* least likely to survive to the next tick.)
|
|
462
|
+
*
|
|
463
|
+
* `.catch` swallows: telemetry is fail-open, and a rejected flush must not leave the chain
|
|
464
|
+
* permanently rejected for every later boundary.
|
|
465
|
+
*/
|
|
466
|
+
flushSoon() {
|
|
467
|
+
this.pending = this.pending.then(() => this.flush()).catch(() => undefined);
|
|
468
|
+
}
|
|
469
|
+
/** Resolves once no flush this sink started is still running. Test and shutdown seam only. */
|
|
470
|
+
async settled() {
|
|
471
|
+
await this.pending;
|
|
472
|
+
}
|
|
473
|
+
/** Events still on disk. Batch close reports a non-zero count rather than implying success. */
|
|
474
|
+
pendingEventCount() {
|
|
475
|
+
return this.options.spool.stats().pendingEvents;
|
|
476
|
+
}
|
|
477
|
+
/**
|
|
478
|
+
* Ship EVERYTHING the outbox is holding, not just one delivery batch.
|
|
479
|
+
*
|
|
480
|
+
* `flush()` sends at most 200 events, which is right for a timer tick and wrong for the two
|
|
481
|
+
* moments a run is trying to leave nothing behind. Both callers are here:
|
|
482
|
+
*
|
|
483
|
+
* SERVER START — the recovery window. Every event is durable the moment it is emitted, so a
|
|
484
|
+
* killed process costs the network hop and not the record — but that promise only pays out if
|
|
485
|
+
* something later actually performs the hop. Nothing did: the only things that flushed were a
|
|
486
|
+
* timer inside the dead process and a batch close it never reached, so "a later run flushes it"
|
|
487
|
+
* was true of the outbox format and false of the code, and a stranded segment sat there until its
|
|
488
|
+
* 30-day age cap dropped it. At start, everything in the outbox is by construction a PRIOR run's
|
|
489
|
+
* — this process has emitted nothing yet.
|
|
490
|
+
*
|
|
491
|
+
* BATCH CLOSE — the explicit end. A real run emits far more than 200 events (a tool event per
|
|
492
|
+
* call, a health tick every 20 seconds), so a single flush there left the remainder to a timer
|
|
493
|
+
* that is about to stop ticking: the same "never reached the ledger" shape, on the path that is
|
|
494
|
+
* supposed to be the reliable one.
|
|
495
|
+
*
|
|
496
|
+
* FAIL-OPEN AND BOUNDED. It stops the moment a flush makes no progress — an unreachable ledger,
|
|
497
|
+
* or a batch whose grant has expired — so an offline machine pays one failed POST and never a
|
|
498
|
+
* retry storm. Events it could not ship stay on disk for the next start, which is the honest
|
|
499
|
+
* state: "could not ship" and "shipped" must not look the same from outside.
|
|
500
|
+
*
|
|
501
|
+
* Returns how many events actually reached the ledger, for the line that reports it.
|
|
502
|
+
*/
|
|
503
|
+
async drainOutbox() {
|
|
504
|
+
const started = this.acceptedEvents;
|
|
505
|
+
// Settle any flush ALREADY on the wire before measuring progress. One started earlier read its
|
|
506
|
+
// batch off the outbox before these events existed, so counting it as this drain's first pass
|
|
507
|
+
// would see the queue unchanged, conclude "no progress" and give up with the outbox full — the
|
|
508
|
+
// drain reporting success while shipping nothing.
|
|
509
|
+
await this.settled();
|
|
510
|
+
// A hard bound as well as the progress check: the loop's exit depends on the outbox shrinking,
|
|
511
|
+
// and a bug that made it not shrink must cost a bounded number of POSTs, not an infinite run.
|
|
512
|
+
for (let pass = 0; pass < MAX_DRAIN_PASSES; pass++) {
|
|
513
|
+
const before = this.options.spool.stats().pendingEvents;
|
|
514
|
+
if (before === 0)
|
|
515
|
+
break;
|
|
516
|
+
await this.flush();
|
|
517
|
+
// Progress is measured on the OUTBOX (a schema rejection retires an event without delivering
|
|
518
|
+
// it, and a pass that only rejects must still terminate); the RETURN is measured on what the
|
|
519
|
+
// ledger accepted, because that is the number the startup line claims reached IN2.
|
|
520
|
+
if (this.options.spool.stats().pendingEvents >= before)
|
|
521
|
+
break;
|
|
522
|
+
}
|
|
523
|
+
return this.acceptedEvents - started;
|
|
524
|
+
}
|
|
525
|
+
/**
|
|
526
|
+
* Ship one batch, at most one flush at a time.
|
|
527
|
+
*
|
|
528
|
+
* A concurrent call AWAITS the running flush rather than returning immediately. It used to be a
|
|
529
|
+
* silent no-op, which is harmless for the 20-second timer (two POSTs racing gain nothing) and
|
|
530
|
+
* wrong for a wave boundary: the running flush read its batch off the outbox BEFORE the wave
|
|
531
|
+
* event was appended, so it cannot be carrying it, and the boundary's own flush would vanish. The
|
|
532
|
+
* awaited promise is what lets `flushSoon` chain a fresh flush behind it.
|
|
533
|
+
*/
|
|
534
|
+
async flush() {
|
|
535
|
+
if (this.active)
|
|
536
|
+
return this.active;
|
|
537
|
+
const run = this.runFlush().finally(() => {
|
|
538
|
+
this.active = null;
|
|
539
|
+
});
|
|
540
|
+
this.active = run;
|
|
541
|
+
// Anything awaiting `settled()` observes this flush even when nothing chained it.
|
|
542
|
+
this.pending = run.catch(() => undefined);
|
|
543
|
+
return run;
|
|
544
|
+
}
|
|
545
|
+
async runFlush() {
|
|
546
|
+
let entries = [];
|
|
547
|
+
let grant = null;
|
|
548
|
+
for (const batchId of this.options.spool.pendingBatchIds()) {
|
|
549
|
+
const candidate = this.options.grants.get(batchId);
|
|
550
|
+
if (!candidate)
|
|
551
|
+
continue;
|
|
552
|
+
entries = this.options.spool.readBatch(200, batchId);
|
|
553
|
+
if (entries.length > 0) {
|
|
554
|
+
grant = candidate;
|
|
555
|
+
break;
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
// An expired/unavailable grant for an older batch must not head-of-line block a newer batch.
|
|
559
|
+
// The old segment remains bounded on disk for explicit reconciliation or later recovery.
|
|
560
|
+
if (entries.length === 0 || !grant)
|
|
561
|
+
return;
|
|
562
|
+
try {
|
|
563
|
+
const res = await this.fetchFn(grant.endpoint, {
|
|
564
|
+
method: "POST",
|
|
565
|
+
headers: {
|
|
566
|
+
"content-type": "application/json",
|
|
567
|
+
authorization: `Bearer ${grant.credential}`
|
|
568
|
+
},
|
|
569
|
+
// Stamp the producing build and the harness the install is RUNNING onto every event. Batches
|
|
570
|
+
// carried a version; individual events did not, so a regression could not be attributed to a
|
|
571
|
+
// release, and a machine running new server code against stale skill files was invisible.
|
|
572
|
+
body: JSON.stringify({
|
|
573
|
+
schemaVersion: 2,
|
|
574
|
+
events: entries.map((entry) => ({
|
|
575
|
+
...entry.event,
|
|
576
|
+
...(installedVersion() ? { serverVersion: installedVersion() } : {}),
|
|
577
|
+
...(harnessVersion() ? { skillVersion: harnessVersion() } : {})
|
|
578
|
+
}))
|
|
579
|
+
}),
|
|
580
|
+
signal: AbortSignal.timeout(10_000)
|
|
581
|
+
});
|
|
582
|
+
if (!res.ok) {
|
|
583
|
+
throw new Error(`HTTP ${res.status}`);
|
|
584
|
+
}
|
|
585
|
+
const response = (await res.json());
|
|
586
|
+
const accepted = Array.isArray(response.acceptedEventIds) ? response.acceptedEventIds : [];
|
|
587
|
+
// A schema rejection is PERMANENT: the same bytes will be refused every time. Retaining a
|
|
588
|
+
// rejected event means re-sending it every flush for the life of the outbox, and once enough
|
|
589
|
+
// of them collect at the head of a segment they crowd out the events that would be accepted.
|
|
590
|
+
// So they are retired from the outbox and counted as lost, which is the truth, rather than
|
|
591
|
+
// retried forever while delivery-health reports "ok".
|
|
592
|
+
const rejected = [];
|
|
593
|
+
for (const entry of response.rejectedEvents ?? []) {
|
|
594
|
+
if (typeof entry.eventId === "string") {
|
|
595
|
+
this.rejectedSchemaEventIds.add(entry.eventId);
|
|
596
|
+
rejected.push(entry.eventId);
|
|
597
|
+
}
|
|
598
|
+
this.reportRejection(entry);
|
|
599
|
+
}
|
|
600
|
+
this.options.spool.acknowledge([...accepted, ...rejected], rejected.length);
|
|
601
|
+
this.acceptedEvents += accepted.length;
|
|
602
|
+
this.lastAcknowledgedEventId = accepted.at(-1);
|
|
603
|
+
this.warned = false;
|
|
604
|
+
}
|
|
605
|
+
catch (error) {
|
|
606
|
+
this.retries += 1;
|
|
607
|
+
this.warnOnce(error);
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* Delivery health, but only when it CARRIES INFORMATION. Spooling a heartbeat every 20 seconds
|
|
612
|
+
* made health events the majority of a run's ledger rows — a 30-minute run emits ~90 of them
|
|
613
|
+
* against a few dozen real tool events — which skews any per-event-type analysis and consumes
|
|
614
|
+
* the 200-event delivery batch. A report is emitted when the meaningful fields change (pending,
|
|
615
|
+
* retries, rejections, drops, last acknowledgement) or when the floor elapses, so silence still
|
|
616
|
+
* proves liveness without drowning the signal.
|
|
617
|
+
*/
|
|
618
|
+
informativeHealthEvent(now) {
|
|
619
|
+
const event = this.healthEvent();
|
|
620
|
+
const p = event.payload;
|
|
621
|
+
// oldestEventAgeMs is excluded deliberately: it advances every tick while a queue is pending,
|
|
622
|
+
// so including it would make every heartbeat look like a change.
|
|
623
|
+
const signature = JSON.stringify([
|
|
624
|
+
event.outcome, p.pendingEvents, p.retries, p.rejectedSchemas, p.droppedEvents,
|
|
625
|
+
p.lastAcknowledgedEventId ?? null
|
|
626
|
+
]);
|
|
627
|
+
if (signature === this.lastHealthSignature && now - this.lastHealthAt < HEALTH_REPORT_FLOOR_MS) {
|
|
628
|
+
return null;
|
|
629
|
+
}
|
|
630
|
+
this.lastHealthSignature = signature;
|
|
631
|
+
this.lastHealthAt = now;
|
|
632
|
+
return event;
|
|
633
|
+
}
|
|
634
|
+
healthEvent() {
|
|
635
|
+
const stats = this.options.spool.stats();
|
|
636
|
+
return {
|
|
637
|
+
eventId: `evt_${randomBytes(16).toString("hex")}`,
|
|
638
|
+
eventType: "telemetry.delivery_health",
|
|
639
|
+
occurredAt: new Date().toISOString(),
|
|
640
|
+
outcome: stats.droppedEvents > 0 || this.rejectedSchemaEventIds.size > 0 ? "degraded" : "ok",
|
|
641
|
+
payload: {
|
|
642
|
+
pendingEvents: stats.pendingEvents,
|
|
643
|
+
oldestEventAgeMs: stats.oldestEventAgeMs,
|
|
644
|
+
retries: this.retries,
|
|
645
|
+
rejectedSchemas: this.rejectedSchemaEventIds.size,
|
|
646
|
+
droppedEvents: stats.droppedEvents,
|
|
647
|
+
...(this.lastAcknowledgedEventId ? { lastAcknowledgedEventId: this.lastAcknowledgedEventId } : {})
|
|
648
|
+
}
|
|
649
|
+
};
|
|
650
|
+
}
|
|
651
|
+
/**
|
|
652
|
+
* SAY WHY AN EVENT WAS REFUSED, ONCE PER DISTINCT REASON.
|
|
653
|
+
*
|
|
654
|
+
* A schema rejection is permanent — the same bytes are refused every time, so the event is
|
|
655
|
+
* retired from the outbox and counted lost. That makes this the ONLY moment the cause exists
|
|
656
|
+
* anywhere: after it, the record is gone and all that remains is `rejectedSchemas: N` in a
|
|
657
|
+
* delivery-health payload that names neither the event nor the reason.
|
|
658
|
+
*
|
|
659
|
+
* DEDUPED BY REASON, NOT BY EVENT. A break usually retires every event of one type (132 in one
|
|
660
|
+
* observed batch), and 132 identical lines would bury a second, different cause arriving later
|
|
661
|
+
* in the same run — which is exactly the case that matters, since #66 and #72 surfaced together.
|
|
662
|
+
* The reason is the diagnostic unit; the first event id is carried along to tie it to a record.
|
|
663
|
+
*
|
|
664
|
+
* NOT `warnOnce`, deliberately: that gate is for a channel that is DOWN, where one line per
|
|
665
|
+
* outage is right and the second line says nothing new. Here every distinct reason is new
|
|
666
|
+
* information about a different contract break.
|
|
667
|
+
*
|
|
668
|
+
* SAFE TO PRINT. These strings are the validator's own vocabulary — a field name and a bound
|
|
669
|
+
* ("writeback.completed.payload has unknown field(s): adopted"). They quote the CONTRACT, never
|
|
670
|
+
* the value that violated it, and the server bounds them to 512 characters. Passed through
|
|
671
|
+
* `redactSecrets` anyway: this lands on a consultant's terminal, and the one thing worse than a
|
|
672
|
+
* silent drop is a diagnostic that leaks.
|
|
673
|
+
*/
|
|
674
|
+
reportRejection(entry) {
|
|
675
|
+
const reason = typeof entry.error === "string" && entry.error.trim() ? entry.error.trim() : "";
|
|
676
|
+
if (!reason || this.reportedRejectionReasons.has(reason))
|
|
677
|
+
return;
|
|
678
|
+
this.reportedRejectionReasons.add(reason);
|
|
679
|
+
const at = typeof entry.eventId === "string" ? ` (${entry.eventId})` : "";
|
|
680
|
+
process.stderr.write(`[cc-mcp] telemetry event REJECTED by the ledger and permanently dropped${at}: ` +
|
|
681
|
+
`${redactSecrets(reason)} — this is a schema contract break between the emitter and ` +
|
|
682
|
+
`convex/lib/events.ts, not a network failure; the event will not be retried\n`);
|
|
683
|
+
}
|
|
684
|
+
warnOnce(error) {
|
|
685
|
+
if (this.warned)
|
|
686
|
+
return;
|
|
687
|
+
this.warned = true;
|
|
688
|
+
process.stderr.write(`[cc-mcp] remote telemetry unavailable (${error instanceof Error ? error.message : String(error)}) — events remain in the local outbox; tools are unaffected\n`);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
/**
|
|
692
|
+
* How much a transcript must have grown before a BOUNDARY re-uploads it. 256 KB is roughly a few
|
|
693
|
+
* dozen turns of a real run, so an ordinary chapter ships and a boundary crossed twice in quick
|
|
694
|
+
* succession does not re-gzip the same file for nothing. Not a correctness threshold — the close
|
|
695
|
+
* uploads unconditionally.
|
|
696
|
+
*/
|
|
697
|
+
export const BOUNDARY_TRANSCRIPT_GROWTH_BYTES = 256 * 1024;
|
|
698
|
+
export class Telemetry {
|
|
699
|
+
previousTool = null;
|
|
700
|
+
previousEnd = null;
|
|
701
|
+
file;
|
|
702
|
+
remote;
|
|
703
|
+
/** Bytes last shipped per transcript file, so a boundary can tell a grown capture from a static one. */
|
|
704
|
+
uploadedTranscriptBytes = new Map();
|
|
705
|
+
runId;
|
|
706
|
+
constructor(now = Date.now) {
|
|
707
|
+
this.now = now;
|
|
708
|
+
this.runId = `run_${new Date().toISOString().replace(/[:.]/g, "-")}_${process.pid}`;
|
|
709
|
+
const machineInstallId = installId();
|
|
710
|
+
const serverVersion = installedVersion();
|
|
711
|
+
setProcessTelemetryIdentity({ installId: machineInstallId, runId: this.runId, serverVersion });
|
|
712
|
+
if (process.env.CC_MCP_TELEMETRY === "0") {
|
|
713
|
+
this.file = null;
|
|
714
|
+
this.remote = null;
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
const dir = process.env.CC_MCP_TELEMETRY_DIR?.trim() || path.join(REPO_ROOT, ".local", "cc-mcp", "telemetry");
|
|
718
|
+
let file = null;
|
|
719
|
+
try {
|
|
720
|
+
mkdirSync(dir, { recursive: true });
|
|
721
|
+
file = path.join(dir, `${this.runId}.jsonl`);
|
|
722
|
+
}
|
|
723
|
+
catch {
|
|
724
|
+
file = null; // telemetry must never break the server
|
|
725
|
+
}
|
|
726
|
+
this.file = file;
|
|
727
|
+
// The sink is ready before activation so it can buffer server-start and activation timing.
|
|
728
|
+
// Its authenticated endpoint arrives with bindBatchContext; no unauthenticated default URL is
|
|
729
|
+
// contacted in a development checkout.
|
|
730
|
+
let durable = null;
|
|
731
|
+
if (process.env.CC_MCP_TELEMETRY_REMOTE !== "0") {
|
|
732
|
+
try {
|
|
733
|
+
const localRoot = localTelemetryRoot();
|
|
734
|
+
durable = new DurableRemoteSink({
|
|
735
|
+
spool: new TelemetrySpool({
|
|
736
|
+
dir: path.join(localRoot, "telemetry-outbox"),
|
|
737
|
+
runId: this.runId
|
|
738
|
+
}),
|
|
739
|
+
grants: new TelemetryGrantStore(path.join(localRoot, "telemetry-grants"))
|
|
740
|
+
});
|
|
741
|
+
}
|
|
742
|
+
catch {
|
|
743
|
+
durable = null;
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
this.remote = durable;
|
|
747
|
+
if (this.remote) {
|
|
748
|
+
onDomainTelemetryEvent((event) => this.remote?.pushDomain(event));
|
|
749
|
+
const timer = setInterval(() => {
|
|
750
|
+
const health = this.remote?.informativeHealthEvent(Date.now());
|
|
751
|
+
if (health)
|
|
752
|
+
this.remote?.pushDomain(health);
|
|
753
|
+
void this.remote?.flush();
|
|
754
|
+
}, 20_000);
|
|
755
|
+
timer.unref(); // never keep the process alive for telemetry
|
|
756
|
+
// THE STALE OUTBOX — every event a PRIOR run on this machine left stranded.
|
|
757
|
+
//
|
|
758
|
+
// Anything in the outbox at this instant belongs to a previous process: this one has not
|
|
759
|
+
// emitted its own server_start yet. The events were durable all along (appendFileSync), and
|
|
760
|
+
// the comments throughout this file promise that "a later run flushes" them — until now
|
|
761
|
+
// nothing did, so a run killed mid-batch left its whole ledger on disk until the outbox's
|
|
762
|
+
// 30-day age cap silently dropped it. That is the Windows incident: eight-minute waves, a
|
|
763
|
+
// process that died before `close_batch`, and a batch that never appeared in IN2.
|
|
764
|
+
//
|
|
765
|
+
// Deliberately NOT awaited and never blocking: startup must not wait on the network, and an
|
|
766
|
+
// offline machine pays exactly one failed POST here (see drainOutbox's progress check).
|
|
767
|
+
// Reported to stderr only when it actually recovered something — a clean start stays quiet.
|
|
768
|
+
void this.remote
|
|
769
|
+
.drainOutbox()
|
|
770
|
+
.then((shipped) => {
|
|
771
|
+
if (shipped > 0) {
|
|
772
|
+
process.stderr.write(`[cc-mcp] shipped ${shipped} telemetry event(s) a previous run left in the local outbox\n`);
|
|
773
|
+
}
|
|
774
|
+
})
|
|
775
|
+
.catch(() => undefined);
|
|
776
|
+
}
|
|
777
|
+
// FULL transcript capture, opt-in and separate from everything above — see transcriptUpload.ts
|
|
778
|
+
// for why it does not ride on telemetry's consent. Announced at startup so a machine where it
|
|
779
|
+
// is on can never be mistaken for one where it is not, and shipped at batch close, when the
|
|
780
|
+
// transcript is as complete as it will get.
|
|
781
|
+
//
|
|
782
|
+
// The endpoint is resolved AT CLOSE, not here: this constructor runs before any batch is bound
|
|
783
|
+
// — activate_session happens later — so transcriptEndpoint() is necessarily null at this
|
|
784
|
+
// moment. Capturing it in a closure and reusing that null at exit meant the upload could never
|
|
785
|
+
// run, on any install, ever (caught in review, 2026-07-31).
|
|
786
|
+
if (transcriptUploadEnabled())
|
|
787
|
+
announce(null);
|
|
788
|
+
// PROVENANCE PARCELS (plan 2026-08-02-002 U6; origin R7) — announced here for the same reason
|
|
789
|
+
// transcripts are, and flushed here for one specific one: a parcel is attributable only through
|
|
790
|
+
// the batch grant that uploads it, so anything still in the outbox when the process ends has to
|
|
791
|
+
// go out BEFORE the batch closes rather than waiting for a run whose grant may have expired.
|
|
792
|
+
//
|
|
793
|
+
// Announcement carries no endpoint at this point: like the transcript line above, this runs
|
|
794
|
+
// before any batch is bound. The once-guard lives in the module, so whichever comes first — this
|
|
795
|
+
// or the first wave's build — is the only line the operator sees.
|
|
796
|
+
//
|
|
797
|
+
// Sits with the transcript block deliberately: CC_MCP_TELEMETRY=0 returns above and takes both
|
|
798
|
+
// with it. One switch that stops everything leaving the machine is the behaviour an operator
|
|
799
|
+
// reaching for it expects, and CC_MCP_PARCEL_UPLOAD=0 remains the narrow way out.
|
|
800
|
+
announceParcelUpload();
|
|
801
|
+
// OPERATOR Q&A — the question cards and what came back, shipped at close by their own route.
|
|
802
|
+
// Announced here beside the other two because it is content leaving the machine, and stated
|
|
803
|
+
// separately from the transcript line for the reason the narrow opt-out exists: the transcript
|
|
804
|
+
// announcement already says the whole session goes, so what this adds is that the questions and
|
|
805
|
+
// answers are also stored in a form somebody can search. Endpoint is null at this point for the
|
|
806
|
+
// same reason as the two above — no batch is bound until activation.
|
|
807
|
+
announceOperatorQa();
|
|
808
|
+
// BETA FEEDBACK — announced beside the other three because it is content leaving the machine,
|
|
809
|
+
// and stated separately because what it adds is a category the other three do not carry: the
|
|
810
|
+
// MODEL's own account of the run, plus the operator's ratings and words if they give them.
|
|
811
|
+
// Silent on an install that is not in beta. Endpoint is null here for the same reason as the
|
|
812
|
+
// three above — no batch is bound until activation.
|
|
813
|
+
//
|
|
814
|
+
// Below the `CC_MCP_TELEMETRY=0` early return with the rest of them, deliberately: one switch
|
|
815
|
+
// that stops everything leaving the machine is what an operator reaching for it expects, and
|
|
816
|
+
// `feedbackUploadEnabled` states the same rule again at the flush rather than inheriting it.
|
|
817
|
+
announceBeta();
|
|
818
|
+
// ONE explicit end for the batch, replacing five separate guesses at one.
|
|
819
|
+
//
|
|
820
|
+
// Everything that used to hang off its own `process.once("beforeExit", …)` — remote flush,
|
|
821
|
+
// transcript ship, parcel flush, finalizeBatch, contamination — now runs here, in order, as
|
|
822
|
+
// awaitable work. `close_batch` drives it while the server is alive; the exit hooks below are
|
|
823
|
+
// only a backstop. See registerBatchCloser in telemetryContext.ts for the measured reason.
|
|
824
|
+
registerBatchCloser(() => this.closeBatch());
|
|
825
|
+
// The mid-run half of the same seam: every chapter boundary ships what has happened so far, so
|
|
826
|
+
// a run that dies before its end is not a run whose evidence never left the machine.
|
|
827
|
+
registerBoundaryFlusher(() => this.flushAtBoundary());
|
|
828
|
+
this.installExitBackstops();
|
|
829
|
+
this.write({
|
|
830
|
+
eventId: `evt_${randomBytes(16).toString("hex")}`,
|
|
831
|
+
ts: new Date().toISOString(),
|
|
832
|
+
event: "server_start",
|
|
833
|
+
pid: process.pid,
|
|
834
|
+
runId: this.runId
|
|
835
|
+
});
|
|
836
|
+
}
|
|
837
|
+
now;
|
|
838
|
+
/**
|
|
839
|
+
* Ship the full transcript, once, at exit. Best effort by contract: this runs after the work is
|
|
840
|
+
* done, so nothing it does may fail a delivery or delay a shutdown that matters. Every outcome —
|
|
841
|
+
* including "found nothing" and "too large" — is written to stderr, because a capture that
|
|
842
|
+
* silently did not happen is worse than one that never claimed to.
|
|
843
|
+
*/
|
|
844
|
+
/**
|
|
845
|
+
* Read this session's wave bundles off disk and state the batch's terminal result.
|
|
846
|
+
*
|
|
847
|
+
* COMPLETE every real wave delivered and nothing needs review
|
|
848
|
+
* NEEDS_ATTENTION some wave was rejected, ambiguous or blocked — a human has to look
|
|
849
|
+
* FAILED real waves ran and none delivered
|
|
850
|
+
* (nothing sent) NO_WAVES: nothing reached the org — a session that only read, that died before
|
|
851
|
+
* deploying, or whose only bundles were accepted practice runs. Emitting COMPLETE
|
|
852
|
+
* there would call a batch finished that never delivered anything.
|
|
853
|
+
*
|
|
854
|
+
* The rule itself is NOT here. `gradeWaves` owns it and `composeBatchReport` calls the same
|
|
855
|
+
* function on the same bundles, because the ledger's terminal result and the operator's local
|
|
856
|
+
* report are two audiences for ONE grade — and when each derived it separately, both counted a
|
|
857
|
+
* `dryRunOnly` practice bundle as a wave needing review (PV Arm B, 2026-08-02).
|
|
858
|
+
*
|
|
859
|
+
* Static import, unlike the report below: `waveGrade.ts` is a dependency-free leaf (node:fs and
|
|
860
|
+
* node:path only), and this must stay SYNCHRONOUS — reached from a signal handler, only work that
|
|
861
|
+
* is already synchronous survives. The report's lazy import is about git plumbing, not layering.
|
|
862
|
+
*/
|
|
863
|
+
finalizeBatch() {
|
|
864
|
+
const sessionId = this.remote?.batchSessionId();
|
|
865
|
+
if (!sessionId)
|
|
866
|
+
return null;
|
|
867
|
+
try {
|
|
868
|
+
const grade = gradeWaves(readWaveBundles(path.join(sessionDir(sessionId), "direct")));
|
|
869
|
+
if (grade.result === "NO_WAVES")
|
|
870
|
+
return null;
|
|
871
|
+
emitDomainTelemetryEvent("batch.completed", {
|
|
872
|
+
result: grade.result,
|
|
873
|
+
reason: grade.reason,
|
|
874
|
+
// A PARTITION of the bundles: delivered + problem + practice === waveCount. The dashboard
|
|
875
|
+
// publishes all four, so a practice run excluded from the problem count has to be visible
|
|
876
|
+
// somewhere or the numbers stop adding up.
|
|
877
|
+
waveCount: grade.waves.length,
|
|
878
|
+
deliveredWaves: grade.deliveredWaves,
|
|
879
|
+
problemWaves: grade.problemWaves,
|
|
880
|
+
practiceRuns: grade.practiceRuns
|
|
881
|
+
}, { stage: "settle", outcome: grade.result });
|
|
882
|
+
// The event is on disk the moment it is emitted (the spool uses appendFileSync), so a SIGKILL
|
|
883
|
+
// after this point costs the network hop, not the record — a later run flushes it.
|
|
884
|
+
return { result: grade.result, waveCount: grade.waves.length };
|
|
885
|
+
}
|
|
886
|
+
catch {
|
|
887
|
+
// Never let finalization break shutdown.
|
|
888
|
+
return null;
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
/**
|
|
892
|
+
* Assemble and persist this batch's closing artifacts — the delivery log, the contamination
|
|
893
|
+
* report (U5; R22), and the batch-level findings that per-wave notes bury.
|
|
894
|
+
*
|
|
895
|
+
* LOCAL ONLY, and that is a boundary rather than a default. The report names components, and
|
|
896
|
+
* component identity is org content; the Convex event tables are shape-only by contract
|
|
897
|
+
* (`convex/lib/events.ts`, origin R14), so nothing here is added to `batch.completed` or to any
|
|
898
|
+
* other event. The output is session artifacts plus the text `close_batch` returns, which is what
|
|
899
|
+
* the operator, the supervisor and the bench harness read.
|
|
900
|
+
*
|
|
901
|
+
* Imported LAZILY on purpose, and for a narrower reason than the layering: `batchReport` drags in
|
|
902
|
+
* the org mirror's git plumbing and the contamination reader, and a static import would pull all
|
|
903
|
+
* of that onto every server start for a module used once at the end of a run. (The grading rule
|
|
904
|
+
* this file also needs is a different case — `waveGrade.ts` is a dependency-free leaf and is
|
|
905
|
+
* imported statically at the top, because `finalizeBatch` must stay synchronous.)
|
|
906
|
+
*/
|
|
907
|
+
async writeReport() {
|
|
908
|
+
const sessionId = this.remote?.batchSessionId();
|
|
909
|
+
if (!sessionId)
|
|
910
|
+
return null;
|
|
911
|
+
// Read the binding off disk rather than through the tools layer, exactly as finalizeBatch
|
|
912
|
+
// reads verdicts: this may run during shutdown and must not drag the session module in.
|
|
913
|
+
const session = JSON.parse(readFileSync(path.join(sessionDir(sessionId), "session.json"), "utf8"));
|
|
914
|
+
const orgId = session.targetOrg?.orgId;
|
|
915
|
+
// No org id means no join key — the mirror and the read-set are both org-keyed. Silence is
|
|
916
|
+
// the honest answer; an unkeyed report would be a guess about which org was measured.
|
|
917
|
+
if (!orgId)
|
|
918
|
+
return null;
|
|
919
|
+
const { composeBatchReport, renderBatchReport, writeBatchReportArtifacts } = await import("../direct/batchReport.js");
|
|
920
|
+
const report = await composeBatchReport({
|
|
921
|
+
sessionId,
|
|
922
|
+
orgId,
|
|
923
|
+
...(session.targetOrg?.alias ? { orgAlias: session.targetOrg.alias } : {})
|
|
924
|
+
});
|
|
925
|
+
const text = renderBatchReport(report);
|
|
926
|
+
const written = await writeBatchReportArtifacts(report, text);
|
|
927
|
+
return written ? { markdown: written.markdown, text } : { markdown: "", text };
|
|
928
|
+
}
|
|
929
|
+
/**
|
|
930
|
+
* Close the batch: state its result, write its evidence, and ship everything that was waiting on
|
|
931
|
+
* the end of the run — in that order, and awaited, so a caller knows what actually happened.
|
|
932
|
+
*
|
|
933
|
+
* ORDER IS DELIBERATE and runs cheapest-and-most-durable first, because any step may be the last
|
|
934
|
+
* one that gets to run if this is reached from a signal rather than from `close_batch`:
|
|
935
|
+
* 1. finalizeBatch — synchronous, lands the terminal event on disk via appendFileSync
|
|
936
|
+
* 2. writeReport — local files only, no network
|
|
937
|
+
* 3. operator Q&A — network, but a few KB and NOT resumable: no outbox, so it goes before the
|
|
938
|
+
* two large uploads rather than behind them
|
|
939
|
+
* 4. beta feedback — network, a few KB, and resumable from its own outbox. Cheap, so it goes
|
|
940
|
+
* ahead of the two large uploads; durable, so being cut off here costs a
|
|
941
|
+
* later run's flush rather than the reports
|
|
942
|
+
* 5. parcels — network, but resumable from the outbox on a later run
|
|
943
|
+
* 6. transcript — network, the largest and least resumable
|
|
944
|
+
* 7. remote flush — drains everything the first six queued
|
|
945
|
+
*
|
|
946
|
+
* Every step is independently guarded: one failing must not cost the others, since they answer
|
|
947
|
+
* different questions and a partial close is far better than none.
|
|
948
|
+
*/
|
|
949
|
+
async closeBatch() {
|
|
950
|
+
const notes = [];
|
|
951
|
+
if (!this.remote?.batchSessionId()) {
|
|
952
|
+
return { finalized: false, notes: ["no batch bound to this server process — nothing to finalize"] };
|
|
953
|
+
}
|
|
954
|
+
let graded = null;
|
|
955
|
+
try {
|
|
956
|
+
graded = this.finalizeBatch();
|
|
957
|
+
notes.push(graded
|
|
958
|
+
? `batch graded ${graded.result} from ${graded.waveCount} wave verdict(s)`
|
|
959
|
+
: "nothing reached the org — batch left non-terminal, which is the honest answer for a run that only read or only practiced");
|
|
960
|
+
}
|
|
961
|
+
catch (error) {
|
|
962
|
+
notes.push(`terminal batch event not emitted: ${describeError(error)}`);
|
|
963
|
+
}
|
|
964
|
+
try {
|
|
965
|
+
const report = await this.writeReport();
|
|
966
|
+
notes.push(report?.markdown ? `batch report written to ${report.markdown}` : "batch report not applicable (no org bound)");
|
|
967
|
+
}
|
|
968
|
+
catch (error) {
|
|
969
|
+
notes.push(`batch report not produced: ${describeError(error)}`);
|
|
970
|
+
}
|
|
971
|
+
try {
|
|
972
|
+
notes.push(...(await this.shipOperatorQa()));
|
|
973
|
+
}
|
|
974
|
+
catch (error) {
|
|
975
|
+
notes.push(`operator Q&A upload failed: ${describeError(error)}`);
|
|
976
|
+
}
|
|
977
|
+
try {
|
|
978
|
+
// Flushes EVERY batch in the outbox, not only this one — that is what makes a report filed by
|
|
979
|
+
// a run that was then killed reach IN2 at all. Silent on an install that is not in beta.
|
|
980
|
+
notes.push(...(await flushFeedback()).notes);
|
|
981
|
+
}
|
|
982
|
+
catch (error) {
|
|
983
|
+
notes.push(`beta feedback flush failed: ${describeError(error)}`);
|
|
984
|
+
}
|
|
985
|
+
try {
|
|
986
|
+
const summary = await flushParcels();
|
|
987
|
+
// A parcel left in the outbox is evidence that has NOT shipped — usually a batch whose grant
|
|
988
|
+
// went before its last wave did. It stays on disk and gets one line, because "the outbox is
|
|
989
|
+
// empty" and "the outbox is holding four waves" must never look the same from outside.
|
|
990
|
+
if (summary.retained > 0) {
|
|
991
|
+
const days = Math.floor(summary.oldestAgeMs / 86_400_000);
|
|
992
|
+
notes.push(`${summary.retained} provenance parcel(s) still queued${days >= 1 ? `, oldest waiting ${days} day(s)` : ""} — ` +
|
|
993
|
+
`they stay in the outbox and go out on a later run under the same batch's grant. The deliveries themselves are complete.`);
|
|
994
|
+
}
|
|
995
|
+
if (summary.outboxDropped > 0) {
|
|
996
|
+
notes.push(`${summary.outboxDropped} provenance parcel(s) were dropped from the outbox by its size/age caps and did NOT reach IN2. Their evidence bundles are still on this machine.`);
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
catch (error) {
|
|
1000
|
+
notes.push(`parcel flush failed: ${describeError(error)}`);
|
|
1001
|
+
}
|
|
1002
|
+
try {
|
|
1003
|
+
notes.push(...(await this.shipTranscript()));
|
|
1004
|
+
}
|
|
1005
|
+
catch (error) {
|
|
1006
|
+
notes.push(`transcript upload failed: ${describeError(error)}`);
|
|
1007
|
+
}
|
|
1008
|
+
try {
|
|
1009
|
+
// DRAIN, not one flush. `flush()` ships at most 200 events, and a real run emits far more —
|
|
1010
|
+
// a tool event per call plus a health tick every 20 seconds — so a single call here left the
|
|
1011
|
+
// remainder for a timer that is about to stop ticking. That is the same "events never reached
|
|
1012
|
+
// the ledger" failure this work exists to close, on the path that is meant to be reliable.
|
|
1013
|
+
await this.remote?.drainOutbox();
|
|
1014
|
+
const left = this.remote?.pendingEventCount() ?? 0;
|
|
1015
|
+
if (left > 0) {
|
|
1016
|
+
notes.push(`${left} telemetry event(s) could not be shipped and remain in the local outbox — they go out on the next run's start`);
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
catch (error) {
|
|
1020
|
+
notes.push(`telemetry flush failed: ${describeError(error)}`);
|
|
1021
|
+
}
|
|
1022
|
+
for (const note of notes)
|
|
1023
|
+
process.stderr.write(`[cc-mcp] ${note}\n`);
|
|
1024
|
+
return {
|
|
1025
|
+
finalized: true,
|
|
1026
|
+
...(graded ? { result: graded.result, waveCount: graded.waveCount } : {}),
|
|
1027
|
+
notes
|
|
1028
|
+
};
|
|
1029
|
+
}
|
|
1030
|
+
/**
|
|
1031
|
+
* SHIP WHAT HAS HAPPENED SO FAR — run at every chapter boundary, while the server is alive.
|
|
1032
|
+
*
|
|
1033
|
+
* THE HOLE THIS CLOSES, stated as the measurement rather than the intention. `close_batch` is the
|
|
1034
|
+
* only thing that drains the telemetry outbox and uploads the transcript, and it runs at the END.
|
|
1035
|
+
* A run that dies at wave three therefore ships NOTHING: its events sit in
|
|
1036
|
+
* `.local/cc-mcp/telemetry-outbox/` with no scheduled drain (the 20-second timer flushes only
|
|
1037
|
+
* while the process lives, and it is the process dying that strands them), and its transcript,
|
|
1038
|
+
* which has no outbox at all, is simply lost. The exit backstop cannot help — it gets at most
|
|
1039
|
+
* 2000 ms and only for synchronous work, and on Windows a client teardown routinely gives it
|
|
1040
|
+
* nothing. Those events then wait for some LATER run of the same install to flush a batch whose
|
|
1041
|
+
* grant has not yet expired, which frequently never happens.
|
|
1042
|
+
*
|
|
1043
|
+
* A boundary is the natural place to fix it: the supervisor is already stopping to say where the
|
|
1044
|
+
* run stands, there are eight of them before the end, and the work is exactly the work `close_batch`
|
|
1045
|
+
* does — the SAME functions, not a second implementation of them, so there is one shipping path to
|
|
1046
|
+
* keep correct.
|
|
1047
|
+
*
|
|
1048
|
+
* ORDER, cheapest and most-recoverable first, because a boundary can also be the last thing this
|
|
1049
|
+
* process ever does:
|
|
1050
|
+
* 1. remote flush — drains the outbox, which is the named hole and the cheapest step
|
|
1051
|
+
* 2. parcels — resumable from their own outbox; a wave's evidence goes with its grant
|
|
1052
|
+
* 3. transcript — the largest, and the one with no outbox, so it goes last but it does go
|
|
1053
|
+
*
|
|
1054
|
+
* NOT INCLUDED, deliberately: the terminal batch event, the batch report, the operator Q&A record
|
|
1055
|
+
* and the beta wrap-up. Each of those is a statement about a FINISHED run — a terminal grade
|
|
1056
|
+
* emitted at wave three would be a lie, and a Q&A record shipped mid-clarification would send
|
|
1057
|
+
* every card twice, once open and once settled, which is the exact reason that upload reads the
|
|
1058
|
+
* round files at the end instead of queueing them as they happen.
|
|
1059
|
+
*/
|
|
1060
|
+
async flushAtBoundary() {
|
|
1061
|
+
const notes = [];
|
|
1062
|
+
if (!this.remote?.batchSessionId()) {
|
|
1063
|
+
return { flushed: false, notes: ["no batch bound to this server process — nothing to ship yet"] };
|
|
1064
|
+
}
|
|
1065
|
+
try {
|
|
1066
|
+
await this.remote?.flush();
|
|
1067
|
+
notes.push("telemetry outbox flushed");
|
|
1068
|
+
}
|
|
1069
|
+
catch (error) {
|
|
1070
|
+
notes.push(`telemetry flush failed: ${describeError(error)} — events stay in the local outbox`);
|
|
1071
|
+
}
|
|
1072
|
+
try {
|
|
1073
|
+
const summary = await flushParcels();
|
|
1074
|
+
if (summary.retained > 0) {
|
|
1075
|
+
notes.push(`${summary.retained} provenance parcel(s) still queued — they go out under the same batch's grant`);
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
catch (error) {
|
|
1079
|
+
notes.push(`parcel flush failed: ${describeError(error)}`);
|
|
1080
|
+
}
|
|
1081
|
+
try {
|
|
1082
|
+
notes.push(...(await this.shipTranscript({ minimumGrowthBytes: BOUNDARY_TRANSCRIPT_GROWTH_BYTES })));
|
|
1083
|
+
}
|
|
1084
|
+
catch (error) {
|
|
1085
|
+
notes.push(`transcript upload failed: ${describeError(error)}`);
|
|
1086
|
+
}
|
|
1087
|
+
for (const note of notes)
|
|
1088
|
+
process.stderr.write(`[cc-mcp] boundary: ${note}\n`);
|
|
1089
|
+
return { flushed: true, notes };
|
|
1090
|
+
}
|
|
1091
|
+
/**
|
|
1092
|
+
* BACKSTOPS — never the plan, and honest about how little they can do.
|
|
1093
|
+
*
|
|
1094
|
+
* MEASURED against the real client (`StdioClientTransport.close()`, SDK 1.29.0): stdin is closed,
|
|
1095
|
+
* then SIGTERM lands **2000 ms** later, then SIGKILL 2000 ms after that. Two consequences, both
|
|
1096
|
+
* verified by subprocess test (`batchClose.test.ts`):
|
|
1097
|
+
*
|
|
1098
|
+
* - `beforeExit` runs ONLY if the event loop drains within that window. Any still-ref'd handle
|
|
1099
|
+
* means it never runs, and the process is signalled instead. This is why five subsystems
|
|
1100
|
+
* hanging off `beforeExit` produced 0 terminal batches across 7 real engagements.
|
|
1101
|
+
* - Inside a SIGNAL handler there is no more time than that: Node will not extend the window,
|
|
1102
|
+
* and the client's SIGKILL is not catchable. Only work that is ALREADY synchronous survives
|
|
1103
|
+
* reliably — for us, `finalizeBatch`'s spool append (appendFileSync), which a later run then
|
|
1104
|
+
* flushes. Network steps here are a bet, not a guarantee, and are labelled as such.
|
|
1105
|
+
*
|
|
1106
|
+
* So these hooks exist to salvage the terminal event when the supervisor never called
|
|
1107
|
+
* `close_batch`. They are not a substitute for it, and nothing that matters may depend on them.
|
|
1108
|
+
*/
|
|
1109
|
+
installExitBackstops() {
|
|
1110
|
+
// Once per PROCESS, not once per instance. Tests construct several Telemetry objects, and three
|
|
1111
|
+
// signal listeners per construction would leak handlers and trip Node's max-listeners warning.
|
|
1112
|
+
if (exitBackstopsInstalled)
|
|
1113
|
+
return;
|
|
1114
|
+
exitBackstopsInstalled = true;
|
|
1115
|
+
process.once("beforeExit", () => {
|
|
1116
|
+
if (batchTelemetryClosed())
|
|
1117
|
+
return;
|
|
1118
|
+
process.stderr.write("[cc-mcp] closing the batch from the exit backstop — close_batch was never called, so this run had no explicit end. Anything still on the network when the client kills this process is lost.\n");
|
|
1119
|
+
void closeBatchTelemetry();
|
|
1120
|
+
});
|
|
1121
|
+
for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"]) {
|
|
1122
|
+
process.once(signal, () => {
|
|
1123
|
+
if (!batchTelemetryClosed()) {
|
|
1124
|
+
process.stderr.write(`[cc-mcp] ${signal} received before close_batch. Emitting the terminal batch event synchronously; ` +
|
|
1125
|
+
`the transcript, parcels, the operator Q&A record and any unflushed events do NOT reliably survive a signal. ` +
|
|
1126
|
+
`Parcels and events are left for a later run; the transcript and the Q&A record have no outbox, so for them this run is the only chance.\n`);
|
|
1127
|
+
// Deliberately NOT awaited: there is nothing to await it with. The synchronous part —
|
|
1128
|
+
// the terminal event's spool append — has already landed by the time this returns.
|
|
1129
|
+
void closeBatchTelemetry();
|
|
1130
|
+
}
|
|
1131
|
+
// Re-raise so the exit status still reports the signal rather than a clean 0 this process
|
|
1132
|
+
// did not earn. `once` has already removed THIS listener, so if nothing else is listening
|
|
1133
|
+
// the default disposition applies and the process dies here.
|
|
1134
|
+
//
|
|
1135
|
+
// Deliberately NOT removeAllListeners: attaching a listener is what suppresses the default
|
|
1136
|
+
// action in the first place, so tearing off every other listener would silently take over a
|
|
1137
|
+
// shutdown this module does not own — a host embedding the server, or a test runner. If
|
|
1138
|
+
// another listener remains it stays in charge, which is the correct outcome.
|
|
1139
|
+
process.kill(process.pid, signal);
|
|
1140
|
+
});
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
/**
|
|
1144
|
+
* Ship the question cards and their answers, once, at close.
|
|
1145
|
+
*
|
|
1146
|
+
* Read from the session's own round files rather than queued as they happen, and that is the
|
|
1147
|
+
* deliberate part: an answer is recorded MINUTES after its question, so a per-event shipment
|
|
1148
|
+
* would send every card twice — once open, once settled — and leave the ledger to work out which
|
|
1149
|
+
* copy is current. Reading at the end sends each round once, in its final state.
|
|
1150
|
+
*
|
|
1151
|
+
* The cost is stated rather than hidden: a run that never reaches `close_batch` ships no Q&A, the
|
|
1152
|
+
* same way it ships no transcript. There is no outbox behind this.
|
|
1153
|
+
*/
|
|
1154
|
+
async shipOperatorQa() {
|
|
1155
|
+
return await shipOperatorQa({
|
|
1156
|
+
sessionId: this.remote?.batchSessionId() ?? null,
|
|
1157
|
+
endpoint: this.remote?.operatorQaEndpoint() ?? null,
|
|
1158
|
+
credential: this.remote?.batchCredential() ?? null
|
|
1159
|
+
});
|
|
1160
|
+
}
|
|
1161
|
+
/**
|
|
1162
|
+
* @param opts.minimumGrowthBytes when set, a transcript this process has already uploaded is
|
|
1163
|
+
* re-sent only once it has grown by at least this much. Boundary uploads pass it; the close does
|
|
1164
|
+
* not, because the final capture must go whatever the last few turns weighed.
|
|
1165
|
+
*/
|
|
1166
|
+
async shipTranscript(opts = {}) {
|
|
1167
|
+
if (!transcriptUploadEnabled())
|
|
1168
|
+
return [];
|
|
1169
|
+
const endpoint = this.remote?.transcriptEndpoint() ?? null;
|
|
1170
|
+
const credential = this.remote?.batchCredential() ?? null;
|
|
1171
|
+
const markers = this.remote?.transcriptMarkers() ?? [];
|
|
1172
|
+
if (!endpoint || !credential) {
|
|
1173
|
+
return ["transcript upload skipped: no batch grant on this install yet"];
|
|
1174
|
+
}
|
|
1175
|
+
// Matched on content — every org-touching tool call carries the IN2 session id as an argument,
|
|
1176
|
+
// so Claude Code records it as tool input. A live check found one session id in THREE
|
|
1177
|
+
// transcripts at once (parallel agents share a session), so this deliberately ships all of
|
|
1178
|
+
// them; picking one would silently drop the others. mtime is only the fallback.
|
|
1179
|
+
const found = findTranscriptsForSession(process.cwd(), markers);
|
|
1180
|
+
const all = found.length > 0 ? found : [findTranscript(process.cwd())].filter((t) => Boolean(t));
|
|
1181
|
+
if (all.length === 0) {
|
|
1182
|
+
// THE SILENT FAILURE, MADE COUNTABLE. This branch is where a week of missing transcripts
|
|
1183
|
+
// actually went: `transcriptDirFor` resolved a directory that does not exist, the scan found
|
|
1184
|
+
// nothing, and the run returned this string to `close_batch`, which prints it — where nobody
|
|
1185
|
+
// read it. Nineteen batches across six installs, and the ledger held no evidence the upload
|
|
1186
|
+
// had even been attempted, so the failure was indistinguishable from the feature being off.
|
|
1187
|
+
//
|
|
1188
|
+
// Emitted BEFORE the early return so "we looked and found nothing" is a fact in the ledger
|
|
1189
|
+
// rather than a line in a report. Shape only — a count and a flag, no path: `process.cwd()`
|
|
1190
|
+
// is a consultant's directory layout and stays on their machine, in the note below.
|
|
1191
|
+
this.reportTranscriptDiscovery({ found: 0, markerMatched: false });
|
|
1192
|
+
return [`transcript upload skipped: no Claude Code transcript found for ${process.cwd()}`];
|
|
1193
|
+
}
|
|
1194
|
+
this.reportTranscriptDiscovery({ found: all.length, markerMatched: found.length > 0 });
|
|
1195
|
+
const notes = [];
|
|
1196
|
+
if (found.length === 0) {
|
|
1197
|
+
notes.push("transcript: no file references this session — falling back to the most recent, which may be the wrong one");
|
|
1198
|
+
}
|
|
1199
|
+
// THE GROWTH GATE — boundary uploads only, and it is about local cost rather than correctness.
|
|
1200
|
+
//
|
|
1201
|
+
// The ledger already refuses to replace a larger capture with a smaller one, so re-uploading is
|
|
1202
|
+
// SAFE; what it is not is free. A transcript is gzipped in full on every send, and a long run
|
|
1203
|
+
// crossing eight boundaries would compress the same tens of megabytes eight times for a handful
|
|
1204
|
+
// of new turns. So a file this process has already shipped is re-sent only once it has grown by
|
|
1205
|
+
// a meaningful amount. The CLOSE passes no threshold at all, which is what keeps the final
|
|
1206
|
+
// capture unconditional — an incremental upload is a safety net under the close, never a
|
|
1207
|
+
// replacement for it.
|
|
1208
|
+
const minimum = opts.minimumGrowthBytes;
|
|
1209
|
+
const transcripts = minimum === undefined
|
|
1210
|
+
? all
|
|
1211
|
+
: all.filter((t) => t.bytes - (this.uploadedTranscriptBytes.get(t.file) ?? 0) >= minimum);
|
|
1212
|
+
if (transcripts.length === 0) {
|
|
1213
|
+
return notes.concat([`transcript: no capture has grown by ${Math.round(minimum / 1024)} KB since the last upload — nothing re-sent`]);
|
|
1214
|
+
}
|
|
1215
|
+
for (const t of transcripts)
|
|
1216
|
+
this.uploadedTranscriptBytes.set(t.file, t.bytes);
|
|
1217
|
+
return notes.concat(await shipSessionTranscripts({ endpoint, credential, transcripts }));
|
|
1218
|
+
}
|
|
1219
|
+
/**
|
|
1220
|
+
* SAY THAT WE LOOKED, AND WHAT WE FOUND — the counter this whole investigation was missing.
|
|
1221
|
+
*
|
|
1222
|
+
* Upload stays non-load-bearing: this never throws, never blocks, and a run whose transcript is
|
|
1223
|
+
* lost still closes exactly as before. But "never blocks" and "never tells anyone" are different
|
|
1224
|
+
* properties, and only the first was ever required. Between 2026-07-31 and 2026-08-06, 19 batches
|
|
1225
|
+
* across 6 installs ran a build that could upload and 4 shipped, and NOTHING in the ledger
|
|
1226
|
+
* distinguished "the discovery scan came back empty" from "this install opted out" from "the
|
|
1227
|
+
* feature was never reached". Three very different causes, one identical absence of rows.
|
|
1228
|
+
*
|
|
1229
|
+
* With this event the difference is one query: a run that emits `found: 0` looked and failed, a
|
|
1230
|
+
* run that emits nothing at all never got as far as looking. That is the signal whose absence let
|
|
1231
|
+
* the defect above run for a week.
|
|
1232
|
+
*
|
|
1233
|
+
* SHAPE ONLY, like every other domain event. A count and a flag — never `process.cwd()`, which is
|
|
1234
|
+
* a consultant's directory layout, and never a file name. The operator-facing note beside this
|
|
1235
|
+
* call carries the path, because that note stays on their machine.
|
|
1236
|
+
*/
|
|
1237
|
+
reportTranscriptDiscovery(opts) {
|
|
1238
|
+
try {
|
|
1239
|
+
emitDomainTelemetryEvent("transcript.discovery", { transcriptsFound: opts.found, markerMatched: opts.markerMatched }, { stage: "settle", outcome: opts.found > 0 ? "found" : "none" });
|
|
1240
|
+
}
|
|
1241
|
+
catch {
|
|
1242
|
+
// Fail-open, like everything else on this path: a run must never break over a counter.
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
write(record) {
|
|
1246
|
+
try {
|
|
1247
|
+
if (this.file)
|
|
1248
|
+
appendFileSync(this.file, `${JSON.stringify(record)}\n`, "utf8");
|
|
1249
|
+
}
|
|
1250
|
+
catch {
|
|
1251
|
+
/* never throw from telemetry */
|
|
1252
|
+
}
|
|
1253
|
+
try {
|
|
1254
|
+
this.remote?.push(record);
|
|
1255
|
+
}
|
|
1256
|
+
catch {
|
|
1257
|
+
/* never throw from telemetry */
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
/** Wrap a tool handler so every invocation is timed and classified. */
|
|
1261
|
+
instrument(tool, handler) {
|
|
1262
|
+
return async (...args) => {
|
|
1263
|
+
const started = this.now();
|
|
1264
|
+
const gapMs = this.previousEnd === null ? undefined : started - this.previousEnd;
|
|
1265
|
+
const gapClass = classifyGap(this.previousTool);
|
|
1266
|
+
let result;
|
|
1267
|
+
let threw = false;
|
|
1268
|
+
try {
|
|
1269
|
+
result = await handler(...args);
|
|
1270
|
+
return result;
|
|
1271
|
+
}
|
|
1272
|
+
catch (error) {
|
|
1273
|
+
threw = true;
|
|
1274
|
+
throw error;
|
|
1275
|
+
}
|
|
1276
|
+
finally {
|
|
1277
|
+
const ended = this.now();
|
|
1278
|
+
this.write({
|
|
1279
|
+
eventId: `evt_${randomBytes(16).toString("hex")}`,
|
|
1280
|
+
ts: new Date().toISOString(),
|
|
1281
|
+
event: "tool",
|
|
1282
|
+
tool,
|
|
1283
|
+
stage: stageForTool(tool),
|
|
1284
|
+
ms: ended - started,
|
|
1285
|
+
...(gapMs !== undefined ? { gapMs } : {}),
|
|
1286
|
+
gapClass,
|
|
1287
|
+
input: summarizeInput(args[0]),
|
|
1288
|
+
output: threw ? { ok: false, errorCode: "TOOL_THREW" } : summarizeOutput(result),
|
|
1289
|
+
runId: this.runId,
|
|
1290
|
+
threw
|
|
1291
|
+
});
|
|
1292
|
+
this.previousTool = tool;
|
|
1293
|
+
this.previousEnd = ended;
|
|
1294
|
+
}
|
|
1295
|
+
};
|
|
1296
|
+
}
|
|
1297
|
+
}
|