in2-agent-server 0.20260821.185429
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +369 -0
- package/agents/plan-requirement.md +32 -0
- package/dist/campfire/configItem.js +201 -0
- package/dist/campfire/configItemDescription.js +379 -0
- package/dist/campfire/deriveFromBundle.js +1013 -0
- package/dist/cli.js +465 -0
- package/dist/core/betaFeedback.js +493 -0
- package/dist/core/chapterInstructions.js +473 -0
- package/dist/core/claudeAssets.js +61 -0
- package/dist/core/gate.js +201 -0
- package/dist/core/ids.js +9 -0
- package/dist/core/install.js +60 -0
- package/dist/core/operatorQa.js +462 -0
- package/dist/core/orgCallMeter.js +178 -0
- package/dist/core/parcels.js +1235 -0
- package/dist/core/paths.js +105 -0
- package/dist/core/phases.js +225 -0
- package/dist/core/planFile.js +251 -0
- package/dist/core/planFragments.js +119 -0
- package/dist/core/readsLog.js +398 -0
- package/dist/core/registration.js +24 -0
- package/dist/core/result.js +38 -0
- package/dist/core/resume.js +233 -0
- package/dist/core/revisionChain.js +351 -0
- package/dist/core/richText.js +61 -0
- package/dist/core/semanticHash.js +89 -0
- package/dist/core/sessionOrg.js +60 -0
- package/dist/core/sfQuery.js +19 -0
- package/dist/core/spawn.js +182 -0
- package/dist/core/supervisorContract.js +230 -0
- package/dist/core/telemetry.js +1297 -0
- package/dist/core/telemetryContext.js +154 -0
- package/dist/core/telemetrySpool.js +244 -0
- package/dist/core/transcriptUpload.js +516 -0
- package/dist/core/update.js +363 -0
- package/dist/core/viewer.js +117 -0
- package/dist/core/workspace.js +92 -0
- package/dist/core/writeGrants.js +93 -0
- package/dist/core/writebackManifest.js +92 -0
- package/dist/direct/applyLoop.js +2516 -0
- package/dist/direct/batchReport.js +510 -0
- package/dist/direct/comparatorNoise.js +75 -0
- package/dist/direct/containers.js +85 -0
- package/dist/direct/contamination.js +221 -0
- package/dist/direct/crud.js +56 -0
- package/dist/direct/destructive.js +168 -0
- package/dist/direct/knownTokens.js +94 -0
- package/dist/direct/lint.js +382 -0
- package/dist/direct/metadataSchema.generated.json +1 -0
- package/dist/direct/metadataSchema.js +179 -0
- package/dist/direct/operatorPage.js +349 -0
- package/dist/direct/orgRepo.js +1531 -0
- package/dist/direct/orgRepoSpawnCount.js +21 -0
- package/dist/direct/plannedScope.js +119 -0
- package/dist/direct/preImageTargets.js +63 -0
- package/dist/direct/refVerify.js +191 -0
- package/dist/direct/registry.js +129 -0
- package/dist/direct/removals.js +262 -0
- package/dist/direct/rollback.js +1513 -0
- package/dist/direct/rollbackRun.js +1360 -0
- package/dist/direct/sessionProjection.js +126 -0
- package/dist/direct/snapshotComparability.js +165 -0
- package/dist/direct/sourceTree.js +113 -0
- package/dist/direct/unstatedChanges.js +106 -0
- package/dist/direct/waveAssembly.js +556 -0
- package/dist/direct/waveCounts.js +19 -0
- package/dist/direct/waveDerivation.js +152 -0
- package/dist/direct/waveGrade.js +185 -0
- package/dist/index.js +90 -0
- package/dist/orgread/customFieldSnapshot.js +48 -0
- package/dist/orgread/fieldMetadata.js +178 -0
- package/dist/orgread/genericSnapshot.js +61 -0
- package/dist/orgread/layoutSnapshot.js +151 -0
- package/dist/orgread/salesforceOrgReader.js +184 -0
- package/dist/orgread/snapshotTypes.js +9 -0
- package/dist/projection/bridge.js +26 -0
- package/dist/projection/diff.js +141 -0
- package/dist/projection/mapmd.js +213 -0
- package/dist/projection/nameAmbiguity.js +106 -0
- package/dist/projection/precedent.js +136 -0
- package/dist/projection/project.js +214 -0
- package/dist/projection/scope.js +110 -0
- package/dist/projection/types.js +5 -0
- package/dist/tools/analyze.js +322 -0
- package/dist/tools/auth.js +915 -0
- package/dist/tools/campfire.js +1742 -0
- package/dist/tools/closeBatch.js +154 -0
- package/dist/tools/closeText.js +35 -0
- package/dist/tools/direct.js +1613 -0
- package/dist/tools/feedback.js +318 -0
- package/dist/tools/operator.js +287 -0
- package/dist/tools/org.js +739 -0
- package/dist/tools/phase.js +249 -0
- package/dist/tools/projection.js +172 -0
- package/dist/tools/session.js +313 -0
- package/dist/tools/update.js +82 -0
- package/dist/tools/waves.js +793 -0
- package/package.json +48 -0
- package/reference/campfire-configuration-items-guide.md +710 -0
- package/reference/deployability-cribs.md +119 -0
- package/reference/operator-voice.md +141 -0
- package/reference/org-read-cribs.md +34 -0
- package/reference/scoped-dml-interim.md +68 -0
- package/reference/supervisor-playbook.md +313 -0
- package/reference/templates.md +233 -0
- package/skills/in2-delivery/SKILL.md +322 -0
- package/skills/in2-delivery/references/campfire-configuration-items-guide.md +710 -0
- package/skills/in2-delivery/references/deployability-cribs.md +119 -0
- package/skills/in2-delivery/references/operator-voice.md +141 -0
- package/skills/in2-delivery/references/org-read-cribs.md +34 -0
- package/skills/in2-delivery/references/scoped-dml-interim.md +68 -0
- package/skills/in2-delivery/references/templates.md +233 -0
- package/skills/plan-requirement/SKILL.md +188 -0
- package/viewer/index.html +899 -0
- package/viewer/server.mjs +433 -0
|
@@ -0,0 +1,1235 @@
|
|
|
1
|
+
import { promises as fs } from "node:fs";
|
|
2
|
+
import { chmodSync, mkdirSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { randomBytes } from "node:crypto";
|
|
6
|
+
import { gunzipSync, gzipSync } from "node:zlib";
|
|
7
|
+
import { REPO_ROOT } from "./paths.js";
|
|
8
|
+
import { log as defaultLog } from "./result.js";
|
|
9
|
+
import { TelemetryGrantStore } from "./telemetrySpool.js";
|
|
10
|
+
import { currentBatchContext } from "./telemetryContext.js";
|
|
11
|
+
// PROVENANCE PARCELS (plan 2026-08-02-002 U6; origin 2026-08-02-001 R7).
|
|
12
|
+
//
|
|
13
|
+
// Each state-changing wave's evidence bundle plus the session's read segment, gzipped into one tar
|
|
14
|
+
// and posted under the batch grant the transcript uploader already uses. The point is aggregation:
|
|
15
|
+
// a wave's provenance is useless if it only ever exists on the consultant laptop that produced it,
|
|
16
|
+
// and R24's cross-machine convergence needs somewhere for a foreign bundle to come FROM.
|
|
17
|
+
//
|
|
18
|
+
// WHY THIS NEEDED ITS OWN DISCLOSURE, and why the default is still on.
|
|
19
|
+
//
|
|
20
|
+
// Most of a parcel restructures content that already leaves the machine: transcripts are on by
|
|
21
|
+
// default and carry every metadata body the model read. `pre/source/` and the pre-image JSON are
|
|
22
|
+
// the exception. They are pre-existing org state retrieved TOOL-SIDE, which the model never saw,
|
|
23
|
+
// so that slice is genuinely new content leaving the machine and no prior consent covers it. The
|
|
24
|
+
// resolution (operator, 2026-08-02) was not to weaken the default but to correct the disclosure:
|
|
25
|
+
// the install prompt now names org state retrieved by the tools, `announceParcelUpload` says the
|
|
26
|
+
// same thing at startup, and `CC_MCP_PARCEL_UPLOAD=0` is the way out. A default the consent text
|
|
27
|
+
// does not mention is the thing to avoid; a default it does mention is a product decision.
|
|
28
|
+
//
|
|
29
|
+
// FAIL-OPEN IS LOAD-BEARING, in the same sense as the org mirror (direct/orgRepo.ts): an upload
|
|
30
|
+
// that cannot happen must never fail, delay, or mask a delivery that landed in the org. Every
|
|
31
|
+
// entry point here returns a result instead of throwing, the apply loop only ever awaits the local
|
|
32
|
+
// build-and-spool step, and the network flush is fire-and-forget. Nothing new reaches the Convex
|
|
33
|
+
// event tables — a parcel is a blob and a row, exactly like a transcript.
|
|
34
|
+
/** Bumped when the parcel LAYOUT changes; readers key on it before trusting the entry paths. */
|
|
35
|
+
export const PARCEL_FORMAT_VERSION = 1;
|
|
36
|
+
/**
|
|
37
|
+
* THE CLIENT HALF OF THE WIRE CONTRACT. The server half landed in U7 —
|
|
38
|
+
* `apps/campfire-launch/convex/parcels.ts` and `src/routes/parcelV1.ts`.
|
|
39
|
+
*
|
|
40
|
+
* `POST /api/parcel/v1`
|
|
41
|
+
* authorization: Bearer <batch grant> the sole authority for batchId/installId/sessionId
|
|
42
|
+
* content-type: application/gzip body is the gzipped tar
|
|
43
|
+
* x-in2-parcel-wave: wave label REQUIRED — a 400 ahead of the body without it
|
|
44
|
+
*
|
|
45
|
+
* That is the whole header set, and the omissions are the interesting part. THE CLIENT DOES NOT
|
|
46
|
+
* STATE THE ORG. A parcel is fetchable by another machine, so `orgId` is a permission boundary
|
|
47
|
+
* rather than a label, and the ledger resolves it from `batches.targetOrgId` on the grant's own
|
|
48
|
+
* batch — an org header would be the client asserting an authority it does not have, and the
|
|
49
|
+
* forwarder drops one if sent. Session id comes from the grant for the same reason. Attempt and
|
|
50
|
+
* verdict travel INSIDE the tar in `parcel.json`, which is where a rebuild reads them anyway.
|
|
51
|
+
*
|
|
52
|
+
* 202 stored — `{ stored, bytes, wave, orgId }`
|
|
53
|
+
* 401 grant not valid — RETRIED (it may simply not have been redeemed on this host yet)
|
|
54
|
+
* 5xx / network — RETRIED
|
|
55
|
+
* other 4xx (400 malformed, 409 batch bound no org, 413 over the 20 MB cap) — permanent; dropped
|
|
56
|
+
* from the spool with a logged reason rather than retried forever
|
|
57
|
+
*
|
|
58
|
+
* The launch service relays an explicit header allowlist, so anything not named above never
|
|
59
|
+
* reaches the ledger no matter what this client sends.
|
|
60
|
+
*
|
|
61
|
+
* THE READ HALF (U8; R24), on the same two routes:
|
|
62
|
+
*
|
|
63
|
+
* `GET /api/parcel/v1?limit=&cursor=` → 200 `{ orgId, parcels[], cursor, isDone }`
|
|
64
|
+
* `GET /api/parcel/v1/blob?parcelId=` → 200 `{ …metadata, url }` — a short-lived signed URL
|
|
65
|
+
*
|
|
66
|
+
* Same single header, `authorization: Bearer <batch grant>`, and again NO ORG ON THE WAY OUT. The
|
|
67
|
+
* org is a permission boundary the ledger resolves from `batches.targetOrgId`, and the launch
|
|
68
|
+
* forwarder deliberately refuses to pass a scope parameter along, so a client cannot ask for
|
|
69
|
+
* somebody else's history by naming it. The org therefore arrives in the RESPONSE, which makes
|
|
70
|
+
* checking it this client's job: a grant bound to another org is skipped, never united in.
|
|
71
|
+
*
|
|
72
|
+
* 401 grant not valid · 404 no parcel for this grant (also: wrong org, malformed id) · 409 the
|
|
73
|
+
* batch bound no org.
|
|
74
|
+
*/
|
|
75
|
+
export const PARCEL_ROUTE = "/api/parcel/v1";
|
|
76
|
+
const DISABLED_VALUES = new Set(["0", "false", "off", "no"]);
|
|
77
|
+
/**
|
|
78
|
+
* ON BY DEFAULT. `CC_MCP_PARCEL_UPLOAD=0` is the way out.
|
|
79
|
+
*
|
|
80
|
+
* Spellings are deliberately generous, for the same reason the transcript switch is: an operator
|
|
81
|
+
* who reaches for `false` and is still uploading org state would be the worst outcome here.
|
|
82
|
+
*/
|
|
83
|
+
export function parcelUploadEnabled(env = process.env) {
|
|
84
|
+
const raw = env.CC_MCP_PARCEL_UPLOAD?.trim().toLowerCase();
|
|
85
|
+
return raw === undefined || raw === "" ? true : !DISABLED_VALUES.has(raw);
|
|
86
|
+
}
|
|
87
|
+
/** Derive the parcel endpoint from the telemetry one — same host, same grant, different route. */
|
|
88
|
+
export function parcelEndpointFrom(telemetryEndpoint) {
|
|
89
|
+
try {
|
|
90
|
+
const url = new URL(telemetryEndpoint);
|
|
91
|
+
url.pathname = PARCEL_ROUTE;
|
|
92
|
+
return url.toString();
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
return telemetryEndpoint.replace(/\/api\/telemetry\/v\d+$/, PARCEL_ROUTE);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
let announced = false;
|
|
99
|
+
/** Test seam only: the once-guard is process-wide by design. */
|
|
100
|
+
export function resetParcelAnnouncementForTests() {
|
|
101
|
+
announced = false;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Say what leaves, and how to stop it — ONCE per process.
|
|
105
|
+
*
|
|
106
|
+
* Called at startup and again from the queue path, so a machine that never reaches startup
|
|
107
|
+
* announcement still says it before the first parcel is built, and a machine that runs forty waves
|
|
108
|
+
* says it once rather than forty times. The opt-out gets a line of its own for the same reason the
|
|
109
|
+
* default does: an operator who set the variable should be able to confirm it took effect without
|
|
110
|
+
* reading source.
|
|
111
|
+
*/
|
|
112
|
+
export function announceParcelUpload(opts = {}) {
|
|
113
|
+
if (announced)
|
|
114
|
+
return;
|
|
115
|
+
announced = true;
|
|
116
|
+
// `defaultLog` adds the `[cc-mcp]` prefix, so no message here carries one — both callers (startup
|
|
117
|
+
// and the first wave's build) go through the same writer and neither double-stamps it.
|
|
118
|
+
const write = opts.log ?? defaultLog;
|
|
119
|
+
if (!parcelUploadEnabled(opts.env ?? process.env)) {
|
|
120
|
+
write("provenance parcels are NOT being uploaded (CC_MCP_PARCEL_UPLOAD=0). Evidence bundles stay on this machine; org history and delivery are unaffected.");
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
write("PROVENANCE PARCEL UPLOAD IS ON (the default). For every wave that changes the org, the " +
|
|
124
|
+
"evidence bundle — including the BEFORE state of each touched component, retrieved from the org by " +
|
|
125
|
+
`these tools rather than read in the chat — and this session's read log are sent to ${opts.endpoint ?? "the configured endpoint"}. ` +
|
|
126
|
+
"Set CC_MCP_PARCEL_UPLOAD=0 to stop this.");
|
|
127
|
+
}
|
|
128
|
+
// ---------------------------------------------------------------------------------------------
|
|
129
|
+
// tar
|
|
130
|
+
// ---------------------------------------------------------------------------------------------
|
|
131
|
+
//
|
|
132
|
+
// Written here rather than taken as a dependency. A parcel is an evidence artifact that has to be
|
|
133
|
+
// openable by anyone with a shell years from now, so the format is plain POSIX ustar with a GNU
|
|
134
|
+
// long-name fallback — and writing 60 lines of header beats adding a package to the install every
|
|
135
|
+
// consultant downloads. `update.ts` already shells out to the system `tar` to EXTRACT; creating is
|
|
136
|
+
// deliberately in-process because bsdtar and GNU tar disagree about defaults and a parcel that
|
|
137
|
+
// depends on which tar the laptop shipped with is not a stable artifact.
|
|
138
|
+
const BLOCK = 512;
|
|
139
|
+
/** Octal, NUL-terminated, in the fixed-width field tar expects. */
|
|
140
|
+
function octalField(value, width) {
|
|
141
|
+
return `${value.toString(8).padStart(width - 1, "0")}\0`;
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Split a long path across ustar's `prefix` and `name` fields.
|
|
145
|
+
*
|
|
146
|
+
* Smallest prefix that leaves a name of at most 100 bytes, so the prefix has the best chance of
|
|
147
|
+
* fitting its own 155. Null when no split works — the caller falls back to a GNU long-name entry.
|
|
148
|
+
*/
|
|
149
|
+
function splitUstarName(name) {
|
|
150
|
+
for (let i = 0; i < name.length; i++) {
|
|
151
|
+
if (name[i] !== "/")
|
|
152
|
+
continue;
|
|
153
|
+
const prefix = name.slice(0, i);
|
|
154
|
+
const rest = name.slice(i + 1);
|
|
155
|
+
if (Buffer.byteLength(rest) <= 100 && Buffer.byteLength(prefix) <= 155)
|
|
156
|
+
return { prefix, name: rest };
|
|
157
|
+
}
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
function ustarHeader(fields) {
|
|
161
|
+
const header = Buffer.alloc(BLOCK, 0);
|
|
162
|
+
header.write(fields.name.slice(0, 100), 0, 100, "utf8");
|
|
163
|
+
header.write(octalField(0o644, 8), 100, 8, "ascii");
|
|
164
|
+
header.write(octalField(0, 8), 108, 8, "ascii"); // uid
|
|
165
|
+
header.write(octalField(0, 8), 116, 8, "ascii"); // gid
|
|
166
|
+
header.write(octalField(fields.size, 12), 124, 12, "ascii");
|
|
167
|
+
// mtime 0. Nothing downstream reads it, and a real mtime would make two builds of one bundle
|
|
168
|
+
// differ in bytes — determinism is cheap here and the local clock is not evidence.
|
|
169
|
+
header.write(octalField(0, 12), 136, 12, "ascii");
|
|
170
|
+
header.write(" ", 148, 8, "ascii"); // checksum is computed over this field as spaces
|
|
171
|
+
header.write(fields.typeflag, 156, 1, "ascii");
|
|
172
|
+
header.write("ustar\0", 257, 6, "ascii");
|
|
173
|
+
header.write("00", 263, 2, "ascii");
|
|
174
|
+
header.write(fields.prefix.slice(0, 155), 345, 155, "utf8");
|
|
175
|
+
let sum = 0;
|
|
176
|
+
for (let i = 0; i < BLOCK; i++)
|
|
177
|
+
sum += header[i];
|
|
178
|
+
header.write(`${sum.toString(8).padStart(6, "0")}\0 `, 148, 8, "ascii");
|
|
179
|
+
return header;
|
|
180
|
+
}
|
|
181
|
+
function padToBlock(content) {
|
|
182
|
+
const remainder = content.length % BLOCK;
|
|
183
|
+
return remainder === 0 ? content : Buffer.concat([content, Buffer.alloc(BLOCK - remainder, 0)]);
|
|
184
|
+
}
|
|
185
|
+
function tarEntry(name, content) {
|
|
186
|
+
const parts = [];
|
|
187
|
+
let headerName = name;
|
|
188
|
+
let prefix = "";
|
|
189
|
+
if (Buffer.byteLength(name) > 100) {
|
|
190
|
+
const split = splitUstarName(name);
|
|
191
|
+
if (split) {
|
|
192
|
+
headerName = split.name;
|
|
193
|
+
prefix = split.prefix;
|
|
194
|
+
}
|
|
195
|
+
else {
|
|
196
|
+
// GNU long name: the real path rides in its own entry ahead of the file. Read by GNU tar,
|
|
197
|
+
// bsdtar, Python's tarfile and node-tar; the truncated name in the following header is the
|
|
198
|
+
// documented placeholder those readers ignore.
|
|
199
|
+
const nameBytes = Buffer.from(`${name}\0`, "utf8");
|
|
200
|
+
parts.push(ustarHeader({ name: "././@LongLink", size: nameBytes.length, typeflag: "L", prefix: "" }));
|
|
201
|
+
parts.push(padToBlock(nameBytes));
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
parts.push(ustarHeader({ name: headerName, size: content.length, typeflag: "0", prefix }));
|
|
205
|
+
parts.push(padToBlock(content));
|
|
206
|
+
return parts;
|
|
207
|
+
}
|
|
208
|
+
/** Entries in, one gzipped tar out. Sorted, so the same inputs always produce the same bytes. */
|
|
209
|
+
function tarGz(entries) {
|
|
210
|
+
const parts = [];
|
|
211
|
+
for (const entry of [...entries].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) {
|
|
212
|
+
parts.push(...tarEntry(entry.name, entry.content));
|
|
213
|
+
}
|
|
214
|
+
parts.push(Buffer.alloc(BLOCK * 2, 0)); // end-of-archive
|
|
215
|
+
let tar = Buffer.concat(parts);
|
|
216
|
+
const remainder = tar.length % (BLOCK * 20);
|
|
217
|
+
if (remainder !== 0)
|
|
218
|
+
tar = Buffer.concat([tar, Buffer.alloc(BLOCK * 20 - remainder, 0)]);
|
|
219
|
+
return { gz: gzipSync(tar, { level: 9 }), rawBytes: tar.length };
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* The reader for the writer above. In-process for the same reason, plus one this side owns alone:
|
|
223
|
+
* a parcel arrives OVER THE NETWORK from a machine we do not control, so every entry path has to be
|
|
224
|
+
* refused before it is joined to anything (`safeParcelEntryPath`). Shelling out to the system `tar`
|
|
225
|
+
* would hand that decision to whichever tar the laptop shipped with — GNU tar strips a leading `/`
|
|
226
|
+
* and warns, bsdtar's `--strip-components` behaviour differs, and neither is a contract.
|
|
227
|
+
*
|
|
228
|
+
* Regular files only. A directory entry is implied by its members, and a symlink, device or hard
|
|
229
|
+
* link in an artifact that is supposed to be JSON and XML is not something to reproduce faithfully.
|
|
230
|
+
*/
|
|
231
|
+
function untar(tar) {
|
|
232
|
+
const out = [];
|
|
233
|
+
const str = (start, length) => {
|
|
234
|
+
const raw = tar.subarray(start, start + length);
|
|
235
|
+
const nul = raw.indexOf(0);
|
|
236
|
+
return raw.subarray(0, nul === -1 ? raw.length : nul).toString("utf8");
|
|
237
|
+
};
|
|
238
|
+
let offset = 0;
|
|
239
|
+
// A GNU long name rides in its own entry AHEAD of the file it names — see `tarEntry`.
|
|
240
|
+
let longName = null;
|
|
241
|
+
while (offset + BLOCK <= tar.length) {
|
|
242
|
+
const header = tar.subarray(offset, offset + BLOCK);
|
|
243
|
+
if (header.every((byte) => byte === 0))
|
|
244
|
+
break; // end-of-archive
|
|
245
|
+
const name = str(offset, 100);
|
|
246
|
+
const size = Number.parseInt(str(offset + 124, 12).trim() || "0", 8);
|
|
247
|
+
const typeflag = header.subarray(156, 157).toString("ascii");
|
|
248
|
+
const prefix = str(offset + 345, 155);
|
|
249
|
+
offset += BLOCK;
|
|
250
|
+
if (!Number.isFinite(size) || size < 0 || offset + size > tar.length) {
|
|
251
|
+
throw new Error("parcel tar declares a member longer than the archive");
|
|
252
|
+
}
|
|
253
|
+
const body = tar.subarray(offset, offset + size);
|
|
254
|
+
offset += Math.ceil(size / BLOCK) * BLOCK;
|
|
255
|
+
if (typeflag === "L") {
|
|
256
|
+
longName = body.toString("utf8").replace(/\0+$/, "");
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
const full = longName ?? (prefix ? `${prefix}/${name}` : name);
|
|
260
|
+
longName = null;
|
|
261
|
+
if (typeflag === "0" || typeflag === "\0" || typeflag === "")
|
|
262
|
+
out.push({ name: full, content: Buffer.from(body) });
|
|
263
|
+
}
|
|
264
|
+
return out;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* A hard wall, not a policy. An oversized parcel still SHIPS (see `queueWaveParcel`); this exists
|
|
268
|
+
* only because the tar is assembled in memory and a pathological bundle should fail loudly rather
|
|
269
|
+
* than take the process down with it.
|
|
270
|
+
*/
|
|
271
|
+
const MAX_UNCOMPRESSED_BYTES = 256 * 1024 * 1024;
|
|
272
|
+
/** Report a compressed parcel past this in the log. Ships regardless — retention wants the data. */
|
|
273
|
+
const DEFAULT_OVERSIZE_BYTES = 8 * 1024 * 1024;
|
|
274
|
+
async function collectFiles(dir, prefix) {
|
|
275
|
+
const out = [];
|
|
276
|
+
const entries = await fs.readdir(dir, { withFileTypes: true });
|
|
277
|
+
for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
278
|
+
const full = path.join(dir, entry.name);
|
|
279
|
+
const name = `${prefix}/${entry.name}`;
|
|
280
|
+
// Symlinks are not followed: a bundle is written by this process, and a link that escaped it
|
|
281
|
+
// would put arbitrary local files into an artifact that leaves the machine.
|
|
282
|
+
if (entry.isDirectory())
|
|
283
|
+
out.push(...(await collectFiles(full, name)));
|
|
284
|
+
else if (entry.isFile())
|
|
285
|
+
out.push({ name, content: await fs.readFile(full) });
|
|
286
|
+
}
|
|
287
|
+
return out;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Two org ids name the same org iff their 15-char prefixes match — the 18-char form only adds a
|
|
291
|
+
* case-insensitivity checksum suffix. Same normalization this file already applies at the manifest
|
|
292
|
+
* check and the grant check below (and sessionOrg.ts's live-binding check); kept as one named
|
|
293
|
+
* function here so the reads filter's intent reads at the call site rather than as a bare slice.
|
|
294
|
+
*/
|
|
295
|
+
function sameOrgPrefix(a, b) {
|
|
296
|
+
return a.slice(0, 15) === b.slice(0, 15);
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Tar the wave's bundle and the session's read segment into one parcel.
|
|
300
|
+
*
|
|
301
|
+
* LAYOUT IS LOAD-BEARING. Entries sit at their real session-relative paths
|
|
302
|
+
* (`<sessionId>/direct/<wave>/…` and `<sessionId>/reads.jsonl`), because U8's rebuild reads a
|
|
303
|
+
* foreign parcel with the SAME `readWaveFromBundle` the live path uses, and that reader derives the
|
|
304
|
+
* session id and wave label from the bundle's own directory. Unpack a parcel over a workspace root
|
|
305
|
+
* and every path lands exactly where a locally-produced wave would have put it.
|
|
306
|
+
*/
|
|
307
|
+
export async function buildParcel(opts) {
|
|
308
|
+
const wave = opts.wave ?? path.basename(opts.bundleDir);
|
|
309
|
+
const sessionId = opts.sessionId ?? path.basename(path.resolve(opts.bundleDir, "..", ".."));
|
|
310
|
+
const readsFile = opts.readsFile ?? path.resolve(opts.bundleDir, "..", "..", "reads.jsonl");
|
|
311
|
+
const entries = await collectFiles(opts.bundleDir, `${sessionId}/direct/${wave}`);
|
|
312
|
+
let readsBytes = 0;
|
|
313
|
+
let readsDropped = 0;
|
|
314
|
+
try {
|
|
315
|
+
const raw = await fs.readFile(readsFile, "utf8");
|
|
316
|
+
// THE PARCEL IS FETCHABLE UNDER ONE ORG'S GRANT, so the read segment it carries is filtered to
|
|
317
|
+
// that org. The session log on disk keeps every record — this filters the export, not the
|
|
318
|
+
// history. A record naming another org (a targetOrg peek) is org content under R14's doctrine
|
|
319
|
+
// and does not travel; an unparseable line cannot be attributed and does not travel either.
|
|
320
|
+
const kept = [];
|
|
321
|
+
for (const line of raw.split("\n")) {
|
|
322
|
+
if (line.trim() === "")
|
|
323
|
+
continue;
|
|
324
|
+
let record;
|
|
325
|
+
try {
|
|
326
|
+
record = JSON.parse(line);
|
|
327
|
+
}
|
|
328
|
+
catch {
|
|
329
|
+
readsDropped += 1;
|
|
330
|
+
continue;
|
|
331
|
+
}
|
|
332
|
+
// A record's orgId comes from the jsforce reader's AuthInfo; opts.orgId comes from the sf CLI's
|
|
333
|
+
// `org display` (see auth.ts:534-538) — the same org, but the two can hand back the 15- vs
|
|
334
|
+
// 18-char form of the same id. Every other org-identity comparison in this file normalizes to
|
|
335
|
+
// the 15-char prefix (parcels.ts's own manifest check and grant check, sessionOrg.ts) before
|
|
336
|
+
// comparing; a bare `===` here would drop a legitimate same-org record as foreign. Two DIFFERENT
|
|
337
|
+
// orgs never share a 15-char prefix, so normalizing cannot reopen the leak this filter closes.
|
|
338
|
+
const belongs = (typeof record.orgId === "string" && sameOrgPrefix(record.orgId, opts.orgId)) ||
|
|
339
|
+
(record.orgId == null && (opts.orgAlias === undefined || record.orgAlias === opts.orgAlias));
|
|
340
|
+
if (belongs)
|
|
341
|
+
kept.push(line);
|
|
342
|
+
else
|
|
343
|
+
readsDropped += 1;
|
|
344
|
+
}
|
|
345
|
+
if (kept.length > 0) {
|
|
346
|
+
const content = Buffer.from(`${kept.join("\n")}\n`, "utf8");
|
|
347
|
+
readsBytes = content.length;
|
|
348
|
+
entries.push({ name: `${sessionId}/reads.jsonl`, content });
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
catch {
|
|
352
|
+
// A session that never read the org has no read log. Absent is the honest record; an empty file
|
|
353
|
+
// would claim a segment exists and say nothing.
|
|
354
|
+
}
|
|
355
|
+
const identity = {
|
|
356
|
+
orgId: opts.orgId,
|
|
357
|
+
sessionId,
|
|
358
|
+
wave,
|
|
359
|
+
attempt: opts.attempt ?? 1,
|
|
360
|
+
verdict: opts.verdict,
|
|
361
|
+
...(opts.batchId ? { batchId: opts.batchId } : {})
|
|
362
|
+
};
|
|
363
|
+
entries.push({
|
|
364
|
+
name: "parcel.json",
|
|
365
|
+
content: Buffer.from(`${JSON.stringify({
|
|
366
|
+
parcelVersion: PARCEL_FORMAT_VERSION,
|
|
367
|
+
...identity,
|
|
368
|
+
builtAt: opts.at ?? new Date().toISOString(),
|
|
369
|
+
bundleFiles: entries.length - (readsBytes > 0 ? 1 : 0),
|
|
370
|
+
readsBytes,
|
|
371
|
+
...(readsDropped > 0 ? { readsDroppedLines: readsDropped } : {})
|
|
372
|
+
}, null, 1)}\n`, "utf8")
|
|
373
|
+
});
|
|
374
|
+
const uncompressed = entries.reduce((sum, entry) => sum + entry.content.length, 0);
|
|
375
|
+
if (uncompressed > MAX_UNCOMPRESSED_BYTES) {
|
|
376
|
+
throw new Error(`bundle is ${Math.round(uncompressed / 1024 / 1024)} MB uncompressed, past the ${Math.round(MAX_UNCOMPRESSED_BYTES / 1024 / 1024)} MB assembly limit — it stays on this machine at ${opts.bundleDir}`);
|
|
377
|
+
}
|
|
378
|
+
const { gz, rawBytes } = tarGz(entries);
|
|
379
|
+
const outDir = opts.outDir ?? os.tmpdir();
|
|
380
|
+
await fs.mkdir(outDir, { recursive: true });
|
|
381
|
+
const file = path.join(outDir, `in2-parcel-${sessionId}-${wave}-${identity.attempt}-${randomBytes(4).toString("hex")}.tgz`);
|
|
382
|
+
await fs.writeFile(file, gz, { mode: 0o600 });
|
|
383
|
+
return { path: file, bytes: gz.length, rawBytes, identity };
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* A durable outbox, shaped like `TelemetrySpool` — a bounded directory that survives a crash and
|
|
387
|
+
* replays on the next flush.
|
|
388
|
+
*
|
|
389
|
+
* It holds THE BUILT BLOB rather than a pointer back at the bundle, which is the one place it
|
|
390
|
+
* departs from the telemetry pattern and does so deliberately: a retry re-uses the same bundle
|
|
391
|
+
* directory (see applyLoop's `deployed/` clear), so a descriptor resolved later would tar attempt
|
|
392
|
+
* two's content under attempt one's identity and no reader could tell.
|
|
393
|
+
*/
|
|
394
|
+
export class ParcelSpool {
|
|
395
|
+
options;
|
|
396
|
+
maxBytes;
|
|
397
|
+
maxAgeMs;
|
|
398
|
+
dropped = 0;
|
|
399
|
+
constructor(options) {
|
|
400
|
+
this.options = options;
|
|
401
|
+
this.maxBytes = options.maxBytes ?? 64 * 1024 * 1024;
|
|
402
|
+
this.maxAgeMs = options.maxAgeMs ?? 30 * 86_400_000;
|
|
403
|
+
mkdirSync(options.dir, { recursive: true, mode: 0o700 });
|
|
404
|
+
try {
|
|
405
|
+
// mkdir's mode is subject to umask; the outbox holds org content, so say it explicitly.
|
|
406
|
+
chmodSync(options.dir, 0o700);
|
|
407
|
+
}
|
|
408
|
+
catch {
|
|
409
|
+
// Best effort on filesystems without POSIX modes.
|
|
410
|
+
}
|
|
411
|
+
// Sweep on open, not only on write: a run that queued parcels and died leaves them here, and an
|
|
412
|
+
// install that never delivers again should not keep them past the age cap forever.
|
|
413
|
+
this.enforceCaps();
|
|
414
|
+
}
|
|
415
|
+
/** Move a built parcel into the outbox. Returns null when it could not be spooled. */
|
|
416
|
+
enqueue(built, now = new Date()) {
|
|
417
|
+
const id = `pcl_${randomBytes(8).toString("hex")}`;
|
|
418
|
+
const blob = path.join(this.options.dir, `${id}.tgz`);
|
|
419
|
+
try {
|
|
420
|
+
try {
|
|
421
|
+
renameSync(built.path, blob);
|
|
422
|
+
}
|
|
423
|
+
catch {
|
|
424
|
+
// Cross-device (a tmpdir on another filesystem): copy, then drop the original.
|
|
425
|
+
writeFileSync(blob, readFileSync(built.path), { mode: 0o600 });
|
|
426
|
+
try {
|
|
427
|
+
unlinkSync(built.path);
|
|
428
|
+
}
|
|
429
|
+
catch {
|
|
430
|
+
// Temp cleanup is best effort.
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
const entry = {
|
|
434
|
+
...built.identity,
|
|
435
|
+
id,
|
|
436
|
+
file: blob,
|
|
437
|
+
bytes: built.bytes,
|
|
438
|
+
rawBytes: built.rawBytes,
|
|
439
|
+
queuedAt: now.toISOString(),
|
|
440
|
+
attempts: 0
|
|
441
|
+
};
|
|
442
|
+
writeFileSync(path.join(this.options.dir, `${id}.json`), `${JSON.stringify(entry)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
443
|
+
this.enforceCaps();
|
|
444
|
+
// Null when the caps retired it on the way in. Saying "queued" for a parcel the outbox just
|
|
445
|
+
// discarded is the one answer that would make the drop invisible.
|
|
446
|
+
return this.descriptor(id);
|
|
447
|
+
}
|
|
448
|
+
catch {
|
|
449
|
+
return null;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
pending() {
|
|
453
|
+
const out = [];
|
|
454
|
+
for (const name of this.descriptorFiles()) {
|
|
455
|
+
const entry = this.descriptor(name.replace(/\.json$/, ""));
|
|
456
|
+
if (entry)
|
|
457
|
+
out.push(entry);
|
|
458
|
+
}
|
|
459
|
+
return out.sort((a, b) => (a.queuedAt < b.queuedAt ? -1 : a.queuedAt > b.queuedAt ? 1 : 0));
|
|
460
|
+
}
|
|
461
|
+
/** Delivered (or permanently rejected) — retire the descriptor and its blob together. */
|
|
462
|
+
acknowledge(id, lost = 0) {
|
|
463
|
+
this.dropped += lost;
|
|
464
|
+
this.remove(id);
|
|
465
|
+
}
|
|
466
|
+
/** Record a failed attempt so a poisoned parcel is visible rather than silently looping. */
|
|
467
|
+
markAttempt(id) {
|
|
468
|
+
const entry = this.descriptor(id);
|
|
469
|
+
if (!entry)
|
|
470
|
+
return;
|
|
471
|
+
try {
|
|
472
|
+
writeFileSync(path.join(this.options.dir, `${id}.json`), `${JSON.stringify({ ...entry, attempts: entry.attempts + 1 })}\n`, {
|
|
473
|
+
encoding: "utf8",
|
|
474
|
+
mode: 0o600
|
|
475
|
+
});
|
|
476
|
+
}
|
|
477
|
+
catch {
|
|
478
|
+
// A lost attempt count only costs a log line's accuracy.
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
stats() {
|
|
482
|
+
const entries = this.pending();
|
|
483
|
+
let bytes = 0;
|
|
484
|
+
for (const entry of entries)
|
|
485
|
+
bytes += entry.bytes;
|
|
486
|
+
const oldest = entries[0];
|
|
487
|
+
return {
|
|
488
|
+
pendingParcels: entries.length,
|
|
489
|
+
bytes,
|
|
490
|
+
oldestAgeMs: oldest ? Math.max(0, Date.now() - Date.parse(oldest.queuedAt)) : 0,
|
|
491
|
+
droppedParcels: this.dropped
|
|
492
|
+
};
|
|
493
|
+
}
|
|
494
|
+
descriptorFiles() {
|
|
495
|
+
try {
|
|
496
|
+
return readdirSync(this.options.dir)
|
|
497
|
+
.filter((name) => name.endsWith(".json"))
|
|
498
|
+
.sort();
|
|
499
|
+
}
|
|
500
|
+
catch {
|
|
501
|
+
return [];
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
descriptor(id) {
|
|
505
|
+
try {
|
|
506
|
+
const entry = JSON.parse(readFileSync(path.join(this.options.dir, `${id}.json`), "utf8"));
|
|
507
|
+
return entry.id === id && typeof entry.file === "string" ? entry : null;
|
|
508
|
+
}
|
|
509
|
+
catch {
|
|
510
|
+
return null;
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
remove(id) {
|
|
514
|
+
for (const file of [path.join(this.options.dir, `${id}.json`), path.join(this.options.dir, `${id}.tgz`)]) {
|
|
515
|
+
try {
|
|
516
|
+
unlinkSync(file);
|
|
517
|
+
}
|
|
518
|
+
catch {
|
|
519
|
+
// Already gone.
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
/**
|
|
524
|
+
* Bound the outbox by bytes and age, oldest first.
|
|
525
|
+
*
|
|
526
|
+
* A parcel dropped here is evidence that never shipped, so it is COUNTED: `stats()` carries the
|
|
527
|
+
* number, `flushParcels` reports it, and the exit hook prints it. A cap that silently ate the
|
|
528
|
+
* backlog would make a clean outbox and an incomplete history look the same from outside.
|
|
529
|
+
*/
|
|
530
|
+
enforceCaps() {
|
|
531
|
+
const now = Date.now();
|
|
532
|
+
const entries = this.pending();
|
|
533
|
+
let total = entries.reduce((sum, entry) => sum + entry.bytes, 0);
|
|
534
|
+
for (const entry of entries) {
|
|
535
|
+
if (now - Date.parse(entry.queuedAt) <= this.maxAgeMs && total <= this.maxBytes)
|
|
536
|
+
continue;
|
|
537
|
+
this.remove(entry.id);
|
|
538
|
+
total -= entry.bytes;
|
|
539
|
+
this.dropped += 1;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
/** Ship one parcel under the batch grant. Never throws; every failure is a described result. */
|
|
544
|
+
export async function uploadParcel(opts) {
|
|
545
|
+
let body;
|
|
546
|
+
try {
|
|
547
|
+
body = await fs.readFile(opts.parcel.file);
|
|
548
|
+
}
|
|
549
|
+
catch (error) {
|
|
550
|
+
// The blob is gone — nothing can be retried, so say so instead of holding a descriptor that
|
|
551
|
+
// will fail identically forever.
|
|
552
|
+
return { uploaded: false, permanent: true, reason: `parcel blob unreadable: ${error instanceof Error ? error.message : String(error)}` };
|
|
553
|
+
}
|
|
554
|
+
try {
|
|
555
|
+
const res = await (opts.fetchFn ?? fetch)(opts.endpoint, {
|
|
556
|
+
method: "POST",
|
|
557
|
+
headers: {
|
|
558
|
+
"content-type": "application/gzip",
|
|
559
|
+
authorization: `Bearer ${opts.credential}`,
|
|
560
|
+
// The only descriptive header the ledger takes. Everything else about this parcel's
|
|
561
|
+
// identity is either derived from the grant (batch, install, session) or inside the tar
|
|
562
|
+
// (org, attempt, verdict) — see the contract note at the top of this file.
|
|
563
|
+
"x-in2-parcel-wave": opts.parcel.wave
|
|
564
|
+
},
|
|
565
|
+
body,
|
|
566
|
+
signal: AbortSignal.timeout(opts.timeoutMs ?? 120_000)
|
|
567
|
+
});
|
|
568
|
+
if (res.ok) {
|
|
569
|
+
const parsed = (await res.json().catch(() => ({})));
|
|
570
|
+
return { uploaded: true, status: res.status, storedBytes: parsed.bytes ?? body.length };
|
|
571
|
+
}
|
|
572
|
+
// 401 is retried on purpose: the grant may not have been redeemed on this host yet, and an
|
|
573
|
+
// expired one is bounded by the spool's own caps rather than by discarding evidence early.
|
|
574
|
+
const permanent = res.status >= 400 && res.status < 500 && res.status !== 401 && res.status !== 408 && res.status !== 429;
|
|
575
|
+
return { uploaded: false, status: res.status, permanent, reason: `HTTP ${res.status}` };
|
|
576
|
+
}
|
|
577
|
+
catch (error) {
|
|
578
|
+
return { uploaded: false, reason: error instanceof Error ? error.message : String(error) };
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
/**
|
|
582
|
+
* The grant for a batch, from the durable store first.
|
|
583
|
+
*
|
|
584
|
+
* TWO RULES, both about attribution rather than convenience.
|
|
585
|
+
*
|
|
586
|
+
* A parcel names the batch it belongs to and resolves THAT batch's grant — never whichever batch
|
|
587
|
+
* happens to be bound right now. A spooled parcel outlives the run that made it, so the loose
|
|
588
|
+
* version of this would let a wave from one engagement upload under another's credential and land
|
|
589
|
+
* in the ledger attributed to it.
|
|
590
|
+
*
|
|
591
|
+
* A parcel with no batch (a development wave against a session that never activated) resolves
|
|
592
|
+
* nothing and stays in the outbox until the caps retire it. There is no batch to attribute it to,
|
|
593
|
+
* and the alternative — borrowing the current one — is exactly the mis-attribution above.
|
|
594
|
+
*/
|
|
595
|
+
export function defaultResolveGrant(batchId) {
|
|
596
|
+
if (!batchId)
|
|
597
|
+
return null;
|
|
598
|
+
try {
|
|
599
|
+
const stored = new TelemetryGrantStore(path.join(REPO_ROOT, ".local", "cc-mcp", "telemetry-grants")).get(batchId);
|
|
600
|
+
if (stored?.credential && stored.endpoint) {
|
|
601
|
+
return { endpoint: parcelEndpointFrom(stored.endpoint), credential: stored.credential };
|
|
602
|
+
}
|
|
603
|
+
// The live context covers the window before a grant reaches disk — same batch only.
|
|
604
|
+
const live = currentBatchContext();
|
|
605
|
+
if (live?.credential && live.endpoint && live.batchId === batchId) {
|
|
606
|
+
return { endpoint: parcelEndpointFrom(live.endpoint), credential: live.credential };
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
catch {
|
|
610
|
+
// A grant store that cannot be read is a reason to retry later, never to fail a caller.
|
|
611
|
+
}
|
|
612
|
+
return null;
|
|
613
|
+
}
|
|
614
|
+
let sharedSpool = null;
|
|
615
|
+
/** The process-wide outbox, beside the telemetry one. Null only when the disk refuses it. */
|
|
616
|
+
export function defaultParcelSpool() {
|
|
617
|
+
if (sharedSpool)
|
|
618
|
+
return sharedSpool;
|
|
619
|
+
try {
|
|
620
|
+
sharedSpool = new ParcelSpool({
|
|
621
|
+
dir: process.env.CC_MCP_PARCEL_OUTBOX?.trim() || path.join(REPO_ROOT, ".local", "cc-mcp", "parcel-outbox")
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
catch {
|
|
625
|
+
sharedSpool = null;
|
|
626
|
+
}
|
|
627
|
+
return sharedSpool;
|
|
628
|
+
}
|
|
629
|
+
const emptySummary = () => ({ uploaded: 0, retained: 0, dropped: 0, outboxDropped: 0, oldestAgeMs: 0, bytesUploaded: 0 });
|
|
630
|
+
/**
|
|
631
|
+
* One flush at a time per outbox.
|
|
632
|
+
*
|
|
633
|
+
* Every wave kicks a fire-and-forget flush and the exit hook kicks another, so two can easily
|
|
634
|
+
* overlap on a fast batch — and both would read the same descriptor and upload the same parcel
|
|
635
|
+
* twice. Chained rather than skipped: a caller that arrives mid-flush waits and then runs its own
|
|
636
|
+
* pass, so a parcel queued after the first pass started is still picked up.
|
|
637
|
+
*/
|
|
638
|
+
const flushChain = new WeakMap();
|
|
639
|
+
/**
|
|
640
|
+
* Ship everything spooled that has a usable grant.
|
|
641
|
+
*
|
|
642
|
+
* Called after each wave and again before the process exits, which is what keeps a parcel from
|
|
643
|
+
* outliving the grant that is the only thing able to attribute it (plan risk: "grant lifetime vs.
|
|
644
|
+
* parcel flush"). A parcel whose grant has gone stays in the outbox and is reported — bounded on
|
|
645
|
+
* disk, never silently dropped.
|
|
646
|
+
*/
|
|
647
|
+
export async function flushParcels(opts = {}) {
|
|
648
|
+
const spool = opts.spool === undefined ? defaultParcelSpool() : opts.spool;
|
|
649
|
+
if (!spool)
|
|
650
|
+
return emptySummary();
|
|
651
|
+
const previous = flushChain.get(spool) ?? Promise.resolve(null);
|
|
652
|
+
const run = previous.catch(() => null).then(() => flushOnce(spool, opts));
|
|
653
|
+
flushChain.set(spool, run);
|
|
654
|
+
try {
|
|
655
|
+
return await run;
|
|
656
|
+
}
|
|
657
|
+
finally {
|
|
658
|
+
if (flushChain.get(spool) === run)
|
|
659
|
+
flushChain.delete(spool);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
async function flushOnce(spool, opts) {
|
|
663
|
+
const summary = emptySummary();
|
|
664
|
+
const write = opts.log ?? ((line) => defaultLog(line));
|
|
665
|
+
const resolve = opts.resolveGrant ?? defaultResolveGrant;
|
|
666
|
+
let pending;
|
|
667
|
+
try {
|
|
668
|
+
pending = spool.pending().slice(0, opts.limit ?? 50);
|
|
669
|
+
}
|
|
670
|
+
catch {
|
|
671
|
+
return summary;
|
|
672
|
+
}
|
|
673
|
+
for (const parcel of pending) {
|
|
674
|
+
let grant = null;
|
|
675
|
+
try {
|
|
676
|
+
grant = resolve(parcel.batchId);
|
|
677
|
+
}
|
|
678
|
+
catch {
|
|
679
|
+
grant = null;
|
|
680
|
+
}
|
|
681
|
+
if (!grant) {
|
|
682
|
+
summary.retained += 1;
|
|
683
|
+
continue;
|
|
684
|
+
}
|
|
685
|
+
const result = await uploadParcel({ endpoint: grant.endpoint, credential: grant.credential, parcel, fetchFn: opts.fetchFn });
|
|
686
|
+
if (result.uploaded) {
|
|
687
|
+
spool.acknowledge(parcel.id);
|
|
688
|
+
summary.uploaded += 1;
|
|
689
|
+
summary.bytesUploaded += result.storedBytes ?? parcel.bytes;
|
|
690
|
+
continue;
|
|
691
|
+
}
|
|
692
|
+
if (result.permanent) {
|
|
693
|
+
spool.acknowledge(parcel.id, 1);
|
|
694
|
+
summary.dropped += 1;
|
|
695
|
+
write(`parcel ${parcel.sessionId}/${parcel.wave} REJECTED and dropped from the outbox (${result.reason}) — ` +
|
|
696
|
+
`its evidence bundle is intact on this machine; org history and the delivery are unaffected.`);
|
|
697
|
+
continue;
|
|
698
|
+
}
|
|
699
|
+
spool.markAttempt(parcel.id);
|
|
700
|
+
summary.retained += 1;
|
|
701
|
+
}
|
|
702
|
+
const stats = spool.stats();
|
|
703
|
+
summary.outboxDropped = stats.droppedParcels;
|
|
704
|
+
summary.oldestAgeMs = stats.oldestAgeMs;
|
|
705
|
+
return summary;
|
|
706
|
+
}
|
|
707
|
+
/**
|
|
708
|
+
* Build one wave's parcel and put it in the outbox. NEVER THROWS.
|
|
709
|
+
*
|
|
710
|
+
* This is the only entry point the apply loop awaits, and it does local work only — tar, gzip,
|
|
711
|
+
* rename. The network is the flush's problem, so a slow or dead endpoint cannot delay a delivery
|
|
712
|
+
* that has already landed in the org.
|
|
713
|
+
*/
|
|
714
|
+
export async function queueWaveParcel(opts) {
|
|
715
|
+
const write = opts.log ?? ((line) => defaultLog(line));
|
|
716
|
+
try {
|
|
717
|
+
announceParcelUpload({ ...(opts.env ? { env: opts.env } : {}), log: write });
|
|
718
|
+
if (!parcelUploadEnabled(opts.env ?? process.env))
|
|
719
|
+
return { queued: false, suppressed: true };
|
|
720
|
+
const built = await buildParcel(opts);
|
|
721
|
+
const oversize = opts.oversizeBytes ?? DEFAULT_OVERSIZE_BYTES;
|
|
722
|
+
// Bytes are logged for EVERY parcel, not only large ones: retention is an open question in the
|
|
723
|
+
// plan and it needs measurements from the first wave onward, not an estimate later.
|
|
724
|
+
write(`parcel ${built.identity.sessionId}/${built.identity.wave} built: ${built.bytes} bytes compressed (${built.rawBytes} raw)` +
|
|
725
|
+
(built.bytes > oversize ? ` — LARGE. Shipping it anyway; the evidence is the point and the server keeps its own cap.` : ""));
|
|
726
|
+
const spool = opts.spool === undefined ? defaultParcelSpool() : opts.spool;
|
|
727
|
+
const entry = spool?.enqueue(built) ?? null;
|
|
728
|
+
if (!entry) {
|
|
729
|
+
// The build wrote a temp file; if it never made it into the outbox, take it back out. A
|
|
730
|
+
// stranded parcel in the system temp dir is org content nobody is tracking.
|
|
731
|
+
await fs.rm(built.path, { force: true }).catch(() => undefined);
|
|
732
|
+
return {
|
|
733
|
+
queued: false,
|
|
734
|
+
bytes: built.bytes,
|
|
735
|
+
rawBytes: built.rawBytes,
|
|
736
|
+
reason: spool ? "parcel could not be spooled" : "parcel outbox unavailable"
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
return { queued: true, bytes: built.bytes, rawBytes: built.rawBytes };
|
|
740
|
+
}
|
|
741
|
+
catch (error) {
|
|
742
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
743
|
+
write(`parcel NOT built for this wave (${reason}) — the delivery and its evidence bundle are unaffected.`);
|
|
744
|
+
return { queued: false, reason };
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
const READ_TIMEOUT_MS = 60_000;
|
|
748
|
+
/** Headroom over the server's own 20 MB upload cap, so a legitimate parcel is never refused here. */
|
|
749
|
+
const MAX_DOWNLOAD_BYTES = 64 * 1024 * 1024;
|
|
750
|
+
/** `…/api/parcel/v1` → `…/api/parcel/v1/blob`, query string and all left alone. */
|
|
751
|
+
function blobEndpointFrom(endpoint) {
|
|
752
|
+
const [base = endpoint] = endpoint.split("?");
|
|
753
|
+
return `${base.replace(/\/+$/, "")}/blob`;
|
|
754
|
+
}
|
|
755
|
+
function withQuery(endpoint, params) {
|
|
756
|
+
const pairs = Object.entries(params).filter(([, value]) => value !== undefined && value !== "");
|
|
757
|
+
if (pairs.length === 0)
|
|
758
|
+
return endpoint;
|
|
759
|
+
const query = pairs.map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`).join("&");
|
|
760
|
+
return `${endpoint}${endpoint.includes("?") ? "&" : "?"}${query}`;
|
|
761
|
+
}
|
|
762
|
+
async function getJson(opts) {
|
|
763
|
+
try {
|
|
764
|
+
const res = await (opts.fetchFn ?? fetch)(opts.url, {
|
|
765
|
+
method: "GET",
|
|
766
|
+
headers: { authorization: `Bearer ${opts.credential}`, accept: "application/json" },
|
|
767
|
+
signal: AbortSignal.timeout(opts.timeoutMs ?? READ_TIMEOUT_MS)
|
|
768
|
+
});
|
|
769
|
+
const text = await res.text();
|
|
770
|
+
if (!res.ok) {
|
|
771
|
+
// The ledger's own message when it has one — a 409 saying "batch has no bound org" is a
|
|
772
|
+
// materially different operator problem from a 401, and collapsing them would hide that.
|
|
773
|
+
let detail = "";
|
|
774
|
+
try {
|
|
775
|
+
detail = String(JSON.parse(text).error ?? "");
|
|
776
|
+
}
|
|
777
|
+
catch {
|
|
778
|
+
detail = text.trim().slice(0, 120);
|
|
779
|
+
}
|
|
780
|
+
return { ok: false, status: res.status, reason: `HTTP ${res.status}${detail ? `: ${detail}` : ""}` };
|
|
781
|
+
}
|
|
782
|
+
return { ok: true, value: JSON.parse(text) };
|
|
783
|
+
}
|
|
784
|
+
catch (error) {
|
|
785
|
+
return { ok: false, reason: error instanceof Error ? error.message : String(error) };
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
/** One page of the parcels this grant's org holds, newest first. */
|
|
789
|
+
export async function listParcels(opts) {
|
|
790
|
+
const url = withQuery(opts.endpoint, {
|
|
791
|
+
...(opts.limit ? { limit: String(opts.limit) } : {}),
|
|
792
|
+
...(opts.cursor ? { cursor: opts.cursor } : {})
|
|
793
|
+
});
|
|
794
|
+
const read = await getJson({ url, credential: opts.credential, ...(opts.fetchFn ? { fetchFn: opts.fetchFn } : {}), ...(opts.timeoutMs ? { timeoutMs: opts.timeoutMs } : {}) });
|
|
795
|
+
if (!read.ok)
|
|
796
|
+
return read;
|
|
797
|
+
const page = read.value;
|
|
798
|
+
// A response that does not name an org cannot be checked against the org being rebuilt, and an
|
|
799
|
+
// unchecked page is exactly how another customer's history would end up in this mirror.
|
|
800
|
+
if (typeof page?.orgId !== "string" || !Array.isArray(page?.parcels)) {
|
|
801
|
+
return { ok: false, reason: "parcel list response did not name an org and its parcels" };
|
|
802
|
+
}
|
|
803
|
+
return {
|
|
804
|
+
ok: true,
|
|
805
|
+
value: {
|
|
806
|
+
orgId: page.orgId,
|
|
807
|
+
parcels: page.parcels,
|
|
808
|
+
cursor: typeof page.cursor === "string" ? page.cursor : null,
|
|
809
|
+
isDone: page.isDone !== false
|
|
810
|
+
}
|
|
811
|
+
};
|
|
812
|
+
}
|
|
813
|
+
/** The signed URL for one parcel's blob, plus the metadata row that names it. */
|
|
814
|
+
export async function parcelDownloadUrl(opts) {
|
|
815
|
+
const url = withQuery(blobEndpointFrom(opts.endpoint), { parcelId: opts.parcelId });
|
|
816
|
+
const read = await getJson({ url, credential: opts.credential, ...(opts.fetchFn ? { fetchFn: opts.fetchFn } : {}), ...(opts.timeoutMs ? { timeoutMs: opts.timeoutMs } : {}) });
|
|
817
|
+
if (!read.ok)
|
|
818
|
+
return read;
|
|
819
|
+
const row = read.value;
|
|
820
|
+
if (typeof row?.url !== "string" || !row.url) {
|
|
821
|
+
// Convex returns a null url when the stored object is gone. A row without a blob is a row.
|
|
822
|
+
return { ok: false, reason: "the ledger has a row for this parcel but no blob behind it" };
|
|
823
|
+
}
|
|
824
|
+
return { ok: true, value: { ...row, url: row.url } };
|
|
825
|
+
}
|
|
826
|
+
/**
|
|
827
|
+
* The gzipped tar behind a signed URL.
|
|
828
|
+
*
|
|
829
|
+
* NO AUTHORIZATION HEADER, deliberately: the URL points at Convex's file-storage host, not at the
|
|
830
|
+
* ledger, and attaching the batch grant would hand this install's write credential to a third
|
|
831
|
+
* party. The URL's own signature and expiry are the authority — that is what makes it signed.
|
|
832
|
+
*/
|
|
833
|
+
export async function downloadParcelBlob(opts) {
|
|
834
|
+
const cap = opts.maxBytes ?? MAX_DOWNLOAD_BYTES;
|
|
835
|
+
try {
|
|
836
|
+
const res = await (opts.fetchFn ?? fetch)(opts.url, {
|
|
837
|
+
method: "GET",
|
|
838
|
+
signal: AbortSignal.timeout(opts.timeoutMs ?? READ_TIMEOUT_MS)
|
|
839
|
+
});
|
|
840
|
+
if (!res.ok)
|
|
841
|
+
return { ok: false, status: res.status, reason: `HTTP ${res.status} fetching the parcel blob` };
|
|
842
|
+
const declared = Number(res.headers.get("content-length") ?? "");
|
|
843
|
+
if (Number.isFinite(declared) && declared > cap) {
|
|
844
|
+
return { ok: false, reason: `parcel blob declares ${declared} bytes, past the ${cap}-byte download cap` };
|
|
845
|
+
}
|
|
846
|
+
const body = Buffer.from(await res.arrayBuffer());
|
|
847
|
+
if (body.length > cap)
|
|
848
|
+
return { ok: false, reason: `parcel blob is ${body.length} bytes, past the ${cap}-byte download cap` };
|
|
849
|
+
return { ok: true, value: body };
|
|
850
|
+
}
|
|
851
|
+
catch (error) {
|
|
852
|
+
return { ok: false, reason: error instanceof Error ? error.message : String(error) };
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
// ---------------------------------------------------------------------------------------------
|
|
856
|
+
// unpacking
|
|
857
|
+
// ---------------------------------------------------------------------------------------------
|
|
858
|
+
/**
|
|
859
|
+
* A tar member path this process is willing to join to a directory, or null.
|
|
860
|
+
*
|
|
861
|
+
* The parcel was built by another machine, so `..`, an absolute path and a Windows drive letter are
|
|
862
|
+
* all things a hostile — or merely broken — writer can put in a header, and `path.join` would
|
|
863
|
+
* happily follow every one of them out of the staging directory. Refused wholesale rather than
|
|
864
|
+
* sanitized: a name that needed rewriting is not a name this reader understands.
|
|
865
|
+
*/
|
|
866
|
+
function safeParcelEntryPath(name) {
|
|
867
|
+
const normalized = name.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
868
|
+
if (!normalized || normalized.startsWith("/") || /^[A-Za-z]:/.test(normalized))
|
|
869
|
+
return null;
|
|
870
|
+
const parts = normalized.split("/");
|
|
871
|
+
if (parts.some((part) => part === "" || part === "." || part === ".."))
|
|
872
|
+
return null;
|
|
873
|
+
return parts.join("/");
|
|
874
|
+
}
|
|
875
|
+
/** A path segment that is a whole segment — a session id or wave label, as the tar states it. */
|
|
876
|
+
const isPlainSegment = (value) => typeof value === "string" && value.length > 0 && !/[/\\]/.test(value) && value !== "." && value !== "..";
|
|
877
|
+
/**
|
|
878
|
+
* A parcel's bytes back into the session shape the wave was written in.
|
|
879
|
+
*
|
|
880
|
+
* Entries land at their stated paths under `destRoot` because `buildParcel` files them at their
|
|
881
|
+
* real session-relative locations for exactly this reason: `readWaveFromBundle` derives a wave's
|
|
882
|
+
* session id and label from its own directory, so an unpacked parcel is indistinguishable from a
|
|
883
|
+
* locally produced bundle — which is the whole mechanism behind identical SHAs on two machines.
|
|
884
|
+
*
|
|
885
|
+
* Throws on anything it will not vouch for: an unknown format version, a path that escapes, a
|
|
886
|
+
* manifest that names a different org than the caller is rebuilding. The caller counts those as
|
|
887
|
+
* parcels it could not stage and says so, rather than quietly building a smaller history.
|
|
888
|
+
*/
|
|
889
|
+
export async function unpackParcel(gz, destRoot, opts = {}) {
|
|
890
|
+
let entries;
|
|
891
|
+
try {
|
|
892
|
+
entries = untar(gunzipSync(gz, { maxOutputLength: MAX_UNCOMPRESSED_BYTES }));
|
|
893
|
+
}
|
|
894
|
+
catch (error) {
|
|
895
|
+
throw new Error(`parcel could not be unpacked: ${error instanceof Error ? error.message : String(error)}`);
|
|
896
|
+
}
|
|
897
|
+
const manifestEntry = entries.find((entry) => entry.name === "parcel.json");
|
|
898
|
+
if (!manifestEntry)
|
|
899
|
+
throw new Error("parcel has no parcel.json — nothing states what wave it is");
|
|
900
|
+
const manifest = JSON.parse(manifestEntry.content.toString("utf8"));
|
|
901
|
+
const parcelVersion = manifest.parcelVersion ?? 0;
|
|
902
|
+
// Refused, not best-efforted. A newer layout may file entries somewhere this reader does not
|
|
903
|
+
// look, and a half-read parcel materializes as a wave with pieces missing — which reads as an
|
|
904
|
+
// org that lost components rather than as an install that is out of date.
|
|
905
|
+
if (parcelVersion > PARCEL_FORMAT_VERSION) {
|
|
906
|
+
throw new Error(`parcel format v${parcelVersion} is newer than this install understands (v${PARCEL_FORMAT_VERSION}) — update, then rebuild`);
|
|
907
|
+
}
|
|
908
|
+
if (!isPlainSegment(manifest.sessionId) || !isPlainSegment(manifest.wave)) {
|
|
909
|
+
throw new Error("parcel.json does not name a plain session id and wave label");
|
|
910
|
+
}
|
|
911
|
+
if (typeof manifest.orgId !== "string" || !manifest.orgId)
|
|
912
|
+
throw new Error("parcel.json names no org");
|
|
913
|
+
// The third check on one fact, and each is worth having: the ledger scoped the LIST by grant, this
|
|
914
|
+
// checks what the producing machine wrote INSIDE the tar, and `rebuild` checks each bundle's own
|
|
915
|
+
// projection manifest. A parcel that disagrees with the grant that served it is not a parcel to
|
|
916
|
+
// file into somebody's org history.
|
|
917
|
+
// The org this parcel actually names is NOT quoted, for the reason `fetchForeignBundles` gives at
|
|
918
|
+
// its own org check: this message ends up in `problems`, and `problems` ends up in front of an
|
|
919
|
+
// operator who is entitled to see their own org id and nobody else's.
|
|
920
|
+
if (opts.expectOrgId && manifest.orgId.slice(0, 15) !== opts.expectOrgId.slice(0, 15)) {
|
|
921
|
+
throw new Error(`parcel.json names a different org than the one being rebuilt (${opts.expectOrgId})`);
|
|
922
|
+
}
|
|
923
|
+
// EVERY path is checked BEFORE the first byte is written. Checking as it goes would leave the
|
|
924
|
+
// entries ahead of a bad one on disk, and a partially unpacked parcel is a bundle that names
|
|
925
|
+
// components it does not carry — which materializes as an org that lost them.
|
|
926
|
+
const planned = [];
|
|
927
|
+
let bytes = 0;
|
|
928
|
+
for (const entry of entries) {
|
|
929
|
+
const rel = safeParcelEntryPath(entry.name);
|
|
930
|
+
if (!rel)
|
|
931
|
+
throw new Error(`parcel entry ${JSON.stringify(entry.name)} escapes the staging directory`);
|
|
932
|
+
planned.push({ abs: path.join(destRoot, rel), content: entry.content });
|
|
933
|
+
bytes += entry.content.length;
|
|
934
|
+
}
|
|
935
|
+
for (const file of planned) {
|
|
936
|
+
await fs.mkdir(path.dirname(file.abs), { recursive: true });
|
|
937
|
+
await fs.writeFile(file.abs, file.content, { mode: 0o600 });
|
|
938
|
+
}
|
|
939
|
+
const bundleDir = path.join(destRoot, manifest.sessionId, "direct", manifest.wave);
|
|
940
|
+
const present = await fs
|
|
941
|
+
.stat(bundleDir)
|
|
942
|
+
.then((stat) => stat.isDirectory())
|
|
943
|
+
.catch(() => false);
|
|
944
|
+
if (!present)
|
|
945
|
+
throw new Error(`parcel names wave ${manifest.sessionId}/${manifest.wave} but carries no bundle for it`);
|
|
946
|
+
return {
|
|
947
|
+
identity: {
|
|
948
|
+
parcelVersion,
|
|
949
|
+
orgId: manifest.orgId,
|
|
950
|
+
sessionId: manifest.sessionId,
|
|
951
|
+
wave: manifest.wave,
|
|
952
|
+
attempt: manifest.attempt ?? 1,
|
|
953
|
+
verdict: manifest.verdict ?? "",
|
|
954
|
+
...(manifest.batchId ? { batchId: manifest.batchId } : {})
|
|
955
|
+
},
|
|
956
|
+
bundleDir,
|
|
957
|
+
files: planned.length,
|
|
958
|
+
bytes
|
|
959
|
+
};
|
|
960
|
+
}
|
|
961
|
+
// ---------------------------------------------------------------------------------------------
|
|
962
|
+
// grants, staging, and the one call the rebuild makes
|
|
963
|
+
// ---------------------------------------------------------------------------------------------
|
|
964
|
+
const grantsDir = () => path.join(REPO_ROOT, ".local", "cc-mcp", "telemetry-grants");
|
|
965
|
+
/**
|
|
966
|
+
* Every live batch grant on this machine, longest-lived first.
|
|
967
|
+
*
|
|
968
|
+
* A grant does NOT record its org — the ledger resolves that from the batch — so this cannot filter
|
|
969
|
+
* by org and does not pretend to. The caller lists through each candidate in turn and keeps the one
|
|
970
|
+
* whose bound org comes back matching. That is one round trip per stale engagement on the laptop
|
|
971
|
+
* and it is the only way to answer the question without asking the client to assert an org.
|
|
972
|
+
*
|
|
973
|
+
* Read straight from the store's files rather than through `get()`, which needs the exact batch id
|
|
974
|
+
* and the id is what a listing is trying to discover — the filename is sanitized on write, so
|
|
975
|
+
* inverting it would be a guess. Expiry is checked here exactly as `get()` checks it.
|
|
976
|
+
*/
|
|
977
|
+
export function storedParcelGrants(dir = grantsDir(), now = Date.now()) {
|
|
978
|
+
const grants = [];
|
|
979
|
+
try {
|
|
980
|
+
for (const name of readdirSync(dir)) {
|
|
981
|
+
if (!name.endsWith(".json"))
|
|
982
|
+
continue;
|
|
983
|
+
try {
|
|
984
|
+
const stored = JSON.parse(readFileSync(path.join(dir, name), "utf8"));
|
|
985
|
+
if (!stored?.batchId || !stored.credential || !stored.endpoint)
|
|
986
|
+
continue;
|
|
987
|
+
if (!(Date.parse(stored.expiresAt) > now))
|
|
988
|
+
continue;
|
|
989
|
+
grants.push({
|
|
990
|
+
batchId: stored.batchId,
|
|
991
|
+
endpoint: parcelEndpointFrom(stored.endpoint),
|
|
992
|
+
credential: stored.credential,
|
|
993
|
+
expiresAt: stored.expiresAt
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
catch {
|
|
997
|
+
// A half-written or hand-edited grant file is not a grant.
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
catch {
|
|
1002
|
+
// No grant directory at all — an install that has never activated. Reported as no grant.
|
|
1003
|
+
}
|
|
1004
|
+
return grants.sort((a, b) => (a.expiresAt < b.expiresAt ? 1 : a.expiresAt > b.expiresAt ? -1 : 0));
|
|
1005
|
+
}
|
|
1006
|
+
/** Where fetched parcels are staged. DELIBERATELY OUTSIDE the session workspace — see below. */
|
|
1007
|
+
export function parcelStageDir(orgId) {
|
|
1008
|
+
return process.env.CC_MCP_PARCEL_STAGE?.trim()
|
|
1009
|
+
? path.join(process.env.CC_MCP_PARCEL_STAGE.trim(), orgId)
|
|
1010
|
+
: path.join(REPO_ROOT, ".local", "cc-mcp", "parcel-fetch", orgId);
|
|
1011
|
+
}
|
|
1012
|
+
/**
|
|
1013
|
+
* Fetch this org's foreign parcels and stage them as replayable bundle directories.
|
|
1014
|
+
*
|
|
1015
|
+
* STAGED OUTSIDE THE SESSION WORKSPACE, and that placement is load-bearing. `workspaceBundleDirs`
|
|
1016
|
+
* enumerates the LOCAL bundle set by walking the session root; a fetched parcel unpacked in there
|
|
1017
|
+
* would be counted as this machine's own work, and "local-only" would silently become a claim about
|
|
1018
|
+
* other people's machines. Each parcel gets its own directory under the stage keyed by parcel id,
|
|
1019
|
+
* so two parcels from one session cannot overwrite each other's copy of `reads.jsonl` and a re-fetch
|
|
1020
|
+
* is a directory replacement rather than a merge.
|
|
1021
|
+
*
|
|
1022
|
+
* NEVER THROWS. Every path out of here is a described result, because the caller's next move is to
|
|
1023
|
+
* rebuild either way — with the foreign set if it arrived, and with an honest scope note if it did
|
|
1024
|
+
* not.
|
|
1025
|
+
*/
|
|
1026
|
+
export async function fetchForeignBundles(opts) {
|
|
1027
|
+
const result = {
|
|
1028
|
+
ok: false,
|
|
1029
|
+
orgId: opts.orgId,
|
|
1030
|
+
bundles: [],
|
|
1031
|
+
listed: 0,
|
|
1032
|
+
downloaded: 0,
|
|
1033
|
+
cached: 0,
|
|
1034
|
+
failed: 0,
|
|
1035
|
+
problems: []
|
|
1036
|
+
};
|
|
1037
|
+
const write = opts.log ?? ((line) => defaultLog(line));
|
|
1038
|
+
const call = { ...(opts.fetchFn ? { fetchFn: opts.fetchFn } : {}), ...(opts.timeoutMs ? { timeoutMs: opts.timeoutMs } : {}) };
|
|
1039
|
+
let candidates;
|
|
1040
|
+
try {
|
|
1041
|
+
candidates = opts.grant
|
|
1042
|
+
? [opts.grant]
|
|
1043
|
+
: opts.batchId
|
|
1044
|
+
? [defaultResolveGrant(opts.batchId)].filter((grant) => grant !== null)
|
|
1045
|
+
: (opts.grants ?? storedParcelGrants());
|
|
1046
|
+
}
|
|
1047
|
+
catch {
|
|
1048
|
+
candidates = [];
|
|
1049
|
+
}
|
|
1050
|
+
if (candidates.length === 0) {
|
|
1051
|
+
result.unavailable = opts.batchId
|
|
1052
|
+
? `no live grant for batch ${opts.batchId} on this machine`
|
|
1053
|
+
: "no live batch grant on this machine — a parcel fetch is authorized by an activated session's grant";
|
|
1054
|
+
return result;
|
|
1055
|
+
}
|
|
1056
|
+
// Which grant speaks for THIS org is the ledger's answer, not ours: list through each in turn and
|
|
1057
|
+
// keep the first whose bound org matches. A grant that answers for another org is skipped without
|
|
1058
|
+
// reading a single parcel out of it.
|
|
1059
|
+
let grant = null;
|
|
1060
|
+
let first = null;
|
|
1061
|
+
const rejections = [];
|
|
1062
|
+
for (const candidate of candidates) {
|
|
1063
|
+
const page = await listParcels({
|
|
1064
|
+
endpoint: candidate.endpoint,
|
|
1065
|
+
credential: candidate.credential,
|
|
1066
|
+
...(opts.pageLimit ? { limit: opts.pageLimit } : {}),
|
|
1067
|
+
...call
|
|
1068
|
+
});
|
|
1069
|
+
if (!page.ok) {
|
|
1070
|
+
rejections.push(page.reason);
|
|
1071
|
+
continue;
|
|
1072
|
+
}
|
|
1073
|
+
if (page.value.orgId.slice(0, 15) !== opts.orgId.slice(0, 15)) {
|
|
1074
|
+
// THE FOREIGN ORG ID IS NOT NAMED, deliberately. `rejections` becomes `unavailable`, which the
|
|
1075
|
+
// rebuild prints in its scope note — so naming the org a stale grant is bound to would put
|
|
1076
|
+
// ANOTHER CUSTOMER'S org id into this engagement's output, on a laptop that legitimately holds
|
|
1077
|
+
// grants for several. The org being rebuilt is the operator's own and stays; the other one is
|
|
1078
|
+
// not this engagement's to see, and "a different org" is all the operator needs to act on
|
|
1079
|
+
// (re-activate for this org). The full id remains available in the grant files themselves.
|
|
1080
|
+
rejections.push(`a grant on this machine is bound to a different org, not ${opts.orgId}`);
|
|
1081
|
+
continue;
|
|
1082
|
+
}
|
|
1083
|
+
grant = candidate;
|
|
1084
|
+
first = page.value;
|
|
1085
|
+
break;
|
|
1086
|
+
}
|
|
1087
|
+
if (!grant || !first) {
|
|
1088
|
+
result.unavailable = rejections.length > 0 ? rejections.join("; ") : "no grant on this machine is bound to this org";
|
|
1089
|
+
return result;
|
|
1090
|
+
}
|
|
1091
|
+
result.ok = true;
|
|
1092
|
+
result.orgId = first.orgId;
|
|
1093
|
+
// Bound to a const before the closure below reads it: `grant` is a `let` the loop reassigns, and
|
|
1094
|
+
// a narrowing that only holds at this line is not one a callback can rely on.
|
|
1095
|
+
const bound = grant;
|
|
1096
|
+
const stage = opts.stageDir ?? parcelStageDir(opts.orgId);
|
|
1097
|
+
const maxParcels = opts.maxParcels ?? 1000;
|
|
1098
|
+
const stageOne = async (entry) => {
|
|
1099
|
+
result.listed += 1;
|
|
1100
|
+
if (!isPlainSegment(entry.parcelId)) {
|
|
1101
|
+
result.failed += 1;
|
|
1102
|
+
result.problems.push(`the ledger listed a parcel whose id is not a plain identifier — skipped`);
|
|
1103
|
+
return;
|
|
1104
|
+
}
|
|
1105
|
+
const dest = path.join(stage, "p", entry.parcelId);
|
|
1106
|
+
const markerFile = path.join(stage, "p", `${entry.parcelId}.staged.json`);
|
|
1107
|
+
const marker = await fs
|
|
1108
|
+
.readFile(markerFile, "utf8")
|
|
1109
|
+
.then((text) => JSON.parse(text))
|
|
1110
|
+
.catch(() => null);
|
|
1111
|
+
if (marker?.bundleDir) {
|
|
1112
|
+
const still = await fs
|
|
1113
|
+
.stat(marker.bundleDir)
|
|
1114
|
+
.then((stat) => stat.isDirectory())
|
|
1115
|
+
.catch(() => false);
|
|
1116
|
+
if (still) {
|
|
1117
|
+
result.cached += 1;
|
|
1118
|
+
result.bundles.push({
|
|
1119
|
+
parcelId: entry.parcelId,
|
|
1120
|
+
sessionId: marker.sessionId,
|
|
1121
|
+
wave: marker.wave,
|
|
1122
|
+
bundleDir: marker.bundleDir,
|
|
1123
|
+
bytes: marker.bytes,
|
|
1124
|
+
uploadedAt: marker.uploadedAt,
|
|
1125
|
+
cached: true
|
|
1126
|
+
});
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
const signed = await parcelDownloadUrl({ endpoint: bound.endpoint, credential: bound.credential, parcelId: entry.parcelId, ...call });
|
|
1131
|
+
if (!signed.ok) {
|
|
1132
|
+
result.failed += 1;
|
|
1133
|
+
result.problems.push(`${entry.sessionId}/${entry.wave}: ${signed.reason}`);
|
|
1134
|
+
return;
|
|
1135
|
+
}
|
|
1136
|
+
// THE LEDGER ALREADY SAID HOW BIG THIS IS, so the download is measured against that rather than
|
|
1137
|
+
// against a global ceiling alone. `present()` returns the byte count the upload was stored with,
|
|
1138
|
+
// and a blob that does not match it is not the parcel the row describes — a truncated transfer,
|
|
1139
|
+
// a proxy that rewrote the body, or a row pointing at the wrong object. Any of those unpacks
|
|
1140
|
+
// into a wave nobody ran, which is worse than a wave that is missing: a missing wave is already
|
|
1141
|
+
// counted and said out loud, and a wrong one is filed as history.
|
|
1142
|
+
//
|
|
1143
|
+
// The cap goes DOWN to the declared size (never up past the module's own ceiling), so an
|
|
1144
|
+
// oversized body is refused off `content-length` before its bytes are read; a SHORT body gets
|
|
1145
|
+
// past the cap by construction and is caught by the equality check below. Both land in `failed`
|
|
1146
|
+
// and in `problems` — never a throw, because the rebuild still has to run and say what it covers.
|
|
1147
|
+
//
|
|
1148
|
+
// A row that declares nothing usable falls back to the module ceiling and is staged unmeasured,
|
|
1149
|
+
// rather than refusing evidence over a field the ledger did not fill in.
|
|
1150
|
+
const declared = Number(entry.bytes);
|
|
1151
|
+
const expectBytes = Number.isFinite(declared) && declared > 0 ? declared : null;
|
|
1152
|
+
const blob = await downloadParcelBlob({
|
|
1153
|
+
url: signed.value.url,
|
|
1154
|
+
...(expectBytes === null ? {} : { maxBytes: Math.min(expectBytes, MAX_DOWNLOAD_BYTES) }),
|
|
1155
|
+
...call
|
|
1156
|
+
});
|
|
1157
|
+
if (!blob.ok) {
|
|
1158
|
+
result.failed += 1;
|
|
1159
|
+
result.problems.push(`${entry.sessionId}/${entry.wave}: ${blob.reason}`);
|
|
1160
|
+
return;
|
|
1161
|
+
}
|
|
1162
|
+
if (expectBytes !== null && blob.value.length !== expectBytes) {
|
|
1163
|
+
result.failed += 1;
|
|
1164
|
+
result.problems.push(`${entry.sessionId}/${entry.wave}: the ledger lists ${expectBytes} bytes but ${blob.value.length} arrived — not staged`);
|
|
1165
|
+
return;
|
|
1166
|
+
}
|
|
1167
|
+
try {
|
|
1168
|
+
// Replaced, not merged: a half-written directory from an interrupted fetch must not be
|
|
1169
|
+
// completed by a later one into a bundle neither parcel ever contained.
|
|
1170
|
+
await fs.rm(dest, { recursive: true, force: true });
|
|
1171
|
+
const unpacked = await unpackParcel(blob.value, dest, { expectOrgId: opts.orgId });
|
|
1172
|
+
const record = {
|
|
1173
|
+
parcelId: entry.parcelId,
|
|
1174
|
+
sessionId: unpacked.identity.sessionId,
|
|
1175
|
+
wave: unpacked.identity.wave,
|
|
1176
|
+
bytes: entry.bytes,
|
|
1177
|
+
uploadedAt: entry.uploadedAt,
|
|
1178
|
+
bundleDir: unpacked.bundleDir,
|
|
1179
|
+
stagedAt: new Date().toISOString()
|
|
1180
|
+
};
|
|
1181
|
+
await fs.mkdir(path.dirname(markerFile), { recursive: true });
|
|
1182
|
+
await fs.writeFile(markerFile, `${JSON.stringify(record)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
1183
|
+
result.downloaded += 1;
|
|
1184
|
+
result.bundles.push({ ...record, cached: false });
|
|
1185
|
+
}
|
|
1186
|
+
catch (error) {
|
|
1187
|
+
result.failed += 1;
|
|
1188
|
+
result.problems.push(`${entry.sessionId}/${entry.wave}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1189
|
+
await fs.rm(dest, { recursive: true, force: true }).catch(() => undefined);
|
|
1190
|
+
}
|
|
1191
|
+
};
|
|
1192
|
+
let page = first;
|
|
1193
|
+
for (;;) {
|
|
1194
|
+
let staged = 0;
|
|
1195
|
+
for (const entry of page.parcels) {
|
|
1196
|
+
if (result.listed >= maxParcels)
|
|
1197
|
+
break;
|
|
1198
|
+
await stageOne(entry);
|
|
1199
|
+
staged += 1;
|
|
1200
|
+
}
|
|
1201
|
+
const moreToCome = !page.isDone && Boolean(page.cursor);
|
|
1202
|
+
if (staged < page.parcels.length || (moreToCome && result.listed >= maxParcels)) {
|
|
1203
|
+
// A CAP THAT TRUNCATES SAYS SO. Stopping here quietly would produce a history that looks
|
|
1204
|
+
// whole and is short by however many parcels were never asked for, which is the one shape
|
|
1205
|
+
// this module exists to prevent. Counted as one loss because the true number is exactly what
|
|
1206
|
+
// the cap stopped us from finding out.
|
|
1207
|
+
result.failed += 1;
|
|
1208
|
+
result.problems.push(`stopped at the ${maxParcels}-parcel ceiling; the ledger holds more for this org`);
|
|
1209
|
+
break;
|
|
1210
|
+
}
|
|
1211
|
+
if (!moreToCome)
|
|
1212
|
+
break;
|
|
1213
|
+
const next = await listParcels({
|
|
1214
|
+
endpoint: bound.endpoint,
|
|
1215
|
+
credential: bound.credential,
|
|
1216
|
+
cursor: page.cursor,
|
|
1217
|
+
...(opts.pageLimit ? { limit: opts.pageLimit } : {}),
|
|
1218
|
+
...call
|
|
1219
|
+
});
|
|
1220
|
+
if (!next.ok) {
|
|
1221
|
+
// The first page landed, so the fetch is not unavailable — it is INCOMPLETE, and that is a
|
|
1222
|
+
// different sentence. Counted as a loss so the scope note cannot read as a whole history.
|
|
1223
|
+
result.failed += 1;
|
|
1224
|
+
result.problems.push(`the parcel listing stopped early: ${next.reason}`);
|
|
1225
|
+
break;
|
|
1226
|
+
}
|
|
1227
|
+
page = next.value;
|
|
1228
|
+
}
|
|
1229
|
+
if (result.failed > 0) {
|
|
1230
|
+
write(`${result.failed} parcel(s) for ${opts.orgId} could NOT be staged — this rebuild may still be missing waves: ${result.problems
|
|
1231
|
+
.slice(0, 5)
|
|
1232
|
+
.join(" | ")}`);
|
|
1233
|
+
}
|
|
1234
|
+
return result;
|
|
1235
|
+
}
|