@bli-cockpit/cli 0.2.39 → 0.2.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/local-sources.js +51 -5
- package/dist/autostart.js +105 -5
- package/dist/commands/brief.js +11 -10
- package/dist/commands/cli-io.js +62 -1
- package/dist/commands/correct.js +5 -17
- package/dist/commands/jarvis.js +8 -20
- package/dist/commands/local-discovery.js +20 -0
- package/dist/commands/local-help.js +1 -1
- package/dist/commands/local.js +34 -1
- package/dist/commands/notes-file.js +27 -0
- package/dist/commands/notes.js +24 -19
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/scout-render.js +20 -18
- package/dist/commands/scout.js +4 -4
- package/dist/commands/sessions.js +45 -1
- package/dist/commands/settings-render.js +9 -4
- package/dist/commands/settings.js +9 -2
- package/dist/commands/text-width.js +108 -0
- package/dist/commands/tower-command.js +7 -8
- package/dist/commands/workbook-render.js +34 -17
- package/dist/commands/workbook.js +6 -4
- package/dist/evidence-upload-client.js +42 -3
- package/dist/local-state.js +27 -2
- package/dist/upload-agent-artifacts.js +27 -2
- package/dist/upload-failure-reason.js +121 -0
- package/dist/upload-http.js +28 -0
- package/dist/upload.js +122 -21
- package/package.json +1 -1
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* on the two supported host families; `cockpit workbook tower workbook | less`
|
|
15
15
|
* is the person's decision to make, not this command's.
|
|
16
16
|
*/
|
|
17
|
-
import { writeLine, writeRaw } from "./cli-io.js";
|
|
17
|
+
import { colorEnabled, writeLine, writeRaw } from "./cli-io.js";
|
|
18
18
|
import { renderDocText, renderShelf, sectionSlice, } from "./workbook-render.js";
|
|
19
19
|
import { loadPairedSession, towerFailureDetail, towerJsonRequest, } from "../tower-client.js";
|
|
20
20
|
const READ_DEADLINE_MS = 30_000;
|
|
@@ -52,6 +52,7 @@ export async function runWorkbook(command, io) {
|
|
|
52
52
|
// -------------------------------------------------------------------- index
|
|
53
53
|
function writeIndex(command, io, payload, width) {
|
|
54
54
|
const projects = payload.projects ?? [];
|
|
55
|
+
const styled = colorEnabled(io);
|
|
55
56
|
if (command.project) {
|
|
56
57
|
const found = projects.find((project) => project.slug === command.project);
|
|
57
58
|
if (!found) {
|
|
@@ -73,7 +74,7 @@ function writeIndex(command, io, payload, width) {
|
|
|
73
74
|
writeLine(io.stdout, JSON.stringify({ ok: true, project: found }));
|
|
74
75
|
}
|
|
75
76
|
else {
|
|
76
|
-
for (const line of renderShelf([found], width))
|
|
77
|
+
for (const line of renderShelf([found], width, styled))
|
|
77
78
|
writeLine(io.stdout, line);
|
|
78
79
|
}
|
|
79
80
|
logIndexRead(io, [found]);
|
|
@@ -83,7 +84,7 @@ function writeIndex(command, io, payload, width) {
|
|
|
83
84
|
writeLine(io.stdout, JSON.stringify({ ok: true, projects }));
|
|
84
85
|
}
|
|
85
86
|
else {
|
|
86
|
-
for (const line of renderShelf(projects, width))
|
|
87
|
+
for (const line of renderShelf(projects, width, styled))
|
|
87
88
|
writeLine(io.stdout, line);
|
|
88
89
|
}
|
|
89
90
|
logIndexRead(io, projects);
|
|
@@ -134,7 +135,8 @@ function writeDoc(command, io, payload, width) {
|
|
|
134
135
|
writeRaw(io.stdout, selected);
|
|
135
136
|
}
|
|
136
137
|
else {
|
|
137
|
-
|
|
138
|
+
const lines = renderDocText(payload, selected, width, colorEnabled(io));
|
|
139
|
+
for (const line of lines)
|
|
138
140
|
writeLine(io.stdout, line);
|
|
139
141
|
}
|
|
140
142
|
writeLine(io.stderr, `[workbook cli] doc read ${JSON.stringify({
|
|
@@ -102,14 +102,39 @@ export async function uploadRawEvidenceFilesChunked(options) {
|
|
|
102
102
|
// (after retries) covers a new dashboard whose ledger migration has not
|
|
103
103
|
// been applied yet. Both still serve the legacy v1 route.
|
|
104
104
|
if (begin.status === 404 || begin.status >= 500) {
|
|
105
|
+
// Until BLI-3483 this downgrade abandoned the entire chunked path
|
|
106
|
+
// without a word, and the comment above named two causes that the fleet
|
|
107
|
+
// had no way to tell apart — an old dashboard versus an unapplied ledger
|
|
108
|
+
// migration. This is the BLI-2528 shape exactly: the code knew, the
|
|
109
|
+
// operator did not. Once per run, because `break` leaves the loop.
|
|
110
|
+
console.error("[evidence-upload] chunked upload unavailable; falling back to the legacy single-shot route", JSON.stringify({
|
|
111
|
+
reason: begin.status === 404
|
|
112
|
+
? "begin_route_absent"
|
|
113
|
+
: "begin_server_error_after_retries",
|
|
114
|
+
http_status: begin.status,
|
|
115
|
+
server_reason: safeFailureDetail(begin.body) ?? "none",
|
|
116
|
+
objects_in_batch: batch.length,
|
|
117
|
+
objects_unresolved: loaded.length - resolvedEntries.size,
|
|
118
|
+
}));
|
|
105
119
|
beginUnavailable = true;
|
|
106
120
|
break;
|
|
107
121
|
}
|
|
108
122
|
if (!begin.ok) {
|
|
123
|
+
// The server's own `{ reason }` sat unread in this body while the ledger
|
|
124
|
+
// recorded the transport status and nothing else (BLI-3483); the commit
|
|
125
|
+
// path has read it since BLI-2528 and this one now does the same.
|
|
126
|
+
const beginDetail = safeFailureDetail(begin.body);
|
|
127
|
+
const beginReason = `begin_failed_http_${begin.status}${beginDetail ? `_${beginDetail}` : ""}`;
|
|
128
|
+
console.error("[evidence-upload] begin refused these objects", JSON.stringify({
|
|
129
|
+
reason: "begin_rejected",
|
|
130
|
+
http_status: begin.status,
|
|
131
|
+
server_reason: beginDetail ?? "none",
|
|
132
|
+
objects_in_batch: batch.length,
|
|
133
|
+
}));
|
|
109
134
|
for (const entry of batch) {
|
|
110
|
-
outcomes.push(failedOutcome(entry.file,
|
|
135
|
+
outcomes.push(failedOutcome(entry.file, beginReason));
|
|
111
136
|
for (const duplicate of entry.duplicates) {
|
|
112
|
-
outcomes.push(failedOutcome(duplicate,
|
|
137
|
+
outcomes.push(failedOutcome(duplicate, beginReason));
|
|
113
138
|
}
|
|
114
139
|
resolvedEntries.add(entry);
|
|
115
140
|
}
|
|
@@ -286,7 +311,21 @@ async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
|
|
|
286
311
|
content_base64: chunk.toString("base64"),
|
|
287
312
|
});
|
|
288
313
|
if (!chunkResponse.ok) {
|
|
289
|
-
|
|
314
|
+
// Same treatment the commit path has had since BLI-2528: the server's
|
|
315
|
+
// own reason rides on the label, so `chunk_3_failed_http_413` becomes
|
|
316
|
+
// `chunk_3_failed_http_413_object_too_large` and the ledger row names
|
|
317
|
+
// the cause instead of the transport (BLI-3483).
|
|
318
|
+
const chunkDetail = safeFailureDetail(chunkResponse.body);
|
|
319
|
+
console.error("[evidence-upload] chunk rejected", JSON.stringify({
|
|
320
|
+
reason: "chunk_rejected",
|
|
321
|
+
upload_id: disposition.upload_id,
|
|
322
|
+
http_status: chunkResponse.status,
|
|
323
|
+
server_reason: chunkDetail ?? "none",
|
|
324
|
+
chunk_index: index,
|
|
325
|
+
chunk_count: entry.chunkCount,
|
|
326
|
+
uploaded_chunk_count: uploadedChunks,
|
|
327
|
+
}));
|
|
328
|
+
return failedOutcome(entry.file, `chunk_${index}_failed_http_${chunkResponse.status}${chunkDetail ? `_${chunkDetail}` : ""}`, uploadedChunks);
|
|
290
329
|
}
|
|
291
330
|
uploadedChunks += 1;
|
|
292
331
|
}
|
package/dist/local-state.js
CHANGED
|
@@ -10,6 +10,7 @@ import { summarizeLocalUploadSpool } from "./spool/local-spool.js";
|
|
|
10
10
|
import { summarizeInstallEventOutbox } from "./spool/install-event-outbox.js";
|
|
11
11
|
import { readRawEvidenceStagingState, summarizeStuckEvidence, } from "./raw-evidence-staging.js";
|
|
12
12
|
import { describeError, isMissingFileFailure } from "./health-detail.js";
|
|
13
|
+
import { serverFailureDetail } from "./upload-http.js";
|
|
13
14
|
const localCollectorPackage = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
|
|
14
15
|
export const LOCAL_COLLECTOR_VERSION = typeof localCollectorPackage.version === "string"
|
|
15
16
|
? localCollectorPackage.version
|
|
@@ -671,7 +672,7 @@ async function postPairStart(fetchImpl, dashboardUrl, body, accessToken) {
|
|
|
671
672
|
});
|
|
672
673
|
const parsed = await readResponseJson(response);
|
|
673
674
|
if (!response.ok) {
|
|
674
|
-
throw new Error(
|
|
675
|
+
throw new Error(pairFailureMessage("Pair request failed", response, parsed));
|
|
675
676
|
}
|
|
676
677
|
return parsePairStartResponse(parsed);
|
|
677
678
|
}
|
|
@@ -692,7 +693,7 @@ async function pollPairRequest(fetchImpl, dashboardUrl, options) {
|
|
|
692
693
|
});
|
|
693
694
|
const parsed = await readResponseJson(response);
|
|
694
695
|
if (!response.ok) {
|
|
695
|
-
throw new Error(
|
|
696
|
+
throw new Error(pairFailureMessage("Pair polling failed", response, parsed));
|
|
696
697
|
}
|
|
697
698
|
const status = readStringField(parsed, "status");
|
|
698
699
|
if (status === "approved") {
|
|
@@ -713,6 +714,30 @@ async function pollPairRequest(fetchImpl, dashboardUrl, options) {
|
|
|
713
714
|
}
|
|
714
715
|
throw new Error("Timed out waiting for dashboard approval. Run `cockpit login` again.");
|
|
715
716
|
}
|
|
717
|
+
/**
|
|
718
|
+
* What `cockpit login` tells the operator when pairing is refused.
|
|
719
|
+
*
|
|
720
|
+
* "Pair request failed" was the whole message — five words, while the status
|
|
721
|
+
* code sat in hand (BLI-3483). A 401 (this build's token is not accepted), a
|
|
722
|
+
* 403 (the dashboard knows the device and is refusing it), a 404 (wrong
|
|
723
|
+
* dashboard URL) and a 502 (something in front of the dashboard answered) are
|
|
724
|
+
* four different next actions, and the operator could not tell them apart.
|
|
725
|
+
* `upload.ts` has named its HTTP status since it was written; this is the same
|
|
726
|
+
* shape. The server's own words come first when it supplied any, and the status
|
|
727
|
+
* always rides at the end so it is never the thing that got dropped.
|
|
728
|
+
*
|
|
729
|
+
* Logged as well as thrown: `cockpit login` failures happen on a machine that
|
|
730
|
+
* is not collecting yet, so the terminal is the only receipt there is.
|
|
731
|
+
*/
|
|
732
|
+
function pairFailureMessage(fallback, response, body) {
|
|
733
|
+
const serverWords = responseErrorMessage(body, fallback);
|
|
734
|
+
console.error("[local-state] pairing request refused", JSON.stringify({
|
|
735
|
+
reason: fallback === "Pair request failed" ? "pair_start_refused" : "pair_poll_refused",
|
|
736
|
+
http_status: response.status,
|
|
737
|
+
server_reason: serverFailureDetail(body) ?? "none",
|
|
738
|
+
}));
|
|
739
|
+
return `${serverWords} (HTTP ${response.status})`;
|
|
740
|
+
}
|
|
716
741
|
async function readResponseJson(response) {
|
|
717
742
|
const text = await response.text();
|
|
718
743
|
if (!text)
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { AgentImageArtifactReportRequestSchema, } from "@bli-cockpit/telemetry-core";
|
|
14
14
|
import { describeError } from "./health-detail.js";
|
|
15
|
-
import { readResponseJson } from "./upload-http.js";
|
|
15
|
+
import { readResponseJson, serverFailureDetail } from "./upload-http.js";
|
|
16
16
|
export async function reportAgentImageArtifacts(options) {
|
|
17
17
|
const artifacts = agentArtifactsFromEvidence(options);
|
|
18
18
|
if (artifacts.length === 0) {
|
|
@@ -37,7 +37,15 @@ export async function reportAgentImageArtifacts(options) {
|
|
|
37
37
|
},
|
|
38
38
|
body: JSON.stringify(payload),
|
|
39
39
|
});
|
|
40
|
+
// Both non-2xx branches returned a label and said nothing (BLI-3483). An
|
|
41
|
+
// image that never gets reported is never redacted and never OCR'd, and
|
|
42
|
+
// the only place that fact existed was a return value the sync discards.
|
|
40
43
|
if (response.status === 404) {
|
|
44
|
+
console.error("[agent-artifacts] the dashboard has no agent-artifact route; these images stay unreported", JSON.stringify({
|
|
45
|
+
reason: "agent_artifact_api_unavailable",
|
|
46
|
+
http_status: 404,
|
|
47
|
+
artifact_count: artifacts.length,
|
|
48
|
+
}));
|
|
41
49
|
return {
|
|
42
50
|
posted: false,
|
|
43
51
|
reason: "agent_artifact_api_unavailable",
|
|
@@ -45,6 +53,13 @@ export async function reportAgentImageArtifacts(options) {
|
|
|
45
53
|
};
|
|
46
54
|
}
|
|
47
55
|
if (!response.ok) {
|
|
56
|
+
const serverReason = serverFailureDetail(await readResponseJson(response));
|
|
57
|
+
console.error("[agent-artifacts] report refused", JSON.stringify({
|
|
58
|
+
reason: "report_failed",
|
|
59
|
+
http_status: response.status,
|
|
60
|
+
server_reason: serverReason ?? "none",
|
|
61
|
+
artifact_count: artifacts.length,
|
|
62
|
+
}));
|
|
48
63
|
return {
|
|
49
64
|
posted: false,
|
|
50
65
|
reason: `report_failed_http_${response.status}`,
|
|
@@ -55,10 +70,20 @@ export async function reportAgentImageArtifacts(options) {
|
|
|
55
70
|
const recordedCount = body && typeof body === "object"
|
|
56
71
|
? Number(body.recorded_count ?? 0)
|
|
57
72
|
: 0;
|
|
73
|
+
const recorded = Number.isFinite(recordedCount) ? recordedCount : 0;
|
|
74
|
+
// The success branch too: "some images reported" and "every image
|
|
75
|
+
// reported" are different facts, and a count that is short of
|
|
76
|
+
// `artifact_count` is the only warning an operator would ever get.
|
|
77
|
+
console.error("[agent-artifacts] report recorded", JSON.stringify({
|
|
78
|
+
reason: "recorded",
|
|
79
|
+
http_status: response.status,
|
|
80
|
+
artifact_count: artifacts.length,
|
|
81
|
+
recorded_count: recorded,
|
|
82
|
+
}));
|
|
58
83
|
return {
|
|
59
84
|
posted: true,
|
|
60
85
|
reason: "recorded",
|
|
61
|
-
recorded_count:
|
|
86
|
+
recorded_count: recorded,
|
|
62
87
|
};
|
|
63
88
|
}
|
|
64
89
|
catch (error) {
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The vocabulary a failed sync is allowed to use, and the bounded cause beside
|
|
3
|
+
* it.
|
|
4
|
+
*
|
|
5
|
+
* Before BLI-3483 the delivery path threw English sentences and the catch in
|
|
6
|
+
* `upload.ts` copied `error.message` straight into the spool row's
|
|
7
|
+
* `failure_reason`. Two consequences, both bad:
|
|
8
|
+
*
|
|
9
|
+
* - **Unbounded cardinality.** Every distinct server message became a distinct
|
|
10
|
+
* "reason", so nothing downstream could count how many machines were failing
|
|
11
|
+
* the same way. A reason label is only useful if it is drawn from a closed
|
|
12
|
+
* set.
|
|
13
|
+
* - **Opposite causes wearing one name.** "Ambient ingest returned an incomplete
|
|
14
|
+
* durable receipt." was the answer for a proxy that returned its own 200 AND
|
|
15
|
+
* for a dashboard that genuinely persisted fewer rows than were sent. The
|
|
16
|
+
* first is a network appliance in the way; the second is data loss. Same
|
|
17
|
+
* sentence, opposite repairs.
|
|
18
|
+
*
|
|
19
|
+
* So a failure now carries two things: a `SyncFailureReason` from the list
|
|
20
|
+
* below, which is the bucket, and a detail of at most
|
|
21
|
+
* {@link SYNC_FAILURE_DETAIL_MAX_CHARS} characters, which is the observed cause
|
|
22
|
+
* — the status, the field that disagreed, the error code. The detail is
|
|
23
|
+
* redacted with the same scrubber the health receipts use, because a caught
|
|
24
|
+
* error on this machine can carry an absolute path or a token-shaped fragment.
|
|
25
|
+
*
|
|
26
|
+
* Spool rows are local JSON, not database rows, so nothing migrates: a row
|
|
27
|
+
* written by an older CLI keeps its prose string and still parses and renders,
|
|
28
|
+
* it simply does not participate in the new buckets.
|
|
29
|
+
*/
|
|
30
|
+
import { describeError, redactedHealthDetail } from "./health-detail.js";
|
|
31
|
+
export const SYNC_FAILURE_REASONS = [
|
|
32
|
+
/** The request never got an answer: DNS, TLS, a reset, an offline machine. */
|
|
33
|
+
"ingest_transport_error",
|
|
34
|
+
/** The ingest route answered and refused the envelope. */
|
|
35
|
+
"ingest_rejected",
|
|
36
|
+
/**
|
|
37
|
+
* Something answered 2xx, but not the ingest route's 202. A proxy, a captive
|
|
38
|
+
* portal, or a gateway that swallowed the real response.
|
|
39
|
+
*/
|
|
40
|
+
"ingest_receipt_not_202",
|
|
41
|
+
/** A 202 from the route whose receipt does not match what was submitted. */
|
|
42
|
+
"ingest_receipt_incomplete",
|
|
43
|
+
/** The sync threw before or after delivery — filesystem, cursor, spool. */
|
|
44
|
+
"sync_failed_local",
|
|
45
|
+
];
|
|
46
|
+
export const SYNC_FAILURE_DETAIL_MAX_CHARS = 250;
|
|
47
|
+
/**
|
|
48
|
+
* A delivery failure that already knows its own bucket.
|
|
49
|
+
*
|
|
50
|
+
* `message` is `reason: detail` so the existing consumers that only ever print
|
|
51
|
+
* `error.message` — `cockpit status`, the health-receipt classifier — keep
|
|
52
|
+
* working and get strictly more than they had.
|
|
53
|
+
*/
|
|
54
|
+
export class SyncDeliveryError extends Error {
|
|
55
|
+
reason;
|
|
56
|
+
detail;
|
|
57
|
+
httpStatus;
|
|
58
|
+
constructor(reason, options = {}) {
|
|
59
|
+
const detail = boundedFailureDetail(options.detail);
|
|
60
|
+
super(composeFailureReason(reason, detail), options.cause === undefined ? undefined : { cause: options.cause });
|
|
61
|
+
this.name = "SyncDeliveryError";
|
|
62
|
+
this.reason = reason;
|
|
63
|
+
this.detail = detail;
|
|
64
|
+
this.httpStatus = options.httpStatus ?? null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Redacts, collapses and caps an observed cause.
|
|
69
|
+
*
|
|
70
|
+
* `redactedHealthDetail` is the boundary rule for anything that leaves this
|
|
71
|
+
* machine (secrets, absolute paths, the hostname, the account name); the cap
|
|
72
|
+
* here is tighter than the health receipt's because this string is also the
|
|
73
|
+
* spool row a person reads in one terminal line.
|
|
74
|
+
*/
|
|
75
|
+
export function boundedFailureDetail(value) {
|
|
76
|
+
if (!value)
|
|
77
|
+
return null;
|
|
78
|
+
const redacted = redactedHealthDetail(value);
|
|
79
|
+
if (!redacted)
|
|
80
|
+
return null;
|
|
81
|
+
return redacted.length > SYNC_FAILURE_DETAIL_MAX_CHARS
|
|
82
|
+
? `${redacted.slice(0, SYNC_FAILURE_DETAIL_MAX_CHARS - 1)}…`
|
|
83
|
+
: redacted;
|
|
84
|
+
}
|
|
85
|
+
/** The single string the spool row stores: the bucket, then the cause. */
|
|
86
|
+
export function composeFailureReason(reason, detail) {
|
|
87
|
+
return detail ? `${reason}: ${detail}` : reason;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Turns anything thrown inside a sync into a bucket plus a cause.
|
|
91
|
+
*
|
|
92
|
+
* A `SyncDeliveryError` already decided; everything else is local — a
|
|
93
|
+
* filesystem error writing the cursor, a spool write, a throw from an adapter —
|
|
94
|
+
* and is described by its error name, code and syscall rather than by its
|
|
95
|
+
* message, which is where the paths live.
|
|
96
|
+
*/
|
|
97
|
+
export function classifySyncFailure(error) {
|
|
98
|
+
if (error instanceof SyncDeliveryError) {
|
|
99
|
+
return {
|
|
100
|
+
reason: error.reason,
|
|
101
|
+
detail: error.detail,
|
|
102
|
+
failure_reason: composeFailureReason(error.reason, error.detail),
|
|
103
|
+
http_status: error.httpStatus,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
const described = describeError(error);
|
|
107
|
+
const parts = [
|
|
108
|
+
described.error_name,
|
|
109
|
+
described.error_code,
|
|
110
|
+
described.error_syscall,
|
|
111
|
+
described.error_detail,
|
|
112
|
+
described.cause_code ?? described.cause_name,
|
|
113
|
+
].filter((part) => Boolean(part));
|
|
114
|
+
const detail = boundedFailureDetail(parts.join(" "));
|
|
115
|
+
return {
|
|
116
|
+
reason: "sync_failed_local",
|
|
117
|
+
detail,
|
|
118
|
+
failure_reason: composeFailureReason("sync_failed_local", detail),
|
|
119
|
+
http_status: null,
|
|
120
|
+
};
|
|
121
|
+
}
|
package/dist/upload-http.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* server sent back without letting a non-JSON body throw, and turn a normal
|
|
7
7
|
* dashboard URL into one that can be concatenated with a path.
|
|
8
8
|
*/
|
|
9
|
+
import { boundedFailureDetail } from "./upload-failure-reason.js";
|
|
9
10
|
/**
|
|
10
11
|
* Reads a reply body as JSON, and never throws doing it.
|
|
11
12
|
*
|
|
@@ -34,6 +35,33 @@ export async function readResponseJson(response) {
|
|
|
34
35
|
return { message: text };
|
|
35
36
|
}
|
|
36
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* The server's own reason for refusing, as something safe to log.
|
|
40
|
+
*
|
|
41
|
+
* The dashboard's error envelopes answer `{ code, message, reason? }`. A
|
|
42
|
+
* `reason`/`code` that already looks like a label travels verbatim — that is
|
|
43
|
+
* the bucket the server chose, and reproducing it is how "storage rejected 116
|
|
44
|
+
* MB" reaches an operator instead of "upload failed" (BLI-2528). Anything else
|
|
45
|
+
* falls back to the message, redacted and capped, because a proxy's error page
|
|
46
|
+
* is unbounded and a route's message can carry a path.
|
|
47
|
+
*
|
|
48
|
+
* `null` means the body said nothing usable — which is itself worth logging as
|
|
49
|
+
* `"none"`, since "the server gave no reason" and "we never looked" are
|
|
50
|
+
* different facts.
|
|
51
|
+
*/
|
|
52
|
+
export function serverFailureDetail(value) {
|
|
53
|
+
if (!value || typeof value !== "object")
|
|
54
|
+
return null;
|
|
55
|
+
const record = value;
|
|
56
|
+
for (const key of ["reason", "code"]) {
|
|
57
|
+
const candidate = record[key];
|
|
58
|
+
if (typeof candidate === "string" && /^[a-z0-9_:.-]{1,80}$/iu.test(candidate)) {
|
|
59
|
+
return candidate;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const message = record["message"] ?? record["error"];
|
|
63
|
+
return typeof message === "string" ? boundedFailureDetail(message) : null;
|
|
64
|
+
}
|
|
37
65
|
/** The server's own words when it supplied any, otherwise our fallback. */
|
|
38
66
|
export function responseErrorMessage(value, fallback) {
|
|
39
67
|
if (value && typeof value === "object") {
|
package/dist/upload.js
CHANGED
|
@@ -7,7 +7,8 @@ import { buildLocalAmbientEnvelope, LocalUploadBlockedError, } from "./upload-en
|
|
|
7
7
|
import { applyRawEvidenceUploadOutcomes, hasRetryableEvidenceGap, logEvidenceBackoffBypassed, logEvidenceHeldByBackoff, logPermanentlyRejectedEvidence, partitionHeldEvidenceFiles, permanentEvidenceFailureReason, persistDeliveryAttempts, retrySourcesForFailedSync, retryableEvidenceGapReason, summarizeRawEvidenceDelivery, } from "./upload-evidence-delivery.js";
|
|
8
8
|
import { reportAgentImageArtifacts } from "./upload-agent-artifacts.js";
|
|
9
9
|
import { describeError } from "./health-detail.js";
|
|
10
|
-
import { isNonEmptyString, readResponseJson,
|
|
10
|
+
import { isNonEmptyString, readResponseJson, serverFailureDetail, } from "./upload-http.js";
|
|
11
|
+
import { SyncDeliveryError, classifySyncFailure, } from "./upload-failure-reason.js";
|
|
11
12
|
export { LocalUploadBlockedError, buildLocalAmbientEnvelope, } from "./upload-envelope.js";
|
|
12
13
|
export { partitionHeldEvidenceFiles } from "./upload-evidence-delivery.js";
|
|
13
14
|
export { reportAgentImageArtifacts } from "./upload-agent-artifacts.js";
|
|
@@ -99,6 +100,17 @@ export async function syncLocalAmbientEnvelope(options = {}) {
|
|
|
99
100
|
uploadOutcomes,
|
|
100
101
|
attemptedAt,
|
|
101
102
|
});
|
|
103
|
+
// The success branch logs too. A line that only fires on failure cannot
|
|
104
|
+
// answer "did anything land at all today?", which is the question that
|
|
105
|
+
// would have caught BLI-2528 in a week instead of 57 days.
|
|
106
|
+
console.error("[cockpit-sync] ingest accepted the envelope", JSON.stringify({
|
|
107
|
+
reason: "ingest_accepted",
|
|
108
|
+
http_status: response.status,
|
|
109
|
+
event_count: built.event_count,
|
|
110
|
+
raw_evidence_file_count: built.raw_evidence_upload_files.length,
|
|
111
|
+
uploaded_chunk_count: uploadedChunkCount,
|
|
112
|
+
failed_object_count: uploadOutcomes.filter((outcome) => outcome.upload_state === "upload_failed").length,
|
|
113
|
+
}));
|
|
102
114
|
return {
|
|
103
115
|
status: "uploaded",
|
|
104
116
|
dashboard_url: built.dashboard_url,
|
|
@@ -113,12 +125,26 @@ export async function syncLocalAmbientEnvelope(options = {}) {
|
|
|
113
125
|
};
|
|
114
126
|
}
|
|
115
127
|
catch (error) {
|
|
116
|
-
|
|
128
|
+
// The spool row used to store `error.message` verbatim, so every distinct
|
|
129
|
+
// server sentence became its own "reason" and nothing could count how many
|
|
130
|
+
// machines were failing the same way (BLI-3483). It now stores a label from
|
|
131
|
+
// a closed set plus a bounded, redacted cause.
|
|
132
|
+
const classified = classifySyncFailure(error);
|
|
117
133
|
// A committed content-addressed object may be shared by concurrent syncs.
|
|
118
134
|
// Never delete it from this error path: an ingest retry can reuse it, while
|
|
119
135
|
// client-side cleanup cannot prove exclusive ownership without racing a
|
|
120
136
|
// second sync that is about to index the same object.
|
|
121
|
-
const spooledFailureReason =
|
|
137
|
+
const spooledFailureReason = classified.failure_reason;
|
|
138
|
+
console.error("[cockpit-sync] sync spooled for retry", JSON.stringify({
|
|
139
|
+
reason: classified.reason,
|
|
140
|
+
...(classified.http_status === null
|
|
141
|
+
? {}
|
|
142
|
+
: { http_status: classified.http_status }),
|
|
143
|
+
...(classified.detail ? { detail: classified.detail } : {}),
|
|
144
|
+
event_count: built.event_count,
|
|
145
|
+
raw_evidence_file_count: built.raw_evidence_upload_files.length,
|
|
146
|
+
uploaded_chunk_count: uploadedChunkCount,
|
|
147
|
+
}));
|
|
122
148
|
const entry = await recordUploadFailure(paths, {
|
|
123
149
|
last_attempt_at: attemptedAt,
|
|
124
150
|
dashboard_url: built.dashboard_url,
|
|
@@ -182,17 +208,50 @@ async function buildEnvelopeOrRecordBlocker(paths, options, cursorObjects, attem
|
|
|
182
208
|
* row counts, which is what the receipt check demands.
|
|
183
209
|
*/
|
|
184
210
|
async function postEnvelopeToIngest(options) {
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
"
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
211
|
+
let response;
|
|
212
|
+
try {
|
|
213
|
+
response = await options.fetchImpl(`${options.built.dashboard_url}/api/ambient/ingest`, {
|
|
214
|
+
method: "POST",
|
|
215
|
+
headers: {
|
|
216
|
+
"Authorization": `Bearer ${options.built.device_token}`,
|
|
217
|
+
"Content-Type": "application/json",
|
|
218
|
+
},
|
|
219
|
+
body: JSON.stringify(options.envelope),
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
catch (error) {
|
|
223
|
+
// No status came back at all, so no server-side reason exists to read. The
|
|
224
|
+
// error's own name and code (`TypeError` / `ENOTFOUND` / `ECONNREFUSED`)
|
|
225
|
+
// are the whole answer, and they are what separates "this machine is
|
|
226
|
+
// offline" from "the dashboard refused us" — two words that used to be the
|
|
227
|
+
// same spool row.
|
|
228
|
+
const described = describeError(error);
|
|
229
|
+
console.error("[cockpit-sync] ingest request failed before a status came back", JSON.stringify({ reason: "ingest_transport_error", ...described }));
|
|
230
|
+
throw new SyncDeliveryError("ingest_transport_error", {
|
|
231
|
+
detail: [
|
|
232
|
+
described.error_name,
|
|
233
|
+
described.error_code,
|
|
234
|
+
described.cause_code ?? described.cause_name,
|
|
235
|
+
described.error_detail,
|
|
236
|
+
]
|
|
237
|
+
.filter(Boolean)
|
|
238
|
+
.join(" "),
|
|
239
|
+
cause: error,
|
|
240
|
+
});
|
|
241
|
+
}
|
|
193
242
|
const responseBody = await readResponseJson(response);
|
|
194
243
|
if (!response.ok) {
|
|
195
|
-
|
|
244
|
+
const serverReason = serverFailureDetail(responseBody);
|
|
245
|
+
console.error("[cockpit-sync] ingest refused the envelope", JSON.stringify({
|
|
246
|
+
reason: "ingest_rejected",
|
|
247
|
+
http_status: response.status,
|
|
248
|
+
server_reason: serverReason ?? "none",
|
|
249
|
+
event_count: options.envelope.events.length,
|
|
250
|
+
}));
|
|
251
|
+
throw new SyncDeliveryError("ingest_rejected", {
|
|
252
|
+
httpStatus: response.status,
|
|
253
|
+
detail: `http ${response.status}; server reason ${serverReason ?? "none"}`,
|
|
254
|
+
});
|
|
196
255
|
}
|
|
197
256
|
assertAmbientIngestReceipt(response, responseBody, options.envelope);
|
|
198
257
|
return response;
|
|
@@ -275,11 +334,26 @@ async function recordIngestedSyncOutcome(options) {
|
|
|
275
334
|
*
|
|
276
335
|
* Anything short of a 202 whose counts match the envelope's own totals is
|
|
277
336
|
* treated as no receipt at all — a proxy's 200, a truncated body, or a
|
|
278
|
-
* dashboard that accepted fewer rows than were sent all land here.
|
|
337
|
+
* dashboard that accepted fewer rows than were sent all land here. They used to
|
|
338
|
+
* land here under the SAME sentence, which is the BLI-3483 complaint: a
|
|
339
|
+
* network appliance answering on the dashboard's behalf and a dashboard that
|
|
340
|
+
* genuinely persisted three of four rows are opposite repairs. They are now two
|
|
341
|
+
* reasons, and the second one names the field that disagreed and by how much.
|
|
279
342
|
*/
|
|
280
343
|
function assertAmbientIngestReceipt(response, value, envelope) {
|
|
281
344
|
if (response.status !== 202 || !value || typeof value !== "object") {
|
|
282
|
-
|
|
345
|
+
const detail = response.status !== 202
|
|
346
|
+
? `http ${response.status}; the ingest route answers 202, so something in front of it replied`
|
|
347
|
+
: "202 with a body that is not an object";
|
|
348
|
+
console.error("[cockpit-sync] ingest answered without a durable receipt", JSON.stringify({
|
|
349
|
+
reason: "ingest_receipt_not_202",
|
|
350
|
+
http_status: response.status,
|
|
351
|
+
body_is_object: Boolean(value) && typeof value === "object",
|
|
352
|
+
}));
|
|
353
|
+
throw new SyncDeliveryError("ingest_receipt_not_202", {
|
|
354
|
+
httpStatus: response.status,
|
|
355
|
+
detail,
|
|
356
|
+
});
|
|
283
357
|
}
|
|
284
358
|
const record = value;
|
|
285
359
|
const ingest = record["ingest"] && typeof record["ingest"] === "object"
|
|
@@ -288,12 +362,39 @@ function assertAmbientIngestReceipt(response, value, envelope) {
|
|
|
288
362
|
const expectedFactCount = envelope.events.length;
|
|
289
363
|
const expectedRiskFlagCount = envelope.events.reduce((total, event) => total + event.risk_flags.length, 0);
|
|
290
364
|
const expectedEvidenceRefCount = envelope.events.reduce((total, event) => total + event.raw_evidence_pointers.length, 0);
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
365
|
+
const mismatches = [];
|
|
366
|
+
if (record["ok"] !== true)
|
|
367
|
+
mismatches.push("ok not true");
|
|
368
|
+
if (!ingest)
|
|
369
|
+
mismatches.push("no ingest block");
|
|
370
|
+
if (ingest && !isNonEmptyString(ingest["work_session_id"])) {
|
|
371
|
+
mismatches.push("blank work_session_id");
|
|
372
|
+
}
|
|
373
|
+
if (ingest) {
|
|
374
|
+
mismatches.push(...countMismatch("fact_count", ingest["fact_count"], expectedFactCount), ...countMismatch("risk_flag_count", ingest["risk_flag_count"], expectedRiskFlagCount), ...countMismatch("evidence_ref_count", ingest["evidence_ref_count"], expectedEvidenceRefCount));
|
|
298
375
|
}
|
|
376
|
+
if (mismatches.length === 0)
|
|
377
|
+
return;
|
|
378
|
+
console.error("[cockpit-sync] ingest receipt does not match what was submitted", JSON.stringify({
|
|
379
|
+
reason: "ingest_receipt_incomplete",
|
|
380
|
+
http_status: response.status,
|
|
381
|
+
mismatch_count: mismatches.length,
|
|
382
|
+
mismatches,
|
|
383
|
+
}));
|
|
384
|
+
throw new SyncDeliveryError("ingest_receipt_incomplete", {
|
|
385
|
+
httpStatus: response.status,
|
|
386
|
+
detail: mismatches.join("; "),
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
/**
|
|
390
|
+
* `submitted 4, receipt 3` — the sentence that tells partial persistence from a
|
|
391
|
+
* receipt that never carried the field at all. Counts are numbers this
|
|
392
|
+
* collector computed and numbers the server echoed; neither is content.
|
|
393
|
+
*/
|
|
394
|
+
function countMismatch(field, received, expected) {
|
|
395
|
+
if (received === expected)
|
|
396
|
+
return [];
|
|
397
|
+
return [
|
|
398
|
+
`${field} submitted ${expected}, receipt ${typeof received === "number" ? received : "absent"}`,
|
|
399
|
+
];
|
|
299
400
|
}
|