@indigoai-us/hq-cli 5.113.1 → 5.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/dist/command-catalog.generated.d.ts +28 -0
- package/dist/command-catalog.generated.js +37 -0
- package/dist/commands/bot-companies.d.ts +40 -0
- package/dist/commands/bot-companies.js +64 -0
- package/dist/commands/bot.d.ts +33 -1
- package/dist/commands/bot.js +155 -7
- package/dist/commands/onboard-bot-membership.d.ts +32 -0
- package/dist/commands/onboard-bot-membership.js +40 -0
- package/dist/commands/onboard.js +10 -0
- package/dist/lib/bot/api.d.ts +6 -0
- package/dist/lib/bot/api.js +6 -1
- package/dist/lib/bot/config.d.ts +35 -0
- package/dist/lib/bot/config.js +69 -2
- package/dist/lib/bot/inflight.d.ts +18 -7
- package/dist/lib/bot/inflight.js +57 -18
- package/dist/lib/bot/prompt.d.ts +13 -1
- package/dist/lib/bot/prompt.js +37 -12
- package/dist/lib/bot/run.d.ts +48 -5
- package/dist/lib/bot/run.js +231 -47
- package/dist/lib/bot/runtime-sign-in.d.ts +24 -0
- package/dist/lib/bot/runtime-sign-in.js +31 -0
- package/dist/lib/bot/status.d.ts +6 -0
- package/dist/utils/feedback-log-bundle.d.ts +61 -11
- package/dist/utils/feedback-log-bundle.js +209 -29
- package/package.json +1 -1
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* body above 64 KiB, and `diagnostics` is written into a DynamoDB item, which
|
|
7
7
|
* caps at 400 KB. Neither can be tuned into carrying a real log history, so the
|
|
8
8
|
* inline blob is deliberately a ~40 KiB summary of 16 KiB tails. This module
|
|
9
|
-
* produces the other half: a
|
|
9
|
+
* produces the other half: a zipped bundle uploaded direct to S3, carrying the
|
|
10
10
|
* files whole rather than in tail-sized slivers.
|
|
11
11
|
*
|
|
12
12
|
* Three properties are load-bearing and none may be traded away:
|
|
@@ -25,17 +25,33 @@
|
|
|
25
25
|
* 3. BOUNDED MEMORY. 50 MB compressed is roughly a gigabyte of raw log text
|
|
26
26
|
* at the ratio these files compress at. Nothing is ever fully materialised:
|
|
27
27
|
* files are read in slices, redacted a line at a time, and streamed into
|
|
28
|
-
*
|
|
28
|
+
* the deflate stream, with only the compressed output retained.
|
|
29
29
|
*
|
|
30
|
-
*
|
|
30
|
+
* WHY ZIP AND NOT GZIP. This bundle is attached to a Slack thread by the
|
|
31
|
+
* hq-pro feedback worker, and Slack's `files.completeUploadExternal` refuses a
|
|
32
|
+
* gzip member from this app at ANY size — a 52-byte gzip was rejected with
|
|
33
|
+
* `internal_error`, while the same bytes as zip were accepted. That was the
|
|
34
|
+
* root cause of roughly 110 consecutive silent attachment failures. The worker
|
|
35
|
+
* carries a gz-to-zip repack so older CLIs still get an attachable bundle;
|
|
36
|
+
* emitting zip here means the common path never pays for that unpack, and the
|
|
37
|
+
* archive is the format the destination actually accepts.
|
|
38
|
+
*
|
|
39
|
+
* A single deflate member wrapped in a ZIP32 container, one entry. The stream
|
|
40
|
+
* is raw deflate so the container's own CRC and sizes are the only integrity
|
|
41
|
+
* record; crc32 is folded in incrementally as lines are written, and the local
|
|
42
|
+
* header, central directory, and EOCD are assembled once at finish. The size
|
|
43
|
+
* budget is charged on the container, not the member, so the ~98 bytes of
|
|
44
|
+
* framing can never be what pushes an upload past the server's ceiling.
|
|
45
|
+
*
|
|
46
|
+
* Output format inside the entry — NDJSON, one JSON record per line:
|
|
31
47
|
* {"kind":"manifest","version":1,...} exactly one, first
|
|
32
48
|
* {"kind":"file","name":"logs/hq-sync.log",...} one per file
|
|
33
49
|
* {"kind":"chunk","name":"logs/hq-sync.log","seq":0,...} many per file
|
|
34
50
|
* {"kind":"summary","fileCount":12,...} exactly one, last
|
|
35
51
|
*
|
|
36
|
-
* NDJSON rather than tar
|
|
37
|
-
* bundle is still parseable line-by-line up to the cut, and so the
|
|
38
|
-
* carry the same redaction metadata the inline blob already reports.
|
|
52
|
+
* NDJSON inside the entry rather than a tar of separate members, so a
|
|
53
|
+
* truncated bundle is still parseable line-by-line up to the cut, and so the
|
|
54
|
+
* records carry the same redaction metadata the inline blob already reports.
|
|
39
55
|
*/
|
|
40
56
|
import { type LogCandidate } from "./feedback-logs.js";
|
|
41
57
|
import { vaultApiFetch } from "./vault-api.js";
|
|
@@ -45,10 +61,17 @@ import { vaultApiFetch } from "./vault-api.js";
|
|
|
45
61
|
* stay in step or the CLI will build bundles the server will not accept.
|
|
46
62
|
*/
|
|
47
63
|
export declare const LOG_BUNDLE_MAX_BYTES: number;
|
|
64
|
+
/**
|
|
65
|
+
* What the presign request declares, and what the server must answer with.
|
|
66
|
+
* Mirrors `LOG_BUNDLE_FORMAT_ZIP` / `LOG_BUNDLE_ZIP_CONTENT_TYPE` in the hq-pro
|
|
67
|
+
* handler `feedback-log-bundles.ts`.
|
|
68
|
+
*/
|
|
69
|
+
export declare const LOG_BUNDLE_FORMAT = "zip";
|
|
70
|
+
export declare const LOG_BUNDLE_CONTENT_TYPE = "application/zip";
|
|
48
71
|
/**
|
|
49
72
|
* Headroom between the size we stop feeding at and the hard cap.
|
|
50
73
|
*
|
|
51
|
-
*
|
|
74
|
+
* deflate reports compressed bytes only as its internal buffer flushes, so the
|
|
52
75
|
* running total lags the bytes actually consumed. The lag is bounded by that
|
|
53
76
|
* buffer (tens of KiB); a 1 MiB margin covers it with three orders of magnitude
|
|
54
77
|
* to spare, and the final size is asserted against the real cap regardless.
|
|
@@ -62,9 +85,9 @@ export declare const LOG_BUNDLE_SAFETY_MARGIN_BYTES: number;
|
|
|
62
85
|
*/
|
|
63
86
|
export declare const LOG_BUNDLE_MAX_FILE_RAW_BYTES: number;
|
|
64
87
|
export interface LogBundleResult {
|
|
65
|
-
/** The
|
|
66
|
-
|
|
67
|
-
/** `
|
|
88
|
+
/** The zipped NDJSON bytes, ready to PUT. */
|
|
89
|
+
bytes: Buffer;
|
|
90
|
+
/** `bytes.byteLength` — what the presign request must declare. */
|
|
68
91
|
sizeBytes: number;
|
|
69
92
|
/** How many files contributed at least one chunk. */
|
|
70
93
|
fileCount: number;
|
|
@@ -95,6 +118,12 @@ export interface BuildLogBundleOptions {
|
|
|
95
118
|
homeDir?: string;
|
|
96
119
|
/** Override the per-file raw ceiling (tests). */
|
|
97
120
|
maxFileRawBytes?: number;
|
|
121
|
+
/**
|
|
122
|
+
* Uncompressed ceiling for the zip entry. Clamped to what ZIP32 can describe,
|
|
123
|
+
* so a caller can lower this but never raise it past the format's limit.
|
|
124
|
+
* Exists so the ceiling is exercisable without a test that allocates 4 GiB.
|
|
125
|
+
*/
|
|
126
|
+
maxEntryBytes?: number;
|
|
98
127
|
}
|
|
99
128
|
/**
|
|
100
129
|
* Order candidates so that, when the cap truncates collection, what survives is
|
|
@@ -107,6 +136,27 @@ export interface BuildLogBundleOptions {
|
|
|
107
136
|
* crowd out a stale claim in the inline collector.
|
|
108
137
|
*/
|
|
109
138
|
export declare function orderBundleCandidates(hqDir: string): LogCandidate[];
|
|
139
|
+
/**
|
|
140
|
+
* Name of the single entry inside the zip. The `.ndjson` extension is what
|
|
141
|
+
* tells a triager (and `jq`) what is inside once it is unzipped.
|
|
142
|
+
*/
|
|
143
|
+
export declare const LOG_BUNDLE_ENTRY_NAME = "hq-logs.ndjson";
|
|
144
|
+
/** The line collection actually stops at. Never above what ZIP32 can describe. */
|
|
145
|
+
export declare const ZIP32_UNCOMPRESSED_STOP_BYTES: number;
|
|
146
|
+
/**
|
|
147
|
+
* Resolve the entry ceiling a caller asked for. A caller may lower it; nothing
|
|
148
|
+
* may raise it past what the container can describe, because a ceiling above
|
|
149
|
+
* the format's limit is not a ceiling at all.
|
|
150
|
+
*/
|
|
151
|
+
export declare function entryCeiling(requested?: number): number;
|
|
152
|
+
export declare function crc32Fallback(buf: Buffer, seed?: number): number;
|
|
153
|
+
/** Exported for the test that proves the fallback agrees with the native one. */
|
|
154
|
+
export declare function crc32(buf: Buffer, seed?: number): number;
|
|
155
|
+
/**
|
|
156
|
+
* Bytes of ZIP framing around one entry: local header, central directory
|
|
157
|
+
* header, end-of-central-directory. The name is stored twice.
|
|
158
|
+
*/
|
|
159
|
+
export declare function zipOverheadBytes(entryName: string): number;
|
|
110
160
|
/**
|
|
111
161
|
* Read `absPath` in slices, redacting whole lines, invoking `onChunk` with
|
|
112
162
|
* roughly {@link CHUNK_TEXT_BYTES} of redacted text at a time.
|
|
@@ -120,7 +170,7 @@ export declare function streamRedactedFile(absPath: string, sizeBytes: number, m
|
|
|
120
170
|
stopped: boolean;
|
|
121
171
|
}>;
|
|
122
172
|
/**
|
|
123
|
-
* Build a
|
|
173
|
+
* Build a zipped NDJSON bundle of the submitter's redacted `~/.hq` logs.
|
|
124
174
|
*
|
|
125
175
|
* Returns `undefined` when nothing eligible exists, so the caller can skip the
|
|
126
176
|
* upload entirely. Never throws: a bug report must not fail over its own
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* body above 64 KiB, and `diagnostics` is written into a DynamoDB item, which
|
|
7
7
|
* caps at 400 KB. Neither can be tuned into carrying a real log history, so the
|
|
8
8
|
* inline blob is deliberately a ~40 KiB summary of 16 KiB tails. This module
|
|
9
|
-
* produces the other half: a
|
|
9
|
+
* produces the other half: a zipped bundle uploaded direct to S3, carrying the
|
|
10
10
|
* files whole rather than in tail-sized slivers.
|
|
11
11
|
*
|
|
12
12
|
* Three properties are load-bearing and none may be traded away:
|
|
@@ -25,17 +25,33 @@
|
|
|
25
25
|
* 3. BOUNDED MEMORY. 50 MB compressed is roughly a gigabyte of raw log text
|
|
26
26
|
* at the ratio these files compress at. Nothing is ever fully materialised:
|
|
27
27
|
* files are read in slices, redacted a line at a time, and streamed into
|
|
28
|
-
*
|
|
28
|
+
* the deflate stream, with only the compressed output retained.
|
|
29
29
|
*
|
|
30
|
-
*
|
|
30
|
+
* WHY ZIP AND NOT GZIP. This bundle is attached to a Slack thread by the
|
|
31
|
+
* hq-pro feedback worker, and Slack's `files.completeUploadExternal` refuses a
|
|
32
|
+
* gzip member from this app at ANY size — a 52-byte gzip was rejected with
|
|
33
|
+
* `internal_error`, while the same bytes as zip were accepted. That was the
|
|
34
|
+
* root cause of roughly 110 consecutive silent attachment failures. The worker
|
|
35
|
+
* carries a gz-to-zip repack so older CLIs still get an attachable bundle;
|
|
36
|
+
* emitting zip here means the common path never pays for that unpack, and the
|
|
37
|
+
* archive is the format the destination actually accepts.
|
|
38
|
+
*
|
|
39
|
+
* A single deflate member wrapped in a ZIP32 container, one entry. The stream
|
|
40
|
+
* is raw deflate so the container's own CRC and sizes are the only integrity
|
|
41
|
+
* record; crc32 is folded in incrementally as lines are written, and the local
|
|
42
|
+
* header, central directory, and EOCD are assembled once at finish. The size
|
|
43
|
+
* budget is charged on the container, not the member, so the ~98 bytes of
|
|
44
|
+
* framing can never be what pushes an upload past the server's ceiling.
|
|
45
|
+
*
|
|
46
|
+
* Output format inside the entry — NDJSON, one JSON record per line:
|
|
31
47
|
* {"kind":"manifest","version":1,...} exactly one, first
|
|
32
48
|
* {"kind":"file","name":"logs/hq-sync.log",...} one per file
|
|
33
49
|
* {"kind":"chunk","name":"logs/hq-sync.log","seq":0,...} many per file
|
|
34
50
|
* {"kind":"summary","fileCount":12,...} exactly one, last
|
|
35
51
|
*
|
|
36
|
-
* NDJSON rather than tar
|
|
37
|
-
* bundle is still parseable line-by-line up to the cut, and so the
|
|
38
|
-
* carry the same redaction metadata the inline blob already reports.
|
|
52
|
+
* NDJSON inside the entry rather than a tar of separate members, so a
|
|
53
|
+
* truncated bundle is still parseable line-by-line up to the cut, and so the
|
|
54
|
+
* records carry the same redaction metadata the inline blob already reports.
|
|
39
55
|
*/
|
|
40
56
|
import * as fs from "node:fs";
|
|
41
57
|
import * as os from "node:os";
|
|
@@ -50,10 +66,17 @@ import { vaultApiFetch } from "./vault-api.js";
|
|
|
50
66
|
* stay in step or the CLI will build bundles the server will not accept.
|
|
51
67
|
*/
|
|
52
68
|
export const LOG_BUNDLE_MAX_BYTES = 50 * 1024 * 1024;
|
|
69
|
+
/**
|
|
70
|
+
* What the presign request declares, and what the server must answer with.
|
|
71
|
+
* Mirrors `LOG_BUNDLE_FORMAT_ZIP` / `LOG_BUNDLE_ZIP_CONTENT_TYPE` in the hq-pro
|
|
72
|
+
* handler `feedback-log-bundles.ts`.
|
|
73
|
+
*/
|
|
74
|
+
export const LOG_BUNDLE_FORMAT = "zip";
|
|
75
|
+
export const LOG_BUNDLE_CONTENT_TYPE = "application/zip";
|
|
53
76
|
/**
|
|
54
77
|
* Headroom between the size we stop feeding at and the hard cap.
|
|
55
78
|
*
|
|
56
|
-
*
|
|
79
|
+
* deflate reports compressed bytes only as its internal buffer flushes, so the
|
|
57
80
|
* running total lags the bytes actually consumed. The lag is bounded by that
|
|
58
81
|
* buffer (tens of KiB); a 1 MiB margin covers it with three orders of magnitude
|
|
59
82
|
* to spare, and the final size is asserted against the real cap regardless.
|
|
@@ -132,24 +155,171 @@ export function orderBundleCandidates(hqDir) {
|
|
|
132
155
|
const logs = discoverLogFiles(hqDir).sort((a, b) => b.modifiedMs - a.modifiedMs || a.name.localeCompare(b.name));
|
|
133
156
|
return [...state, ...logs];
|
|
134
157
|
}
|
|
135
|
-
|
|
136
|
-
|
|
158
|
+
/**
|
|
159
|
+
* Name of the single entry inside the zip. The `.ndjson` extension is what
|
|
160
|
+
* tells a triager (and `jq`) what is inside once it is unzipped.
|
|
161
|
+
*/
|
|
162
|
+
export const LOG_BUNDLE_ENTRY_NAME = "hq-logs.ndjson";
|
|
163
|
+
/**
|
|
164
|
+
* ZIP32 stores the uncompressed size in 32 bits, so a bundle whose contents
|
|
165
|
+
* exceed 4 GiB cannot be described by the container at all.
|
|
166
|
+
*
|
|
167
|
+
* The compressed cap does not prevent this. Log text compresses at roughly 24x
|
|
168
|
+
* in practice but there is no upper bound on the ratio — repetitive logs go
|
|
169
|
+
* far higher — and discovery admits an unbounded number of files at up to
|
|
170
|
+
* {@link LOG_BUNDLE_MAX_FILE_RAW_BYTES} each. Collection stops at the line
|
|
171
|
+
* below and reports `truncated`, which is what the caller already handles;
|
|
172
|
+
* without it `writeUInt32LE` throws at finish and the outer catch discards the
|
|
173
|
+
* whole bundle, so the installation with the most logs would send none.
|
|
174
|
+
*/
|
|
175
|
+
const ZIP32_MAX_UNCOMPRESSED_BYTES = 0xffffffff;
|
|
176
|
+
/**
|
|
177
|
+
* Room left below the ZIP32 ceiling for the chunk in flight plus the closing
|
|
178
|
+
* summary record. A chunk is written whole before the budget is re-checked.
|
|
179
|
+
*/
|
|
180
|
+
const ZIP32_UNCOMPRESSED_MARGIN_BYTES = 1024 * 1024;
|
|
181
|
+
/** The line collection actually stops at. Never above what ZIP32 can describe. */
|
|
182
|
+
export const ZIP32_UNCOMPRESSED_STOP_BYTES = ZIP32_MAX_UNCOMPRESSED_BYTES - ZIP32_UNCOMPRESSED_MARGIN_BYTES;
|
|
183
|
+
/**
|
|
184
|
+
* Resolve the entry ceiling a caller asked for. A caller may lower it; nothing
|
|
185
|
+
* may raise it past what the container can describe, because a ceiling above
|
|
186
|
+
* the format's limit is not a ceiling at all.
|
|
187
|
+
*/
|
|
188
|
+
export function entryCeiling(requested) {
|
|
189
|
+
if (requested === undefined || !Number.isFinite(requested)) {
|
|
190
|
+
return ZIP32_UNCOMPRESSED_STOP_BYTES;
|
|
191
|
+
}
|
|
192
|
+
return Math.max(0, Math.min(requested, ZIP32_UNCOMPRESSED_STOP_BYTES));
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* crc32 with a fallback, because `zlib.crc32` is newer than this package's
|
|
196
|
+
* floor: it landed in Node 22.2.0 and was backported only to 20.15.0, while
|
|
197
|
+
* `engines.node` is `>=20.0.0`. On Node 20.0-20.14 and 21.x the native call is
|
|
198
|
+
* undefined, the first sink write throws, and `buildLogBundle` catches it and
|
|
199
|
+
* returns undefined — every report on those runtimes silently loses its log
|
|
200
|
+
* bundle. The table is built once, on first use, and only when needed.
|
|
201
|
+
*/
|
|
202
|
+
let crcTable = null;
|
|
203
|
+
export function crc32Fallback(buf, seed = 0) {
|
|
204
|
+
if (!crcTable) {
|
|
205
|
+
crcTable = new Uint32Array(256);
|
|
206
|
+
for (let i = 0; i < 256; i++) {
|
|
207
|
+
let c = i;
|
|
208
|
+
for (let k = 0; k < 8; k++)
|
|
209
|
+
c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
|
210
|
+
crcTable[i] = c >>> 0;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
let crc = ~seed >>> 0;
|
|
214
|
+
for (let i = 0; i < buf.length; i++) {
|
|
215
|
+
crc = (crcTable[(crc ^ buf[i]) & 0xff] ^ (crc >>> 8)) >>> 0;
|
|
216
|
+
}
|
|
217
|
+
return (~crc >>> 0) >>> 0;
|
|
218
|
+
}
|
|
219
|
+
/** Exported for the test that proves the fallback agrees with the native one. */
|
|
220
|
+
export function crc32(buf, seed = 0) {
|
|
221
|
+
const native = zlib.crc32;
|
|
222
|
+
return typeof native === "function" ? native(buf, seed) : crc32Fallback(buf, seed);
|
|
223
|
+
}
|
|
224
|
+
/** Local file header, central directory header, and EOCD signatures. */
|
|
225
|
+
const ZIP_LOCAL_SIG = 0x04034b50;
|
|
226
|
+
const ZIP_CENTRAL_SIG = 0x02014b50;
|
|
227
|
+
const ZIP_EOCD_SIG = 0x06054b50;
|
|
228
|
+
/** Deflate, no data descriptor: sizes and CRC are known before a header is written. */
|
|
229
|
+
const ZIP_METHOD_DEFLATE = 8;
|
|
230
|
+
const ZIP_VERSION = 20;
|
|
231
|
+
/**
|
|
232
|
+
* A fixed MS-DOS timestamp, so the same logs produce the same bytes.
|
|
233
|
+
*
|
|
234
|
+
* Deliberately not the wall clock: a bundle's timing lives in the manifest
|
|
235
|
+
* record, which is redacted content the recipient can read, rather than in
|
|
236
|
+
* archive metadata that no tool here surfaces. 0x0021 is 1980-01-01, the
|
|
237
|
+
* earliest the format can represent.
|
|
238
|
+
*/
|
|
239
|
+
const ZIP_DOS_TIME = 0;
|
|
240
|
+
const ZIP_DOS_DATE = 0x0021;
|
|
241
|
+
/**
|
|
242
|
+
* Bytes of ZIP framing around one entry: local header, central directory
|
|
243
|
+
* header, end-of-central-directory. The name is stored twice.
|
|
244
|
+
*/
|
|
245
|
+
export function zipOverheadBytes(entryName) {
|
|
246
|
+
return 30 + 46 + 22 + Buffer.byteLength(entryName, "utf8") * 2;
|
|
247
|
+
}
|
|
248
|
+
function createZipSink(entryName = LOG_BUNDLE_ENTRY_NAME, maxUncompressed = ZIP32_UNCOMPRESSED_STOP_BYTES) {
|
|
249
|
+
const name = Buffer.from(entryName, "utf8");
|
|
250
|
+
const deflate = zlib.createDeflateRaw({ level: 9 });
|
|
137
251
|
const parts = [];
|
|
252
|
+
const overhead = zipOverheadBytes(entryName);
|
|
138
253
|
let compressed = 0;
|
|
254
|
+
let uncompressed = 0;
|
|
255
|
+
let crc = 0;
|
|
139
256
|
let failure = null;
|
|
140
|
-
|
|
257
|
+
deflate.on("data", (chunk) => {
|
|
141
258
|
parts.push(chunk);
|
|
142
259
|
compressed += chunk.byteLength;
|
|
143
260
|
});
|
|
144
|
-
|
|
261
|
+
deflate.on("error", (err) => {
|
|
145
262
|
failure = err;
|
|
146
263
|
});
|
|
264
|
+
function container(body) {
|
|
265
|
+
const local = Buffer.alloc(30);
|
|
266
|
+
local.writeUInt32LE(ZIP_LOCAL_SIG, 0);
|
|
267
|
+
local.writeUInt16LE(ZIP_VERSION, 4);
|
|
268
|
+
local.writeUInt16LE(0, 6); // flags
|
|
269
|
+
local.writeUInt16LE(ZIP_METHOD_DEFLATE, 8);
|
|
270
|
+
local.writeUInt16LE(ZIP_DOS_TIME, 10);
|
|
271
|
+
local.writeUInt16LE(ZIP_DOS_DATE, 12);
|
|
272
|
+
local.writeUInt32LE(crc >>> 0, 14);
|
|
273
|
+
local.writeUInt32LE(body.byteLength, 18);
|
|
274
|
+
local.writeUInt32LE(uncompressed, 22);
|
|
275
|
+
local.writeUInt16LE(name.byteLength, 26);
|
|
276
|
+
local.writeUInt16LE(0, 28); // extra length
|
|
277
|
+
const central = Buffer.alloc(46);
|
|
278
|
+
central.writeUInt32LE(ZIP_CENTRAL_SIG, 0);
|
|
279
|
+
central.writeUInt16LE(ZIP_VERSION, 4); // version made by
|
|
280
|
+
central.writeUInt16LE(ZIP_VERSION, 6); // version needed
|
|
281
|
+
central.writeUInt16LE(0, 8); // flags
|
|
282
|
+
central.writeUInt16LE(ZIP_METHOD_DEFLATE, 10);
|
|
283
|
+
central.writeUInt16LE(ZIP_DOS_TIME, 12);
|
|
284
|
+
central.writeUInt16LE(ZIP_DOS_DATE, 14);
|
|
285
|
+
central.writeUInt32LE(crc >>> 0, 16);
|
|
286
|
+
central.writeUInt32LE(body.byteLength, 20);
|
|
287
|
+
central.writeUInt32LE(uncompressed, 24);
|
|
288
|
+
central.writeUInt16LE(name.byteLength, 28);
|
|
289
|
+
central.writeUInt16LE(0, 30); // extra
|
|
290
|
+
central.writeUInt16LE(0, 32); // comment
|
|
291
|
+
central.writeUInt16LE(0, 34); // disk number start
|
|
292
|
+
central.writeUInt16LE(0, 36); // internal attrs
|
|
293
|
+
central.writeUInt32LE(0, 38); // external attrs
|
|
294
|
+
central.writeUInt32LE(0, 42); // local header offset
|
|
295
|
+
const centralSize = central.byteLength + name.byteLength;
|
|
296
|
+
const centralOffset = local.byteLength + name.byteLength + body.byteLength;
|
|
297
|
+
const eocd = Buffer.alloc(22);
|
|
298
|
+
eocd.writeUInt32LE(ZIP_EOCD_SIG, 0);
|
|
299
|
+
eocd.writeUInt16LE(0, 4); // this disk
|
|
300
|
+
eocd.writeUInt16LE(0, 6); // disk with central dir
|
|
301
|
+
eocd.writeUInt16LE(1, 8); // entries on this disk
|
|
302
|
+
eocd.writeUInt16LE(1, 10); // entries total
|
|
303
|
+
eocd.writeUInt32LE(centralSize, 12);
|
|
304
|
+
eocd.writeUInt32LE(centralOffset, 16);
|
|
305
|
+
eocd.writeUInt16LE(0, 20); // comment length
|
|
306
|
+
return Buffer.concat([local, name, body, central, name, eocd]);
|
|
307
|
+
}
|
|
147
308
|
return {
|
|
148
|
-
|
|
309
|
+
// The budget is charged on what will be PUT, framing included, so the
|
|
310
|
+
// container can never be what carries the upload over the server's cap.
|
|
311
|
+
compressedBytes: () => overhead + compressed,
|
|
312
|
+
atUncompressedLimit: () => uncompressed > maxUncompressed,
|
|
149
313
|
write: (line) => new Promise((resolve, reject) => {
|
|
150
314
|
if (failure)
|
|
151
315
|
return reject(failure);
|
|
152
|
-
|
|
316
|
+
const buf = Buffer.from(line, "utf8");
|
|
317
|
+
// Folded in as we go: the whole point of this module is that the
|
|
318
|
+
// uncompressed text is never materialised, so there is nothing to
|
|
319
|
+
// checksum at the end.
|
|
320
|
+
crc = crc32(buf, crc);
|
|
321
|
+
uncompressed += buf.byteLength;
|
|
322
|
+
deflate.write(buf, (err) => (err ? reject(err) : resolve()));
|
|
153
323
|
}),
|
|
154
324
|
// Z_SYNC_FLUSH costs a handful of bytes per boundary and a little ratio.
|
|
155
325
|
// That is the price of a budget that is enforced rather than estimated:
|
|
@@ -159,13 +329,13 @@ function createGzipSink() {
|
|
|
159
329
|
sync: () => new Promise((resolve, reject) => {
|
|
160
330
|
if (failure)
|
|
161
331
|
return reject(failure);
|
|
162
|
-
|
|
332
|
+
deflate.flush(zlib.constants.Z_SYNC_FLUSH, () => resolve());
|
|
163
333
|
}),
|
|
164
334
|
finish: () => new Promise((resolve, reject) => {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
335
|
+
deflate.on("end", () => failure ? reject(failure) : resolve(container(Buffer.concat(parts))));
|
|
336
|
+
deflate.on("error", reject);
|
|
337
|
+
deflate.end();
|
|
338
|
+
deflate.resume();
|
|
169
339
|
}),
|
|
170
340
|
};
|
|
171
341
|
}
|
|
@@ -268,7 +438,7 @@ export async function streamRedactedFile(absPath, sizeBytes, maxRawBytes, chunkB
|
|
|
268
438
|
}
|
|
269
439
|
}
|
|
270
440
|
/**
|
|
271
|
-
* Build a
|
|
441
|
+
* Build a zipped NDJSON bundle of the submitter's redacted `~/.hq` logs.
|
|
272
442
|
*
|
|
273
443
|
* Returns `undefined` when nothing eligible exists, so the caller can skip the
|
|
274
444
|
* upload entirely. Never throws: a bug report must not fail over its own
|
|
@@ -294,7 +464,7 @@ export async function buildLogBundle(opts = {}) {
|
|
|
294
464
|
if (candidates.length === 0)
|
|
295
465
|
return undefined;
|
|
296
466
|
try {
|
|
297
|
-
const sink =
|
|
467
|
+
const sink = createZipSink(LOG_BUNDLE_ENTRY_NAME, entryCeiling(opts.maxEntryBytes));
|
|
298
468
|
let fileCount = 0;
|
|
299
469
|
let rawBytes = 0;
|
|
300
470
|
let redactions = 0;
|
|
@@ -309,7 +479,7 @@ export async function buildLogBundle(opts = {}) {
|
|
|
309
479
|
}));
|
|
310
480
|
for (const candidate of candidates) {
|
|
311
481
|
await sink.sync();
|
|
312
|
-
if (sink.compressedBytes() > stopAt) {
|
|
482
|
+
if (sink.compressedBytes() > stopAt || sink.atUncompressedLimit()) {
|
|
313
483
|
truncated = true;
|
|
314
484
|
break;
|
|
315
485
|
}
|
|
@@ -334,7 +504,7 @@ export async function buildLogBundle(opts = {}) {
|
|
|
334
504
|
// Stop feeding once the compressed total reaches the stop line.
|
|
335
505
|
// The sync is what makes that total trustworthy.
|
|
336
506
|
await sink.sync();
|
|
337
|
-
return sink.compressedBytes() <= stopAt;
|
|
507
|
+
return sink.compressedBytes() <= stopAt && !sink.atUncompressedLimit();
|
|
338
508
|
});
|
|
339
509
|
stopped = result.stopped;
|
|
340
510
|
}
|
|
@@ -349,18 +519,18 @@ export async function buildLogBundle(opts = {}) {
|
|
|
349
519
|
}
|
|
350
520
|
}
|
|
351
521
|
await sink.write(record({ kind: "summary", fileCount, rawBytes, redactions, truncated }));
|
|
352
|
-
const
|
|
522
|
+
const bytes = await sink.finish();
|
|
353
523
|
// Nothing but a manifest and a summary is not worth uploading.
|
|
354
524
|
if (fileCount === 0)
|
|
355
525
|
return undefined;
|
|
356
526
|
// Final authority. The stop line plus margin should make this unreachable,
|
|
357
527
|
// but the server refuses to presign above the cap, so a bundle that
|
|
358
528
|
// overshot is useless and must not be offered.
|
|
359
|
-
if (
|
|
529
|
+
if (bytes.byteLength > maxBytes)
|
|
360
530
|
return undefined;
|
|
361
531
|
return {
|
|
362
|
-
|
|
363
|
-
sizeBytes:
|
|
532
|
+
bytes,
|
|
533
|
+
sizeBytes: bytes.byteLength,
|
|
364
534
|
fileCount,
|
|
365
535
|
truncated,
|
|
366
536
|
rawBytes,
|
|
@@ -395,7 +565,10 @@ export async function uploadLogBundle(opts) {
|
|
|
395
565
|
token: opts.token,
|
|
396
566
|
path: "/v1/feedback/logs/presign",
|
|
397
567
|
method: "POST",
|
|
398
|
-
|
|
568
|
+
// The server owns the key and the signed content type, so it has to be
|
|
569
|
+
// told which format is coming; without this it mints a `.ndjson.gz` slot
|
|
570
|
+
// signed `application/gzip` and the PUT would be a lie on both counts.
|
|
571
|
+
body: { sizeBytes: bundle.sizeBytes, format: LOG_BUNDLE_FORMAT },
|
|
399
572
|
});
|
|
400
573
|
if (!res.ok)
|
|
401
574
|
return undefined;
|
|
@@ -404,15 +577,22 @@ export async function uploadLogBundle(opts) {
|
|
|
404
577
|
if (!slot || typeof slot.key !== "string" || typeof slot.url !== "string") {
|
|
405
578
|
return undefined;
|
|
406
579
|
}
|
|
580
|
+
// A server too old to know about `format` answers with a gzip slot. Its
|
|
581
|
+
// signature binds `application/gzip` and its key claims `.gz`, and these
|
|
582
|
+
// bytes are neither. Skipping is the honest outcome and costs only the
|
|
583
|
+
// bundle — the submission still carries the inline logs, which is exactly
|
|
584
|
+
// what happens when bundles are disabled entirely.
|
|
585
|
+
if (slot.contentType !== LOG_BUNDLE_CONTENT_TYPE)
|
|
586
|
+
return undefined;
|
|
407
587
|
const doFetch = opts.fetchImpl ?? fetch;
|
|
408
588
|
const put = await doFetch(slot.url, {
|
|
409
589
|
method: "PUT",
|
|
410
590
|
headers: {
|
|
411
|
-
"Content-Type":
|
|
591
|
+
"Content-Type": slot.contentType,
|
|
412
592
|
// Must match the length bound into the signature, or S3 rejects it.
|
|
413
593
|
"Content-Length": String(bundle.sizeBytes),
|
|
414
594
|
},
|
|
415
|
-
body: new Uint8Array(bundle.
|
|
595
|
+
body: new Uint8Array(bundle.bytes),
|
|
416
596
|
});
|
|
417
597
|
if (!put.ok)
|
|
418
598
|
return undefined;
|