@indigoai-us/hq-cli 5.108.5 → 5.108.7

package/dist/utils/feedback-log-bundle.js

@@ -0,0 +1,430 @@
+/**
+ * Full-history log bundle for `hq feedback` — the out-of-band companion to the
+ * inline `diagnostics.logs` blob in `feedback-logs.ts`.
+ *
+ * Inline logs are capped twice over: the feedback endpoint rejects a request
+ * body above 64 KiB, and `diagnostics` is written into a DynamoDB item, which
+ * caps at 400 KB. Neither can be tuned into carrying a real log history, so the
+ * inline blob is deliberately a ~40 KiB summary of 16 KiB tails. This module
+ * produces the other half: a gzipped bundle uploaded direct to S3, carrying the
+ * files whole rather than in tail-sized slivers.
+ *
+ * Three properties are load-bearing and none may be traded away:
+ *
+ *   1. SAME ALLOWLIST. Discovery reuses `feedback-logs.ts` verbatim —
+ *      `discoverLogFiles`, `discoverStateFiles`, `discoverFlatDirFiles`,
+ *      `walkJsonFiles`. The bundle carries MORE OF the same files, never more
+ *      files. Credential material lives outside `~/.hq` (`~/.codex/auth.json`,
+ *      `~/.hq-agent/machine-creds.json`), so widening discovery here — not
+ *      raising the size — is what would turn this into an exfiltration bug.
+ *
+ *   2. SAME REDACTION. Every line goes through `redactLogText` before it is
+ *      compressed. Shipping a raw archive would be far simpler and would
+ *      silently undo the entire security model of the inline path.
+ *
+ *   3. BOUNDED MEMORY. 50 MB compressed is roughly a gigabyte of raw log text
+ *      at the ratio these files compress at. Nothing is ever fully materialised:
+ *      files are read in slices, redacted a line at a time, and streamed into
+ *      gzip, with only the compressed output retained.
+ *
+ * Output format — gzipped NDJSON, one JSON record per line:
+ *   {"kind":"manifest","version":1,...}                    exactly one, first
+ *   {"kind":"file","name":"logs/hq-sync.log",...}          one per file
+ *   {"kind":"chunk","name":"logs/hq-sync.log","seq":0,...} many per file
+ *   {"kind":"summary","fileCount":12,...}                  exactly one, last
+ *
+ * NDJSON rather than tar so there is no archive dependency, so a truncated
+ * bundle is still parseable line-by-line up to the cut, and so the records
+ * carry the same redaction metadata the inline blob already reports.
+ */
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import * as zlib from "node:zlib";
+import { StringDecoder } from "node:string_decoder";
+import { FEEDBACK_JOBS_MAX_DEPTH, FEEDBACK_JOBS_MAX_FILES, FEEDBACK_JOBS_DIRNAME, FEEDBACK_LOCKS_DIRNAME, FEEDBACK_LOCKS_MAX_FILES, discoverFlatDirFiles, discoverLogFiles, discoverStateFiles, redactLogText, walkJsonFiles, } from "./feedback-logs.js";
+import { vaultApiFetch } from "./vault-api.js";
+/**
+ * Compressed ceiling. Mirrors `MAX_LOG_BUNDLE_BYTES` in the hq-pro handler
+ * `feedback-log-bundles.ts`, which refuses to presign above it — the two must
+ * stay in step or the CLI will build bundles the server will not accept.
+ */
+export const LOG_BUNDLE_MAX_BYTES = 50 * 1024 * 1024;
+/**
+ * Headroom between the size we stop feeding at and the hard cap.
+ *
+ * gzip reports compressed bytes only as its internal buffer flushes, so the
+ * running total lags the bytes actually consumed. The lag is bounded by that
+ * buffer (tens of KiB); a 1 MiB margin covers it with three orders of magnitude
+ * to spare, and the final size is asserted against the real cap regardless.
+ */
+export const LOG_BUNDLE_SAFETY_MARGIN_BYTES = 1024 * 1024;
+/**
+ * Per-file raw ceiling. The desktop logger rotates at 32 MiB (hq-desktop-core
+ * `logfile.rs`), so this admits a full generation with headroom while stopping
+ * one pathological file from consuming the whole bundle. Files above it are
+ * read from the TAIL — the end of a log is what explains a failure.
+ */
+export const LOG_BUNDLE_MAX_FILE_RAW_BYTES = 64 * 1024 * 1024;
+/** Read slice size. Bounds peak memory; unrelated to the output chunking. */
+const READ_SLICE_BYTES = 4 * 1024 * 1024;
+/** Target raw text per emitted chunk record. Keeps NDJSON lines manageable. */
+const CHUNK_TEXT_BYTES = 256 * 1024;
+/**
+ * A chunk is written whole before the budget is re-checked, so the cap can only
+ * be honoured if one chunk is small relative to it. Scale the chunk down for
+ * small budgets; at the production 50 MB cap this returns CHUNK_TEXT_BYTES.
+ */
+function chunkBytesForCap(maxBytes) {
+    return Math.min(CHUNK_TEXT_BYTES, Math.max(4096, Math.floor(maxBytes / 8)));
+}
+/**
+ * Split point for a "line" that is longer than a whole chunk.
+ *
+ * Some logs are a single line: one-line JSON, a file with CRLF-only or no line
+ * terminators at all. Without this the reader would buffer the entire file as
+ * one line and emit it as one chunk, defeating both the memory bound and the
+ * size cap.
+ *
+ * The cut is taken at the last WHITESPACE inside the limit, because the
+ * secrets redaction looks for are unbroken tokens — a JWT, a key id, a bearer
+ * value. Cutting on whitespace means a split can never bisect one and let the
+ * halves slip past a pattern that would have matched the whole. Only a segment
+ * with no whitespace at all falls back to a hard cut, and that is already
+ * unparseable content rather than a log line.
+ */
+function splitOversizedLine(text, limit) {
+    const window = text.slice(0, limit);
+    const ws = Math.max(window.lastIndexOf(" "), window.lastIndexOf("\t"), window.lastIndexOf("\r"));
+    return ws > 0 ? ws + 1 : limit;
+}
+/**
+ * True when `text` opens a PEM block it does not close.
+ *
+ * Redaction runs per chunk rather than per line — redacting each line
+ * separately re-runs a dozen patterns hundreds of thousands of times over a
+ * large log, which is the dominant cost of building a bundle. The one pattern
+ * that genuinely spans lines is a PEM block, so a chunk boundary is held open
+ * rather than cutting one in half and letting the halves escape the
+ * whole-block pattern.
+ */
+function hasOpenPemBlock(text) {
+    const begins = text.match(/-----BEGIN [^-\n]+-----/g)?.length ?? 0;
+    const ends = text.match(/-----END [^-\n]+-----/g)?.length ?? 0;
+    return begins > ends;
+}
+/**
+ * Order candidates so that, when the cap truncates collection, what survives is
+ * what a triager reads first.
+ *
+ * State documents lead: they are tiny and answer questions a log cannot (which
+ * operation is claimed, what the sync cursor is). Log files follow, newest
+ * first. Reversing this would let one large old log crowd out every status
+ * file — the same class of ordering defect that let ordinary lock entries
+ * crowd out a stale claim in the inline collector.
+ */
+export function orderBundleCandidates(hqDir) {
+    const state = [
+        ...discoverStateFiles(hqDir),
+        ...discoverFlatDirFiles(hqDir, FEEDBACK_LOCKS_DIRNAME, FEEDBACK_LOCKS_MAX_FILES),
+        ...walkJsonFiles(hqDir, FEEDBACK_JOBS_DIRNAME, FEEDBACK_JOBS_MAX_DEPTH, FEEDBACK_JOBS_MAX_FILES),
+    ];
+    const logs = discoverLogFiles(hqDir).sort((a, b) => b.modifiedMs - a.modifiedMs || a.name.localeCompare(b.name));
+    return [...state, ...logs];
+}
+function createGzipSink() {
+    const gzip = zlib.createGzip({ level: 9 });
+    const parts = [];
+    let compressed = 0;
+    let failure = null;
+    gzip.on("data", (chunk) => {
+        parts.push(chunk);
+        compressed += chunk.byteLength;
+    });
+    gzip.on("error", (err) => {
+        failure = err;
+    });
+    return {
+        compressedBytes: () => compressed,
+        write: (line) => new Promise((resolve, reject) => {
+            if (failure)
+                return reject(failure);
+            gzip.write(line, "utf8", (err) => (err ? reject(err) : resolve()));
+        }),
+        // Z_SYNC_FLUSH costs a handful of bytes per boundary and a little ratio.
+        // That is the price of a budget that is enforced rather than estimated:
+        // without it deflate can hold the bulk of its output until end-of-stream,
+        // the running total reads near zero throughout, and the cap is only
+        // discovered to have been blown once the whole bundle has been built.
+        sync: () => new Promise((resolve, reject) => {
+            if (failure)
+                return reject(failure);
+            gzip.flush(zlib.constants.Z_SYNC_FLUSH, () => resolve());
+        }),
+        finish: () => new Promise((resolve, reject) => {
+            gzip.on("end", () => (failure ? reject(failure) : resolve(Buffer.concat(parts))));
+            gzip.on("error", reject);
+            gzip.end();
+            gzip.resume();
+        }),
+    };
+}
+/** One NDJSON line. Newline-terminated; JSON.stringify escapes any interior one. */
+function record(value) {
+    return `${JSON.stringify(value)}\n`;
+}
+/**
+ * Read `absPath` in slices, redacting whole lines, invoking `onChunk` with
+ * roughly {@link CHUNK_TEXT_BYTES} of redacted text at a time.
+ *
+ * Reads from the tail when the file exceeds `maxRawBytes`, and drops the first
+ * partial line after seeking so a chunk never begins mid-record. `onChunk`
+ * returns false to stop early (the cap was hit).
+ */
+export async function streamRedactedFile(absPath, sizeBytes, maxRawBytes, chunkBytes, onChunk) {
+    const fromTail = sizeBytes > maxRawBytes;
+    let position = fromTail ? sizeBytes - maxRawBytes : 0;
+    // The ceiling has to bound the bytes CONSUMED, not just where reading starts.
+    // Seeking alone happens to read the right amount when the on-disk size is
+    // accurate, but a file being appended to while we read it would otherwise
+    // stream without limit.
+    let remaining = maxRawBytes;
+    const fd = fs.openSync(absPath, "r");
+    try {
+        const decoder = new StringDecoder("utf8");
+        const buffer = Buffer.allocUnsafe(READ_SLICE_BYTES);
+        let carry = "";
+        let pending = "";
+        // Tracked incrementally: measuring the whole accumulator on every line is
+        // quadratic in the chunk size and dominates the runtime on a large log.
+        let pendingBytes = 0;
+        // After seeking into the middle of a file the first line is a fragment;
+        // drop it so every emitted line is whole.
+        let dropPartialLine = fromTail;
+        const flush = async () => {
+            if (pendingBytes === 0)
+                return true;
+            const { text, redactions } = redactLogText(pending);
+            pending = "";
+            pendingBytes = 0;
+            return onChunk(text, redactions);
+        };
+        const push = async (line) => {
+            pending += `${line}\n`;
+            pendingBytes += Buffer.byteLength(line, "utf8") + 1;
+            if (pendingBytes < chunkBytes)
+                return true;
+            // Hold the boundary open rather than bisect a PEM block — but not
+            // without limit, so a stray unclosed BEGIN cannot grow the buffer.
+            if (pendingBytes < chunkBytes * 4 && hasOpenPemBlock(pending))
+                return true;
+            return flush();
+        };
+        for (;;) {
+            const want = Math.min(READ_SLICE_BYTES, remaining);
+            const read = fs.readSync(fd, buffer, 0, want, position);
+            if (read === 0)
+                break;
+            position += read;
+            remaining -= read;
+            carry += decoder.write(buffer.subarray(0, read));
+            const lines = carry.split("\n");
+            // The final element is an incomplete line (or ""); hold it for the next slice.
+            carry = lines.pop() ?? "";
+            // ...unless it has grown past a whole chunk with no terminator in sight,
+            // in which case it is not a line and must be cut so the bundle stays
+            // bounded. See splitOversizedLine for why the cut lands on whitespace.
+            while (Buffer.byteLength(carry, "utf8") >= chunkBytes) {
+                const at = splitOversizedLine(carry, chunkBytes);
+                const segment = carry.slice(0, at);
+                carry = carry.slice(at);
+                if (dropPartialLine) {
+                    dropPartialLine = false;
+                    continue;
+                }
+                if (!(await push(segment)))
+                    return { fromTail, stopped: true };
+            }
+            for (const line of lines) {
+                if (dropPartialLine) {
+                    dropPartialLine = false;
+                    continue;
+                }
+                if (!(await push(line)))
+                    return { fromTail, stopped: true };
+            }
+        }
+        carry += decoder.end();
+        if (carry.length > 0 && !dropPartialLine) {
+            if (!(await push(carry)))
+                return { fromTail, stopped: true };
+        }
+        if (!(await flush()))
+            return { fromTail, stopped: true };
+        return { fromTail, stopped: false };
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+}
+/**
+ * Build a gzipped NDJSON bundle of the submitter's redacted `~/.hq` logs.
+ *
+ * Returns `undefined` when nothing eligible exists, so the caller can skip the
+ * upload entirely. Never throws: a bug report must not fail over its own
+ * diagnostics, so any unexpected error yields `undefined` and the submission
+ * proceeds with inline logs alone.
+ */
+export async function buildLogBundle(opts = {}) {
+    const maxBytes = Math.max(0, Math.min(opts.maxBytes ?? LOG_BUNDLE_MAX_BYTES, LOG_BUNDLE_MAX_BYTES));
+    // Scale the margin down for very small caps so the stop line never lands
+    // at zero (which would truncate before the first byte). At the production
+    // 50 MB cap this is exactly LOG_BUNDLE_SAFETY_MARGIN_BYTES.
+    const margin = Math.min(LOG_BUNDLE_SAFETY_MARGIN_BYTES, Math.floor(maxBytes / 2));
+    const stopAt = Math.max(0, maxBytes - margin);
+    const chunkBytes = chunkBytesForCap(maxBytes);
+    const hqDir = opts.hqDir ?? path.join(opts.homeDir ?? os.homedir(), ".hq");
+    let candidates;
+    try {
+        candidates = orderBundleCandidates(hqDir);
+    }
+    catch {
+        return undefined;
+    }
+    if (candidates.length === 0)
+        return undefined;
+    try {
+        const sink = createGzipSink();
+        let fileCount = 0;
+        let rawBytes = 0;
+        let redactions = 0;
+        let truncated = false;
+        await sink.write(record({
+            kind: "manifest",
+            version: 1,
+            tool: "hq-cli",
+            generatedAt: new Date().toISOString(),
+            maxBytes,
+            candidateCount: candidates.length,
+        }));
+        for (const candidate of candidates) {
+            await sink.sync();
+            if (sink.compressedBytes() > stopAt) {
+                truncated = true;
+                break;
+            }
+            let seq = 0;
+            let wroteHeader = false;
+            let stopped = false;
+            try {
+                const result = await streamRedactedFile(candidate.absPath, candidate.sizeBytes, opts.maxFileRawBytes ?? LOG_BUNDLE_MAX_FILE_RAW_BYTES, chunkBytes, async (text, chunkRedactions) => {
+                    if (!wroteHeader) {
+                        await sink.write(record({
+                            kind: "file",
+                            name: candidate.name,
+                            sizeBytes: candidate.sizeBytes,
+                            modifiedIso: new Date(candidate.modifiedMs).toISOString(),
+                        }));
+                        wroteHeader = true;
+                        fileCount++;
+                    }
+                    await sink.write(record({ kind: "chunk", name: candidate.name, seq: seq++, text }));
+                    rawBytes += Buffer.byteLength(text, "utf8");
+                    redactions += chunkRedactions;
+                    // Stop feeding once the compressed total reaches the stop line.
+                    // The sync is what makes that total trustworthy.
+                    await sink.sync();
+                    return sink.compressedBytes() <= stopAt;
+                });
+                stopped = result.stopped;
+            }
+            catch {
+                // An unreadable or vanished file is not a reason to lose the bundle;
+                // the inline blob already reports skips, so drop it silently here.
+                continue;
+            }
+            if (stopped) {
+                truncated = true;
+                break;
+            }
+        }
+        await sink.write(record({ kind: "summary", fileCount, rawBytes, redactions, truncated }));
+        const gzip = await sink.finish();
+        // Nothing but a manifest and a summary is not worth uploading.
+        if (fileCount === 0)
+            return undefined;
+        // Final authority. The stop line plus margin should make this unreachable,
+        // but the server refuses to presign above the cap, so a bundle that
+        // overshot is useless and must not be offered.
+        if (gzip.byteLength > maxBytes)
+            return undefined;
+        return {
+            gzip,
+            sizeBytes: gzip.byteLength,
+            fileCount,
+            truncated,
+            rawBytes,
+            redactions,
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Build, presign, and upload a log bundle; return the reference the submission
+ * should carry, or `undefined` if anything at all did not work out.
+ *
+ * Every failure path is silent and non-fatal by design. The bundle is an
+ * enrichment on top of the inline logs that already ship in the request body,
+ * so a missing endpoint, a disabled bucket, a refused presign, or a failed PUT
+ * must all degrade to "submit without it" rather than cost the user their bug
+ * report. In particular a 404 is expected and unremarkable while a CLI that
+ * knows about bundles is running against a server that does not yet.
+ */
+export async function uploadLogBundle(opts) {
+    if (!opts.enabled)
+        return undefined;
+    const build = opts.build ?? buildLogBundle;
+    const apiFetch = opts.apiFetch ?? vaultApiFetch;
+    try {
+        const bundle = await build();
+        if (!bundle)
+            return undefined;
+        const res = await apiFetch({
+            token: opts.token,
+            path: "/v1/feedback/logs/presign",
+            method: "POST",
+            body: { sizeBytes: bundle.sizeBytes },
+        });
+        if (!res.ok)
+            return undefined;
+        const parsed = (await res.json());
+        const slot = parsed?.bundle;
+        if (!slot || typeof slot.key !== "string" || typeof slot.url !== "string") {
+            return undefined;
+        }
+        const doFetch = opts.fetchImpl ?? fetch;
+        const put = await doFetch(slot.url, {
+            method: "PUT",
+            headers: {
+                "Content-Type": typeof slot.contentType === "string" ? slot.contentType : "application/gzip",
+                // Must match the length bound into the signature, or S3 rejects it.
+                "Content-Length": String(bundle.sizeBytes),
+            },
+            body: new Uint8Array(bundle.gzip),
+        });
+        if (!put.ok)
+            return undefined;
+        return {
+            key: slot.key,
+            sizeBytes: bundle.sizeBytes,
+            fileCount: bundle.fileCount,
+            truncated: bundle.truncated,
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+//# sourceMappingURL=feedback-log-bundle.js.map