hippo-memory 1.69.0 → 1.71.0

package/dist/connectors/github/dlq.js

@@ -1,107 +1,18 @@
-import { openHippoDb, closeHippoDb } from '../../db.js';
+import { bumpGitHubDlqRetryCount, githubDlqEntry, listGitHubDlq, } from '../../store/connectors/github.js';
+import { replayFailed } from '../dlq.js';
 import { verifyGitHubSignature } from './signature.js';
 import { isGitHubWebhookEnvelope } from './types.js';
-import { redactPayload, DLQ_REDACTED_NOTE } from '../../secret-detect.js';
-export function writeToDlq(db, opts) {
-    const rawPayload = redactPayload(opts.rawPayload);
-    const redacted = rawPayload !== opts.rawPayload;
-    const result = db
-        .prepare(`INSERT INTO github_dlq
-        (tenant_id, raw_payload, error, event_name, delivery_id, signature,
-         installation_id, repo_full_name, received_at, bucket)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
-        .run(opts.tenantId ?? '__unroutable__', rawPayload, redacted ? `${opts.error}; ${DLQ_REDACTED_NOTE}` : opts.error, opts.eventName ?? null, opts.deliveryId ?? null, redacted ? null : opts.signature ?? null, opts.installationId ?? null, opts.repoFullName ?? null, new Date().toISOString(), opts.bucket ?? 'parse_error');
-    return Number(result.lastInsertRowid);
-}
-const SELECT_COLUMNS = `id, tenant_id, raw_payload, error, event_name, delivery_id,
-            signature, installation_id, repo_full_name, retry_count,
-            received_at, retried_at, bucket`;
-export function listDlq(db, opts) {
-    // SAFETY: rows come from the SELECT above (SELECT_COLUMNS), which projects
-    // exactly DlqRawRow's fields, in the same order github_dlq defines them.
-    const rows = db
-        .prepare(`SELECT ${SELECT_COLUMNS}
-         FROM github_dlq
-        WHERE tenant_id = ?
-        ORDER BY received_at ASC
-        LIMIT ?`)
-        .all(opts.tenantId, opts.limit ?? 100);
-    return rows.map(rowToItem);
-}
-export function getDlqEntry(db, id) {
-    // SAFETY: the row comes from the SELECT above (SELECT_COLUMNS), which
-    // projects exactly DlqRawRow's fields, in the same order github_dlq defines them.
-    const row = db
-        .prepare(`SELECT ${SELECT_COLUMNS}
-         FROM github_dlq
-        WHERE id = ?`)
-        .get(id);
-    if (!row)
-        return null;
-    return rowToItem(row);
-}
-function rowToItem(r) {
-    return {
-        id: Number(r.id),
-        tenantId: String(r.tenant_id),
-        rawPayload: String(r.raw_payload),
-        error: String(r.error),
-        eventName: r.event_name == null ? null : String(r.event_name),
-        deliveryId: r.delivery_id == null ? null : String(r.delivery_id),
-        signature: r.signature == null ? null : String(r.signature),
-        installationId: r.installation_id == null ? null : String(r.installation_id),
-        repoFullName: r.repo_full_name == null ? null : String(r.repo_full_name),
-        retryCount: Number(r.retry_count ?? 0),
-        receivedAt: String(r.received_at),
-        retriedAt: r.retried_at == null ? null : String(r.retried_at),
-        bucket: r.bucket == null ? 'parse_error' : String(r.bucket),
-    };
-}
-function bumpRetryCount(hippoRoot, id) {
-    const db = openHippoDb(hippoRoot);
-    try {
-        db.prepare(`UPDATE github_dlq
-          SET retry_count = retry_count + 1,
-              retried_at = ?
-        WHERE id = ?`).run(new Date().toISOString(), id);
-    }
-    finally {
-        closeHippoDb(db);
-    }
-}
-/**
- * Replay a DLQ row through the normal ingest path. Behavior:
- *   1. Fetch row by id. Not found → `not_found`.
- *   2. If !force and webhookSecret provided, verify signature with the
- *      current secret. Fail → bump retry_count, `sig_fail`.
- *      Missing signature on the row → `sig_missing` (no bump; --force required).
- *   3. JSON.parse the raw payload. Fail → bump, `parse_error`.
- *   4. Type-guard the envelope. Fail → bump, `unhandled`.
- *   5. If an `ingestHook` is supplied, call it and return its memoryId.
- *      If not (dry-run path), bump retry_count and return status `replayed`
- *      with memoryId=null. The webhook route wires the real hook.
- *
- * Mirrors Slack's "always use current routing" policy: replays use the
- * deployment state NOW, not at the time of original DLQing.
- */
+/** GitHub's dead-letter table; the event, delivery, installation and repo columns let an operator triage a row without re-reading the payload. */
+export const githubDlq = {
+    letter: (row) => ({ connector: 'github', ...row }),
+    list: listGitHubDlq,
+};
+/** Replay a DLQ row through the normal ingest path; a failed signature, parse or envelope check bumps retry_count.
+ *  Without an `ingestHook` (dry-run) it bumps retry_count and returns `replayed` with memoryId=null; replays use today's routing, not the original's. */
 export async function replayDlqEntry(ctx, id, opts = {}) {
-    const db = openHippoDb(ctx.hippoRoot);
-    let row;
-    try {
-        row = getDlqEntry(db, id);
-    }
-    finally {
-        closeHippoDb(db);
-    }
-    if (!row) {
-        return {
-            ok: false,
-            status: 'not_found',
-            memoryId: null,
-            retryCount: 0,
-            reason: `dlq id ${id} not found`,
-        };
-    }
+    const row = githubDlqEntry(ctx.hippoRoot, id);
+    if (!row)
+        return replayFailed('not_found', 0, `dlq id ${id} not found`);
     // Signature verification (current secret, not the one in effect when DLQed).
     if (!opts.force && opts.webhookSecret) {
         const sigFailure = checkReplaySignature(ctx.hippoRoot, id, row, opts.webhookSecret, opts.previousSecret);
@@ -113,7 +24,7 @@ export async function replayDlqEntry(ctx, id, opts = {}) {
         return envelopeFailure;
     // Without an ingest hook this is a dry-run validation. Bump and report.
     if (!opts.ingestHook) {
-        bumpRetryCount(ctx.hippoRoot, id);
+        bumpGitHubDlqRetryCount(ctx.hippoRoot, id);
         return {
             ok: true,
             status: 'replayed',
@@ -122,9 +33,7 @@ export async function replayDlqEntry(ctx, id, opts = {}) {
             reason: 'dry-run: no ingest hook supplied',
         };
     }
-    // Real replay path. The route's IngestHook is responsible for routing,
-    // idempotency, and writing the memory. The DLQ module only validates the
-    // surface and bumps the retry counter.
+    // Real replay path: the route's IngestHook owns routing, idempotency and the memory write; this module only validates and bumps the retry counter.
     // No idempotencyKey arg: the hook re-derives it from the parsed event (artifact_ref + updated_at).
     const eventName = row.eventName ?? '';
     const deliveryId = row.deliveryId ?? '';
@@ -133,7 +42,7 @@ export async function replayDlqEntry(ctx, id, opts = {}) {
         eventName,
         deliveryId,
     });
-    bumpRetryCount(ctx.hippoRoot, id);
+    bumpGitHubDlqRetryCount(ctx.hippoRoot, id);
     return {
         ok: true,
         status: 'replayed',
@@ -144,13 +53,7 @@ export async function replayDlqEntry(ctx, id, opts = {}) {
 /** The failure result when the row cannot pass the signature gate, else null. */
 function checkReplaySignature(hippoRoot, id, row, webhookSecret, previousSecret) {
     if (!row.signature) {
-        return {
-            ok: false,
-            status: 'sig_missing',
-            memoryId: null,
-            retryCount: row.retryCount,
-            reason: 'row has no signature (legacy, or redacted before storing); pass --force to replay',
-        };
+        return replayFailed('sig_missing', row.retryCount, 'row has no signature (legacy, or redacted before storing); pass --force to replay');
     }
     const sigOk = verifyGitHubSignature({
         rawBody: row.rawPayload,
@@ -159,14 +62,8 @@ function checkReplaySignature(hippoRoot, id, row, webhookSecret, previousSecret)
         previousSecret,
     });
     if (!sigOk) {
-        bumpRetryCount(hippoRoot, id);
-        return {
-            ok: false,
-            status: 'sig_fail',
-            memoryId: null,
-            retryCount: row.retryCount + 1,
-            reason: 'signature did not verify against current GITHUB_WEBHOOK_SECRET; pass --force to replay anyway',
-        };
+        bumpGitHubDlqRetryCount(hippoRoot, id);
+        return replayFailed('sig_fail', row.retryCount + 1, 'signature did not verify against current GITHUB_WEBHOOK_SECRET; pass --force to replay anyway');
     }
     return null;
 }
@@ -177,28 +74,14 @@ function checkReplayEnvelope(hippoRoot, id, row) {
         parsed = JSON.parse(row.rawPayload);
     }
     catch (e) {
-        bumpRetryCount(hippoRoot, id);
-        // SAFETY: this is a best-effort error message only; property access on
-        // any JS value is safe (undefined if absent), preserving the existing
-        // lenient formatting even when something non-Error was thrown.
+        bumpGitHubDlqRetryCount(hippoRoot, id);
+        // SAFETY: best-effort error message only; property access on any JS value is safe, even for a non-Error throw.
         const message = e.message;
-        return {
-            ok: false,
-            status: 'parse_error',
-            memoryId: null,
-            retryCount: row.retryCount + 1,
-            reason: `still unparseable: ${message}`,
-        };
+        return replayFailed('parse_error', row.retryCount + 1, `still unparseable: ${message}`);
     }
     if (!isGitHubWebhookEnvelope(parsed)) {
-        bumpRetryCount(hippoRoot, id);
-        return {
-            ok: false,
-            status: 'unhandled',
-            memoryId: null,
-            retryCount: row.retryCount + 1,
-            reason: 'not a GitHub webhook envelope',
-        };
+        bumpGitHubDlqRetryCount(hippoRoot, id);
+        return replayFailed('unhandled', row.retryCount + 1, 'not a GitHub webhook envelope');
     }
     return null;
 }

package/dist/connectors/github/ingest.d.ts

@@ -1,37 +1,12 @@
-/**
- * GitHub event ingest with afterWrite race-safe idempotency.
- *
- * Mirrors src/connectors/slack/ingest.ts. The dedupe key is sha256(eventName +
- * ':' + rawBody), derived from the signed body, not from the
- * unsigned X-GitHub-Delivery header, so a replay attacker cannot bypass
- * idempotency by rotating the delivery UUID.
- *
- * Race semantics:
- *   - Fast path: hasSeenKey pre-check returns 'duplicate' for the common case.
- *   - Slow path: hasSeenKey passes (no row yet). Two workers may race into
- *     remember() concurrently. Inside the writeEntry SAVEPOINT, INSERT OR
- *     IGNORE on github_event_log either inserts (changes=1, commit) or
- *     collides (changes=0, throw DuplicateIdempotencyError -> SAVEPOINT
- *     rolls back this worker's memory row). Exactly one memory exists per
- *     idempotency_key.
- *
- * The `__testInjectBeforeLog` hook below lets tests pre-populate github_event_log
- * inside the SAVEPOINT to exercise the changes=0 -> rollback path, not just the fast path.
- */
-import { type Context } from '../../api.js';
-import { type DatabaseSyncLike } from '../../db.js';
+import { type Context } from '../../api/index.js';
 import type { GitHubIssueEvent, GitHubIssueCommentEvent, GitHubPullRequestEvent, GitHubPullRequestReviewCommentEvent } from './types.js';
 export type IngestStatus = 'ingested' | 'duplicate' | 'skipped' | 'skipped_duplicate';
 export interface IngestResult {
     status: IngestStatus;
     memoryId: string | null;
 }
-/**
- * Discriminated union of the four event shapes V1 ingests. The eventName
- * field MUST equal the X-GitHub-Event header value; computeIdempotencyKey
- * folds it into the dedupe key so the same body posted under two different
- * X-GitHub-Event headers produces two distinct keys (test 7).
- */
+/** Discriminated union of the four event shapes V1 ingests; eventName MUST equal the X-GitHub-Event header,
+ *  since computeIdempotencyKey folds it into the dedupe key. */
 export type IngestEvent = {
     eventName: 'issues';
     payload: GitHubIssueEvent;
@@ -52,14 +27,6 @@ export interface IngestInput {
     rawBody: string;
     /** X-GitHub-Delivery header value, audit metadata only. */
     deliveryId: string;
-    /**
-     * Test-only hook fired inside the SAVEPOINT, AFTER the memory row is
-     * inserted but BEFORE the github_event_log INSERT OR IGNORE. Tests pass
-     * a function that pre-inserts the github_event_log row using the
-     * provided db handle to simulate a concurrent worker winning the race.
-     * Production callers leave this undefined.
-     */
-    __testInjectBeforeLog?: (db: DatabaseSyncLike, idempotencyKey: string) => void;
 }
-export declare function ingestEvent(ctx: Context, input: IngestInput): IngestResult;
+export declare function ingestEvent(ctx: Context, input: IngestInput): Promise<IngestResult>;
 //# sourceMappingURL=ingest.d.ts.map

package/dist/connectors/github/ingest.js

@@ -1,27 +1,9 @@
-/**
- * GitHub event ingest with afterWrite race-safe idempotency.
- *
- * Mirrors src/connectors/slack/ingest.ts. The dedupe key is sha256(eventName +
- * ':' + rawBody), derived from the signed body, not from the
- * unsigned X-GitHub-Delivery header, so a replay attacker cannot bypass
- * idempotency by rotating the delivery UUID.
- *
- * Race semantics:
- *   - Fast path: hasSeenKey pre-check returns 'duplicate' for the common case.
- *   - Slow path: hasSeenKey passes (no row yet). Two workers may race into
- *     remember() concurrently. Inside the writeEntry SAVEPOINT, INSERT OR
- *     IGNORE on github_event_log either inserts (changes=1, commit) or
- *     collides (changes=0, throw DuplicateIdempotencyError -> SAVEPOINT
- *     rolls back this worker's memory row). Exactly one memory exists per
- *     idempotency_key.
- *
- * The `__testInjectBeforeLog` hook below lets tests pre-populate github_event_log
- * inside the SAVEPOINT to exercise the changes=0 -> rollback path, not just the fast path.
- */
-import { remember } from '../../api.js';
-import { openHippoDb, closeHippoDb } from '../../db.js';
-import { RejectedValueError } from '../../rejection.js';
-import { hasSeenKey, lookupMemoryByKey, DuplicateIdempotencyError } from './idempotency.js';
+// GitHub event ingest, once per idempotency key. The key comes from the event's own content, never from the unsigned
+// X-GitHub-Delivery header, so a replay cannot get past it by rotating the delivery id. The pre-check is only the fast path:
+// the store logs the key in the memory's own transaction, which is what holds when two workers race.
+import { remember } from '../../api/index.js';
+import { requireGroup, storeFor } from '../../store/index.js';
+import { RejectedValueError } from '../../store/rejection.js';
 import { computeIdempotencyKey } from './signature.js';
 import { issueEventToRememberOpts, issueCommentEventToRememberOpts, pullRequestEventToRememberOpts, prReviewCommentEventToRememberOpts, } from './transform.js';
 function transformEvent(event) {
@@ -36,12 +18,8 @@ function transformEvent(event) {
             return prReviewCommentEventToRememberOpts(event.payload);
     }
 }
-/**
- * Extract the source-normalized identifier the idempotency key needs.
- * Backfill and webhook both produce IngestEvent objects describing the same
- * source revision, so deriving the key from these fields collapses both paths
- * onto the same dedupe row. Mirrors the artifactRef strings in transform.ts.
- */
+/** Extract the source-normalized identifier the idempotency key needs, so backfill and webhook deliveries of the same
+ *  source revision collapse onto one dedupe row. Mirrors the artifactRef strings in transform.ts. */
 function eventArtifactRef(event) {
     const repo = event.payload.repository?.full_name ?? 'unknown/unknown';
     switch (event.eventName) {
@@ -67,87 +45,40 @@ function eventUpdatedAt(event) {
             return event.payload.comment.updated_at ?? null;
     }
 }
-const INSERT_EVENT_LOG_SQL = `INSERT OR IGNORE INTO github_event_log (idempotency_key, delivery_id, event_name, ingested_at, memory_id) VALUES (?, ?, ?, ?, ?)`;
-export function ingestEvent(ctx, input) {
+export async function ingestEvent(ctx, input) {
+    const events = requireGroup(storeFor(ctx), 'connectorEvents');
     const idempotencyKey = computeIdempotencyKey(eventArtifactRef(input.event), eventUpdatedAt(input.event));
+    const event = { connector: 'github', idempotencyKey, deliveryId: input.deliveryId, eventName: input.event.eventName };
     // Fast path: pre-check. Avoids running the transform / opening a write tx
     // for the common already-seen case (GitHub auto-retries with the same body).
-    const db = openHippoDb(ctx.hippoRoot);
-    try {
-        if (hasSeenKey(db, idempotencyKey)) {
-            return { status: 'duplicate', memoryId: lookupMemoryByKey(db, idempotencyKey) };
-        }
-    }
-    finally {
-        closeHippoDb(db);
-    }
+    const seen = await events.eventRecord(event);
+    if (seen.seen)
+        return { status: 'duplicate', memoryId: seen.memoryId };
     const opts = transformEvent(input.event);
     if (!opts) {
-        // Empty body: no memory to write, but mark seen so a retry of the same
-        // empty event returns 'duplicate' (not 'skipped' again — that would
-        // re-run the transform on every retry).
-        markKeySeenWithoutMemory(ctx.hippoRoot, idempotencyKey, input);
+        // Empty body: no memory to write, but mark seen so a retry returns 'duplicate' instead of re-running the transform.
+        await events.markEventSeen(event);
         return { status: 'skipped', memoryId: null };
     }
-    // Atomic write via afterWrite. Inside writeEntry's SAVEPOINT:
-    //   1. memories row INSERT lands.
-    //   2. (test-only) __testInjectBeforeLog can race-inject a colliding key.
-    //   3. INSERT OR IGNORE on github_event_log; if a concurrent worker (or
-    //      the test injection) beat us, changes=0 -> throw -> SAVEPOINT rolls
-    //      back our memory row. Other worker's commit stands.
     try {
-        return rememberWithEventLog(ctx, input, idempotencyKey, opts);
+        return await rememberWithEventLog(ctx, event, opts);
     }
     catch (e) {
-        if (e instanceof DuplicateIdempotencyError) {
-            // Other worker's row is committed. Return its memory_id so callers
-            // behave identically to the fast-path 'duplicate' branch.
-            const db3 = openHippoDb(ctx.hippoRoot);
-            try {
-                return { status: 'skipped_duplicate', memoryId: lookupMemoryByKey(db3, idempotencyKey) };
-            }
-            finally {
-                closeHippoDb(db3);
-            }
-        }
         if (e instanceof RejectedValueError) {
             // A tombstone hit is a PERMANENT skip, never DLQ-retried: mark the key seen like the empty-body
             // branch above so a GitHub retry of the same delivery acks as done, not error.
-            markKeySeenWithoutMemory(ctx.hippoRoot, idempotencyKey, input);
+            await events.markEventSeen(event);
             return { status: 'skipped', memoryId: null };
         }
         throw e;
     }
 }
-function markKeySeenWithoutMemory(hippoRoot, idempotencyKey, input) {
-    const db = openHippoDb(hippoRoot);
-    try {
-        db
-            .prepare(INSERT_EVENT_LOG_SQL)
-            .run(idempotencyKey, input.deliveryId, input.event.eventName, new Date().toISOString(), null);
-    }
-    finally {
-        closeHippoDb(db);
-    }
-}
-function rememberWithEventLog(ctx, input, idempotencyKey, opts) {
-    // No `|| 'connector:github'` fallback (see rememberWithEventLog in slack/ingest.ts for rationale).
-    const result = remember({ ...ctx, store: undefined }, // the event log row commits with the memory on hippo.db's own handle, never through a store
-    {
-        ...opts,
-        untrusted: true,
-        afterWrite: (innerDb, memoryId) => {
-            if (input.__testInjectBeforeLog) {
-                input.__testInjectBeforeLog(innerDb, idempotencyKey);
-            }
-            const inserted = innerDb
-                .prepare(INSERT_EVENT_LOG_SQL)
-                .run(idempotencyKey, input.deliveryId, input.event.eventName, new Date().toISOString(), memoryId);
-            if (Number(inserted.changes ?? 0) === 0) {
-                throw new DuplicateIdempotencyError(idempotencyKey);
-            }
-        },
-    });
+async function rememberWithEventLog(ctx, event, opts) {
+    // No `|| 'connector:github'` fallback: the caller always builds ctx with the connector subject (same as slack/ingest.ts).
+    const result = await remember(ctx, { ...opts, untrusted: true, event });
+    // Another worker logged this key between the pre-check and the write: its memory stands and ours was not stored.
+    if (result.duplicate)
+        return { status: 'skipped_duplicate', memoryId: result.duplicate.memoryId };
     return { status: 'ingested', memoryId: result.id };
 }
 //# sourceMappingURL=ingest.js.map

package/dist/connectors/github/octokit-client.d.ts

@@ -1,19 +1,12 @@
-/**
- * Octokit-shaped HTTP fetcher for the GitHub connector backfill.
- *
- * Production uses `realGitHubFetcher` against `https://api.github.com`.
- * Tests inject a fake `GitHubFetcher` so they never hit the network.
- *
- * Any non-200 response that is NOT a recognized rate-limit pause MUST throw
- * `GitHubFetchError`: an empty page would be an empty backfill with no operator signal.
- */
+/** Octokit-shaped HTTP fetcher for the GitHub backfill; tests inject a fake `GitHubFetcher`.
+ *  Any non-200 that is not a recognised rate-limit pause MUST throw `GitHubFetchError`, since an empty page would be a silent empty backfill. */
 import { type RateLimitInfo } from './ratelimit.js';
-import type { JsonValue } from '../../json.js';
+import type { JsonValue } from '../../util/json.js';
 export declare class GitHubFetchError extends Error {
     readonly status: number;
     readonly bodyExcerpt: string;
     readonly url: string;
-    constructor(status: number, bodyExcerpt: string, url: string);
+    constructor(status: number, bodyExcerpt: string, url: string, options?: ErrorOptions);
 }
 export interface GitHubBackfillPage {
     readonly items: ReadonlyArray<JsonValue>;
@@ -24,12 +17,7 @@ export type GitHubFetcher = (args: {
     url: string;
     token: string;
 }) => Promise<GitHubBackfillPage>;
-/**
- * Parse the rel="next" URL from an RFC 5988 `Link` header.
- *
- * Header format: `<url1>; rel="next", <url2>; rel="last"`.
- * Returns the URL whose rel parameter is exactly `"next"`, or null.
- */
+/** Parse the rel="next" URL from an RFC 5988 `Link` header, or null. */
 export declare function parseNextLink(linkHeader: string): string | null;
 export declare const realGitHubFetcher: GitHubFetcher;
 //# sourceMappingURL=octokit-client.d.ts.map

package/dist/connectors/github/octokit-client.js

@@ -1,32 +1,22 @@
-/**
- * Octokit-shaped HTTP fetcher for the GitHub connector backfill.
- *
- * Production uses `realGitHubFetcher` against `https://api.github.com`.
- * Tests inject a fake `GitHubFetcher` so they never hit the network.
- *
- * Any non-200 response that is NOT a recognized rate-limit pause MUST throw
- * `GitHubFetchError`: an empty page would be an empty backfill with no operator signal.
- */
+/** Octokit-shaped HTTP fetcher for the GitHub backfill; tests inject a fake `GitHubFetcher`.
+ *  Any non-200 that is not a recognised rate-limit pause MUST throw `GitHubFetchError`, since an empty page would be a silent empty backfill. */
 import { parseRateLimit } from './ratelimit.js';
-import { fetchWithRetry } from '../../http-retry.js';
+import { fetchWithRetry } from '../../util/http-retry.js';
+import { readCappedJson, readCappedText } from '../../util/capped-json.js';
+import { errorMessage } from '../../util/log.js';
 export class GitHubFetchError extends Error {
     status;
     bodyExcerpt;
     url;
-    constructor(status, bodyExcerpt, url) {
-        super(`GitHub ${status} on ${url}: ${bodyExcerpt}`);
+    constructor(status, bodyExcerpt, url, options) {
+        super(`GitHub ${status} on ${url}: ${bodyExcerpt}`, options);
         this.status = status;
         this.bodyExcerpt = bodyExcerpt;
         this.url = url;
         this.name = 'GitHubFetchError';
     }
 }
-/**
- * Parse the rel="next" URL from an RFC 5988 `Link` header.
- *
- * Header format: `<url1>; rel="next", <url2>; rel="last"`.
- * Returns the URL whose rel parameter is exactly `"next"`, or null.
- */
+/** Parse the rel="next" URL from an RFC 5988 `Link` header, or null. */
 export function parseNextLink(linkHeader) {
     if (!linkHeader)
         return null;
@@ -43,6 +33,24 @@ function headersToRecord(h) {
 }
 const GITHUB_TIMEOUT_MS = 30_000;
 const ERROR_BODY_SNIPPET_CHARS = 256;
+const UTF8_MAX_BYTES_PER_CHAR = 4;
+// A page is at most 100 items (per_page=100), each with a body of up to 65,536 characters at 4 bytes, plus its fields.
+const GITHUB_MAX_REPLY_BYTES = 32 * 1024 * 1024;
+/** The items of a 200 list reply. A body that is not JSON, is over the cap or is not an array throws, so it never reaches ingest as items. */
+async function readItems(res, url) {
+    // The path only: a `next` link comes from the server, so its query string is not ours to print.
+    const path = new URL(url).pathname;
+    let body;
+    try {
+        body = await readCappedJson(res, GITHUB_MAX_REPLY_BYTES);
+    }
+    catch (err) {
+        throw new GitHubFetchError(res.status, errorMessage(err), path, { cause: err });
+    }
+    if (!Array.isArray(body))
+        throw new GitHubFetchError(res.status, 'the reply is not a JSON array', path);
+    return body;
+}
 export const realGitHubFetcher = async ({ url, token }) => {
     const res = await fetchWithRetry(url, {
         headers: {
@@ -55,15 +63,11 @@ export const realGitHubFetcher = async ({ url, token }) => {
     const rateLimit = parseRateLimit(headers, res.status);
     // Don't silently turn 401/403/404/500 into empty pages.
     if (res.status !== 200 && rateLimit.reason === 'none') {
-        const body = await res.text().catch(() => '');
+        // The status is the error reported; a body that cannot be read only costs its snippet.
+        const body = await readCappedText(res, ERROR_BODY_SNIPPET_CHARS * UTF8_MAX_BYTES_PER_CHAR).catch(() => '');
         throw new GitHubFetchError(res.status, body.slice(0, ERROR_BODY_SNIPPET_CHARS), url);
     }
-    // SAFETY: GitHub's list/paginated endpoints (issues, comments, etc. — the
-    // only endpoints this backfill fetcher targets) return a JSON array body
-    // on 200; per-item shape is intentionally left as `unknown` here since
-    // GitHubBackfillPage.items is opaque to this seam (callers parse the
-    // shape they expect downstream).
-    const items = res.status === 200 ? (await res.json()) : [];
+    const items = res.status === 200 ? await readItems(res, url) : [];
     const link = res.headers.get('link') ?? '';
     const next = parseNextLink(link);
     return { items, next, rateLimit };

package/dist/connectors/github/ratelimit.d.ts

@@ -1,20 +1,9 @@
-/**
- * GitHub rate-limit header parser.
- *
- * GitHub returns 403 for primary rate-limit (with `X-RateLimit-Remaining: 0`
- * and `X-RateLimit-Reset: <epoch>`) and 429 + `Retry-After: <seconds>` for
- * secondary rate-limit. Backfill must pause and resume rather than error.
- */
+/** GitHub rate-limit header parser: 403 with `X-RateLimit-Remaining: 0` is primary, 429 with `Retry-After` is secondary.
+ *  Backfill must pause and resume rather than error. */
 export interface RateLimitInfo {
     readonly sleepSeconds: number;
     readonly reason: 'primary' | 'secondary' | 'none';
 }
-/**
- * Parse rate-limit signal from a GitHub HTTP response.
- *
- * @param headers Lower-cased HTTP response headers.
- * @param status  HTTP status code.
- * @param now     Optional current epoch seconds (for deterministic tests).
- */
+/** Parse the rate-limit signal from a GitHub response; `headers` are lower-cased and `now` (epoch seconds) is for deterministic tests. */
 export declare function parseRateLimit(headers: Record<string, string | undefined>, status: number, now?: number): RateLimitInfo;
 //# sourceMappingURL=ratelimit.d.ts.map

package/dist/connectors/github/ratelimit.js

@@ -1,17 +1,6 @@
-/**
- * GitHub rate-limit header parser.
- *
- * GitHub returns 403 for primary rate-limit (with `X-RateLimit-Remaining: 0`
- * and `X-RateLimit-Reset: <epoch>`) and 429 + `Retry-After: <seconds>` for
- * secondary rate-limit. Backfill must pause and resume rather than error.
- */
-/**
- * Parse rate-limit signal from a GitHub HTTP response.
- *
- * @param headers Lower-cased HTTP response headers.
- * @param status  HTTP status code.
- * @param now     Optional current epoch seconds (for deterministic tests).
- */
+/** GitHub rate-limit header parser: 403 with `X-RateLimit-Remaining: 0` is primary, 429 with `Retry-After` is secondary.
+ *  Backfill must pause and resume rather than error. */
+/** Parse the rate-limit signal from a GitHub response; `headers` are lower-cased and `now` (epoch seconds) is for deterministic tests. */
 export function parseRateLimit(headers, status, now) {
     const _now = now ?? Math.floor(Date.now() / 1000);
     if (status === 429) {

package/dist/connectors/github/scope.d.ts

@@ -1,8 +1,6 @@
 import type { GitHubRepository } from './types.js';
-/**
- * Map a GitHub repository to a hippo scope string. Default-private when
- * privacy is undetermined: cost of leaking public into private (recall returns
- * nothing) << cost of leaking private into public (data exposure).
- */
+/** Map a GitHub repository to a hippo scope string. Default-private when
+ *  privacy is undetermined: cost of leaking public into private (recall returns
+ *  nothing) << cost of leaking private into public (data exposure). */
 export declare function scopeFromRepository(repo: GitHubRepository | undefined): string;
 //# sourceMappingURL=scope.d.ts.map

package/dist/connectors/github/scope.js

@@ -1,8 +1,6 @@
-/**
- * Map a GitHub repository to a hippo scope string. Default-private when
- * privacy is undetermined: cost of leaking public into private (recall returns
- * nothing) << cost of leaking private into public (data exposure).
- */
+/** Map a GitHub repository to a hippo scope string. Default-private when
+ *  privacy is undetermined: cost of leaking public into private (recall returns
+ *  nothing) << cost of leaking private into public (data exposure). */
 export function scopeFromRepository(repo) {
     if (!repo)
         return 'github:private:unknown';