hippo-memory 1.70.0 → 1.71.0

package/dist/connectors/github/backfill.d.ts

@@ -1,28 +1,6 @@
-/**
- * Paginated backfill of three GitHub REST streams with per-stream
- * high-water marks (HWMs):
- *   - /repos/{repo}/issues               -> issues_hwm
- *   - /repos/{repo}/issues/comments      -> issue_comments_hwm
- *   - /repos/{repo}/pulls/comments       -> pr_review_comments_hwm
- *
- * Crash safety: each stream's HWM is persisted ONLY after
- * the stream fully drains. If stream 2 throws mid-flight, stream 1's HWM
- * is committed and stream 2's stays NULL (or its prior value). Rerun
- * picks up from stream 1's saved HWM and re-fetches stream 2 from its
- * last committed point.
- *
- * The /issues endpoint returns BOTH issues and PRs (a PR is
- * an issue with a `pull_request` field). We skip PRs here so they don't
- * get ingested under the issues schema. PRs are handled via webhook in
- * V1 (no /pulls backfill stream — review comments cover the discussion
- * surface).
- *
- * Privacy (V1 limitation): the REST list endpoints don't reliably set
- * `repository.private`, and resolving it requires an extra API call per
- * repo. Backfill leaves the field undefined so scopeFromRepository falls
- * through to private. Callers who know the repo is public can re-tag
- * downstream; the fail-safe default protects private orgs.
- */
+/** Paginated backfill of the GitHub issues, issue-comments and PR-review-comments streams, each with its own high-water mark (HWM).
+ *  An HWM is persisted only after its stream fully drains, so a crash in stream 2 leaves stream 1 committed and stream 2 resumes from its prior HWM.
+ *  `repository.private` is left undefined (REST lists omit it), so scopeFromRepository falls through to private. */
 import type { Context } from '../../api/index.js';
 import type { GitHubFetcher } from './octokit-client.js';
 export interface BackfillOpts {
@@ -32,6 +10,8 @@ export interface BackfillOpts {
     token: string;
     /** Optional cap on items per stream. Useful for tests. */
     maxPerStream?: number;
+    /** First-run `--since`; seeds all three HWMs, COALESCE keeps any HWM a stream already saved. */
+    sinceIso?: string;
     /** sleep ms — injectable so tests don't actually wait. */
     sleepMs?: (ms: number) => Promise<void>;
 }

package/dist/connectors/github/backfill.js

@@ -1,38 +1,12 @@
-/**
- * Paginated backfill of three GitHub REST streams with per-stream
- * high-water marks (HWMs):
- *   - /repos/{repo}/issues               -> issues_hwm
- *   - /repos/{repo}/issues/comments      -> issue_comments_hwm
- *   - /repos/{repo}/pulls/comments       -> pr_review_comments_hwm
- *
- * Crash safety: each stream's HWM is persisted ONLY after
- * the stream fully drains. If stream 2 throws mid-flight, stream 1's HWM
- * is committed and stream 2's stays NULL (or its prior value). Rerun
- * picks up from stream 1's saved HWM and re-fetches stream 2 from its
- * last committed point.
- *
- * The /issues endpoint returns BOTH issues and PRs (a PR is
- * an issue with a `pull_request` field). We skip PRs here so they don't
- * get ingested under the issues schema. PRs are handled via webhook in
- * V1 (no /pulls backfill stream — review comments cover the discussion
- * surface).
- *
- * Privacy (V1 limitation): the REST list endpoints don't reliably set
- * `repository.private`, and resolving it requires an extra API call per
- * repo. Backfill leaves the field undefined so scopeFromRepository falls
- * through to private. Callers who know the repo is public can re-tag
- * downstream; the fail-safe default protects private orgs.
- */
-import { readCursors, writeHwm } from '../../store/connectors/github.js';
+/** Paginated backfill of the GitHub issues, issue-comments and PR-review-comments streams, each with its own high-water mark (HWM).
+ *  An HWM is persisted only after its stream fully drains, so a crash in stream 2 leaves stream 1 committed and stream 2 resumes from its prior HWM.
+ *  `repository.private` is left undefined (REST lists omit it), so scopeFromRepository falls through to private. */
+import { readCursors, seedCursors, writeHwm } from '../../store/connectors/github.js';
 import { ingestEvent } from './ingest.js';
 import { isJsonObject } from '../../util/json.js';
 const API = 'https://api.github.com';
-/**
- * Build a synthetic `repository` object from the known repo full name.
- * REST list endpoints often omit the full repository object on each item;
- * we already know the repo since the caller passed it. `private` left
- * undefined so scopeFromRepository falls through to private.
- */
+/** Build a synthetic `repository` from the known repo full name, because REST list items often omit it.
+ *  `private` stays undefined so scopeFromRepository falls through to private. */
 function syntheticRepository(repoFullName) {
     const [owner, name] = repoFullName.split('/');
     return {
@@ -42,11 +16,7 @@ function syntheticRepository(repoFullName) {
         // private intentionally omitted — fail-safe to private scope.
     };
 }
-/**
- * Parse the trailing issue/PR number from a REST API URL.
- * e.g. https://api.github.com/repos/o/r/issues/42 -> 42
- *      https://api.github.com/repos/o/r/pulls/7   -> 7
- */
+/** Parse the trailing issue/PR number from a REST API URL, e.g. .../issues/42 -> 42. */
 function parseTrailingNumber(url) {
     if (!url)
         return null;
@@ -86,9 +56,7 @@ async function drainStream(ctx, url0, options) {
         pages++;
         for (const item of page.items) {
             // Track updated_at on EVERY item, before the toIngestEvent filter, so PR-only pages don't loop forever.
-            // SAFETY: updated_at is GitHub's ISO-timestamp field on every item
-            // shape this stream returns; a non-string value would only fail the
-            // string comparisons below, matching pre-migration passthrough.
+            // SAFETY: updated_at is GitHub's ISO-timestamp field on every item shape this stream returns.
             const updatedAt = isJsonObject(item)
                 ? (item.updated_at ?? null)
                 : null;
@@ -146,9 +114,8 @@ function issueItemToEvent(item, repository) {
 function issueCommentItemToEvent(item, repository) {
     if (!isCommentItem(item))
         return null;
-    // SAFETY: this closure only runs against the /issues/comments stream,
-    // so every item isCommentItem validates here is genuinely an
-    // IssueCommentItem; a missing issue_url is handled defensively below.
+    // SAFETY: this closure only runs against the /issues/comments stream, so every item isCommentItem
+    // validates is an IssueCommentItem; a missing issue_url is handled below.
     const c = item;
     const issueNumber = parseTrailingNumber(c.issue_url);
     if (issueNumber === null)
@@ -169,9 +136,8 @@ function issueCommentItemToEvent(item, repository) {
 function prReviewCommentItemToEvent(item, repository) {
     if (!isCommentItem(item))
         return null;
-    // SAFETY: this closure only runs against the /pulls/comments stream,
-    // so every item isCommentItem validates here is genuinely a
-    // PrReviewCommentItem; a missing pull_request_url is handled below.
+    // SAFETY: this closure only runs against the /pulls/comments stream, so every item isCommentItem
+    // validates is a PrReviewCommentItem; a missing pull_request_url is handled below.
     const c = item;
     const prNumber = parseTrailingNumber(c.pull_request_url);
     if (prNumber === null)
@@ -206,13 +172,14 @@ async function backfillStream(ctx, opts, sleep, stream) {
 }
 export async function backfillRepo(ctx, opts) {
     const sleep = opts.sleepMs ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    if (opts.sinceIso)
+        seedCursors(ctx.hippoRoot, ctx.tenantId, opts.repoFullName, opts.sinceIso);
     const cursors = readCursors(ctx.hippoRoot, ctx.tenantId, opts.repoFullName);
     const repository = syntheticRepository(opts.repoFullName);
     const result = {
         ingested: { issues: 0, issueComments: 0, prReviewComments: 0 },
         pages: { issues: 0, issueComments: 0, prReviewComments: 0 },
     };
-    // ---------- Stream 1: issues (skip PRs) ----------
     const issuesRes = await backfillStream(ctx, opts, sleep, {
         url: `${API}/repos/${opts.repoFullName}/issues?state=all&per_page=100` +
             (cursors.issues ? `&since=${encodeURIComponent(cursors.issues)}` : ''),
@@ -221,7 +188,6 @@ export async function backfillRepo(ctx, opts) {
     });
     result.ingested.issues = issuesRes.ingested;
     result.pages.issues = issuesRes.pages;
-    // ---------- Stream 2: repo-level issue comments ----------
     const commentsRes = await backfillStream(ctx, opts, sleep, {
         url: `${API}/repos/${opts.repoFullName}/issues/comments?per_page=100` +
             (cursors.issueComments ? `&since=${encodeURIComponent(cursors.issueComments)}` : ''),
@@ -230,7 +196,6 @@ export async function backfillRepo(ctx, opts) {
     });
     result.ingested.issueComments = commentsRes.ingested;
     result.pages.issueComments = commentsRes.pages;
-    // ---------- Stream 3: repo-level PR review comments ----------
     const prCommentsRes = await backfillStream(ctx, opts, sleep, {
         url: `${API}/repos/${opts.repoFullName}/pulls/comments?per_page=100` +
             (cursors.prReviewComments ? `&since=${encodeURIComponent(cursors.prReviewComments)}` : ''),

package/dist/connectors/github/deletion.d.ts

@@ -1,8 +1,7 @@
 import { type Context } from '../../api/index.js';
 export interface DeletionInput {
-    /** artifact_ref of the comment, e.g.,
-     *  'github://acme/repo/issue/42/comment/123' or
-     *  'github://acme/repo/pull/7/review_comment/456'. */
+    /** artifact_ref of the comment, e.g. 'github://acme/repo/issue/42/comment/123'
+     *  or 'github://acme/repo/pull/7/review_comment/456'. */
     artifactRef: string;
     /** Source idempotency key for this delete event (sha256 of artifact_ref + ':' + updated_at). */
     idempotencyKey: string;
@@ -16,20 +15,7 @@ export interface DeletionResult {
     status: DeletionStatus;
     archivedCount: number;
 }
-/**
- * Handle GitHub `issue_comment.deleted` and `pull_request_review_comment.deleted`.
- *
- * Filter by tenant_id + kind='raw'. Multi-row archive:
- * GitHub edits keep the same artifact_ref, so multiple active raw rows can
- * match a single deletion event. Archive ALL of them.
- *
- * The store runs ALL archives + the idempotency mark as one write: a per-row
- * failure rolls back the whole batch, idempotency included, so a retry
- * re-attempts cleanly instead of leaving searchable survivors.
- *
- * Tenant scope and kind='raw' filtering are load-bearing: without them a
- * deletion event from tenant A could archive tenant B's row sharing the same
- * artifact_ref, or accidentally target a distilled row.
- */
+/** Handle `issue_comment.deleted` and `pull_request_review_comment.deleted`: archive ALL active raw rows for the artifact_ref in one write.
+ *  Tenant and kind='raw' filtering are load-bearing: without them a deletion could archive another tenant's row or a distilled row. */
 export declare function handleCommentDeleted(ctx: Context, input: DeletionInput): Promise<DeletionResult>;
 //# sourceMappingURL=deletion.d.ts.map

package/dist/connectors/github/deletion.js

@@ -1,19 +1,6 @@
 import { requireGroup, storeFor } from '../../store/index.js';
-/**
- * Handle GitHub `issue_comment.deleted` and `pull_request_review_comment.deleted`.
- *
- * Filter by tenant_id + kind='raw'. Multi-row archive:
- * GitHub edits keep the same artifact_ref, so multiple active raw rows can
- * match a single deletion event. Archive ALL of them.
- *
- * The store runs ALL archives + the idempotency mark as one write: a per-row
- * failure rolls back the whole batch, idempotency included, so a retry
- * re-attempts cleanly instead of leaving searchable survivors.
- *
- * Tenant scope and kind='raw' filtering are load-bearing: without them a
- * deletion event from tenant A could archive tenant B's row sharing the same
- * artifact_ref, or accidentally target a distilled row.
- */
+/** Handle `issue_comment.deleted` and `pull_request_review_comment.deleted`: archive ALL active raw rows for the artifact_ref in one write.
+ *  Tenant and kind='raw' filtering are load-bearing: without them a deletion could archive another tenant's row or a distilled row. */
 export async function handleCommentDeleted(ctx, input) {
     const { artifactRef, idempotencyKey, deliveryId, eventName } = input;
     const done = await requireGroup(storeFor(ctx), 'connectorEvents').archiveDeletedArtifact({

package/dist/connectors/github/dlq.d.ts

@@ -1,32 +1,20 @@
 import type { Context } from '../../api/index.js';
-import { type DlqBucket, type DlqItem, type GithubDlqWrite } from '../../store/connectors/github.js';
+import { type DlqBucket, type DlqItem, type GitHubDlqInsert } from '../../store/connectors/github.js';
 import { type ConnectorDlq, type ReplayResult } from '../dlq.js';
 export type { DlqBucket, DlqItem };
-type GithubOwnColumns = Pick<GithubDlqWrite, 'eventName' | 'deliveryId' | 'installationId' | 'repoFullName'>;
+type GitHubOwnColumns = Pick<GitHubDlqInsert, 'eventName' | 'deliveryId' | 'installationId' | 'repoFullName'>;
 /** GitHub's dead-letter table; the event, delivery, installation and repo columns let an operator triage a row without re-reading the payload. */
-export declare const githubDlq: ConnectorDlq<GithubOwnColumns, DlqBucket, DlqItem>;
+export declare const githubDlq: ConnectorDlq<GitHubOwnColumns, DlqBucket, DlqItem>;
 export interface ReplayDlqOpts {
     /** Current webhook secret. If omitted, signature check is skipped (force-only path). */
     webhookSecret?: string;
-    /**
-     * Previous webhook secret during rotation.
-     * Operators rotating GITHUB_WEBHOOK_SECRET would otherwise be forced into
-     * --force on DLQ rows written under the old secret. Plumbed through to
-     * verifyGitHubSignature.previousSecret.
-     */
+    /** Previous webhook secret during rotation, passed to verifyGitHubSignature.previousSecret so old-secret DLQ rows need no --force. */
     previousSecret?: string;
     /** When true, skip signature verification (used for legacy entries after secret rotation). */
     force?: boolean;
 }
-/**
- * Hook the webhook route injects to actually re-ingest a row. Decoupling
- * the dispatch keeps this module free of every event-type handler — the
- * route already knows how to route an envelope, so it passes that capability
- * back in.
- *
- * No `idempotencyKey` field: ingest re-derives the key from the parsed event,
- * so a hook trusting a passed-in key would dedupe against a stale one.
- */
+/** Hook the webhook route injects to re-ingest a row, which keeps this module free of every event-type handler.
+ *  No `idempotencyKey` field: ingest re-derives it from the parsed event, and a passed-in key could be stale. */
 export type IngestHook = (ctx: Context, args: {
     rawPayload: string;
     eventName: string;
@@ -34,21 +22,8 @@ export type IngestHook = (ctx: Context, args: {
 }) => Promise<{
     memoryId: string | null;
 }>;
-/**
- * Replay a DLQ row through the normal ingest path. Behavior:
- *   1. Fetch row by id. Not found → `not_found`.
- *   2. If !force and webhookSecret provided, verify signature with the
- *      current secret. Fail → bump retry_count, `sig_fail`.
- *      Missing signature on the row → `sig_missing` (no bump; --force required).
- *   3. JSON.parse the raw payload. Fail → bump, `parse_error`.
- *   4. Type-guard the envelope. Fail → bump, `unhandled`.
- *   5. If an `ingestHook` is supplied, call it and return its memoryId.
- *      If not (dry-run path), bump retry_count and return status `replayed`
- *      with memoryId=null. The webhook route wires the real hook.
- *
- * Mirrors Slack's "always use current routing" policy: replays use the
- * deployment state NOW, not at the time of original DLQing.
- */
+/** Replay a DLQ row through the normal ingest path; a failed signature, parse or envelope check bumps retry_count.
+ *  Without an `ingestHook` (dry-run) it bumps retry_count and returns `replayed` with memoryId=null; replays use today's routing, not the original's. */
 export declare function replayDlqEntry(ctx: Context, id: number, opts?: ReplayDlqOpts & {
     ingestHook?: IngestHook;
 }): Promise<ReplayResult>;

package/dist/connectors/github/dlq.js

@@ -1,29 +1,16 @@
-import { bumpDlqRetry, dlqEntry, listDlqRows, } from '../../store/connectors/github.js';
+import { bumpGitHubDlqRetryCount, githubDlqEntry, listGitHubDlq, } from '../../store/connectors/github.js';
 import { replayFailed } from '../dlq.js';
 import { verifyGitHubSignature } from './signature.js';
 import { isGitHubWebhookEnvelope } from './types.js';
 /** GitHub's dead-letter table; the event, delivery, installation and repo columns let an operator triage a row without re-reading the payload. */
 export const githubDlq = {
     letter: (row) => ({ connector: 'github', ...row }),
-    list: listDlqRows,
+    list: listGitHubDlq,
 };
-/**
- * Replay a DLQ row through the normal ingest path. Behavior:
- *   1. Fetch row by id. Not found → `not_found`.
- *   2. If !force and webhookSecret provided, verify signature with the
- *      current secret. Fail → bump retry_count, `sig_fail`.
- *      Missing signature on the row → `sig_missing` (no bump; --force required).
- *   3. JSON.parse the raw payload. Fail → bump, `parse_error`.
- *   4. Type-guard the envelope. Fail → bump, `unhandled`.
- *   5. If an `ingestHook` is supplied, call it and return its memoryId.
- *      If not (dry-run path), bump retry_count and return status `replayed`
- *      with memoryId=null. The webhook route wires the real hook.
- *
- * Mirrors Slack's "always use current routing" policy: replays use the
- * deployment state NOW, not at the time of original DLQing.
- */
+/** Replay a DLQ row through the normal ingest path; a failed signature, parse or envelope check bumps retry_count.
+ *  Without an `ingestHook` (dry-run) it bumps retry_count and returns `replayed` with memoryId=null; replays use today's routing, not the original's. */
 export async function replayDlqEntry(ctx, id, opts = {}) {
-    const row = dlqEntry(ctx.hippoRoot, id);
+    const row = githubDlqEntry(ctx.hippoRoot, id);
     if (!row)
         return replayFailed('not_found', 0, `dlq id ${id} not found`);
     // Signature verification (current secret, not the one in effect when DLQed).
@@ -37,7 +24,7 @@ export async function replayDlqEntry(ctx, id, opts = {}) {
         return envelopeFailure;
     // Without an ingest hook this is a dry-run validation. Bump and report.
     if (!opts.ingestHook) {
-        bumpDlqRetry(ctx.hippoRoot, id);
+        bumpGitHubDlqRetryCount(ctx.hippoRoot, id);
         return {
             ok: true,
             status: 'replayed',
@@ -46,9 +33,7 @@ export async function replayDlqEntry(ctx, id, opts = {}) {
             reason: 'dry-run: no ingest hook supplied',
         };
     }
-    // Real replay path. The route's IngestHook is responsible for routing,
-    // idempotency, and writing the memory. The DLQ module only validates the
-    // surface and bumps the retry counter.
+    // Real replay path: the route's IngestHook owns routing, idempotency and the memory write; this module only validates and bumps the retry counter.
     // No idempotencyKey arg: the hook re-derives it from the parsed event (artifact_ref + updated_at).
     const eventName = row.eventName ?? '';
     const deliveryId = row.deliveryId ?? '';
@@ -57,7 +42,7 @@ export async function replayDlqEntry(ctx, id, opts = {}) {
         eventName,
         deliveryId,
     });
-    bumpDlqRetry(ctx.hippoRoot, id);
+    bumpGitHubDlqRetryCount(ctx.hippoRoot, id);
     return {
         ok: true,
         status: 'replayed',
@@ -77,7 +62,7 @@ function checkReplaySignature(hippoRoot, id, row, webhookSecret, previousSecret)
         previousSecret,
     });
     if (!sigOk) {
-        bumpDlqRetry(hippoRoot, id);
+        bumpGitHubDlqRetryCount(hippoRoot, id);
         return replayFailed('sig_fail', row.retryCount + 1, 'signature did not verify against current GITHUB_WEBHOOK_SECRET; pass --force to replay anyway');
     }
     return null;
@@ -89,15 +74,13 @@ function checkReplayEnvelope(hippoRoot, id, row) {
         parsed = JSON.parse(row.rawPayload);
     }
     catch (e) {
-        bumpDlqRetry(hippoRoot, id);
-        // SAFETY: this is a best-effort error message only; property access on
-        // any JS value is safe (undefined if absent), preserving the existing
-        // lenient formatting even when something non-Error was thrown.
+        bumpGitHubDlqRetryCount(hippoRoot, id);
+        // SAFETY: best-effort error message only; property access on any JS value is safe, even for a non-Error throw.
         const message = e.message;
         return replayFailed('parse_error', row.retryCount + 1, `still unparseable: ${message}`);
     }
     if (!isGitHubWebhookEnvelope(parsed)) {
-        bumpDlqRetry(hippoRoot, id);
+        bumpGitHubDlqRetryCount(hippoRoot, id);
         return replayFailed('unhandled', row.retryCount + 1, 'not a GitHub webhook envelope');
     }
     return null;

package/dist/connectors/github/ingest.d.ts

@@ -5,12 +5,8 @@ export interface IngestResult {
     status: IngestStatus;
     memoryId: string | null;
 }
-/**
- * Discriminated union of the four event shapes V1 ingests. The eventName
- * field MUST equal the X-GitHub-Event header value; computeIdempotencyKey
- * folds it into the dedupe key so the same body posted under two different
- * X-GitHub-Event headers produces two distinct keys (test 7).
- */
+/** Discriminated union of the four event shapes V1 ingests; eventName MUST equal the X-GitHub-Event header,
+ *  since computeIdempotencyKey folds it into the dedupe key. */
 export type IngestEvent = {
     eventName: 'issues';
     payload: GitHubIssueEvent;

package/dist/connectors/github/ingest.js

@@ -18,12 +18,8 @@ function transformEvent(event) {
             return prReviewCommentEventToRememberOpts(event.payload);
     }
 }
-/**
- * Extract the source-normalized identifier the idempotency key needs.
- * Backfill and webhook both produce IngestEvent objects describing the same
- * source revision, so deriving the key from these fields collapses both paths
- * onto the same dedupe row. Mirrors the artifactRef strings in transform.ts.
- */
+/** Extract the source-normalized identifier the idempotency key needs, so backfill and webhook deliveries of the same
+ *  source revision collapse onto one dedupe row. Mirrors the artifactRef strings in transform.ts. */
 function eventArtifactRef(event) {
     const repo = event.payload.repository?.full_name ?? 'unknown/unknown';
     switch (event.eventName) {
@@ -60,9 +56,7 @@ export async function ingestEvent(ctx, input) {
         return { status: 'duplicate', memoryId: seen.memoryId };
     const opts = transformEvent(input.event);
     if (!opts) {
-        // Empty body: no memory to write, but mark seen so a retry of the same
-        // empty event returns 'duplicate' (not 'skipped' again — that would
-        // re-run the transform on every retry).
+        // Empty body: no memory to write, but mark seen so a retry returns 'duplicate' instead of re-running the transform.
         await events.markEventSeen(event);
         return { status: 'skipped', memoryId: null };
     }
@@ -80,7 +74,7 @@ export async function ingestEvent(ctx, input) {
     }
 }
 async function rememberWithEventLog(ctx, event, opts) {
-    // No `|| 'connector:github'` fallback (see rememberWithEventLog in slack/ingest.ts for rationale).
+    // No `|| 'connector:github'` fallback: the caller always builds ctx with the connector subject (same as slack/ingest.ts).
     const result = await remember(ctx, { ...opts, untrusted: true, event });
     // Another worker logged this key between the pre-check and the write: its memory stands and ours was not stored.
     if (result.duplicate)

package/dist/connectors/github/octokit-client.d.ts

@@ -1,19 +1,12 @@
-/**
- * Octokit-shaped HTTP fetcher for the GitHub connector backfill.
- *
- * Production uses `realGitHubFetcher` against `https://api.github.com`.
- * Tests inject a fake `GitHubFetcher` so they never hit the network.
- *
- * Any non-200 response that is NOT a recognized rate-limit pause MUST throw
- * `GitHubFetchError`: an empty page would be an empty backfill with no operator signal.
- */
+/** Octokit-shaped HTTP fetcher for the GitHub backfill; tests inject a fake `GitHubFetcher`.
+ *  Any non-200 that is not a recognised rate-limit pause MUST throw `GitHubFetchError`, since an empty page would be a silent empty backfill. */
 import { type RateLimitInfo } from './ratelimit.js';
 import type { JsonValue } from '../../util/json.js';
 export declare class GitHubFetchError extends Error {
     readonly status: number;
     readonly bodyExcerpt: string;
     readonly url: string;
-    constructor(status: number, bodyExcerpt: string, url: string);
+    constructor(status: number, bodyExcerpt: string, url: string, options?: ErrorOptions);
 }
 export interface GitHubBackfillPage {
     readonly items: ReadonlyArray<JsonValue>;
@@ -24,12 +17,7 @@ export type GitHubFetcher = (args: {
     url: string;
     token: string;
 }) => Promise<GitHubBackfillPage>;
-/**
- * Parse the rel="next" URL from an RFC 5988 `Link` header.
- *
- * Header format: `<url1>; rel="next", <url2>; rel="last"`.
- * Returns the URL whose rel parameter is exactly `"next"`, or null.
- */
+/** Parse the rel="next" URL from an RFC 5988 `Link` header, or null. */
 export declare function parseNextLink(linkHeader: string): string | null;
 export declare const realGitHubFetcher: GitHubFetcher;
 //# sourceMappingURL=octokit-client.d.ts.map

package/dist/connectors/github/octokit-client.js

@@ -1,32 +1,22 @@
-/**
- * Octokit-shaped HTTP fetcher for the GitHub connector backfill.
- *
- * Production uses `realGitHubFetcher` against `https://api.github.com`.
- * Tests inject a fake `GitHubFetcher` so they never hit the network.
- *
- * Any non-200 response that is NOT a recognized rate-limit pause MUST throw
- * `GitHubFetchError`: an empty page would be an empty backfill with no operator signal.
- */
+/** Octokit-shaped HTTP fetcher for the GitHub backfill; tests inject a fake `GitHubFetcher`.
+ *  Any non-200 that is not a recognised rate-limit pause MUST throw `GitHubFetchError`, since an empty page would be a silent empty backfill. */
 import { parseRateLimit } from './ratelimit.js';
 import { fetchWithRetry } from '../../util/http-retry.js';
+import { readCappedJson, readCappedText } from '../../util/capped-json.js';
+import { errorMessage } from '../../util/log.js';
 export class GitHubFetchError extends Error {
     status;
     bodyExcerpt;
     url;
-    constructor(status, bodyExcerpt, url) {
-        super(`GitHub ${status} on ${url}: ${bodyExcerpt}`);
+    constructor(status, bodyExcerpt, url, options) {
+        super(`GitHub ${status} on ${url}: ${bodyExcerpt}`, options);
         this.status = status;
         this.bodyExcerpt = bodyExcerpt;
         this.url = url;
         this.name = 'GitHubFetchError';
     }
 }
-/**
- * Parse the rel="next" URL from an RFC 5988 `Link` header.
- *
- * Header format: `<url1>; rel="next", <url2>; rel="last"`.
- * Returns the URL whose rel parameter is exactly `"next"`, or null.
- */
+/** Parse the rel="next" URL from an RFC 5988 `Link` header, or null. */
 export function parseNextLink(linkHeader) {
     if (!linkHeader)
         return null;
@@ -43,6 +33,24 @@ function headersToRecord(h) {
 }
 const GITHUB_TIMEOUT_MS = 30_000;
 const ERROR_BODY_SNIPPET_CHARS = 256;
+const UTF8_MAX_BYTES_PER_CHAR = 4;
+// A page is at most 100 items (per_page=100), each with a body of up to 65,536 characters at 4 bytes, plus its fields.
+const GITHUB_MAX_REPLY_BYTES = 32 * 1024 * 1024;
+/** The items of a 200 list reply. A body that is not JSON, is over the cap or is not an array throws, so it never reaches ingest as items. */
+async function readItems(res, url) {
+    // The path only: a `next` link comes from the server, so its query string is not ours to print.
+    const path = new URL(url).pathname;
+    let body;
+    try {
+        body = await readCappedJson(res, GITHUB_MAX_REPLY_BYTES);
+    }
+    catch (err) {
+        throw new GitHubFetchError(res.status, errorMessage(err), path, { cause: err });
+    }
+    if (!Array.isArray(body))
+        throw new GitHubFetchError(res.status, 'the reply is not a JSON array', path);
+    return body;
+}
 export const realGitHubFetcher = async ({ url, token }) => {
     const res = await fetchWithRetry(url, {
         headers: {
@@ -56,15 +64,10 @@ export const realGitHubFetcher = async ({ url, token }) => {
     // Don't silently turn 401/403/404/500 into empty pages.
     if (res.status !== 200 && rateLimit.reason === 'none') {
         // The status is the error reported; a body that cannot be read only costs its snippet.
-        const body = await res.text().catch(() => '');
+        const body = await readCappedText(res, ERROR_BODY_SNIPPET_CHARS * UTF8_MAX_BYTES_PER_CHAR).catch(() => '');
         throw new GitHubFetchError(res.status, body.slice(0, ERROR_BODY_SNIPPET_CHARS), url);
     }
-    // SAFETY: GitHub's list/paginated endpoints (issues, comments, etc. — the
-    // only endpoints this backfill fetcher targets) return a JSON array body
-    // on 200; per-item shape is intentionally left as `unknown` here since
-    // GitHubBackfillPage.items is opaque to this seam (callers parse the
-    // shape they expect downstream).
-    const items = res.status === 200 ? (await res.json()) : [];
+    const items = res.status === 200 ? await readItems(res, url) : [];
     const link = res.headers.get('link') ?? '';
     const next = parseNextLink(link);
     return { items, next, rateLimit };

package/dist/connectors/github/ratelimit.d.ts

@@ -1,20 +1,9 @@
-/**
- * GitHub rate-limit header parser.
- *
- * GitHub returns 403 for primary rate-limit (with `X-RateLimit-Remaining: 0`
- * and `X-RateLimit-Reset: <epoch>`) and 429 + `Retry-After: <seconds>` for
- * secondary rate-limit. Backfill must pause and resume rather than error.
- */
+/** GitHub rate-limit header parser: 403 with `X-RateLimit-Remaining: 0` is primary, 429 with `Retry-After` is secondary.
+ *  Backfill must pause and resume rather than error. */
 export interface RateLimitInfo {
     readonly sleepSeconds: number;
     readonly reason: 'primary' | 'secondary' | 'none';
 }
-/**
- * Parse rate-limit signal from a GitHub HTTP response.
- *
- * @param headers Lower-cased HTTP response headers.
- * @param status  HTTP status code.
- * @param now     Optional current epoch seconds (for deterministic tests).
- */
+/** Parse the rate-limit signal from a GitHub response; `headers` are lower-cased and `now` (epoch seconds) is for deterministic tests. */
 export declare function parseRateLimit(headers: Record<string, string | undefined>, status: number, now?: number): RateLimitInfo;
 //# sourceMappingURL=ratelimit.d.ts.map

package/dist/connectors/github/ratelimit.js

@@ -1,17 +1,6 @@
-/**
- * GitHub rate-limit header parser.
- *
- * GitHub returns 403 for primary rate-limit (with `X-RateLimit-Remaining: 0`
- * and `X-RateLimit-Reset: <epoch>`) and 429 + `Retry-After: <seconds>` for
- * secondary rate-limit. Backfill must pause and resume rather than error.
- */
-/**
- * Parse rate-limit signal from a GitHub HTTP response.
- *
- * @param headers Lower-cased HTTP response headers.
- * @param status  HTTP status code.
- * @param now     Optional current epoch seconds (for deterministic tests).
- */
+/** GitHub rate-limit header parser: 403 with `X-RateLimit-Remaining: 0` is primary, 429 with `Retry-After` is secondary.
+ *  Backfill must pause and resume rather than error. */
+/** Parse the rate-limit signal from a GitHub response; `headers` are lower-cased and `now` (epoch seconds) is for deterministic tests. */
 export function parseRateLimit(headers, status, now) {
     const _now = now ?? Math.floor(Date.now() / 1000);
     if (status === 429) {

package/dist/connectors/github/scope.d.ts

@@ -1,8 +1,6 @@
 import type { GitHubRepository } from './types.js';
-/**
- * Map a GitHub repository to a hippo scope string. Default-private when
- * privacy is undetermined: cost of leaking public into private (recall returns
- * nothing) << cost of leaking private into public (data exposure).
- */
+/** Map a GitHub repository to a hippo scope string. Default-private when
+ *  privacy is undetermined: cost of leaking public into private (recall returns
+ *  nothing) << cost of leaking private into public (data exposure). */
 export declare function scopeFromRepository(repo: GitHubRepository | undefined): string;
 //# sourceMappingURL=scope.d.ts.map

package/dist/connectors/github/scope.js

@@ -1,8 +1,6 @@
-/**
- * Map a GitHub repository to a hippo scope string. Default-private when
- * privacy is undetermined: cost of leaking public into private (recall returns
- * nothing) << cost of leaking private into public (data exposure).
- */
+/** Map a GitHub repository to a hippo scope string. Default-private when
+ *  privacy is undetermined: cost of leaking public into private (recall returns
+ *  nothing) << cost of leaking private into public (data exposure). */
 export function scopeFromRepository(repo) {
     if (!repo)
         return 'github:private:unknown';