hippo-memory 1.68.0 → 1.70.0

package/dist/connectors/github/webhook.js

@@ -1,14 +1,14 @@
-import { envGithubWebhookSecret, envGithubWebhookSecretPrevious } from '../../env.js';
+import { envGithubWebhookSecret, envGithubWebhookSecretPrevious } from '../../util/env.js';
 import { verifyGitHubSignature } from './signature.js';
 import { isGitHubWebhookEnvelope, isGitHubIssueEvent, isGitHubIssueCommentEvent, isGitHubPullRequestEvent, isGitHubPullRequestReviewCommentEvent, } from './types.js';
 import { ingestEvent as ingestGitHubEvent } from './ingest.js';
 import { handleCommentDeleted as handleGitHubCommentDeleted } from './deletion.js';
-import { writeToDlq as writeToGitHubDlq } from './dlq.js';
+import { parkInDlq } from '../dlq.js';
+import { githubDlq } from './dlq.js';
 import { resolveTenantForGitHub } from './tenant-routing.js';
 import { computeDeletionKey as computeGitHubDeletionKey } from './signature.js';
-import { resolveTenantId } from '../../tenant.js';
-import { openHippoDb, closeHippoDb } from '../../db.js';
-import { HttpError, JSON_HEADERS, isHeaderString, readBody, sendJson, } from '../../http-util.js';
+import { resolveTenantId } from '../../store/tenant.js';
+import { HttpError, JSON_HEADERS, isHeaderString, closeIfBodyUnread, readWebhookBody, sendJson, } from '../../util/http-util.js';
 /**
  * GitHub webhook receiver. Mirrors the Slack route shape but with
  * GitHub-specific idioms:
@@ -25,10 +25,12 @@ import { HttpError, JSON_HEADERS, isHeaderString, readBody, sendJson, } from '..
  *
  * Bearer auth is skipped on purpose: server.ts checks isPublicRoute before calling this.
  */
-export async function handleGitHubEventsWebhook({ req, res, opts }) {
-    const rawBody = await readBody(req);
+export async function handleGitHubEventsWebhook(request, store) {
+    const { req, res, opts } = request;
+    // Secret and signature header before the body, so a caller with neither cannot make the server buffer one.
     const secret = envGithubWebhookSecret();
     if (!secret) {
+        closeIfBodyUnread(request);
         res.writeHead(404, JSON_HEADERS);
         res.end(JSON.stringify({ error: 'not found' }));
         return;
@@ -40,18 +42,22 @@ export async function handleGitHubEventsWebhook({ req, res, opts }) {
     const sigStr = isHeaderString(sigHdr) ? sigHdr : null;
     const eventName = isHeaderString(eventHdr) ? eventHdr : null;
     const deliveryId = isHeaderString(deliveryHdr) ? deliveryHdr : null;
-    if (sigStr === null ||
-        !verifyGitHubSignature({
-            rawBody,
-            signature: sigStr,
-            webhookSecret: secret,
-            previousSecret,
-        })) {
+    if (sigStr === null) {
+        closeIfBodyUnread(request);
+        throw new HttpError(401, 'invalid GitHub signature');
+    }
+    const rawBody = await readWebhookBody(request);
+    if (!verifyGitHubSignature({
+        rawBody,
+        signature: sigStr,
+        webhookSecret: secret,
+        previousSecret,
+    })) {
         throw new HttpError(401, 'invalid GitHub signature');
     }
     // Signature OK from here on. Everything else is ACK-200; bad envelopes go
     // to the DLQ and a human can replay later.
-    routeSignedDelivery({ hippoRoot: opts.hippoRoot, res, rawBody, eventName, deliveryId, signature: sigStr });
+    await routeSignedDelivery({ hippoRoot: opts.hippoRoot, store, res, rawBody, eventName, deliveryId, signature: sigStr });
 }
 const ALLOWED_EVENTS = new Set([
     'issues',
@@ -59,28 +65,23 @@ const ALLOWED_EVENTS = new Set([
     'pull_request',
     'pull_request_review_comment',
 ]);
-function parkInDlq(d, park) {
-    const db = openHippoDb(d.hippoRoot);
-    try {
-        writeToGitHubDlq(db, {
-            tenantId: park.tenantId,
-            rawPayload: d.rawBody,
-            error: park.error,
-            bucket: park.bucket,
-            eventName: d.eventName,
-            deliveryId: d.deliveryId,
-            signature: d.signature,
-            installationId: park.installationId,
-            repoFullName: park.repoFullName,
-        });
-    }
-    finally {
-        closeHippoDb(db);
-    }
+async function parkAndAck(d, park) {
+    const row = {
+        tenantId: park.tenantId,
+        rawPayload: d.rawBody,
+        error: park.error,
+        bucket: park.bucket,
+        eventName: d.eventName,
+        deliveryId: d.deliveryId,
+        signature: d.signature,
+        installationId: park.installationId,
+        repoFullName: park.repoFullName,
+    };
+    await parkInDlq(githubDlq, d.hippoRoot, row, d.store);
     sendJson(d.res, 200, { ok: true, status: 'dlq' });
 }
-function routeSignedDelivery(d) {
-    const body = parseEnvelopeOrPark(d);
+async function routeSignedDelivery(d) {
+    const body = await parseEnvelopeOrPark(d);
     if (body === null || d.deliveryId === null)
         return;
     const routing = {
@@ -89,18 +90,9 @@ function routeSignedDelivery(d) {
     };
     // Tenant resolution. Fail closed on multi-tenant installs with unknown
     // routing - same policy as Slack.
-    let resolvedTenant;
-    {
-        const db = openHippoDb(d.hippoRoot);
-        try {
-            resolvedTenant = resolveTenantForGitHub(db, routing);
-        }
-        finally {
-            closeHippoDb(db);
-        }
-    }
+    const resolvedTenant = await resolveTenantForGitHub(d.hippoRoot, routing, d.store);
     if (resolvedTenant === null) {
-        parkInDlq(d, {
+        await parkAndAck(d, {
             ...routing,
             tenantId: null,
             error: `unroutable: installation_id=${routing.installationId ?? '(none)'} repo=${routing.repoFullName ?? '(none)'}`,
@@ -112,11 +104,12 @@ function routeSignedDelivery(d) {
         hippoRoot: d.hippoRoot,
         tenantId: resolvedTenant,
         actor: { subject: 'connector:github', role: 'admin' },
+        store: d.store,
     };
-    if (dispatchGitHubEvent(d, ctx, body, d.deliveryId, routing))
+    if (await dispatchGitHubEvent(d, ctx, body, d.deliveryId, routing))
         return;
     // Header allow-listed but body shape didn't satisfy the matching guard.
-    parkInDlq(d, {
+    await parkAndAck(d, {
         ...routing,
         tenantId: resolvedTenant,
         error: `body shape did not match X-GitHub-Event=${d.eventName}`,
@@ -124,7 +117,7 @@ function routeSignedDelivery(d) {
     });
 }
 /** Answers ping and parks every unusable body; returns the envelope only when ingest should go on. */
-function parseEnvelopeOrPark(d) {
+async function parseEnvelopeOrPark(d) {
     // Cheap regex extraction of installation_id / repo for DLQ rows that fail
     // to JSON.parse - gives operators something to triage.
     const installationFromRaw = (() => {
@@ -135,8 +128,8 @@ function parseEnvelopeOrPark(d) {
         const m = d.rawBody.match(/"full_name"\s*:\s*"([^"]+)"/);
         return m ? m[1] : null;
     })();
-    const parkRaw = (error, bucket) => {
-        parkInDlq(d, {
+    const parkRaw = async (error, bucket) => {
+        await parkAndAck(d, {
             tenantId: resolveTenantId({}),
             error,
             bucket,
@@ -169,53 +162,57 @@ function parseEnvelopeOrPark(d) {
     return body;
 }
 /** Ingests, archives or parks by event header; false when no guard matched the body. */
-function dispatchGitHubEvent(d, ctx, body, deliveryId, routing) {
-    const { res, rawBody, eventName } = d;
-    const ingest = (event) => {
-        const r = ingestGitHubEvent(ctx, { event, rawBody, deliveryId });
-        sendJson(res, 200, { ok: true, status: r.status, memoryId: r.memoryId });
-        return true;
-    };
-    const archiveComment = (event, artifactRef, updatedAt) => {
+async function dispatchGitHubEvent(d, ctx, body, deliveryId, routing) {
+    const action = deliveryAction(d.eventName, body);
+    if (action === null)
+        return false;
+    if (action.kind === 'manual-review') {
+        // GitHub does fire issues.deleted (admin-initiated). Don't archive - V1
+        // policy is to log and let an operator decide. Archive could lose the
+        // memory if the issue is being moved between accounts.
+        await parkAndAck(d, {
+            ...routing,
+            tenantId: ctx.tenantId,
+            error: 'issues.deleted requires manual review',
+            bucket: 'unhandled',
+        });
+    }
+    else if (action.kind === 'archive-comment') {
         // The "deleted:" key namespace keeps this from colliding with the ingest key for the
         // same artifact; a shared key would make hasSeenKey skip the archive.
-        const idempotencyKey = computeGitHubDeletionKey(artifactRef, updatedAt);
-        const r = handleGitHubCommentDeleted(ctx, {
-            artifactRef,
+        const idempotencyKey = computeGitHubDeletionKey(action.artifactRef, action.updatedAt);
+        const r = await handleGitHubCommentDeleted(ctx, {
+            artifactRef: action.artifactRef,
             idempotencyKey,
             deliveryId,
-            eventName: event,
+            eventName: action.eventName,
         });
-        sendJson(res, 200, { ok: true, status: r.status, archivedCount: r.archivedCount });
-        return true;
-    };
-    // Dispatch by event header. Type guards cross-check the body shape against
-    // the header so a payload of one event type cannot satisfy another's guard.
+        sendJson(d.res, 200, { ok: true, status: r.status, archivedCount: r.archivedCount });
+    }
+    else {
+        const r = await ingestGitHubEvent(ctx, { event: action.event, rawBody: d.rawBody, deliveryId });
+        sendJson(d.res, 200, { ok: true, status: r.status, memoryId: r.memoryId });
+    }
+    return true;
+}
+// Dispatch by event header. Type guards cross-check the body shape against
+// the header so a payload of one event type cannot satisfy another's guard.
+function deliveryAction(eventName, body) {
     if (eventName === 'issues' && isGitHubIssueEvent(body, 'issues')) {
-        if (body.action === 'deleted') {
-            // GitHub does fire issues.deleted (admin-initiated). Don't archive - V1
-            // policy is to log and let an operator decide. Archive could lose the
-            // memory if the issue is being moved between accounts.
-            parkInDlq(d, {
-                ...routing,
-                tenantId: ctx.tenantId,
-                error: 'issues.deleted requires manual review',
-                bucket: 'unhandled',
-            });
-            return true;
-        }
-        return ingest({ eventName: 'issues', payload: body });
+        if (body.action === 'deleted')
+            return { kind: 'manual-review' };
+        return { kind: 'ingest', event: { eventName: 'issues', payload: body } };
     }
     if (eventName === 'issue_comment' && isGitHubIssueCommentEvent(body, 'issue_comment')) {
         if (body.action === 'deleted') {
             const repo = body.repository?.full_name ?? '';
             const artifactRef = `github://${repo}/issue/${body.issue.number}/comment/${body.comment.id}`;
-            return archiveComment(eventName, artifactRef, body.comment.updated_at ?? null);
+            return { kind: 'archive-comment', eventName, artifactRef, updatedAt: body.comment.updated_at ?? null };
         }
-        return ingest({ eventName: 'issue_comment', payload: body });
+        return { kind: 'ingest', event: { eventName: 'issue_comment', payload: body } };
     }
     if (eventName === 'pull_request' && isGitHubPullRequestEvent(body, 'pull_request')) {
-        return ingest({ eventName: 'pull_request', payload: body });
+        return { kind: 'ingest', event: { eventName: 'pull_request', payload: body } };
     }
     if (eventName === 'pull_request_review_comment' &&
         isGitHubPullRequestReviewCommentEvent(body, 'pull_request_review_comment')) {
@@ -223,10 +220,10 @@ function dispatchGitHubEvent(d, ctx, body, deliveryId, routing) {
             const repo = body.repository?.full_name ?? '';
             const artifactRef = `github://${repo}/pull/${body.pull_request.number}/review_comment/${body.comment.id}`;
             // Same "deleted:" key namespace as the issue_comment branch above.
-            return archiveComment(eventName, artifactRef, body.comment.updated_at ?? null);
+            return { kind: 'archive-comment', eventName, artifactRef, updatedAt: body.comment.updated_at ?? null };
         }
-        return ingest({ eventName: 'pull_request_review_comment', payload: body });
+        return { kind: 'ingest', event: { eventName: 'pull_request_review_comment', payload: body } };
     }
-    return false;
+    return null;
 }
 //# sourceMappingURL=webhook.js.map

package/dist/connectors/slack/backfill.d.ts

@@ -1,4 +1,4 @@
-import type { Context } from '../../api.js';
+import type { Context } from '../../api/index.js';
 import type { SlackMessageEvent } from './types.js';
 import type { ChannelMeta } from './scope.js';
 export interface SlackHistoryPage {

package/dist/connectors/slack/backfill.js

@@ -1,29 +1,5 @@
-import { openHippoDb, closeHippoDb } from '../../db.js';
+import { saveSlackCursor, slackCursor } from '../../store/connectors/slack.js';
 import { ingestMessage } from './ingest.js';
-function readCursor(root, tenantId, channelId) {
-    const db = openHippoDb(root);
-    try {
-        // SAFETY: row shape matches the single `latest_ts` column named in the
-        // SELECT above; sqlite returns undefined when no row matches.
-        const row = db
-            .prepare(`SELECT latest_ts FROM slack_cursors WHERE tenant_id=? AND channel_id=?`)
-            .get(tenantId, channelId);
-        return row?.latest_ts ?? null;
-    }
-    finally {
-        closeHippoDb(db);
-    }
-}
-function writeCursor(root, tenantId, channelId, latestTs) {
-    const db = openHippoDb(root);
-    try {
-        db.prepare(`INSERT INTO slack_cursors (tenant_id, channel_id, latest_ts, updated_at) VALUES (?,?,?,?)
-       ON CONFLICT(tenant_id, channel_id) DO UPDATE SET latest_ts = excluded.latest_ts, updated_at = excluded.updated_at`).run(tenantId, channelId, latestTs, new Date().toISOString());
-    }
-    finally {
-        closeHippoDb(db);
-    }
-}
 /**
  * Page through `conversations.history` via the injected fetcher and ingest each
  * message. The cursor is persisted to `slack_cursors` after every page so a
@@ -38,7 +14,7 @@ export async function backfillChannel(ctx, opts) {
     // first page only. `cursor` starts null — Slack mints the next-page token
     // and we feed it back. Mixing the two would feed a numeric ts as an opaque
     // cursor and break against the live API on rerun.
-    const resumeFrom = readCursor(ctx.hippoRoot, ctx.tenantId, opts.channel.id);
+    const resumeFrom = slackCursor(ctx.hippoRoot, ctx.tenantId, opts.channel.id);
     let cursor = null;
     let ingested = 0;
     let pages = 0;
@@ -51,7 +27,7 @@ export async function backfillChannel(ctx, opts) {
         });
         pages++;
         for (const msg of page.messages) {
-            const r = ingestMessage(ctx, {
+            const r = await ingestMessage(ctx, {
                 teamId: opts.teamId,
                 channel: opts.channel,
                 message: msg,
@@ -63,12 +39,12 @@ export async function backfillChannel(ctx, opts) {
                 latestTs = msg.ts;
             if (opts.maxMessages && ingested >= opts.maxMessages) {
                 if (latestTs)
-                    writeCursor(ctx.hippoRoot, ctx.tenantId, opts.channel.id, latestTs);
+                    saveSlackCursor(ctx.hippoRoot, ctx.tenantId, opts.channel.id, latestTs);
                 return { ingested, pages };
             }
         }
         if (latestTs)
-            writeCursor(ctx.hippoRoot, ctx.tenantId, opts.channel.id, latestTs);
+            saveSlackCursor(ctx.hippoRoot, ctx.tenantId, opts.channel.id, latestTs);
         if (!page.next_cursor)
             break;
         cursor = page.next_cursor;

package/dist/connectors/slack/deletion.d.ts

@@ -1,4 +1,4 @@
-import { type Context } from '../../api.js';
+import { type Context } from '../../api/index.js';
 export interface DeletionInput {
     teamId: string;
     channelId: string;
@@ -10,10 +10,7 @@ export interface DeletionResult {
     status: DeletionStatus;
     memoryId: string | null;
 }
-/**
- * Handle Slack `message_deleted`. `afterArchive` runs inside the archive's own
- * SAVEPOINT, so the slack_event_log row commits with the archive or not at all
- * and a crash cannot leave a retry hitting `not_found` instead of `duplicate`.
- */
-export declare function handleMessageDeleted(ctx: Context, input: DeletionInput): DeletionResult;
+/** Handles Slack `message_deleted`. The store logs the event in the archive's own transaction, so a crash cannot leave a retry
+ *  answering `not_found` where `duplicate` is due. */
+export declare function handleMessageDeleted(ctx: Context, input: DeletionInput): Promise<DeletionResult>;
 //# sourceMappingURL=deletion.d.ts.map

package/dist/connectors/slack/deletion.js

@@ -1,54 +1,26 @@
-import { archiveRaw } from '../../api.js';
-import { openHippoDb, closeHippoDb } from '../../db.js';
-import { markEventSeen, hasSeenEvent } from './idempotency.js';
-/**
- * Handle Slack `message_deleted`. `afterArchive` runs inside the archive's own
- * SAVEPOINT, so the slack_event_log row commits with the archive or not at all
- * and a crash cannot leave a retry hitting `not_found` instead of `duplicate`.
- */
-export function handleMessageDeleted(ctx, input) {
-    const db = openHippoDb(ctx.hippoRoot);
-    let memoryId = null;
-    try {
-        if (hasSeenEvent(db, input.eventId)) {
-            return { status: 'duplicate', memoryId: null };
-        }
-        const ref = `slack://${input.teamId}/${input.channelId}/${input.deletedTs}`;
-        // Tenant scope is load-bearing: without `tenant_id = ?` a deletion event
-        // from tenant A could archive tenant B's raw row sharing the same
-        // artifact_ref. The `kind = 'raw'` filter prevents accidentally targeting
-        // distilled rows.
-        // SAFETY: row comes from the SELECT above, which projects only the
-        // `id` column; `.get` returns undefined when no row matches.
-        const row = db
-            .prepare(`SELECT id FROM memories WHERE artifact_ref = ? AND tenant_id = ? AND kind = 'raw'`)
-            .get(ref, ctx.tenantId);
-        memoryId = row?.id ?? null;
-    }
-    finally {
-        closeHippoDb(db);
-    }
+import { archiveRaw } from '../../api/index.js';
+import { requireGroup, storeFor } from '../../store/index.js';
+/** Handles Slack `message_deleted`. The store logs the event in the archive's own transaction, so a crash cannot leave a retry
+ *  answering `not_found` where `duplicate` is due. */
+export async function handleMessageDeleted(ctx, input) {
+    const events = requireGroup(storeFor(ctx), 'connectorEvents');
+    const event = { connector: 'slack', eventId: input.eventId };
+    // The tenant in the lookup is load-bearing: without it a deletion event from
+    // tenant A could archive tenant B's raw row sharing the same artifact_ref.
+    const target = await events.deletionTarget({
+        event,
+        artifactRef: `slack://${input.teamId}/${input.channelId}/${input.deletedTs}`,
+        tenantId: ctx.tenantId,
+    });
+    if (target.seen)
+        return { status: 'duplicate', memoryId: null };
+    const memoryId = target.memoryId;
     if (!memoryId) {
-        // No row to archive — still mark the deletion event seen so a retry returns
-        // 'duplicate'. There is nothing to roll back here, so the second-handle
-        // pattern is fine for this branch.
-        const db2 = openHippoDb(ctx.hippoRoot);
-        try {
-            markEventSeen(db2, input.eventId, null);
-        }
-        finally {
-            closeHippoDb(db2);
-        }
+        // No row to archive: the event is still marked seen, so a retry answers 'duplicate'.
+        await events.markEventSeen(event);
         return { status: 'not_found', memoryId: null };
     }
-    // Archive + event-log mark commit together via afterArchive. The hook
-    // receives the same db handle the archive is using, so the INSERT lives
-    // inside the SAVEPOINT.
-    archiveRaw(ctx, memoryId, `source_deleted:slack:${input.teamId}:${input.channelId}:${input.deletedTs}`, {
-        afterArchive: (sameDb, archivedId) => {
-            markEventSeen(sameDb, input.eventId, archivedId);
-        },
-    });
+    await archiveRaw(ctx, memoryId, `source_deleted:slack:${input.teamId}:${input.channelId}:${input.deletedTs}`, { event });
     return { status: 'archived', memoryId };
 }
 //# sourceMappingURL=deletion.js.map

package/dist/connectors/slack/dlq.d.ts

@@ -1,44 +1,9 @@
-import type { DatabaseSyncLike } from '../../db.js';
-import { type Context } from '../../api.js';
-export type DlqBucket = 'parse_error' | 'unroutable' | 'signature_fail';
-export interface DlqItem {
-    id: number;
-    tenantId: string;
-    teamId: string | null;
-    rawPayload: string;
-    error: string;
-    receivedAt: string;
-    retriedAt: string | null;
-    bucket: DlqBucket | string;
-    retryCount: number;
-    signature: string | null;
-    slackTimestamp: string | null;
-}
-/**
- * writeToDlq is bucket-aware. `bucket` defaults to
- * 'parse_error' to preserve the legacy single-arg call sites while letting
- * new callers tag rows for `hippo slack dlq replay` triage.
- *
- * `tenantId: null` means "no tenant resolved" (unroutable team). Stored as
- * the sentinel '__unroutable__' so the column stays NOT NULL — a mismatch
- * the listing CLI surfaces explicitly.
- */
-export interface WriteDlqOpts {
-    tenantId: string | null;
-    rawPayload: string;
-    error: string;
-    bucket?: DlqBucket;
-    teamId?: string | null;
-    signature?: string | null;
-    slackTimestamp?: string | null;
-}
-export declare function writeToDlq(db: DatabaseSyncLike, opts: WriteDlqOpts): number;
-export declare function listDlq(db: DatabaseSyncLike, opts: {
-    tenantId: string;
-    limit?: number;
-}): DlqItem[];
-export declare function getDlqEntry(db: DatabaseSyncLike, id: number): DlqItem | null;
-export declare function markDlqRetried(db: DatabaseSyncLike, id: number): void;
+import { type Context } from '../../api/index.js';
+import { type DlqBucket, type DlqItem, type SlackDlqInsert } from '../../store/connectors/slack.js';
+import { type ConnectorDlq, type ReplayResult } from '../dlq.js';
+export type { DlqBucket, DlqItem };
+/** Slack's dead-letter table; `teamId` and `slackTimestamp` are the columns only it has. */
+export declare const slackDlq: ConnectorDlq<Pick<SlackDlqInsert, 'teamId' | 'slackTimestamp'>, DlqBucket, DlqItem>;
 export interface ReplayDlqOpts {
     /** Skip signature verification when the row's signature/timestamp are missing or stale. */
     force?: boolean;
@@ -49,13 +14,6 @@ export interface ReplayDlqOpts {
     /** Skew window override for tests. */
     skewSeconds?: number;
 }
-export interface ReplayDlqResult {
-    ok: boolean;
-    status: string;
-    memoryId: string | null;
-    retryCount: number;
-    reason?: string;
-}
 /**
  * Replay a DLQ row through the normal ingest path. Used by `hippo slack dlq
  * replay <id> [--force]`.
@@ -75,5 +33,5 @@ export interface ReplayDlqResult {
  * envelope was DLQed. That is intentional: the DLQ exists to be drained after
  * the operator fixed the routing.
  */
-export declare function replayDlqEntry(ctx: Pick<Context, 'hippoRoot'>, id: number, opts?: ReplayDlqOpts): ReplayDlqResult;
+export declare function replayDlqEntry(ctx: Pick<Context, 'hippoRoot'>, id: number, opts?: ReplayDlqOpts): Promise<ReplayResult>;
 //# sourceMappingURL=dlq.d.ts.map