hippo-memory 1.56.0 → 1.58.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/agent-memories/claude-code.js +1 -1
- package/dist/agent-memories/gemini.js +1 -1
- package/dist/api-errors.d.ts +27 -0
- package/dist/api-errors.js +37 -0
- package/dist/api.d.ts +21 -14
- package/dist/api.js +97 -71
- package/dist/audit.d.ts +4 -0
- package/dist/audit.js +11 -0
- package/dist/autolearn.d.ts +1 -1
- package/dist/autolearn.js +7 -5
- package/dist/capture-contract.d.ts +47 -0
- package/dist/capture-contract.js +49 -0
- package/dist/capture-error.js +2 -1
- package/dist/capture.d.ts +0 -13
- package/dist/capture.js +5 -66
- package/dist/card-detail.d.ts +1 -1
- package/dist/card-detail.js +1 -1
- package/dist/cli/shared.d.ts +137 -0
- package/dist/cli/shared.js +834 -0
- package/dist/cli/sleep.d.ts +10 -0
- package/dist/cli/sleep.js +171 -0
- package/dist/cli.d.ts +0 -7
- package/dist/cli.js +322 -1827
- package/dist/client.js +9 -0
- package/dist/codex-patch.js +1 -1
- package/dist/compaction-record.d.ts +1 -1
- package/dist/compaction-record.js +3 -2
- package/dist/config.d.ts +5 -0
- package/dist/config.js +17 -0
- package/dist/connectors/github/dlq.js +5 -2
- package/dist/connectors/github/octokit-client.js +4 -2
- package/dist/connectors/github/webhook.d.ts +19 -0
- package/dist/connectors/github/webhook.js +313 -0
- package/dist/connectors/slack/dlq.js +6 -2
- package/dist/connectors/slack/web-client.js +7 -5
- package/dist/connectors/slack/webhook.d.ts +22 -0
- package/dist/connectors/slack/webhook.js +203 -0
- package/dist/consolidate.d.ts +10 -0
- package/dist/consolidate.js +38 -35
- package/dist/context-auto.d.ts +3 -0
- package/dist/context-auto.js +34 -0
- package/dist/customer-notes.js +16 -14
- package/dist/dag.js +3 -2
- package/dist/dashboard.js +3 -2
- package/dist/db.d.ts +12 -0
- package/dist/db.js +62 -1
- package/dist/decisions.js +11 -9
- package/dist/doctor.js +5 -0
- package/dist/embedding-provider.js +3 -3
- package/dist/embeddings.d.ts +4 -4
- package/dist/embeddings.js +72 -16
- package/dist/eval-stats.d.ts +58 -0
- package/dist/eval-stats.js +111 -0
- package/dist/extract.js +3 -2
- package/dist/goals.d.ts +49 -25
- package/dist/goals.js +39 -22
- package/dist/graph-extract.js +1 -1
- package/dist/graph-recall.d.ts +1 -1
- package/dist/graph-recall.js +1 -1
- package/dist/graph.js +1 -1
- package/dist/hooks.d.ts +1 -3
- package/dist/hooks.js +2 -4
- package/dist/http-retry.d.ts +21 -0
- package/dist/http-retry.js +50 -0
- package/dist/http-util.d.ts +39 -0
- package/dist/http-util.js +56 -0
- package/dist/importers.d.ts +2 -0
- package/dist/importers.js +16 -5
- package/dist/incidents.js +13 -11
- package/dist/index.d.ts +5 -2
- package/dist/index.js +5 -2
- package/dist/judgment.js +10 -17
- package/dist/log.d.ts +25 -0
- package/dist/log.js +48 -0
- package/dist/mcp/server.js +224 -308
- package/dist/mcp/tool-args.d.ts +21 -0
- package/dist/mcp/tool-args.js +80 -0
- package/dist/memory.d.ts +19 -0
- package/dist/memory.js +41 -2
- package/dist/overlap-index.d.ts +7 -0
- package/dist/overlap-index.js +38 -0
- package/dist/pilot-arm.d.ts +9 -0
- package/dist/pilot-arm.js +47 -0
- package/dist/policies.js +14 -12
- package/dist/predictions.js +11 -9
- package/dist/processes.js +16 -14
- package/dist/project-briefs.js +19 -16
- package/dist/project-identity.d.ts +1 -1
- package/dist/project-identity.js +25 -1
- package/dist/prompt-recall.js +1 -1
- package/dist/raw-archive.js +7 -6
- package/dist/recall-history.d.ts +5 -0
- package/dist/recall-history.js +9 -0
- package/dist/recall-pipeline.d.ts +101 -0
- package/dist/recall-pipeline.js +313 -0
- package/dist/recall-scope.d.ts +24 -1
- package/dist/recall-scope.js +29 -2
- package/dist/refine-llm.js +3 -2
- package/dist/reject-flow.js +6 -9
- package/dist/rejection.d.ts +2 -1
- package/dist/rejection.js +2 -1
- package/dist/search.d.ts +0 -20
- package/dist/search.js +16 -51
- package/dist/secret-detect.d.ts +13 -1
- package/dist/secret-detect.js +33 -1
- package/dist/server.d.ts +3 -1
- package/dist/server.js +1854 -2566
- package/dist/session-digest.js +2 -1
- package/dist/shared.js +7 -6
- package/dist/skills.js +17 -15
- package/dist/store-cards.d.ts +53 -0
- package/dist/store-cards.js +512 -0
- package/dist/store.d.ts +2 -89
- package/dist/store.js +10 -566
- package/dist/tenant.d.ts +22 -0
- package/dist/tenant.js +26 -0
- package/dist/token-ledger.d.ts +4 -2
- package/dist/token-ledger.js +2 -2
- package/dist/tokenize.d.ts +2 -0
- package/dist/tokenize.js +8 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/dist/connectors/slack/ratelimit.d.ts +0 -9
- package/dist/connectors/slack/ratelimit.js +0 -18
package/dist/client.js
CHANGED
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
* Network errors (ECONNREFUSED on a stale pidfile, etc.) propagate as the
|
|
14
14
|
* native fetch failure so the caller can detect them and self-heal.
|
|
15
15
|
*/
|
|
16
|
+
/** A write the local server has not answered in this long is stuck; the caller treats it as delivery-unknown. */
|
|
17
|
+
const SERVER_TIMEOUT_MS = 30_000;
|
|
16
18
|
function buildHeaders(apiKey, withBody) {
|
|
17
19
|
const headers = {};
|
|
18
20
|
if (withBody)
|
|
@@ -58,6 +60,7 @@ export async function remember(serverUrl, apiKey, opts) {
|
|
|
58
60
|
const res = await fetch(`${serverUrl}/v1/memories`, {
|
|
59
61
|
method: 'POST',
|
|
60
62
|
headers: buildHeaders(apiKey, true),
|
|
63
|
+
signal: AbortSignal.timeout(SERVER_TIMEOUT_MS),
|
|
61
64
|
body: JSON.stringify(opts),
|
|
62
65
|
});
|
|
63
66
|
if (!res.ok)
|
|
@@ -69,6 +72,7 @@ export async function forget(serverUrl, apiKey, id) {
|
|
|
69
72
|
const res = await fetch(`${serverUrl}/v1/memories/${encodeURIComponent(id)}`, {
|
|
70
73
|
method: 'DELETE',
|
|
71
74
|
headers: buildHeaders(apiKey, false),
|
|
75
|
+
signal: AbortSignal.timeout(SERVER_TIMEOUT_MS),
|
|
72
76
|
});
|
|
73
77
|
if (!res.ok)
|
|
74
78
|
await throwForStatus(res);
|
|
@@ -79,6 +83,7 @@ export async function promote(serverUrl, apiKey, id) {
|
|
|
79
83
|
const res = await fetch(`${serverUrl}/v1/memories/${encodeURIComponent(id)}/promote`, {
|
|
80
84
|
method: 'POST',
|
|
81
85
|
headers: buildHeaders(apiKey, false),
|
|
86
|
+
signal: AbortSignal.timeout(SERVER_TIMEOUT_MS),
|
|
82
87
|
});
|
|
83
88
|
if (!res.ok)
|
|
84
89
|
await throwForStatus(res);
|
|
@@ -89,6 +94,7 @@ export async function archiveRaw(serverUrl, apiKey, id, reason) {
|
|
|
89
94
|
const res = await fetch(`${serverUrl}/v1/memories/${encodeURIComponent(id)}/archive`, {
|
|
90
95
|
method: 'POST',
|
|
91
96
|
headers: buildHeaders(apiKey, true),
|
|
97
|
+
signal: AbortSignal.timeout(SERVER_TIMEOUT_MS),
|
|
92
98
|
body: JSON.stringify({ reason }),
|
|
93
99
|
});
|
|
94
100
|
if (!res.ok)
|
|
@@ -123,6 +129,9 @@ export function classifyTransportFailure(err) {
|
|
|
123
129
|
}
|
|
124
130
|
if (message.includes('socket hang up') || message.includes('fetch failed'))
|
|
125
131
|
return 'delivery-unknown';
|
|
132
|
+
// The request was sent and the server went quiet, so it may have committed.
|
|
133
|
+
if (err.name === 'TimeoutError')
|
|
134
|
+
return 'delivery-unknown';
|
|
126
135
|
return 'none';
|
|
127
136
|
}
|
|
128
137
|
//# sourceMappingURL=client.js.map
|
package/dist/codex-patch.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Files named by a Codex apply_patch call, read the way openai/codex codex-rs/apply-patch reads them
|
|
2
2
|
// (parser.rs for the body, invocation.rs for the shell forms it runs as apply_patch).
|
|
3
3
|
import * as path from 'path';
|
|
4
|
-
import { isStringValue } from './capture.js';
|
|
4
|
+
import { isStringValue } from './capture-contract.js';
|
|
5
5
|
/** Codex prints this line only when a patch applied (apply-patch lib.rs). */
|
|
6
6
|
export const PATCH_SUCCESS_LINE = 'Success. Updated the following files:';
|
|
7
7
|
const BEGIN_PATCH = '*** Begin Patch';
|
|
@@ -2,7 +2,7 @@ import { type DatabaseSyncLike } from './db.js';
|
|
|
2
2
|
/** PostCompact has 10 s in all (PreCompact 30 s), so a locked store must be given up on early. */
|
|
3
3
|
export declare const COMPACTION_DB_WAIT_MS = 2000;
|
|
4
4
|
/** Tested verbatim: Claude Code hands PreCompact stdout to the summariser as instructions. */
|
|
5
|
-
export declare const PRE_COMPACT_INSTRUCTION = "In your summary, add a last section titled 'Memories for hippo'. List, one per line starting with '- ', each lesson learned, decision made (with its reason) and correction the user gave in this session that should outlive it. Write each as a standalone sentence that names its subject. Leave out anything an earlier summary already listed under 'Memories for hippo'
|
|
5
|
+
export declare const PRE_COMPACT_INSTRUCTION = "In your summary, add a last section titled 'Memories for hippo'. List, one per line starting with '- ', each lesson learned, decision made (with its reason) and correction the user gave in this session that should outlive it. Write each as a standalone sentence that names its subject. Leave out anything an earlier summary already listed under 'Memories for hippo', and anything this session already saved with `hippo remember`. Write '- none' if nothing new remains.";
|
|
6
6
|
/** Long enough that a live hook has finished with its own record. */
|
|
7
7
|
export declare const REPLAY_AFTER_MS: number;
|
|
8
8
|
/** Claude Code deletes transcripts after 30 days, so an older gap can never be filled. */
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// One record per Claude Code compaction, and what turns its summary into kept memories.
|
|
2
2
|
import * as fs from 'fs';
|
|
3
3
|
import * as path from 'path';
|
|
4
|
-
import { errorMessage,
|
|
4
|
+
import { errorMessage, readTranscriptTail, truncateCodePointSafe } from './capture.js';
|
|
5
|
+
import { isObjectLike, isStringValue } from './capture-contract.js';
|
|
5
6
|
import { compactSummaryBody, parseCompactionItems, selectItemRows } from './compaction-items.js';
|
|
6
7
|
import { loadConfig } from './config.js';
|
|
7
8
|
import { closeHippoDb, isSqliteBusy, openHippoDb } from './db.js';
|
|
@@ -14,7 +15,7 @@ import { resolveTenantId } from './tenant.js';
|
|
|
14
15
|
/** PostCompact has 10 s in all (PreCompact 30 s), so a locked store must be given up on early. */
|
|
15
16
|
export const COMPACTION_DB_WAIT_MS = 2000;
|
|
16
17
|
/** Tested verbatim: Claude Code hands PreCompact stdout to the summariser as instructions. */
|
|
17
|
-
export const PRE_COMPACT_INSTRUCTION = "In your summary, add a last section titled 'Memories for hippo'. List, one per line starting with '- ', each lesson learned, decision made (with its reason) and correction the user gave in this session that should outlive it. Write each as a standalone sentence that names its subject. Leave out anything an earlier summary already listed under 'Memories for hippo'
|
|
18
|
+
export const PRE_COMPACT_INSTRUCTION = "In your summary, add a last section titled 'Memories for hippo'. List, one per line starting with '- ', each lesson learned, decision made (with its reason) and correction the user gave in this session that should outlive it. Write each as a standalone sentence that names its subject. Leave out anything an earlier summary already listed under 'Memories for hippo', and anything this session already saved with `hippo remember`. Write '- none' if nothing new remains.";
|
|
18
19
|
const SUMMARY_MAX_CHARS = 256 * 1024;
|
|
19
20
|
/** Long enough that a live hook has finished with its own record. */
|
|
20
21
|
export const REPLAY_AFTER_MS = 10 * 60_000;
|
package/dist/config.d.ts
CHANGED
|
@@ -145,6 +145,11 @@ export interface HippoConfig {
|
|
|
145
145
|
deliveryLedger: {
|
|
146
146
|
enabled: boolean;
|
|
147
147
|
};
|
|
148
|
+
/** Pilot holdout (src/pilot-arm.ts): share of sessions, in basis points 0-10000, that get no memories pushed.
|
|
149
|
+
* Default 0 = off. Read from the store the token ledger writes to. */
|
|
150
|
+
pilot: {
|
|
151
|
+
holdoutRateBp: number;
|
|
152
|
+
};
|
|
148
153
|
}
|
|
149
154
|
export declare function loadConfig(hippoRoot: string): HippoConfig;
|
|
150
155
|
//# sourceMappingURL=config.d.ts.map
|
package/dist/config.js
CHANGED
|
@@ -88,6 +88,9 @@ const DEFAULT_CONFIG = {
|
|
|
88
88
|
deliveryLedger: {
|
|
89
89
|
enabled: false,
|
|
90
90
|
},
|
|
91
|
+
pilot: {
|
|
92
|
+
holdoutRateBp: 0,
|
|
93
|
+
},
|
|
91
94
|
};
|
|
92
95
|
function isMemoryValueConfig(value) {
|
|
93
96
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
@@ -115,6 +118,19 @@ function deliveryLedgerEnabled(value) {
|
|
|
115
118
|
`(got ${JSON.stringify(value)}) - using false.`);
|
|
116
119
|
return false;
|
|
117
120
|
}
|
|
121
|
+
// Only an integer 0..10000 counts; anything else warns and turns the pilot off.
|
|
122
|
+
function pilotHoldoutRate(value) {
|
|
123
|
+
if (value === undefined)
|
|
124
|
+
return 0;
|
|
125
|
+
const rate = value?.holdoutRateBp;
|
|
126
|
+
if (rate === undefined && value !== null && value.constructor === Object)
|
|
127
|
+
return 0;
|
|
128
|
+
if (Number.isInteger(rate) && rate >= 0 && rate <= 10000)
|
|
129
|
+
return rate;
|
|
130
|
+
console.error(`Warning: config.json's "pilot" must be an object like {"holdoutRateBp": 2000}, an integer from 0 to 10000 ` +
|
|
131
|
+
`(got ${JSON.stringify(value)}) - using 0 (pilot off).`);
|
|
132
|
+
return 0;
|
|
133
|
+
}
|
|
118
134
|
function agentMemoryTools(value) {
|
|
119
135
|
if (value === undefined || value === null)
|
|
120
136
|
return null;
|
|
@@ -226,6 +242,7 @@ export function loadConfig(hippoRoot) {
|
|
|
226
242
|
},
|
|
227
243
|
agentMemories: { tools: agentMemoryTools(raw.agentMemories?.tools) },
|
|
228
244
|
deliveryLedger: { enabled: deliveryLedgerEnabled(raw.deliveryLedger) },
|
|
245
|
+
pilot: { holdoutRateBp: pilotHoldoutRate(raw.pilot) },
|
|
229
246
|
};
|
|
230
247
|
}
|
|
231
248
|
catch (err) {
|
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import { openHippoDb, closeHippoDb } from '../../db.js';
|
|
2
2
|
import { verifyGitHubSignature } from './signature.js';
|
|
3
3
|
import { isGitHubWebhookEnvelope } from './types.js';
|
|
4
|
+
import { redactPayload, DLQ_REDACTED_NOTE } from '../../secret-detect.js';
|
|
4
5
|
export function writeToDlq(db, opts) {
|
|
6
|
+
const rawPayload = redactPayload(opts.rawPayload);
|
|
7
|
+
const redacted = rawPayload !== opts.rawPayload;
|
|
5
8
|
const result = db
|
|
6
9
|
.prepare(`INSERT INTO github_dlq
|
|
7
10
|
(tenant_id, raw_payload, error, event_name, delivery_id, signature,
|
|
8
11
|
installation_id, repo_full_name, received_at, bucket)
|
|
9
12
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
10
|
-
.run(opts.tenantId ?? '__unroutable__',
|
|
13
|
+
.run(opts.tenantId ?? '__unroutable__', rawPayload, redacted ? `${opts.error}; ${DLQ_REDACTED_NOTE}` : opts.error, opts.eventName ?? null, opts.deliveryId ?? null, redacted ? null : opts.signature ?? null, opts.installationId ?? null, opts.repoFullName ?? null, new Date().toISOString(), opts.bucket ?? 'parse_error');
|
|
11
14
|
return Number(result.lastInsertRowid);
|
|
12
15
|
}
|
|
13
16
|
const SELECT_COLUMNS = `id, tenant_id, raw_payload, error, event_name, delivery_id,
|
|
@@ -107,7 +110,7 @@ export async function replayDlqEntry(ctx, id, opts = {}) {
|
|
|
107
110
|
status: 'sig_missing',
|
|
108
111
|
memoryId: null,
|
|
109
112
|
retryCount: row.retryCount,
|
|
110
|
-
reason: 'legacy
|
|
113
|
+
reason: 'row has no signature (legacy, or redacted before storing); pass --force to replay',
|
|
111
114
|
};
|
|
112
115
|
}
|
|
113
116
|
const sigOk = verifyGitHubSignature({
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* operator signal, so this code path is now load-bearing.
|
|
11
11
|
*/
|
|
12
12
|
import { parseRateLimit } from './ratelimit.js';
|
|
13
|
+
import { fetchWithRetry } from '../../http-retry.js';
|
|
13
14
|
export class GitHubFetchError extends Error {
|
|
14
15
|
status;
|
|
15
16
|
bodyExcerpt;
|
|
@@ -42,14 +43,15 @@ function headersToRecord(h) {
|
|
|
42
43
|
});
|
|
43
44
|
return Object.fromEntries(entries);
|
|
44
45
|
}
|
|
46
|
+
const GITHUB_TIMEOUT_MS = 30_000;
|
|
45
47
|
export const realGitHubFetcher = async ({ url, token }) => {
|
|
46
|
-
const res = await
|
|
48
|
+
const res = await fetchWithRetry(url, {
|
|
47
49
|
headers: {
|
|
48
50
|
Authorization: `Bearer ${token}`,
|
|
49
51
|
Accept: 'application/vnd.github+json',
|
|
50
52
|
'X-GitHub-Api-Version': '2022-11-28',
|
|
51
53
|
},
|
|
52
|
-
});
|
|
54
|
+
}, { timeoutMs: GITHUB_TIMEOUT_MS });
|
|
53
55
|
const headers = headersToRecord(res.headers);
|
|
54
56
|
const rateLimit = parseRateLimit(headers, res.status);
|
|
55
57
|
// Codex P1 #4: don't silently turn 401/403/404/500 into empty pages.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { type WebhookRequest } from '../../http-util.js';
|
|
2
|
+
/**
|
|
3
|
+
* GitHub webhook receiver. Mirrors the Slack route shape but with
|
|
4
|
+
* GitHub-specific idioms:
|
|
5
|
+
* 1. HMAC SHA-256 over the raw body (X-Hub-Signature-256), no timestamp.
|
|
6
|
+
* 2. Event type discriminated by the X-GitHub-Event header (not body.type).
|
|
7
|
+
* 3. X-GitHub-Delivery is required audit metadata (NOT the dedupe seam - see
|
|
8
|
+
* computeIdempotencyKey, which folds the signed body into the key so a
|
|
9
|
+
* replayed body with a fresh delivery UUID still dedupes).
|
|
10
|
+
* 4. Tenant resolved by installation.id → github_installations, then by
|
|
11
|
+
* repository.full_name → github_repositories (PAT-mode multi-tenant).
|
|
12
|
+
* 5. ALWAYS ACK 200 on signed envelopes (DLQ included). 401 only on bad
|
|
13
|
+
* signature; 404 only when GITHUB_WEBHOOK_SECRET is unset (don't expose
|
|
14
|
+
* the route's existence on builds where it's gated off).
|
|
15
|
+
*
|
|
16
|
+
* Bearer auth is skipped on purpose: server.ts checks isPublicRoute before calling this.
|
|
17
|
+
*/
|
|
18
|
+
export declare function handleGitHubEventsWebhook({ req, res, opts }: WebhookRequest): Promise<void>;
|
|
19
|
+
//# sourceMappingURL=webhook.d.ts.map
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
import { verifyGitHubSignature } from './signature.js';
|
|
2
|
+
import { isGitHubWebhookEnvelope, isGitHubIssueEvent, isGitHubIssueCommentEvent, isGitHubPullRequestEvent, isGitHubPullRequestReviewCommentEvent, } from './types.js';
|
|
3
|
+
import { ingestEvent as ingestGitHubEvent } from './ingest.js';
|
|
4
|
+
import { handleCommentDeleted as handleGitHubCommentDeleted } from './deletion.js';
|
|
5
|
+
import { writeToDlq as writeToGitHubDlq } from './dlq.js';
|
|
6
|
+
import { resolveTenantForGitHub } from './tenant-routing.js';
|
|
7
|
+
import { computeDeletionKey as computeGitHubDeletionKey } from './signature.js';
|
|
8
|
+
import { resolveTenantId } from '../../tenant.js';
|
|
9
|
+
import { openHippoDb, closeHippoDb } from '../../db.js';
|
|
10
|
+
import { adminActor } from '../../api.js';
|
|
11
|
+
import { HttpError, JSON_HEADERS, isHeaderString, readBody, sendJson, } from '../../http-util.js';
|
|
12
|
+
/**
|
|
13
|
+
* GitHub webhook receiver. Mirrors the Slack route shape but with
|
|
14
|
+
* GitHub-specific idioms:
|
|
15
|
+
* 1. HMAC SHA-256 over the raw body (X-Hub-Signature-256), no timestamp.
|
|
16
|
+
* 2. Event type discriminated by the X-GitHub-Event header (not body.type).
|
|
17
|
+
* 3. X-GitHub-Delivery is required audit metadata (NOT the dedupe seam - see
|
|
18
|
+
* computeIdempotencyKey, which folds the signed body into the key so a
|
|
19
|
+
* replayed body with a fresh delivery UUID still dedupes).
|
|
20
|
+
* 4. Tenant resolved by installation.id → github_installations, then by
|
|
21
|
+
* repository.full_name → github_repositories (PAT-mode multi-tenant).
|
|
22
|
+
* 5. ALWAYS ACK 200 on signed envelopes (DLQ included). 401 only on bad
|
|
23
|
+
* signature; 404 only when GITHUB_WEBHOOK_SECRET is unset (don't expose
|
|
24
|
+
* the route's existence on builds where it's gated off).
|
|
25
|
+
*
|
|
26
|
+
* Bearer auth is skipped on purpose: server.ts checks isPublicRoute before calling this.
|
|
27
|
+
*/
|
|
28
|
+
export async function handleGitHubEventsWebhook({ req, res, opts }) {
|
|
29
|
+
const rawBody = await readBody(req);
|
|
30
|
+
const secret = process.env.GITHUB_WEBHOOK_SECRET;
|
|
31
|
+
if (!secret) {
|
|
32
|
+
res.writeHead(404, JSON_HEADERS);
|
|
33
|
+
res.end(JSON.stringify({ error: 'not found' }));
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
const previousSecret = process.env.GITHUB_WEBHOOK_SECRET_PREVIOUS;
|
|
37
|
+
const sigHdr = req.headers['x-hub-signature-256'];
|
|
38
|
+
const eventHdr = req.headers['x-github-event'];
|
|
39
|
+
const deliveryHdr = req.headers['x-github-delivery'];
|
|
40
|
+
const sigStr = isHeaderString(sigHdr) ? sigHdr : null;
|
|
41
|
+
const eventName = isHeaderString(eventHdr) ? eventHdr : null;
|
|
42
|
+
const deliveryId = isHeaderString(deliveryHdr) ? deliveryHdr : null;
|
|
43
|
+
if (sigStr === null ||
|
|
44
|
+
!verifyGitHubSignature({
|
|
45
|
+
rawBody,
|
|
46
|
+
signature: sigStr,
|
|
47
|
+
webhookSecret: secret,
|
|
48
|
+
previousSecret,
|
|
49
|
+
})) {
|
|
50
|
+
throw new HttpError(401, 'invalid GitHub signature');
|
|
51
|
+
}
|
|
52
|
+
// Signature OK from here on. Everything else is ACK-200; bad envelopes go
|
|
53
|
+
// to the DLQ and a human can replay later.
|
|
54
|
+
// Cheap regex extraction of installation_id / repo for DLQ rows that fail
|
|
55
|
+
// to JSON.parse - gives operators something to triage.
|
|
56
|
+
const installationFromRaw = (() => {
|
|
57
|
+
const m = rawBody.match(/"installation"\s*:\s*\{[^}]*"id"\s*:\s*(\d+)/);
|
|
58
|
+
return m ? m[1] : null;
|
|
59
|
+
})();
|
|
60
|
+
const repoFromRaw = (() => {
|
|
61
|
+
const m = rawBody.match(/"full_name"\s*:\s*"([^"]+)"/);
|
|
62
|
+
return m ? m[1] : null;
|
|
63
|
+
})();
|
|
64
|
+
if (deliveryId === null) {
|
|
65
|
+
// Body was signed but caller omitted the audit header. Park.
|
|
66
|
+
const db = openHippoDb(opts.hippoRoot);
|
|
67
|
+
try {
|
|
68
|
+
writeToGitHubDlq(db, {
|
|
69
|
+
tenantId: resolveTenantId({}),
|
|
70
|
+
rawPayload: rawBody,
|
|
71
|
+
error: 'missing X-GitHub-Delivery header',
|
|
72
|
+
bucket: 'parse_error',
|
|
73
|
+
eventName,
|
|
74
|
+
deliveryId: null,
|
|
75
|
+
signature: sigStr,
|
|
76
|
+
installationId: installationFromRaw,
|
|
77
|
+
repoFullName: repoFromRaw,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
closeHippoDb(db);
|
|
82
|
+
}
|
|
83
|
+
sendJson(res, 200, { ok: true, status: 'dlq' });
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
// Ping fires once at hook creation. Don't ingest, don't DLQ - just pong.
|
|
87
|
+
if (eventName === 'ping') {
|
|
88
|
+
sendJson(res, 200, { pong: true });
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
const ALLOWED_EVENTS = new Set([
|
|
92
|
+
'issues',
|
|
93
|
+
'issue_comment',
|
|
94
|
+
'pull_request',
|
|
95
|
+
'pull_request_review_comment',
|
|
96
|
+
]);
|
|
97
|
+
if (eventName === null || !ALLOWED_EVENTS.has(eventName)) {
|
|
98
|
+
const db = openHippoDb(opts.hippoRoot);
|
|
99
|
+
try {
|
|
100
|
+
writeToGitHubDlq(db, {
|
|
101
|
+
tenantId: resolveTenantId({}),
|
|
102
|
+
rawPayload: rawBody,
|
|
103
|
+
error: `unhandled event: ${eventName ?? '(missing X-GitHub-Event)'}`,
|
|
104
|
+
bucket: 'unhandled',
|
|
105
|
+
eventName,
|
|
106
|
+
deliveryId,
|
|
107
|
+
signature: sigStr,
|
|
108
|
+
installationId: installationFromRaw,
|
|
109
|
+
repoFullName: repoFromRaw,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
finally {
|
|
113
|
+
closeHippoDb(db);
|
|
114
|
+
}
|
|
115
|
+
sendJson(res, 200, { ok: true, status: 'dlq' });
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
let body;
|
|
119
|
+
try {
|
|
120
|
+
body = JSON.parse(rawBody);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
const db = openHippoDb(opts.hippoRoot);
|
|
124
|
+
try {
|
|
125
|
+
writeToGitHubDlq(db, {
|
|
126
|
+
tenantId: resolveTenantId({}),
|
|
127
|
+
rawPayload: rawBody,
|
|
128
|
+
error: 'invalid JSON',
|
|
129
|
+
bucket: 'parse_error',
|
|
130
|
+
eventName,
|
|
131
|
+
deliveryId,
|
|
132
|
+
signature: sigStr,
|
|
133
|
+
installationId: installationFromRaw,
|
|
134
|
+
repoFullName: repoFromRaw,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
finally {
|
|
138
|
+
closeHippoDb(db);
|
|
139
|
+
}
|
|
140
|
+
sendJson(res, 200, { ok: true, status: 'dlq' });
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (body === undefined || !isGitHubWebhookEnvelope(body)) {
|
|
144
|
+
const db = openHippoDb(opts.hippoRoot);
|
|
145
|
+
try {
|
|
146
|
+
writeToGitHubDlq(db, {
|
|
147
|
+
tenantId: resolveTenantId({}),
|
|
148
|
+
rawPayload: rawBody,
|
|
149
|
+
error: 'not a GitHub webhook envelope',
|
|
150
|
+
bucket: 'parse_error',
|
|
151
|
+
eventName,
|
|
152
|
+
deliveryId,
|
|
153
|
+
signature: sigStr,
|
|
154
|
+
installationId: installationFromRaw,
|
|
155
|
+
repoFullName: repoFromRaw,
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
finally {
|
|
159
|
+
closeHippoDb(db);
|
|
160
|
+
}
|
|
161
|
+
sendJson(res, 200, { ok: true, status: 'dlq' });
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const installationId = body.installation?.id != null ? String(body.installation.id) : null;
|
|
165
|
+
const repoFullName = body.repository?.full_name ?? null;
|
|
166
|
+
// Tenant resolution. Fail closed on multi-tenant installs with unknown
|
|
167
|
+
// routing - same policy as Slack.
|
|
168
|
+
let resolvedTenant;
|
|
169
|
+
{
|
|
170
|
+
const db = openHippoDb(opts.hippoRoot);
|
|
171
|
+
try {
|
|
172
|
+
resolvedTenant = resolveTenantForGitHub(db, {
|
|
173
|
+
installationId,
|
|
174
|
+
repoFullName,
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
finally {
|
|
178
|
+
closeHippoDb(db);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
if (resolvedTenant === null) {
|
|
182
|
+
const db = openHippoDb(opts.hippoRoot);
|
|
183
|
+
try {
|
|
184
|
+
writeToGitHubDlq(db, {
|
|
185
|
+
tenantId: null,
|
|
186
|
+
rawPayload: rawBody,
|
|
187
|
+
error: `unroutable: installation_id=${installationId ?? '(none)'} repo=${repoFullName ?? '(none)'}`,
|
|
188
|
+
bucket: 'unroutable',
|
|
189
|
+
eventName,
|
|
190
|
+
deliveryId,
|
|
191
|
+
signature: sigStr,
|
|
192
|
+
installationId,
|
|
193
|
+
repoFullName,
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
finally {
|
|
197
|
+
closeHippoDb(db);
|
|
198
|
+
}
|
|
199
|
+
sendJson(res, 200, { ok: true, status: 'dlq' });
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
const ctx = {
|
|
203
|
+
hippoRoot: opts.hippoRoot,
|
|
204
|
+
tenantId: resolvedTenant,
|
|
205
|
+
actor: adminActor('connector:github'),
|
|
206
|
+
};
|
|
207
|
+
// Dispatch by event header. Type guards cross-check the body shape against
|
|
208
|
+
// the header so a payload of one event type cannot satisfy another's guard.
|
|
209
|
+
if (eventName === 'issues' && isGitHubIssueEvent(body, 'issues')) {
|
|
210
|
+
if (body.action === 'deleted') {
|
|
211
|
+
// GitHub does fire issues.deleted (admin-initiated). Don't archive - V1
|
|
212
|
+
// policy is to log and let an operator decide. Archive could lose the
|
|
213
|
+
// memory if the issue is being moved between accounts.
|
|
214
|
+
const db = openHippoDb(opts.hippoRoot);
|
|
215
|
+
try {
|
|
216
|
+
writeToGitHubDlq(db, {
|
|
217
|
+
tenantId: resolvedTenant,
|
|
218
|
+
rawPayload: rawBody,
|
|
219
|
+
error: 'issues.deleted requires manual review',
|
|
220
|
+
bucket: 'unhandled',
|
|
221
|
+
eventName,
|
|
222
|
+
deliveryId,
|
|
223
|
+
signature: sigStr,
|
|
224
|
+
installationId,
|
|
225
|
+
repoFullName,
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
finally {
|
|
229
|
+
closeHippoDb(db);
|
|
230
|
+
}
|
|
231
|
+
sendJson(res, 200, { ok: true, status: 'dlq' });
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
const ingestInput = { eventName: 'issues', payload: body };
|
|
235
|
+
const r = ingestGitHubEvent(ctx, { event: ingestInput, rawBody, deliveryId });
|
|
236
|
+
sendJson(res, 200, { ok: true, status: r.status, memoryId: r.memoryId });
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
if (eventName === 'issue_comment' && isGitHubIssueCommentEvent(body, 'issue_comment')) {
|
|
240
|
+
if (body.action === 'deleted') {
|
|
241
|
+
const repo = body.repository?.full_name ?? '';
|
|
242
|
+
const artifactRef = `github://${repo}/issue/${body.issue.number}/comment/${body.comment.id}`;
|
|
243
|
+
// The "deleted:" key namespace keeps this from colliding with the ingest key for the
|
|
244
|
+
// same artifact; a shared key would make hasSeenKey skip the archive.
|
|
245
|
+
const idempotencyKey = computeGitHubDeletionKey(artifactRef, body.comment.updated_at ?? null);
|
|
246
|
+
const r = handleGitHubCommentDeleted(ctx, {
|
|
247
|
+
artifactRef,
|
|
248
|
+
idempotencyKey,
|
|
249
|
+
deliveryId,
|
|
250
|
+
eventName,
|
|
251
|
+
});
|
|
252
|
+
sendJson(res, 200, { ok: true, status: r.status, archivedCount: r.archivedCount });
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
const ingestInput = { eventName: 'issue_comment', payload: body };
|
|
256
|
+
const r = ingestGitHubEvent(ctx, { event: ingestInput, rawBody, deliveryId });
|
|
257
|
+
sendJson(res, 200, { ok: true, status: r.status, memoryId: r.memoryId });
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
if (eventName === 'pull_request' && isGitHubPullRequestEvent(body, 'pull_request')) {
|
|
261
|
+
const ingestInput = { eventName: 'pull_request', payload: body };
|
|
262
|
+
const r = ingestGitHubEvent(ctx, { event: ingestInput, rawBody, deliveryId });
|
|
263
|
+
sendJson(res, 200, { ok: true, status: r.status, memoryId: r.memoryId });
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
if (eventName === 'pull_request_review_comment' &&
|
|
267
|
+
isGitHubPullRequestReviewCommentEvent(body, 'pull_request_review_comment')) {
|
|
268
|
+
if (body.action === 'deleted') {
|
|
269
|
+
const repo = body.repository?.full_name ?? '';
|
|
270
|
+
const artifactRef = `github://${repo}/pull/${body.pull_request.number}/review_comment/${body.comment.id}`;
|
|
271
|
+
// Same "deleted:" key namespace as the issue_comment branch above.
|
|
272
|
+
const idempotencyKey = computeGitHubDeletionKey(artifactRef, body.comment.updated_at ?? null);
|
|
273
|
+
const r = handleGitHubCommentDeleted(ctx, {
|
|
274
|
+
artifactRef,
|
|
275
|
+
idempotencyKey,
|
|
276
|
+
deliveryId,
|
|
277
|
+
eventName,
|
|
278
|
+
});
|
|
279
|
+
sendJson(res, 200, { ok: true, status: r.status, archivedCount: r.archivedCount });
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
const ingestInput = {
|
|
283
|
+
eventName: 'pull_request_review_comment',
|
|
284
|
+
payload: body,
|
|
285
|
+
};
|
|
286
|
+
const r = ingestGitHubEvent(ctx, { event: ingestInput, rawBody, deliveryId });
|
|
287
|
+
sendJson(res, 200, { ok: true, status: r.status, memoryId: r.memoryId });
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
// Header allow-listed but body shape didn't satisfy the matching guard.
|
|
291
|
+
{
|
|
292
|
+
const db = openHippoDb(opts.hippoRoot);
|
|
293
|
+
try {
|
|
294
|
+
writeToGitHubDlq(db, {
|
|
295
|
+
tenantId: resolvedTenant,
|
|
296
|
+
rawPayload: rawBody,
|
|
297
|
+
error: `body shape did not match X-GitHub-Event=${eventName}`,
|
|
298
|
+
bucket: 'parse_error',
|
|
299
|
+
eventName,
|
|
300
|
+
deliveryId,
|
|
301
|
+
signature: sigStr,
|
|
302
|
+
installationId,
|
|
303
|
+
repoFullName,
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
finally {
|
|
307
|
+
closeHippoDb(db);
|
|
308
|
+
}
|
|
309
|
+
sendJson(res, 200, { ok: true, status: 'dlq' });
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
//# sourceMappingURL=webhook.js.map
|
|
@@ -5,12 +5,16 @@ import { resolveTenantForTeam } from './tenant-routing.js';
|
|
|
5
5
|
import { verifySlackSignature } from './signature.js';
|
|
6
6
|
import { isSlackEventEnvelope, isSlackMessageEvent } from './types.js';
|
|
7
7
|
import { handleMessageDeleted } from './deletion.js';
|
|
8
|
+
import { redactPayload, DLQ_REDACTED_NOTE } from '../../secret-detect.js';
|
|
8
9
|
export function writeToDlq(db, opts) {
|
|
10
|
+
const rawPayload = redactPayload(opts.rawPayload);
|
|
11
|
+
// A redacted body can never match its signature, so the row keeps none and replay needs --force.
|
|
12
|
+
const redacted = rawPayload !== opts.rawPayload;
|
|
9
13
|
const result = db
|
|
10
14
|
.prepare(`INSERT INTO slack_dlq
|
|
11
15
|
(tenant_id, team_id, raw_payload, error, received_at, bucket, signature, slack_timestamp)
|
|
12
16
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
13
|
-
.run(opts.tenantId ?? '__unroutable__', opts.teamId ?? null,
|
|
17
|
+
.run(opts.tenantId ?? '__unroutable__', opts.teamId ?? null, rawPayload, redacted ? `${opts.error}; ${DLQ_REDACTED_NOTE}` : opts.error, new Date().toISOString(), opts.bucket ?? 'parse_error', redacted ? null : opts.signature ?? null, opts.slackTimestamp ?? null);
|
|
14
18
|
return Number(result.lastInsertRowid);
|
|
15
19
|
}
|
|
16
20
|
export function listDlq(db, opts) {
|
|
@@ -95,7 +99,7 @@ export function replayDlqEntry(ctx, id, opts = {}) {
|
|
|
95
99
|
status: 'sig_missing',
|
|
96
100
|
memoryId: null,
|
|
97
101
|
retryCount: row.retryCount,
|
|
98
|
-
reason: '
|
|
102
|
+
reason: 'row has no signature/timestamp (legacy, or redacted before storing); pass --force to replay',
|
|
99
103
|
};
|
|
100
104
|
}
|
|
101
105
|
if (opts.signingSecret) {
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { fetchWithRetry } from '
|
|
1
|
+
import { fetchWithRetry, isRetryableStatus } from '../../http-retry.js';
|
|
2
2
|
function isJsonString(value) {
|
|
3
3
|
return typeof value === 'string';
|
|
4
4
|
}
|
|
5
|
+
const SLACK_TIMEOUT_MS = 30_000;
|
|
5
6
|
export function slackHistoryFetcher(token, fetchImpl) {
|
|
6
7
|
return async ({ channelId, cursor, oldest }) => {
|
|
7
8
|
const url = new URL('https://slack.com/api/conversations.history');
|
|
@@ -11,11 +12,12 @@ export function slackHistoryFetcher(token, fetchImpl) {
|
|
|
11
12
|
url.searchParams.set('cursor', cursor);
|
|
12
13
|
if (oldest)
|
|
13
14
|
url.searchParams.set('oldest', oldest);
|
|
14
|
-
const r = await fetchWithRetry({
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
fetchImpl,
|
|
15
|
+
const r = await fetchWithRetry(url, { method: 'GET', headers: { authorization: `Bearer ${token}` } }, {
|
|
16
|
+
timeoutMs: SLACK_TIMEOUT_MS,
|
|
17
|
+
fetchFn: fetchImpl,
|
|
18
18
|
});
|
|
19
|
+
if (isRetryableStatus(r.status))
|
|
20
|
+
throw new Error(`slack: still rate-limited or unavailable (HTTP ${r.status})`);
|
|
19
21
|
// SAFETY: body is the Slack `conversations.history` response; per the
|
|
20
22
|
// documented shape it's `{ ok, error?, messages?, response_metadata? }`.
|
|
21
23
|
const body = (await r.json());
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type WebhookRequest } from '../../http-util.js';
|
|
2
|
+
/**
|
|
3
|
+
* Slack Events API webhook. Auth is signature-based (HMAC over the raw
|
|
4
|
+
* body with SLACK_SIGNING_SECRET); Bearer is NOT required, which is why
|
|
5
|
+
* this route is in PUBLIC_ROUTES. The route is responsible for:
|
|
6
|
+
* 1. Echoing the one-time url_verification challenge.
|
|
7
|
+
* 2. Verifying the HMAC on every other inbound payload.
|
|
8
|
+
* 3. Resolving body.team_id → tenantId via slack_workspaces, falling
|
|
9
|
+
* back to HIPPO_TENANT then 'default'.
|
|
10
|
+
* 4. Dispatching event_callback envelopes to ingestMessage /
|
|
11
|
+
* handleMessageDeleted.
|
|
12
|
+
* 5. Parking malformed or unhandled payloads in slack_dlq and STILL
|
|
13
|
+
* ACKing 200 - Slack retries forever otherwise.
|
|
14
|
+
*
|
|
15
|
+
* When SLACK_SIGNING_SECRET is unset we return 404, not
|
|
16
|
+
* 503, so an external probe cannot distinguish "route gated off by config"
|
|
17
|
+
* from "route does not exist on this build".
|
|
18
|
+
*
|
|
19
|
+
* Bearer auth is skipped on purpose: server.ts checks isPublicRoute before calling this.
|
|
20
|
+
*/
|
|
21
|
+
export declare function handleSlackEventsWebhook({ req, res, opts }: WebhookRequest): Promise<void>;
|
|
22
|
+
//# sourceMappingURL=webhook.d.ts.map
|