hippo-memory 1.62.0 → 1.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/agent-memories/apply.d.ts +2 -0
- package/dist/agent-memories/apply.js +23 -1
- package/dist/agent-memories/claude-code.d.ts +2 -1
- package/dist/agent-memories/claude-code.js +4 -4
- package/dist/agent-memories/report.d.ts +1 -0
- package/dist/agent-memories/report.js +2 -2
- package/dist/agent-memories/sync.js +37 -23
- package/dist/ambient-store.d.ts +4 -4
- package/dist/ambient-store.js +4 -3
- package/dist/api/assemble.d.ts +7 -10
- package/dist/api/assemble.js +7 -12
- package/dist/api/audit.d.ts +2 -2
- package/dist/api/audit.js +2 -2
- package/dist/api/auth.d.ts +6 -6
- package/dist/api/auth.js +5 -7
- package/dist/api/context-select.d.ts +4 -3
- package/dist/api/context-select.js +9 -11
- package/dist/api/context-types.d.ts +12 -17
- package/dist/api/context.d.ts +3 -2
- package/dist/api/context.js +20 -20
- package/dist/api/dormant.js +1 -1
- package/dist/api/drill-down.d.ts +4 -4
- package/dist/api/drill-down.js +22 -10
- package/dist/api/outcome.d.ts +8 -13
- package/dist/api/outcome.js +13 -17
- package/dist/api/promote.d.ts +5 -8
- package/dist/api/quarantine.js +3 -2
- package/dist/api/recall-types.d.ts +43 -55
- package/dist/api/recall.d.ts +3 -3
- package/dist/api/recall.js +7 -18
- package/dist/api/remember.d.ts +2 -2
- package/dist/api/sleep.d.ts +7 -35
- package/dist/api/sleep.js +3 -2
- package/dist/api/tokens.d.ts +2 -2
- package/dist/api/tokens.js +2 -2
- package/dist/api/types.d.ts +6 -10
- package/dist/api/types.js +2 -4
- package/dist/audit-prune.d.ts +4 -6
- package/dist/audit-prune.js +3 -5
- package/dist/audit.js +11 -33
- package/dist/auth.d.ts +8 -9
- package/dist/auth.js +5 -7
- package/dist/autolearn.d.ts +1 -1
- package/dist/autolearn.js +1 -1
- package/dist/availability.js +3 -5
- package/dist/capture/command.d.ts +5 -4
- package/dist/capture/command.js +8 -22
- package/dist/capture/compact.d.ts +1 -1
- package/dist/capture/compact.js +9 -13
- package/dist/capture/extract.js +37 -115
- package/dist/capture-error.d.ts +1 -1
- package/dist/capture-error.js +1 -1
- package/dist/churn-git.d.ts +1 -1
- package/dist/churn-git.js +1 -1
- package/dist/cli/audit.js +3 -4
- package/dist/cli/auth.js +3 -6
- package/dist/cli/curate.d.ts +1 -1
- package/dist/cli/curate.js +8 -24
- package/dist/cli/dag.js +5 -9
- package/dist/cli/decisions.js +10 -21
- package/dist/cli/explain.js +2 -1
- package/dist/cli/goals.js +1 -1
- package/dist/cli/init.js +1 -1
- package/dist/cli/playbooks.js +4 -9
- package/dist/cli/projects.js +3 -1
- package/dist/cli/recall.js +2 -1
- package/dist/cli/remember.js +6 -18
- package/dist/cli/session-hooks.js +21 -40
- package/dist/cli/setup.d.ts +1 -1
- package/dist/cli/setup.js +2 -6
- package/dist/cli/shared.js +3 -3
- package/dist/cli/slack.js +1 -1
- package/dist/cli/sleep.js +27 -1
- package/dist/cli/status.d.ts +4 -4
- package/dist/cli/status.js +7 -12
- package/dist/cli/transfer.js +8 -13
- package/dist/cli/usage.js +9 -6
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +4 -9
- package/dist/compaction-record.d.ts +2 -0
- package/dist/compaction-record.js +89 -68
- package/dist/compare.d.ts +11 -16
- package/dist/compare.js +11 -16
- package/dist/config.d.ts +19 -18
- package/dist/config.js +90 -67
- package/dist/connectors/github/backfill.d.ts +2 -2
- package/dist/connectors/github/backfill.js +8 -15
- package/dist/connectors/github/cli-impl.js +3 -8
- package/dist/connectors/github/deletion.d.ts +5 -12
- package/dist/connectors/github/deletion.js +5 -12
- package/dist/connectors/github/dlq.d.ts +6 -9
- package/dist/connectors/github/dlq.js +2 -3
- package/dist/connectors/github/ingest.d.ts +5 -7
- package/dist/connectors/github/ingest.js +8 -12
- package/dist/connectors/github/octokit-client.d.ts +3 -5
- package/dist/connectors/github/octokit-client.js +5 -6
- package/dist/connectors/github/signature.d.ts +9 -39
- package/dist/connectors/github/signature.js +9 -39
- package/dist/connectors/github/tenant-routing.d.ts +1 -1
- package/dist/connectors/github/tenant-routing.js +1 -1
- package/dist/connectors/github/transform.js +2 -2
- package/dist/connectors/github/types.d.ts +2 -10
- package/dist/connectors/github/types.js +1 -3
- package/dist/connectors/slack/deletion.d.ts +3 -8
- package/dist/connectors/slack/deletion.js +3 -8
- package/dist/connectors/slack/dlq.d.ts +1 -1
- package/dist/connectors/slack/ingest.d.ts +1 -1
- package/dist/connectors/slack/ingest.js +7 -16
- package/dist/connectors/slack/signature.d.ts +1 -1
- package/dist/connectors/slack/tenant-routing.d.ts +3 -5
- package/dist/connectors/slack/tenant-routing.js +3 -5
- package/dist/connectors/slack/transform.d.ts +5 -6
- package/dist/connectors/slack/transform.js +5 -6
- package/dist/connectors/slack/types.d.ts +2 -6
- package/dist/connectors/slack/types.js +1 -3
- package/dist/connectors/slack/web-client.js +10 -3
- package/dist/connectors/slack/workspaces.d.ts +3 -5
- package/dist/connectors/slack/workspaces.js +3 -5
- package/dist/consolidate/conflicts.js +3 -14
- package/dist/consolidate/decay.js +9 -29
- package/dist/consolidate/llm-passes.js +4 -5
- package/dist/consolidate/merge.js +8 -23
- package/dist/consolidate/run.d.ts +1 -8
- package/dist/consolidate/run.js +3 -25
- package/dist/consolidate/sleep.js +5 -17
- package/dist/consolidate/traces.js +9 -21
- package/dist/customer-notes.d.ts +5 -7
- package/dist/customer-notes.js +6 -9
- package/dist/dag.d.ts +10 -21
- package/dist/dag.js +23 -73
- package/dist/db/continuity.js +2 -2
- package/dist/db/migrations/v14.js +1 -1
- package/dist/db/migrations/v15.js +1 -2
- package/dist/db/migrations/v16.js +3 -4
- package/dist/db/migrations/v17.js +2 -3
- package/dist/db/migrations/v19.js +1 -1
- package/dist/db/migrations/v20.js +1 -1
- package/dist/db/migrations/v21.js +2 -6
- package/dist/db/migrations/v22.js +2 -4
- package/dist/db/migrations/v23.js +1 -1
- package/dist/db/migrations/v24.js +4 -6
- package/dist/db/migrations/v25.js +2 -3
- package/dist/db/migrations/v26.js +3 -3
- package/dist/db/migrations/v27.js +2 -10
- package/dist/db/migrations/v28.js +5 -8
- package/dist/db/migrations/v29.js +3 -4
- package/dist/db/migrations/v30.js +2 -2
- package/dist/db/migrations/v31.js +1 -1
- package/dist/db/migrations/v32.js +1 -1
- package/dist/db/migrations/v33.js +3 -3
- package/dist/db/migrations/v34.js +1 -1
- package/dist/db/migrations/v35.js +3 -4
- package/dist/db/migrations/v36.js +3 -4
- package/dist/db/migrations/v37.js +5 -5
- package/dist/db/migrations/v38.js +7 -8
- package/dist/db/migrations/v39.js +1 -1
- package/dist/db/migrations/v40.js +4 -16
- package/dist/db/migrations/v41.js +3 -4
- package/dist/db/migrations/v42.js +3 -4
- package/dist/db/migrations/v45.js +1 -1
- package/dist/db/migrations/v46.js +1 -1
- package/dist/db/migrations/v47.js +1 -1
- package/dist/db/migrations/v48.js +1 -1
- package/dist/decisions.d.ts +2 -2
- package/dist/decisions.js +6 -6
- package/dist/dedupe.js +86 -61
- package/dist/delivery-recorder.js +154 -135
- package/dist/doctor.js +119 -104
- package/dist/dormant.js +1 -4
- package/dist/embedding-provider.d.ts +4 -8
- package/dist/embedding-provider.js +4 -8
- package/dist/embeddings.js +55 -47
- package/dist/env.d.ts +1 -1
- package/dist/env.js +12 -12
- package/dist/escape.d.ts +5 -0
- package/dist/escape.js +10 -0
- package/dist/eval-stats.d.ts +1 -2
- package/dist/eval-stats.js +1 -2
- package/dist/eval-suite.js +27 -21
- package/dist/extract.js +4 -9
- package/dist/failure-log.d.ts +3 -3
- package/dist/failure-log.js +1 -1
- package/dist/forward-claim-detector.d.ts +2 -4
- package/dist/forward-claim-detector.js +6 -11
- package/dist/goals.d.ts +3 -3
- package/dist/goals.js +5 -6
- package/dist/graph/read.d.ts +2 -2
- package/dist/graph/read.js +5 -6
- package/dist/graph/types.d.ts +8 -8
- package/dist/graph/write.d.ts +7 -14
- package/dist/graph/write.js +16 -23
- package/dist/graph-extract.d.ts +7 -8
- package/dist/graph-extract.js +62 -72
- package/dist/graph-recall.d.ts +2 -2
- package/dist/graph-recall.js +55 -49
- package/dist/graph-stream.d.ts +5 -6
- package/dist/graph-stream.js +66 -57
- package/dist/graph-view.d.ts +2 -2
- package/dist/graph-view.js +7 -7
- package/dist/half-life-migration.d.ts +1 -2
- package/dist/half-life-migration.js +2 -3
- package/dist/hooks/codex-session.js +1 -1
- package/dist/hooks/codex-wrapper.d.ts +1 -1
- package/dist/hooks/codex-wrapper.js +3 -2
- package/dist/hooks/json-hooks.d.ts +2 -2
- package/dist/hooks/json-hooks.js +5 -4
- package/dist/hooks/opencode.d.ts +1 -1
- package/dist/hooks/opencode.js +5 -4
- package/dist/hooks/shared.d.ts +3 -7
- package/dist/hooks/shared.js +1 -8
- package/dist/http-util.d.ts +2 -3
- package/dist/http-util.js +3 -0
- package/dist/importers/core.d.ts +2 -9
- package/dist/importers/core.js +15 -30
- package/dist/importers/sources.js +2 -1
- package/dist/importers/vault.js +2 -20
- package/dist/incidents.d.ts +1 -1
- package/dist/incidents.js +1 -1
- package/dist/instruction-detect.d.ts +1 -1
- package/dist/instruction-detect.js +1 -1
- package/dist/invalidation.d.ts +3 -0
- package/dist/invalidation.js +160 -114
- package/dist/json.d.ts +5 -0
- package/dist/json.js +4 -0
- package/dist/judgment.js +1 -2
- package/dist/local-embedding.js +1 -1
- package/dist/mcp/admin-tools.js +7 -17
- package/dist/mcp/format.js +1 -1
- package/dist/mcp/framing.js +3 -6
- package/dist/mcp/protocol.d.ts +2 -5
- package/dist/mcp/protocol.js +1 -3
- package/dist/mcp/recall-tools.js +12 -15
- package/dist/mcp/request.js +4 -3
- package/dist/mcp/session-state.js +2 -3
- package/dist/mcp/stdio.js +2 -1
- package/dist/mcp/tools.js +9 -6
- package/dist/memory-value-weights.d.ts +5 -8
- package/dist/memory-value-weights.js +5 -8
- package/dist/memory-value.d.ts +13 -13
- package/dist/memory-value.js +26 -37
- package/dist/memory.d.ts +20 -22
- package/dist/memory.js +24 -48
- package/dist/multihop.d.ts +1 -1
- package/dist/multihop.js +3 -2
- package/dist/owner-validation.d.ts +4 -5
- package/dist/owner-validation.js +4 -5
- package/dist/physics.d.ts +4 -4
- package/dist/physics.js +7 -9
- package/dist/policies.d.ts +9 -10
- package/dist/policies.js +12 -14
- package/dist/postinstall.js +3 -6
- package/dist/predictions/planning-fallacy.d.ts +9 -14
- package/dist/predictions/planning-fallacy.js +10 -16
- package/dist/predictions/store.d.ts +15 -23
- package/dist/predictions/store.js +36 -33
- package/dist/processes.d.ts +2 -7
- package/dist/processes.js +3 -3
- package/dist/project-briefs.d.ts +2 -3
- package/dist/project-briefs.js +9 -13
- package/dist/project-identity.d.ts +22 -9
- package/dist/project-identity.js +47 -12
- package/dist/project-merge.d.ts +17 -2
- package/dist/project-merge.js +109 -27
- package/dist/project-remote.d.ts +12 -0
- package/dist/project-remote.js +138 -0
- package/dist/prompt-recall.js +1 -2
- package/dist/rate-limit.d.ts +1 -1
- package/dist/rate-limit.js +1 -1
- package/dist/raw-archive.d.ts +9 -0
- package/dist/raw-archive.js +70 -53
- package/dist/recall-history.d.ts +19 -20
- package/dist/recall-history.js +24 -42
- package/dist/recall-pipeline.js +4 -28
- package/dist/recall-scope.d.ts +7 -8
- package/dist/recall-scope.js +7 -8
- package/dist/recall-trace.d.ts +5 -9
- package/dist/recall-trace.js +6 -10
- package/dist/refine-llm.d.ts +1 -1
- package/dist/refine-llm.js +2 -2
- package/dist/reject-flow.d.ts +3 -4
- package/dist/reject-flow.js +122 -117
- package/dist/rejection.d.ts +5 -6
- package/dist/rejection.js +7 -15
- package/dist/rerankers/clef.d.ts +1 -1
- package/dist/rerankers/jev.d.ts +1 -2
- package/dist/rerankers/jev.js +4 -5
- package/dist/rerankers/llm.d.ts +1 -2
- package/dist/rerankers/llm.js +1 -2
- package/dist/rerankers/types.d.ts +1 -2
- package/dist/rrf.d.ts +2 -2
- package/dist/rrf.js +2 -2
- package/dist/search/bm25-search.d.ts +1 -1
- package/dist/search/bm25-search.js +2 -1
- package/dist/search/boosts.js +2 -1
- package/dist/search/hybrid.d.ts +1 -1
- package/dist/search/hybrid.js +2 -1
- package/dist/search/physics-search.d.ts +1 -1
- package/dist/search/physics-search.js +2 -1
- package/dist/search/types.d.ts +2 -0
- package/dist/search/types.js +3 -1
- package/dist/secret-detect.d.ts +4 -5
- package/dist/secret-detect.js +6 -10
- package/dist/server/auth.js +5 -5
- package/dist/server/client-ip.js +1 -1
- package/dist/server/cursor.js +2 -1
- package/dist/server/mcp-http.js +4 -4
- package/dist/server/request.d.ts +3 -6
- package/dist/server/request.js +6 -7
- package/dist/server/routes/admin.js +5 -4
- package/dist/server/routes/customer-notes.js +6 -5
- package/dist/server/routes/decisions.js +4 -3
- package/dist/server/routes/incidents.js +7 -5
- package/dist/server/routes/memories.js +7 -7
- package/dist/server/routes/policies.js +3 -2
- package/dist/server/routes/predictions.js +12 -15
- package/dist/server/routes/processes.js +3 -2
- package/dist/server/routes/project-briefs.js +8 -7
- package/dist/server/routes/recall.js +95 -93
- package/dist/server/routes/skills.js +6 -5
- package/dist/server/types.d.ts +1 -1
- package/dist/server/validation.d.ts +1 -2
- package/dist/server/validation.js +7 -14
- package/dist/server-detect.js +72 -58
- package/dist/server.d.ts +2 -2
- package/dist/server.js +131 -117
- package/dist/shared.d.ts +17 -17
- package/dist/shared.js +93 -97
- package/dist/skills.d.ts +3 -3
- package/dist/skills.js +8 -8
- package/dist/store/audit-event.d.ts +2 -2
- package/dist/store/audit-event.js +1 -1
- package/dist/store/candidates.d.ts +2 -2
- package/dist/store/candidates.js +4 -3
- package/dist/store/conflicts.js +30 -22
- package/dist/store/delete-and-batch.d.ts +11 -14
- package/dist/store/delete-and-batch.js +40 -91
- package/dist/store/entry-reads.d.ts +17 -25
- package/dist/store/entry-reads.js +59 -40
- package/dist/store/entry-row.d.ts +6 -24
- package/dist/store/entry-row.js +6 -24
- package/dist/store/entry-writes.d.ts +6 -7
- package/dist/store/entry-writes.js +13 -11
- package/dist/store/handoffs.d.ts +1 -1
- package/dist/store/handoffs.js +7 -10
- package/dist/store/index-and-stats.d.ts +2 -6
- package/dist/store/index-and-stats.js +4 -10
- package/dist/store/mirrors.d.ts +6 -19
- package/dist/store/mirrors.js +14 -39
- package/dist/store/open.js +9 -31
- package/dist/store/rows.d.ts +5 -11
- package/dist/store/rows.js +6 -11
- package/dist/store/search-rows.d.ts +17 -34
- package/dist/store/search-rows.js +31 -59
- package/dist/store/sessions.d.ts +4 -5
- package/dist/store/sessions.js +5 -6
- package/dist/store/summaries.d.ts +13 -17
- package/dist/store/summaries.js +26 -70
- package/dist/support-bundle.js +4 -8
- package/dist/tenant.d.ts +1 -5
- package/dist/token-ledger.d.ts +1 -1
- package/dist/token-ledger.js +3 -5
- package/dist/trace.js +1 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/working-memory.d.ts +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
package/dist/config.js
CHANGED
|
@@ -5,10 +5,11 @@ import * as fs from 'fs';
|
|
|
5
5
|
import * as path from 'path';
|
|
6
6
|
import { DEFAULT_PHYSICS_CONFIG, mergePhysicsConfig } from './physics-config.js';
|
|
7
7
|
import { DEFAULT_HALF_LIFE_DAYS } from './memory.js';
|
|
8
|
+
import { DEFAULT_LOCAL_BUMP, DEFAULT_RECALL_BUDGET } from './search/types.js';
|
|
8
9
|
import { log } from './log.js';
|
|
9
10
|
const DEFAULT_CONFIG = {
|
|
10
11
|
defaultHalfLifeDays: DEFAULT_HALF_LIFE_DAYS,
|
|
11
|
-
defaultBudget:
|
|
12
|
+
defaultBudget: DEFAULT_RECALL_BUDGET,
|
|
12
13
|
defaultContextBudget: 3000,
|
|
13
14
|
decayBasis: 'adaptive',
|
|
14
15
|
autoLearnOnSleep: true,
|
|
@@ -36,7 +37,7 @@ const DEFAULT_CONFIG = {
|
|
|
36
37
|
lambda: 0.7,
|
|
37
38
|
},
|
|
38
39
|
search: {
|
|
39
|
-
localBump:
|
|
40
|
+
localBump: DEFAULT_LOCAL_BUMP,
|
|
40
41
|
},
|
|
41
42
|
replay: {
|
|
42
43
|
count: 5,
|
|
@@ -92,6 +93,9 @@ const DEFAULT_CONFIG = {
|
|
|
92
93
|
pilot: {
|
|
93
94
|
holdoutRateBp: 0,
|
|
94
95
|
},
|
|
96
|
+
projectIdentity: {
|
|
97
|
+
remote: true,
|
|
98
|
+
},
|
|
95
99
|
};
|
|
96
100
|
function isMemoryValueConfig(value) {
|
|
97
101
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
@@ -132,6 +136,17 @@ function pilotHoldoutRate(value) {
|
|
|
132
136
|
`(got ${JSON.stringify(value)}) - using 0 (pilot off).`);
|
|
133
137
|
return 0;
|
|
134
138
|
}
|
|
139
|
+
// Only a real false turns the remote rule off; anything else malformed warns and keeps it on.
|
|
140
|
+
function projectIdentityRemote(value) {
|
|
141
|
+
const remote = value?.remote;
|
|
142
|
+
if (remote === true || remote === false)
|
|
143
|
+
return remote;
|
|
144
|
+
if (value === undefined || (remote === undefined && value !== null && value.constructor === Object))
|
|
145
|
+
return true;
|
|
146
|
+
log.warn(`config.json's "projectIdentity" must be an object like {"remote": false} ` +
|
|
147
|
+
`(got ${JSON.stringify(value)}) - using true (projects named by their origin remote).`);
|
|
148
|
+
return true;
|
|
149
|
+
}
|
|
135
150
|
function agentMemoryTools(value) {
|
|
136
151
|
if (value === undefined || value === null)
|
|
137
152
|
return null;
|
|
@@ -141,6 +156,70 @@ function agentMemoryTools(value) {
|
|
|
141
156
|
`(got ${JSON.stringify(value)}) - importing none.`);
|
|
142
157
|
return [];
|
|
143
158
|
}
|
|
159
|
+
function memoryValueOverride(raw) {
|
|
160
|
+
// Spreading a non-object raw.memoryValue (e.g. {"memoryValue": true}) silently leaves `enabled` false;
|
|
161
|
+
// this feature must never be silently off, so warn loudly and fall back to defaults instead.
|
|
162
|
+
const memoryValueRaw = raw.memoryValue;
|
|
163
|
+
const validMemoryValueConfig = memoryValueRaw === undefined || isMemoryValueConfig(memoryValueRaw);
|
|
164
|
+
if (!validMemoryValueConfig) {
|
|
165
|
+
log.warn(`config.json's "memoryValue" must be an object like {"enabled": true} ` +
|
|
166
|
+
`(got ${JSON.stringify(memoryValueRaw)}) - using defaults.`);
|
|
167
|
+
}
|
|
168
|
+
return memoryValueRaw !== undefined && isMemoryValueConfig(memoryValueRaw) ? memoryValueRaw : {};
|
|
169
|
+
}
|
|
170
|
+
function dormantSettings(raw) {
|
|
171
|
+
// Same rule as memoryValue above: a malformed value never silently
|
|
172
|
+
// changes what sleep does. Anything but a real object / boolean / number
|
|
173
|
+
// warns and falls back to the default, which keeps faded memories.
|
|
174
|
+
const dormantRaw = raw.dormant;
|
|
175
|
+
if (dormantRaw !== undefined && !isDormantConfig(dormantRaw)) {
|
|
176
|
+
log.warn(`config.json's "dormant" must be an object like {"enabled": false} ` +
|
|
177
|
+
`(got ${JSON.stringify(dormantRaw)}) - using the default (faded memories kept dormant).`);
|
|
178
|
+
}
|
|
179
|
+
const dormantOverride = dormantRaw !== undefined && isDormantConfig(dormantRaw) ? dormantRaw : {};
|
|
180
|
+
// Only a real boolean counts: {"enabled": "false"} is a truthy string.
|
|
181
|
+
let dormantEnabled = dormantOverride.enabled ?? DEFAULT_CONFIG.dormant.enabled;
|
|
182
|
+
if (dormantEnabled !== true && dormantEnabled !== false) {
|
|
183
|
+
log.warn(`config.json's "dormant.enabled" must be true or false ` +
|
|
184
|
+
`(got ${JSON.stringify(dormantEnabled)}) - using the default (faded memories kept dormant).`);
|
|
185
|
+
dormantEnabled = DEFAULT_CONFIG.dormant.enabled;
|
|
186
|
+
}
|
|
187
|
+
let dormantRetentionDays = dormantOverride.retentionDays ?? DEFAULT_CONFIG.dormant.retentionDays;
|
|
188
|
+
if (!Number.isFinite(dormantRetentionDays) || dormantRetentionDays < 0) {
|
|
189
|
+
log.warn(`config.json's "dormant.retentionDays" must be a number of days, 0 or more ` +
|
|
190
|
+
`(got ${JSON.stringify(dormantRetentionDays)}) - using ${DEFAULT_CONFIG.dormant.retentionDays}.`);
|
|
191
|
+
dormantRetentionDays = DEFAULT_CONFIG.dormant.retentionDays;
|
|
192
|
+
}
|
|
193
|
+
return { enabled: dormantEnabled, retentionDays: dormantRetentionDays };
|
|
194
|
+
}
|
|
195
|
+
function churnStalenessEnabled(raw) {
|
|
196
|
+
// Same "never silently wrong" rule as memoryValue/dormant above.
|
|
197
|
+
const churnStalenessRaw = raw.churnStaleness;
|
|
198
|
+
const validChurnStalenessConfig = churnStalenessRaw === undefined || isChurnStalenessConfig(churnStalenessRaw);
|
|
199
|
+
if (!validChurnStalenessConfig) {
|
|
200
|
+
log.warn(`config.json's "churnStaleness" must be an object like {"enabled": true} ` +
|
|
201
|
+
`(got ${JSON.stringify(churnStalenessRaw)}) - using defaults.`);
|
|
202
|
+
}
|
|
203
|
+
const enabled = churnStalenessRaw !== undefined && isChurnStalenessConfig(churnStalenessRaw)
|
|
204
|
+
? churnStalenessRaw.enabled
|
|
205
|
+
: DEFAULT_CONFIG.churnStaleness.enabled;
|
|
206
|
+
if (enabled !== true && enabled !== false) {
|
|
207
|
+
log.warn(`config.json's "churnStaleness.enabled" must be true or false ` +
|
|
208
|
+
`(got ${JSON.stringify(enabled)}) - using false.`);
|
|
209
|
+
return false;
|
|
210
|
+
}
|
|
211
|
+
return enabled;
|
|
212
|
+
}
|
|
213
|
+
function defaultHalfLifeDays(raw) {
|
|
214
|
+
// Every writer starts a memory on this, and a zero or negative half-life scores zero strength, so sleep would retire it.
|
|
215
|
+
const days = raw.defaultHalfLifeDays ?? DEFAULT_CONFIG.defaultHalfLifeDays;
|
|
216
|
+
if (!Number.isFinite(days) || days <= 0) {
|
|
217
|
+
log.warn(`config.json's "defaultHalfLifeDays" must be a number of days above 0 ` +
|
|
218
|
+
`(got ${JSON.stringify(days)}) - using ${DEFAULT_CONFIG.defaultHalfLifeDays}.`);
|
|
219
|
+
return DEFAULT_CONFIG.defaultHalfLifeDays;
|
|
220
|
+
}
|
|
221
|
+
return days;
|
|
222
|
+
}
|
|
144
223
|
export function loadConfig(hippoRoot) {
|
|
145
224
|
const configPath = path.join(hippoRoot, 'config.json');
|
|
146
225
|
if (!fs.existsSync(configPath))
|
|
@@ -149,66 +228,12 @@ export function loadConfig(hippoRoot) {
|
|
|
149
228
|
const raw = JSON.parse(fs.readFileSync(configPath, 'utf8'));
|
|
150
229
|
const basis = raw.decayBasis;
|
|
151
230
|
const validBasis = basis === 'clock' || basis === 'session' || basis === 'adaptive';
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
// default `false` with zero indication anything was wrong. This
|
|
157
|
-
// feature's whole point is "never silently off": warn loudly and fall
|
|
158
|
-
// back to defaults instead of merging garbage.
|
|
159
|
-
const memoryValueRaw = raw.memoryValue;
|
|
160
|
-
const validMemoryValueConfig = memoryValueRaw === undefined || isMemoryValueConfig(memoryValueRaw);
|
|
161
|
-
if (!validMemoryValueConfig) {
|
|
162
|
-
log.warn(`config.json's "memoryValue" must be an object like {"enabled": true} ` +
|
|
163
|
-
`(got ${JSON.stringify(memoryValueRaw)}) - using defaults.`);
|
|
164
|
-
}
|
|
165
|
-
const memoryValueOverride = memoryValueRaw !== undefined && isMemoryValueConfig(memoryValueRaw) ? memoryValueRaw : {};
|
|
166
|
-
// Same rule as memoryValue above: a malformed value never silently
|
|
167
|
-
// changes what sleep does. Anything but a real object / boolean / number
|
|
168
|
-
// warns and falls back to the default, which keeps faded memories.
|
|
169
|
-
const dormantRaw = raw.dormant;
|
|
170
|
-
if (dormantRaw !== undefined && !isDormantConfig(dormantRaw)) {
|
|
171
|
-
log.warn(`config.json's "dormant" must be an object like {"enabled": false} ` +
|
|
172
|
-
`(got ${JSON.stringify(dormantRaw)}) - using the default (faded memories kept dormant).`);
|
|
173
|
-
}
|
|
174
|
-
const dormantOverride = dormantRaw !== undefined && isDormantConfig(dormantRaw) ? dormantRaw : {};
|
|
175
|
-
// Only a real boolean counts: {"enabled": "false"} is a truthy string.
|
|
176
|
-
let dormantEnabled = dormantOverride.enabled ?? DEFAULT_CONFIG.dormant.enabled;
|
|
177
|
-
if (dormantEnabled !== true && dormantEnabled !== false) {
|
|
178
|
-
log.warn(`config.json's "dormant.enabled" must be true or false ` +
|
|
179
|
-
`(got ${JSON.stringify(dormantEnabled)}) - using the default (faded memories kept dormant).`);
|
|
180
|
-
dormantEnabled = DEFAULT_CONFIG.dormant.enabled;
|
|
181
|
-
}
|
|
182
|
-
let dormantRetentionDays = dormantOverride.retentionDays ?? DEFAULT_CONFIG.dormant.retentionDays;
|
|
183
|
-
if (!Number.isFinite(dormantRetentionDays) || dormantRetentionDays < 0) {
|
|
184
|
-
log.warn(`config.json's "dormant.retentionDays" must be a number of days, 0 or more ` +
|
|
185
|
-
`(got ${JSON.stringify(dormantRetentionDays)}) - using ${DEFAULT_CONFIG.dormant.retentionDays}.`);
|
|
186
|
-
dormantRetentionDays = DEFAULT_CONFIG.dormant.retentionDays;
|
|
187
|
-
}
|
|
188
|
-
// Same "never silently wrong" rule as memoryValue/dormant above.
|
|
189
|
-
const churnStalenessRaw = raw.churnStaleness;
|
|
190
|
-
const validChurnStalenessConfig = churnStalenessRaw === undefined || isChurnStalenessConfig(churnStalenessRaw);
|
|
191
|
-
if (!validChurnStalenessConfig) {
|
|
192
|
-
log.warn(`config.json's "churnStaleness" must be an object like {"enabled": true} ` +
|
|
193
|
-
`(got ${JSON.stringify(churnStalenessRaw)}) - using defaults.`);
|
|
194
|
-
}
|
|
195
|
-
let churnStalenessEnabled = churnStalenessRaw !== undefined && isChurnStalenessConfig(churnStalenessRaw)
|
|
196
|
-
? churnStalenessRaw.enabled
|
|
197
|
-
: DEFAULT_CONFIG.churnStaleness.enabled;
|
|
198
|
-
if (churnStalenessEnabled !== true && churnStalenessEnabled !== false) {
|
|
199
|
-
log.warn(`config.json's "churnStaleness.enabled" must be true or false ` +
|
|
200
|
-
`(got ${JSON.stringify(churnStalenessEnabled)}) - using false.`);
|
|
201
|
-
churnStalenessEnabled = false;
|
|
202
|
-
}
|
|
203
|
-
// Every writer starts a memory on this, and a zero or negative half-life scores zero strength, so sleep would retire it.
|
|
204
|
-
let defaultHalfLifeDays = raw.defaultHalfLifeDays ?? DEFAULT_CONFIG.defaultHalfLifeDays;
|
|
205
|
-
if (!Number.isFinite(defaultHalfLifeDays) || defaultHalfLifeDays <= 0) {
|
|
206
|
-
log.warn(`config.json's "defaultHalfLifeDays" must be a number of days above 0 ` +
|
|
207
|
-
`(got ${JSON.stringify(defaultHalfLifeDays)}) - using ${DEFAULT_CONFIG.defaultHalfLifeDays}.`);
|
|
208
|
-
defaultHalfLifeDays = DEFAULT_CONFIG.defaultHalfLifeDays;
|
|
209
|
-
}
|
|
231
|
+
const memoryValue = memoryValueOverride(raw);
|
|
232
|
+
const dormant = dormantSettings(raw);
|
|
233
|
+
const churnEnabled = churnStalenessEnabled(raw);
|
|
234
|
+
const halfLifeDays = defaultHalfLifeDays(raw);
|
|
210
235
|
return {
|
|
211
|
-
defaultHalfLifeDays,
|
|
236
|
+
defaultHalfLifeDays: halfLifeDays,
|
|
212
237
|
defaultBudget: raw.defaultBudget ?? DEFAULT_CONFIG.defaultBudget,
|
|
213
238
|
defaultContextBudget: raw.defaultContextBudget ?? DEFAULT_CONFIG.defaultContextBudget,
|
|
214
239
|
decayBasis: validBasis ? basis : DEFAULT_CONFIG.decayBasis,
|
|
@@ -232,18 +257,16 @@ export function loadConfig(hippoRoot) {
|
|
|
232
257
|
ambient: { ...DEFAULT_CONFIG.ambient, ...(raw.ambient ?? {}) },
|
|
233
258
|
memoryValue: {
|
|
234
259
|
...DEFAULT_CONFIG.memoryValue,
|
|
235
|
-
...
|
|
236
|
-
},
|
|
237
|
-
dormant: {
|
|
238
|
-
enabled: dormantEnabled,
|
|
239
|
-
retentionDays: dormantRetentionDays,
|
|
260
|
+
...memoryValue,
|
|
240
261
|
},
|
|
262
|
+
dormant,
|
|
241
263
|
churnStaleness: {
|
|
242
|
-
enabled:
|
|
264
|
+
enabled: churnEnabled,
|
|
243
265
|
},
|
|
244
266
|
agentMemories: { tools: agentMemoryTools(raw.agentMemories?.tools) },
|
|
245
267
|
deliveryLedger: { enabled: deliveryLedgerEnabled(raw.deliveryLedger) },
|
|
246
268
|
pilot: { holdoutRateBp: pilotHoldoutRate(raw.pilot) },
|
|
269
|
+
projectIdentity: { remote: projectIdentityRemote(raw.projectIdentity) },
|
|
247
270
|
};
|
|
248
271
|
}
|
|
249
272
|
catch (err) {
|
|
@@ -5,13 +5,13 @@
|
|
|
5
5
|
* - /repos/{repo}/issues/comments -> issue_comments_hwm
|
|
6
6
|
* - /repos/{repo}/pulls/comments -> pr_review_comments_hwm
|
|
7
7
|
*
|
|
8
|
-
* Crash safety
|
|
8
|
+
* Crash safety: each stream's HWM is persisted ONLY after
|
|
9
9
|
* the stream fully drains. If stream 2 throws mid-flight, stream 1's HWM
|
|
10
10
|
* is committed and stream 2's stays NULL (or its prior value). Rerun
|
|
11
11
|
* picks up from stream 1's saved HWM and re-fetches stream 2 from its
|
|
12
12
|
* last committed point.
|
|
13
13
|
*
|
|
14
|
-
*
|
|
14
|
+
* The /issues endpoint returns BOTH issues and PRs (a PR is
|
|
15
15
|
* an issue with a `pull_request` field). We skip PRs here so they don't
|
|
16
16
|
* get ingested under the issues schema. PRs are handled via webhook in
|
|
17
17
|
* V1 (no /pulls backfill stream — review comments cover the discussion
|
|
@@ -5,13 +5,13 @@
|
|
|
5
5
|
* - /repos/{repo}/issues/comments -> issue_comments_hwm
|
|
6
6
|
* - /repos/{repo}/pulls/comments -> pr_review_comments_hwm
|
|
7
7
|
*
|
|
8
|
-
* Crash safety
|
|
8
|
+
* Crash safety: each stream's HWM is persisted ONLY after
|
|
9
9
|
* the stream fully drains. If stream 2 throws mid-flight, stream 1's HWM
|
|
10
10
|
* is committed and stream 2's stays NULL (or its prior value). Rerun
|
|
11
11
|
* picks up from stream 1's saved HWM and re-fetches stream 2 from its
|
|
12
12
|
* last committed point.
|
|
13
13
|
*
|
|
14
|
-
*
|
|
14
|
+
* The /issues endpoint returns BOTH issues and PRs (a PR is
|
|
15
15
|
* an issue with a `pull_request` field). We skip PRs here so they don't
|
|
16
16
|
* get ingested under the issues schema. PRs are handled via webhook in
|
|
17
17
|
* V1 (no /pulls backfill stream — review comments cover the discussion
|
|
@@ -110,18 +110,15 @@ function isCommentItem(x) {
|
|
|
110
110
|
}
|
|
111
111
|
/**
|
|
112
112
|
* Drain one stream end-to-end. Pauses and retries on rate-limit. Throws on
|
|
113
|
-
* any other fetch error so the caller leaves the HWM unchanged
|
|
114
|
-
* codex P1 #3 crash safety).
|
|
113
|
+
* any other fetch error so the caller leaves the HWM unchanged.
|
|
115
114
|
*
|
|
116
|
-
* v1.3.1 (round 2 codex P1s + claude P1):
|
|
117
115
|
* - Tracks max(updated_at) across ALL fetched items, including ones
|
|
118
116
|
* `toIngestEvent` rejects (e.g., PRs returned via `/issues`). Otherwise
|
|
119
117
|
* a page of pure PRs would never advance the HWM and the next run would
|
|
120
118
|
* re-fetch the same window forever.
|
|
121
119
|
* - Returns `drained: boolean` — true only when the stream actually ran
|
|
122
|
-
* to next=null. Callers MUST NOT advance the HWM when drained=false
|
|
123
|
-
*
|
|
124
|
-
* maxUpdatedAt and the caller persisted it, skipping the unfetched tail.
|
|
120
|
+
* to next=null. Callers MUST NOT advance the HWM when drained=false,
|
|
121
|
+
* or a capped run would persist a partial HWM and skip the unfetched tail.
|
|
125
122
|
*/
|
|
126
123
|
async function drainStream(ctx, url0, toIngestEvent, fetcher, token, sleep, maxItems) {
|
|
127
124
|
let url = url0;
|
|
@@ -141,9 +138,7 @@ async function drainStream(ctx, url0, toIngestEvent, fetcher, token, sleep, maxI
|
|
|
141
138
|
}
|
|
142
139
|
pages++;
|
|
143
140
|
for (const item of page.items) {
|
|
144
|
-
//
|
|
145
|
-
// filter. Skipped PRs from /issues still contribute to the HWM so
|
|
146
|
-
// PR-only pages don't loop forever.
|
|
141
|
+
// Track updated_at on EVERY item, before the toIngestEvent filter, so PR-only pages don't loop forever.
|
|
147
142
|
// SAFETY: updated_at is GitHub's ISO-timestamp field on every item
|
|
148
143
|
// shape this stream returns; a non-string value would only fail the
|
|
149
144
|
// string comparisons below, matching pre-migration passthrough.
|
|
@@ -175,7 +170,7 @@ async function drainStream(ctx, url0, toIngestEvent, fetcher, token, sleep, maxI
|
|
|
175
170
|
function issueItemToEvent(item, repository) {
|
|
176
171
|
if (!isIssuesItem(item))
|
|
177
172
|
return null;
|
|
178
|
-
//
|
|
173
|
+
// /issues returns PRs too; skip them.
|
|
179
174
|
if (item.pull_request)
|
|
180
175
|
return null;
|
|
181
176
|
const payload = {
|
|
@@ -240,9 +235,7 @@ function prReviewCommentItemToEvent(item, repository) {
|
|
|
240
235
|
/** Drains one stream, then persists its HWM only when the stream ran to the end. */
|
|
241
236
|
async function backfillStream(ctx, opts, sleep, stream) {
|
|
242
237
|
const res = await drainStream(ctx, stream.url, stream.toIngestEvent, opts.fetcher, opts.token, sleep, opts.maxPerStream);
|
|
243
|
-
//
|
|
244
|
-
// (--max) must leave the HWM at its previous value so the next invocation
|
|
245
|
-
// re-fetches the unprocessed tail.
|
|
238
|
+
// A capped run (--max) must leave the HWM at its previous value so the next run re-fetches the unprocessed tail.
|
|
246
239
|
if (res.drained && res.maxUpdatedAt) {
|
|
247
240
|
writeOneHwm(ctx.hippoRoot, ctx.tenantId, opts.repoFullName, stream.column, res.maxUpdatedAt);
|
|
248
241
|
}
|
|
@@ -162,9 +162,7 @@ export async function cmdGithubDlqReplay(hippoRoot, args, flags) {
|
|
|
162
162
|
tenantId: resolveTenantId({}),
|
|
163
163
|
actor: adminActor('cli:github-dlq-replay'),
|
|
164
164
|
};
|
|
165
|
-
//
|
|
166
|
-
// dry-run that printed "replay ok" while only bumping retry_count. Wire the
|
|
167
|
-
// real hook so `replay` actually re-runs the ingest path.
|
|
165
|
+
// Without an ingestHook replay only bumps retry_count while printing "replay ok"; the real hook re-runs ingest.
|
|
168
166
|
const result = await replayDlqEntry(ctx, id, {
|
|
169
167
|
force,
|
|
170
168
|
webhookSecret: envGithubWebhookSecret(),
|
|
@@ -175,11 +173,8 @@ export async function cmdGithubDlqReplay(hippoRoot, args, flags) {
|
|
|
175
173
|
if (!event) {
|
|
176
174
|
return { memoryId: null };
|
|
177
175
|
}
|
|
178
|
-
//
|
|
179
|
-
//
|
|
180
|
-
// handler, NOT to ingestEvent. The v1.3.1 hook unconditionally called
|
|
181
|
-
// ingestEvent and would have written the deleted comment as a NEW raw
|
|
182
|
-
// memory instead of archiving the matching ones.
|
|
176
|
+
// A replayed comment `.deleted` row must route to the deletion handler, NOT to ingestEvent,
|
|
177
|
+
// which would write the deleted comment as a NEW raw memory instead of archiving the matching ones.
|
|
183
178
|
if ((event.eventName === 'issue_comment' || event.eventName === 'pull_request_review_comment') &&
|
|
184
179
|
event.payload.action === 'deleted') {
|
|
185
180
|
const repo = event.payload.repository?.full_name ?? '';
|
|
@@ -19,21 +19,14 @@ export interface DeletionResult {
|
|
|
19
19
|
/**
|
|
20
20
|
* Handle GitHub `issue_comment.deleted` and `pull_request_review_comment.deleted`.
|
|
21
21
|
*
|
|
22
|
-
*
|
|
22
|
+
* Filter by tenant_id + kind='raw'. Multi-row archive:
|
|
23
23
|
* GitHub edits keep the same artifact_ref, so multiple active raw rows can
|
|
24
24
|
* match a single deletion event. Archive ALL of them.
|
|
25
25
|
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
* with archivedCount=0 — survivors stayed searchable, leaking private bodies.
|
|
31
|
-
*
|
|
32
|
-
* v1.3.1 fix: ONE shared DB handle wrapping ALL archives + the idempotency
|
|
33
|
-
* mark in a single outer SAVEPOINT. Any per-row failure rolls back the entire
|
|
34
|
-
* batch (including idempotency), so retry re-attempts cleanly. archiveRawMemory
|
|
35
|
-
* (the lower-level function from raw-archive.js) runs its own inner SAVEPOINT
|
|
36
|
-
* which nests safely inside the outer one.
|
|
26
|
+
* ONE shared DB handle wraps ALL archives + the idempotency mark in a single
|
|
27
|
+
* outer SAVEPOINT: a per-row failure rolls back the whole batch, idempotency
|
|
28
|
+
* included, so a retry re-attempts cleanly instead of leaving searchable
|
|
29
|
+
* survivors. archiveRawMemory's own inner SAVEPOINT nests safely inside it.
|
|
37
30
|
*
|
|
38
31
|
* Tenant scope and kind='raw' filtering are load-bearing: without them a
|
|
39
32
|
* deletion event from tenant A could archive tenant B's row sharing the same
|
|
@@ -4,21 +4,14 @@ import { hasSeenKey, markKeySeen } from './idempotency.js';
|
|
|
4
4
|
/**
|
|
5
5
|
* Handle GitHub `issue_comment.deleted` and `pull_request_review_comment.deleted`.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
7
|
+
* Filter by tenant_id + kind='raw'. Multi-row archive:
|
|
8
8
|
* GitHub edits keep the same artifact_ref, so multiple active raw rows can
|
|
9
9
|
* match a single deletion event. Archive ALL of them.
|
|
10
10
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* with archivedCount=0 — survivors stayed searchable, leaking private bodies.
|
|
16
|
-
*
|
|
17
|
-
* v1.3.1 fix: ONE shared DB handle wrapping ALL archives + the idempotency
|
|
18
|
-
* mark in a single outer SAVEPOINT. Any per-row failure rolls back the entire
|
|
19
|
-
* batch (including idempotency), so retry re-attempts cleanly. archiveRawMemory
|
|
20
|
-
* (the lower-level function from raw-archive.js) runs its own inner SAVEPOINT
|
|
21
|
-
* which nests safely inside the outer one.
|
|
11
|
+
* ONE shared DB handle wraps ALL archives + the idempotency mark in a single
|
|
12
|
+
* outer SAVEPOINT: a per-row failure rolls back the whole batch, idempotency
|
|
13
|
+
* included, so a retry re-attempts cleanly instead of leaving searchable
|
|
14
|
+
* survivors. archiveRawMemory's own inner SAVEPOINT nests safely inside it.
|
|
22
15
|
*
|
|
23
16
|
* Tenant scope and kind='raw' filtering are load-bearing: without them a
|
|
24
17
|
* deletion event from tenant A could archive tenant B's row sharing the same
|
|
@@ -3,9 +3,8 @@ import type { Context } from '../../api.js';
|
|
|
3
3
|
/**
|
|
4
4
|
* GitHub webhook DLQ. Mirrors the Slack DLQ shape (src/connectors/slack/dlq.ts)
|
|
5
5
|
* but carries GitHub-specific metadata: event_name, delivery_id, signature,
|
|
6
|
-
* installation_id, repo_full_name
|
|
7
|
-
*
|
|
8
|
-
* from the raw payload.
|
|
6
|
+
* installation_id, repo_full_name, so a `hippo gh dlq replay` operator can
|
|
7
|
+
* triage without re-deriving anything from the raw payload.
|
|
9
8
|
*
|
|
10
9
|
* Buckets:
|
|
11
10
|
* - parse_error — raw_payload was not valid JSON
|
|
@@ -56,7 +55,7 @@ export interface ReplayDlqOpts {
|
|
|
56
55
|
/** Current webhook secret. If omitted, signature check is skipped (force-only path). */
|
|
57
56
|
webhookSecret?: string;
|
|
58
57
|
/**
|
|
59
|
-
* Previous webhook secret during rotation
|
|
58
|
+
* Previous webhook secret during rotation.
|
|
60
59
|
* Operators rotating GITHUB_WEBHOOK_SECRET would otherwise be forced into
|
|
61
60
|
* --force on DLQ rows written under the old secret. Plumbed through to
|
|
62
61
|
* verifyGitHubSignature.previousSecret.
|
|
@@ -79,10 +78,8 @@ export interface ReplayResult {
|
|
|
79
78
|
* route already knows how to route an envelope, so it passes that capability
|
|
80
79
|
* back in.
|
|
81
80
|
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
* itself, so the field was a phantom — any future hook that trusted the
|
|
85
|
-
* passed-in value would dedupe against a stale key. Field removed.
|
|
81
|
+
* No `idempotencyKey` field: ingest re-derives the key from the parsed event,
|
|
82
|
+
* so a hook trusting a passed-in key would dedupe against a stale one.
|
|
86
83
|
*/
|
|
87
84
|
export type IngestHook = (ctx: Context, args: {
|
|
88
85
|
rawPayload: string;
|
|
@@ -101,7 +98,7 @@ export type IngestHook = (ctx: Context, args: {
|
|
|
101
98
|
* 4. Type-guard the envelope. Fail → bump, `unhandled`.
|
|
102
99
|
* 5. If an `ingestHook` is supplied, call it and return its memoryId.
|
|
103
100
|
* If not (dry-run path), bump retry_count and return status `replayed`
|
|
104
|
-
* with memoryId=null. The webhook route wires the real hook
|
|
101
|
+
* with memoryId=null. The webhook route wires the real hook.
|
|
105
102
|
*
|
|
106
103
|
* Mirrors Slack's "always use current routing" policy: replays use the
|
|
107
104
|
* deployment state NOW, not at the time of original DLQing.
|
|
@@ -79,7 +79,7 @@ function bumpRetryCount(hippoRoot, id) {
|
|
|
79
79
|
* 4. Type-guard the envelope. Fail → bump, `unhandled`.
|
|
80
80
|
* 5. If an `ingestHook` is supplied, call it and return its memoryId.
|
|
81
81
|
* If not (dry-run path), bump retry_count and return status `replayed`
|
|
82
|
-
* with memoryId=null. The webhook route wires the real hook
|
|
82
|
+
* with memoryId=null. The webhook route wires the real hook.
|
|
83
83
|
*
|
|
84
84
|
* Mirrors Slack's "always use current routing" policy: replays use the
|
|
85
85
|
* deployment state NOW, not at the time of original DLQing.
|
|
@@ -125,8 +125,7 @@ export async function replayDlqEntry(ctx, id, opts = {}) {
|
|
|
125
125
|
// Real replay path. The route's IngestHook is responsible for routing,
|
|
126
126
|
// idempotency, and writing the memory. The DLQ module only validates the
|
|
127
127
|
// surface and bumps the retry counter.
|
|
128
|
-
//
|
|
129
|
-
// from the parsed event (artifact_ref + updated_at) since v1.3.1.
|
|
128
|
+
// No idempotencyKey arg: the hook re-derives it from the parsed event (artifact_ref + updated_at).
|
|
130
129
|
const eventName = row.eventName ?? '';
|
|
131
130
|
const deliveryId = row.deliveryId ?? '';
|
|
132
131
|
const { memoryId } = await opts.ingestHook(ctx, {
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
* GitHub event ingest with afterWrite race-safe idempotency.
|
|
3
3
|
*
|
|
4
4
|
* Mirrors src/connectors/slack/ingest.ts. The dedupe key is sha256(eventName +
|
|
5
|
-
* ':' + rawBody)
|
|
5
|
+
* ':' + rawBody), derived from the signed body, not from the
|
|
6
6
|
* unsigned X-GitHub-Delivery header, so a replay attacker cannot bypass
|
|
7
7
|
* idempotency by rotating the delivery UUID.
|
|
8
8
|
*
|
|
9
|
-
* Race semantics
|
|
9
|
+
* Race semantics:
|
|
10
10
|
* - Fast path: hasSeenKey pre-check returns 'duplicate' for the common case.
|
|
11
11
|
* - Slow path: hasSeenKey passes (no row yet). Two workers may race into
|
|
12
12
|
* remember() concurrently. Inside the writeEntry SAVEPOINT, INSERT OR
|
|
@@ -15,10 +15,8 @@
|
|
|
15
15
|
* rolls back this worker's memory row). Exactly one memory exists per
|
|
16
16
|
* idempotency_key.
|
|
17
17
|
*
|
|
18
|
-
* The
|
|
19
|
-
*
|
|
20
|
-
* tests pre-populate github_event_log inside the SAVEPOINT to actually
|
|
21
|
-
* exercise the changes=0 -> rollback path.
|
|
18
|
+
* The `__testInjectBeforeLog` hook below lets tests pre-populate github_event_log
|
|
19
|
+
* inside the SAVEPOINT to exercise the changes=0 -> rollback path, not just the fast path.
|
|
22
20
|
*/
|
|
23
21
|
import { type Context } from '../../api.js';
|
|
24
22
|
import { type DatabaseSyncLike } from '../../db.js';
|
|
@@ -50,7 +48,7 @@ export type IngestEvent = {
|
|
|
50
48
|
export interface IngestInput {
|
|
51
49
|
/** The X-GitHub-Event header value + parsed body, discriminated. */
|
|
52
50
|
event: IngestEvent;
|
|
53
|
-
/** The raw HTTP body
|
|
51
|
+
/** The raw HTTP body, used for the idempotency key (replay-safe). */
|
|
54
52
|
rawBody: string;
|
|
55
53
|
/** X-GitHub-Delivery header value, audit metadata only. */
|
|
56
54
|
deliveryId: string;
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
* GitHub event ingest with afterWrite race-safe idempotency.
|
|
3
3
|
*
|
|
4
4
|
* Mirrors src/connectors/slack/ingest.ts. The dedupe key is sha256(eventName +
|
|
5
|
-
* ':' + rawBody)
|
|
5
|
+
* ':' + rawBody), derived from the signed body, not from the
|
|
6
6
|
* unsigned X-GitHub-Delivery header, so a replay attacker cannot bypass
|
|
7
7
|
* idempotency by rotating the delivery UUID.
|
|
8
8
|
*
|
|
9
|
-
* Race semantics
|
|
9
|
+
* Race semantics:
|
|
10
10
|
* - Fast path: hasSeenKey pre-check returns 'duplicate' for the common case.
|
|
11
11
|
* - Slow path: hasSeenKey passes (no row yet). Two workers may race into
|
|
12
12
|
* remember() concurrently. Inside the writeEntry SAVEPOINT, INSERT OR
|
|
@@ -15,10 +15,8 @@
|
|
|
15
15
|
* rolls back this worker's memory row). Exactly one memory exists per
|
|
16
16
|
* idempotency_key.
|
|
17
17
|
*
|
|
18
|
-
* The
|
|
19
|
-
*
|
|
20
|
-
* tests pre-populate github_event_log inside the SAVEPOINT to actually
|
|
21
|
-
* exercise the changes=0 -> rollback path.
|
|
18
|
+
* The `__testInjectBeforeLog` hook below lets tests pre-populate github_event_log
|
|
19
|
+
* inside the SAVEPOINT to exercise the changes=0 -> rollback path, not just the fast path.
|
|
22
20
|
*/
|
|
23
21
|
import { remember } from '../../api.js';
|
|
24
22
|
import { openHippoDb, closeHippoDb } from '../../db.js';
|
|
@@ -39,7 +37,7 @@ function transformEvent(event) {
|
|
|
39
37
|
}
|
|
40
38
|
}
|
|
41
39
|
/**
|
|
42
|
-
*
|
|
40
|
+
* Extract the source-normalized identifier the idempotency key needs.
|
|
43
41
|
* Backfill and webhook both produce IngestEvent objects describing the same
|
|
44
42
|
* source revision, so deriving the key from these fields collapses both paths
|
|
45
43
|
* onto the same dedupe row. Mirrors the artifactRef strings in transform.ts.
|
|
@@ -113,10 +111,8 @@ export function ingestEvent(ctx, input) {
|
|
|
113
111
|
}
|
|
114
112
|
}
|
|
115
113
|
if (e instanceof RejectedValueError) {
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
// seen exactly like the empty-body branch above so a GitHub retry of
|
|
119
|
-
// the same delivery acks as done, not error.
|
|
114
|
+
// A tombstone hit is a PERMANENT skip, never DLQ-retried: mark the key seen like the empty-body
|
|
115
|
+
// branch above so a GitHub retry of the same delivery acks as done, not error.
|
|
120
116
|
markKeySeenWithoutMemory(ctx.hippoRoot, idempotencyKey, input);
|
|
121
117
|
return { status: 'skipped', memoryId: null };
|
|
122
118
|
}
|
|
@@ -135,7 +131,7 @@ function markKeySeenWithoutMemory(hippoRoot, idempotencyKey, input) {
|
|
|
135
131
|
}
|
|
136
132
|
}
|
|
137
133
|
function rememberWithEventLog(ctx, input, idempotencyKey, opts) {
|
|
138
|
-
//
|
|
134
|
+
// No `|| 'connector:github'` fallback (see rememberWithEventLog in slack/ingest.ts for rationale).
|
|
139
135
|
const result = remember(ctx, {
|
|
140
136
|
...opts,
|
|
141
137
|
untrusted: true,
|
|
@@ -4,13 +4,11 @@
|
|
|
4
4
|
* Production uses `realGitHubFetcher` against `https://api.github.com`.
|
|
5
5
|
* Tests inject a fake `GitHubFetcher` so they never hit the network.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* 401/403/404/500 into empty pages produced empty backfills with no
|
|
10
|
-
* operator signal, so this code path is now load-bearing.
|
|
7
|
+
* Any non-200 response that is NOT a recognized rate-limit pause MUST throw
|
|
8
|
+
* `GitHubFetchError`: an empty page would be an empty backfill with no operator signal.
|
|
11
9
|
*/
|
|
12
|
-
import type { JsonValue } from './types.js';
|
|
13
10
|
import { type RateLimitInfo } from './ratelimit.js';
|
|
11
|
+
import type { JsonValue } from '../../json.js';
|
|
14
12
|
export declare class GitHubFetchError extends Error {
|
|
15
13
|
readonly status: number;
|
|
16
14
|
readonly bodyExcerpt: string;
|
|
@@ -4,10 +4,8 @@
|
|
|
4
4
|
* Production uses `realGitHubFetcher` against `https://api.github.com`.
|
|
5
5
|
* Tests inject a fake `GitHubFetcher` so they never hit the network.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* 401/403/404/500 into empty pages produced empty backfills with no
|
|
10
|
-
* operator signal, so this code path is now load-bearing.
|
|
7
|
+
* Any non-200 response that is NOT a recognized rate-limit pause MUST throw
|
|
8
|
+
* `GitHubFetchError`: an empty page would be an empty backfill with no operator signal.
|
|
11
9
|
*/
|
|
12
10
|
import { parseRateLimit } from './ratelimit.js';
|
|
13
11
|
import { fetchWithRetry } from '../../http-retry.js';
|
|
@@ -44,6 +42,7 @@ function headersToRecord(h) {
|
|
|
44
42
|
return Object.fromEntries(entries);
|
|
45
43
|
}
|
|
46
44
|
const GITHUB_TIMEOUT_MS = 30_000;
|
|
45
|
+
const ERROR_BODY_SNIPPET_CHARS = 256;
|
|
47
46
|
export const realGitHubFetcher = async ({ url, token }) => {
|
|
48
47
|
const res = await fetchWithRetry(url, {
|
|
49
48
|
headers: {
|
|
@@ -54,10 +53,10 @@ export const realGitHubFetcher = async ({ url, token }) => {
|
|
|
54
53
|
}, { timeoutMs: GITHUB_TIMEOUT_MS });
|
|
55
54
|
const headers = headersToRecord(res.headers);
|
|
56
55
|
const rateLimit = parseRateLimit(headers, res.status);
|
|
57
|
-
//
|
|
56
|
+
// Don't silently turn 401/403/404/500 into empty pages.
|
|
58
57
|
if (res.status !== 200 && rateLimit.reason === 'none') {
|
|
59
58
|
const body = await res.text().catch(() => '');
|
|
60
|
-
throw new GitHubFetchError(res.status, body.slice(0,
|
|
59
|
+
throw new GitHubFetchError(res.status, body.slice(0, ERROR_BODY_SNIPPET_CHARS), url);
|
|
61
60
|
}
|
|
62
61
|
// SAFETY: GitHub's list/paginated endpoints (issues, comments, etc. — the
|
|
63
62
|
// only endpoints this backfill fetcher targets) return a JSON array body
|