hippo-memory 1.72.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/agent-memories/apply.js +5 -5
- package/dist/api/assemble.js +2 -1
- package/dist/api/audit.d.ts +7 -7
- package/dist/api/audit.js +19 -17
- package/dist/api/auth.d.ts +5 -6
- package/dist/api/auth.js +28 -33
- package/dist/api/conflicts.js +1 -1
- package/dist/api/context-auto.js +4 -2
- package/dist/api/context-select.js +8 -20
- package/dist/api/context.js +12 -10
- package/dist/api/drill-down.js +2 -1
- package/dist/api/forget.d.ts +2 -2
- package/dist/api/forget.js +6 -7
- package/dist/api/graph.js +2 -1
- package/dist/api/learn.d.ts +1 -1
- package/dist/api/learn.js +45 -44
- package/dist/api/memories.d.ts +4 -2
- package/dist/api/memories.js +16 -5
- package/dist/api/objects.js +2 -1
- package/dist/api/outcome.d.ts +5 -5
- package/dist/api/outcome.js +16 -15
- package/dist/api/predictions.js +2 -1
- package/dist/api/promote.d.ts +3 -3
- package/dist/api/promote.js +25 -27
- package/dist/api/prompt-hook.d.ts +3 -1
- package/dist/api/prompt-hook.js +1 -0
- package/dist/api/quarantine.js +2 -1
- package/dist/api/recall-core.js +8 -3
- package/dist/api/recall-pipeline.js +17 -17
- package/dist/api/recall-types.d.ts +3 -1
- package/dist/api/recall.js +32 -29
- package/dist/api/remember-local.d.ts +5 -2
- package/dist/api/remember-local.js +14 -6
- package/dist/api/remember.d.ts +5 -2
- package/dist/api/remember.js +16 -14
- package/dist/api/session-end.d.ts +24 -0
- package/dist/api/session-end.js +31 -1
- package/dist/api/sleep-run.js +5 -4
- package/dist/api/sleep.js +4 -3
- package/dist/api/tokens.js +1 -1
- package/dist/api/types.d.ts +1 -7
- package/dist/capture/command.js +1 -1
- package/dist/capture/compaction-spool.d.ts +3 -5
- package/dist/capture/compaction-spool.js +41 -46
- package/dist/capture/session-end-caller.js +1 -1
- package/dist/capture/session-worker.js +19 -6
- package/dist/cli/audit.js +10 -10
- package/dist/cli/auth.js +4 -4
- package/dist/cli/client.d.ts +1 -2
- package/dist/cli/client.js +13 -13
- package/dist/cli/context.js +1 -1
- package/dist/cli/curate.d.ts +1 -1
- package/dist/cli/curate.js +9 -9
- package/dist/cli/decisions.js +4 -5
- package/dist/cli/hook-runtime.d.ts +3 -1
- package/dist/cli/hook-runtime.js +2 -2
- package/dist/cli/init.d.ts +1 -1
- package/dist/cli/init.js +9 -9
- package/dist/cli/object-verbs.d.ts +2 -2
- package/dist/cli/object-verbs.js +4 -4
- package/dist/cli/predictions.js +2 -3
- package/dist/cli/quality-repair-once.d.ts +1 -1
- package/dist/cli/quality-repair-once.js +2 -2
- package/dist/cli/refine-llm.js +2 -1
- package/dist/cli/remember.js +5 -5
- package/dist/cli/serve.js +9 -4
- package/dist/cli/session-hooks.d.ts +0 -4
- package/dist/cli/session-hooks.js +57 -66
- package/dist/cli/setup.d.ts +1 -1
- package/dist/cli/setup.js +2 -2
- package/dist/cli/shared.d.ts +1 -1
- package/dist/cli/shared.js +2 -2
- package/dist/cli/sleep.js +2 -2
- package/dist/cli/transfer.d.ts +3 -3
- package/dist/cli/transfer.js +34 -15
- package/dist/cli/usage.d.ts +1 -1
- package/dist/cli/usage.js +1 -1
- package/dist/cli/verb-row.d.ts +2 -0
- package/dist/cli/verbs/health.js +2 -0
- package/dist/cli/verbs/memory.js +43 -51
- package/dist/cli/verbs/objects.js +6 -3
- package/dist/cli/verbs/sessions.js +4 -2
- package/dist/cli.js +1 -2
- package/dist/connectors/dlq.js +2 -1
- package/dist/connectors/github/deletion.js +2 -1
- package/dist/connectors/github/ingest.js +2 -1
- package/dist/connectors/github/tenant-routing.js +2 -1
- package/dist/connectors/github/webhook.js +4 -2
- package/dist/connectors/ingest.js +2 -1
- package/dist/connectors/slack/deletion.js +2 -1
- package/dist/connectors/slack/ingest.js +2 -1
- package/dist/connectors/slack/tenant-routing.js +2 -1
- package/dist/consolidate/conflicts.js +1 -1
- package/dist/consolidate/dag.js +14 -9
- package/dist/consolidate/dedupe.js +1 -1
- package/dist/consolidate/memory-value.js +1 -1
- package/dist/consolidate/merge.js +6 -6
- package/dist/consolidate/sleep.js +1 -1
- package/dist/consolidate/traces.js +2 -2
- package/dist/core/config.js +15 -16
- package/dist/core/markdown.js +2 -2
- package/dist/core/memory.d.ts +4 -4
- package/dist/core/memory.js +10 -5
- package/dist/core/physics.d.ts +3 -1
- package/dist/core/physics.js +17 -2
- package/dist/core/project-identity.js +2 -1
- package/dist/core/search-types.d.ts +1 -1
- package/dist/dashboard/dashboard-actions.d.ts +2 -2
- package/dist/dashboard/dashboard-actions.js +5 -5
- package/dist/dashboard/dashboard-queries.js +2 -7
- package/dist/dashboard/dashboard-snapshot.js +1 -3
- package/dist/dashboard/dashboard.js +16 -5
- package/dist/db/connect.d.ts +2 -1
- package/dist/db/connect.js +2 -1
- package/dist/db/migrate.js +14 -7
- package/dist/db/migrations/v11.js +1 -0
- package/dist/db/migrations/v14.js +1 -0
- package/dist/db/physics-state.d.ts +0 -4
- package/dist/db/physics-state.js +5 -31
- package/dist/db/request-stores.d.ts +1 -0
- package/dist/db/request-stores.js +2 -1
- package/dist/db/vector-store.js +2 -1
- package/dist/doctor.d.ts +3 -0
- package/dist/doctor.js +7 -7
- package/dist/embeddings/local.js +2 -2
- package/dist/embeddings/provider.js +1 -10
- package/dist/entry/store-kit.d.ts +37 -0
- package/dist/entry/store-kit.js +25 -0
- package/dist/entry/testing.d.ts +6 -0
- package/dist/entry/testing.js +7 -0
- package/dist/eval/judgment.js +1 -1
- package/dist/hooks/json-hooks.js +6 -2
- package/dist/importers/core.js +3 -3
- package/dist/importers/vault.d.ts +1 -1
- package/dist/importers/vault.js +18 -18
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/learn/autolearn.d.ts +6 -2
- package/dist/learn/autolearn.js +8 -6
- package/dist/learn/extract.js +1 -1
- package/dist/learn/invalidation.d.ts +2 -1
- package/dist/learn/invalidation.js +10 -5
- package/dist/mcp/memory-tools.d.ts +1 -1
- package/dist/mcp/memory-tools.js +6 -9
- package/dist/mcp/request.js +20 -19
- package/dist/mcp/stdio.js +13 -8
- package/dist/rerankers/cross-encoder.js +10 -3
- package/dist/search/embed-memories.d.ts +5 -2
- package/dist/search/embed-memories.js +40 -28
- package/dist/search/engine.d.ts +8 -0
- package/dist/search/engine.js +10 -0
- package/dist/search/vector.js +5 -5
- package/dist/server/auth.d.ts +1 -1
- package/dist/server/auth.js +7 -8
- package/dist/server/boot.d.ts +2 -1
- package/dist/server/boot.js +28 -19
- package/dist/server/client-ip.d.ts +9 -4
- package/dist/server/client-ip.js +35 -17
- package/dist/server/route-table.js +5 -6
- package/dist/server/routes/memories.js +4 -2
- package/dist/server/sleep-offload.js +3 -3
- package/dist/server/tls.d.ts +7 -2
- package/dist/server/tls.js +17 -5
- package/dist/server/types.d.ts +7 -2
- package/dist/server.d.ts +4 -41
- package/dist/server.js +3 -30
- package/dist/sharing/global-store.d.ts +4 -5
- package/dist/sharing/global-store.js +6 -11
- package/dist/sharing/global-sync.js +1 -1
- package/dist/sharing/search-both.js +9 -2
- package/dist/sharing/share.js +2 -4
- package/dist/store/ambient.js +2 -1
- package/dist/store/candidates.js +18 -8
- package/dist/store/cards.js +5 -4
- package/dist/store/compactions-record.js +8 -7
- package/dist/store/conflicts.d.ts +7 -1
- package/dist/store/conflicts.js +6 -5
- package/dist/store/connectors/dlq.js +2 -1
- package/dist/store/connectors/event-log.d.ts +13 -0
- package/dist/store/connectors/event-log.js +24 -0
- package/dist/store/connectors/github.js +6 -8
- package/dist/store/connectors/slack.js +6 -10
- package/dist/store/dashboard-reads.js +1 -0
- package/dist/store/delete-and-batch.js +5 -4
- package/dist/store/delivery-recorder.d.ts +2 -2
- package/dist/store/delivery-recorder.js +1 -6
- package/dist/store/descendant-page.js +2 -1
- package/dist/store/dormant.js +7 -3
- package/dist/store/entry-reads.d.ts +25 -3
- package/dist/store/entry-reads.js +43 -12
- package/dist/store/entry-row.js +9 -2
- package/dist/store/entry-writes.d.ts +10 -7
- package/dist/store/entry-writes.js +23 -11
- package/dist/store/goals.js +3 -2
- package/dist/store/graph-reads.js +5 -4
- package/dist/store/graph-view-rows.js +2 -1
- package/dist/store/half-life.js +2 -2
- package/dist/store/handoffs.d.ts +9 -3
- package/dist/store/handoffs.js +4 -3
- package/dist/store/index.d.ts +2 -2
- package/dist/store/index.js +1 -1
- package/dist/store/mirrors.js +9 -4
- package/dist/store/open.d.ts +2 -1
- package/dist/store/open.js +2 -1
- package/dist/store/port.d.ts +59 -3
- package/dist/store/quality-repair.js +1 -1
- package/dist/store/recall-trace.js +2 -1
- package/dist/store/rejected-values.js +14 -9
- package/dist/store/rows.js +2 -2
- package/dist/store/rule-sql.d.ts +2 -0
- package/dist/store/rule-sql.js +12 -6
- package/dist/store/search-rows.d.ts +5 -5
- package/dist/store/search-rows.js +9 -12
- package/dist/store/set-aside.js +1 -1
- package/dist/store/sqlite/connector-writes-group.js +3 -1
- package/dist/store/sqlite/entry-writes-group.d.ts +3 -3
- package/dist/store/sqlite/entry-writes-group.js +1 -1
- package/dist/store/sqlite/held-connection.d.ts +2 -0
- package/dist/store/sqlite/held-connection.js +1 -1
- package/dist/store/sqlite/last-recall-group.d.ts +13 -0
- package/dist/store/sqlite/last-recall-group.js +57 -0
- package/dist/store/sqlite/local.d.ts +0 -17
- package/dist/store/sqlite/local.js +0 -31
- package/dist/store/sqlite/maintenance-groups.d.ts +6 -0
- package/dist/store/sqlite/maintenance-groups.js +60 -0
- package/dist/store/sqlite/objects-group.js +14 -6
- package/dist/store/sqlite/store.d.ts +1 -8
- package/dist/store/sqlite/store.js +13 -24
- package/dist/store/summaries.d.ts +7 -0
- package/dist/store/summaries.js +49 -7
- package/dist/store/token-ledger-rows.js +3 -2
- package/dist/store/vector-index.js +5 -2
- package/dist/util/anthropic-messages.d.ts +3 -1
- package/dist/util/anthropic-messages.js +4 -2
- package/dist/util/atomic-write.d.ts +5 -2
- package/dist/util/atomic-write.js +12 -8
- package/dist/util/env.d.ts +10 -1
- package/dist/util/env.js +10 -1
- package/dist/util/format.d.ts +2 -0
- package/dist/util/format.js +4 -0
- package/dist/util/request-scope.d.ts +6 -2
- package/dist/util/request-scope.js +7 -3
- package/dist/util/spawn-argv.d.ts +9 -0
- package/dist/util/spawn-argv.js +57 -0
- package/dist/util/sql.d.ts +3 -0
- package/dist/util/sql.js +6 -0
- package/dist/util/vector-codec.d.ts +4 -0
- package/dist/util/vector-codec.js +15 -0
- package/dist/util/version.d.ts +1 -1
- package/dist/util/version.js +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +3 -1
- package/dist/api/on-store.d.ts +0 -16
- package/dist/api/on-store.js +0 -38
- package/dist/cli/lenient-id.d.ts +0 -2
- package/dist/cli/lenient-id.js +0 -13
package/dist/api/recall.js
CHANGED
|
@@ -5,9 +5,8 @@ import { recallScopeFilter } from '../store/search-rows.js';
|
|
|
5
5
|
import { estimateTokens } from '../util/token-text.js';
|
|
6
6
|
import { formatHandoffEvidenceLine } from '../core/handoff.js';
|
|
7
7
|
import { boostByGoals } from '../search/goal-boost.js';
|
|
8
|
-
import { storeFor } from '../store/
|
|
9
|
-
import {
|
|
10
|
-
import { physicsSearch } from '../search/physics-search.js';
|
|
8
|
+
import { storeFor } from '../store/sqlite/store.js';
|
|
9
|
+
import { searchByEngine } from '../search/engine.js';
|
|
11
10
|
import { churnStaleFactor } from '../search/boosts.js';
|
|
12
11
|
import { compareEntryIdentity } from '../core/compare.js';
|
|
13
12
|
import { dropHeldCopies, duplicateKey, storedTextKeys } from '../util/same-text.js';
|
|
@@ -64,23 +63,28 @@ function shownBand(opts, result) {
|
|
|
64
63
|
async function retrieveBand(ctx, store, opts, windowSize, own) {
|
|
65
64
|
let candidates = await store.searchRecallEntries(opts.query, recallSearchArgs(ctx, opts, windowSize, own));
|
|
66
65
|
if (opts.mode === 'hybrid' || opts.mode === 'physics') {
|
|
67
|
-
const searchOpts = { budget: Infinity,
|
|
68
|
-
const ranked = await
|
|
66
|
+
const searchOpts = { budget: Infinity, scope: opts.scope ?? null, vectorCandidates: recallVectorSpec(ctx, opts, own) };
|
|
67
|
+
const ranked = await searchByEngine({ root: ctx.hippoRoot, store }, opts.query, candidates, physicsFor(ctx, opts), searchOpts);
|
|
69
68
|
const rankedIds = new Set(ranked.map((r) => r.entry.id));
|
|
70
69
|
candidates = [...ranked.map((r) => r.entry), ...candidates.filter((e) => !rankedIds.has(e.id))];
|
|
71
70
|
}
|
|
72
71
|
const plan = planRecall(ctx, opts, candidates, own);
|
|
73
|
-
const
|
|
72
|
+
const reads = await readRecall(store, ctx, opts, plan);
|
|
73
|
+
const { result, writes } = composeRecall(ctx, opts, { windowSize, all: candidates, plan, reads });
|
|
74
|
+
await finishBandRecall(store, ctx, opts, writes, result.results.map((r) => r.id));
|
|
75
|
+
return result;
|
|
76
|
+
}
|
|
77
|
+
/** The one recall write: the caller's leading audit rows go first, and the ids shown are the ones strengthened. */
|
|
78
|
+
async function finishBandRecall(store, ctx, opts, writes, shownIds) {
|
|
74
79
|
await store.finishRecall({
|
|
75
|
-
...writes,
|
|
80
|
+
...writes,
|
|
81
|
+
audit: [...(opts.leadingAudit ?? []), ...writes.audit],
|
|
82
|
+
strengthen: strengthenOf(ctx, shownIds),
|
|
76
83
|
});
|
|
77
|
-
return result;
|
|
78
84
|
}
|
|
79
|
-
/**
|
|
80
|
-
function
|
|
81
|
-
return opts.mode === 'physics'
|
|
82
|
-
? physicsSearch(opts.query, pool, { ...searchOpts, physicsConfig: loadConfig(ctx.hippoRoot).physics })
|
|
83
|
-
: hybridSearch(opts.query, pool, searchOpts);
|
|
85
|
+
/** `mode` names the engine: physics with the store's config, hybrid otherwise. */
|
|
86
|
+
function physicsFor(ctx, opts) {
|
|
87
|
+
return opts.mode === 'physics' ? loadConfig(ctx.hippoRoot).physics : null;
|
|
84
88
|
}
|
|
85
89
|
/** The recall load's arguments: exact scope, current rows only, the caller's project and personal scope. */
|
|
86
90
|
function recallSearchArgs(ctx, opts, limit, own) {
|
|
@@ -112,8 +116,8 @@ async function retrieveFromStore(ctx, opts, windowSize, show, own) {
|
|
|
112
116
|
const loaded = await store.searchRecallEntries(opts.query, recallSearchArgs(ctx, opts, Math.max(windowSize, SHOW_RANKED_LEXICAL_WINDOW), own));
|
|
113
117
|
const pool = loaded.filter((e) => passesScopeFilterForRecall(e.scope ?? null, opts.scope, own));
|
|
114
118
|
// No scope option: the scope boost follows HIPPO_SCOPE and the skill env, as MCP recall always ranked.
|
|
115
|
-
const searchOpts = { budget: Infinity,
|
|
116
|
-
let ranked = await
|
|
119
|
+
const searchOpts = { budget: Infinity, vectorCandidates: recallVectorSpec(ctx, opts, own) };
|
|
120
|
+
let ranked = await searchByEngine({ root: ctx.hippoRoot, store }, opts.query, pool, physicsFor(ctx, opts), searchOpts);
|
|
117
121
|
// One goal read serves both boosts; the ranked list's log rows come first, so its scores win the log's first-write.
|
|
118
122
|
const goals = opts.sessionId && !opts.goalTag ? await store.activeGoals({ sessionId: opts.sessionId, tenantId: ctx.tenantId }) : null;
|
|
119
123
|
let rankedLog = [];
|
|
@@ -125,27 +129,26 @@ async function retrieveFromStore(ctx, opts, windowSize, show, own) {
|
|
|
125
129
|
const window = ranked.slice(0, windowSize).map((r) => r.entry);
|
|
126
130
|
const bandOpts = { ...opts, suppressRecallTrace: true };
|
|
127
131
|
const plan = planRecall(ctx, bandOpts, window, own);
|
|
128
|
-
const
|
|
129
|
-
|
|
130
|
-
});
|
|
132
|
+
const reads = await readRecall(store, ctx, bandOpts, plan, goals);
|
|
133
|
+
const { result, writes } = composeRecall(ctx, bandOpts, { windowSize, all: window, plan, reads, auditBand: false });
|
|
131
134
|
// Rows the vector arm added count as candidates too.
|
|
132
135
|
const inPool = new Set(pool.map((e) => e.id));
|
|
133
136
|
const candidates = [...pool, ...ranked.map((r) => r.entry).filter((e) => !inPool.has(e.id))];
|
|
134
137
|
const shown = show({ ranked, pool: candidates, droppedByScope: loaded.length - pool.length }, result);
|
|
135
138
|
const scores = new Map(ranked.map((r) => [r.entry.id, r.score]));
|
|
136
139
|
const shownRow = recallAuditRow(callerOf(ctx), 'recall', undefined, recallAuditMetadata(opts.query, shown.ids.length));
|
|
137
|
-
|
|
140
|
+
const trace = opts.suppressRecallTrace ? undefined : {
|
|
141
|
+
tenantId: ctx.tenantId,
|
|
142
|
+
sessionId: opts.sessionId ?? null,
|
|
143
|
+
pipeline: 'mcp',
|
|
144
|
+
query: opts.query,
|
|
145
|
+
results: shown.ids.map((id) => ({ memoryId: id, score: scores.get(id) ?? 0 })),
|
|
146
|
+
};
|
|
147
|
+
await finishBandRecall(store, ctx, opts, {
|
|
138
148
|
goalLog: [...rankedLog, ...writes.goalLog],
|
|
139
|
-
audit: [...
|
|
140
|
-
trace
|
|
141
|
-
|
|
142
|
-
sessionId: opts.sessionId ?? null,
|
|
143
|
-
pipeline: 'mcp',
|
|
144
|
-
query: opts.query,
|
|
145
|
-
results: shown.ids.map((id) => ({ memoryId: id, score: scores.get(id) ?? 0 })),
|
|
146
|
-
},
|
|
147
|
-
strengthen: strengthenOf(ctx, shown.ids),
|
|
148
|
-
});
|
|
149
|
+
audit: [...writes.audit, shownRow, ...shown.audit],
|
|
150
|
+
trace,
|
|
151
|
+
}, shown.ids);
|
|
149
152
|
return result;
|
|
150
153
|
}
|
|
151
154
|
/** Contract preflight: throws before any store-touching work. */
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ConfidenceLevel, Layer, MemoryEntry, MemoryKind } from '../core/memory.js';
|
|
2
|
+
import { type EmbedOutcome } from '../search/embed-memories.js';
|
|
2
3
|
import type { HippoDbContext } from './types.js';
|
|
3
4
|
/** One write by a caller in this process, every field already parsed and checked by it. */
|
|
4
5
|
export interface LocalRememberInput {
|
|
@@ -23,10 +24,12 @@ export interface StartedWeak {
|
|
|
23
24
|
readonly reason: string;
|
|
24
25
|
readonly strength: number;
|
|
25
26
|
}
|
|
26
|
-
/** `stored` carries the row as the store holds it after the write
|
|
27
|
+
/** `stored` carries the row as the store holds it after the write, and `embedding`, which settles with whether the row reached the vector index
|
|
28
|
+
* (a caller may ignore it; a failure is also logged). `skipped` is the gate dropping the write, with nothing stored or counted. */
|
|
27
29
|
export type LocalRememberOutcome = {
|
|
28
30
|
readonly status: 'stored';
|
|
29
31
|
readonly entry: MemoryEntry;
|
|
32
|
+
readonly embedding: Promise<EmbedOutcome>;
|
|
30
33
|
readonly startedWeak?: StartedWeak;
|
|
31
34
|
} | {
|
|
32
35
|
readonly status: 'skipped';
|
|
@@ -34,7 +37,7 @@ export type LocalRememberOutcome = {
|
|
|
34
37
|
readonly score: number;
|
|
35
38
|
};
|
|
36
39
|
/** One local write on hippo.db: gate, write, read-back, counter, then the embedding, started and never awaited. A refused value throws as `remember` does. */
|
|
37
|
-
export declare function rememberLocally(ctx: HippoDbContext, input: LocalRememberInput): LocalRememberOutcome
|
|
40
|
+
export declare function rememberLocally(ctx: HippoDbContext, input: LocalRememberInput): Promise<LocalRememberOutcome>;
|
|
38
41
|
/** What a caller asks of fact extraction. The key comes from the caller, which read it from its own environment. */
|
|
39
42
|
export interface FactExtractionRequest {
|
|
40
43
|
/** The caller asked for extraction (`--extract`); the store's config can turn it on without that. */
|
|
@@ -7,7 +7,7 @@ import { loadNewestEntries, schemaFitInStore } from '../store/candidates.js';
|
|
|
7
7
|
import { embedMemory } from '../search/embed-memories.js';
|
|
8
8
|
import { loadAllEntries, readEntry } from '../store/entry-reads.js';
|
|
9
9
|
import { updateStats } from '../store/index-and-stats.js';
|
|
10
|
-
import { errorMessage } from '../util/log.js';
|
|
10
|
+
import { errorMessage, log } from '../util/log.js';
|
|
11
11
|
import { remember } from './remember.js';
|
|
12
12
|
const WEAK_HALF_LIFE_FACTOR = 0.5;
|
|
13
13
|
/** The gate's verdict on one write, or null where it does not judge: the store has it off, or the write is pinned or forced. */
|
|
@@ -34,14 +34,23 @@ function storedEntry(ctx, id) {
|
|
|
34
34
|
throw new Error(`memory ${id} was written but cannot be read back`);
|
|
35
35
|
return entry;
|
|
36
36
|
}
|
|
37
|
+
/** Best-effort and slow (a model load or a network call), so the caller is not held for it; a row left out of the index is logged. */
|
|
38
|
+
function indexInBackground(hippoRoot, entry) {
|
|
39
|
+
return embedMemory(hippoRoot, entry).then((outcome) => {
|
|
40
|
+
if (outcome === 'failed') {
|
|
41
|
+
log.warn(`memory ${entry.id} is stored but not in the vector index, so recall by meaning will miss it; run 'hippo embed' to index it`);
|
|
42
|
+
}
|
|
43
|
+
return outcome;
|
|
44
|
+
});
|
|
45
|
+
}
|
|
37
46
|
/** One local write on hippo.db: gate, write, read-back, counter, then the embedding, started and never awaited. A refused value throws as `remember` does. */
|
|
38
|
-
export function rememberLocally(ctx, input) {
|
|
47
|
+
export async function rememberLocally(ctx, input) {
|
|
39
48
|
const schemaFit = schemaFitInStore(ctx.hippoRoot, ctx.tenantId, input.content, input.fitTags ?? input.tags);
|
|
40
49
|
const verdict = salienceVerdict(ctx, input);
|
|
41
50
|
if (verdict?.decision === 'skip')
|
|
42
51
|
return { status: 'skipped', reason: verdict.reason, score: verdict.score };
|
|
43
52
|
const startedWeak = verdict?.decision === 'start_weak' ? { reason: verdict.reason, strength: verdict.score } : undefined;
|
|
44
|
-
const { id } = remember(ctx, {
|
|
53
|
+
const { id } = await remember(ctx, {
|
|
45
54
|
content: input.content,
|
|
46
55
|
kind: input.kind,
|
|
47
56
|
scope: input.scope,
|
|
@@ -59,9 +68,8 @@ export function rememberLocally(ctx, input) {
|
|
|
59
68
|
});
|
|
60
69
|
const entry = storedEntry(ctx, id);
|
|
61
70
|
updateStats(ctx.hippoRoot, { remembered: 1 });
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
return startedWeak ? { status: 'stored', entry, startedWeak } : { status: 'stored', entry };
|
|
71
|
+
const embedding = indexInBackground(ctx.hippoRoot, entry);
|
|
72
|
+
return startedWeak ? { status: 'stored', entry, embedding, startedWeak } : { status: 'stored', entry, embedding };
|
|
65
73
|
}
|
|
66
74
|
/** Extracts facts from a row just stored and stores them beside it. Best-effort: a failure is named in the result, never thrown. */
|
|
67
75
|
export async function extractRememberedFacts(ctx, entry, request) {
|
package/dist/api/remember.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ConnectorEvent } from '../store/port.js';
|
|
2
2
|
import { type ConfidenceLevel, type Layer, type MemoryKind, type TraceOutcome } from '../core/memory.js';
|
|
3
|
-
import type { Context
|
|
3
|
+
import type { Context } from './types.js';
|
|
4
4
|
/** Row fields the caller's own process decides, as the CLI does from its flags and its salience gate. */
|
|
5
5
|
export interface RememberLocal {
|
|
6
6
|
layer?: Layer;
|
|
@@ -42,6 +42,9 @@ export interface RememberResult {
|
|
|
42
42
|
id: string;
|
|
43
43
|
kind: MemoryKind;
|
|
44
44
|
tenantId: string;
|
|
45
|
+
/** The row as written, so a caller can show it without reading it back. */
|
|
46
|
+
halfLifeDays: number;
|
|
47
|
+
tags: string[];
|
|
45
48
|
/** Set only when untrusted content was flagged and quarantined instead of stored under its requested scope. */
|
|
46
49
|
quarantined?: {
|
|
47
50
|
reason: string;
|
|
@@ -55,5 +58,5 @@ export interface RememberResult {
|
|
|
55
58
|
}
|
|
56
59
|
/** Store one memory: the store's entryWrites group writes the row and its one remember audit row, under the caller's subject.
|
|
57
60
|
* A write that carries a connector event or untrusted content goes through connectorWrites, which commits its companion rows with it. */
|
|
58
|
-
export declare function remember
|
|
61
|
+
export declare function remember(ctx: Context, opts: RememberOpts): Promise<RememberResult>;
|
|
59
62
|
//# sourceMappingURL=remember.d.ts.map
|
package/dist/api/remember.js
CHANGED
|
@@ -8,7 +8,8 @@ import { vetSecrets } from '../util/secret-detect.js';
|
|
|
8
8
|
import { assertCallerProject } from '../core/project-identity.js';
|
|
9
9
|
import { BadRequestError } from '../core/api-errors.js';
|
|
10
10
|
import { assertClientScope, personalScopeOf } from '../core/recall-scope.js';
|
|
11
|
-
import {
|
|
11
|
+
import { requireGroup } from '../store/index.js';
|
|
12
|
+
import { storeFor } from '../store/sqlite/store.js';
|
|
12
13
|
/** The scope a write lands in: the caller's own personal scope when asked for, else the client's, which may not be a personal one. */
|
|
13
14
|
function rememberScope(ctx, opts) {
|
|
14
15
|
if (!opts.personal) {
|
|
@@ -60,7 +61,7 @@ function weakened(entry, weaken) {
|
|
|
60
61
|
}
|
|
61
62
|
function rememberResult(ctx, prepared, written) {
|
|
62
63
|
const { entry, detection, warnings } = prepared;
|
|
63
|
-
const result = { id: entry.id, kind: entry.kind, tenantId: ctx.tenantId };
|
|
64
|
+
const result = { id: entry.id, kind: entry.kind, tenantId: ctx.tenantId, halfLifeDays: entry.half_life_days, tags: [...entry.tags] };
|
|
64
65
|
if (detection.flagged)
|
|
65
66
|
result.quarantined = { reason: detection.reason ?? 'unknown' };
|
|
66
67
|
if (warnings.length > 0)
|
|
@@ -75,17 +76,18 @@ function quarantineOf({ requestedScope, detection }) {
|
|
|
75
76
|
}
|
|
76
77
|
/** Store one memory: the store's entryWrites group writes the row and its one remember audit row, under the caller's subject.
|
|
77
78
|
* A write that carries a connector event or untrusted content goes through connectorWrites, which commits its companion rows with it. */
|
|
78
|
-
export function remember(ctx, opts) {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
});
|
|
79
|
+
export async function remember(ctx, opts) {
|
|
80
|
+
const store = storeFor(ctx);
|
|
81
|
+
const entryWrites = requireGroup(store, 'entryWrites');
|
|
82
|
+
const prepared = prepareRemember(ctx, opts);
|
|
83
|
+
// A store writes origin_project as given, so the served folder's fallback is stamped here.
|
|
84
|
+
const write = { entry: stampOriginProject(ctx.hippoRoot, prepared.entry), actor: ctx.actor.subject };
|
|
85
|
+
if (opts.untrusted !== true && opts.event === undefined) {
|
|
86
|
+
await entryWrites.writeEntry(write);
|
|
87
|
+
return rememberResult(ctx, prepared);
|
|
88
|
+
}
|
|
89
|
+
const connectorWrites = requireGroup(store, 'connectorWrites');
|
|
90
|
+
const written = await connectorWrites.writeConnectorEntry({ ...write, event: opts.event, quarantine: quarantineOf(prepared) });
|
|
91
|
+
return rememberResult(ctx, prepared, written);
|
|
90
92
|
}
|
|
91
93
|
//# sourceMappingURL=remember.js.map
|
|
@@ -19,4 +19,28 @@ export declare function closeEndedSession(ctx: Context, close: SessionClose): vo
|
|
|
19
19
|
export declare function bookSessionRereads(ctx: Context, transcriptPath: string | undefined, sessionId: string | null): Promise<string[]>;
|
|
20
20
|
/** The block a compaction lost: the fresh task snapshot and its session trail, or null with none or another session's snapshot. */
|
|
21
21
|
export declare function compactResumeBlock(ctx: Context, payloadSessionId: string | null): string | null;
|
|
22
|
+
/** What one session-end run needs from its host; the host owns sleep, transcript reading and capture, this module owns the order. */
|
|
23
|
+
export interface SessionEndWork<Scan> {
|
|
24
|
+
mode: 'turn' | 'full';
|
|
25
|
+
hippoRoot: string;
|
|
26
|
+
tenantId: string;
|
|
27
|
+
sessionId: string | null;
|
|
28
|
+
transcriptPath: string | undefined;
|
|
29
|
+
/** Builds the call context for one store root. */
|
|
30
|
+
contextFor: (root: string) => Context;
|
|
31
|
+
/** The store the run captures into: the project store, else global. */
|
|
32
|
+
storeRoot: () => string;
|
|
33
|
+
/** One line to the session-end log; `startFresh` restarts the file. */
|
|
34
|
+
log: (line: string, opts?: {
|
|
35
|
+
startFresh?: boolean;
|
|
36
|
+
}) => void;
|
|
37
|
+
sleep: () => Promise<void>;
|
|
38
|
+
readScan: (log: (line: string) => void) => Scan | null;
|
|
39
|
+
capture: (store: string, scan: Scan | null) => void;
|
|
40
|
+
digest: (scan: Scan | null, log: (line: string) => void) => void;
|
|
41
|
+
workingState: (log: (line: string) => void) => ReturnType<SessionClose['workingState']>;
|
|
42
|
+
evidence: () => HandoffEvidence;
|
|
43
|
+
}
|
|
44
|
+
/** Books re-reads, sleeps, captures, digests and closes one ended session, in that order. */
|
|
45
|
+
export declare function sessionEndWork<Scan>(w: SessionEndWork<Scan>): Promise<void>;
|
|
22
46
|
//# sourceMappingURL=session-end.d.ts.map
|
package/dist/api/session-end.js
CHANGED
|
@@ -29,7 +29,7 @@ function writeEndHandoff(ctx, sessionId, close) {
|
|
|
29
29
|
close.log('skip: no snapshot or transcript for session');
|
|
30
30
|
return;
|
|
31
31
|
}
|
|
32
|
-
const handoff = writeSessionEndHandoff(ctx.hippoRoot, ctx.tenantId, sessionId, close.evidence(), derived,
|
|
32
|
+
const handoff = writeSessionEndHandoff(ctx.hippoRoot, { tenantId: ctx.tenantId, sessionId, evidence: close.evidence(), derived, inPlace: close.turn });
|
|
33
33
|
close.log(handoff ? `wrote handoff for session ${sessionId}` : `skip: kept the existing handoff for session ${sessionId}`);
|
|
34
34
|
}
|
|
35
35
|
catch (err) {
|
|
@@ -98,4 +98,34 @@ export function compactResumeBlock(ctx, payloadSessionId) {
|
|
|
98
98
|
}
|
|
99
99
|
return compactResumeText(snapshot, events);
|
|
100
100
|
}
|
|
101
|
+
/** Books re-reads, sleeps, captures, digests and closes one ended session, in that order. */
|
|
102
|
+
export async function sessionEndWork(w) {
|
|
103
|
+
const rereadLog = await bookSessionRereads(w.contextFor(w.hippoRoot), w.transcriptPath, w.sessionId)
|
|
104
|
+
.catch((err) => [`re-read count failed: ${errorMessage(err)}`]);
|
|
105
|
+
// Sleep starts the log file afresh, so the lines go in after it; on exit too, in case sleep exits the process.
|
|
106
|
+
const flushRereadLog = () => { for (const line of rereadLog.splice(0))
|
|
107
|
+
w.log(line); };
|
|
108
|
+
process.once('exit', flushRereadLog);
|
|
109
|
+
// A folder with neither a project nor a global store must not get one made.
|
|
110
|
+
const store = w.storeRoot();
|
|
111
|
+
if (!isInitialized(store)) {
|
|
112
|
+
w.log('skip: no hippo store for this folder or globally', { startFresh: true });
|
|
113
|
+
flushRereadLog();
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
await w.sleep();
|
|
117
|
+
flushRereadLog();
|
|
118
|
+
const scan = w.readScan((line) => w.log(line));
|
|
119
|
+
w.capture(store, scan);
|
|
120
|
+
w.digest(scan, (line) => w.log(line));
|
|
121
|
+
// Only this session's snapshot, and only after sleep and capture.
|
|
122
|
+
const closeLog = (line) => w.log(line);
|
|
123
|
+
closeEndedSession(w.contextFor(store), {
|
|
124
|
+
sessionId: w.sessionId,
|
|
125
|
+
turn: w.mode === 'turn',
|
|
126
|
+
workingState: () => (w.transcriptPath ? w.workingState(closeLog) : null),
|
|
127
|
+
evidence: w.evidence,
|
|
128
|
+
log: closeLog,
|
|
129
|
+
});
|
|
130
|
+
}
|
|
101
131
|
//# sourceMappingURL=session-end.js.map
|
package/dist/api/sleep-run.js
CHANGED
|
@@ -52,7 +52,8 @@ async function runSleepPhases(ctx, opts, phases, counts) {
|
|
|
52
52
|
const consolidateResult = await phases.consolidate(ctx.hippoRoot, { dryRun });
|
|
53
53
|
counts.consolidation = consolidateResult.semanticCreated + consolidateResult.merged;
|
|
54
54
|
const result = sleepResultFrom(consolidateResult, dryRun);
|
|
55
|
-
//
|
|
55
|
+
// Consolidate's own read predates its writes, so it cannot stand in for this one.
|
|
56
|
+
// One read after consolidation serves dedup, the audit and ambient: dedup reports every row it deletes, so later phases subtract.
|
|
56
57
|
const loaded = { entries: phases.loadAllEntries(ctx.hippoRoot), backing: memoriesBackingObjects(ctx.hippoRoot) };
|
|
57
58
|
// Phase 2: Dedup (post-consolidate near-duplicate cleanup).
|
|
58
59
|
const dedupResult = phases.deduplicateStore(ctx.hippoRoot, { dryRun, actor: ctx.actor.subject, loaded });
|
|
@@ -60,7 +61,7 @@ async function runSleepPhases(ctx, opts, phases, counts) {
|
|
|
60
61
|
if (dedupResult.removed > 0)
|
|
61
62
|
result.deduped = dedupSummary(dedupResult);
|
|
62
63
|
// Phase 3: Quality audit (remove junk, report warnings; a dry run skips rows earlier phases would remove).
|
|
63
|
-
const audited = runQualityAudit(ctx, phases, { dryRun, consolidateResult, dedupResult, result, loaded });
|
|
64
|
+
const audited = await runQualityAudit(ctx, phases, { dryRun, consolidateResult, dedupResult, result, loaded });
|
|
64
65
|
counts.auditDeleted = audited.removed;
|
|
65
66
|
if (dryRun)
|
|
66
67
|
return result;
|
|
@@ -117,7 +118,7 @@ function dedupSummary(dedupResult) {
|
|
|
117
118
|
};
|
|
118
119
|
}
|
|
119
120
|
/** `removed` counts audit errors deleted (or that would be under dryRun); `remaining` is what the audit read, less the rows it deleted. */
|
|
120
|
-
function runQualityAudit(ctx, phases, options) {
|
|
121
|
+
async function runQualityAudit(ctx, phases, options) {
|
|
121
122
|
const { dryRun, consolidateResult, dedupResult, result, loaded } = options;
|
|
122
123
|
// The load followed consolidation, so only a dry run still holds the rows consolidation would remove; dedup's pairs are gone or would be.
|
|
123
124
|
const planned = new Set([...(dryRun ? consolidateResult.removedIds ?? [] : []), ...dedupResult.pairs.map((p) => p.removed)]);
|
|
@@ -127,7 +128,7 @@ function runQualityAudit(ctx, phases, options) {
|
|
|
127
128
|
return { removed: 0, remaining: allEntries };
|
|
128
129
|
const errors = auditOut.issues.filter((i) => i.severity === 'error');
|
|
129
130
|
const warnings = auditOut.issues.filter((i) => i.severity === 'warning');
|
|
130
|
-
const gone = dryRun ? errors.map((issue) => issue.memoryId) : removeAuditErrors(ctx, 'sleep-audit', errors);
|
|
131
|
+
const gone = dryRun ? errors.map((issue) => issue.memoryId) : await removeAuditErrors(ctx, 'sleep-audit', errors);
|
|
131
132
|
const removed = gone.length;
|
|
132
133
|
const deleted = new Set(gone);
|
|
133
134
|
if (removed > 0 || warnings.length > 0) {
|
package/dist/api/sleep.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
// The sleep cycle: decay, consolidation, dedupe, graph extraction and the other maintenance passes.
|
|
2
2
|
import { runSleep } from './sleep-run.js';
|
|
3
|
+
import { requireGroup } from '../store/index.js';
|
|
4
|
+
import { storeFor } from '../store/sqlite/store.js';
|
|
3
5
|
/** Sleeps the WHOLE hippoRoot, every tenant, so /v1/sleep stays loopback-only; SQLite only; never auto-deletes pinned, raw, kept or object-backing rows. */
|
|
4
6
|
export async function sleep(ctx, opts = {}) {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
}
|
|
7
|
+
// Sleep reopens hippo.db by folder, here and in its child process.
|
|
8
|
+
requireGroup(storeFor(ctx), 'localFiles');
|
|
8
9
|
return runSleep(ctx, opts);
|
|
9
10
|
}
|
|
10
11
|
//# sourceMappingURL=sleep.js.map
|
package/dist/api/tokens.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Token-use and failure reports.
|
|
2
2
|
import { rethrowIfSqliteBlocked } from '../util/sqlite-blocked.js';
|
|
3
|
-
import { storeFor } from '../store/
|
|
3
|
+
import { storeFor } from '../store/sqlite/store.js';
|
|
4
4
|
import { failureLogSummary, tokenUseSummary } from '../store/usage-reports.js';
|
|
5
5
|
import { errorMessage, log } from '../util/log.js';
|
|
6
6
|
import { DAY_MS } from '../util/time.js';
|
package/dist/api/types.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ export interface Actor {
|
|
|
10
10
|
scopes?: readonly string[];
|
|
11
11
|
/** An auth resolver vouched for this caller, so its admin role stops at its own tenant. */
|
|
12
12
|
viaAuthResolver?: true;
|
|
13
|
-
/** The host's operator (CLI, stdio MCP, keyless loopback, host-tenant admin key): may act beyond its tenant. */
|
|
13
|
+
/** The host's operator (CLI, stdio MCP, keyless loopback set to admin, host-tenant admin key): may act beyond its tenant. */
|
|
14
14
|
hostAdmin?: true;
|
|
15
15
|
owner?: string;
|
|
16
16
|
}
|
|
@@ -23,12 +23,6 @@ export interface Context {
|
|
|
23
23
|
export type HippoDbContext = Context & {
|
|
24
24
|
store?: undefined;
|
|
25
25
|
};
|
|
26
|
-
export type StoreReply<C extends Context, R> = C extends {
|
|
27
|
-
readonly store: HippoStore;
|
|
28
|
-
} ? Promise<R> : C extends {
|
|
29
|
-
readonly hippoRoot: string;
|
|
30
|
-
readonly store?: undefined;
|
|
31
|
-
} ? R : R | Promise<R>;
|
|
32
26
|
/** Builds process-local Actor values (admin and host admin) for CLI and CLI-run connector Contexts; Bearer-authed callers build Actor from the api_keys row
|
|
33
27
|
* instead. */
|
|
34
28
|
export declare function adminActor(subject: string): Actor;
|
package/dist/capture/command.js
CHANGED
|
@@ -164,7 +164,7 @@ export function captureExtractedItems(targetRoot, options, extracted, keys) {
|
|
|
164
164
|
continue;
|
|
165
165
|
}
|
|
166
166
|
const entry = captureEntry(item, options, baseHalfLifeDays);
|
|
167
|
-
if (wouldReject(entry.
|
|
167
|
+
if (wouldReject(entry.tenant_id ?? 'default', entry.id, entry.content)) {
|
|
168
168
|
console.log(` [reject] (${item.category}) ${item.content.slice(0, 80)} - matches a rejected value`);
|
|
169
169
|
tally.rejected++;
|
|
170
170
|
continue;
|
|
@@ -22,8 +22,6 @@ export interface SpoolFs {
|
|
|
22
22
|
flag?: 'wx';
|
|
23
23
|
}): void;
|
|
24
24
|
}
|
|
25
|
-
/** Test-only seam, so a test can make one fs call fail; null restores the real fs. */
|
|
26
|
-
export declare function _setSpoolFsForTests(next: SpoolFs | null): void;
|
|
27
25
|
export interface SpooledCompaction {
|
|
28
26
|
tenantId: string;
|
|
29
27
|
payload: PostCompactPayload;
|
|
@@ -32,7 +30,7 @@ export interface SpooledCompaction {
|
|
|
32
30
|
}
|
|
33
31
|
/** Records one spooled compaction; it calls `recorded` once the store holds the summary, so the file is done even if a later step throws. */
|
|
34
32
|
export type SpoolImporter = (spooled: SpooledCompaction, recorded: () => void) => void;
|
|
35
|
-
export declare function spool(hippoRoot: string, tenantId: string, payload: PostCompactPayload, text: CompactionText, at: Date): void;
|
|
33
|
+
export declare function spool(hippoRoot: string, tenantId: string, payload: PostCompactPayload, text: CompactionText, at: Date, fsx?: SpoolFs): void;
|
|
36
34
|
/** null when the text is not a spooled compaction. */
|
|
37
35
|
export declare function parseSpooled(text: string, fallbackTenantId: string): SpooledCompaction | null;
|
|
38
36
|
export interface SpoolCounts {
|
|
@@ -41,7 +39,7 @@ export interface SpoolCounts {
|
|
|
41
39
|
bad: number;
|
|
42
40
|
}
|
|
43
41
|
/** What doctor reports, read only: files a replay should have taken by now, claims and tmp files a dead process left, and `.bad` files. */
|
|
44
|
-
export declare function spoolCounts(hippoRoot: string, now: Date, waitingAfterMs: number): SpoolCounts;
|
|
42
|
+
export declare function spoolCounts(hippoRoot: string, now: Date, waitingAfterMs: number, fsx?: SpoolFs): SpoolCounts;
|
|
45
43
|
/** One replayer at a time holds `replay.lock`, so only one process claims a file even where Windows lets two renames of it both land. */
|
|
46
|
-
export declare function importSpool(hippoRoot: string, tenantId: string, log: Log, deadline: number, importOne: SpoolImporter): number;
|
|
44
|
+
export declare function importSpool(hippoRoot: string, tenantId: string, log: Log, deadline: number, importOne: SpoolImporter, fsx?: SpoolFs): number;
|
|
47
45
|
//# sourceMappingURL=compaction-spool.d.ts.map
|