hippo-memory 1.65.0 → 1.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/recall-types.d.ts +1 -0
- package/dist/api/recall.js +5 -5
- package/dist/db/open.d.ts +5 -1
- package/dist/db/open.js +28 -7
- package/dist/db/sqlite-blocked.d.ts +11 -1
- package/dist/db/sqlite-blocked.js +20 -2
- package/dist/db.d.ts +2 -2
- package/dist/db.js +2 -2
- package/dist/embeddings.d.ts +11 -2
- package/dist/embeddings.js +20 -11
- package/dist/quality-repair.js +3 -0
- package/dist/search/hybrid.d.ts +3 -0
- package/dist/search/physics-search.d.ts +3 -0
- package/dist/search/physics-search.js +18 -19
- package/dist/search/vector.d.ts +7 -2
- package/dist/search/vector.js +27 -10
- package/dist/server/routes/recall.js +6 -9
- package/dist/server.d.ts +7 -2
- package/dist/server.js +5 -0
- package/dist/store/open.js +2 -1
- package/dist/store-port.d.ts +19 -2
- package/dist/store-port.js +18 -1
- package/dist/vector-store.d.ts +8 -0
- package/dist/vector-store.js +31 -19
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
|
@@ -152,6 +152,7 @@ export interface RecallOpts {
|
|
|
152
152
|
/** Set only by the MCP recall tool, which ranks with its own scorer and drops copies from its own final list: this call
|
|
153
153
|
* then keeps a memory that a merged row in the same result holds word for word. Other callers leave it unset. */
|
|
154
154
|
keepHeldCopies?: boolean;
|
|
155
|
+
leadingAudit?: readonly AppendAuditOpts[];
|
|
155
156
|
/** MCP recall only: `retrieve` ranks the whole scoped store and strengthens and traces (pipeline 'mcp') just the ids this returns; `results` stays the window band. */
|
|
156
157
|
showRanked?: (ranking: StoreRanking, result: RecallResult) => ShownRanking;
|
|
157
158
|
}
|
package/dist/api/recall.js
CHANGED
|
@@ -62,7 +62,7 @@ export function recall(ctx, opts) {
|
|
|
62
62
|
const all = loadRecallSearchEntries(ctx.hippoRoot, opts.query, windowSize, ctx.tenantId, opts.scope, 'exact', false, recallOrigin(opts), own);
|
|
63
63
|
const plan = planRecall(ctx, opts, all, own);
|
|
64
64
|
const { result, writes } = composeRecall(ctx, opts, windowSize, all, plan, readRecallSync(ctx, opts, plan));
|
|
65
|
-
finishRecallAt(ctx.hippoRoot, writes);
|
|
65
|
+
finishRecallAt(ctx.hippoRoot, { ...writes, audit: [...(opts.leadingAudit ?? []), ...writes.audit] });
|
|
66
66
|
return result;
|
|
67
67
|
}
|
|
68
68
|
function recallOrigin(opts) {
|
|
@@ -82,7 +82,7 @@ export async function retrieve(ctx, opts) {
|
|
|
82
82
|
const store = storeFor(ctx);
|
|
83
83
|
let candidates = await store.searchRecallEntries(opts.query, recallSearchArgs(ctx, opts, windowSize, own));
|
|
84
84
|
if (opts.mode === 'hybrid' || opts.mode === 'physics') {
|
|
85
|
-
const searchOpts = { budget: Infinity, hippoRoot: ctx.hippoRoot, scope: opts.scope ?? null, vectorCandidates: recallVectorSpec(ctx, opts, own) };
|
|
85
|
+
const searchOpts = { budget: Infinity, hippoRoot: ctx.hippoRoot, scope: opts.scope ?? null, vectorCandidates: recallVectorSpec(ctx, opts, own), store };
|
|
86
86
|
const ranked = opts.mode === 'physics'
|
|
87
87
|
? await physicsSearch(opts.query, candidates, { ...searchOpts, physicsConfig: loadConfig(ctx.hippoRoot).physics })
|
|
88
88
|
: await hybridSearch(opts.query, candidates, searchOpts);
|
|
@@ -91,7 +91,7 @@ export async function retrieve(ctx, opts) {
|
|
|
91
91
|
}
|
|
92
92
|
const plan = planRecall(ctx, opts, candidates, own);
|
|
93
93
|
const { result, writes } = composeRecall(ctx, opts, windowSize, candidates, plan, await readRecall(store, ctx, opts, plan));
|
|
94
|
-
await store.finishRecall({ ...writes, strengthen: strengthenOf(ctx, result.results.map((r) => r.id)) });
|
|
94
|
+
await store.finishRecall({ ...writes, audit: [...(opts.leadingAudit ?? []), ...writes.audit], strengthen: strengthenOf(ctx, result.results.map((r) => r.id)) });
|
|
95
95
|
return result;
|
|
96
96
|
}
|
|
97
97
|
/** The recall load's arguments: exact scope, current rows only, the caller's project and personal scope. */
|
|
@@ -127,7 +127,7 @@ async function retrieveFromStore(ctx, opts, windowSize, show, own) {
|
|
|
127
127
|
const loaded = await store.searchRecallEntries(opts.query, recallSearchArgs(ctx, opts, Math.max(windowSize, SHOW_RANKED_LEXICAL_WINDOW), own));
|
|
128
128
|
const pool = loaded.filter((e) => passesScopeFilterForRecall(e.scope ?? null, opts.scope, own));
|
|
129
129
|
// No scope option: the scope boost follows HIPPO_SCOPE and the skill env, as MCP recall always ranked.
|
|
130
|
-
const searchOpts = { budget: Infinity, hippoRoot: ctx.hippoRoot, vectorCandidates: recallVectorSpec(ctx, opts, own) };
|
|
130
|
+
const searchOpts = { budget: Infinity, hippoRoot: ctx.hippoRoot, vectorCandidates: recallVectorSpec(ctx, opts, own), store };
|
|
131
131
|
let ranked = opts.mode === 'physics'
|
|
132
132
|
? await physicsSearch(opts.query, pool, { ...searchOpts, physicsConfig: loadConfig(ctx.hippoRoot).physics })
|
|
133
133
|
: await hybridSearch(opts.query, pool, searchOpts);
|
|
@@ -151,7 +151,7 @@ async function retrieveFromStore(ctx, opts, windowSize, show, own) {
|
|
|
151
151
|
const shownRow = recallAuditRow(callerOf(ctx), 'recall', undefined, recallAuditMetadata(opts.query, shown.ids.length));
|
|
152
152
|
await store.finishRecall({
|
|
153
153
|
goalLog: [...rankedLog, ...writes.goalLog],
|
|
154
|
-
audit: [...writes.audit, shownRow, ...shown.audit],
|
|
154
|
+
audit: [...(opts.leadingAudit ?? []), ...writes.audit, shownRow, ...shown.audit],
|
|
155
155
|
trace: opts.suppressRecallTrace ? undefined : {
|
|
156
156
|
tenantId: ctx.tenantId,
|
|
157
157
|
sessionId: opts.sessionId ?? null,
|
package/dist/db/open.d.ts
CHANGED
|
@@ -18,10 +18,14 @@ export declare const SLEEP_DB_WAIT_MS = 5000;
|
|
|
18
18
|
export declare function scopedBusyWait(): number | undefined;
|
|
19
19
|
/** Runs `fn` so that every hippo.db open inside it, across awaits, throws; otherwise a missed port would create and write a hippo.db nobody reads. */
|
|
20
20
|
export declare function withSqliteBlocked<T>(storeKind: string, fn: () => T): T;
|
|
21
|
-
/** Runs `fn` with hippo.db opens allowed again, for a store whose own methods are backed by hippo.db. */
|
|
21
|
+
/** Runs `fn` with hippo.db opens allowed again, for a store whose own methods are backed by hippo.db, or a copy that reads the old hippo.db. */
|
|
22
22
|
export declare function withSqliteAllowed<T>(fn: () => T): T;
|
|
23
23
|
/** First line of a best-effort catch around a hippo.db open: an unported path must fail closed, not fall back silently. */
|
|
24
24
|
export declare function rethrowIfSqliteBlocked<E>(err: E): void;
|
|
25
|
+
/** store init --db, store copy --db and serve --db write this file in the hippo root; its text is the store kind. */
|
|
26
|
+
export declare const OTHER_STORE_MARKER = "other-store";
|
|
27
|
+
/** Throws where hippo.db must not open; checked on every open, not cached, since the marker can appear while a process runs. */
|
|
28
|
+
export declare function assertSqliteAllowed(hippoRoot: string): void;
|
|
25
29
|
/** `busyWaitMs` shortens every lock wait of this open, for a hook that must finish inside its own timeout. */
|
|
26
30
|
export declare function openHippoDb(hippoRoot: string, opts?: {
|
|
27
31
|
busyWaitMs?: number;
|
package/dist/db/open.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
-
import {
|
|
2
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { errorMessage, log } from '../log.js';
|
|
3
5
|
import { DatabaseSync } from './sqlite.js';
|
|
4
6
|
import { tableExists } from './tables.js';
|
|
5
7
|
import { assertBinaryCompatible } from './migrate.js';
|
|
6
8
|
import { connectHippoDb, getHippoDbPath } from './connect.js';
|
|
7
9
|
import { currentRequestStores, isScopedHandle, runWithRequestStores } from './request-stores.js';
|
|
8
|
-
import { SqliteBlockedError } from './sqlite-blocked.js';
|
|
10
|
+
import { OtherStoreFolderError, SqliteBlockedError } from './sqlite-blocked.js';
|
|
9
11
|
export { getHippoDbPath };
|
|
10
12
|
/** Lock wait for hook commands: under the 5 s prompt-hook budget even after a few skipped writes, and far above a normal write's hold. */
|
|
11
13
|
export const HOOK_DB_WAIT_MS = 1000;
|
|
@@ -32,29 +34,48 @@ const sqliteBlockedBy = new AsyncLocalStorage();
|
|
|
32
34
|
export function withSqliteBlocked(storeKind, fn) {
|
|
33
35
|
return sqliteBlockedBy.run(storeKind, fn);
|
|
34
36
|
}
|
|
35
|
-
/** Runs `fn` with hippo.db opens allowed again, for a store whose own methods are backed by hippo.db. */
|
|
37
|
+
/** Runs `fn` with hippo.db opens allowed again, for a store whose own methods are backed by hippo.db, or a copy that reads the old hippo.db. */
|
|
36
38
|
export function withSqliteAllowed(fn) {
|
|
37
|
-
return sqliteBlockedBy.exit(fn);
|
|
39
|
+
return sqliteBlockedBy.exit(() => markerWaived.run(true, fn));
|
|
38
40
|
}
|
|
39
41
|
/** First line of a best-effort catch around a hippo.db open: an unported path must fail closed, not fall back silently. */
|
|
40
42
|
export function rethrowIfSqliteBlocked(err) {
|
|
41
43
|
if (err instanceof SqliteBlockedError)
|
|
42
44
|
throw err;
|
|
43
45
|
}
|
|
44
|
-
|
|
46
|
+
/** store init --db, store copy --db and serve --db write this file in the hippo root; its text is the store kind. */
|
|
47
|
+
export const OTHER_STORE_MARKER = 'other-store';
|
|
48
|
+
const markerWaived = new AsyncLocalStorage();
|
|
49
|
+
// An empty or unreadable marker still refuses: falling through would write the hippo.db the marker forbids.
|
|
50
|
+
function markerKind(marker) {
|
|
51
|
+
try {
|
|
52
|
+
return readFileSync(marker, 'utf8').trim() || 'unknown';
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
log.debug(`${marker} is unreadable, so its store kind reads as unknown: ${errorMessage(err)}`);
|
|
56
|
+
return 'unknown';
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/** Throws where hippo.db must not open; checked on every open, not cached, since the marker can appear while a process runs. */
|
|
60
|
+
export function assertSqliteAllowed(hippoRoot) {
|
|
45
61
|
const storeKind = sqliteBlockedBy.getStore();
|
|
46
62
|
if (storeKind !== undefined)
|
|
47
63
|
throw new SqliteBlockedError(storeKind);
|
|
64
|
+
if (markerWaived.getStore())
|
|
65
|
+
return;
|
|
66
|
+
const marker = join(hippoRoot, OTHER_STORE_MARKER);
|
|
67
|
+
if (existsSync(marker))
|
|
68
|
+
throw new OtherStoreFolderError(markerKind(marker), marker);
|
|
48
69
|
}
|
|
49
70
|
/** `busyWaitMs` shortens every lock wait of this open, for a hook that must finish inside its own timeout. */
|
|
50
71
|
export function openHippoDb(hippoRoot, opts) {
|
|
51
|
-
assertSqliteAllowed();
|
|
72
|
+
assertSqliteAllowed(hippoRoot);
|
|
52
73
|
const stores = currentRequestStores();
|
|
53
74
|
return stores ? stores.get(hippoRoot, opts) : connectHippoDb(hippoRoot, opts?.busyWaitMs);
|
|
54
75
|
}
|
|
55
76
|
/** Open an existing store without changing it: no mkdir, WAL switch, migration or mirror cleanup. Throws when hippo.db is missing. */
|
|
56
77
|
export function openHippoDbReadOnly(hippoRoot) {
|
|
57
|
-
assertSqliteAllowed();
|
|
78
|
+
assertSqliteAllowed(hippoRoot);
|
|
58
79
|
const db = new DatabaseSync(getHippoDbPath(hippoRoot), { readOnly: true });
|
|
59
80
|
try {
|
|
60
81
|
db.exec('PRAGMA busy_timeout = 5000');
|
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
/** Thrown by a hippo.db open inside a request served from another store: the code path is not ported to the store port yet. */
|
|
2
2
|
export declare class SqliteBlockedError extends Error {
|
|
3
3
|
readonly storeKind: string;
|
|
4
|
-
constructor(storeKind: string);
|
|
4
|
+
constructor(storeKind: string, message?: string);
|
|
5
|
+
}
|
|
6
|
+
/** A store that lacks a whole group of port methods; it maps to the same 501, and the log names the group instead of hippo.db. */
|
|
7
|
+
export declare class StoreNotPortedError extends SqliteBlockedError {
|
|
8
|
+
readonly group: string;
|
|
9
|
+
constructor(storeKind: string, group: string);
|
|
10
|
+
}
|
|
11
|
+
/** Thrown by a hippo.db open in a folder whose marker file names the store that holds its memories, from the CLI as from a request. */
|
|
12
|
+
export declare class OtherStoreFolderError extends SqliteBlockedError {
|
|
13
|
+
readonly markerPath: string;
|
|
14
|
+
constructor(storeKind: string, markerPath: string);
|
|
5
15
|
}
|
|
6
16
|
//# sourceMappingURL=sqlite-blocked.d.ts.map
|
|
@@ -2,10 +2,28 @@
|
|
|
2
2
|
/** Thrown by a hippo.db open inside a request served from another store: the code path is not ported to the store port yet. */
|
|
3
3
|
export class SqliteBlockedError extends Error {
|
|
4
4
|
storeKind;
|
|
5
|
-
constructor(storeKind) {
|
|
6
|
-
super(
|
|
5
|
+
constructor(storeKind, message = `hippo.db is not opened while the '${storeKind}' store serves this request; this code path is not ported to the store yet`) {
|
|
6
|
+
super(message);
|
|
7
7
|
this.storeKind = storeKind;
|
|
8
8
|
this.name = 'SqliteBlockedError';
|
|
9
9
|
}
|
|
10
10
|
}
|
|
11
|
+
/** A store that lacks a whole group of port methods; it maps to the same 501, and the log names the group instead of hippo.db. */
|
|
12
|
+
export class StoreNotPortedError extends SqliteBlockedError {
|
|
13
|
+
group;
|
|
14
|
+
constructor(storeKind, group) {
|
|
15
|
+
super(storeKind, `the '${storeKind}' store has no '${group}' reads; this code path is not ported to the store yet`);
|
|
16
|
+
this.group = group;
|
|
17
|
+
this.name = 'StoreNotPortedError';
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** Thrown by a hippo.db open in a folder whose marker file names the store that holds its memories, from the CLI as from a request. */
|
|
21
|
+
export class OtherStoreFolderError extends SqliteBlockedError {
|
|
22
|
+
markerPath;
|
|
23
|
+
constructor(storeKind, markerPath) {
|
|
24
|
+
super(storeKind, `This folder's memories live in the '${storeKind}' store, so hippo.db is not used here (marker: ${markerPath})`);
|
|
25
|
+
this.markerPath = markerPath;
|
|
26
|
+
this.name = 'OtherStoreFolderError';
|
|
27
|
+
}
|
|
28
|
+
}
|
|
11
29
|
//# sourceMappingURL=sqlite-blocked.js.map
|
package/dist/db.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ export { isSqliteBusy, isStoreBusy, StoreBusyError, STORE_BUSY_MESSAGE, execWith
|
|
|
3
3
|
export { getSchemaVersion, getMeta, setMeta, isFtsAvailable } from './db/meta.js';
|
|
4
4
|
export { countTableRows, pruneConsolidationRuns } from './db/tables.js';
|
|
5
5
|
export { getCurrentSchemaVersion, IncompatibleBinaryError, ftsRowCounts, repairFtsDrift } from './db/migrate.js';
|
|
6
|
-
export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS, scopedBusyWait, withSqliteBlocked, withSqliteAllowed, rethrowIfSqliteBlocked, openHippoDb, openHippoDbReadOnly, closeHippoDb, } from './db/open.js';
|
|
6
|
+
export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS, scopedBusyWait, withSqliteBlocked, withSqliteAllowed, rethrowIfSqliteBlocked, OTHER_STORE_MARKER, openHippoDb, openHippoDbReadOnly, closeHippoDb, } from './db/open.js';
|
|
7
7
|
export { RequestStores, runWithRequestStores, currentRequestStores, outsideRequestStores } from './db/request-stores.js';
|
|
8
|
-
export { SqliteBlockedError } from './db/sqlite-blocked.js';
|
|
8
|
+
export { OtherStoreFolderError, SqliteBlockedError } from './db/sqlite-blocked.js';
|
|
9
9
|
//# sourceMappingURL=db.d.ts.map
|
package/dist/db.js
CHANGED
|
@@ -2,7 +2,7 @@ export { isSqliteBusy, isStoreBusy, StoreBusyError, STORE_BUSY_MESSAGE, execWith
|
|
|
2
2
|
export { getSchemaVersion, getMeta, setMeta, isFtsAvailable } from './db/meta.js';
|
|
3
3
|
export { countTableRows, pruneConsolidationRuns } from './db/tables.js';
|
|
4
4
|
export { getCurrentSchemaVersion, IncompatibleBinaryError, ftsRowCounts, repairFtsDrift } from './db/migrate.js';
|
|
5
|
-
export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS, scopedBusyWait, withSqliteBlocked, withSqliteAllowed, rethrowIfSqliteBlocked, openHippoDb, openHippoDbReadOnly, closeHippoDb, } from './db/open.js';
|
|
5
|
+
export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS, scopedBusyWait, withSqliteBlocked, withSqliteAllowed, rethrowIfSqliteBlocked, OTHER_STORE_MARKER, openHippoDb, openHippoDbReadOnly, closeHippoDb, } from './db/open.js';
|
|
6
6
|
export { RequestStores, runWithRequestStores, currentRequestStores, outsideRequestStores } from './db/request-stores.js';
|
|
7
|
-
export { SqliteBlockedError } from './db/sqlite-blocked.js';
|
|
7
|
+
export { OtherStoreFolderError, SqliteBlockedError } from './db/sqlite-blocked.js';
|
|
8
8
|
//# sourceMappingURL=db.js.map
|
package/dist/embeddings.d.ts
CHANGED
|
@@ -64,8 +64,19 @@ export declare function embeddingInputText(entry: {
|
|
|
64
64
|
* consistent with the compare side in `embeddingModelRequiresReindex`.
|
|
65
65
|
*/
|
|
66
66
|
export declare function saveStoredEmbeddingModel(hippoRoot: string, model: string): void;
|
|
67
|
+
/** What a store's vector index was built by: the stored identity, and whether any vector exists. */
|
|
68
|
+
export interface EmbeddingIndexState {
|
|
69
|
+
readonly storedModel: string | null;
|
|
70
|
+
readonly hasVectors: boolean;
|
|
71
|
+
}
|
|
72
|
+
/** The model the index was built by: the stored identity, else the default model when vectors exist. */
|
|
73
|
+
export declare function indexedModel(state: EmbeddingIndexState): string | null;
|
|
74
|
+
/** The one rebuild rule every store applies to its indexed model. */
|
|
75
|
+
export declare function indexNeedsRebuild(indexed: string | null, providerId: string): boolean;
|
|
67
76
|
export declare function resolveIndexedEmbeddingModel(hippoRoot: string, index?: Record<string, number[]>): string | null;
|
|
68
77
|
export declare function embeddingModelRequiresReindex(hippoRoot: string, model: string, index?: Record<string, number[]>): boolean;
|
|
78
|
+
/** hippo.db's index state, the meta row and the EXISTS on one handle. */
|
|
79
|
+
export declare function embeddingIndexStateAt(hippoRoot: string): EmbeddingIndexState;
|
|
69
80
|
/**
|
|
70
81
|
* Cosine similarity between two vectors. Handles unnormalized vectors.
|
|
71
82
|
* Returns 0 for empty or mismatched vectors.
|
|
@@ -75,8 +86,6 @@ export declare function cosineSimilarity(a: number[], b: number[]): number;
|
|
|
75
86
|
export declare function loadEmbeddingIndex(hippoRoot: string): Record<string, number[]>;
|
|
76
87
|
/** Stored vectors for `ids` only. */
|
|
77
88
|
export declare function loadStoredVectors(hippoRoot: string, ids: readonly string[]): Map<string, number[]>;
|
|
78
|
-
/** Whether the store holds any vector at all. */
|
|
79
|
-
export declare function hasEmbeddings(hippoRoot: string): boolean;
|
|
80
89
|
/** Replace every stored vector with `index`; `model` defaults to the stored index identity. */
|
|
81
90
|
export declare function saveEmbeddingIndex(hippoRoot: string, index: Record<string, number[]>, model?: string): void;
|
|
82
91
|
/**
|
package/dist/embeddings.js
CHANGED
|
@@ -101,16 +101,29 @@ export function saveStoredEmbeddingModel(hippoRoot, model) {
|
|
|
101
101
|
closeHippoDb(db);
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
|
+
/** The model the index was built by: the stored identity, else the default model when vectors exist. */
|
|
105
|
+
export function indexedModel(state) {
|
|
106
|
+
return state.storedModel ?? (state.hasVectors ? DEFAULT_EMBEDDING_MODEL : null);
|
|
107
|
+
}
|
|
108
|
+
/** The one rebuild rule every store applies to its indexed model. */
|
|
109
|
+
export function indexNeedsRebuild(indexed, providerId) {
|
|
110
|
+
return indexed !== null && indexed !== embeddingIndexIdentity(providerId);
|
|
111
|
+
}
|
|
104
112
|
export function resolveIndexedEmbeddingModel(hippoRoot, index) {
|
|
105
|
-
const
|
|
106
|
-
if (
|
|
107
|
-
return
|
|
108
|
-
|
|
109
|
-
return hasVectors ? DEFAULT_EMBEDDING_MODEL : null;
|
|
113
|
+
const storedModel = loadStoredEmbeddingModel(hippoRoot);
|
|
114
|
+
if (storedModel)
|
|
115
|
+
return storedModel;
|
|
116
|
+
return indexedModel({ storedModel, hasVectors: index ? Object.keys(index).length > 0 : withVectorDb(hippoRoot, hasStoredVectors) });
|
|
110
117
|
}
|
|
111
118
|
export function embeddingModelRequiresReindex(hippoRoot, model, index) {
|
|
112
|
-
|
|
113
|
-
|
|
119
|
+
return indexNeedsRebuild(resolveIndexedEmbeddingModel(hippoRoot, index), model);
|
|
120
|
+
}
|
|
121
|
+
/** hippo.db's index state, the meta row and the EXISTS on one handle. */
|
|
122
|
+
export function embeddingIndexStateAt(hippoRoot) {
|
|
123
|
+
return withVectorDb(hippoRoot, (db) => ({
|
|
124
|
+
storedModel: getMeta(db, EMBEDDING_MODEL_META_KEY, '').trim() || null,
|
|
125
|
+
hasVectors: hasStoredVectors(db),
|
|
126
|
+
}));
|
|
114
127
|
}
|
|
115
128
|
async function rebuildEmbeddingIndex(entries, provider) {
|
|
116
129
|
const rebuilt = {};
|
|
@@ -189,10 +202,6 @@ export function loadEmbeddingIndex(hippoRoot) {
|
|
|
189
202
|
export function loadStoredVectors(hippoRoot, ids) {
|
|
190
203
|
return ids.length === 0 ? new Map() : withVectorDb(hippoRoot, (db) => loadVectors(db, ids));
|
|
191
204
|
}
|
|
192
|
-
/** Whether the store holds any vector at all. */
|
|
193
|
-
export function hasEmbeddings(hippoRoot) {
|
|
194
|
-
return withVectorDb(hippoRoot, hasStoredVectors);
|
|
195
|
-
}
|
|
196
205
|
/** Replace every stored vector with `index`; `model` defaults to the stored index identity. */
|
|
197
206
|
export function saveEmbeddingIndex(hippoRoot, index, model) {
|
|
198
207
|
withVectorDb(hippoRoot, (db) => replaceAllVectors(db, index, model ?? getMeta(db, EMBEDDING_MODEL_META_KEY, '')));
|
package/dist/quality-repair.js
CHANGED
|
@@ -2,6 +2,7 @@ import fs from 'node:fs';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { appendAuditEvent } from './audit.js';
|
|
4
4
|
import { withBackup } from './db/backup.js';
|
|
5
|
+
import { assertSqliteAllowed } from './db/open.js';
|
|
5
6
|
import { DatabaseSync } from './db/sqlite.js';
|
|
6
7
|
import { getMeta, setMeta } from './db/meta.js';
|
|
7
8
|
import { tableColumns } from './db/tables.js';
|
|
@@ -187,6 +188,8 @@ function repairOn(db, root, opts) {
|
|
|
187
188
|
return { ...result, warnings };
|
|
188
189
|
}
|
|
189
190
|
function openForRepair(root, readOnly, fn) {
|
|
191
|
+
// Opens without openHippoDb to skip migrations, so it takes the same refusal itself.
|
|
192
|
+
assertSqliteAllowed(root);
|
|
190
193
|
const file = path.join(root, 'hippo.db');
|
|
191
194
|
if (!fs.existsSync(file))
|
|
192
195
|
throw new Error(`No existing Hippo database at ${file}`);
|
package/dist/search/hybrid.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { type BM25Corpus } from './bm25.js';
|
|
|
4
4
|
import { type HybridVectorCandidates } from './vector.js';
|
|
5
5
|
import { type GraphStreamOptions } from './fusion.js';
|
|
6
6
|
import { type ResultCost, type SearchResult } from './types.js';
|
|
7
|
+
import type { HippoStore } from '../store-port.js';
|
|
7
8
|
export interface HybridSearchOptions {
|
|
8
9
|
budget?: number;
|
|
9
10
|
now?: Date;
|
|
@@ -40,6 +41,8 @@ export interface HybridSearchOptions {
|
|
|
40
41
|
graphStream?: GraphStreamOptions;
|
|
41
42
|
/** Add the rows nearest the query vector, not only rescore `entries`; without it a row no query word matches cannot surface. */
|
|
42
43
|
vectorCandidates?: HybridVectorCandidates;
|
|
44
|
+
/** Where vectors are read; hippo.db under `hippoRoot` when unset. */
|
|
45
|
+
store?: HippoStore;
|
|
43
46
|
}
|
|
44
47
|
/** BM25 blended with cosine similarity when stored vectors and a provider are available, BM25 * strength * recency otherwise. */
|
|
45
48
|
export declare function hybridSearch(query: string, entries: MemoryEntry[], options?: HybridSearchOptions): Promise<SearchResult[]>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type MemoryEntry } from '../memory.js';
|
|
2
2
|
import { type PhysicsConfig } from '../physics-config.js';
|
|
3
|
+
import { type HippoStore } from '../store-port.js';
|
|
3
4
|
import { type HybridVectorCandidates } from './vector.js';
|
|
4
5
|
import { type ResultCost, type SearchResult } from './types.js';
|
|
5
6
|
export interface PhysicsSearchOptions {
|
|
@@ -23,6 +24,8 @@ export interface PhysicsSearchOptions {
|
|
|
23
24
|
summaryFreshness?: boolean;
|
|
24
25
|
/** Same as hybridSearch's: rows nearest the query join the pool before physics scoring. */
|
|
25
26
|
vectorCandidates?: HybridVectorCandidates;
|
|
27
|
+
/** Where vectors and particles are read; hippo.db under `hippoRoot` when unset. */
|
|
28
|
+
store?: HippoStore;
|
|
26
29
|
}
|
|
27
30
|
/** Scores memories by gravity, momentum and cluster amplification; rows without physics state rank through hybridSearch. */
|
|
28
31
|
export declare function physicsSearch(query: string, entries: MemoryEntry[], options?: PhysicsSearchOptions): Promise<SearchResult[]>;
|
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import { calculateStrength } from '../memory.js';
|
|
2
2
|
import { estimateTokens } from '../token-ledger.js';
|
|
3
3
|
import { evalNow, isRecallBoostAblated } from '../ablation.js';
|
|
4
|
-
import {
|
|
4
|
+
import { indexedModel, indexNeedsRebuild } from '../embeddings.js';
|
|
5
5
|
import { resolveEmbeddingProvider } from '../embedding-provider.js';
|
|
6
6
|
import { physicsScore as computePhysicsScores, computeMass } from '../physics.js';
|
|
7
7
|
import { DEFAULT_PHYSICS_CONFIG } from '../physics-config.js';
|
|
8
|
-
import {
|
|
9
|
-
import { openHippoDb, closeHippoDb, rethrowIfSqliteBlocked } from '../db.js';
|
|
8
|
+
import { rethrowIfSqliteBlocked } from '../db.js';
|
|
10
9
|
import { compareScoredResults } from '../compare.js';
|
|
11
10
|
import { log } from '../log.js';
|
|
11
|
+
import { sqliteStore } from '../store-port.js';
|
|
12
12
|
import { churnStaleFactor, summaryMultipliers, summaryScoring } from './boosts.js';
|
|
13
13
|
import { addDagFields, ageInDays } from './breakdown.js';
|
|
14
14
|
import { fitBudget } from './finalize.js';
|
|
15
15
|
import { hybridSearch } from './hybrid.js';
|
|
16
16
|
import { currentEntries } from './as-of.js';
|
|
17
|
-
import { vectorCandidatesOutside } from './vector.js';
|
|
17
|
+
import { requireVectorReads, vectorCandidatesOutside } from './vector.js';
|
|
18
18
|
import { DEFAULT_RECALL_BUDGET } from './types.js';
|
|
19
19
|
/** Scores memories by gravity, momentum and cluster amplification; rows without physics state rank through hybridSearch. */
|
|
20
20
|
export async function physicsSearch(query, entries, options = {}) {
|
|
@@ -27,11 +27,14 @@ export async function physicsSearch(query, entries, options = {}) {
|
|
|
27
27
|
if (options.asOf)
|
|
28
28
|
return hybridSearch(query, entries, options);
|
|
29
29
|
const root = options.hippoRoot;
|
|
30
|
-
const
|
|
30
|
+
const store = options.store ?? sqliteStore(root);
|
|
31
|
+
const queryVector = await physicsQueryVector(query, root, store, options.queryEmbedding);
|
|
31
32
|
if (!queryVector)
|
|
32
33
|
return hybridSearch(query, entries, options);
|
|
33
|
-
|
|
34
|
-
const
|
|
34
|
+
// Checked here too, since a caller's own query vector skips the check inside physicsQueryVector.
|
|
35
|
+
const reads = requireVectorReads(store);
|
|
36
|
+
const pool = currentEntries(await withVectorCandidates(reads, entries, queryVector, options.vectorCandidates), options);
|
|
37
|
+
const physicsMap = await loadCandidateParticles(reads, pool);
|
|
35
38
|
if (!physicsMap)
|
|
36
39
|
return hybridSearch(query, pool, options);
|
|
37
40
|
const pools = splitByParticle(pool, physicsMap, queryVector, now);
|
|
@@ -45,13 +48,15 @@ export async function physicsSearch(query, entries, options = {}) {
|
|
|
45
48
|
return fitBudget(merged, options.budget ?? DEFAULT_RECALL_BUDGET, options.minResults ?? 1, options.cost);
|
|
46
49
|
}
|
|
47
50
|
/** The caller's vector, else the provider's; null sends the caller to hybridSearch. */
|
|
48
|
-
async function physicsQueryVector(query, root, given) {
|
|
51
|
+
async function physicsQueryVector(query, root, store, given) {
|
|
49
52
|
if (given && given.length > 0)
|
|
50
53
|
return given;
|
|
51
54
|
// Physics scores against particle positions, not the stored vector index, so a pruned index must not block it.
|
|
52
55
|
try {
|
|
53
56
|
const provider = resolveEmbeddingProvider(root);
|
|
54
|
-
if (!provider.isAvailable()
|
|
57
|
+
if (!provider.isAvailable())
|
|
58
|
+
return null;
|
|
59
|
+
if (indexNeedsRebuild(indexedModel(await requireVectorReads(store).embeddingIndexState()), provider.id))
|
|
55
60
|
return null;
|
|
56
61
|
const [vec] = await provider.embed([query], 'query');
|
|
57
62
|
return vec && vec.length > 0 ? vec : null;
|
|
@@ -62,11 +67,11 @@ async function physicsQueryVector(query, root, given) {
|
|
|
62
67
|
return null;
|
|
63
68
|
}
|
|
64
69
|
}
|
|
65
|
-
function withVectorCandidates(
|
|
70
|
+
async function withVectorCandidates(reads, entries, queryVector, spec) {
|
|
66
71
|
if (!spec)
|
|
67
72
|
return entries;
|
|
68
73
|
try {
|
|
69
|
-
return [...entries, ...vectorCandidatesOutside(
|
|
74
|
+
return [...entries, ...await vectorCandidatesOutside(reads, entries, queryVector, spec)];
|
|
70
75
|
}
|
|
71
76
|
catch (err) {
|
|
72
77
|
rethrowIfSqliteBlocked(err);
|
|
@@ -75,15 +80,9 @@ function withVectorCandidates(root, entries, queryVector, spec) {
|
|
|
75
80
|
}
|
|
76
81
|
}
|
|
77
82
|
/** Particles for the candidate rows only, so one tenant's search never reads every tenant's physics state. */
|
|
78
|
-
function loadCandidateParticles(
|
|
83
|
+
async function loadCandidateParticles(reads, pool) {
|
|
79
84
|
try {
|
|
80
|
-
|
|
81
|
-
try {
|
|
82
|
-
return loadPhysicsState(db, pool.map((e) => e.id));
|
|
83
|
-
}
|
|
84
|
-
finally {
|
|
85
|
-
closeHippoDb(db);
|
|
86
|
-
}
|
|
85
|
+
return await reads.physicsParticles(pool.map((e) => e.id));
|
|
87
86
|
}
|
|
88
87
|
catch (err) {
|
|
89
88
|
rethrowIfSqliteBlocked(err);
|
package/dist/search/vector.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { MemoryEntry } from '../memory.js';
|
|
2
|
-
import {
|
|
2
|
+
import type { VectorCandidateSpec } from '../store/search-rows.js';
|
|
3
|
+
import { type HippoStore, type VectorReads } from '../store-port.js';
|
|
3
4
|
import { type CurrentnessOptions } from './as-of.js';
|
|
4
5
|
/** hybridSearch's vector arm: which rows it may add, plus the caller's JS admission rules (exact private regex, entry filters). */
|
|
5
6
|
export type HybridVectorCandidates = VectorCandidateSpec & {
|
|
@@ -16,9 +17,13 @@ export interface VectorArm {
|
|
|
16
17
|
export interface VectorArmOptions extends CurrentnessOptions {
|
|
17
18
|
hippoRoot?: string;
|
|
18
19
|
vectorCandidates?: HybridVectorCandidates;
|
|
20
|
+
/** Where vectors are read; hippo.db under `hippoRoot` when unset. */
|
|
21
|
+
store?: HippoStore;
|
|
19
22
|
}
|
|
23
|
+
/** The store's vector reads; a store without them answers 501, as any unported path does. */
|
|
24
|
+
export declare function requireVectorReads(store: HippoStore): VectorReads;
|
|
20
25
|
/** The nearest admitted rows not already in `entries`. */
|
|
21
|
-
export declare function vectorCandidatesOutside(
|
|
26
|
+
export declare function vectorCandidatesOutside(reads: VectorReads, entries: readonly MemoryEntry[], queryVector: readonly number[], spec: HybridVectorCandidates): Promise<MemoryEntry[]>;
|
|
22
27
|
/** Embeds the query and loads stored vectors; any failure leaves BM25 to rank alone. */
|
|
23
28
|
export declare function resolveVectorArm(query: string, entries: MemoryEntry[], options: VectorArmOptions): Promise<VectorArm>;
|
|
24
29
|
export interface DenseScores {
|
package/dist/search/vector.js
CHANGED
|
@@ -1,18 +1,32 @@
|
|
|
1
|
-
import { cosineSimilarity,
|
|
2
|
-
import { loadVectorCandidateEntries } from '../store/search-rows.js';
|
|
1
|
+
import { cosineSimilarity, indexedModel, indexNeedsRebuild } from '../embeddings.js';
|
|
3
2
|
import { resolveEmbeddingProvider } from '../embedding-provider.js';
|
|
4
3
|
import { rethrowIfSqliteBlocked } from '../db.js';
|
|
4
|
+
import { StoreNotPortedError } from '../db/sqlite-blocked.js';
|
|
5
5
|
import { log } from '../log.js';
|
|
6
|
+
import { sqliteStore } from '../store-port.js';
|
|
6
7
|
import { redactSecretsStrict } from '../secret-detect.js';
|
|
7
8
|
import { currentEntries } from './as-of.js';
|
|
8
9
|
// Search runs on every hook prompt, so one line per reason per process says why vectors went unused without flooding stderr.
|
|
9
10
|
function warnBm25Fallback(reason, detail) {
|
|
10
11
|
log.once(`search.bm25-fallback.${reason}`, 'warn', `hybrid search fell back to BM25 only: ${redactSecretsStrict(detail)}`);
|
|
11
12
|
}
|
|
13
|
+
function reindexHint(storeKind) {
|
|
14
|
+
const why = 'the embedding index was built by another model or is being rebuilt';
|
|
15
|
+
// `hippo embed` writes only hippo.db, so another store gets its vectors by a rebuild from it.
|
|
16
|
+
return storeKind === 'sqlite'
|
|
17
|
+
? `${why}; run 'hippo embed'`
|
|
18
|
+
: `${why}; run 'hippo embed' on the SQLite store, then rebuild the '${storeKind}' database from it`;
|
|
19
|
+
}
|
|
20
|
+
/** The store's vector reads; a store without them answers 501, as any unported path does. */
|
|
21
|
+
export function requireVectorReads(store) {
|
|
22
|
+
if (!store.vectors)
|
|
23
|
+
throw new StoreNotPortedError(store.kind, 'vectors');
|
|
24
|
+
return store.vectors;
|
|
25
|
+
}
|
|
12
26
|
/** The nearest admitted rows not already in `entries`. */
|
|
13
|
-
export function vectorCandidatesOutside(
|
|
27
|
+
export async function vectorCandidatesOutside(reads, entries, queryVector, spec) {
|
|
14
28
|
const inPool = new Set(entries.map((e) => e.id));
|
|
15
|
-
return
|
|
29
|
+
return (await reads.nearestEntries(queryVector, spec)).filter((e) => !inPool.has(e.id) && (spec.admit?.(e) ?? true));
|
|
16
30
|
}
|
|
17
31
|
/** Embeds the query and loads stored vectors; any failure leaves BM25 to rank alone. */
|
|
18
32
|
export async function resolveVectorArm(query, entries, options) {
|
|
@@ -32,14 +46,17 @@ async function fillVectorArm(arm, query, root, options) {
|
|
|
32
46
|
const provider = resolveEmbeddingProvider(root);
|
|
33
47
|
if (!provider.isAvailable())
|
|
34
48
|
return;
|
|
35
|
-
|
|
36
|
-
|
|
49
|
+
const store = options.store ?? sqliteStore(root);
|
|
50
|
+
const reads = requireVectorReads(store);
|
|
51
|
+
const index = await reads.embeddingIndexState();
|
|
52
|
+
if (indexNeedsRebuild(indexedModel(index), provider.id)) {
|
|
53
|
+
warnBm25Fallback('reindex', reindexHint(store.kind));
|
|
37
54
|
return;
|
|
38
55
|
}
|
|
39
56
|
const spec = options.vectorCandidates;
|
|
40
|
-
const vectors =
|
|
57
|
+
const vectors = await reads.storedVectors(arm.entries.map((e) => e.id));
|
|
41
58
|
// Only spend a (possibly paid, off-box) query embedding when there is a stored vector this search can use.
|
|
42
|
-
if (vectors.size === 0 && !(spec !== undefined &&
|
|
59
|
+
if (vectors.size === 0 && !(spec !== undefined && index.hasVectors))
|
|
43
60
|
return;
|
|
44
61
|
const [vec] = await provider.embed([query], 'query');
|
|
45
62
|
arm.queryVector = vec ?? [];
|
|
@@ -47,11 +64,11 @@ async function fillVectorArm(arm, query, root, options) {
|
|
|
47
64
|
warnBm25Fallback('empty-query-vector', 'the embedding provider returned no vector for the query');
|
|
48
65
|
return;
|
|
49
66
|
}
|
|
50
|
-
const added = spec ? vectorCandidatesOutside(
|
|
67
|
+
const added = spec ? await vectorCandidatesOutside(reads, arm.entries, arm.queryVector, spec) : [];
|
|
51
68
|
if (added.length > 0) {
|
|
52
69
|
arm.addedRows = true;
|
|
53
70
|
arm.entries = currentEntries([...arm.entries, ...added], options);
|
|
54
|
-
for (const [id, v] of
|
|
71
|
+
for (const [id, v] of await reads.storedVectors(added.map((e) => e.id)))
|
|
55
72
|
vectors.set(id, v);
|
|
56
73
|
}
|
|
57
74
|
arm.embeddingIndex = Object.fromEntries(vectors);
|
|
@@ -16,21 +16,18 @@ export function __resetSessionRecallHistoryHttp() {
|
|
|
16
16
|
resetSessionRings('http');
|
|
17
17
|
}
|
|
18
18
|
// HTTP threads the ring through opts.recallHistory, so the hint retrieve() returns is the one the caller sees.
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
return undefined;
|
|
22
|
-
if (sessionId)
|
|
23
|
-
return peekSessionRing('http', ctx.tenantId, sessionId);
|
|
24
|
-
await storeFor(ctx).appendAuditEvents(anchorSkippedRows({ tenantId: ctx.tenantId, actor: ctx.actor.subject }, q));
|
|
25
|
-
return undefined;
|
|
19
|
+
function recallHistoryFor(ctx, sessionId) {
|
|
20
|
+
return sessionId && biasHintEnabled('anchoring') ? peekSessionRing('http', ctx.tenantId, sessionId) : undefined;
|
|
26
21
|
}
|
|
27
22
|
// GET /v1/memories?q=...&limit=...&mode=...&scope=...&include_continuity=1
|
|
28
23
|
export async function handleRecallMemories({ req, res, opts, query }) {
|
|
29
24
|
const { opts: recallOpts, limit, mode, explain } = parseRecallRequest(httpParams(query));
|
|
30
25
|
const { query: q, includeContinuity, sessionId } = recallOpts;
|
|
31
26
|
const ctx = await buildContextWithAuth(req, opts);
|
|
32
|
-
const recallHistory =
|
|
33
|
-
|
|
27
|
+
const recallHistory = recallHistoryFor(ctx, sessionId);
|
|
28
|
+
// Written first in the recall's own write, so a recall that fails leaves no row.
|
|
29
|
+
const leadingAudit = sessionId ? [] : anchorSkippedRows({ tenantId: ctx.tenantId, actor: ctx.actor.subject }, q);
|
|
30
|
+
const result = await retrieve(ctx, { ...recallOpts, limit, mode, explain, recallHistory, leadingAudit });
|
|
34
31
|
// The ring is created only after recall succeeds, so a 400 cannot LRU-evict a live session.
|
|
35
32
|
const ring = sessionRing('http', ctx.tenantId, sessionId);
|
|
36
33
|
if (ring)
|
package/dist/server.d.ts
CHANGED
|
@@ -11,7 +11,7 @@ export { HttpError } from './http-util.js';
|
|
|
11
11
|
export { promptHookContext, type CallerProject } from './prompt-hook.js';
|
|
12
12
|
export type { JsonValue } from './json.js';
|
|
13
13
|
export { captureSessionTexts, type SessionCaptureRequest, type SessionCaptureResult } from './capture/session-texts.js';
|
|
14
|
-
export { sqliteStore, type HippoStore, type RecallSearchArgs, type RecallWrites } from './store-port.js';
|
|
14
|
+
export { sqliteStore, type HippoStore, type RecallSearchArgs, type RecallWrites, type VectorReads } from './store-port.js';
|
|
15
15
|
export type { ApiKeyRecord } from './auth.js';
|
|
16
16
|
export type { AppendAuditOpts } from './audit.js';
|
|
17
17
|
export type { ContinuityBlock } from './api/recall-types.js';
|
|
@@ -21,10 +21,15 @@ export type { ClassResolution, PlanningFallacyEvidence } from './predictions/pla
|
|
|
21
21
|
export type { PredictionBaserate } from './predictions/store.js';
|
|
22
22
|
export type { RecallTraceInput } from './recall-trace.js';
|
|
23
23
|
export type { StrengthenOptions } from './store/entry-writes.js';
|
|
24
|
-
export type { OriginFilter } from './store/search-rows.js';
|
|
24
|
+
export type { OriginFilter, RecallScopeFilter, VectorCandidateSpec } from './store/search-rows.js';
|
|
25
25
|
export type { ContinuityKey } from './store/sessions.js';
|
|
26
26
|
export type { TokenUse } from './token-ledger.js';
|
|
27
|
+
export type { EmbeddingIndexState } from './embeddings.js';
|
|
28
|
+
export type { PhysicsParticle } from './physics.js';
|
|
27
29
|
export { StoreBusyError } from './db.js';
|
|
30
|
+
export { decodeVector, EMBEDDING_MODEL_META_KEY, rankVectorRows, type VectorMatch, type VectorRow } from './vector-store.js';
|
|
31
|
+
export { bufferToFloat32 } from './physics-state.js';
|
|
32
|
+
export { OTHER_STORE_MARKER, OtherStoreFolderError, withSqliteAllowed } from './db.js';
|
|
28
33
|
export { authCreate, type AuthCreateOpts, type AuthCreateResult } from './api.js';
|
|
29
34
|
export { isSharedStore } from './config.js';
|
|
30
35
|
export { ownerOrSubject } from './api.js';
|
package/dist/server.js
CHANGED
|
@@ -48,6 +48,11 @@ export { captureSessionTexts } from './capture/session-texts.js';
|
|
|
48
48
|
// An add-on serves from another database by passing serve() its own HippoStore.
|
|
49
49
|
export { sqliteStore } from './store-port.js';
|
|
50
50
|
export { StoreBusyError } from './db.js';
|
|
51
|
+
// An add-on store decodes and ranks with hippo.db's own code, so both return the same ids in the same order.
|
|
52
|
+
export { decodeVector, EMBEDDING_MODEL_META_KEY, rankVectorRows } from './vector-store.js';
|
|
53
|
+
export { bufferToFloat32 } from './physics-state.js';
|
|
54
|
+
// store copy --db writes the marker and reads the old hippo.db under the waiver.
|
|
55
|
+
export { OTHER_STORE_MARKER, OtherStoreFolderError, withSqliteAllowed } from './db.js';
|
|
51
56
|
// An add-on's install step mints the first admin key into a store folder it names, which `hippo auth create` cannot reach.
|
|
52
57
|
export { authCreate } from './api.js';
|
|
53
58
|
// An add-on that serves a team store checks the flag before it starts.
|
package/dist/store/open.js
CHANGED
|
@@ -25,9 +25,10 @@ export function initStore(hippoRoot) {
|
|
|
25
25
|
}
|
|
26
26
|
/** One open connection with init done on it, for callers who used to pay for `initStore` + a second `openHippoDb`. */
|
|
27
27
|
export function openStore(hippoRoot, opts) {
|
|
28
|
-
|
|
28
|
+
// Open first: a folder marked for another store must refuse before any mirror folder appears.
|
|
29
29
|
const db = openHippoDb(hippoRoot, opts);
|
|
30
30
|
try {
|
|
31
|
+
ensureMirrorDirectories(hippoRoot);
|
|
31
32
|
const bootstrapped = bootstrapLegacyStore(db, hippoRoot);
|
|
32
33
|
if (bootstrapped) {
|
|
33
34
|
syncMirrorFiles(hippoRoot, db);
|
package/dist/store-port.d.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { type ApiKeyRecord } from './auth.js';
|
|
2
2
|
import { type AppendAuditOpts } from './audit.js';
|
|
3
3
|
import type { ContinuityBlock } from './api/recall-types.js';
|
|
4
|
+
import { type EmbeddingIndexState } from './embeddings.js';
|
|
4
5
|
import { type ActiveGoals, type GetActiveGoalsOpts, type GoalRecallLogRow } from './goals.js';
|
|
5
6
|
import type { MemoryEntry } from './memory.js';
|
|
7
|
+
import type { PhysicsParticle } from './physics.js';
|
|
6
8
|
import { type PlanningFallacyEvidence } from './predictions/planning-fallacy.js';
|
|
7
9
|
import { type RecallTraceInput } from './recall-trace.js';
|
|
8
10
|
import { type StrengthenOptions } from './store/entry-writes.js';
|
|
9
|
-
import { type OriginFilter } from './store/search-rows.js';
|
|
11
|
+
import { type OriginFilter, type VectorCandidateSpec } from './store/search-rows.js';
|
|
10
12
|
import { type ContinuityKey } from './store/sessions.js';
|
|
11
13
|
import { type TokenUse } from './token-ledger.js';
|
|
12
14
|
/** The arguments of `loadRecallSearchEntries` after the query, by name. */
|
|
@@ -35,6 +37,17 @@ export interface RecallWrites {
|
|
|
35
37
|
readonly opts: StrengthenOptions;
|
|
36
38
|
};
|
|
37
39
|
}
|
|
40
|
+
/** The reads behind recall's vector arm, one group a store sets in full as `HippoStore.vectors` or leaves unset. */
|
|
41
|
+
export interface VectorReads {
|
|
42
|
+
/** The stored model and whether any vector exists, in one snapshot. */
|
|
43
|
+
embeddingIndexState(): Promise<EmbeddingIndexState>;
|
|
44
|
+
storedVectors(ids: readonly string[]): Promise<Map<string, number[]>>;
|
|
45
|
+
/** The `spec.limit ?? 50` rows nearest `queryVector` by `rankVectorRows` among those where the tenant matches (when set), kind is not 'archived', a superseded row
|
|
46
|
+
* shows only with `includeSuperseded`, scope follows `RecallScopeFilter`, and origin is '' or listed (when `spec.origin` is set). */
|
|
47
|
+
nearestEntries(queryVector: readonly number[], spec: VectorCandidateSpec): Promise<MemoryEntry[]>;
|
|
48
|
+
/** The particles of `ids` only; an empty list reads nothing. */
|
|
49
|
+
physicsParticles(ids: readonly string[]): Promise<Map<string, PhysicsParticle>>;
|
|
50
|
+
}
|
|
38
51
|
/** What `serve()` reads and writes through. Each method is atomic and no transaction spans an await, since SQLite's lock wait blocks the event loop; a lock timeout throws `StoreBusyError`. */
|
|
39
52
|
export interface HippoStore {
|
|
40
53
|
/** 'sqlite' is hippo.db under the served root. Under any other kind, an unported route answers 501 and a hippo.db open inside a request throws. */
|
|
@@ -65,6 +78,8 @@ export interface HippoStore {
|
|
|
65
78
|
bumpRecallStats(recalled: number): Promise<void>;
|
|
66
79
|
/** One token-ledger row. Throws, so the caller decides whether a ledger failure matters. */
|
|
67
80
|
recordTokens(use: TokenUse): Promise<void>;
|
|
81
|
+
/** Unset on a store built before them, where hybrid and physics recall under an embedding provider answer 501. */
|
|
82
|
+
readonly vectors?: VectorReads;
|
|
68
83
|
/** Releases the store's connections; `serve()` closes only a store it made itself. */
|
|
69
84
|
close(): Promise<void>;
|
|
70
85
|
}
|
|
@@ -74,7 +89,9 @@ export declare function storeFor(ctx: {
|
|
|
74
89
|
readonly store?: HippoStore;
|
|
75
90
|
}): HippoStore;
|
|
76
91
|
/** The built-in store: today's synchronous hippo.db functions behind the port, each call on its own short-lived handles, so close has nothing to release. */
|
|
77
|
-
export declare function sqliteStore(hippoRoot: string): HippoStore
|
|
92
|
+
export declare function sqliteStore(hippoRoot: string): HippoStore & {
|
|
93
|
+
readonly vectors: VectorReads;
|
|
94
|
+
};
|
|
78
95
|
/** sqliteStore's continuity read, for the synchronous recall that cannot await the port. */
|
|
79
96
|
export declare function continuityAt(hippoRoot: string, tenantId: string, eventLimit: number, key: ContinuityKey | null): ContinuityBlock;
|
|
80
97
|
/** sqliteStore's finishRecall, for the synchronous recall that cannot await the port. */
|
package/dist/store-port.js
CHANGED
|
@@ -2,14 +2,16 @@
|
|
|
2
2
|
import { readApiKeyRecord } from './auth.js';
|
|
3
3
|
import { appendAuditEvent } from './audit.js';
|
|
4
4
|
import { closeHippoDb, openHippoDb, withWriteScope } from './db.js';
|
|
5
|
+
import { embeddingIndexStateAt, loadStoredVectors } from './embeddings.js';
|
|
5
6
|
import { activeGoalsWithPolicies, localGoalRecallRows, writeGoalRecallLog, } from './goals.js';
|
|
7
|
+
import { loadPhysicsState } from './physics-state.js';
|
|
6
8
|
import { planningFallacyEvidenceAt } from './predictions/planning-fallacy.js';
|
|
7
9
|
import { writeRecallTrace } from './recall-trace.js';
|
|
8
10
|
import { loadEntriesByIds, loadFreshRawMemories } from './store/entry-reads.js';
|
|
9
11
|
import { strengthenRetrievedInOwnTx } from './store/entry-writes.js';
|
|
10
12
|
import { loadLatestHandoff } from './store/handoffs.js';
|
|
11
13
|
import { updateStats } from './store/index-and-stats.js';
|
|
12
|
-
import { loadRecallSearchEntries } from './store/search-rows.js';
|
|
14
|
+
import { loadRecallSearchEntries, loadVectorCandidateEntries } from './store/search-rows.js';
|
|
13
15
|
import { listSessionEvents, loadActiveTaskSnapshot } from './store/sessions.js';
|
|
14
16
|
import { recordTokenUse } from './token-ledger.js';
|
|
15
17
|
function onHandle(hippoRoot, fn) {
|
|
@@ -67,6 +69,21 @@ export function sqliteStore(hippoRoot) {
|
|
|
67
69
|
async recordTokens(use) {
|
|
68
70
|
onHandle(hippoRoot, (db) => recordTokenUse(db, use));
|
|
69
71
|
},
|
|
72
|
+
vectors: {
|
|
73
|
+
async embeddingIndexState() {
|
|
74
|
+
return embeddingIndexStateAt(hippoRoot);
|
|
75
|
+
},
|
|
76
|
+
async storedVectors(ids) {
|
|
77
|
+
return loadStoredVectors(hippoRoot, ids);
|
|
78
|
+
},
|
|
79
|
+
async nearestEntries(queryVector, spec) {
|
|
80
|
+
return loadVectorCandidateEntries(hippoRoot, queryVector, spec);
|
|
81
|
+
},
|
|
82
|
+
async physicsParticles(ids) {
|
|
83
|
+
// loadPhysicsState reads every row for an empty list.
|
|
84
|
+
return ids.length === 0 ? new Map() : onHandle(hippoRoot, (db) => loadPhysicsState(db, [...ids]));
|
|
85
|
+
},
|
|
86
|
+
},
|
|
70
87
|
async close() { },
|
|
71
88
|
};
|
|
72
89
|
}
|
package/dist/vector-store.d.ts
CHANGED
|
@@ -18,6 +18,14 @@ export interface VectorMatch {
|
|
|
18
18
|
id: string;
|
|
19
19
|
score: number;
|
|
20
20
|
}
|
|
21
|
+
/** A stored vector, decoded. */
|
|
22
|
+
export interface VectorRow {
|
|
23
|
+
readonly id: string;
|
|
24
|
+
readonly vector: Float32Array;
|
|
25
|
+
}
|
|
26
|
+
/** The `k` rows closest to `query` by cosine, best first; ties go to the smaller id, so row order never changes the top k.
|
|
27
|
+
* The query rounds to Float32 as stored vectors are, and scores add up in float64, so every store that feeds it the same bytes ranks alike. */
|
|
28
|
+
export declare function rankVectorRows(query: readonly number[], rows: Iterable<VectorRow>, k: number): VectorMatch[];
|
|
21
29
|
/** The `k` stored vectors closest to `query` among `memories m` rows passing `where` (SQL starting with ` AND`).
|
|
22
30
|
* Filters run before the cut, so rows a caller may not see can never push admitted rows out of the top k. */
|
|
23
31
|
export declare function topVectorMatches(db: DatabaseSyncLike, query: readonly number[], k: number, where: string, params: readonly (string | number)[]): VectorMatch[];
|
package/dist/vector-store.js
CHANGED
|
@@ -24,8 +24,8 @@ export function encodeVector(vector) {
|
|
|
24
24
|
return new Uint8Array(Float32Array.from(vector).buffer);
|
|
25
25
|
}
|
|
26
26
|
export function decodeVector(blob) {
|
|
27
|
-
// A view needs 4-byte alignment;
|
|
28
|
-
const bytes = blob.byteOffset % 4 === 0 ? blob : blob
|
|
27
|
+
// A view needs 4-byte alignment; Buffer#slice is a view too, so copy with the constructor.
|
|
28
|
+
const bytes = blob.byteOffset % 4 === 0 ? blob : new Uint8Array(blob);
|
|
29
29
|
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
30
30
|
}
|
|
31
31
|
/** Insert or replace vectors, skipping empty or non-finite ones; the caller owns any surrounding transaction. */
|
|
@@ -92,28 +92,22 @@ export function storedVectorIds(db) {
|
|
|
92
92
|
export function hasStoredVectors(db) {
|
|
93
93
|
return db.prepare('SELECT 1 FROM memory_vectors LIMIT 1').get() !== undefined;
|
|
94
94
|
}
|
|
95
|
-
/** The `k`
|
|
96
|
-
*
|
|
97
|
-
export function
|
|
95
|
+
/** The `k` rows closest to `query` by cosine, best first; ties go to the smaller id, so row order never changes the top k.
|
|
96
|
+
* The query rounds to Float32 as stored vectors are, and scores add up in float64, so every store that feeds it the same bytes ranks alike. */
|
|
97
|
+
export function rankVectorRows(query, rows, k) {
|
|
98
|
+
const top = [];
|
|
98
99
|
if (k <= 0 || query.length === 0)
|
|
99
|
-
return
|
|
100
|
+
return top;
|
|
100
101
|
const q = Float32Array.from(query);
|
|
101
102
|
let qNorm = 0;
|
|
102
103
|
for (let i = 0; i < q.length; i++)
|
|
103
104
|
qNorm += q[i] * q[i];
|
|
104
105
|
qNorm = Math.sqrt(qNorm);
|
|
105
106
|
if (qNorm < 1e-10)
|
|
106
|
-
return
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
FROM memory_vectors v JOIN memories m ON m.id = v.memory_id
|
|
111
|
-
WHERE v.dim = ?${where}
|
|
112
|
-
`).iterate(q.length, ...params);
|
|
113
|
-
const top = [];
|
|
114
|
-
// SAFETY: the SELECT above names exactly these two columns; node:sqlite returns BLOBs as Uint8Array.
|
|
115
|
-
for (const row of rows) {
|
|
116
|
-
const v = decodeVector(row.vector);
|
|
107
|
+
return top;
|
|
108
|
+
for (const { id, vector: v } of rows) {
|
|
109
|
+
if (v.length !== q.length)
|
|
110
|
+
continue;
|
|
117
111
|
let dot = 0;
|
|
118
112
|
let norm = 0;
|
|
119
113
|
for (let i = 0; i < v.length; i++) {
|
|
@@ -123,12 +117,30 @@ export function topVectorMatches(db, query, k, where, params) {
|
|
|
123
117
|
if (norm < 1e-20)
|
|
124
118
|
continue;
|
|
125
119
|
const score = dot / (qNorm * Math.sqrt(norm));
|
|
126
|
-
if (top.length === k && !beats(score,
|
|
120
|
+
if (top.length === k && !beats(score, id, top[k - 1]))
|
|
127
121
|
continue;
|
|
128
|
-
insertSorted(top, { id
|
|
122
|
+
insertSorted(top, { id, score }, k);
|
|
129
123
|
}
|
|
130
124
|
return top;
|
|
131
125
|
}
|
|
126
|
+
function* decodedRows(rows) {
|
|
127
|
+
for (const row of rows)
|
|
128
|
+
yield { id: row.id, vector: decodeVector(row.vector) };
|
|
129
|
+
}
|
|
130
|
+
/** The `k` stored vectors closest to `query` among `memories m` rows passing `where` (SQL starting with ` AND`).
|
|
131
|
+
* Filters run before the cut, so rows a caller may not see can never push admitted rows out of the top k. */
|
|
132
|
+
export function topVectorMatches(db, query, k, where, params) {
|
|
133
|
+
if (k <= 0 || query.length === 0)
|
|
134
|
+
return [];
|
|
135
|
+
// SHORTCUT: brute-force cosine over every vector of matching dim, fine to ~100k rows; an ANN index (sqlite-vec, HNSW) is the upgrade.
|
|
136
|
+
const rows = db.prepare(`
|
|
137
|
+
SELECT v.memory_id AS id, v.vector AS vector
|
|
138
|
+
FROM memory_vectors v JOIN memories m ON m.id = v.memory_id
|
|
139
|
+
WHERE v.dim = ?${where}
|
|
140
|
+
`).iterate(query.length, ...params);
|
|
141
|
+
// SAFETY: the SELECT above names exactly these two columns; node:sqlite returns BLOBs as Uint8Array.
|
|
142
|
+
return rankVectorRows(query, decodedRows(rows), k);
|
|
143
|
+
}
|
|
132
144
|
// Ties break on id so the cut is the same on every run.
|
|
133
145
|
function beats(score, id, other) {
|
|
134
146
|
return score > other.score || (score === other.score && id < other.id);
|
package/dist/version.d.ts
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* an ESM `import` can resolve cleanly, and a hardcoded constant survives
|
|
17
17
|
* any packager that drops .json files.
|
|
18
18
|
*/
|
|
19
|
-
export declare const PACKAGE_VERSION = "1.
|
|
19
|
+
export declare const PACKAGE_VERSION = "1.66.0";
|
|
20
20
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by hand in that release; scripts/check-expiring-keys-floor.mjs gates it. */
|
|
21
21
|
export declare const EXPIRING_KEYS_MIN_BINARY = "1.64.0";
|
|
22
22
|
/** The floor a store takes on its first session bind or owner snapshot: first release with v54; set by hand in that release. */
|
package/dist/version.js
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* an ESM `import` can resolve cleanly, and a hardcoded constant survives
|
|
17
17
|
* any packager that drops .json files.
|
|
18
18
|
*/
|
|
19
|
-
export const PACKAGE_VERSION = '1.
|
|
19
|
+
export const PACKAGE_VERSION = '1.66.0';
|
|
20
20
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by hand in that release; scripts/check-expiring-keys-floor.mjs gates it. */
|
|
21
21
|
export const EXPIRING_KEYS_MIN_BINARY = '1.64.0';
|
|
22
22
|
/** The floor a store takes on its first session bind or owner snapshot: first release with v54; set by hand in that release. */
|
package/openclaw.plugin.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"id": "hippo-memory",
|
|
3
3
|
"name": "Hippo Memory",
|
|
4
4
|
"description": "Make your agent's memory work like a brain. Hippo is long-term memory for coding agents.",
|
|
5
|
-
"version": "1.
|
|
5
|
+
"version": "1.66.0",
|
|
6
6
|
"configSchema": {
|
|
7
7
|
"type": "object",
|
|
8
8
|
"additionalProperties": false,
|
package/package.json
CHANGED