hippo-memory 1.64.0 → 1.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/agent-memories/sync.js +3 -2
- package/dist/api/recall-record.d.ts +46 -0
- package/dist/api/recall-record.js +98 -0
- package/dist/api/recall-request.d.ts +32 -0
- package/dist/api/recall-request.js +121 -0
- package/dist/api/recall-types.d.ts +7 -1
- package/dist/api/recall.d.ts +3 -2
- package/dist/api/recall.js +174 -181
- package/dist/api/tokens.d.ts +1 -1
- package/dist/api/tokens.js +13 -17
- package/dist/cli/maintenance.d.ts +2 -1
- package/dist/cli/maintenance.js +17 -11
- package/dist/cli/recall.d.ts +1 -1
- package/dist/cli/recall.js +18 -49
- package/dist/cli/shared.js +2 -2
- package/dist/cli.d.ts +16 -0
- package/dist/cli.js +18 -1
- package/dist/config.d.ts +2 -0
- package/dist/config.js +4 -0
- package/dist/db/connect.d.ts +5 -0
- package/dist/db/connect.js +52 -0
- package/dist/db/open.d.ts +9 -11
- package/dist/db/open.js +23 -116
- package/dist/db/request-stores.d.ts +30 -0
- package/dist/db/request-stores.js +100 -0
- package/dist/db/sqlite-blocked.d.ts +6 -0
- package/dist/db/sqlite-blocked.js +11 -0
- package/dist/db.d.ts +3 -1
- package/dist/db.js +3 -1
- package/dist/embedding-provider.js +4 -3
- package/dist/embeddings.js +2 -1
- package/dist/goals.d.ts +18 -11
- package/dist/goals.js +38 -36
- package/dist/http-util.d.ts +2 -0
- package/dist/http-util.js +5 -0
- package/dist/local-embedding.d.ts +2 -0
- package/dist/local-embedding.js +20 -4
- package/dist/mcp/memory-tools.js +4 -1
- package/dist/mcp/recall-tools.js +28 -133
- package/dist/mcp/request.d.ts +7 -0
- package/dist/mcp/request.js +63 -41
- package/dist/mcp/session-state.d.ts +1 -3
- package/dist/mcp/session-state.js +3 -5
- package/dist/predictions/planning-fallacy.d.ts +52 -0
- package/dist/predictions/planning-fallacy.js +70 -55
- package/dist/recall-trace.d.ts +1 -1
- package/dist/recall-trace.js +5 -4
- package/dist/search/physics-search.js +4 -1
- package/dist/search/vector.js +2 -0
- package/dist/server/routes/recall.d.ts +1 -1
- package/dist/server/routes/recall.js +30 -218
- package/dist/server.d.ts +12 -1
- package/dist/server.js +33 -22
- package/dist/store/entry-writes.d.ts +2 -0
- package/dist/store/entry-writes.js +24 -16
- package/dist/store-port.d.ts +70 -1
- package/dist/store-port.js +87 -9
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
package/dist/db/open.js
CHANGED
|
@@ -1,84 +1,46 @@
|
|
|
1
|
-
import * as fs from 'fs';
|
|
2
|
-
import * as path from 'path';
|
|
3
1
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
4
|
-
import { cleanupArchivedMirrors } from '../raw-archive-mirror-cleanup.js';
|
|
5
2
|
import { log } from '../log.js';
|
|
6
3
|
import { DatabaseSync } from './sqlite.js';
|
|
7
|
-
import { execWithBusyRetry } from './busy.js';
|
|
8
4
|
import { tableExists } from './tables.js';
|
|
9
|
-
import { assertBinaryCompatible
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
const sharedHandles = new Map();
|
|
15
|
-
const sharedSet = new WeakSet();
|
|
16
|
-
let shareDepth = 0;
|
|
17
|
-
let shareBusyWaitMs;
|
|
5
|
+
import { assertBinaryCompatible } from './migrate.js';
|
|
6
|
+
import { connectHippoDb, getHippoDbPath } from './connect.js';
|
|
7
|
+
import { currentRequestStores, isScopedHandle, runWithRequestStores } from './request-stores.js';
|
|
8
|
+
import { SqliteBlockedError } from './sqlite-blocked.js';
|
|
9
|
+
export { getHippoDbPath };
|
|
18
10
|
/** Lock wait for hook commands: under the 5 s prompt-hook budget even after a few skipped writes, and far above a normal write's hold. */
|
|
19
11
|
export const HOOK_DB_WAIT_MS = 1000;
|
|
12
|
+
/** Lock wait inside an HTTP request: SQLite waits synchronously, so a long wait would stall every other request on the event loop. */
|
|
13
|
+
export const SERVER_DB_WAIT_MS = 250;
|
|
20
14
|
/** A busy store made a command skip work: warn once per process (the holder is usually `hippo sleep`). */
|
|
21
15
|
export function noteStoreBusy(skipped) {
|
|
22
16
|
log.once('store-busy', 'warn', `store busy (another hippo process holds the write lock); ${skipped}`);
|
|
23
|
-
|
|
24
|
-
for (const db of sharedHandles.values()) {
|
|
25
|
-
if (db.isOpen !== false)
|
|
26
|
-
db.exec('PRAGMA busy_timeout = 0');
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
function closeSharedStoreHandles() {
|
|
30
|
-
for (const db of sharedHandles.values()) {
|
|
31
|
-
sharedSet.delete(db);
|
|
32
|
-
if (db.isOpen !== false)
|
|
33
|
-
db.close();
|
|
34
|
-
}
|
|
35
|
-
sharedHandles.clear();
|
|
17
|
+
currentRequestStores()?.noteBusy();
|
|
36
18
|
}
|
|
37
19
|
/** Runs `fn` with one handle per store: openHippoDb reuses it and closeHippoDb leaves it open until `fn` settles or the process exits.
|
|
38
20
|
* `busyWaitMs` is the lock wait of every open inside `fn` that does not pass its own. */
|
|
39
21
|
export async function withSharedStoreHandles(fn, opts) {
|
|
40
|
-
|
|
41
|
-
process.once('exit', closeSharedStoreHandles);
|
|
42
|
-
shareBusyWaitMs = opts?.busyWaitMs;
|
|
43
|
-
}
|
|
44
|
-
try {
|
|
45
|
-
return await fn();
|
|
46
|
-
}
|
|
47
|
-
finally {
|
|
48
|
-
if (--shareDepth === 0) {
|
|
49
|
-
process.off('exit', closeSharedStoreHandles);
|
|
50
|
-
closeSharedStoreHandles();
|
|
51
|
-
shareBusyWaitMs = undefined;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
22
|
+
return runWithRequestStores(fn, { busyWaitMs: opts?.busyWaitMs });
|
|
54
23
|
}
|
|
55
|
-
/** Lock wait inside an HTTP request: SQLite waits synchronously, so a long wait would stall every other request on the event loop. */
|
|
56
|
-
export const SERVER_DB_WAIT_MS = 250;
|
|
57
24
|
/** Lock wait for each of sleep's short write transactions, even inside a server request: a run stops only on a writer that holds the lock longer. */
|
|
58
25
|
export const SLEEP_DB_WAIT_MS = 5000;
|
|
59
|
-
|
|
60
|
-
/** Runs `fn` so that every store opened inside it, across awaits, waits at most `busyWaitMs` for a lock unless the open passes its own. */
|
|
61
|
-
export function withBusyWait(busyWaitMs, fn) {
|
|
62
|
-
return scopedBusyWaitMs.run(busyWaitMs, fn);
|
|
63
|
-
}
|
|
64
|
-
/** The lock wait an open here would get without its own `busyWaitMs`, in openHippoDb's order; undefined outside both scopes. */
|
|
26
|
+
/** The lock wait an open here would get without its own `busyWaitMs`: the request scope's; undefined outside every scope. */
|
|
65
27
|
export function scopedBusyWait() {
|
|
66
|
-
return
|
|
67
|
-
}
|
|
68
|
-
/** Thrown by a hippo.db open inside a request served from another store: the code path is not ported to the store port yet. */
|
|
69
|
-
export class SqliteBlockedError extends Error {
|
|
70
|
-
storeKind;
|
|
71
|
-
constructor(storeKind) {
|
|
72
|
-
super(`hippo.db is not opened while the '${storeKind}' store serves this request; this code path is not ported to the store yet`);
|
|
73
|
-
this.storeKind = storeKind;
|
|
74
|
-
this.name = 'SqliteBlockedError';
|
|
75
|
-
}
|
|
28
|
+
return currentRequestStores()?.busyWaitMs;
|
|
76
29
|
}
|
|
77
30
|
const sqliteBlockedBy = new AsyncLocalStorage();
|
|
78
31
|
/** Runs `fn` so that every hippo.db open inside it, across awaits, throws; otherwise a missed port would create and write a hippo.db nobody reads. */
|
|
79
32
|
export function withSqliteBlocked(storeKind, fn) {
|
|
80
33
|
return sqliteBlockedBy.run(storeKind, fn);
|
|
81
34
|
}
|
|
35
|
+
/** Runs `fn` with hippo.db opens allowed again, for a store whose own methods are backed by hippo.db. */
|
|
36
|
+
export function withSqliteAllowed(fn) {
|
|
37
|
+
return sqliteBlockedBy.exit(fn);
|
|
38
|
+
}
|
|
39
|
+
/** First line of a best-effort catch around a hippo.db open: an unported path must fail closed, not fall back silently. */
|
|
40
|
+
export function rethrowIfSqliteBlocked(err) {
|
|
41
|
+
if (err instanceof SqliteBlockedError)
|
|
42
|
+
throw err;
|
|
43
|
+
}
|
|
82
44
|
function assertSqliteAllowed() {
|
|
83
45
|
const storeKind = sqliteBlockedBy.getStore();
|
|
84
46
|
if (storeKind !== undefined)
|
|
@@ -87,63 +49,8 @@ function assertSqliteAllowed() {
|
|
|
87
49
|
/** `busyWaitMs` shortens every lock wait of this open, for a hook that must finish inside its own timeout. */
|
|
88
50
|
export function openHippoDb(hippoRoot, opts) {
|
|
89
51
|
assertSqliteAllowed();
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
const busyWaitMs = opts?.busyWaitMs ?? shareBusyWaitMs ?? scopedBusyWaitMs.getStore();
|
|
93
|
-
const key = `${path.resolve(getHippoDbPath(hippoRoot))}\0${busyWaitMs ?? ''}`;
|
|
94
|
-
const shared = sharedHandles.get(key);
|
|
95
|
-
if (shared?.isOpen && !shared.isTransaction)
|
|
96
|
-
return shared;
|
|
97
|
-
const db = openOwnHippoDb(hippoRoot, { busyWaitMs });
|
|
98
|
-
// An open nested inside a transaction gets its own connection, as it did before sharing.
|
|
99
|
-
if (!shared?.isOpen) {
|
|
100
|
-
sharedHandles.set(key, db);
|
|
101
|
-
sharedSet.add(db);
|
|
102
|
-
}
|
|
103
|
-
return db;
|
|
104
|
-
}
|
|
105
|
-
// Owner-only on create; SQLite gives the WAL and SHM files the db's mode. Existing paths keep theirs.
|
|
106
|
-
function createStoreFilesOwnerOnly(hippoRoot) {
|
|
107
|
-
fs.mkdirSync(hippoRoot, { recursive: true, mode: 0o700 });
|
|
108
|
-
try {
|
|
109
|
-
fs.closeSync(fs.openSync(getHippoDbPath(hippoRoot), 'wx', 0o600));
|
|
110
|
-
}
|
|
111
|
-
catch (err) {
|
|
112
|
-
if (!(err instanceof Error && 'code' in err && err.code === 'EEXIST'))
|
|
113
|
-
throw err;
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
function openOwnHippoDb(hippoRoot, opts) {
|
|
117
|
-
createStoreFilesOwnerOnly(hippoRoot);
|
|
118
|
-
const db = new DatabaseSync(getHippoDbPath(hippoRoot));
|
|
119
|
-
const busyWaitMs = opts?.busyWaitMs;
|
|
120
|
-
try {
|
|
121
|
-
db.exec(`PRAGMA busy_timeout = ${busyWaitMs ?? 5000}`);
|
|
122
|
-
execWithBusyRetry(db, 'PRAGMA journal_mode = WAL', busyWaitMs);
|
|
123
|
-
db.exec('PRAGMA synchronous = NORMAL');
|
|
124
|
-
db.exec('PRAGMA wal_autocheckpoint = 100');
|
|
125
|
-
db.exec('PRAGMA foreign_keys = ON');
|
|
126
|
-
runMigrations(db, hippoRoot, busyWaitMs);
|
|
127
|
-
// Path A backfill: delete any orphan markdown mirrors for already-archived
|
|
128
|
-
// raw_archive rows. Idempotent via per-row raw_archive.mirror_cleaned_at
|
|
129
|
-
// (v21). Wrapped in try/catch — a filesystem failure must not prevent DB open.
|
|
130
|
-
try {
|
|
131
|
-
cleanupArchivedMirrors(hippoRoot, db);
|
|
132
|
-
}
|
|
133
|
-
catch (cleanupErr) {
|
|
134
|
-
log.error(`openHippoDb: cleanupArchivedMirrors failed (non-fatal): ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`);
|
|
135
|
-
}
|
|
136
|
-
return db;
|
|
137
|
-
}
|
|
138
|
-
catch (error) {
|
|
139
|
-
try {
|
|
140
|
-
db.close();
|
|
141
|
-
}
|
|
142
|
-
catch {
|
|
143
|
-
// Best effort only.
|
|
144
|
-
}
|
|
145
|
-
throw error;
|
|
146
|
-
}
|
|
52
|
+
const stores = currentRequestStores();
|
|
53
|
+
return stores ? stores.get(hippoRoot, opts) : connectHippoDb(hippoRoot, opts?.busyWaitMs);
|
|
147
54
|
}
|
|
148
55
|
/** Open an existing store without changing it: no mkdir, WAL switch, migration or mirror cleanup. Throws when hippo.db is missing. */
|
|
149
56
|
export function openHippoDbReadOnly(hippoRoot) {
|
|
@@ -166,7 +73,7 @@ export function openHippoDbReadOnly(hippoRoot) {
|
|
|
166
73
|
}
|
|
167
74
|
}
|
|
168
75
|
export function closeHippoDb(db) {
|
|
169
|
-
if (
|
|
76
|
+
if (isScopedHandle(db))
|
|
170
77
|
return;
|
|
171
78
|
db.close();
|
|
172
79
|
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { DatabaseSyncLike } from './sqlite.js';
|
|
2
|
+
export interface RequestStoresOptions {
|
|
3
|
+
/** Lock wait of every open in the scope that does not pass its own. */
|
|
4
|
+
readonly busyWaitMs?: number;
|
|
5
|
+
/** Set for hook scopes: after one lock wait runs out, {@link RequestStores.noteBusy} makes the scope's later writes skip at once. */
|
|
6
|
+
readonly failFastWhenBusy?: boolean;
|
|
7
|
+
}
|
|
8
|
+
/** The handles one request opened, keyed by resolved store path and lock wait. */
|
|
9
|
+
export declare class RequestStores {
|
|
10
|
+
#private;
|
|
11
|
+
readonly busyWaitMs: number | undefined;
|
|
12
|
+
constructor(opts?: RequestStoresOptions);
|
|
13
|
+
get closed(): boolean;
|
|
14
|
+
/** The scope's handle on `hippoRoot`, opened on first use. `opts.busyWaitMs` overrides the scope's lock wait. */
|
|
15
|
+
get(hippoRoot: string, opts?: {
|
|
16
|
+
busyWaitMs?: number;
|
|
17
|
+
}): DatabaseSyncLike;
|
|
18
|
+
/** A lock held past one full wait belongs to a long transaction, so a hook scope's later writes skip at once. */
|
|
19
|
+
noteBusy(): void;
|
|
20
|
+
close(): void;
|
|
21
|
+
}
|
|
22
|
+
/** Runs `fn` in a request scope whose handles close when `fn` settles; inside an open scope `fn` joins it and `opts` is ignored. */
|
|
23
|
+
export declare function runWithRequestStores<T>(fn: () => T | Promise<T>, opts?: RequestStoresOptions): Promise<T>;
|
|
24
|
+
/** The scope this code runs in, if any. A closed scope still answers: its `get` falls through to a plain open. */
|
|
25
|
+
export declare function currentRequestStores(): RequestStores | undefined;
|
|
26
|
+
/** Runs `fn` outside every scope, for a connection that must outlive the request that happens to open it. */
|
|
27
|
+
export declare function outsideRequestStores<T>(fn: () => T): T;
|
|
28
|
+
/** Whether a scope owns `db` and will close it itself. */
|
|
29
|
+
export declare function isScopedHandle(db: DatabaseSyncLike): boolean;
|
|
30
|
+
//# sourceMappingURL=request-stores.d.ts.map
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// One store handle per store for each request (HTTP call, MCP tool call, scoped CLI command), opened lazily and closed with it.
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
4
|
+
import { connectHippoDb, getHippoDbPath } from './connect.js';
|
|
5
|
+
// Handles a scope owns, so closeHippoDb leaves them open for the rest of the request.
|
|
6
|
+
const scopedHandles = new WeakSet();
|
|
7
|
+
// process.exit skips every finally, so one exit listener closes the handles of scopes still open.
|
|
8
|
+
const liveScopes = new Set();
|
|
9
|
+
let exitListenerInstalled = false;
|
|
10
|
+
function closeLiveScopes() {
|
|
11
|
+
for (const scope of liveScopes)
|
|
12
|
+
scope.close();
|
|
13
|
+
}
|
|
14
|
+
function trackLive(scope) {
|
|
15
|
+
liveScopes.add(scope);
|
|
16
|
+
if (exitListenerInstalled)
|
|
17
|
+
return;
|
|
18
|
+
process.on('exit', closeLiveScopes);
|
|
19
|
+
exitListenerInstalled = true;
|
|
20
|
+
}
|
|
21
|
+
/** The handles one request opened, keyed by resolved store path and lock wait. */
|
|
22
|
+
export class RequestStores {
|
|
23
|
+
busyWaitMs;
|
|
24
|
+
#failFastWhenBusy;
|
|
25
|
+
#handles = new Map();
|
|
26
|
+
#closed = false;
|
|
27
|
+
constructor(opts = {}) {
|
|
28
|
+
this.busyWaitMs = opts.busyWaitMs;
|
|
29
|
+
this.#failFastWhenBusy = opts.failFastWhenBusy === true;
|
|
30
|
+
}
|
|
31
|
+
get closed() {
|
|
32
|
+
return this.#closed;
|
|
33
|
+
}
|
|
34
|
+
/** The scope's handle on `hippoRoot`, opened on first use. `opts.busyWaitMs` overrides the scope's lock wait. */
|
|
35
|
+
get(hippoRoot, opts) {
|
|
36
|
+
const busyWaitMs = opts?.busyWaitMs ?? this.busyWaitMs;
|
|
37
|
+
// A listener or timer that outlives its request still sees this scope, so it gets a connection of its own.
|
|
38
|
+
if (this.#closed)
|
|
39
|
+
return connectHippoDb(hippoRoot, busyWaitMs);
|
|
40
|
+
const key = `${path.resolve(getHippoDbPath(hippoRoot))}\0${busyWaitMs ?? ''}`;
|
|
41
|
+
const cached = this.#handles.get(key);
|
|
42
|
+
if (cached?.isOpen && !cached.isTransaction)
|
|
43
|
+
return cached;
|
|
44
|
+
const db = connectHippoDb(hippoRoot, busyWaitMs);
|
|
45
|
+
// An open nested inside a transaction gets its own connection, which its caller closes.
|
|
46
|
+
if (cached?.isOpen)
|
|
47
|
+
return db;
|
|
48
|
+
this.#handles.set(key, db);
|
|
49
|
+
scopedHandles.add(db);
|
|
50
|
+
trackLive(this);
|
|
51
|
+
return db;
|
|
52
|
+
}
|
|
53
|
+
/** A lock held past one full wait belongs to a long transaction, so a hook scope's later writes skip at once. */
|
|
54
|
+
noteBusy() {
|
|
55
|
+
if (!this.#failFastWhenBusy)
|
|
56
|
+
return;
|
|
57
|
+
for (const db of this.#handles.values()) {
|
|
58
|
+
if (db.isOpen !== false)
|
|
59
|
+
db.exec('PRAGMA busy_timeout = 0');
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
close() {
|
|
63
|
+
this.#closed = true;
|
|
64
|
+
liveScopes.delete(this);
|
|
65
|
+
for (const db of this.#handles.values()) {
|
|
66
|
+
if (db.isOpen !== false)
|
|
67
|
+
db.close();
|
|
68
|
+
}
|
|
69
|
+
this.#handles.clear();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
const scopes = new AsyncLocalStorage();
|
|
73
|
+
/** Runs `fn` in a request scope whose handles close when `fn` settles; inside an open scope `fn` joins it and `opts` is ignored. */
|
|
74
|
+
export async function runWithRequestStores(fn, opts) {
|
|
75
|
+
const outer = scopes.getStore();
|
|
76
|
+
if (outer && !outer.closed)
|
|
77
|
+
return fn();
|
|
78
|
+
const stores = new RequestStores(opts);
|
|
79
|
+
return scopes.run(stores, async () => {
|
|
80
|
+
try {
|
|
81
|
+
return await fn();
|
|
82
|
+
}
|
|
83
|
+
finally {
|
|
84
|
+
stores.close();
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
/** The scope this code runs in, if any. A closed scope still answers: its `get` falls through to a plain open. */
|
|
89
|
+
export function currentRequestStores() {
|
|
90
|
+
return scopes.getStore();
|
|
91
|
+
}
|
|
92
|
+
/** Runs `fn` outside every scope, for a connection that must outlive the request that happens to open it. */
|
|
93
|
+
export function outsideRequestStores(fn) {
|
|
94
|
+
return scopes.exit(fn);
|
|
95
|
+
}
|
|
96
|
+
/** Whether a scope owns `db` and will close it itself. */
|
|
97
|
+
export function isScopedHandle(db) {
|
|
98
|
+
return scopedHandles.has(db);
|
|
99
|
+
}
|
|
100
|
+
//# sourceMappingURL=request-stores.js.map
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/** Thrown by a hippo.db open inside a request served from another store: the code path is not ported to the store port yet. */
|
|
2
|
+
export declare class SqliteBlockedError extends Error {
|
|
3
|
+
readonly storeKind: string;
|
|
4
|
+
constructor(storeKind: string);
|
|
5
|
+
}
|
|
6
|
+
//# sourceMappingURL=sqlite-blocked.d.ts.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// A leaf module, so http-util can map this error to a 501 without importing db/open, whose migrations reach back to http-util.
|
|
2
|
+
/** Thrown by a hippo.db open inside a request served from another store: the code path is not ported to the store port yet. */
|
|
3
|
+
export class SqliteBlockedError extends Error {
|
|
4
|
+
storeKind;
|
|
5
|
+
constructor(storeKind) {
|
|
6
|
+
super(`hippo.db is not opened while the '${storeKind}' store serves this request; this code path is not ported to the store yet`);
|
|
7
|
+
this.storeKind = storeKind;
|
|
8
|
+
this.name = 'SqliteBlockedError';
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=sqlite-blocked.js.map
|
package/dist/db.d.ts
CHANGED
|
@@ -3,5 +3,7 @@ export { isSqliteBusy, isStoreBusy, StoreBusyError, STORE_BUSY_MESSAGE, execWith
|
|
|
3
3
|
export { getSchemaVersion, getMeta, setMeta, isFtsAvailable } from './db/meta.js';
|
|
4
4
|
export { countTableRows, pruneConsolidationRuns } from './db/tables.js';
|
|
5
5
|
export { getCurrentSchemaVersion, IncompatibleBinaryError, ftsRowCounts, repairFtsDrift } from './db/migrate.js';
|
|
6
|
-
export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS,
|
|
6
|
+
export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS, scopedBusyWait, withSqliteBlocked, withSqliteAllowed, rethrowIfSqliteBlocked, openHippoDb, openHippoDbReadOnly, closeHippoDb, } from './db/open.js';
|
|
7
|
+
export { RequestStores, runWithRequestStores, currentRequestStores, outsideRequestStores } from './db/request-stores.js';
|
|
8
|
+
export { SqliteBlockedError } from './db/sqlite-blocked.js';
|
|
7
9
|
//# sourceMappingURL=db.d.ts.map
|
package/dist/db.js
CHANGED
|
@@ -2,5 +2,7 @@ export { isSqliteBusy, isStoreBusy, StoreBusyError, STORE_BUSY_MESSAGE, execWith
|
|
|
2
2
|
export { getSchemaVersion, getMeta, setMeta, isFtsAvailable } from './db/meta.js';
|
|
3
3
|
export { countTableRows, pruneConsolidationRuns } from './db/tables.js';
|
|
4
4
|
export { getCurrentSchemaVersion, IncompatibleBinaryError, ftsRowCounts, repairFtsDrift } from './db/migrate.js';
|
|
5
|
-
export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS,
|
|
5
|
+
export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS, scopedBusyWait, withSqliteBlocked, withSqliteAllowed, rethrowIfSqliteBlocked, openHippoDb, openHippoDbReadOnly, closeHippoDb, } from './db/open.js';
|
|
6
|
+
export { RequestStores, runWithRequestStores, currentRequestStores, outsideRequestStores } from './db/request-stores.js';
|
|
7
|
+
export { SqliteBlockedError } from './db/sqlite-blocked.js';
|
|
6
8
|
//# sourceMappingURL=db.js.map
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
* egress-blocked in the build sandbox).
|
|
28
28
|
*/
|
|
29
29
|
import { envByName } from './env.js';
|
|
30
|
-
import { getEmbedding, isEmbeddingAvailable, resolveEmbeddingModel, DEFAULT_EMBEDDING_MODEL, } from './local-embedding.js';
|
|
30
|
+
import { getEmbedding, isEmbeddingAvailable, requireLocalPipeline, resolveEmbeddingModel, DEFAULT_EMBEDDING_MODEL, } from './local-embedding.js';
|
|
31
31
|
import { loadConfig } from './config.js';
|
|
32
32
|
import { redactSecretsStrict } from './secret-detect.js';
|
|
33
33
|
import { fetchWithRetry } from './http-retry.js';
|
|
@@ -57,8 +57,9 @@ class LocalEmbeddingProvider {
|
|
|
57
57
|
return this.enabled && isEmbeddingAvailable();
|
|
58
58
|
}
|
|
59
59
|
async embed(texts, role) {
|
|
60
|
-
//
|
|
61
|
-
|
|
60
|
+
// A model that cannot load fails the call, as an API outage does; items then run one at a time on the shared pipeline.
|
|
61
|
+
if (texts.length > 0)
|
|
62
|
+
await requireLocalPipeline(this.model);
|
|
62
63
|
const out = [];
|
|
63
64
|
for (const text of texts) {
|
|
64
65
|
out.push(await getEmbedding(text, this.model, role));
|
package/dist/embeddings.js
CHANGED
|
@@ -8,7 +8,7 @@ import * as fs from 'fs';
|
|
|
8
8
|
import * as path from 'path';
|
|
9
9
|
import { randomUUID } from 'crypto';
|
|
10
10
|
import { loadAllEntries } from './store/entry-reads.js';
|
|
11
|
-
import { openHippoDb, closeHippoDb, getMeta, setMeta } from './db.js';
|
|
11
|
+
import { openHippoDb, closeHippoDb, getMeta, rethrowIfSqliteBlocked, setMeta } from './db.js';
|
|
12
12
|
import { EMBEDDING_MODEL_META_KEY, deleteOrphanVectors, hasStoredVectors, loadVectors, replaceAllVectors, storedVectorIds, upsertVectors, } from './vector-store.js';
|
|
13
13
|
import { initializeParticle, savePhysicsState, loadPhysicsState, resetAllPhysicsState } from './physics-state.js';
|
|
14
14
|
import { loadConfig } from './config.js';
|
|
@@ -81,6 +81,7 @@ function loadStoredEmbeddingModel(hippoRoot) {
|
|
|
81
81
|
}
|
|
82
82
|
}
|
|
83
83
|
catch (err) {
|
|
84
|
+
rethrowIfSqliteBlocked(err);
|
|
84
85
|
log.debug(`stored embedding model unreadable: ${err instanceof Error ? err.message : String(err)}`);
|
|
85
86
|
return null;
|
|
86
87
|
}
|
package/dist/goals.d.ts
CHANGED
|
@@ -78,6 +78,13 @@ export interface GetActiveGoalsOpts {
|
|
|
78
78
|
tenantId: string;
|
|
79
79
|
}
|
|
80
80
|
export declare function getActiveGoals(hippoRoot: string, opts: GetActiveGoalsOpts): Goal[];
|
|
81
|
+
/** A session's active goals, oldest first, and the retrieval policy of each goal that names one. */
|
|
82
|
+
export interface ActiveGoals {
|
|
83
|
+
readonly goals: Goal[];
|
|
84
|
+
readonly policies: ReadonlyMap<string, RetrievalPolicy>;
|
|
85
|
+
}
|
|
86
|
+
/** The active goals and their policies on one handle, so a goal and its policy never disagree. */
|
|
87
|
+
export declare function activeGoalsWithPolicies(hippoRoot: string, opts: GetActiveGoalsOpts): ActiveGoals;
|
|
81
88
|
/** Every goal of a (tenant, session), whatever its status, oldest first. */
|
|
82
89
|
export declare function getSessionGoals(hippoRoot: string, opts: GetActiveGoalsOpts): Goal[];
|
|
83
90
|
export declare function getActiveGoalsWithDb(db: DatabaseSyncLike, opts: GetActiveGoalsOpts): Goal[];
|
|
@@ -90,7 +97,7 @@ export interface GoalRecallLogRow {
|
|
|
90
97
|
recalledAt: string;
|
|
91
98
|
score: number;
|
|
92
99
|
}
|
|
93
|
-
/** Options shared by {@link computeGoalStackBoost} and {@link
|
|
100
|
+
/** Options shared by {@link computeGoalStackBoost} and {@link boostByGoals}. */
|
|
94
101
|
export interface GoalStackBoostOpts {
|
|
95
102
|
sessionId: string;
|
|
96
103
|
tenantId: string;
|
|
@@ -107,6 +114,10 @@ export interface GoalStackBoost<R> {
|
|
|
107
114
|
results: R[];
|
|
108
115
|
log: GoalRecallLogRow[];
|
|
109
116
|
}
|
|
117
|
+
export declare function loadGoalPolicies(db: DatabaseSyncLike, active: readonly Goal[]): Map<string, RetrievalPolicy>;
|
|
118
|
+
/** The rows whose memory lives in this store: goal_recall_log.memory_id references memories, so a global row would fail
|
|
119
|
+
* the insert. A global result keeps its boost; it only earns no log row, so no outcome propagation. */
|
|
120
|
+
export declare function localGoalRecallRows(db: DatabaseSyncLike, rows: readonly GoalRecallLogRow[]): GoalRecallLogRow[];
|
|
110
121
|
/**
|
|
111
122
|
* dlPFC goal-stack boost. Applies the multi-goal boost to entry-backed scored
|
|
112
123
|
* rows when (tenant, session) has active goals and returns the log rows to
|
|
@@ -130,21 +141,17 @@ export declare function computeGoalStackBoost<R extends {
|
|
|
130
141
|
entry: MemoryEntry;
|
|
131
142
|
score: number;
|
|
132
143
|
}>(db: DatabaseSyncLike, results: R[], opts: GoalStackBoostOpts): GoalStackBoost<R>;
|
|
144
|
+
/** {@link computeGoalStackBoost} over goals already read, touching no store. Its log keeps rows whose memory is global,
|
|
145
|
+
* since only the store can tell them apart; {@link localGoalRecallRows} drops those before the write. */
|
|
146
|
+
export declare function boostByGoals<R extends {
|
|
147
|
+
entry: MemoryEntry;
|
|
148
|
+
score: number;
|
|
149
|
+
}>(results: R[], active: ActiveGoals, opts: GoalStackBoostOpts): GoalStackBoost<R>;
|
|
133
150
|
/**
|
|
134
151
|
* Writes goal-boost log rows. INSERT OR IGNORE because UNIQUE(memory_id, goal_id)
|
|
135
152
|
* makes a re-recall during the same goal life a no-op for outcome attribution.
|
|
136
153
|
*/
|
|
137
154
|
export declare function writeGoalRecallLog(db: DatabaseSyncLike, rows: readonly GoalRecallLogRow[]): void;
|
|
138
|
-
/**
|
|
139
|
-
* {@link computeGoalStackBoost} plus {@link writeGoalRecallLog} in one call, for
|
|
140
|
-
* pipelines that boost and log on the same handle.
|
|
141
|
-
*
|
|
142
|
-
* @internal Recall ranking helper. Subject to change.
|
|
143
|
-
*/
|
|
144
|
-
export declare function applyGoalStackBoost<R extends {
|
|
145
|
-
entry: MemoryEntry;
|
|
146
|
-
score: number;
|
|
147
|
-
}>(db: DatabaseSyncLike, results: R[], opts: GoalStackBoostOpts): R[];
|
|
148
155
|
export interface CompleteGoalOpts {
|
|
149
156
|
outcomeScore?: number;
|
|
150
157
|
/**
|
package/dist/goals.js
CHANGED
|
@@ -129,6 +129,17 @@ export function getActiveGoals(hippoRoot, opts) {
|
|
|
129
129
|
closeHippoDb(db);
|
|
130
130
|
}
|
|
131
131
|
}
|
|
132
|
+
/** The active goals and their policies on one handle, so a goal and its policy never disagree. */
|
|
133
|
+
export function activeGoalsWithPolicies(hippoRoot, opts) {
|
|
134
|
+
const db = openHippoDb(hippoRoot);
|
|
135
|
+
try {
|
|
136
|
+
const goals = getActiveGoalsWithDb(db, opts);
|
|
137
|
+
return { goals, policies: loadGoalPolicies(db, goals) };
|
|
138
|
+
}
|
|
139
|
+
finally {
|
|
140
|
+
closeHippoDb(db);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
132
143
|
/** Every goal of a (tenant, session), whatever its status, oldest first. */
|
|
133
144
|
export function getSessionGoals(hippoRoot, opts) {
|
|
134
145
|
const db = openHippoDb(hippoRoot);
|
|
@@ -163,7 +174,7 @@ export function getActiveGoalsWithDb(db, opts) {
|
|
|
163
174
|
// can compose onto the base goal-tag boost. Composed result is hard-capped
|
|
164
175
|
// at MAX_FINAL_MULTIPLIER (3.0x) BEFORE applying to score -- even an
|
|
165
176
|
// `errorPriority: 9.0` policy cannot exceed 3.0x.
|
|
166
|
-
function loadGoalPolicies(db, active) {
|
|
177
|
+
export function loadGoalPolicies(db, active) {
|
|
167
178
|
const policiesByGoalId = new Map();
|
|
168
179
|
for (const g of active) {
|
|
169
180
|
if (!g.retrievalPolicyId)
|
|
@@ -220,28 +231,11 @@ function goalBoostMultiplier(entry, tags, matches, goalsByTag, policiesByGoalId)
|
|
|
220
231
|
return Math.min(multiplier, MAX_FINAL_MULTIPLIER);
|
|
221
232
|
}
|
|
222
233
|
/** Log rows for the top `limit` boosted rows, one per matched goal. */
|
|
223
|
-
function buildGoalRecallLog(
|
|
234
|
+
function buildGoalRecallLog(boosted, matchesByEntryId, goalsByTag, opts) {
|
|
224
235
|
const { sessionId, tenantId, limit } = opts;
|
|
225
|
-
// Filter to local memories only -- global memory IDs aren't in this DB's
|
|
226
|
-
// memories table, so the FK on goal_recall_log.memory_id would fail.
|
|
227
|
-
// dlPFC depth's outcome propagation is session-scoped to local; boost on
|
|
228
|
-
// ranking still applies to global results, just no log row -> no
|
|
229
|
-
// propagation.
|
|
230
|
-
const topKIds = boosted.slice(0, limit).map((r) => r.entry.id);
|
|
231
|
-
const localIds = new Set();
|
|
232
|
-
if (topKIds.length > 0) {
|
|
233
|
-
const placeholders = topKIds.map(() => '?').join(',');
|
|
234
|
-
// SAFETY: rows come from the SELECT above, which projects exactly one
|
|
235
|
-
// column, `id`, from memories.
|
|
236
|
-
const localRows = db.prepare(`SELECT id FROM memories WHERE id IN (${placeholders})`).all(...topKIds);
|
|
237
|
-
for (const row of localRows)
|
|
238
|
-
localIds.add(row.id);
|
|
239
|
-
}
|
|
240
236
|
const recalledAt = new Date().toISOString();
|
|
241
237
|
const log = [];
|
|
242
238
|
for (const r of boosted.slice(0, limit)) {
|
|
243
|
-
if (!localIds.has(r.entry.id))
|
|
244
|
-
continue; // global -> skip log insert
|
|
245
239
|
const matches = matchesByEntryId.get(r.entry.id);
|
|
246
240
|
if (!matches || matches.length === 0)
|
|
247
241
|
continue;
|
|
@@ -254,6 +248,17 @@ function buildGoalRecallLog(db, boosted, matchesByEntryId, goalsByTag, opts) {
|
|
|
254
248
|
}
|
|
255
249
|
return log;
|
|
256
250
|
}
|
|
251
|
+
/** The rows whose memory lives in this store: goal_recall_log.memory_id references memories, so a global row would fail
|
|
252
|
+
* the insert. A global result keeps its boost; it only earns no log row, so no outcome propagation. */
|
|
253
|
+
export function localGoalRecallRows(db, rows) {
|
|
254
|
+
if (rows.length === 0)
|
|
255
|
+
return [];
|
|
256
|
+
const ids = [...new Set(rows.map((r) => r.memoryId))];
|
|
257
|
+
// SAFETY: the SELECT projects exactly one column, `id`, from memories.
|
|
258
|
+
const local = db.prepare(`SELECT id FROM memories WHERE id IN (${ids.map(() => '?').join(',')})`).all(...ids);
|
|
259
|
+
const localIds = new Set(local.map((r) => r.id));
|
|
260
|
+
return rows.filter((r) => localIds.has(r.memoryId));
|
|
261
|
+
}
|
|
257
262
|
/**
|
|
258
263
|
* dlPFC goal-stack boost. Applies the multi-goal boost to entry-backed scored
|
|
259
264
|
* rows when (tenant, session) has active goals and returns the log rows to
|
|
@@ -274,12 +279,20 @@ function buildGoalRecallLog(db, boosted, matchesByEntryId, goalsByTag, opts) {
|
|
|
274
279
|
* @internal Recall ranking helper. Subject to change.
|
|
275
280
|
*/
|
|
276
281
|
export function computeGoalStackBoost(db, results, opts) {
|
|
277
|
-
const { sessionId, tenantId
|
|
278
|
-
|
|
279
|
-
|
|
282
|
+
const goals = getActiveGoalsWithDb(db, { sessionId: opts.sessionId, tenantId: opts.tenantId });
|
|
283
|
+
if (goals.length === 0)
|
|
284
|
+
return { results, log: [] };
|
|
285
|
+
const boost = boostByGoals(results, { goals, policies: loadGoalPolicies(db, goals) }, opts);
|
|
286
|
+
return { results: boost.results, log: localGoalRecallRows(db, boost.log) };
|
|
287
|
+
}
|
|
288
|
+
/** {@link computeGoalStackBoost} over goals already read, touching no store. Its log keeps rows whose memory is global,
|
|
289
|
+
* since only the store can tell them apart; {@link localGoalRecallRows} drops those before the write. */
|
|
290
|
+
export function boostByGoals(results, active, opts) {
|
|
291
|
+
const { trace } = opts;
|
|
292
|
+
if (active.goals.length === 0)
|
|
280
293
|
return { results, log: [] };
|
|
281
|
-
const goalsByTag = new Map(active.map((g) => [g.goalName, g]));
|
|
282
|
-
const policiesByGoalId =
|
|
294
|
+
const goalsByTag = new Map(active.goals.map((g) => [g.goalName, g]));
|
|
295
|
+
const policiesByGoalId = active.policies;
|
|
283
296
|
// Goal-tag matches per boosted row, keyed by entry id. Kept as a side table
|
|
284
297
|
// (rather than a spread-on `_goalMatches` marker property) so `boosted`
|
|
285
298
|
// stays exactly R[] end to end, with no cast-tag-then-strip round trip.
|
|
@@ -313,7 +326,7 @@ export function computeGoalStackBoost(db, results, opts) {
|
|
|
313
326
|
// inherits its determinism via sort stability, and ties preserve the
|
|
314
327
|
// prior (meaningful) rank instead of reordering by content.
|
|
315
328
|
.sort((a, b) => b.score - a.score);
|
|
316
|
-
return { results: boosted, log: buildGoalRecallLog(
|
|
329
|
+
return { results: boosted, log: buildGoalRecallLog(boosted, matchesByEntryId, goalsByTag, opts) };
|
|
317
330
|
}
|
|
318
331
|
/**
|
|
319
332
|
* Writes goal-boost log rows. INSERT OR IGNORE because UNIQUE(memory_id, goal_id)
|
|
@@ -331,17 +344,6 @@ export function writeGoalRecallLog(db, rows) {
|
|
|
331
344
|
insertLog.run(row.goalId, row.memoryId, row.tenantId, row.sessionId, row.recalledAt, row.score);
|
|
332
345
|
}
|
|
333
346
|
}
|
|
334
|
-
/**
|
|
335
|
-
* {@link computeGoalStackBoost} plus {@link writeGoalRecallLog} in one call, for
|
|
336
|
-
* pipelines that boost and log on the same handle.
|
|
337
|
-
*
|
|
338
|
-
* @internal Recall ranking helper. Subject to change.
|
|
339
|
-
*/
|
|
340
|
-
export function applyGoalStackBoost(db, results, opts) {
|
|
341
|
-
const boost = computeGoalStackBoost(db, results, opts);
|
|
342
|
-
writeGoalRecallLog(db, boost.log);
|
|
343
|
-
return boost.results;
|
|
344
|
-
}
|
|
345
347
|
const POSITIVE_OUTCOME_THRESHOLD = 0.7;
|
|
346
348
|
const NEGATIVE_OUTCOME_THRESHOLD = 0.3;
|
|
347
349
|
const STRENGTH_BOOST = 1.10;
|
package/dist/http-util.d.ts
CHANGED
|
@@ -23,6 +23,8 @@ export interface ApiErrorReply {
|
|
|
23
23
|
message: string;
|
|
24
24
|
}
|
|
25
25
|
export declare const INTERNAL_ERROR_MESSAGE = "internal server error";
|
|
26
|
+
/** The reply when a request under another store reaches code that still opens hippo.db. */
|
|
27
|
+
export declare const STORE_NOT_PORTED_MESSAGE = "store_not_ported";
|
|
26
28
|
/** Maps by class so rewording a message never moves a status; an untyped error is a 500 whose text stays in the server log. */
|
|
27
29
|
export declare function mapApiError<E>(err: E): ApiErrorReply;
|
|
28
30
|
/** Serialises before the head goes out, so a body that is not JSON still reaches the caller's error reply. */
|
package/dist/http-util.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ApiError } from './api-errors.js';
|
|
2
|
+
import { SqliteBlockedError } from './db/sqlite-blocked.js';
|
|
2
3
|
// Leaf module shared by server.ts and the connector webhook receivers; it must not import either.
|
|
3
4
|
export function isJsonObjectRecord(value) {
|
|
4
5
|
return value !== undefined && value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
@@ -35,6 +36,8 @@ export class BodyTooLargeError extends Error {
|
|
|
35
36
|
export class BodyTimeoutError extends Error {
|
|
36
37
|
}
|
|
37
38
|
export const INTERNAL_ERROR_MESSAGE = 'internal server error';
|
|
39
|
+
/** The reply when a request under another store reaches code that still opens hippo.db. */
|
|
40
|
+
export const STORE_NOT_PORTED_MESSAGE = 'store_not_ported';
|
|
38
41
|
/** Maps by class so rewording a message never moves a status; an untyped error is a 500 whose text stays in the server log. */
|
|
39
42
|
export function mapApiError(err) {
|
|
40
43
|
// An add-on can build an HttpError from any number, and writeHead throws on one outside 100-999.
|
|
@@ -47,6 +50,8 @@ export function mapApiError(err) {
|
|
|
47
50
|
return { status: 413, message: err.message };
|
|
48
51
|
if (err instanceof BodyTimeoutError)
|
|
49
52
|
return { status: 408, message: err.message };
|
|
53
|
+
if (err instanceof SqliteBlockedError)
|
|
54
|
+
return { status: 501, message: STORE_NOT_PORTED_MESSAGE };
|
|
50
55
|
return { status: 500, message: INTERNAL_ERROR_MESSAGE };
|
|
51
56
|
}
|
|
52
57
|
/** Serialises before the head goes out, so a body that is not JSON still reaches the caller's error reply. */
|
|
@@ -7,6 +7,8 @@ export type EmbeddingRole = 'query' | 'passage';
|
|
|
7
7
|
export declare function prefixFor(model: string, role?: EmbeddingRole): string;
|
|
8
8
|
/** Check (synchronously) if @xenova/transformers or @huggingface/transformers is installed. */
|
|
9
9
|
export declare function isEmbeddingAvailable(): boolean;
|
|
10
|
+
/** Throws, with the reason, when the model's pipeline cannot load: a provider-level failure, not a per-item skip. */
|
|
11
|
+
export declare function requireLocalPipeline(model: string): Promise<void>;
|
|
10
12
|
export declare function resolveEmbeddingModel(hippoRoot: string, explicitModel?: string): string;
|
|
11
13
|
/** Embeds `text` with the local model, or returns `[]` when Transformers.js is missing or fails; `role` adds e5-style prefixes. */
|
|
12
14
|
export declare function getEmbedding(text: string, model?: string, role?: EmbeddingRole): Promise<number[]>;
|