hippo-memory 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/api/detached.d.ts +6 -0
- package/dist/api/detached.js +16 -0
- package/dist/api/embeddings.d.ts +4 -3
- package/dist/api/embeddings.js +4 -4
- package/dist/api/memories/promote.js +2 -1
- package/dist/api/memories/remember-local.d.ts +5 -4
- package/dist/api/memories/remember-local.js +13 -10
- package/dist/api/session-end.js +1 -0
- package/dist/api/sharing.js +9 -7
- package/dist/api/sleep/auto-sleep.d.ts +1 -1
- package/dist/api/sleep/auto-sleep.js +5 -3
- package/dist/api/sleep/learn.js +5 -2
- package/dist/api/sleep/sleep-run.js +4 -2
- package/dist/api/types.d.ts +4 -0
- package/dist/base/util/detached-set.d.ts +8 -0
- package/dist/base/util/detached-set.js +35 -0
- package/dist/base/util/version.d.ts +1 -1
- package/dist/base/util/version.js +1 -1
- package/dist/db/store-runtime.d.ts +10 -3
- package/dist/db/store-runtime.js +19 -3
- package/dist/domain/search/detached-work.d.ts +6 -0
- package/dist/domain/search/detached-work.js +2 -0
- package/dist/domain/search/embed-memories.d.ts +4 -4
- package/dist/domain/search/embed-memories.js +4 -5
- package/dist/domain/sharing/global-sync.d.ts +8 -2
- package/dist/domain/sharing/global-sync.js +5 -5
- package/dist/domain/sharing/share.d.ts +4 -0
- package/dist/domain/sharing/share.js +2 -2
- package/dist/entry/eval-scripts.d.ts +1 -0
- package/dist/entry/sleep.d.ts +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/server.d.ts +1 -0
- package/dist/store/sqlite/runtime/server-runtime.d.ts +6 -1
- package/dist/store/sqlite/runtime/server-runtime.js +15 -3
- package/dist/store/sqlite/runtime/worker-jobs.js +2 -2
- package/dist/surface/capture/compact.d.ts +3 -3
- package/dist/surface/capture/compact.js +4 -4
- package/dist/surface/capture/session-texts.js +4 -6
- package/dist/surface/cli/api-context.d.ts +5 -0
- package/dist/surface/cli/api-context.js +12 -0
- package/dist/surface/cli/goals.js +6 -8
- package/dist/surface/cli/hook-runtime.js +2 -2
- package/dist/surface/cli/memory-import.js +20 -15
- package/dist/surface/mcp/protocol.d.ts +6 -0
- package/dist/surface/mcp/protocol.js +1 -0
- package/dist/surface/mcp/stdio.js +11 -5
- package/dist/surface/open-store.d.ts +2 -2
- package/dist/surface/open-store.js +11 -7
- package/dist/surface/server/auth.d.ts +1 -1
- package/dist/surface/server/auth.js +13 -2
- package/dist/surface/server/boot.js +3 -2
- package/dist/surface/server/dashboard-listener.js +2 -2
- package/dist/surface/server/lifecycle.d.ts +2 -0
- package/dist/surface/server/lifecycle.js +4 -2
- package/dist/surface/server/mcp-http.js +1 -0
- package/dist/surface/server/types.d.ts +3 -0
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -56,6 +56,7 @@ Every measurement we have ever published is indexed in [`docs/evals/`](docs/eval
|
|
|
56
56
|
pre-registrations kept next to their results, including the runs that failed and the one
|
|
57
57
|
claim we retracted.
|
|
58
58
|
|
|
59
|
+
- **Z0 round 2, 16 repositories, Claude Code with hippo against its built-in memory: no extra cost (ratio 0.999 [0.962, 1.037]) and about 12% less spent on teach and correction turns. A drop in repeat mistakes is not shown (-8.9 points [-18.5, +0.7], under the pre-registered 15-point minimum).** ([result](docs/evals/2026-11-01-z0-round2-result.md))
|
|
59
60
|
- **Sequential Learning Benchmark.** [benchmarks/sequential-learning/](benchmarks/sequential-learning/). 50 tasks, 10 buried traps. Measures whether agents learn from past mistakes, not just retrieve text. v0.11.0 informal magnitude RETRACTED v1.7.9; mechanism remains shipped. See [CHANGELOG.md](./CHANGELOG.md) v1.7.9 entry.
|
|
60
61
|
- **LongMemEval oracle split, all 500 questions in one pooled store, BM25 only, no embeddings, v0.11: R@5 = 74.0%** ([benchmarks/README.md](benchmarks/README.md), scripts in [benchmarks/longmemeval/](benchmarks/longmemeval/)). A different setup from the per-haystack results under [Benchmarks](#benchmarks), so the two are not a before and after.
|
|
61
62
|
- **On a private 300-query developer store, R@1 0.41 to 0.62 with `hippo recall "<query>" --reranker jev`**, the free local cross-encoder against Jev ([full eval](docs/evals/2026-09-19-jev-reranker.md)). The opt-in [TypeSafe Jev](https://typesafe.ai) reranker, off by default, about 0.0004 USD a recall. 2000-draw paired bootstrap; the margin held in 20 of 20 seeds and a permutation null reached it in 0 of 200 runs. Ranking only: three graded tests on one 150-question LongMemEval set did **not** show a better answer rate than the free local cross-encoder, and that negative result is in the same doc. What it buys today is a shorter context: on that set, 2 memories ranked by Jev answered as well as 5 ranked by the cross-encoder.
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { DetachedWork } from '../domain/search/detached-work.js';
|
|
2
|
+
import type { Context } from './types.js';
|
|
3
|
+
/** Runs `start` with the root's keeper of detached work; a context with none gets one of the
|
|
4
|
+
* call's own, which settles before the call returns, so no work outlives the call unowned. */
|
|
5
|
+
export declare function withDetached<T>(ctx: Pick<Context, 'detached'>, start: (detached: DetachedWork) => T | Promise<T>): Promise<T>;
|
|
6
|
+
//# sourceMappingURL=detached.d.ts.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// Work an api call starts and does not wait on goes to the keeper its root set on the Context.
|
|
2
|
+
import { DetachedSet } from '../base/util/detached-set.js';
|
|
3
|
+
/** Runs `start` with the root's keeper of detached work; a context with none gets one of the
|
|
4
|
+
* call's own, which settles before the call returns, so no work outlives the call unowned. */
|
|
5
|
+
export async function withDetached(ctx, start) {
|
|
6
|
+
if (ctx.detached !== undefined)
|
|
7
|
+
return start(ctx.detached);
|
|
8
|
+
const own = new DetachedSet();
|
|
9
|
+
try {
|
|
10
|
+
return await start(own);
|
|
11
|
+
}
|
|
12
|
+
finally {
|
|
13
|
+
await own.settle();
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=detached.js.map
|
package/dist/api/embeddings.d.ts
CHANGED
|
@@ -2,12 +2,13 @@ import type { ConfiguredFolder } from '../base/config/configured-folder.js';
|
|
|
2
2
|
import type { MemoryEntry } from '../base/memory/entry.js';
|
|
3
3
|
import type { EmbeddingProvider } from '../base/ports/embedder.js';
|
|
4
4
|
import { type EmbedRun } from '../domain/search/embed-memories.js';
|
|
5
|
+
import type { DetachedWork } from '../domain/search/detached-work.js';
|
|
5
6
|
import type { RequiredStore, VectorCoverage } from '../base/ports/store.js';
|
|
6
7
|
import { type Context, type ContextWith } from './types.js';
|
|
7
8
|
/** Embeds each row a capture writes into `folder`'s index without holding the capture up; a
|
|
8
|
-
* failure is a warning. A capture runs outside any Context, so its root hands the eval switches
|
|
9
|
-
*
|
|
10
|
-
export declare function detachedEmbedder(folder: ConfiguredFolder<RequiredStore>, run: EmbedRun): (entry: MemoryEntry) => void;
|
|
9
|
+
* failure is a warning. A capture runs outside any Context, so its root hands the eval switches,
|
|
10
|
+
* the clock and the embeds' keeper. */
|
|
11
|
+
export declare function detachedEmbedder(folder: ConfiguredFolder<RequiredStore>, run: EmbedRun, detached: DetachedWork): (entry: MemoryEntry) => void;
|
|
11
12
|
/** Embed every memory that has no vector yet; resolves to how many it embedded. */
|
|
12
13
|
export declare function memoryEmbedMissing(ctx: Context, provider?: EmbeddingProvider): Promise<number>;
|
|
13
14
|
/** How many vectors the store holds, live memory or not. */
|
package/dist/api/embeddings.js
CHANGED
|
@@ -5,11 +5,11 @@ import { embedAll, embedMemoryDetached, folderEmbedTarget, } from '../domain/sea
|
|
|
5
5
|
import { indexStep } from '../domain/search/embed-index-target.js';
|
|
6
6
|
import { requireHostAdmin } from './types.js';
|
|
7
7
|
/** Embeds each row a capture writes into `folder`'s index without holding the capture up; a
|
|
8
|
-
* failure is a warning. A capture runs outside any Context, so its root hands the eval switches
|
|
9
|
-
*
|
|
10
|
-
export function detachedEmbedder(folder, run) {
|
|
8
|
+
* failure is a warning. A capture runs outside any Context, so its root hands the eval switches,
|
|
9
|
+
* the clock and the embeds' keeper. */
|
|
10
|
+
export function detachedEmbedder(folder, run, detached) {
|
|
11
11
|
const target = folderEmbedTarget(folder, run);
|
|
12
|
-
return (entry) => embedMemoryDetached(target, entry);
|
|
12
|
+
return (entry) => embedMemoryDetached(target, entry, detached);
|
|
13
13
|
}
|
|
14
14
|
/** Embed every memory that has no vector yet; resolves to how many it embedded. */
|
|
15
15
|
export async function memoryEmbedMissing(ctx, provider) {
|
|
@@ -4,6 +4,7 @@ import { createSuccessor, generateId, } from '../../base/memory/entry.js';
|
|
|
4
4
|
import { detectSecret } from '../../base/util/secret-detect.js';
|
|
5
5
|
import { embedMemoryDetached, folderEmbedTarget } from '../../domain/search/embed-memories.js';
|
|
6
6
|
import { globalTargetOf } from '../infra.js';
|
|
7
|
+
import { withDetached } from '../detached.js';
|
|
7
8
|
import { sharedStoreOf } from '../types.js';
|
|
8
9
|
import { assertWriteRight, canReadScope, isPersonalScope, isQuarantineScope, scopeReaderOf, } from '../../base/scope/recall-scope.js';
|
|
9
10
|
export async function promote(ctx, id) {
|
|
@@ -29,7 +30,7 @@ export async function promote(ctx, id) {
|
|
|
29
30
|
auditAs: { tenantId: ctx.tenantId, actor: ctx.actor.subject },
|
|
30
31
|
});
|
|
31
32
|
// A promote may create the global store, which only the write target names before it exists.
|
|
32
|
-
embedMemoryDetached(folderEmbedTarget(target, ctx.infra), globalEntry);
|
|
33
|
+
await withDetached(ctx, (detached) => embedMemoryDetached(folderEmbedTarget(target, ctx.infra), globalEntry, detached));
|
|
33
34
|
return { ok: true, sourceId: id, globalId: globalEntry.id };
|
|
34
35
|
}
|
|
35
36
|
/** The copy of a local memory, with a new `g_` id so it cannot collide. A project store's NULL
|
|
@@ -29,8 +29,9 @@ export interface StartedWeak {
|
|
|
29
29
|
readonly strength: number;
|
|
30
30
|
}
|
|
31
31
|
/** `stored` carries the row as the store holds it after the write, and `embedding`, which
|
|
32
|
-
* settles with whether the row reached the vector index (
|
|
33
|
-
*
|
|
32
|
+
* settles with whether the row reached the vector index (the Context's keeper also holds it, so a
|
|
33
|
+
* caller may drop it; a failure is logged). `skipped` is the gate dropping the write, with nothing
|
|
34
|
+
* stored or counted. */
|
|
34
35
|
export type LocalRememberOutcome = {
|
|
35
36
|
readonly status: 'stored';
|
|
36
37
|
readonly entry: MemoryEntry;
|
|
@@ -43,8 +44,8 @@ export type LocalRememberOutcome = {
|
|
|
43
44
|
readonly reason: string;
|
|
44
45
|
readonly score: number;
|
|
45
46
|
};
|
|
46
|
-
/** One local write on hippo.db: gate, write, read-back, counter, then the embedding,
|
|
47
|
-
*
|
|
47
|
+
/** One local write on hippo.db: gate, write, read-back, counter, then the embedding, which the
|
|
48
|
+
* Context's keeper owns; with none, the call waits for it. A refused value throws as `remember` does. */
|
|
48
49
|
export declare function rememberLocally(ctx: Context, input: LocalRememberInput): Promise<LocalRememberOutcome>;
|
|
49
50
|
/** What a caller asks of fact extraction; the model is the one its Context carries. */
|
|
50
51
|
export interface FactExtractionRequest {
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
// The local remember pipeline for a caller in this process (`hippo remember`, `hippo watch`):
|
|
2
2
|
// schema fit, the salience gate, the write, the read-back, the `remembered` counter and the
|
|
3
|
-
// embedding
|
|
4
|
-
// it has reported the write. Nothing here prints
|
|
3
|
+
// embedding, which the Context's keeper owns, in that order. Fact extraction is a second step,
|
|
4
|
+
// which the caller runs after it has reported the write. Nothing here prints.
|
|
5
5
|
import { embedMemoryInto, folderEmbedTarget, } from '../../domain/search/embed-memories.js';
|
|
6
6
|
import { captureError } from '../../domain/learn/autolearn.js';
|
|
7
7
|
import { extractFacts, storeExtractedFacts } from '../../domain/learn/extract.js';
|
|
8
8
|
import { errorFields, errorMessage, log } from '../../base/util/log.js';
|
|
9
|
+
import { withDetached } from '../detached.js';
|
|
9
10
|
import { memoryGet } from './memories.js';
|
|
10
11
|
import { remember } from './remember.js';
|
|
11
12
|
// The tags a caller in this process adds from the folder it runs in.
|
|
@@ -20,8 +21,7 @@ async function storedEntry(ctx, id) {
|
|
|
20
21
|
throw new Error(`memory ${id} was written but cannot be read back`);
|
|
21
22
|
return entry;
|
|
22
23
|
}
|
|
23
|
-
/** Best-effort and slow (a model load or a network call)
|
|
24
|
-
* row left out of the index is logged. */
|
|
24
|
+
/** Best-effort and slow (a model load or a network call); a row left out of the index is logged. */
|
|
25
25
|
function indexInBackground(ctx, entry) {
|
|
26
26
|
const folder = { store: ctx.store, config: ctx.config, embedderFor: ctx.infra.embedderFor };
|
|
27
27
|
const target = folderEmbedTarget(folder, ctx.infra);
|
|
@@ -32,8 +32,8 @@ function indexInBackground(ctx, entry) {
|
|
|
32
32
|
return outcome;
|
|
33
33
|
});
|
|
34
34
|
}
|
|
35
|
-
/** One local write on hippo.db: gate, write, read-back, counter, then the embedding,
|
|
36
|
-
*
|
|
35
|
+
/** One local write on hippo.db: gate, write, read-back, counter, then the embedding, which the
|
|
36
|
+
* Context's keeper owns; with none, the call waits for it. A refused value throws as `remember` does. */
|
|
37
37
|
export async function rememberLocally(ctx, input) {
|
|
38
38
|
const schemaFit = await ctx.store.contextReads.schemaFit(ctx.tenantId, input.content, input.fitTags ?? input.tags);
|
|
39
39
|
const written = await remember(ctx, {
|
|
@@ -57,10 +57,13 @@ export async function rememberLocally(ctx, input) {
|
|
|
57
57
|
const { id, warnings = [], startedWeak } = written;
|
|
58
58
|
const entry = await storedEntry(ctx, id);
|
|
59
59
|
await ctx.store.bumpRemembered(1);
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
60
|
+
return withDetached(ctx, (detached) => {
|
|
61
|
+
const embedding = indexInBackground(ctx, entry);
|
|
62
|
+
detached.detach(embedding);
|
|
63
|
+
return startedWeak
|
|
64
|
+
? { status: 'stored', entry, embedding, warnings, startedWeak }
|
|
65
|
+
: { status: 'stored', entry, embedding, warnings };
|
|
66
|
+
});
|
|
64
67
|
}
|
|
65
68
|
/** Extracts facts from a row just stored and stores them beside it.
|
|
66
69
|
* Best-effort: a failure is named in the result, never thrown. */
|
package/dist/api/session-end.js
CHANGED
package/dist/api/sharing.js
CHANGED
|
@@ -5,6 +5,7 @@ import { ALL_TENANTS } from '../base/util/tenant.js';
|
|
|
5
5
|
import { syncGlobalToLocal } from '../domain/sharing/global-sync.js';
|
|
6
6
|
import { autoShare, listPeers, shareMemory } from '../domain/sharing/share.js';
|
|
7
7
|
import { globalTargetOf } from './infra.js';
|
|
8
|
+
import { withDetached } from './detached.js';
|
|
8
9
|
import { requireHostAdmin } from './types.js';
|
|
9
10
|
export { AUTO_SHARE_MIN_SCORE, transferScore } from '../domain/sharing/share.js';
|
|
10
11
|
// A root's reach sets it for a store kept in a folder, so only a context built without a reach
|
|
@@ -19,17 +20,16 @@ function requireGlobalTarget(ctx, action) {
|
|
|
19
20
|
/** Copies one of the caller's memories to the global store; null when its transfer score is too
|
|
20
21
|
* low and `force` is off. */
|
|
21
22
|
export async function shareToGlobal(ctx, id, opts) {
|
|
23
|
+
const target = requireGlobalTarget(ctx, 'share');
|
|
22
24
|
const options = { force: opts.force, tenantId: ctx.tenantId, embed: ctx.infra };
|
|
23
|
-
return
|
|
25
|
+
return withDetached(ctx, (detached) => shareMemory(ctx, target, id, { ...options, detached }));
|
|
24
26
|
}
|
|
25
27
|
/** The caller's memories whose transfer score reaches `minScore`, copied to the global store
|
|
26
28
|
* unless `dryRun`. */
|
|
27
29
|
export async function autoShareToGlobal(ctx, opts) {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
embed: ctx.infra,
|
|
32
|
-
});
|
|
30
|
+
const target = requireGlobalTarget(ctx, 'auto-share');
|
|
31
|
+
const options = { ...opts, tenantId: ctx.tenantId, embed: ctx.infra };
|
|
32
|
+
return withDetached(ctx, (detached) => autoShare(ctx, target, { ...options, detached }));
|
|
33
33
|
}
|
|
34
34
|
/** The projects that shared into the global store: the caller's tenant, or every tenant for the
|
|
35
35
|
* host's operator. */
|
|
@@ -42,6 +42,8 @@ export async function sharePeers(ctx, opts = {}) {
|
|
|
42
42
|
* `includeCrossProject`. */
|
|
43
43
|
export async function syncFromGlobal(ctx, opts) {
|
|
44
44
|
const local = { store: ctx.store, config: ctx.config, embedderFor: ctx.infra.embedderFor };
|
|
45
|
-
|
|
45
|
+
const target = requireGlobalTarget(ctx, 'sync');
|
|
46
|
+
const options = { ...opts, embed: ctx.infra };
|
|
47
|
+
return withDetached(ctx, (detached) => syncGlobalToLocal(local, target, { ...options, detached }));
|
|
46
48
|
}
|
|
47
49
|
//# sourceMappingURL=sharing.js.map
|
|
@@ -20,7 +20,7 @@ export declare function autoSleepDue(newSince: (tenantId: string, now: Date) =>
|
|
|
20
20
|
* only the host tenant's writes start one, one per store at a time. Never rejects: the write it
|
|
21
21
|
* follows has landed.
|
|
22
22
|
*/
|
|
23
|
-
export declare function autoSleepAfterWrite(ctx: Pick<Context, 'tenantId' | 'config' | 'callers' | 'infra'>): Promise<void>;
|
|
23
|
+
export declare function autoSleepAfterWrite(ctx: Pick<Context, 'tenantId' | 'config' | 'callers' | 'infra' | 'detached'>): Promise<void>;
|
|
24
24
|
/** What one auto-sleep runs: the consolidation alone, its skipped passes logged by the process
|
|
25
25
|
* that ran it. */
|
|
26
26
|
export declare function autoSleepConsolidate(ctx: Pick<ContextWith<'consolidation' | 'learning' | 'lessonWrites' | 'auditMaintenance' | 'vectors'>, 'store' | 'config' | 'tenantId' | 'infra'>): Promise<ConsolidationResult>;
|
|
@@ -4,6 +4,7 @@ import { errorFields, errorMessage, log } from '../../base/util/log.js';
|
|
|
4
4
|
import { logPhaseFailures } from '../../domain/consolidate/run.js';
|
|
5
5
|
import { runConsolidation } from '../../domain/consolidate/sleep.js';
|
|
6
6
|
import { strengthOf } from '../types.js';
|
|
7
|
+
import { withDetached } from '../detached.js';
|
|
7
8
|
/** Counts only when enabled: the count reads the store. */
|
|
8
9
|
export async function autoSleepDue(newSince, tenantId, config, now) {
|
|
9
10
|
const { enabled, threshold } = config;
|
|
@@ -37,9 +38,9 @@ export async function autoSleepAfterWrite(ctx) {
|
|
|
37
38
|
}
|
|
38
39
|
if (!due)
|
|
39
40
|
return;
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
host
|
|
41
|
+
// The root's keeper holds the sleep so its stores outlast it; a context with no keeper waits for
|
|
42
|
+
// it. A failure is logged here once, with the tenant.
|
|
43
|
+
const sleep = host
|
|
43
44
|
.run()
|
|
44
45
|
.catch((err) => {
|
|
45
46
|
log.error(`auto-sleep consolidate failed (tenant ${tenantId}): ${errorMessage(err)}`, errorFields(err));
|
|
@@ -47,6 +48,7 @@ export async function autoSleepAfterWrite(ctx) {
|
|
|
47
48
|
.finally(() => {
|
|
48
49
|
host.latch.held = false;
|
|
49
50
|
});
|
|
51
|
+
await withDetached(ctx, (detached) => detached.detach(sleep));
|
|
50
52
|
}
|
|
51
53
|
/** What one auto-sleep runs: the consolidation alone, its skipped passes logged by the process
|
|
52
54
|
* that ran it. */
|
package/dist/api/sleep/learn.js
CHANGED
|
@@ -4,6 +4,7 @@ import { ForbiddenError } from '../../base/errors/api-errors.js';
|
|
|
4
4
|
import { fetchGitLog, extractLessons, partitionLessons } from '../../domain/learn/autolearn.js';
|
|
5
5
|
import { isGitRepo } from '../../base/ports/git.js';
|
|
6
6
|
import { embedMemoryDetached } from '../../domain/search/embed-memories.js';
|
|
7
|
+
import { withDetached } from '../detached.js';
|
|
7
8
|
import { extractInvalidationTarget, invalidateMatching, invalidationMatcher, invalidateMatchingAmong, } from '../../domain/learn/invalidation.js';
|
|
8
9
|
import { createMemory, ERROR_TAG, Layer, schemaFitFrom, } from '../../base/memory/entry.js';
|
|
9
10
|
import { extractPathTags } from '../../domain/search/path-context.js';
|
|
@@ -85,8 +86,10 @@ async function landLessons(batch, queued) {
|
|
|
85
86
|
strength: strengthOf(ctx),
|
|
86
87
|
clock,
|
|
87
88
|
};
|
|
88
|
-
|
|
89
|
-
|
|
89
|
+
await withDetached(ctx, (detached) => {
|
|
90
|
+
for (const entry of landed)
|
|
91
|
+
embedMemoryDetached(target, entry, detached);
|
|
92
|
+
});
|
|
90
93
|
}
|
|
91
94
|
/** The keys of the rows that may hold a lesson, the only rows that can be its copy. */
|
|
92
95
|
async function knownKeys(ctx, writes, lessons) {
|
|
@@ -10,6 +10,7 @@ import { extractGraphChunked } from '../../domain/graph/extract.js';
|
|
|
10
10
|
import { strengthOf } from '../types.js';
|
|
11
11
|
import { auditRemoveErrors } from '../auth/audit.js';
|
|
12
12
|
import { globalTargetOf } from '../infra.js';
|
|
13
|
+
import { withDetached } from '../detached.js';
|
|
13
14
|
import { errorFields, errorMessage, log } from '../../base/util/log.js';
|
|
14
15
|
import { ALL_TENANTS } from '../../base/util/tenant.js';
|
|
15
16
|
import { failedUnitOf } from '../../base/errors/failed-unit.js';
|
|
@@ -186,7 +187,7 @@ async function shareOnSleep(ctx, phases, result) {
|
|
|
186
187
|
// Both skip counters are surfaced so the secret veto and the rejection tombstone are observable,
|
|
187
188
|
// not silent.
|
|
188
189
|
const autoShareStats = { secretSkipped: 0, rejectedSkipped: 0 };
|
|
189
|
-
const shared = await phases.autoShare({
|
|
190
|
+
const shared = await withDetached(ctx, (detached) => phases.autoShare({
|
|
190
191
|
store: {
|
|
191
192
|
sharing,
|
|
192
193
|
folderFiles,
|
|
@@ -197,7 +198,8 @@ async function shareOnSleep(ctx, phases, result) {
|
|
|
197
198
|
minScore: AUTO_SHARE_MIN_SCORE,
|
|
198
199
|
stats: autoShareStats,
|
|
199
200
|
embed: ctx.infra,
|
|
200
|
-
|
|
201
|
+
detached,
|
|
202
|
+
}));
|
|
201
203
|
if (shared.length > 0) {
|
|
202
204
|
result.shared = shared.length;
|
|
203
205
|
}
|
package/dist/api/types.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ import type { InfraPorts } from '../base/ports/infra.js';
|
|
|
8
8
|
import type { FailureNotices } from '../base/util/failure-notices.js';
|
|
9
9
|
import type { StrengthModel } from '../base/memory/strength.js';
|
|
10
10
|
import type { EmbedderFor } from '../base/ports/embedder.js';
|
|
11
|
+
import type { DetachedWork } from '../domain/search/detached-work.js';
|
|
11
12
|
/** Actor identity + role for a Context: the audit-log subject (pass `ctx.actor.subject` to audit helpers) and the role
|
|
12
13
|
* for admin gating.
|
|
13
14
|
* Role checks happen at the request boundary (e.g. /v1/sleep), except in authCreate and authRevoke (ForbiddenError).
|
|
@@ -74,6 +75,9 @@ export interface Context {
|
|
|
74
75
|
* for. Set by a server or stdio root; the deadline itself rides in the store, whose worker
|
|
75
76
|
* calls wait on it. */
|
|
76
77
|
readonly signal?: AbortSignal | undefined;
|
|
78
|
+
/** Where the root keeps work a call starts and does not wait on, such as an embed, and settles it
|
|
79
|
+
* before closing its stores. Unset, the call waits for that work itself before it returns. */
|
|
80
|
+
readonly detached?: DetachedWork | undefined;
|
|
77
81
|
}
|
|
78
82
|
/** The strength model `ctx` scores by: its root's eval switches with its config's loss-aversion
|
|
79
83
|
* ratio. */
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare class DetachedSet {
|
|
2
|
+
#private;
|
|
3
|
+
/** Keeps `work` until settle(); a rejection nothing caught is logged here, never dropped. */
|
|
4
|
+
detach(work: Promise<unknown>): void;
|
|
5
|
+
/** Waits for every kept promise, work detached meanwhile included, up to `withinMs`; returns how many still run. */
|
|
6
|
+
settle(withinMs?: number): Promise<number>;
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=detached-set.d.ts.map
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// Work a caller starts and does not wait on, kept by whoever owns the stores it writes to, so that
|
|
2
|
+
// owner settles it before closing them.
|
|
3
|
+
import { errorFields, errorMessage, log } from './log.js';
|
|
4
|
+
export class DetachedSet {
|
|
5
|
+
#work = new Set();
|
|
6
|
+
/** Keeps `work` until settle(); a rejection nothing caught is logged here, never dropped. */
|
|
7
|
+
detach(work) {
|
|
8
|
+
const kept = work
|
|
9
|
+
.then(() => undefined, (err) => {
|
|
10
|
+
log.error(`detached work failed: ${errorMessage(err)}`, errorFields(err));
|
|
11
|
+
})
|
|
12
|
+
.finally(() => this.#work.delete(kept));
|
|
13
|
+
this.#work.add(kept);
|
|
14
|
+
}
|
|
15
|
+
/** Waits for every kept promise, work detached meanwhile included, up to `withinMs`; returns how many still run. */
|
|
16
|
+
async settle(withinMs) {
|
|
17
|
+
const drained = this.#drain();
|
|
18
|
+
if (withinMs === undefined) {
|
|
19
|
+
await drained;
|
|
20
|
+
return 0;
|
|
21
|
+
}
|
|
22
|
+
let bound;
|
|
23
|
+
const timedOut = new Promise((resolve) => {
|
|
24
|
+
bound = setTimeout(resolve, withinMs);
|
|
25
|
+
});
|
|
26
|
+
await Promise.race([drained, timedOut]);
|
|
27
|
+
clearTimeout(bound);
|
|
28
|
+
return this.#work.size;
|
|
29
|
+
}
|
|
30
|
+
async #drain() {
|
|
31
|
+
while (this.#work.size > 0)
|
|
32
|
+
await Promise.all(this.#work);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=detached-set.js.map
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Single source of truth for the package version, written by scripts/sync-version.mjs; never
|
|
3
3
|
* edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
|
|
4
4
|
* package.json read: the published bundle may lack package.json on a path ESM can resolve. */
|
|
5
|
-
export declare const PACKAGE_VERSION = "3.
|
|
5
|
+
export declare const PACKAGE_VERSION = "3.1.0";
|
|
6
6
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by
|
|
7
7
|
* hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
|
|
8
8
|
export declare const EXPIRING_KEYS_MIN_BINARY = "1.64.0";
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Single source of truth for the package version, written by scripts/sync-version.mjs; never
|
|
3
3
|
* edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
|
|
4
4
|
* package.json read: the published bundle may lack package.json on a path ESM can resolve. */
|
|
5
|
-
export const PACKAGE_VERSION = '3.
|
|
5
|
+
export const PACKAGE_VERSION = '3.1.0';
|
|
6
6
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by
|
|
7
7
|
* hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
|
|
8
8
|
export const EXPIRING_KEYS_MIN_BINARY = '1.64.0';
|
|
@@ -37,9 +37,16 @@ export declare class StoreRuntime {
|
|
|
37
37
|
giveIdle(poolKey: string, db: DatabaseSyncLike): boolean;
|
|
38
38
|
/** Closes every idle handle, so a held connection can be the store's last. */
|
|
39
39
|
closeIdle(): void;
|
|
40
|
-
/**
|
|
41
|
-
|
|
42
|
-
|
|
40
|
+
/** Keeps work nothing waits on, such as a command's embed, until dispose settles it. */
|
|
41
|
+
detach(work: Promise<unknown>): void;
|
|
42
|
+
/** Waits for the detached work, up to `withinMs` when given, for a root that closes its stores
|
|
43
|
+
* itself; resolves with how many were still running. */
|
|
44
|
+
settle(withinMs?: number): Promise<number>;
|
|
45
|
+
/** Settles the detached work, which may still write through the stores, then runs closeNow. */
|
|
46
|
+
dispose(): Promise<void>;
|
|
47
|
+
/** Runs the held writes, then closes the open sessions and the idle handles, each step even if
|
|
48
|
+
* the one before throws; it skips detached work, so only a root that cannot wait calls it. */
|
|
49
|
+
closeNow(): void;
|
|
43
50
|
}
|
|
44
51
|
export {};
|
|
45
52
|
//# sourceMappingURL=store-runtime.d.ts.map
|
package/dist/db/store-runtime.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import * as path from 'node:path';
|
|
6
6
|
import { errorFields, errorMessage, log } from '../base/util/log.js';
|
|
7
7
|
import { HeldWrites } from './held-writes.js';
|
|
8
|
+
import { DetachedSet } from '../base/util/detached-set.js';
|
|
8
9
|
// Per pool key: a server answers several requests at once, and each closing session hands its
|
|
9
10
|
// handle back.
|
|
10
11
|
const IDLE_PER_KEY = 4;
|
|
@@ -20,6 +21,7 @@ export class StoreRuntime {
|
|
|
20
21
|
#idle = new Map();
|
|
21
22
|
#background = new Set();
|
|
22
23
|
#stats = new Map();
|
|
24
|
+
#detached = new DetachedSet();
|
|
23
25
|
/** Whether `step` has run to its end on the store at `key`. */
|
|
24
26
|
hasRun(key, step) {
|
|
25
27
|
return this.#settled.get(key)?.has(step) ?? false;
|
|
@@ -96,9 +98,23 @@ export class StoreRuntime {
|
|
|
96
98
|
closeQuietly(db);
|
|
97
99
|
this.#idle.clear();
|
|
98
100
|
}
|
|
99
|
-
/**
|
|
100
|
-
|
|
101
|
-
|
|
101
|
+
/** Keeps work nothing waits on, such as a command's embed, until dispose settles it. */
|
|
102
|
+
detach(work) {
|
|
103
|
+
this.#detached.detach(work);
|
|
104
|
+
}
|
|
105
|
+
/** Waits for the detached work, up to `withinMs` when given, for a root that closes its stores
|
|
106
|
+
* itself; resolves with how many were still running. */
|
|
107
|
+
settle(withinMs) {
|
|
108
|
+
return this.#detached.settle(withinMs);
|
|
109
|
+
}
|
|
110
|
+
/** Settles the detached work, which may still write through the stores, then runs closeNow. */
|
|
111
|
+
async dispose() {
|
|
112
|
+
await this.settle();
|
|
113
|
+
this.closeNow();
|
|
114
|
+
}
|
|
115
|
+
/** Runs the held writes, then closes the open sessions and the idle handles, each step even if
|
|
116
|
+
* the one before throws; it skips detached work, so only a root that cannot wait calls it. */
|
|
117
|
+
closeNow() {
|
|
102
118
|
try {
|
|
103
119
|
this.held.flush();
|
|
104
120
|
}
|
|
@@ -7,6 +7,7 @@ import type { StoreWith } from '../../base/ports/store.js';
|
|
|
7
7
|
import type { FailureNotices } from '../../base/util/failure-notices.js';
|
|
8
8
|
import { type IndexEmbedTarget } from './embed-index-target.js';
|
|
9
9
|
import { type StoreEmbedTarget } from './embed-store-target.js';
|
|
10
|
+
import type { DetachedWork } from './detached-work.js';
|
|
10
11
|
/** `off` is a provider that is disabled or not installed, which is a choice; `failed` is a
|
|
11
12
|
* memory that stays out of the vector index. */
|
|
12
13
|
export type EmbedOutcome = 'indexed' | 'off' | 'failed';
|
|
@@ -39,10 +40,9 @@ export interface EmbedAdapter {
|
|
|
39
40
|
* through its vectorWrites. Resolves whatever fails, so a caller that fires and forgets stays
|
|
40
41
|
* safe; the outcome says whether the memory reached the index. */
|
|
41
42
|
export declare function embedMemoryInto(on: EmbedTarget | undefined, entry: MemoryEntry, model?: string): Promise<EmbedOutcome>;
|
|
42
|
-
/** embedMemory for a write that does not wait on its vector
|
|
43
|
-
*
|
|
44
|
-
|
|
45
|
-
export declare function embedMemoryDetached(on: EmbedTarget | undefined, entry: MemoryEntry): void;
|
|
43
|
+
/** embedMemory for a write that does not wait on its vector: `detached` keeps the embed until its
|
|
44
|
+
* root settles it. A rejection outside embedMemory's own handling is logged here. */
|
|
45
|
+
export declare function embedMemoryDetached(on: EmbedTarget | undefined, entry: MemoryEntry, detached: DetachedWork): void;
|
|
46
46
|
/** Embeds every entry with no cached vector, prunes vectors of deleted memories, and returns how
|
|
47
47
|
* many it embedded; `provider` defaults to the one the target's embedder builds. With a store,
|
|
48
48
|
* every tenant's memories with no vector for the provider's model go through its `vectorWrites`. */
|
|
@@ -42,11 +42,10 @@ export async function embedMemoryInto(on, entry, model) {
|
|
|
42
42
|
return 'off';
|
|
43
43
|
return adapterFor(on).one(provider, entry);
|
|
44
44
|
}
|
|
45
|
-
/** embedMemory for a write that does not wait on its vector
|
|
46
|
-
*
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
embedMemoryInto(on, entry).catch((err) => log.warn(`embedding ${entry.id} failed unexpectedly`, errorFields(err)));
|
|
45
|
+
/** embedMemory for a write that does not wait on its vector: `detached` keeps the embed until its
|
|
46
|
+
* root settles it. A rejection outside embedMemory's own handling is logged here. */
|
|
47
|
+
export function embedMemoryDetached(on, entry, detached) {
|
|
48
|
+
detached.detach(embedMemoryInto(on, entry).catch((err) => log.warn(`embedding ${entry.id} failed unexpectedly`, errorFields(err))));
|
|
50
49
|
}
|
|
51
50
|
/** Throws when an unavailable provider is a misconfiguration rather than an intentional no-op. */
|
|
52
51
|
function throwIfProviderKeyMissing(cfg, provider) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type EmbedRun } from '../search/embed-memories.js';
|
|
2
|
+
import type { DetachedWork } from '../search/detached-work.js';
|
|
2
3
|
import type { ConfiguredFolder } from '../../base/config/configured-folder.js';
|
|
3
4
|
import { type SharingStore } from './text-keys.js';
|
|
4
5
|
/**
|
|
@@ -9,6 +10,11 @@ export declare function syncGlobalToLocal(into: ConfiguredFolder<SharingStore>,
|
|
|
9
10
|
readonly store: SharingStore;
|
|
10
11
|
}, opts: {
|
|
11
12
|
includeCrossProject?: boolean;
|
|
12
|
-
|
|
13
|
-
|
|
13
|
+
} & SyncEmbed): Promise<number>;
|
|
14
|
+
/** The root's ports that score the copies, and the keeper of the embed that follows them. */
|
|
15
|
+
interface SyncEmbed {
|
|
16
|
+
readonly embed: EmbedRun;
|
|
17
|
+
readonly detached: DetachedWork;
|
|
18
|
+
}
|
|
19
|
+
export {};
|
|
14
20
|
//# sourceMappingURL=global-sync.d.ts.map
|
|
@@ -22,7 +22,7 @@ export async function syncGlobalToLocal(into, global, opts) {
|
|
|
22
22
|
if (!(await global.store.folderFiles.home()).exists)
|
|
23
23
|
return 0;
|
|
24
24
|
const rules = { project: home.project, includeCrossProject: opts.includeCrossProject };
|
|
25
|
-
return copyMissingRows(into, global.store, rules, opts
|
|
25
|
+
return copyMissingRows(into, global.store, rules, opts);
|
|
26
26
|
}
|
|
27
27
|
const textKey = (e) => `${e.tenantId}\n${e.content}`;
|
|
28
28
|
/** Global rows of every tenant that the local store holds by neither id nor tenant and text,
|
|
@@ -78,7 +78,7 @@ async function refusedTexts(store, { copies, twins }) {
|
|
|
78
78
|
}
|
|
79
79
|
/** One transaction: id, tenant and text decide which rows to read whole, and the copies commit
|
|
80
80
|
* together; each copy keeps its own tenant. */
|
|
81
|
-
async function copyMissingRows(into, from, rules,
|
|
81
|
+
async function copyMissingRows(into, from, rules, embedding) {
|
|
82
82
|
const candidates = await missingRows(into.store.sharing, from.sharing);
|
|
83
83
|
if (candidates.length === 0)
|
|
84
84
|
return 0;
|
|
@@ -98,17 +98,17 @@ async function copyMissingRows(into, from, rules, embed) {
|
|
|
98
98
|
const written = copies.length === 0
|
|
99
99
|
? []
|
|
100
100
|
: await into.store.lessonWrites.writeSkippingRejected(copies, CLI_ACTOR);
|
|
101
|
-
finishSyncDown(into, written.length, copies.length - written.length,
|
|
101
|
+
finishSyncDown(into, written.length, copies.length - written.length, embedding);
|
|
102
102
|
return written.length;
|
|
103
103
|
}
|
|
104
|
-
function finishSyncDown(into, count, rejected, embed) {
|
|
104
|
+
function finishSyncDown(into, count, rejected, { embed, detached }) {
|
|
105
105
|
if (rejected > 0) {
|
|
106
106
|
log.warn(`syncGlobalToLocal: skipped ${rejected} rejected value(s) (run \`hippo unreject\` on the local store to allow).`);
|
|
107
107
|
}
|
|
108
108
|
// Batch producer: one embedAll() on the destination rather than an embedMemory() per copied row
|
|
109
109
|
// (same batching invariant as autoShare).
|
|
110
110
|
if (count > 0) {
|
|
111
|
-
|
|
111
|
+
detached.detach(embedAll(folderEmbedTarget(into, embed)).catch((err) => logEmbedAllFailure('syncGlobalToLocal', err)));
|
|
112
112
|
}
|
|
113
113
|
}
|
|
114
114
|
//# sourceMappingURL=global-sync.js.map
|
|
@@ -2,6 +2,7 @@ import { MemoryEntry } from '../../base/memory/entry.js';
|
|
|
2
2
|
import { type EmbedRun } from '../search/embed-memories.js';
|
|
3
3
|
import type { ConfiguredFolder } from '../../base/config/configured-folder.js';
|
|
4
4
|
import { type TenantId } from '../../base/util/tenant.js';
|
|
5
|
+
import type { DetachedWork } from '../search/detached-work.js';
|
|
5
6
|
import type { SharingStore } from './text-keys.js';
|
|
6
7
|
import type { RequiredStore, StoreGroups, StoreWith } from '../../base/ports/store.js';
|
|
7
8
|
/** Tags whose rows only a hand-run share or promote may copy to the global store; derived rows
|
|
@@ -26,6 +27,7 @@ export declare function shareMemory(local: {
|
|
|
26
27
|
tenantId: string;
|
|
27
28
|
skipEmbed?: boolean;
|
|
28
29
|
embed: EmbedRun;
|
|
30
|
+
detached: DetachedWork;
|
|
29
31
|
}): Promise<MemoryEntry | null>;
|
|
30
32
|
/** List all projects that contributed memories to the global store, parsed from
|
|
31
33
|
* 'shared:<project>:' or 'promoted:<path>' sources; ALL_TENANTS is the host-wide view. */
|
|
@@ -54,6 +56,8 @@ export declare function autoShare(from: {
|
|
|
54
56
|
tenantId: TenantId;
|
|
55
57
|
stats?: AutoShareStats;
|
|
56
58
|
embed: EmbedRun;
|
|
59
|
+
/** Keeps the batch embed of the shared rows until the root settles it. */
|
|
60
|
+
detached: DetachedWork;
|
|
57
61
|
}): Promise<MemoryEntry[]>;
|
|
58
62
|
export {};
|
|
59
63
|
//# sourceMappingURL=share.d.ts.map
|
|
@@ -131,7 +131,7 @@ export async function shareMemory(local, global, id, options) {
|
|
|
131
131
|
strength: strengthModelFor(options.embed.evalSwitches, global.config.runtime),
|
|
132
132
|
clock: options.embed.clock,
|
|
133
133
|
};
|
|
134
|
-
embedMemoryDetached(target, globalEntry);
|
|
134
|
+
embedMemoryDetached(target, globalEntry, options.detached);
|
|
135
135
|
}
|
|
136
136
|
return globalEntry;
|
|
137
137
|
}
|
|
@@ -313,7 +313,7 @@ export async function autoShare(from, global, options) {
|
|
|
313
313
|
now: options.embed.clock(),
|
|
314
314
|
});
|
|
315
315
|
if (shared.length > 0) {
|
|
316
|
-
|
|
316
|
+
options.detached.detach(embedAll(folderEmbedTarget(global, options.embed)).catch((err) => logEmbedAllFailure('autoShare', err)));
|
|
317
317
|
}
|
|
318
318
|
return shared;
|
|
319
319
|
}
|
|
@@ -36,4 +36,5 @@ export declare const ingestEvent: (ctx: EntryContext, input: import("../surface/
|
|
|
36
36
|
export declare const ingestMessage: (ctx: EntryContext, input: import("../surface/connectors/slack/ingest.js").IngestInput) => Promise<import("../surface/connectors/ingest.js").IngestResult>;
|
|
37
37
|
export { adminActor, goalPush, outcome } from '../api/index.js';
|
|
38
38
|
export type { EntryContext as Context } from '../api/index.js';
|
|
39
|
+
export type { DetachedWork } from '../domain/search/detached-work.js';
|
|
39
40
|
//# sourceMappingURL=eval-scripts.d.ts.map
|