hippo-memory 3.0.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/api/context/context-plan.js +2 -4
- package/dist/api/context/context-select.js +12 -7
- package/dist/api/detached.d.ts +6 -0
- package/dist/api/detached.js +16 -0
- package/dist/api/embeddings.d.ts +4 -3
- package/dist/api/embeddings.js +4 -4
- package/dist/api/memories/promote.js +2 -1
- package/dist/api/memories/remember-local.d.ts +5 -4
- package/dist/api/memories/remember-local.js +13 -10
- package/dist/api/recall/rank-stages.js +2 -2
- package/dist/api/session-end.js +1 -0
- package/dist/api/sharing.js +9 -7
- package/dist/api/sleep/auto-sleep.d.ts +1 -1
- package/dist/api/sleep/auto-sleep.js +5 -3
- package/dist/api/sleep/learn.js +5 -2
- package/dist/api/sleep/sleep-run.js +4 -2
- package/dist/api/types.d.ts +4 -0
- package/dist/base/memory/entry.js +2 -2
- package/dist/base/memory/markdown-delimiters.d.ts +15 -0
- package/dist/base/memory/markdown-delimiters.js +94 -0
- package/dist/base/memory/memory-audit.js +4 -4
- package/dist/base/memory/memory-quality.d.ts +12 -0
- package/dist/base/memory/memory-quality.js +92 -3
- package/dist/base/ports/store/types.d.ts +2 -1
- package/dist/base/rank/prompt-recall.d.ts +16 -3
- package/dist/base/rank/prompt-recall.js +47 -7
- package/dist/base/util/detached-set.d.ts +8 -0
- package/dist/base/util/detached-set.js +35 -0
- package/dist/base/util/tokenize.d.ts +29 -0
- package/dist/base/util/tokenize.js +68 -5
- package/dist/base/util/version.d.ts +1 -1
- package/dist/base/util/version.js +1 -1
- package/dist/db/meta.d.ts +1 -0
- package/dist/db/meta.js +1 -0
- package/dist/db/store-runtime.d.ts +10 -3
- package/dist/db/store-runtime.js +19 -3
- package/dist/domain/consolidate/conflicts.js +2 -2
- package/dist/domain/learn/forward-claim-detector.js +4 -2
- package/dist/domain/learn/invalidation.js +5 -2
- package/dist/domain/search/bm25.js +3 -3
- package/dist/domain/search/detached-work.d.ts +6 -0
- package/dist/domain/search/detached-work.js +2 -0
- package/dist/domain/search/embed-memories.d.ts +4 -4
- package/dist/domain/search/embed-memories.js +4 -5
- package/dist/domain/sharing/global-sync.d.ts +8 -2
- package/dist/domain/sharing/global-sync.js +5 -5
- package/dist/domain/sharing/share.d.ts +4 -0
- package/dist/domain/sharing/share.js +2 -2
- package/dist/entry/eval-scripts.d.ts +1 -0
- package/dist/entry/sleep.d.ts +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/server.d.ts +1 -0
- package/dist/store/rules/windows.d.ts +3 -0
- package/dist/store/rules/windows.js +3 -0
- package/dist/store/sqlite/audit/audit-maintenance.js +16 -13
- package/dist/store/sqlite/recall/candidates.js +1 -0
- package/dist/store/sqlite/recall/search-rows.d.ts +2 -2
- package/dist/store/sqlite/recall/search-rows.js +62 -26
- package/dist/store/sqlite/runtime/server-runtime.d.ts +6 -1
- package/dist/store/sqlite/runtime/server-runtime.js +15 -3
- package/dist/store/sqlite/runtime/worker-jobs.js +2 -2
- package/dist/store/sqlite/sharing/copy-rows.d.ts +1 -0
- package/dist/store/sqlite/sharing/copy-rows.js +2 -1
- package/dist/surface/capture/command.d.ts +8 -1
- package/dist/surface/capture/command.js +38 -8
- package/dist/surface/capture/compact.d.ts +3 -3
- package/dist/surface/capture/compact.js +4 -4
- package/dist/surface/capture/extract.d.ts +10 -0
- package/dist/surface/capture/extract.js +199 -26
- package/dist/surface/capture/session-texts.js +11 -11
- package/dist/surface/cli/api-context.d.ts +5 -0
- package/dist/surface/cli/api-context.js +12 -0
- package/dist/surface/cli/goals.js +6 -8
- package/dist/surface/cli/hook-runtime.js +2 -2
- package/dist/surface/cli/memory-import.js +20 -15
- package/dist/surface/mcp/protocol.d.ts +6 -0
- package/dist/surface/mcp/protocol.js +1 -0
- package/dist/surface/mcp/stdio.js +11 -5
- package/dist/surface/open-store.d.ts +2 -2
- package/dist/surface/open-store.js +11 -7
- package/dist/surface/server/auth.d.ts +1 -1
- package/dist/surface/server/auth.js +13 -2
- package/dist/surface/server/boot.js +3 -2
- package/dist/surface/server/dashboard-listener.js +2 -2
- package/dist/surface/server/lifecycle.d.ts +2 -0
- package/dist/surface/server/lifecycle.js +4 -2
- package/dist/surface/server/mcp-http.js +1 -0
- package/dist/surface/server/types.d.ts +3 -0
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
|
@@ -7,6 +7,7 @@ import type { StoreWith } from '../../base/ports/store.js';
|
|
|
7
7
|
import type { FailureNotices } from '../../base/util/failure-notices.js';
|
|
8
8
|
import { type IndexEmbedTarget } from './embed-index-target.js';
|
|
9
9
|
import { type StoreEmbedTarget } from './embed-store-target.js';
|
|
10
|
+
import type { DetachedWork } from './detached-work.js';
|
|
10
11
|
/** `off` is a provider that is disabled or not installed, which is a choice; `failed` is a
|
|
11
12
|
* memory that stays out of the vector index. */
|
|
12
13
|
export type EmbedOutcome = 'indexed' | 'off' | 'failed';
|
|
@@ -39,10 +40,9 @@ export interface EmbedAdapter {
|
|
|
39
40
|
* through its vectorWrites. Resolves whatever fails, so a caller that fires and forgets stays
|
|
40
41
|
* safe; the outcome says whether the memory reached the index. */
|
|
41
42
|
export declare function embedMemoryInto(on: EmbedTarget | undefined, entry: MemoryEntry, model?: string): Promise<EmbedOutcome>;
|
|
42
|
-
/** embedMemory for a write that does not wait on its vector
|
|
43
|
-
*
|
|
44
|
-
|
|
45
|
-
export declare function embedMemoryDetached(on: EmbedTarget | undefined, entry: MemoryEntry): void;
|
|
43
|
+
/** embedMemory for a write that does not wait on its vector: `detached` keeps the embed until its
|
|
44
|
+
* root settles it. A rejection outside embedMemory's own handling is logged here. */
|
|
45
|
+
export declare function embedMemoryDetached(on: EmbedTarget | undefined, entry: MemoryEntry, detached: DetachedWork): void;
|
|
46
46
|
/** Embeds every entry with no cached vector, prunes vectors of deleted memories, and returns how
|
|
47
47
|
* many it embedded; `provider` defaults to the one the target's embedder builds. With a store,
|
|
48
48
|
* every tenant's memories with no vector for the provider's model go through its `vectorWrites`. */
|
|
@@ -42,11 +42,10 @@ export async function embedMemoryInto(on, entry, model) {
|
|
|
42
42
|
return 'off';
|
|
43
43
|
return adapterFor(on).one(provider, entry);
|
|
44
44
|
}
|
|
45
|
-
/** embedMemory for a write that does not wait on its vector
|
|
46
|
-
*
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
embedMemoryInto(on, entry).catch((err) => log.warn(`embedding ${entry.id} failed unexpectedly`, errorFields(err)));
|
|
45
|
+
/** embedMemory for a write that does not wait on its vector: `detached` keeps the embed until its
|
|
46
|
+
* root settles it. A rejection outside embedMemory's own handling is logged here. */
|
|
47
|
+
export function embedMemoryDetached(on, entry, detached) {
|
|
48
|
+
detached.detach(embedMemoryInto(on, entry).catch((err) => log.warn(`embedding ${entry.id} failed unexpectedly`, errorFields(err))));
|
|
50
49
|
}
|
|
51
50
|
/** Throws when an unavailable provider is a misconfiguration rather than an intentional no-op. */
|
|
52
51
|
function throwIfProviderKeyMissing(cfg, provider) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type EmbedRun } from '../search/embed-memories.js';
|
|
2
|
+
import type { DetachedWork } from '../search/detached-work.js';
|
|
2
3
|
import type { ConfiguredFolder } from '../../base/config/configured-folder.js';
|
|
3
4
|
import { type SharingStore } from './text-keys.js';
|
|
4
5
|
/**
|
|
@@ -9,6 +10,11 @@ export declare function syncGlobalToLocal(into: ConfiguredFolder<SharingStore>,
|
|
|
9
10
|
readonly store: SharingStore;
|
|
10
11
|
}, opts: {
|
|
11
12
|
includeCrossProject?: boolean;
|
|
12
|
-
|
|
13
|
-
|
|
13
|
+
} & SyncEmbed): Promise<number>;
|
|
14
|
+
/** The root's ports that score the copies, and the keeper of the embed that follows them. */
|
|
15
|
+
interface SyncEmbed {
|
|
16
|
+
readonly embed: EmbedRun;
|
|
17
|
+
readonly detached: DetachedWork;
|
|
18
|
+
}
|
|
19
|
+
export {};
|
|
14
20
|
//# sourceMappingURL=global-sync.d.ts.map
|
|
@@ -22,7 +22,7 @@ export async function syncGlobalToLocal(into, global, opts) {
|
|
|
22
22
|
if (!(await global.store.folderFiles.home()).exists)
|
|
23
23
|
return 0;
|
|
24
24
|
const rules = { project: home.project, includeCrossProject: opts.includeCrossProject };
|
|
25
|
-
return copyMissingRows(into, global.store, rules, opts
|
|
25
|
+
return copyMissingRows(into, global.store, rules, opts);
|
|
26
26
|
}
|
|
27
27
|
const textKey = (e) => `${e.tenantId}\n${e.content}`;
|
|
28
28
|
/** Global rows of every tenant that the local store holds by neither id nor tenant and text,
|
|
@@ -78,7 +78,7 @@ async function refusedTexts(store, { copies, twins }) {
|
|
|
78
78
|
}
|
|
79
79
|
/** One transaction: id, tenant and text decide which rows to read whole, and the copies commit
|
|
80
80
|
* together; each copy keeps its own tenant. */
|
|
81
|
-
async function copyMissingRows(into, from, rules,
|
|
81
|
+
async function copyMissingRows(into, from, rules, embedding) {
|
|
82
82
|
const candidates = await missingRows(into.store.sharing, from.sharing);
|
|
83
83
|
if (candidates.length === 0)
|
|
84
84
|
return 0;
|
|
@@ -98,17 +98,17 @@ async function copyMissingRows(into, from, rules, embed) {
|
|
|
98
98
|
const written = copies.length === 0
|
|
99
99
|
? []
|
|
100
100
|
: await into.store.lessonWrites.writeSkippingRejected(copies, CLI_ACTOR);
|
|
101
|
-
finishSyncDown(into, written.length, copies.length - written.length,
|
|
101
|
+
finishSyncDown(into, written.length, copies.length - written.length, embedding);
|
|
102
102
|
return written.length;
|
|
103
103
|
}
|
|
104
|
-
function finishSyncDown(into, count, rejected, embed) {
|
|
104
|
+
function finishSyncDown(into, count, rejected, { embed, detached }) {
|
|
105
105
|
if (rejected > 0) {
|
|
106
106
|
log.warn(`syncGlobalToLocal: skipped ${rejected} rejected value(s) (run \`hippo unreject\` on the local store to allow).`);
|
|
107
107
|
}
|
|
108
108
|
// Batch producer: one embedAll() on the destination rather than an embedMemory() per copied row
|
|
109
109
|
// (same batching invariant as autoShare).
|
|
110
110
|
if (count > 0) {
|
|
111
|
-
|
|
111
|
+
detached.detach(embedAll(folderEmbedTarget(into, embed)).catch((err) => logEmbedAllFailure('syncGlobalToLocal', err)));
|
|
112
112
|
}
|
|
113
113
|
}
|
|
114
114
|
//# sourceMappingURL=global-sync.js.map
|
|
@@ -2,6 +2,7 @@ import { MemoryEntry } from '../../base/memory/entry.js';
|
|
|
2
2
|
import { type EmbedRun } from '../search/embed-memories.js';
|
|
3
3
|
import type { ConfiguredFolder } from '../../base/config/configured-folder.js';
|
|
4
4
|
import { type TenantId } from '../../base/util/tenant.js';
|
|
5
|
+
import type { DetachedWork } from '../search/detached-work.js';
|
|
5
6
|
import type { SharingStore } from './text-keys.js';
|
|
6
7
|
import type { RequiredStore, StoreGroups, StoreWith } from '../../base/ports/store.js';
|
|
7
8
|
/** Tags whose rows only a hand-run share or promote may copy to the global store; derived rows
|
|
@@ -26,6 +27,7 @@ export declare function shareMemory(local: {
|
|
|
26
27
|
tenantId: string;
|
|
27
28
|
skipEmbed?: boolean;
|
|
28
29
|
embed: EmbedRun;
|
|
30
|
+
detached: DetachedWork;
|
|
29
31
|
}): Promise<MemoryEntry | null>;
|
|
30
32
|
/** List all projects that contributed memories to the global store, parsed from
|
|
31
33
|
* 'shared:<project>:' or 'promoted:<path>' sources; ALL_TENANTS is the host-wide view. */
|
|
@@ -54,6 +56,8 @@ export declare function autoShare(from: {
|
|
|
54
56
|
tenantId: TenantId;
|
|
55
57
|
stats?: AutoShareStats;
|
|
56
58
|
embed: EmbedRun;
|
|
59
|
+
/** Keeps the batch embed of the shared rows until the root settles it. */
|
|
60
|
+
detached: DetachedWork;
|
|
57
61
|
}): Promise<MemoryEntry[]>;
|
|
58
62
|
export {};
|
|
59
63
|
//# sourceMappingURL=share.d.ts.map
|
|
@@ -131,7 +131,7 @@ export async function shareMemory(local, global, id, options) {
|
|
|
131
131
|
strength: strengthModelFor(options.embed.evalSwitches, global.config.runtime),
|
|
132
132
|
clock: options.embed.clock,
|
|
133
133
|
};
|
|
134
|
-
embedMemoryDetached(target, globalEntry);
|
|
134
|
+
embedMemoryDetached(target, globalEntry, options.detached);
|
|
135
135
|
}
|
|
136
136
|
return globalEntry;
|
|
137
137
|
}
|
|
@@ -313,7 +313,7 @@ export async function autoShare(from, global, options) {
|
|
|
313
313
|
now: options.embed.clock(),
|
|
314
314
|
});
|
|
315
315
|
if (shared.length > 0) {
|
|
316
|
-
|
|
316
|
+
options.detached.detach(embedAll(folderEmbedTarget(global, options.embed)).catch((err) => logEmbedAllFailure('autoShare', err)));
|
|
317
317
|
}
|
|
318
318
|
return shared;
|
|
319
319
|
}
|
|
@@ -36,4 +36,5 @@ export declare const ingestEvent: (ctx: EntryContext, input: import("../surface/
|
|
|
36
36
|
export declare const ingestMessage: (ctx: EntryContext, input: import("../surface/connectors/slack/ingest.js").IngestInput) => Promise<import("../surface/connectors/ingest.js").IngestResult>;
|
|
37
37
|
export { adminActor, goalPush, outcome } from '../api/index.js';
|
|
38
38
|
export type { EntryContext as Context } from '../api/index.js';
|
|
39
|
+
export type { DetachedWork } from '../domain/search/detached-work.js';
|
|
39
40
|
//# sourceMappingURL=eval-scripts.d.ts.map
|
package/dist/entry/sleep.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ export type { SleepOptions, SleepResult } from '../api/sleep/sleep.js';
|
|
|
8
8
|
export type { SleepEvent } from '../api/sleep/events.js';
|
|
9
9
|
export type { RescueRank } from '../domain/consolidate/events.js';
|
|
10
10
|
export type { SleepContext, SleepStore };
|
|
11
|
+
export type { DetachedWork } from '../domain/search/detached-work.js';
|
|
11
12
|
/** Sleeps the whole store under `ctx`, whose store sets the groups a sleep pass works on. */
|
|
12
13
|
export declare function sleep(ctx: SleepContext, opts?: SleepOptions): Promise<SleepResult>;
|
|
13
14
|
/** consolidate's options as published: the LLM phases call the model ANTHROPIC_API_KEY opens,
|
package/dist/index.d.ts
CHANGED
|
@@ -4,6 +4,6 @@
|
|
|
4
4
|
* hippo-memory/importers and hippo-memory/eval.
|
|
5
5
|
*/
|
|
6
6
|
export { MemoryEntry, Layer, EmotionalValence, ConfidenceLevel, DecayOptions, calculateStrength, resolveConfidence, confidenceFacets, type ConfidenceFacets, applyOutcome, generateId, computeSchemaFit, createMemory, markRetrieved, estimateTokens, SessionHandoff, HandoffOutcome, HandoffEvidence, isHandoffOutcome, Card, CardStatus, CardRun, CardComment, CardTransitions, isCardStatus, CARD_TRANSITIONS, CARD_LEASE_MS, computeSalience, SalienceDecision, SalienceResult, SalienceOptions, computeAmbientState, renderAmbientSummary, formatAmbientVector, AmbientState, tokenize, textOverlap, isEmbeddingAvailable, getEmbedding, } from './entry/memory.js';
|
|
7
|
-
export { ConsolidationResult, type SleepOptions, type SleepResult, type SleepEvent, type RescueRank, type SleepContext, type SleepStore, sleep, consolidate, strengthBucket, } from './entry/sleep.js';
|
|
7
|
+
export { ConsolidationResult, type SleepOptions, type SleepResult, type SleepEvent, type RescueRank, type SleepContext, type SleepStore, type DetachedWork, sleep, consolidate, strengthBucket, } from './entry/sleep.js';
|
|
8
8
|
export { captureError, extractLessons, partitionLessons, getGlobalRoot, transferScore, } from './entry/learning.js';
|
|
9
9
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/server.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ export { clientIpForRateLimit } from './surface/server/client-ip.js';
|
|
|
14
14
|
export { isLoopback, isReservedActor } from './surface/server/auth.js';
|
|
15
15
|
export type { AddonCall, AddonRoute, AuthResolver, RateLimitSpec, ResolvedBearer, ServeOptions, ServerHandle, } from './surface/server/types.js';
|
|
16
16
|
export type { HippoStore } from './base/ports/store-index.js';
|
|
17
|
+
export type { DetachedWork } from './domain/search/detached-work.js';
|
|
17
18
|
export { HttpError } from './base/errors/http-errors.js';
|
|
18
19
|
export type { CallerProject } from './api/prompt-hook.js';
|
|
19
20
|
export type { JsonValue } from './base/util/json.js';
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
/** Newest admitted rows LIKE tests for a term under 3 characters, which no trigram lookup can
|
|
2
2
|
* answer. */
|
|
3
3
|
export declare const LIKE_WINDOW_ROWS = 2000;
|
|
4
|
+
/** Most terms LIKE tests on each of those rows: the trigram index looks up every longer term over
|
|
5
|
+
* the whole store, so a long CJK query's 1,000s of bigrams need not all be tested here. */
|
|
6
|
+
export declare const LIKE_WINDOW_TERMS = 64;
|
|
4
7
|
/** How many of a tenant's newest rows schemaFit reads; older rows no longer move the fit. */
|
|
5
8
|
export declare const SCHEMA_FIT_WINDOW_ROWS = 2000;
|
|
6
9
|
/** Events of one completed session a sleep reads to render its trace, the newest kept. */
|
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
/** Newest admitted rows LIKE tests for a term under 3 characters, which no trigram lookup can
|
|
4
4
|
* answer. */
|
|
5
5
|
export const LIKE_WINDOW_ROWS = 2000;
|
|
6
|
+
/** Most terms LIKE tests on each of those rows: the trigram index looks up every longer term over
|
|
7
|
+
* the whole store, so a long CJK query's 1,000s of bigrams need not all be tested here. */
|
|
8
|
+
export const LIKE_WINDOW_TERMS = 64;
|
|
6
9
|
/** How many of a tenant's newest rows schemaFit reads; older rows no longer move the fit. */
|
|
7
10
|
export const SCHEMA_FIT_WINDOW_ROWS = 2000;
|
|
8
11
|
/** Events of one completed session a sleep reads to render its trace, the newest kept. */
|
|
@@ -17,7 +17,8 @@ import { purgeMirrorBestEffort } from '../folder-files/mirrors.js';
|
|
|
17
17
|
import { openStore } from '../runtime/handles.js';
|
|
18
18
|
import { answered, bound, skipEmpty } from '../runtime/bound.js';
|
|
19
19
|
import { READ_PAGE_ROWS } from '../../../base/util/pages.js';
|
|
20
|
-
import { getMeta, pragmaUserVersion, setMeta, META_KEY } from '../../../db/meta.js';
|
|
20
|
+
import { getMeta, getMetaNumber, pragmaUserVersion, setMeta, META_KEY } from '../../../db/meta.js';
|
|
21
|
+
import { QUALITY_GATE_VERSION } from '../../../base/memory/memory-quality.js';
|
|
21
22
|
// Quality repair's issue list: which rows are defects and which of those a protection keeps.
|
|
22
23
|
/** The issues among `entries` in their order, each defect kept when `protect` names the row. */
|
|
23
24
|
function repairIssues(entries, parents, backing, protect) {
|
|
@@ -223,8 +224,8 @@ function applyPlan(db, { run, plan, backup, rules }) {
|
|
|
223
224
|
else
|
|
224
225
|
warnings.push(`Kept ${issue.id}: the store's delete guard protects it.`);
|
|
225
226
|
}
|
|
226
|
-
|
|
227
|
-
setMeta(db,
|
|
227
|
+
for (const [key, value] of run.doneWrites ?? [])
|
|
228
|
+
setMeta(db, key, value);
|
|
228
229
|
return { ...plan, issues, appliedIds, backup, warnings };
|
|
229
230
|
});
|
|
230
231
|
}
|
|
@@ -248,21 +249,23 @@ function repairOn(db, files, run) {
|
|
|
248
249
|
}
|
|
249
250
|
return { ...result, warnings };
|
|
250
251
|
}
|
|
251
|
-
|
|
252
|
+
// The old key stays '1' so a binary from before the gate version keeps skipping the repair.
|
|
253
|
+
const DONE_WRITES = [
|
|
254
|
+
[META_KEY.qualityRepairAuto, '1'],
|
|
255
|
+
[META_KEY.qualityRepairGate, String(QUALITY_GATE_VERSION)],
|
|
256
|
+
];
|
|
257
|
+
/** Applies the repair once per gate version, so stores that predate a quality check are cleaned on
|
|
252
258
|
* upgrade with no command; null once done. `db` is opened writable without running migrations. */
|
|
253
259
|
function repairQualityOnce(db, files, tenantId, protect) {
|
|
254
|
-
if (
|
|
260
|
+
if (getMetaNumber(db, META_KEY.qualityRepairGate) >= QUALITY_GATE_VERSION)
|
|
255
261
|
return null;
|
|
256
262
|
// The apply commits the flag with the moves, so a lock taken between them cannot hide what moved.
|
|
257
|
-
const result = repairOn(db, files, {
|
|
258
|
-
tenantId,
|
|
259
|
-
protect,
|
|
260
|
-
apply: true,
|
|
261
|
-
doneKey: META_KEY.qualityRepairAuto,
|
|
262
|
-
});
|
|
263
|
+
const result = repairOn(db, files, { tenantId, protect, apply: true, doneWrites: DONE_WRITES });
|
|
263
264
|
// An unsupported schema changed nothing, so the next run tries again once a migration has run.
|
|
264
|
-
if (result.supported && result.backup === null)
|
|
265
|
-
|
|
265
|
+
if (result.supported && result.backup === null) {
|
|
266
|
+
for (const [key, value] of DONE_WRITES)
|
|
267
|
+
setMeta(db, key, value);
|
|
268
|
+
}
|
|
266
269
|
return result;
|
|
267
270
|
}
|
|
268
271
|
export function sqliteAuditMaintenance({ on, files }) {
|
|
@@ -16,7 +16,7 @@ export declare function listRecallEntries(db: DatabaseSyncLike, query: string, o
|
|
|
16
16
|
export declare function listVectorCandidateEntries(db: DatabaseSyncLike, queryVector: readonly number[], spec: VectorCandidateSpec, index?: VectorIndex): Promise<MemoryEntry[]>;
|
|
17
17
|
/** The rows of `ids` in that order; an id deleted since it was matched is skipped. */
|
|
18
18
|
export declare function entriesInOrder(db: DatabaseSyncLike, ids: readonly string[]): MemoryEntry[];
|
|
19
|
-
/** Rarest-K prompt terms for this connection's FTS
|
|
20
|
-
*
|
|
19
|
+
/** Rarest-K prompt terms for this connection's FTS indexes, as a space-joined query string: words
|
|
20
|
+
* count in the word index, CJK terms in the trigram index. Without FTS, the first 32 terms. */
|
|
21
21
|
export declare function pickRarestFtsQuery(db: DatabaseSyncLike, terms: readonly string[], maxTerms?: number): string;
|
|
22
22
|
//# sourceMappingURL=search-rows.d.ts.map
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { isFtsAvailable } from '../../../db/index.js';
|
|
2
|
-
import { tokenize } from '../../../base/util/tokenize.js';
|
|
2
|
+
import { cjkIndexTerms, partitionCjkTerms, tokenize } from '../../../base/util/tokenize.js';
|
|
3
3
|
import { recallScopeFilter } from '../../../base/scope/recall-scope.js';
|
|
4
4
|
import { originInSql, scopeAdmitSql } from '../runtime/rule-sql.js';
|
|
5
|
-
import { ftsTermParts, RAREST_TERM_COUNT, rarestFtsQuery, } from '../../../base/rank/prompt-recall.js';
|
|
5
|
+
import { ftsTermParts, RAREST_TERM_COUNT, rarestFtsQuery, spreadCjkTerms, } from '../../../base/rank/prompt-recall.js';
|
|
6
6
|
import { errorMessage, log, errorFields } from '../../../base/util/log.js';
|
|
7
7
|
import { topVectorMatches } from '../../../db/vector-match.js';
|
|
8
8
|
import { MEMORY_SELECT_COLUMNS, MEMORY_SEARCH_COLUMNS, rowToEntry, } from '../runtime/rows.js';
|
|
@@ -14,7 +14,7 @@ import { placeholders } from '../../../db/sql.js';
|
|
|
14
14
|
import { VECTOR_CANDIDATE_LIMIT, } from '../../../base/ports/store/vectors.js';
|
|
15
15
|
import { ALL_TENANTS } from '../../../base/util/tenant.js';
|
|
16
16
|
import { tenantAnd } from '../runtime/tenant.js';
|
|
17
|
-
import { LIKE_WINDOW_ROWS } from '../../rules/windows.js';
|
|
17
|
+
import { LIKE_WINDOW_ROWS, LIKE_WINDOW_TERMS } from '../../rules/windows.js';
|
|
18
18
|
/** The recall scope rule for a table column prefix (`m.` or none); `passesScopeFilterForRecall`
|
|
19
19
|
* in recall-scope.ts is its JS twin. */
|
|
20
20
|
function recallScopeClause(col, scopeFilter) {
|
|
@@ -55,18 +55,40 @@ function listSearchRows(db, { query, limit, tenantId, options = {} }) {
|
|
|
55
55
|
// LIMIT here too, so every candidate path honours the caller's cap.
|
|
56
56
|
return listAllCandidates(db, p, limit);
|
|
57
57
|
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
const rows = listLikeCandidates(db, terms, p, limit);
|
|
58
|
+
// The run's trigrams reach the trigram index, which the 2-letter tokens cannot.
|
|
59
|
+
const { words, cjk } = partitionCjkTerms(terms);
|
|
60
|
+
const rows = interleaveRows(words.length > 0 ? listWordCandidates(db, words, p, limit) : [], cjk.length > 0
|
|
61
|
+
? listLikeCandidates(db, [...new Set([...cjk, ...cjkIndexTerms(query)])], p, limit)
|
|
62
|
+
: [], limit);
|
|
64
63
|
if (rows.length > 0)
|
|
65
64
|
return rows;
|
|
66
65
|
// LIMIT the full-store fallback too: RecallResult reports the scorer window, so an unbounded
|
|
67
66
|
// fallback would misstate the candidate-pool size.
|
|
68
67
|
return listAllCandidates(db, p, limit);
|
|
69
68
|
}
|
|
69
|
+
/** FTS5 matches for `terms`, else their substring matches. */
|
|
70
|
+
function listWordCandidates(db, terms, p, limit) {
|
|
71
|
+
if (isFtsAvailable(db)) {
|
|
72
|
+
const rows = listFtsCandidates(db, terms, p, limit);
|
|
73
|
+
if (rows.length > 0)
|
|
74
|
+
return rows;
|
|
75
|
+
}
|
|
76
|
+
return listLikeCandidates(db, terms, p, limit);
|
|
77
|
+
}
|
|
78
|
+
/** `a` and `b` taken in turn, each row once, up to `limit`; with one list empty, the other as it
|
|
79
|
+
* is. */
|
|
80
|
+
function interleaveRows(a, b, limit) {
|
|
81
|
+
if (a.length === 0 || b.length === 0)
|
|
82
|
+
return a.length === 0 ? b : a;
|
|
83
|
+
const out = new Map();
|
|
84
|
+
for (let i = 0; out.size < limit && i < Math.max(a.length, b.length); i++) {
|
|
85
|
+
for (const row of [a[i], b[i]]) {
|
|
86
|
+
if (row !== undefined && out.size < limit)
|
|
87
|
+
out.set(row.id, row);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return [...out.values()];
|
|
91
|
+
}
|
|
70
92
|
function searchPredicates(tenantId, scopeFilter, includeSuperseded, originProjects) {
|
|
71
93
|
// One tenant reads through the composite idx_memories_tenant_created.
|
|
72
94
|
const one = tenantId !== ALL_TENANTS;
|
|
@@ -164,21 +186,18 @@ function trigramArm(terms, p) {
|
|
|
164
186
|
/** LIKE over the newest admitted rows: the bound is what keeps a short term from reading the
|
|
165
187
|
* whole store. */
|
|
166
188
|
function windowArm(terms, p, limit) {
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
const likes = terms.flatMap((term) => {
|
|
171
|
-
const like = `%${escapeLike(term)}%`;
|
|
172
|
-
return [like, like];
|
|
173
|
-
});
|
|
189
|
+
// One JSON array, not an OR per term: a long CJK query has 1,000s of terms, past SQLite's
|
|
190
|
+
// expression depth.
|
|
191
|
+
const likes = JSON.stringify(terms.slice(0, LIKE_WINDOW_TERMS).map((term) => `%${escapeLike(term)}%`));
|
|
174
192
|
// idx_memories_tenant_created serves a tenant's newest rows; with no tenant only the table's own
|
|
175
193
|
// order needs no sort.
|
|
176
194
|
const newestFirst = p.tenantParams.length > 0 ? 'created DESC' : 'rowid DESC';
|
|
177
195
|
return {
|
|
178
196
|
sql: `SELECT * FROM (SELECT ${MEMORY_SELECT_COLUMNS}, updated_at ` +
|
|
179
197
|
`FROM memories${p.tenantOnlyPredicate}${p.archivedClauseTenantOnly}${p.plainScope.sql}${p.currentNoAlias} ` +
|
|
180
|
-
`ORDER BY ${newestFirst} LIMIT ?) WHERE (
|
|
181
|
-
|
|
198
|
+
`ORDER BY ${newestFirst} LIMIT ?) WHERE EXISTS (SELECT 1 FROM json_each(?) j WHERE ` +
|
|
199
|
+
`LOWER(content) LIKE j.value ESCAPE '\\' OR LOWER(tags_json) LIKE j.value ESCAPE '\\')`,
|
|
200
|
+
params: [...p.tenantParams, ...p.scopeParams, Math.max(limit, LIKE_WINDOW_ROWS), likes],
|
|
182
201
|
};
|
|
183
202
|
}
|
|
184
203
|
/** The arms' rows, each once, most recently updated first. */
|
|
@@ -232,28 +251,45 @@ export function entriesInOrder(db, ids) {
|
|
|
232
251
|
const byId = new Map(rows.map((r) => [r.id, rowToEntry(r)]));
|
|
233
252
|
return ids.flatMap((id) => byId.get(id) ?? []);
|
|
234
253
|
}
|
|
235
|
-
// The vocabulary
|
|
254
|
+
// The vocabulary views are temp tables, so they live and die with their connection.
|
|
236
255
|
const vocabReady = new WeakSet();
|
|
237
256
|
function ensureRarestVocab(db) {
|
|
238
257
|
if (vocabReady.has(db))
|
|
239
258
|
return;
|
|
240
259
|
db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS temp.z1_rarest_vocab USING fts5vocab(main, 'memories_fts', 'row')`);
|
|
260
|
+
db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS temp.z1_trigram_vocab USING fts5vocab(main, 'memories_trigram', 'row')`);
|
|
241
261
|
vocabReady.add(db);
|
|
242
262
|
}
|
|
243
|
-
/** Rarest-K prompt terms for this connection's FTS
|
|
244
|
-
*
|
|
263
|
+
/** Rarest-K prompt terms for this connection's FTS indexes, as a space-joined query string: words
|
|
264
|
+
* count in the word index, CJK terms in the trigram index. Without FTS, the first 32 terms. */
|
|
245
265
|
export function pickRarestFtsQuery(db, terms, maxTerms = RAREST_TERM_COUNT) {
|
|
246
266
|
// The LIKE path has no bm25 ranking to bound, so it keeps the pre-rarest 32-term query.
|
|
247
267
|
if (!isFtsAvailable(db))
|
|
248
|
-
return terms
|
|
268
|
+
return spreadCjkTerms(terms, 32).join(' ');
|
|
249
269
|
ensureRarestVocab(db);
|
|
250
|
-
const
|
|
270
|
+
const { words, cjk } = partitionCjkTerms(terms);
|
|
271
|
+
const wordCounts = wordDocCounts(db, words);
|
|
272
|
+
const cjkCounts = trigramDocCounts(db, cjk);
|
|
273
|
+
return rarestFtsQuery(terms, (part) => wordCounts.get(part) ?? 0, maxTerms, (term) => cjkCounts.get(term) ?? 0);
|
|
274
|
+
}
|
|
275
|
+
function wordDocCounts(db, words) {
|
|
276
|
+
const vocab = Array.from(new Set(words.flatMap(ftsTermParts)));
|
|
251
277
|
if (vocab.length === 0)
|
|
252
|
-
return
|
|
278
|
+
return new Map();
|
|
253
279
|
// One cached shape per term count: an IN list over a virtual table needs one mark per term to
|
|
254
280
|
// keep its lookups.
|
|
255
281
|
const rows = cachedStatement(db, `SELECT term, doc FROM temp.z1_rarest_vocab WHERE term IN (${placeholders(vocab.length)})`).all(...vocab);
|
|
256
|
-
|
|
257
|
-
|
|
282
|
+
return new Map(rows.map((r) => [r.term, r.doc]));
|
|
283
|
+
}
|
|
284
|
+
/** Rows holding each CJK term by the trigram index: a trigram's own count, and for a shorter run
|
|
285
|
+
* the summed counts of the trigrams it opens, a rarity estimate, not an exact row count. */
|
|
286
|
+
function trigramDocCounts(db, cjk) {
|
|
287
|
+
// SHORTCUT: prefix seeks only, as a suffix count scans the whole vocabulary; a 0 keeps a short run.
|
|
288
|
+
if (cjk.length === 0)
|
|
289
|
+
return new Map();
|
|
290
|
+
// CROSS JOIN keeps the term list outside, so each term is one range seek into the vocabulary.
|
|
291
|
+
const rows = cachedStatement(db, `SELECT j.value AS term, SUM(v.doc) AS doc FROM json_each(?) j CROSS JOIN temp.z1_trigram_vocab v ` +
|
|
292
|
+
`ON v.term >= j.value AND v.term < j.value || char(1114111) GROUP BY j.value`).all(JSON.stringify(cjk));
|
|
293
|
+
return new Map(rows.map((r) => [r.term, r.doc]));
|
|
258
294
|
}
|
|
259
295
|
//# sourceMappingURL=search-rows.js.map
|
|
@@ -3,6 +3,7 @@ import type { HippoStore, StoreReach } from '../../../base/ports/store.js';
|
|
|
3
3
|
import { type ExecutorOptions } from './executor.js';
|
|
4
4
|
import { workerCounts } from './executor-counts.js';
|
|
5
5
|
import { type StoreHolder } from './held-connection.js';
|
|
6
|
+
import { StoreRuntime } from '../../../db/store-runtime.js';
|
|
6
7
|
import type { RequestStores } from '../../../db/index.js';
|
|
7
8
|
import type { RequestLimits } from '../../../base/util/request-scope.js';
|
|
8
9
|
export interface ServerRuntimeOptions extends ExecutorOptions {
|
|
@@ -30,7 +31,11 @@ export interface SqliteServerRuntime extends Omit<StoreHolder, 'release'> {
|
|
|
30
31
|
readonly runRequest: <T>(run: (session: RequestStores) => Promise<T>) => Promise<T>;
|
|
31
32
|
/** What the store workers of this process refused, expired and replaced; /health reports them. */
|
|
32
33
|
readonly workerCounts: Readonly<typeof workerCounts>;
|
|
33
|
-
|
|
34
|
+
/** Keeps the work a request starts and does not wait on, such as an embed; close() settles it. */
|
|
35
|
+
readonly detached: Pick<StoreRuntime, 'detach'>;
|
|
36
|
+
/** Waits up to `settleWithinMs` for the detached work, then closes the stores, the held
|
|
37
|
+
* connection last. */
|
|
38
|
+
readonly close: (settleWithinMs: number) => Promise<void>;
|
|
34
39
|
/** What the root's exit hook runs when the process exits before close(): the held writes and
|
|
35
40
|
* handles of the server's store runtime. */
|
|
36
41
|
readonly closeOnExit: () => void;
|
|
@@ -10,12 +10,22 @@ import { inSqliteSession, sqliteReach } from '../store.js';
|
|
|
10
10
|
import { runSqliteRequest } from './request-scope.js';
|
|
11
11
|
import { globalStoresFor, storeForRequest, workerSqliteStore } from './worker-store.js';
|
|
12
12
|
import { StoreRuntime } from '../../../db/store-runtime.js';
|
|
13
|
+
import { log } from '../../../base/util/log.js';
|
|
13
14
|
/** `own` as one request calls it: worker calls carry the request's limits, and an in-process
|
|
14
15
|
* store opens on its session. */
|
|
15
16
|
function boundToRequest(own, limits, session) {
|
|
16
17
|
const bound = storeForRequest(own, limits);
|
|
17
18
|
return session === undefined ? bound : inSqliteSession(bound, session);
|
|
18
19
|
}
|
|
20
|
+
/** Detached work still writes through the stores, so it settles before they close; one past the
|
|
21
|
+
* bound fails against the closed stores, and DetachedSet logs it. */
|
|
22
|
+
async function settleDetached(runtime, withinMs) {
|
|
23
|
+
const left = await runtime.settle(withinMs);
|
|
24
|
+
if (left === 0)
|
|
25
|
+
return;
|
|
26
|
+
const line = `shutdown: ${left} handed-off task(s) still running after ${withinMs} ms; closing the stores`;
|
|
27
|
+
log.warn(line, { left });
|
|
28
|
+
}
|
|
19
29
|
/** hippo.db under `hippoRoot` as a server runs it: `opts.store` when given, else a worker store
|
|
20
30
|
* on `opts`' executor. */
|
|
21
31
|
export function sqliteServerRuntime(hippoRoot, opts = {}) {
|
|
@@ -41,18 +51,20 @@ export function sqliteServerRuntime(hippoRoot, opts = {}) {
|
|
|
41
51
|
}),
|
|
42
52
|
runRequest: (run) => runSqliteRequest(run, 'server', runtime),
|
|
43
53
|
workerCounts,
|
|
54
|
+
detached: runtime,
|
|
44
55
|
hold: holder.hold,
|
|
45
56
|
afterResponse: holder.afterResponse,
|
|
46
|
-
close: async () => {
|
|
57
|
+
close: async (settleWithinMs) => {
|
|
58
|
+
await settleDetached(runtime, settleWithinMs);
|
|
47
59
|
// The store's worker threads and the pooled handles close first, so the held one is still
|
|
48
60
|
// SQLite's last.
|
|
49
61
|
if (given === undefined)
|
|
50
62
|
await store.close();
|
|
51
63
|
await globalStores.close();
|
|
52
|
-
runtime.
|
|
64
|
+
runtime.closeNow();
|
|
53
65
|
await holder.release();
|
|
54
66
|
},
|
|
55
|
-
closeOnExit: () => runtime.
|
|
67
|
+
closeOnExit: () => runtime.closeNow(),
|
|
56
68
|
};
|
|
57
69
|
}
|
|
58
70
|
//# sourceMappingURL=server-runtime.js.map
|
|
@@ -127,7 +127,7 @@ function post(port, reply, end) {
|
|
|
127
127
|
});
|
|
128
128
|
}
|
|
129
129
|
// Each step still closes what is left when the one before it throws, so a failed flush leaves no
|
|
130
|
-
// handle on the files. Held writes run first, then the thread's session closes, and
|
|
130
|
+
// handle on the files. Held writes run first, then the thread's session closes, and closeNow finds
|
|
131
131
|
// no session left to close twice.
|
|
132
132
|
function closeThread({ runtime, stores }, port) {
|
|
133
133
|
try {
|
|
@@ -136,7 +136,7 @@ function closeThread({ runtime, stores }, port) {
|
|
|
136
136
|
}
|
|
137
137
|
finally {
|
|
138
138
|
try {
|
|
139
|
-
runtime.
|
|
139
|
+
runtime.closeNow();
|
|
140
140
|
}
|
|
141
141
|
finally {
|
|
142
142
|
port.close();
|
|
@@ -33,7 +33,8 @@ const MAY_HOLD_COPY_SQL = `(${bareTextSql('content')} IN (SELECT value FROM json
|
|
|
33
33
|
function copyChunk(db, read, texts, phrases) {
|
|
34
34
|
const tenant = tenantAnd(read.tenantId, phrases ? TENANT_IS : 'tenant_id = ?');
|
|
35
35
|
const bares = JSON.stringify([...new Set(texts.map(bareText))]);
|
|
36
|
-
return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project FROM memories
|
|
36
|
+
return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project, pinned FROM memories ` +
|
|
37
|
+
`WHERE ${phrases ? IN_PHRASE_HITS : '1'}` +
|
|
37
38
|
`${tenant.sql}${read.filter.sql} AND ${MAY_HOLD_COPY_SQL}`).all(...(phrases ? [phrases.join(' OR ')] : []), ...tenant.params, ...read.filter.params, bares, bares);
|
|
38
39
|
}
|
|
39
40
|
/** The rows `read` admits that may hold one of `texts`, a superset of the copies heldTextKeys
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { type MemoryEntry } from '../../base/memory/entry.js';
|
|
2
2
|
import type { HippoConfig } from '../../base/config/config.js';
|
|
3
3
|
import { type ProjectRef } from '../../base/scope/project-identity.js';
|
|
4
|
-
import { type ExtractedItem } from './extract.js';
|
|
4
|
+
import { type BoldTwin, type ExtractedItem } from './extract.js';
|
|
5
5
|
import { type SessionTurn } from './transcript.js';
|
|
6
6
|
import type { CaptureDedup } from '../../base/ports/store/capture.js';
|
|
7
|
+
import type { HeldText } from '../../base/ports/store/types.js';
|
|
7
8
|
import type { StoreGroups } from '../../base/ports/store.js';
|
|
8
9
|
export interface CaptureOptions {
|
|
9
10
|
source: 'stdin' | 'file' | 'last-session';
|
|
@@ -41,6 +42,12 @@ export interface CaptureTarget {
|
|
|
41
42
|
readonly embed: (entry: MemoryEntry) => void;
|
|
42
43
|
}
|
|
43
44
|
export declare function runCapture(target: CaptureTarget, options: CaptureOptions): Promise<void>;
|
|
45
|
+
/** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
|
|
46
|
+
* items one, and the first stays. */
|
|
47
|
+
export declare function scrubbedItems(items: readonly ExtractedItem[], scrub: (text: string) => string): ExtractedItem[];
|
|
48
|
+
/** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
|
|
49
|
+
* row: a pinned row, or one whose text is no certain defect. */
|
|
50
|
+
export declare function heldKeysOf(rows: readonly HeldText[], twins: readonly BoldTwin[]): Set<string>;
|
|
44
51
|
export interface CaptureTally {
|
|
45
52
|
captured: number;
|
|
46
53
|
skipped: number;
|