hippo-memory 3.0.0 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +1 -0
  2. package/dist/api/context/context-plan.js +2 -4
  3. package/dist/api/context/context-select.js +12 -7
  4. package/dist/api/detached.d.ts +6 -0
  5. package/dist/api/detached.js +16 -0
  6. package/dist/api/embeddings.d.ts +4 -3
  7. package/dist/api/embeddings.js +4 -4
  8. package/dist/api/memories/promote.js +2 -1
  9. package/dist/api/memories/remember-local.d.ts +5 -4
  10. package/dist/api/memories/remember-local.js +13 -10
  11. package/dist/api/recall/rank-stages.js +2 -2
  12. package/dist/api/session-end.js +1 -0
  13. package/dist/api/sharing.js +9 -7
  14. package/dist/api/sleep/auto-sleep.d.ts +1 -1
  15. package/dist/api/sleep/auto-sleep.js +5 -3
  16. package/dist/api/sleep/learn.js +5 -2
  17. package/dist/api/sleep/sleep-run.js +4 -2
  18. package/dist/api/types.d.ts +4 -0
  19. package/dist/base/memory/entry.js +2 -2
  20. package/dist/base/memory/markdown-delimiters.d.ts +15 -0
  21. package/dist/base/memory/markdown-delimiters.js +94 -0
  22. package/dist/base/memory/memory-audit.js +4 -4
  23. package/dist/base/memory/memory-quality.d.ts +12 -0
  24. package/dist/base/memory/memory-quality.js +92 -3
  25. package/dist/base/ports/store/types.d.ts +2 -1
  26. package/dist/base/rank/prompt-recall.d.ts +16 -3
  27. package/dist/base/rank/prompt-recall.js +47 -7
  28. package/dist/base/util/detached-set.d.ts +8 -0
  29. package/dist/base/util/detached-set.js +35 -0
  30. package/dist/base/util/tokenize.d.ts +29 -0
  31. package/dist/base/util/tokenize.js +68 -5
  32. package/dist/base/util/version.d.ts +1 -1
  33. package/dist/base/util/version.js +1 -1
  34. package/dist/db/meta.d.ts +1 -0
  35. package/dist/db/meta.js +1 -0
  36. package/dist/db/store-runtime.d.ts +10 -3
  37. package/dist/db/store-runtime.js +19 -3
  38. package/dist/domain/consolidate/conflicts.js +2 -2
  39. package/dist/domain/learn/forward-claim-detector.js +4 -2
  40. package/dist/domain/learn/invalidation.js +5 -2
  41. package/dist/domain/search/bm25.js +3 -3
  42. package/dist/domain/search/detached-work.d.ts +6 -0
  43. package/dist/domain/search/detached-work.js +2 -0
  44. package/dist/domain/search/embed-memories.d.ts +4 -4
  45. package/dist/domain/search/embed-memories.js +4 -5
  46. package/dist/domain/sharing/global-sync.d.ts +8 -2
  47. package/dist/domain/sharing/global-sync.js +5 -5
  48. package/dist/domain/sharing/share.d.ts +4 -0
  49. package/dist/domain/sharing/share.js +2 -2
  50. package/dist/entry/eval-scripts.d.ts +1 -0
  51. package/dist/entry/sleep.d.ts +1 -0
  52. package/dist/index.d.ts +1 -1
  53. package/dist/server.d.ts +1 -0
  54. package/dist/store/rules/windows.d.ts +3 -0
  55. package/dist/store/rules/windows.js +3 -0
  56. package/dist/store/sqlite/audit/audit-maintenance.js +16 -13
  57. package/dist/store/sqlite/recall/candidates.js +1 -0
  58. package/dist/store/sqlite/recall/search-rows.d.ts +2 -2
  59. package/dist/store/sqlite/recall/search-rows.js +62 -26
  60. package/dist/store/sqlite/runtime/server-runtime.d.ts +6 -1
  61. package/dist/store/sqlite/runtime/server-runtime.js +15 -3
  62. package/dist/store/sqlite/runtime/worker-jobs.js +2 -2
  63. package/dist/store/sqlite/sharing/copy-rows.d.ts +1 -0
  64. package/dist/store/sqlite/sharing/copy-rows.js +2 -1
  65. package/dist/surface/capture/command.d.ts +8 -1
  66. package/dist/surface/capture/command.js +38 -8
  67. package/dist/surface/capture/compact.d.ts +3 -3
  68. package/dist/surface/capture/compact.js +4 -4
  69. package/dist/surface/capture/extract.d.ts +10 -0
  70. package/dist/surface/capture/extract.js +199 -26
  71. package/dist/surface/capture/session-texts.js +11 -11
  72. package/dist/surface/cli/api-context.d.ts +5 -0
  73. package/dist/surface/cli/api-context.js +12 -0
  74. package/dist/surface/cli/goals.js +6 -8
  75. package/dist/surface/cli/hook-runtime.js +2 -2
  76. package/dist/surface/cli/memory-import.js +20 -15
  77. package/dist/surface/mcp/protocol.d.ts +6 -0
  78. package/dist/surface/mcp/protocol.js +1 -0
  79. package/dist/surface/mcp/stdio.js +11 -5
  80. package/dist/surface/open-store.d.ts +2 -2
  81. package/dist/surface/open-store.js +11 -7
  82. package/dist/surface/server/auth.d.ts +1 -1
  83. package/dist/surface/server/auth.js +13 -2
  84. package/dist/surface/server/boot.js +3 -2
  85. package/dist/surface/server/dashboard-listener.js +2 -2
  86. package/dist/surface/server/lifecycle.d.ts +2 -0
  87. package/dist/surface/server/lifecycle.js +4 -2
  88. package/dist/surface/server/mcp-http.js +1 -0
  89. package/dist/surface/server/types.d.ts +3 -0
  90. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  91. package/extensions/openclaw-plugin/package.json +1 -1
  92. package/openclaw.plugin.json +1 -1
  93. package/package.json +1 -1
@@ -7,6 +7,7 @@ import type { StoreWith } from '../../base/ports/store.js';
7
7
  import type { FailureNotices } from '../../base/util/failure-notices.js';
8
8
  import { type IndexEmbedTarget } from './embed-index-target.js';
9
9
  import { type StoreEmbedTarget } from './embed-store-target.js';
10
+ import type { DetachedWork } from './detached-work.js';
10
11
  /** `off` is a provider that is disabled or not installed, which is a choice; `failed` is a
11
12
  * memory that stays out of the vector index. */
12
13
  export type EmbedOutcome = 'indexed' | 'off' | 'failed';
@@ -39,10 +40,9 @@ export interface EmbedAdapter {
39
40
  * through its vectorWrites. Resolves whatever fails, so a caller that fires and forgets stays
40
41
  * safe; the outcome says whether the memory reached the index. */
41
42
  export declare function embedMemoryInto(on: EmbedTarget | undefined, entry: MemoryEntry, model?: string): Promise<EmbedOutcome>;
42
- /** embedMemory for a write that does not wait on its vector. A rejection outside embedMemory's
43
- * own handling is logged here, since nothing else holds it and an unhandled one ends the
44
- * process. */
45
- export declare function embedMemoryDetached(on: EmbedTarget | undefined, entry: MemoryEntry): void;
43
+ /** embedMemory for a write that does not wait on its vector: `detached` keeps the embed until its
44
+ * root settles it. A rejection outside embedMemory's own handling is logged here. */
45
+ export declare function embedMemoryDetached(on: EmbedTarget | undefined, entry: MemoryEntry, detached: DetachedWork): void;
46
46
  /** Embeds every entry with no cached vector, prunes vectors of deleted memories, and returns how
47
47
  * many it embedded; `provider` defaults to the one the target's embedder builds. With a store,
48
48
  * every tenant's memories with no vector for the provider's model go through its `vectorWrites`. */
@@ -42,11 +42,10 @@ export async function embedMemoryInto(on, entry, model) {
42
42
  return 'off';
43
43
  return adapterFor(on).one(provider, entry);
44
44
  }
45
- /** embedMemory for a write that does not wait on its vector. A rejection outside embedMemory's
46
- * own handling is logged here, since nothing else holds it and an unhandled one ends the
47
- * process. */
48
- export function embedMemoryDetached(on, entry) {
49
- embedMemoryInto(on, entry).catch((err) => log.warn(`embedding ${entry.id} failed unexpectedly`, errorFields(err)));
45
+ /** embedMemory for a write that does not wait on its vector: `detached` keeps the embed until its
46
+ * root settles it. A rejection outside embedMemory's own handling is logged here. */
47
+ export function embedMemoryDetached(on, entry, detached) {
48
+ detached.detach(embedMemoryInto(on, entry).catch((err) => log.warn(`embedding ${entry.id} failed unexpectedly`, errorFields(err))));
50
49
  }
51
50
  /** Throws when an unavailable provider is a misconfiguration rather than an intentional no-op. */
52
51
  function throwIfProviderKeyMissing(cfg, provider) {
@@ -1,4 +1,5 @@
1
1
  import { type EmbedRun } from '../search/embed-memories.js';
2
+ import type { DetachedWork } from '../search/detached-work.js';
2
3
  import type { ConfiguredFolder } from '../../base/config/configured-folder.js';
3
4
  import { type SharingStore } from './text-keys.js';
4
5
  /**
@@ -9,6 +10,11 @@ export declare function syncGlobalToLocal(into: ConfiguredFolder<SharingStore>,
9
10
  readonly store: SharingStore;
10
11
  }, opts: {
11
12
  includeCrossProject?: boolean;
12
- embed: EmbedRun;
13
- }): Promise<number>;
13
+ } & SyncEmbed): Promise<number>;
14
+ /** The root's ports that score the copies, and the keeper of the embed that follows them. */
15
+ interface SyncEmbed {
16
+ readonly embed: EmbedRun;
17
+ readonly detached: DetachedWork;
18
+ }
19
+ export {};
14
20
  //# sourceMappingURL=global-sync.d.ts.map
@@ -22,7 +22,7 @@ export async function syncGlobalToLocal(into, global, opts) {
22
22
  if (!(await global.store.folderFiles.home()).exists)
23
23
  return 0;
24
24
  const rules = { project: home.project, includeCrossProject: opts.includeCrossProject };
25
- return copyMissingRows(into, global.store, rules, opts.embed);
25
+ return copyMissingRows(into, global.store, rules, opts);
26
26
  }
27
27
  const textKey = (e) => `${e.tenantId}\n${e.content}`;
28
28
  /** Global rows of every tenant that the local store holds by neither id nor tenant and text,
@@ -78,7 +78,7 @@ async function refusedTexts(store, { copies, twins }) {
78
78
  }
79
79
  /** One transaction: id, tenant and text decide which rows to read whole, and the copies commit
80
80
  * together; each copy keeps its own tenant. */
81
- async function copyMissingRows(into, from, rules, embed) {
81
+ async function copyMissingRows(into, from, rules, embedding) {
82
82
  const candidates = await missingRows(into.store.sharing, from.sharing);
83
83
  if (candidates.length === 0)
84
84
  return 0;
@@ -98,17 +98,17 @@ async function copyMissingRows(into, from, rules, embed) {
98
98
  const written = copies.length === 0
99
99
  ? []
100
100
  : await into.store.lessonWrites.writeSkippingRejected(copies, CLI_ACTOR);
101
- finishSyncDown(into, written.length, copies.length - written.length, embed);
101
+ finishSyncDown(into, written.length, copies.length - written.length, embedding);
102
102
  return written.length;
103
103
  }
104
- function finishSyncDown(into, count, rejected, embed) {
104
+ function finishSyncDown(into, count, rejected, { embed, detached }) {
105
105
  if (rejected > 0) {
106
106
  log.warn(`syncGlobalToLocal: skipped ${rejected} rejected value(s) (run \`hippo unreject\` on the local store to allow).`);
107
107
  }
108
108
  // Batch producer: one embedAll() on the destination rather than an embedMemory() per copied row
109
109
  // (same batching invariant as autoShare).
110
110
  if (count > 0) {
111
- void embedAll(folderEmbedTarget(into, embed)).catch((err) => logEmbedAllFailure('syncGlobalToLocal', err));
111
+ detached.detach(embedAll(folderEmbedTarget(into, embed)).catch((err) => logEmbedAllFailure('syncGlobalToLocal', err)));
112
112
  }
113
113
  }
114
114
  //# sourceMappingURL=global-sync.js.map
@@ -2,6 +2,7 @@ import { MemoryEntry } from '../../base/memory/entry.js';
2
2
  import { type EmbedRun } from '../search/embed-memories.js';
3
3
  import type { ConfiguredFolder } from '../../base/config/configured-folder.js';
4
4
  import { type TenantId } from '../../base/util/tenant.js';
5
+ import type { DetachedWork } from '../search/detached-work.js';
5
6
  import type { SharingStore } from './text-keys.js';
6
7
  import type { RequiredStore, StoreGroups, StoreWith } from '../../base/ports/store.js';
7
8
  /** Tags whose rows only a hand-run share or promote may copy to the global store; derived rows
@@ -26,6 +27,7 @@ export declare function shareMemory(local: {
26
27
  tenantId: string;
27
28
  skipEmbed?: boolean;
28
29
  embed: EmbedRun;
30
+ detached: DetachedWork;
29
31
  }): Promise<MemoryEntry | null>;
30
32
  /** List all projects that contributed memories to the global store, parsed from
31
33
  * 'shared:<project>:' or 'promoted:<path>' sources; ALL_TENANTS is the host-wide view. */
@@ -54,6 +56,8 @@ export declare function autoShare(from: {
54
56
  tenantId: TenantId;
55
57
  stats?: AutoShareStats;
56
58
  embed: EmbedRun;
59
+ /** Keeps the batch embed of the shared rows until the root settles it. */
60
+ detached: DetachedWork;
57
61
  }): Promise<MemoryEntry[]>;
58
62
  export {};
59
63
  //# sourceMappingURL=share.d.ts.map
@@ -131,7 +131,7 @@ export async function shareMemory(local, global, id, options) {
131
131
  strength: strengthModelFor(options.embed.evalSwitches, global.config.runtime),
132
132
  clock: options.embed.clock,
133
133
  };
134
- embedMemoryDetached(target, globalEntry);
134
+ embedMemoryDetached(target, globalEntry, options.detached);
135
135
  }
136
136
  return globalEntry;
137
137
  }
@@ -313,7 +313,7 @@ export async function autoShare(from, global, options) {
313
313
  now: options.embed.clock(),
314
314
  });
315
315
  if (shared.length > 0) {
316
- void embedAll(folderEmbedTarget(global, options.embed)).catch((err) => logEmbedAllFailure('autoShare', err));
316
+ options.detached.detach(embedAll(folderEmbedTarget(global, options.embed)).catch((err) => logEmbedAllFailure('autoShare', err)));
317
317
  }
318
318
  return shared;
319
319
  }
@@ -36,4 +36,5 @@ export declare const ingestEvent: (ctx: EntryContext, input: import("../surface/
36
36
  export declare const ingestMessage: (ctx: EntryContext, input: import("../surface/connectors/slack/ingest.js").IngestInput) => Promise<import("../surface/connectors/ingest.js").IngestResult>;
37
37
  export { adminActor, goalPush, outcome } from '../api/index.js';
38
38
  export type { EntryContext as Context } from '../api/index.js';
39
+ export type { DetachedWork } from '../domain/search/detached-work.js';
39
40
  //# sourceMappingURL=eval-scripts.d.ts.map
@@ -8,6 +8,7 @@ export type { SleepOptions, SleepResult } from '../api/sleep/sleep.js';
8
8
  export type { SleepEvent } from '../api/sleep/events.js';
9
9
  export type { RescueRank } from '../domain/consolidate/events.js';
10
10
  export type { SleepContext, SleepStore };
11
+ export type { DetachedWork } from '../domain/search/detached-work.js';
11
12
  /** Sleeps the whole store under `ctx`, whose store sets the groups a sleep pass works on. */
12
13
  export declare function sleep(ctx: SleepContext, opts?: SleepOptions): Promise<SleepResult>;
13
14
  /** consolidate's options as published: the LLM phases call the model ANTHROPIC_API_KEY opens,
package/dist/index.d.ts CHANGED
@@ -4,6 +4,6 @@
4
4
  * hippo-memory/importers and hippo-memory/eval.
5
5
  */
6
6
  export { MemoryEntry, Layer, EmotionalValence, ConfidenceLevel, DecayOptions, calculateStrength, resolveConfidence, confidenceFacets, type ConfidenceFacets, applyOutcome, generateId, computeSchemaFit, createMemory, markRetrieved, estimateTokens, SessionHandoff, HandoffOutcome, HandoffEvidence, isHandoffOutcome, Card, CardStatus, CardRun, CardComment, CardTransitions, isCardStatus, CARD_TRANSITIONS, CARD_LEASE_MS, computeSalience, SalienceDecision, SalienceResult, SalienceOptions, computeAmbientState, renderAmbientSummary, formatAmbientVector, AmbientState, tokenize, textOverlap, isEmbeddingAvailable, getEmbedding, } from './entry/memory.js';
7
- export { ConsolidationResult, type SleepOptions, type SleepResult, type SleepEvent, type RescueRank, type SleepContext, type SleepStore, sleep, consolidate, strengthBucket, } from './entry/sleep.js';
7
+ export { ConsolidationResult, type SleepOptions, type SleepResult, type SleepEvent, type RescueRank, type SleepContext, type SleepStore, type DetachedWork, sleep, consolidate, strengthBucket, } from './entry/sleep.js';
8
8
  export { captureError, extractLessons, partitionLessons, getGlobalRoot, transferScore, } from './entry/learning.js';
9
9
  //# sourceMappingURL=index.d.ts.map
package/dist/server.d.ts CHANGED
@@ -14,6 +14,7 @@ export { clientIpForRateLimit } from './surface/server/client-ip.js';
14
14
  export { isLoopback, isReservedActor } from './surface/server/auth.js';
15
15
  export type { AddonCall, AddonRoute, AuthResolver, RateLimitSpec, ResolvedBearer, ServeOptions, ServerHandle, } from './surface/server/types.js';
16
16
  export type { HippoStore } from './base/ports/store-index.js';
17
+ export type { DetachedWork } from './domain/search/detached-work.js';
17
18
  export { HttpError } from './base/errors/http-errors.js';
18
19
  export type { CallerProject } from './api/prompt-hook.js';
19
20
  export type { JsonValue } from './base/util/json.js';
@@ -1,6 +1,9 @@
1
1
  /** Newest admitted rows LIKE tests for a term under 3 characters, which no trigram lookup can
2
2
  * answer. */
3
3
  export declare const LIKE_WINDOW_ROWS = 2000;
4
+ /** Most terms LIKE tests on each of those rows: the trigram index looks up every longer term over
5
+ * the whole store, so a long CJK query's 1,000s of bigrams need not all be tested here. */
6
+ export declare const LIKE_WINDOW_TERMS = 64;
4
7
  /** How many of a tenant's newest rows schemaFit reads; older rows no longer move the fit. */
5
8
  export declare const SCHEMA_FIT_WINDOW_ROWS = 2000;
6
9
  /** Events of one completed session a sleep reads to render its trace, the newest kept. */
@@ -3,6 +3,9 @@
3
3
  /** Newest admitted rows LIKE tests for a term under 3 characters, which no trigram lookup can
4
4
  * answer. */
5
5
  export const LIKE_WINDOW_ROWS = 2000;
6
+ /** Most terms LIKE tests on each of those rows: the trigram index looks up every longer term over
7
+ * the whole store, so a long CJK query's 1,000s of bigrams need not all be tested here. */
8
+ export const LIKE_WINDOW_TERMS = 64;
6
9
  /** How many of a tenant's newest rows schemaFit reads; older rows no longer move the fit. */
7
10
  export const SCHEMA_FIT_WINDOW_ROWS = 2000;
8
11
  /** Events of one completed session a sleep reads to render its trace, the newest kept. */
@@ -17,7 +17,8 @@ import { purgeMirrorBestEffort } from '../folder-files/mirrors.js';
17
17
  import { openStore } from '../runtime/handles.js';
18
18
  import { answered, bound, skipEmpty } from '../runtime/bound.js';
19
19
  import { READ_PAGE_ROWS } from '../../../base/util/pages.js';
20
- import { getMeta, pragmaUserVersion, setMeta, META_KEY } from '../../../db/meta.js';
20
+ import { getMeta, getMetaNumber, pragmaUserVersion, setMeta, META_KEY } from '../../../db/meta.js';
21
+ import { QUALITY_GATE_VERSION } from '../../../base/memory/memory-quality.js';
21
22
  // Quality repair's issue list: which rows are defects and which of those a protection keeps.
22
23
  /** The issues among `entries` in their order, each defect kept when `protect` names the row. */
23
24
  function repairIssues(entries, parents, backing, protect) {
@@ -223,8 +224,8 @@ function applyPlan(db, { run, plan, backup, rules }) {
223
224
  else
224
225
  warnings.push(`Kept ${issue.id}: the store's delete guard protects it.`);
225
226
  }
226
- if (run.doneKey)
227
- setMeta(db, run.doneKey, '1');
227
+ for (const [key, value] of run.doneWrites ?? [])
228
+ setMeta(db, key, value);
228
229
  return { ...plan, issues, appliedIds, backup, warnings };
229
230
  });
230
231
  }
@@ -248,21 +249,23 @@ function repairOn(db, files, run) {
248
249
  }
249
250
  return { ...result, warnings };
250
251
  }
251
- /** Applies the repair once per store, so stores that predate the quality gate are cleaned on
252
+ // The old key stays '1' so a binary from before the gate version keeps skipping the repair.
253
+ const DONE_WRITES = [
254
+ [META_KEY.qualityRepairAuto, '1'],
255
+ [META_KEY.qualityRepairGate, String(QUALITY_GATE_VERSION)],
256
+ ];
257
+ /** Applies the repair once per gate version, so stores that predate a quality check are cleaned on
252
258
  * upgrade with no command; null once done. `db` is opened writable without running migrations. */
253
259
  function repairQualityOnce(db, files, tenantId, protect) {
254
- if (getMeta(db, META_KEY.qualityRepairAuto) === '1')
260
+ if (getMetaNumber(db, META_KEY.qualityRepairGate) >= QUALITY_GATE_VERSION)
255
261
  return null;
256
262
  // The apply commits the flag with the moves, so a lock taken between them cannot hide what moved.
257
- const result = repairOn(db, files, {
258
- tenantId,
259
- protect,
260
- apply: true,
261
- doneKey: META_KEY.qualityRepairAuto,
262
- });
263
+ const result = repairOn(db, files, { tenantId, protect, apply: true, doneWrites: DONE_WRITES });
263
264
  // An unsupported schema changed nothing, so the next run tries again once a migration has run.
264
- if (result.supported && result.backup === null)
265
- setMeta(db, META_KEY.qualityRepairAuto, '1');
265
+ if (result.supported && result.backup === null) {
266
+ for (const [key, value] of DONE_WRITES)
267
+ setMeta(db, key, value);
268
+ }
266
269
  return result;
267
270
  }
268
271
  export function sqliteAuditMaintenance({ on, files }) {
@@ -298,6 +298,7 @@ export function listTextsHolding(db, { tenantId, texts, project, admit = TEAM_VI
298
298
  content: row.content,
299
299
  source: row.source,
300
300
  originProject: row.origin_project,
301
+ pinned: row.pinned === 1,
301
302
  });
302
303
  }
303
304
  return [...byId.values()];
@@ -16,7 +16,7 @@ export declare function listRecallEntries(db: DatabaseSyncLike, query: string, o
16
16
  export declare function listVectorCandidateEntries(db: DatabaseSyncLike, queryVector: readonly number[], spec: VectorCandidateSpec, index?: VectorIndex): Promise<MemoryEntry[]>;
17
17
  /** The rows of `ids` in that order; an id deleted since it was matched is skipped. */
18
18
  export declare function entriesInOrder(db: DatabaseSyncLike, ids: readonly string[]): MemoryEntry[];
19
- /** Rarest-K prompt terms for this connection's FTS index, as a space-joined query string.
20
- * Without FTS, returns the first 32 terms, as before rarest-term selection. */
19
+ /** Rarest-K prompt terms for this connection's FTS indexes, as a space-joined query string: words
20
+ * count in the word index, CJK terms in the trigram index. Without FTS, the first 32 terms. */
21
21
  export declare function pickRarestFtsQuery(db: DatabaseSyncLike, terms: readonly string[], maxTerms?: number): string;
22
22
  //# sourceMappingURL=search-rows.d.ts.map
@@ -1,8 +1,8 @@
1
1
  import { isFtsAvailable } from '../../../db/index.js';
2
- import { tokenize } from '../../../base/util/tokenize.js';
2
+ import { cjkIndexTerms, partitionCjkTerms, tokenize } from '../../../base/util/tokenize.js';
3
3
  import { recallScopeFilter } from '../../../base/scope/recall-scope.js';
4
4
  import { originInSql, scopeAdmitSql } from '../runtime/rule-sql.js';
5
- import { ftsTermParts, RAREST_TERM_COUNT, rarestFtsQuery, } from '../../../base/rank/prompt-recall.js';
5
+ import { ftsTermParts, RAREST_TERM_COUNT, rarestFtsQuery, spreadCjkTerms, } from '../../../base/rank/prompt-recall.js';
6
6
  import { errorMessage, log, errorFields } from '../../../base/util/log.js';
7
7
  import { topVectorMatches } from '../../../db/vector-match.js';
8
8
  import { MEMORY_SELECT_COLUMNS, MEMORY_SEARCH_COLUMNS, rowToEntry, } from '../runtime/rows.js';
@@ -14,7 +14,7 @@ import { placeholders } from '../../../db/sql.js';
14
14
  import { VECTOR_CANDIDATE_LIMIT, } from '../../../base/ports/store/vectors.js';
15
15
  import { ALL_TENANTS } from '../../../base/util/tenant.js';
16
16
  import { tenantAnd } from '../runtime/tenant.js';
17
- import { LIKE_WINDOW_ROWS } from '../../rules/windows.js';
17
+ import { LIKE_WINDOW_ROWS, LIKE_WINDOW_TERMS } from '../../rules/windows.js';
18
18
  /** The recall scope rule for a table column prefix (`m.` or none); `passesScopeFilterForRecall`
19
19
  * in recall-scope.ts is its JS twin. */
20
20
  function recallScopeClause(col, scopeFilter) {
@@ -55,18 +55,40 @@ function listSearchRows(db, { query, limit, tenantId, options = {} }) {
55
55
  // LIMIT here too, so every candidate path honours the caller's cap.
56
56
  return listAllCandidates(db, p, limit);
57
57
  }
58
- if (isFtsAvailable(db)) {
59
- const rows = listFtsCandidates(db, terms, p, limit);
60
- if (rows.length > 0)
61
- return rows;
62
- }
63
- const rows = listLikeCandidates(db, terms, p, limit);
58
+ // The run's trigrams reach the trigram index, which the 2-letter tokens cannot.
59
+ const { words, cjk } = partitionCjkTerms(terms);
60
+ const rows = interleaveRows(words.length > 0 ? listWordCandidates(db, words, p, limit) : [], cjk.length > 0
61
+ ? listLikeCandidates(db, [...new Set([...cjk, ...cjkIndexTerms(query)])], p, limit)
62
+ : [], limit);
64
63
  if (rows.length > 0)
65
64
  return rows;
66
65
  // LIMIT the full-store fallback too: RecallResult reports the scorer window, so an unbounded
67
66
  // fallback would misstate the candidate-pool size.
68
67
  return listAllCandidates(db, p, limit);
69
68
  }
69
+ /** FTS5 matches for `terms`, else their substring matches. */
70
+ function listWordCandidates(db, terms, p, limit) {
71
+ if (isFtsAvailable(db)) {
72
+ const rows = listFtsCandidates(db, terms, p, limit);
73
+ if (rows.length > 0)
74
+ return rows;
75
+ }
76
+ return listLikeCandidates(db, terms, p, limit);
77
+ }
78
+ /** `a` and `b` taken in turn, each row once, up to `limit`; with one list empty, the other as it
79
+ * is. */
80
+ function interleaveRows(a, b, limit) {
81
+ if (a.length === 0 || b.length === 0)
82
+ return a.length === 0 ? b : a;
83
+ const out = new Map();
84
+ for (let i = 0; out.size < limit && i < Math.max(a.length, b.length); i++) {
85
+ for (const row of [a[i], b[i]]) {
86
+ if (row !== undefined && out.size < limit)
87
+ out.set(row.id, row);
88
+ }
89
+ }
90
+ return [...out.values()];
91
+ }
70
92
  function searchPredicates(tenantId, scopeFilter, includeSuperseded, originProjects) {
71
93
  // One tenant reads through the composite idx_memories_tenant_created.
72
94
  const one = tenantId !== ALL_TENANTS;
@@ -164,21 +186,18 @@ function trigramArm(terms, p) {
164
186
  /** LIKE over the newest admitted rows: the bound is what keeps a short term from reading the
165
187
  * whole store. */
166
188
  function windowArm(terms, p, limit) {
167
- const where = terms
168
- .map(() => `(LOWER(content) LIKE ? ESCAPE '\\' OR LOWER(tags_json) LIKE ? ESCAPE '\\')`)
169
- .join(' OR ');
170
- const likes = terms.flatMap((term) => {
171
- const like = `%${escapeLike(term)}%`;
172
- return [like, like];
173
- });
189
+ // One JSON array, not an OR per term: a long CJK query has 1,000s of terms, past SQLite's
190
+ // expression depth.
191
+ const likes = JSON.stringify(terms.slice(0, LIKE_WINDOW_TERMS).map((term) => `%${escapeLike(term)}%`));
174
192
  // idx_memories_tenant_created serves a tenant's newest rows; with no tenant only the table's own
175
193
  // order needs no sort.
176
194
  const newestFirst = p.tenantParams.length > 0 ? 'created DESC' : 'rowid DESC';
177
195
  return {
178
196
  sql: `SELECT * FROM (SELECT ${MEMORY_SELECT_COLUMNS}, updated_at ` +
179
197
  `FROM memories${p.tenantOnlyPredicate}${p.archivedClauseTenantOnly}${p.plainScope.sql}${p.currentNoAlias} ` +
180
- `ORDER BY ${newestFirst} LIMIT ?) WHERE (${where})`,
181
- params: [...p.tenantParams, ...p.scopeParams, Math.max(limit, LIKE_WINDOW_ROWS), ...likes],
198
+ `ORDER BY ${newestFirst} LIMIT ?) WHERE EXISTS (SELECT 1 FROM json_each(?) j WHERE ` +
199
+ `LOWER(content) LIKE j.value ESCAPE '\\' OR LOWER(tags_json) LIKE j.value ESCAPE '\\')`,
200
+ params: [...p.tenantParams, ...p.scopeParams, Math.max(limit, LIKE_WINDOW_ROWS), likes],
182
201
  };
183
202
  }
184
203
  /** The arms' rows, each once, most recently updated first. */
@@ -232,28 +251,45 @@ export function entriesInOrder(db, ids) {
232
251
  const byId = new Map(rows.map((r) => [r.id, rowToEntry(r)]));
233
252
  return ids.flatMap((id) => byId.get(id) ?? []);
234
253
  }
235
- // The vocabulary view is a temp table, so it lives and dies with its connection.
254
+ // The vocabulary views are temp tables, so they live and die with their connection.
236
255
  const vocabReady = new WeakSet();
237
256
  function ensureRarestVocab(db) {
238
257
  if (vocabReady.has(db))
239
258
  return;
240
259
  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS temp.z1_rarest_vocab USING fts5vocab(main, 'memories_fts', 'row')`);
260
+ db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS temp.z1_trigram_vocab USING fts5vocab(main, 'memories_trigram', 'row')`);
241
261
  vocabReady.add(db);
242
262
  }
243
- /** Rarest-K prompt terms for this connection's FTS index, as a space-joined query string.
244
- * Without FTS, returns the first 32 terms, as before rarest-term selection. */
263
+ /** Rarest-K prompt terms for this connection's FTS indexes, as a space-joined query string: words
264
+ * count in the word index, CJK terms in the trigram index. Without FTS, the first 32 terms. */
245
265
  export function pickRarestFtsQuery(db, terms, maxTerms = RAREST_TERM_COUNT) {
246
266
  // The LIKE path has no bm25 ranking to bound, so it keeps the pre-rarest 32-term query.
247
267
  if (!isFtsAvailable(db))
248
- return terms.slice(0, 32).join(' ');
268
+ return spreadCjkTerms(terms, 32).join(' ');
249
269
  ensureRarestVocab(db);
250
- const vocab = Array.from(new Set(terms.flatMap(ftsTermParts)));
270
+ const { words, cjk } = partitionCjkTerms(terms);
271
+ const wordCounts = wordDocCounts(db, words);
272
+ const cjkCounts = trigramDocCounts(db, cjk);
273
+ return rarestFtsQuery(terms, (part) => wordCounts.get(part) ?? 0, maxTerms, (term) => cjkCounts.get(term) ?? 0);
274
+ }
275
+ function wordDocCounts(db, words) {
276
+ const vocab = Array.from(new Set(words.flatMap(ftsTermParts)));
251
277
  if (vocab.length === 0)
252
- return '';
278
+ return new Map();
253
279
  // One cached shape per term count: an IN list over a virtual table needs one mark per term to
254
280
  // keep its lookups.
255
281
  const rows = cachedStatement(db, `SELECT term, doc FROM temp.z1_rarest_vocab WHERE term IN (${placeholders(vocab.length)})`).all(...vocab);
256
- const counts = new Map(rows.map((r) => [r.term, r.doc]));
257
- return rarestFtsQuery(terms, (part) => counts.get(part) ?? 0, maxTerms);
282
+ return new Map(rows.map((r) => [r.term, r.doc]));
283
+ }
284
+ /** Rows holding each CJK term by the trigram index: a trigram's own count, and for a shorter run
285
+ * the summed counts of the trigrams it opens, a rarity estimate, not an exact row count. */
286
+ function trigramDocCounts(db, cjk) {
287
+ // SHORTCUT: prefix seeks only, as a suffix count scans the whole vocabulary; a 0 keeps a short run.
288
+ if (cjk.length === 0)
289
+ return new Map();
290
+ // CROSS JOIN keeps the term list outside, so each term is one range seek into the vocabulary.
291
+ const rows = cachedStatement(db, `SELECT j.value AS term, SUM(v.doc) AS doc FROM json_each(?) j CROSS JOIN temp.z1_trigram_vocab v ` +
292
+ `ON v.term >= j.value AND v.term < j.value || char(1114111) GROUP BY j.value`).all(JSON.stringify(cjk));
293
+ return new Map(rows.map((r) => [r.term, r.doc]));
258
294
  }
259
295
  //# sourceMappingURL=search-rows.js.map
@@ -3,6 +3,7 @@ import type { HippoStore, StoreReach } from '../../../base/ports/store.js';
3
3
  import { type ExecutorOptions } from './executor.js';
4
4
  import { workerCounts } from './executor-counts.js';
5
5
  import { type StoreHolder } from './held-connection.js';
6
+ import { StoreRuntime } from '../../../db/store-runtime.js';
6
7
  import type { RequestStores } from '../../../db/index.js';
7
8
  import type { RequestLimits } from '../../../base/util/request-scope.js';
8
9
  export interface ServerRuntimeOptions extends ExecutorOptions {
@@ -30,7 +31,11 @@ export interface SqliteServerRuntime extends Omit<StoreHolder, 'release'> {
30
31
  readonly runRequest: <T>(run: (session: RequestStores) => Promise<T>) => Promise<T>;
31
32
  /** What the store workers of this process refused, expired and replaced; /health reports them. */
32
33
  readonly workerCounts: Readonly<typeof workerCounts>;
33
- readonly close: () => Promise<void>;
34
+ /** Keeps the work a request starts and does not wait on, such as an embed; close() settles it. */
35
+ readonly detached: Pick<StoreRuntime, 'detach'>;
36
+ /** Waits up to `settleWithinMs` for the detached work, then closes the stores, the held
37
+ * connection last. */
38
+ readonly close: (settleWithinMs: number) => Promise<void>;
34
39
  /** What the root's exit hook runs when the process exits before close(): the held writes and
35
40
  * handles of the server's store runtime. */
36
41
  readonly closeOnExit: () => void;
@@ -10,12 +10,22 @@ import { inSqliteSession, sqliteReach } from '../store.js';
10
10
  import { runSqliteRequest } from './request-scope.js';
11
11
  import { globalStoresFor, storeForRequest, workerSqliteStore } from './worker-store.js';
12
12
  import { StoreRuntime } from '../../../db/store-runtime.js';
13
+ import { log } from '../../../base/util/log.js';
13
14
  /** `own` as one request calls it: worker calls carry the request's limits, and an in-process
14
15
  * store opens on its session. */
15
16
  function boundToRequest(own, limits, session) {
16
17
  const bound = storeForRequest(own, limits);
17
18
  return session === undefined ? bound : inSqliteSession(bound, session);
18
19
  }
20
+ /** Detached work still writes through the stores, so it settles before they close; one past the
21
+ * bound fails against the closed stores, and DetachedSet logs it. */
22
+ async function settleDetached(runtime, withinMs) {
23
+ const left = await runtime.settle(withinMs);
24
+ if (left === 0)
25
+ return;
26
+ const line = `shutdown: ${left} handed-off task(s) still running after ${withinMs} ms; closing the stores`;
27
+ log.warn(line, { left });
28
+ }
19
29
  /** hippo.db under `hippoRoot` as a server runs it: `opts.store` when given, else a worker store
20
30
  * on `opts`' executor. */
21
31
  export function sqliteServerRuntime(hippoRoot, opts = {}) {
@@ -41,18 +51,20 @@ export function sqliteServerRuntime(hippoRoot, opts = {}) {
41
51
  }),
42
52
  runRequest: (run) => runSqliteRequest(run, 'server', runtime),
43
53
  workerCounts,
54
+ detached: runtime,
44
55
  hold: holder.hold,
45
56
  afterResponse: holder.afterResponse,
46
- close: async () => {
57
+ close: async (settleWithinMs) => {
58
+ await settleDetached(runtime, settleWithinMs);
47
59
  // The store's worker threads and the pooled handles close first, so the held one is still
48
60
  // SQLite's last.
49
61
  if (given === undefined)
50
62
  await store.close();
51
63
  await globalStores.close();
52
- runtime.dispose();
64
+ runtime.closeNow();
53
65
  await holder.release();
54
66
  },
55
- closeOnExit: () => runtime.dispose(),
67
+ closeOnExit: () => runtime.closeNow(),
56
68
  };
57
69
  }
58
70
  //# sourceMappingURL=server-runtime.js.map
@@ -127,7 +127,7 @@ function post(port, reply, end) {
127
127
  });
128
128
  }
129
129
  // Each step still closes what is left when the one before it throws, so a failed flush leaves no
130
- // handle on the files. Held writes run first, then the thread's session closes, and dispose finds
130
+ // handle on the files. Held writes run first, then the thread's session closes, and closeNow finds
131
131
  // no session left to close twice.
132
132
  function closeThread({ runtime, stores }, port) {
133
133
  try {
@@ -136,7 +136,7 @@ function closeThread({ runtime, stores }, port) {
136
136
  }
137
137
  finally {
138
138
  try {
139
- runtime.dispose();
139
+ runtime.closeNow();
140
140
  }
141
141
  finally {
142
142
  port.close();
@@ -9,6 +9,7 @@ export type CopyDbRow = {
9
9
  content: string;
10
10
  source: string;
11
11
  origin_project: string | null;
12
+ pinned: number;
12
13
  };
13
14
  /** Which rows of the tenant a copy read looks at: `filter` adds predicates, each written ` AND
14
15
  * ...`. */
@@ -33,7 +33,8 @@ const MAY_HOLD_COPY_SQL = `(${bareTextSql('content')} IN (SELECT value FROM json
33
33
  function copyChunk(db, read, texts, phrases) {
34
34
  const tenant = tenantAnd(read.tenantId, phrases ? TENANT_IS : 'tenant_id = ?');
35
35
  const bares = JSON.stringify([...new Set(texts.map(bareText))]);
36
- return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project FROM memories WHERE ${phrases ? IN_PHRASE_HITS : '1'}` +
36
+ return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project, pinned FROM memories ` +
37
+ `WHERE ${phrases ? IN_PHRASE_HITS : '1'}` +
37
38
  `${tenant.sql}${read.filter.sql} AND ${MAY_HOLD_COPY_SQL}`).all(...(phrases ? [phrases.join(' OR ')] : []), ...tenant.params, ...read.filter.params, bares, bares);
38
39
  }
39
40
  /** The rows `read` admits that may hold one of `texts`, a superset of the copies heldTextKeys
@@ -1,9 +1,10 @@
1
1
  import { type MemoryEntry } from '../../base/memory/entry.js';
2
2
  import type { HippoConfig } from '../../base/config/config.js';
3
3
  import { type ProjectRef } from '../../base/scope/project-identity.js';
4
- import { type ExtractedItem } from './extract.js';
4
+ import { type BoldTwin, type ExtractedItem } from './extract.js';
5
5
  import { type SessionTurn } from './transcript.js';
6
6
  import type { CaptureDedup } from '../../base/ports/store/capture.js';
7
+ import type { HeldText } from '../../base/ports/store/types.js';
7
8
  import type { StoreGroups } from '../../base/ports/store.js';
8
9
  export interface CaptureOptions {
9
10
  source: 'stdin' | 'file' | 'last-session';
@@ -41,6 +42,12 @@ export interface CaptureTarget {
41
42
  readonly embed: (entry: MemoryEntry) => void;
42
43
  }
43
44
  export declare function runCapture(target: CaptureTarget, options: CaptureOptions): Promise<void>;
45
+ /** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
46
+ * items one, and the first stays. */
47
+ export declare function scrubbedItems(items: readonly ExtractedItem[], scrub: (text: string) => string): ExtractedItem[];
48
+ /** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
49
+ * row: a pinned row, or one whose text is no certain defect. */
50
+ export declare function heldKeysOf(rows: readonly HeldText[], twins: readonly BoldTwin[]): Set<string>;
44
51
  export interface CaptureTally {
45
52
  captured: number;
46
53
  skipped: number;