hippo-memory 1.11.2 → 1.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +132 -1
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +396 -4
- package/dist/api.js.map +1 -1
- package/dist/cli.js +160 -369
- package/dist/cli.js.map +1 -1
- package/dist/dedupe.d.ts +34 -0
- package/dist/dedupe.d.ts.map +1 -0
- package/dist/dedupe.js +61 -0
- package/dist/dedupe.js.map +1 -0
- package/dist/src/api.js +396 -4
- package/dist/src/api.js.map +1 -1
- package/dist/src/cli.js +160 -369
- package/dist/src/cli.js.map +1 -1
- package/dist/src/dedupe.js +61 -0
- package/dist/src/dedupe.js.map +1 -0
- package/dist/src/version.js +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
package/dist/dedupe.d.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Store-level deduplication. Scans for near-duplicate memories by content
|
|
3
|
+
* Jaccard overlap, keeps the stronger copy (by strength + retrieval count),
|
|
4
|
+
* removes the rest.
|
|
5
|
+
*
|
|
6
|
+
* Extracted from cli.ts in Episode A (v1.11.3) so `api.sleep` can dedupe
|
|
7
|
+
* during the consolidation pipeline without violating the cli -> api
|
|
8
|
+
* dependency direction. `cmdDedup` in cli.ts continues to import and use
|
|
9
|
+
* this function unchanged.
|
|
10
|
+
*/
|
|
11
|
+
export interface DedupPair {
|
|
12
|
+
kept: string;
|
|
13
|
+
keptContent: string;
|
|
14
|
+
keptLayer: string;
|
|
15
|
+
keptStrength: number;
|
|
16
|
+
removed: string;
|
|
17
|
+
removedContent: string;
|
|
18
|
+
removedLayer: string;
|
|
19
|
+
removedStrength: number;
|
|
20
|
+
similarity: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Scan the store for near-duplicate memories and remove the weaker copy.
|
|
24
|
+
* Two memories are duplicates if their content has > threshold Jaccard overlap.
|
|
25
|
+
* Keeps the one with higher strength (or more retrievals if tied).
|
|
26
|
+
*/
|
|
27
|
+
export declare function deduplicateStore(hippoRoot: string, options?: {
|
|
28
|
+
threshold?: number;
|
|
29
|
+
dryRun?: boolean;
|
|
30
|
+
}): {
|
|
31
|
+
removed: number;
|
|
32
|
+
pairs: DedupPair[];
|
|
33
|
+
};
|
|
34
|
+
//# sourceMappingURL=dedupe.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedupe.d.ts","sourceRoot":"","sources":["../src/dedupe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAKH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAO,GACrD;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,SAAS,EAAE,CAAA;CAAE,CA6CzC"}
|
package/dist/dedupe.js
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Store-level deduplication. Scans for near-duplicate memories by content
|
|
3
|
+
* Jaccard overlap, keeps the stronger copy (by strength + retrieval count),
|
|
4
|
+
* removes the rest.
|
|
5
|
+
*
|
|
6
|
+
* Extracted from cli.ts in Episode A (v1.11.3) so `api.sleep` can dedupe
|
|
7
|
+
* during the consolidation pipeline without violating the cli -> api
|
|
8
|
+
* dependency direction. `cmdDedup` in cli.ts continues to import and use
|
|
9
|
+
* this function unchanged.
|
|
10
|
+
*/
|
|
11
|
+
import { textOverlap } from './search.js';
|
|
12
|
+
import { loadAllEntries, deleteEntry } from './store.js';
|
|
13
|
+
/**
|
|
14
|
+
* Scan the store for near-duplicate memories and remove the weaker copy.
|
|
15
|
+
* Two memories are duplicates if their content has > threshold Jaccard overlap.
|
|
16
|
+
* Keeps the one with higher strength (or more retrievals if tied).
|
|
17
|
+
*/
|
|
18
|
+
export function deduplicateStore(hippoRoot, options = {}) {
|
|
19
|
+
const threshold = options.threshold ?? 0.7;
|
|
20
|
+
const dryRun = options.dryRun ?? false;
|
|
21
|
+
const entries = loadAllEntries(hippoRoot);
|
|
22
|
+
// Sort by strength desc, then retrieval count, so we keep the most valuable copy
|
|
23
|
+
entries.sort((a, b) => {
|
|
24
|
+
const sDiff = (b.strength ?? 0) - (a.strength ?? 0);
|
|
25
|
+
if (Math.abs(sDiff) > 0.01)
|
|
26
|
+
return sDiff;
|
|
27
|
+
return (b.retrieval_count ?? 0) - (a.retrieval_count ?? 0);
|
|
28
|
+
});
|
|
29
|
+
const removed = new Set();
|
|
30
|
+
const pairs = [];
|
|
31
|
+
for (let i = 0; i < entries.length; i++) {
|
|
32
|
+
if (removed.has(entries[i].id))
|
|
33
|
+
continue;
|
|
34
|
+
for (let j = i + 1; j < entries.length; j++) {
|
|
35
|
+
if (removed.has(entries[j].id))
|
|
36
|
+
continue;
|
|
37
|
+
const similarity = textOverlap(entries[i].content, entries[j].content);
|
|
38
|
+
if (similarity <= threshold)
|
|
39
|
+
continue;
|
|
40
|
+
removed.add(entries[j].id);
|
|
41
|
+
pairs.push({
|
|
42
|
+
kept: entries[i].id,
|
|
43
|
+
keptContent: entries[i].content,
|
|
44
|
+
keptLayer: entries[i].layer,
|
|
45
|
+
keptStrength: entries[i].strength ?? 0,
|
|
46
|
+
removed: entries[j].id,
|
|
47
|
+
removedContent: entries[j].content,
|
|
48
|
+
removedLayer: entries[j].layer,
|
|
49
|
+
removedStrength: entries[j].strength ?? 0,
|
|
50
|
+
similarity,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
if (!dryRun) {
|
|
55
|
+
for (const id of removed) {
|
|
56
|
+
deleteEntry(hippoRoot, id);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return { removed: removed.size, pairs };
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=dedupe.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedupe.js","sourceRoot":"","sources":["../src/dedupe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAczD;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAC9B,SAAiB,EACjB,UAAoD,EAAE;IAEtD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;IAC3C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;IACvC,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;IAE1C,iFAAiF;IACjF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACpB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI;YAAE,OAAO,KAAK,CAAC;QACzC,OAAO,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAAE,SAAS;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAAE,SAAS;YAEzC,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACvE,IAAI,UAAU,IAAI,SAAS;gBAAE,SAAS;YAEtC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;gBACnB,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;gBAC/B,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK;gBAC3B,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC;gBACtC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;gBACtB,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;gBAClC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK;gBAC9B,eAAe,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC;gBACzC,UAAU;aACX,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,KAAK,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;YACzB,WAAW,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;AAC1C,CAAC"}
|
package/dist/src/api.js
CHANGED
|
@@ -8,13 +8,19 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { createHash } from 'node:crypto';
|
|
10
10
|
import { openHippoDb, closeHippoDb } from './db.js';
|
|
11
|
-
import { writeEntry, writeEntryDbOnly, writeEntryMirrors, readEntry, deleteEntry, loadRecallSearchEntries, loadEntriesByIds, loadChildrenOf, loadFreshRawMemories, loadSessionRawMemories, countSessionRawMemories, DEFAULT_SEARCH_CANDIDATE_LIMIT, RECALL_DEFAULT_DENY_SCOPES, removeEntryMirrors, loadActiveTaskSnapshot, loadLatestHandoff, listSessionEvents, } from './store.js';
|
|
12
|
-
import { createMemory, applyOutcome, Layer, } from './memory.js';
|
|
13
|
-
import { appendAuditEvent, queryAuditEvents, } from './audit.js';
|
|
14
|
-
import { promoteToGlobal, getGlobalRoot } from './shared.js';
|
|
11
|
+
import { writeEntry, writeEntryDbOnly, writeEntryMirrors, readEntry, deleteEntry, loadRecallSearchEntries, loadEntriesByIds, loadChildrenOf, loadFreshRawMemories, loadSessionRawMemories, countSessionRawMemories, DEFAULT_SEARCH_CANDIDATE_LIMIT, RECALL_DEFAULT_DENY_SCOPES, removeEntryMirrors, loadActiveTaskSnapshot, loadLatestHandoff, listSessionEvents, loadIndex, saveIndex, loadAllEntries, updateStats, isInitialized, } from './store.js';
|
|
12
|
+
import { createMemory, applyOutcome, calculateStrength, Layer, } from './memory.js';
|
|
13
|
+
import { appendAuditEvent, queryAuditEvents, auditMemories, } from './audit.js';
|
|
14
|
+
import { promoteToGlobal, getGlobalRoot, autoShare, searchBothHybrid } from './shared.js';
|
|
15
15
|
import { archiveRawMemory } from './raw-archive.js';
|
|
16
16
|
import { createApiKey, listApiKeys, revokeApiKey, } from './auth.js';
|
|
17
17
|
import { applyGoalStackBoost } from './goals.js';
|
|
18
|
+
import { markRetrieved, estimateTokens, hybridSearch, physicsSearch } from './search.js';
|
|
19
|
+
import { scopeMatch } from './scope.js';
|
|
20
|
+
import { consolidate } from './consolidate.js';
|
|
21
|
+
import { loadConfig } from './config.js';
|
|
22
|
+
import { deduplicateStore } from './dedupe.js';
|
|
23
|
+
import { computeAmbientState } from './ambient.js';
|
|
18
24
|
/**
|
|
19
25
|
* Thrown by `api.recall` when a caller's options violate a recall contract
|
|
20
26
|
* that has been opted into via env. Carries a stable `code` field for HTTP /
|
|
@@ -998,4 +1004,390 @@ export function auditList(ctx, opts) {
|
|
|
998
1004
|
closeHippoDb(db);
|
|
999
1005
|
}
|
|
1000
1006
|
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Assemble a context bundle: recalled memories (pinned-only / strength-sorted
|
|
1009
|
+
* fallback / hybrid search) + active task snapshot + session handoff + recent
|
|
1010
|
+
* session events. Budget-bounded, tenant-scoped. Mutates `last_retrieval_ids`
|
|
1011
|
+
* + emits a 'recall' audit row for non-pinned, non-'*' queries.
|
|
1012
|
+
*
|
|
1013
|
+
* Behaves like the pre-extraction `cmdContext` data-loading + selection
|
|
1014
|
+
* pipeline. CLI presentation (markdown / json / additional-context rendering)
|
|
1015
|
+
* stays in `cli.ts`.
|
|
1016
|
+
*
|
|
1017
|
+
* Tenant scope: all `loadAllEntries` / snapshot / handoff / events reads use
|
|
1018
|
+
* `ctx.tenantId`. Cross-tenant rows are filtered out.
|
|
1019
|
+
*
|
|
1020
|
+
* Returns an empty result (`entries: []`, snapshot/handoff/events undefined)
|
|
1021
|
+
* when there's nothing to surface (no memories AND no snapshot AND no handoff
|
|
1022
|
+
* AND no recent events).
|
|
1023
|
+
*/
|
|
1024
|
+
export async function getContext(ctx, opts = {}) {
|
|
1025
|
+
const pinnedOnly = opts.pinnedOnly === true;
|
|
1026
|
+
const budget = opts.budget ?? 1500;
|
|
1027
|
+
const limit = opts.limit ?? Number.POSITIVE_INFINITY;
|
|
1028
|
+
const includeRecent = opts.includeRecent ?? 0;
|
|
1029
|
+
const activeScope = opts.scope ?? '';
|
|
1030
|
+
if (budget <= 0) {
|
|
1031
|
+
return { entries: [], tokens: 0 };
|
|
1032
|
+
}
|
|
1033
|
+
// Pinned-only path is allowed against an un-initialised local store (the
|
|
1034
|
+
// UserPromptSubmit hook can run in directories without a .hippo). Non-pinned
|
|
1035
|
+
// path requires an initialised local store; callers should check first.
|
|
1036
|
+
const hasLocal = isInitialized(ctx.hippoRoot);
|
|
1037
|
+
const query = (opts.q ?? '').trim() || '*';
|
|
1038
|
+
const globalRoot = getGlobalRoot();
|
|
1039
|
+
const hasGlobal = isInitialized(globalRoot);
|
|
1040
|
+
// Tenant-scoped loads (v1.11.1 lesson: NEVER resolveTenantId({}) here).
|
|
1041
|
+
let localEntries = hasLocal ? loadAllEntries(ctx.hippoRoot, ctx.tenantId) : [];
|
|
1042
|
+
let globalEntries = hasGlobal ? loadAllEntries(globalRoot, ctx.tenantId) : [];
|
|
1043
|
+
// Filter superseded — context never includes superseded rows.
|
|
1044
|
+
localEntries = localEntries.filter((e) => !e.superseded_by);
|
|
1045
|
+
globalEntries = globalEntries.filter((e) => !e.superseded_by);
|
|
1046
|
+
const activeSnapshot = hasLocal
|
|
1047
|
+
? loadActiveTaskSnapshot(ctx.hippoRoot, ctx.tenantId)
|
|
1048
|
+
: null;
|
|
1049
|
+
const sessionHandoff = hasLocal && activeSnapshot?.session_id
|
|
1050
|
+
? loadLatestHandoff(ctx.hippoRoot, ctx.tenantId, activeSnapshot.session_id)
|
|
1051
|
+
: null;
|
|
1052
|
+
const recentSessionEvents = hasLocal && activeSnapshot?.session_id
|
|
1053
|
+
? listSessionEvents(ctx.hippoRoot, ctx.tenantId, {
|
|
1054
|
+
session_id: activeSnapshot.session_id,
|
|
1055
|
+
limit: 5,
|
|
1056
|
+
})
|
|
1057
|
+
: [];
|
|
1058
|
+
if (localEntries.length === 0 &&
|
|
1059
|
+
globalEntries.length === 0 &&
|
|
1060
|
+
!activeSnapshot &&
|
|
1061
|
+
!sessionHandoff &&
|
|
1062
|
+
recentSessionEvents.length === 0) {
|
|
1063
|
+
return { entries: [], tokens: 0 };
|
|
1064
|
+
}
|
|
1065
|
+
let selectedItems = [];
|
|
1066
|
+
let totalTokens = 0;
|
|
1067
|
+
if (pinnedOnly) {
|
|
1068
|
+
// loadConfig is safe even when local isn't initialised — returns defaults.
|
|
1069
|
+
const pinnedCfg = loadConfig(ctx.hippoRoot);
|
|
1070
|
+
if (!pinnedCfg.pinnedInject.enabled) {
|
|
1071
|
+
return { entries: [], tokens: 0 };
|
|
1072
|
+
}
|
|
1073
|
+
// Effective budget: explicit opts.budget wins over config.
|
|
1074
|
+
const effBudget = opts.budget !== undefined ? budget : pinnedCfg.pinnedInject.budget;
|
|
1075
|
+
const nowP = new Date();
|
|
1076
|
+
const selectedIds = new Set();
|
|
1077
|
+
let usedP = 0;
|
|
1078
|
+
if (includeRecent > 0) {
|
|
1079
|
+
const recent = [
|
|
1080
|
+
...localEntries.map((entry) => ({ entry, isGlobal: false })),
|
|
1081
|
+
...globalEntries.map((entry) => ({ entry, isGlobal: true })),
|
|
1082
|
+
]
|
|
1083
|
+
.sort((a, b) => {
|
|
1084
|
+
const byCreated = Date.parse(b.entry.created) - Date.parse(a.entry.created);
|
|
1085
|
+
return byCreated !== 0 ? byCreated : b.entry.id.localeCompare(a.entry.id);
|
|
1086
|
+
})
|
|
1087
|
+
.slice(0, includeRecent)
|
|
1088
|
+
.map(({ entry, isGlobal }) => ({
|
|
1089
|
+
entry,
|
|
1090
|
+
score: calculateStrength(entry, nowP) * (isGlobal ? 1 / 1.2 : 1),
|
|
1091
|
+
tokens: estimateTokens(entry.content),
|
|
1092
|
+
isGlobal,
|
|
1093
|
+
}));
|
|
1094
|
+
for (const r of recent) {
|
|
1095
|
+
if (selectedIds.has(r.entry.id))
|
|
1096
|
+
continue;
|
|
1097
|
+
if (usedP + r.tokens > effBudget)
|
|
1098
|
+
continue;
|
|
1099
|
+
selectedItems.push(r);
|
|
1100
|
+
selectedIds.add(r.entry.id);
|
|
1101
|
+
usedP += r.tokens;
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
const pinnedLocal = localEntries.filter((e) => e.pinned);
|
|
1105
|
+
const pinnedGlobal = globalEntries.filter((e) => e.pinned);
|
|
1106
|
+
if (pinnedLocal.length === 0 &&
|
|
1107
|
+
pinnedGlobal.length === 0 &&
|
|
1108
|
+
selectedItems.length === 0) {
|
|
1109
|
+
return { entries: [], tokens: 0 };
|
|
1110
|
+
}
|
|
1111
|
+
const rankedPinned = [
|
|
1112
|
+
...pinnedLocal.map((e) => ({ entry: e, isGlobal: false })),
|
|
1113
|
+
...pinnedGlobal.map((e) => ({ entry: e, isGlobal: true })),
|
|
1114
|
+
]
|
|
1115
|
+
.map(({ entry, isGlobal }) => {
|
|
1116
|
+
const scopeSig = scopeMatch(entry.tags, activeScope);
|
|
1117
|
+
const sBst = scopeSig === 1 ? 1.5 : scopeSig === -1 ? 0.5 : 1.0;
|
|
1118
|
+
return {
|
|
1119
|
+
entry,
|
|
1120
|
+
score: calculateStrength(entry, nowP) * (isGlobal ? 1 / 1.2 : 1) * sBst,
|
|
1121
|
+
tokens: estimateTokens(entry.content),
|
|
1122
|
+
isGlobal,
|
|
1123
|
+
};
|
|
1124
|
+
})
|
|
1125
|
+
.sort((a, b) => b.score - a.score);
|
|
1126
|
+
for (const r of rankedPinned) {
|
|
1127
|
+
if (selectedIds.has(r.entry.id))
|
|
1128
|
+
continue;
|
|
1129
|
+
if (usedP + r.tokens > effBudget)
|
|
1130
|
+
continue;
|
|
1131
|
+
selectedItems.push(r);
|
|
1132
|
+
selectedIds.add(r.entry.id);
|
|
1133
|
+
usedP += r.tokens;
|
|
1134
|
+
}
|
|
1135
|
+
totalTokens = usedP;
|
|
1136
|
+
}
|
|
1137
|
+
else if (query === '*') {
|
|
1138
|
+
// No query: return strongest memories by strength, up to budget.
|
|
1139
|
+
const now = new Date();
|
|
1140
|
+
const localRanked = localEntries
|
|
1141
|
+
.map((e) => ({
|
|
1142
|
+
entry: e,
|
|
1143
|
+
score: calculateStrength(e, now),
|
|
1144
|
+
tokens: estimateTokens(e.content),
|
|
1145
|
+
isGlobal: false,
|
|
1146
|
+
}))
|
|
1147
|
+
.sort((a, b) => b.score - a.score);
|
|
1148
|
+
const globalRanked = globalEntries
|
|
1149
|
+
.map((e) => ({
|
|
1150
|
+
entry: e,
|
|
1151
|
+
score: calculateStrength(e, now) * (1 / 1.2),
|
|
1152
|
+
tokens: estimateTokens(e.content),
|
|
1153
|
+
isGlobal: true,
|
|
1154
|
+
}))
|
|
1155
|
+
.sort((a, b) => b.score - a.score);
|
|
1156
|
+
const combined = [...localRanked, ...globalRanked].sort((a, b) => b.score - a.score);
|
|
1157
|
+
let used = 0;
|
|
1158
|
+
for (const r of combined) {
|
|
1159
|
+
if (used + r.tokens > budget)
|
|
1160
|
+
continue;
|
|
1161
|
+
selectedItems.push(r);
|
|
1162
|
+
used += r.tokens;
|
|
1163
|
+
}
|
|
1164
|
+
totalTokens = used;
|
|
1165
|
+
}
|
|
1166
|
+
else {
|
|
1167
|
+
// Real query: hybrid search (global + local) or physics+hybrid (local only).
|
|
1168
|
+
let results;
|
|
1169
|
+
if (hasGlobal) {
|
|
1170
|
+
const merged = await searchBothHybrid(query, ctx.hippoRoot, globalRoot, {
|
|
1171
|
+
budget,
|
|
1172
|
+
scope: activeScope,
|
|
1173
|
+
tenantId: ctx.tenantId,
|
|
1174
|
+
});
|
|
1175
|
+
const localIndex = loadIndex(ctx.hippoRoot);
|
|
1176
|
+
results = merged.map((r) => ({
|
|
1177
|
+
entry: r.entry,
|
|
1178
|
+
score: r.score,
|
|
1179
|
+
tokens: r.tokens,
|
|
1180
|
+
isGlobal: !localIndex.entries[r.entry.id],
|
|
1181
|
+
}));
|
|
1182
|
+
}
|
|
1183
|
+
else {
|
|
1184
|
+
const ctxConfig = loadConfig(ctx.hippoRoot);
|
|
1185
|
+
const usePhysicsCtx = ctxConfig.physics?.enabled !== false;
|
|
1186
|
+
const ctxResults = usePhysicsCtx
|
|
1187
|
+
? await physicsSearch(query, localEntries, {
|
|
1188
|
+
budget,
|
|
1189
|
+
hippoRoot: ctx.hippoRoot,
|
|
1190
|
+
physicsConfig: ctxConfig.physics,
|
|
1191
|
+
scope: activeScope,
|
|
1192
|
+
})
|
|
1193
|
+
: await hybridSearch(query, localEntries, {
|
|
1194
|
+
budget,
|
|
1195
|
+
hippoRoot: ctx.hippoRoot,
|
|
1196
|
+
scope: activeScope,
|
|
1197
|
+
});
|
|
1198
|
+
results = ctxResults.map((r) => ({
|
|
1199
|
+
entry: r.entry,
|
|
1200
|
+
score: r.score,
|
|
1201
|
+
tokens: r.tokens,
|
|
1202
|
+
isGlobal: false,
|
|
1203
|
+
}));
|
|
1204
|
+
}
|
|
1205
|
+
selectedItems = results;
|
|
1206
|
+
totalTokens = results.reduce((sum, r) => sum + r.tokens, 0);
|
|
1207
|
+
// A5 H4: emit recall audit row for context-mode searches (matches the
|
|
1208
|
+
// 'recall' op emitted by api.recall for parity). pinnedOnly + '*' fallback
|
|
1209
|
+
// never hit the search engines, so they don't emit (matches cmdContext).
|
|
1210
|
+
const ctxRecallMetadata = {
|
|
1211
|
+
query: query.slice(0, 200),
|
|
1212
|
+
results: selectedItems.length,
|
|
1213
|
+
mode: 'context',
|
|
1214
|
+
};
|
|
1215
|
+
if (hasLocal) {
|
|
1216
|
+
const localDb = openHippoDb(ctx.hippoRoot);
|
|
1217
|
+
try {
|
|
1218
|
+
appendAuditEvent(localDb, {
|
|
1219
|
+
tenantId: ctx.tenantId,
|
|
1220
|
+
actor: ctx.actor,
|
|
1221
|
+
op: 'recall',
|
|
1222
|
+
metadata: ctxRecallMetadata,
|
|
1223
|
+
});
|
|
1224
|
+
}
|
|
1225
|
+
finally {
|
|
1226
|
+
closeHippoDb(localDb);
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
1229
|
+
if (hasGlobal) {
|
|
1230
|
+
const globalDb = openHippoDb(globalRoot);
|
|
1231
|
+
try {
|
|
1232
|
+
appendAuditEvent(globalDb, {
|
|
1233
|
+
tenantId: ctx.tenantId,
|
|
1234
|
+
actor: ctx.actor,
|
|
1235
|
+
op: 'recall',
|
|
1236
|
+
metadata: ctxRecallMetadata,
|
|
1237
|
+
});
|
|
1238
|
+
}
|
|
1239
|
+
finally {
|
|
1240
|
+
closeHippoDb(globalDb);
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
if (limit < selectedItems.length) {
|
|
1245
|
+
selectedItems = selectedItems.slice(0, limit);
|
|
1246
|
+
totalTokens = selectedItems.reduce((sum, r) => sum + r.tokens, 0);
|
|
1247
|
+
}
|
|
1248
|
+
if (selectedItems.length === 0 &&
|
|
1249
|
+
!activeSnapshot &&
|
|
1250
|
+
!sessionHandoff &&
|
|
1251
|
+
recentSessionEvents.length === 0) {
|
|
1252
|
+
return { entries: [], tokens: 0 };
|
|
1253
|
+
}
|
|
1254
|
+
// pinnedOnly is the UserPromptSubmit hot path — read-only so pinned
|
|
1255
|
+
// memories don't inflate retrieval_count or extend half_life by 2 days per
|
|
1256
|
+
// turn over a long session.
|
|
1257
|
+
if (!pinnedOnly) {
|
|
1258
|
+
const toUpdate = selectedItems.map((s) => s.entry);
|
|
1259
|
+
const updatedEntries = markRetrieved(toUpdate);
|
|
1260
|
+
const localIndex = loadIndex(ctx.hippoRoot);
|
|
1261
|
+
for (const u of updatedEntries) {
|
|
1262
|
+
const targetRoot = localIndex.entries[u.id]
|
|
1263
|
+
? ctx.hippoRoot
|
|
1264
|
+
: hasGlobal
|
|
1265
|
+
? globalRoot
|
|
1266
|
+
: ctx.hippoRoot;
|
|
1267
|
+
writeEntry(targetRoot, u);
|
|
1268
|
+
}
|
|
1269
|
+
localIndex.last_retrieval_ids = updatedEntries.map((u) => u.id);
|
|
1270
|
+
saveIndex(ctx.hippoRoot, localIndex);
|
|
1271
|
+
updateStats(ctx.hippoRoot, { recalled: selectedItems.length });
|
|
1272
|
+
// Replace selectedItems entries with markRetrieved-updated copies so
|
|
1273
|
+
// the returned ContextResult reflects post-recall state.
|
|
1274
|
+
selectedItems = selectedItems.map((s) => ({
|
|
1275
|
+
...s,
|
|
1276
|
+
entry: updatedEntries.find((u) => u.id === s.entry.id) ?? s.entry,
|
|
1277
|
+
}));
|
|
1278
|
+
}
|
|
1279
|
+
return {
|
|
1280
|
+
entries: selectedItems,
|
|
1281
|
+
tokens: totalTokens,
|
|
1282
|
+
activeSnapshot: activeSnapshot ?? undefined,
|
|
1283
|
+
sessionHandoff: sessionHandoff ?? undefined,
|
|
1284
|
+
recentEvents: recentSessionEvents.length > 0 ? recentSessionEvents : undefined,
|
|
1285
|
+
};
|
|
1286
|
+
}
|
|
1287
|
+
/**
|
|
1288
|
+
* Run the pure-storage consolidation pipeline.
|
|
1289
|
+
*
|
|
1290
|
+
* Tenant scope note: sleep operates on the WHOLE hippoRoot (all tenants in
|
|
1291
|
+
* it), matching the pre-refactor cmdSleepCore behavior. Correct for a CLI
|
|
1292
|
+
* maintenance op invoked by the operator. But once Episode B exposes this
|
|
1293
|
+
* over HTTP `/v1/sleep`, the route MUST gate to a global-admin actor or
|
|
1294
|
+
* scope dedup/audit/delete by ctx.tenantId — otherwise a tenant-A Bearer
|
|
1295
|
+
* could dedupe and delete tenant-B's rows. See TODOS.md "Episode A
|
|
1296
|
+
* follow-ups" for the Episode B preflight checklist.
|
|
1297
|
+
*
|
|
1298
|
+
* Audit emission gap: the consolidation phases (dedup, audit-delete) do
|
|
1299
|
+
* NOT emit audit_log rows today, matching pre-refactor cmdSleepCore. Same
|
|
1300
|
+
* CLI/MCP parity gap that T6 fixed for cmdOutcome, now visible at the api
|
|
1301
|
+
* surface. Episode B should decide whether `/v1/sleep` writes a single
|
|
1302
|
+
* 'consolidate' audit row per invocation or per-phase rows per deletion.
|
|
1303
|
+
*/
|
|
1304
|
+
export async function sleep(ctx, opts = {}) {
|
|
1305
|
+
const dryRun = Boolean(opts.dryRun);
|
|
1306
|
+
// Phase 1: Consolidation.
|
|
1307
|
+
const consolidateResult = await consolidate(ctx.hippoRoot, { dryRun });
|
|
1308
|
+
const result = {
|
|
1309
|
+
active: consolidateResult.decayed,
|
|
1310
|
+
removed: consolidateResult.removed,
|
|
1311
|
+
mergedEpisodic: consolidateResult.merged,
|
|
1312
|
+
newSemantic: consolidateResult.semanticCreated,
|
|
1313
|
+
dryRun,
|
|
1314
|
+
details: consolidateResult.details,
|
|
1315
|
+
};
|
|
1316
|
+
if (dryRun)
|
|
1317
|
+
return result;
|
|
1318
|
+
// Phase 2: Dedup (post-consolidate near-duplicate cleanup).
|
|
1319
|
+
const dedupResult = deduplicateStore(ctx.hippoRoot);
|
|
1320
|
+
if (dedupResult.removed > 0) {
|
|
1321
|
+
const semDups = dedupResult.pairs.filter((p) => p.keptLayer === 'semantic' && p.removedLayer === 'semantic').length;
|
|
1322
|
+
const epiDups = dedupResult.pairs.filter((p) => p.keptLayer === 'episodic' && p.removedLayer === 'episodic').length;
|
|
1323
|
+
const crossDups = dedupResult.pairs.filter((p) => p.keptLayer !== p.removedLayer).length;
|
|
1324
|
+
result.deduped = {
|
|
1325
|
+
removed: dedupResult.removed,
|
|
1326
|
+
semDups,
|
|
1327
|
+
epiDups,
|
|
1328
|
+
crossDups,
|
|
1329
|
+
};
|
|
1330
|
+
}
|
|
1331
|
+
// Phase 3: Quality audit (remove junk, report warnings).
|
|
1332
|
+
const allEntries = loadAllEntries(ctx.hippoRoot);
|
|
1333
|
+
const auditOut = auditMemories(allEntries);
|
|
1334
|
+
if (auditOut.issues.length > 0) {
|
|
1335
|
+
const errors = auditOut.issues.filter((i) => i.severity === 'error');
|
|
1336
|
+
const warnings = auditOut.issues.filter((i) => i.severity === 'warning');
|
|
1337
|
+
if (errors.length > 0) {
|
|
1338
|
+
for (const issue of errors) {
|
|
1339
|
+
deleteEntry(ctx.hippoRoot, issue.memoryId);
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
if (errors.length > 0 || warnings.length > 0) {
|
|
1343
|
+
result.audit = {
|
|
1344
|
+
errorsRemoved: errors.length,
|
|
1345
|
+
warningCount: warnings.length,
|
|
1346
|
+
};
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
// Phase 4: Auto-share high-transfer-score memories to global.
|
|
1350
|
+
if (!opts.noShare) {
|
|
1351
|
+
const sleepConfig = loadConfig(ctx.hippoRoot);
|
|
1352
|
+
if (sleepConfig.autoShareOnSleep) {
|
|
1353
|
+
const shared = autoShare(ctx.hippoRoot, { minScore: 0.6 });
|
|
1354
|
+
if (shared.length > 0) {
|
|
1355
|
+
result.shared = shared.length;
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
// Phase 5: Post-sleep ambient state summary.
|
|
1360
|
+
const postSleepConfig = loadConfig(ctx.hippoRoot);
|
|
1361
|
+
if (postSleepConfig.ambient.enabled) {
|
|
1362
|
+
const postSleepEntries = loadAllEntries(ctx.hippoRoot).filter((e) => !e.superseded_by);
|
|
1363
|
+
if (postSleepEntries.length > 0) {
|
|
1364
|
+
result.ambient = computeAmbientState(postSleepEntries);
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
1367
|
+
return result;
|
|
1368
|
+
}
|
|
1369
|
+
// ---------------------------------------------------------------------------
|
|
1370
|
+
// outcomeForLastRecall (last-recall wrapper around outcome — Task 3)
|
|
1371
|
+
// ---------------------------------------------------------------------------
|
|
1372
|
+
/**
|
|
1373
|
+
* Apply an outcome to the ids most recently returned by `recall()`.
|
|
1374
|
+
*
|
|
1375
|
+
* Reads `loadIndex(ctx.hippoRoot).last_retrieval_ids` (per-hippoRoot local
|
|
1376
|
+
* state; not tenant-scoped at the index layer) and forwards to `outcome()`,
|
|
1377
|
+
* which DOES tenant-filter via `readEntry(..., ctx.tenantId)` — cross-tenant
|
|
1378
|
+
* ids in `last_retrieval_ids` are silently skipped, matching the MCP
|
|
1379
|
+
* `hippo_outcome` semantics.
|
|
1380
|
+
*
|
|
1381
|
+
* Do NOT tighten `loadIndex` with `tenantId` inside this helper — doing so
|
|
1382
|
+
* would break the (correct) cross-tenant-silent-skip behavior covered by
|
|
1383
|
+
* the test in `tests/api-outcome-for-last-recall.test.ts`.
|
|
1384
|
+
*/
|
|
1385
|
+
export function outcomeForLastRecall(ctx, good) {
|
|
1386
|
+
const idx = loadIndex(ctx.hippoRoot);
|
|
1387
|
+
const ids = idx.last_retrieval_ids;
|
|
1388
|
+
if (ids.length === 0)
|
|
1389
|
+
return { applied: 0, ids: [] };
|
|
1390
|
+
const { applied } = outcome(ctx, ids, good);
|
|
1391
|
+
return { applied, ids };
|
|
1392
|
+
}
|
|
1001
1393
|
//# sourceMappingURL=api.js.map
|