hippo-memory 1.55.0 → 1.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/api.d.ts +19 -9
- package/dist/api.js +112 -35
- package/dist/card-detail.d.ts +1 -1
- package/dist/card-detail.js +1 -1
- package/dist/cli/shared.d.ts +137 -0
- package/dist/cli/shared.js +830 -0
- package/dist/cli/sleep.d.ts +10 -0
- package/dist/cli/sleep.js +171 -0
- package/dist/cli.d.ts +0 -7
- package/dist/cli.js +313 -1806
- package/dist/config.d.ts +5 -0
- package/dist/config.js +21 -0
- package/dist/connectors/github/webhook.d.ts +19 -0
- package/dist/connectors/github/webhook.js +313 -0
- package/dist/connectors/slack/webhook.d.ts +22 -0
- package/dist/connectors/slack/webhook.js +203 -0
- package/dist/consolidate.js +3 -2
- package/dist/context-auto.d.ts +3 -0
- package/dist/context-auto.js +34 -0
- package/dist/customer-notes.js +2 -1
- package/dist/dashboard.js +2 -1
- package/dist/db.js +67 -1
- package/dist/decisions.js +2 -1
- package/dist/delivery-recorder.d.ts +127 -0
- package/dist/delivery-recorder.js +218 -0
- package/dist/eval-stats.d.ts +58 -0
- package/dist/eval-stats.js +111 -0
- package/dist/goals.d.ts +49 -25
- package/dist/goals.js +39 -22
- package/dist/graph-extract.js +1 -1
- package/dist/graph-recall.d.ts +1 -1
- package/dist/graph-recall.js +1 -1
- package/dist/graph.js +1 -1
- package/dist/hooks.d.ts +1 -3
- package/dist/hooks.js +2 -4
- package/dist/http-util.d.ts +31 -0
- package/dist/http-util.js +46 -0
- package/dist/incidents.js +2 -1
- package/dist/index.d.ts +5 -2
- package/dist/index.js +5 -2
- package/dist/mcp/server.js +173 -285
- package/dist/memory.d.ts +19 -0
- package/dist/memory.js +38 -0
- package/dist/policies.js +2 -1
- package/dist/predictions.js +2 -1
- package/dist/processes.js +2 -1
- package/dist/project-briefs.js +3 -1
- package/dist/prompt-recall.js +1 -1
- package/dist/recall-history.d.ts +5 -0
- package/dist/recall-history.js +9 -0
- package/dist/recall-pipeline.d.ts +101 -0
- package/dist/recall-pipeline.js +313 -0
- package/dist/recall-scope.d.ts +22 -0
- package/dist/recall-scope.js +27 -1
- package/dist/recall-trace.d.ts +69 -0
- package/dist/recall-trace.js +136 -0
- package/dist/search.d.ts +0 -20
- package/dist/search.js +2 -49
- package/dist/server.js +1901 -2384
- package/dist/skills.js +2 -1
- package/dist/store-cards.d.ts +53 -0
- package/dist/store-cards.js +512 -0
- package/dist/store.d.ts +2 -89
- package/dist/store.js +6 -562
- package/dist/tenant.d.ts +22 -0
- package/dist/tenant.js +26 -0
- package/dist/token-ledger.d.ts +2 -0
- package/dist/token-ledger.js +5 -0
- package/dist/tokenize.d.ts +2 -0
- package/dist/tokenize.js +8 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +2 -1
package/dist/memory.js
CHANGED
|
@@ -471,4 +471,42 @@ function inferValence(tags) {
|
|
|
471
471
|
return 'positive';
|
|
472
472
|
return 'neutral';
|
|
473
473
|
}
|
|
474
|
+
/**
|
|
475
|
+
* Update retrieval metadata on entries that were returned by a search.
|
|
476
|
+
* Returns the mutated copies (caller must persist to disk).
|
|
477
|
+
*
|
|
478
|
+
* EVAL-ONLY ablation (see ablation.ts): with HIPPO_ABLATE_RECALL_BOOST set,
|
|
479
|
+
* this returns the entries UNMUTATED - neutralizing all three strengthening
|
|
480
|
+
* sub-effects (clock reset, retrieval_count, half-life increment) at the
|
|
481
|
+
* single shared write site. The entries (not an empty array) must be
|
|
482
|
+
* returned because callers derive `last_retrieval_ids` from the return
|
|
483
|
+
* value, and a later `hippo outcome --good/--bad` targets those ids - an
|
|
484
|
+
* empty return would silently co-ablate the outcome channel in the
|
|
485
|
+
* strengthen-off arm. PERSISTENCE is gated separately at
|
|
486
|
+
* each persisting caller (CLI recall, api context, MCP recall/context,
|
|
487
|
+
* consolidation replay): writeEntry on identical rows still refreshes
|
|
488
|
+
* updated_at, rewrites mirrors, and marks DAG parents dirty,
|
|
489
|
+
* so those write loops skip under the flag.
|
|
490
|
+
* The default `now` honors HIPPO_FAKE_NOW (simulated-time protocols).
|
|
491
|
+
*/
|
|
492
|
+
// Confidence is deliberately absent below: it is an epistemic tier, not a
|
|
493
|
+
// recency signal, and a stored 'stale' is always a deliberate mark.
|
|
494
|
+
export function markRetrieved(entries, now = evalNow()) {
|
|
495
|
+
if (isRecallBoostAblated())
|
|
496
|
+
return entries;
|
|
497
|
+
return entries.map((e) => {
|
|
498
|
+
if (e.superseded_by)
|
|
499
|
+
return e;
|
|
500
|
+
const wrong = netWrong(e) > 0;
|
|
501
|
+
const updated = {
|
|
502
|
+
...e,
|
|
503
|
+
retrieval_count: e.retrieval_count + 1,
|
|
504
|
+
last_retrieved: wrong ? e.last_retrieved : now.toISOString(),
|
|
505
|
+
// +2 days half-life per retrieval (PLAN.md); a wrong memory keeps both, since last_retrieved is the decay anchor
|
|
506
|
+
half_life_days: wrong ? e.half_life_days : e.half_life_days + 2,
|
|
507
|
+
};
|
|
508
|
+
updated.strength = calculateStrength(updated, now);
|
|
509
|
+
return updated;
|
|
510
|
+
});
|
|
511
|
+
}
|
|
474
512
|
//# sourceMappingURL=memory.js.map
|
package/dist/policies.js
CHANGED
|
@@ -37,7 +37,8 @@
|
|
|
37
37
|
* supersede, the predecessor's UPDATE) inside writeEntry's SAVEPOINT.
|
|
38
38
|
*/
|
|
39
39
|
import { openHippoDb, closeHippoDb } from './db.js';
|
|
40
|
-
import { writeEntry
|
|
40
|
+
import { writeEntry } from './store.js';
|
|
41
|
+
import { assertTenantId } from './tenant.js';
|
|
41
42
|
import { markGraphDirty, removeGraphEntitiesForObject } from './graph.js';
|
|
42
43
|
import { createMemory, Layer } from './memory.js';
|
|
43
44
|
import { appendAuditEvent } from './audit.js';
|
package/dist/predictions.js
CHANGED
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
* this module ships the data layer.
|
|
26
26
|
*/
|
|
27
27
|
import { openHippoDb, closeHippoDb } from './db.js';
|
|
28
|
-
import { writeEntry
|
|
28
|
+
import { writeEntry } from './store.js';
|
|
29
|
+
import { assertTenantId } from './tenant.js';
|
|
29
30
|
import { createMemory, Layer } from './memory.js';
|
|
30
31
|
import { appendAuditEvent } from './audit.js';
|
|
31
32
|
import { loadConfig } from './config.js';
|
package/dist/processes.js
CHANGED
|
@@ -31,7 +31,8 @@
|
|
|
31
31
|
* them back. Pattern matches saveDecision (decisions.ts).
|
|
32
32
|
*/
|
|
33
33
|
import { openHippoDb, closeHippoDb } from './db.js';
|
|
34
|
-
import { writeEntry
|
|
34
|
+
import { writeEntry } from './store.js';
|
|
35
|
+
import { assertTenantId } from './tenant.js';
|
|
35
36
|
import { createMemory, Layer } from './memory.js';
|
|
36
37
|
import { appendAuditEvent } from './audit.js';
|
|
37
38
|
import { objectHalfLifeDays } from './half-life-migration.js';
|
package/dist/project-briefs.js
CHANGED
|
@@ -22,7 +22,9 @@
|
|
|
22
22
|
* closed (retired).
|
|
23
23
|
*/
|
|
24
24
|
import { openHippoDb, closeHippoDb } from './db.js';
|
|
25
|
-
import { writeEntry
|
|
25
|
+
import { writeEntry } from './store.js';
|
|
26
|
+
import { assertTenantId } from './tenant.js';
|
|
27
|
+
import { RECALL_DEFAULT_DENY_SCOPES } from './recall-scope.js';
|
|
26
28
|
import { markGraphDirty, removeGraphEntitiesForObject } from './graph.js';
|
|
27
29
|
import { createMemory, Layer } from './memory.js';
|
|
28
30
|
import { appendAuditEvent } from './audit.js';
|
package/dist/prompt-recall.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** Z1: recall gated on the hook prompt, not the five newest memories (pure, no I/O).
|
|
2
2
|
* See docs/plans/2026-09-26-z1-prompt-recall.md. */
|
|
3
|
-
import { tokenize } from './
|
|
3
|
+
import { tokenize } from './tokenize.js';
|
|
4
4
|
import { STOP_WORDS } from './audit.js';
|
|
5
5
|
// Latency bound, not tuned: fixed in the prereg regardless of gate config.
|
|
6
6
|
export const PROMPT_RECALL_MAX_CHARS = 4000;
|
package/dist/recall-history.d.ts
CHANGED
|
@@ -124,4 +124,9 @@ export declare function getOrCreateRing(map: Map<string, RingBuffer>, key: strin
|
|
|
124
124
|
export declare function appendRecall(ring: RingBuffer, queryHash: number, topMemoryId: string | null, anchoredOn?: string): void;
|
|
125
125
|
/** Snapshot a ring as a readonly RecallHistorySnapshot. */
|
|
126
126
|
export declare function snapshotRing(ring: RingBuffer): RecallHistorySnapshot;
|
|
127
|
+
/**
|
|
128
|
+
* Whether a recall bias hint is enabled. Reads the env at call time, so
|
|
129
|
+
* `HIPPO_ANCHORING=off` or `HIPPO_AVAILABILITY=off` disables only that kind.
|
|
130
|
+
*/
|
|
131
|
+
export declare function biasHintEnabled(kind: 'anchoring' | 'availability'): boolean;
|
|
127
132
|
//# sourceMappingURL=recall-history.d.ts.map
|
package/dist/recall-history.js
CHANGED
|
@@ -232,4 +232,13 @@ export function appendRecall(ring, queryHash, topMemoryId, anchoredOn) {
|
|
|
232
232
|
export function snapshotRing(ring) {
|
|
233
233
|
return ring.snapshot();
|
|
234
234
|
}
|
|
235
|
+
/**
|
|
236
|
+
* Whether a recall bias hint is enabled. Reads the env at call time, so
|
|
237
|
+
* `HIPPO_ANCHORING=off` or `HIPPO_AVAILABILITY=off` disables only that kind.
|
|
238
|
+
*/
|
|
239
|
+
export function biasHintEnabled(kind) {
|
|
240
|
+
return kind === 'anchoring'
|
|
241
|
+
? process.env.HIPPO_ANCHORING !== 'off'
|
|
242
|
+
: process.env.HIPPO_AVAILABILITY !== 'off';
|
|
243
|
+
}
|
|
235
244
|
//# sourceMappingURL=recall-history.js.map
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { type GoalRecallLogRow } from './goals.js';
|
|
2
|
+
import { type MemoryEntry } from './memory.js';
|
|
3
|
+
import type { PhysicsConfig } from './physics-config.js';
|
|
4
|
+
import type { RerankerFn } from './rerankers/types.js';
|
|
5
|
+
import { type ResultCost, type SearchResult } from './search.js';
|
|
6
|
+
/** Stores rankRecall reads and where it sends operator notes. */
|
|
7
|
+
export interface RankRecallCtx {
|
|
8
|
+
hippoRoot: string;
|
|
9
|
+
/** A second store searched beside `hippoRoot`; leave undefined when there is none. */
|
|
10
|
+
globalRoot?: string;
|
|
11
|
+
tenantId: string;
|
|
12
|
+
/** Receives each operator note when the pipeline reaches it, so it interleaves with other stderr in order. */
|
|
13
|
+
note?: (line: string) => void;
|
|
14
|
+
}
|
|
15
|
+
/** Search-engine choice and tuning. */
|
|
16
|
+
export interface RecallSearchOpts {
|
|
17
|
+
usePhysics: boolean;
|
|
18
|
+
physicsConfig: PhysicsConfig;
|
|
19
|
+
multihop: boolean;
|
|
20
|
+
/** Graph-stream rrf fusion over the local store; hop and seed counts fall back to the engine defaults. */
|
|
21
|
+
graphStream?: RecallGraphStream;
|
|
22
|
+
mmr: boolean;
|
|
23
|
+
mmrLambda: number;
|
|
24
|
+
localBump: number;
|
|
25
|
+
minResults?: number;
|
|
26
|
+
/** Score breakdowns for `hippo explain`; explain's physics call also leaves out the bi-temporal flags. */
|
|
27
|
+
explain: boolean;
|
|
28
|
+
}
|
|
29
|
+
/** `--graph-hops` and `--graph-seeds`. */
|
|
30
|
+
export interface RecallGraphStream {
|
|
31
|
+
hops?: number;
|
|
32
|
+
seeds?: number;
|
|
33
|
+
}
|
|
34
|
+
/** `--hops` and `--max-neighbors`. */
|
|
35
|
+
export interface RecallGraphHops {
|
|
36
|
+
hops: number;
|
|
37
|
+
maxNeighbors: number;
|
|
38
|
+
}
|
|
39
|
+
/** A reranker from the registry and how many head rows it sees. */
|
|
40
|
+
export interface RecallReranker {
|
|
41
|
+
fn: RerankerFn;
|
|
42
|
+
topK: number;
|
|
43
|
+
}
|
|
44
|
+
/** A stage rankRecall can stop before, in pipeline order. */
|
|
45
|
+
export type RankStage = 'expand' | 'rerank' | 'salience' | 'outcome' | 'layer';
|
|
46
|
+
/** Everything that shapes one ranking, already parsed and validated. */
|
|
47
|
+
export interface RankRecallOpts {
|
|
48
|
+
query: string;
|
|
49
|
+
/** Tokens the engines may spend, priced by `cost`. */
|
|
50
|
+
budget: number;
|
|
51
|
+
/** Price of one result, usually the tokens its printed line takes. */
|
|
52
|
+
cost: ResultCost;
|
|
53
|
+
limit: number;
|
|
54
|
+
/** Record a rerank trace step for each score change. */
|
|
55
|
+
why?: boolean;
|
|
56
|
+
includeSuperseded: boolean;
|
|
57
|
+
asOf?: string;
|
|
58
|
+
/** `--scope`: unlocks that envelope scope on top of the default-admitted set. */
|
|
59
|
+
explicitScope: string | null;
|
|
60
|
+
/** Scope used for boosting only, never for filtering. */
|
|
61
|
+
activeScope: string | null;
|
|
62
|
+
search: RecallSearchOpts;
|
|
63
|
+
graphHops?: RecallGraphHops;
|
|
64
|
+
evcAdaptive?: boolean;
|
|
65
|
+
filterConflicts?: boolean;
|
|
66
|
+
valueAware?: boolean;
|
|
67
|
+
rerankUtility?: boolean;
|
|
68
|
+
reranker?: RecallReranker;
|
|
69
|
+
/** Explicit goal tag; when set, the session goal stack is skipped. */
|
|
70
|
+
goalTag?: string;
|
|
71
|
+
/** Session whose active goals boost matching rows. */
|
|
72
|
+
sessionId?: string;
|
|
73
|
+
salienceThreshold?: number;
|
|
74
|
+
outcome?: string;
|
|
75
|
+
layer?: string;
|
|
76
|
+
/** The caller rejected a flag this stage reads; ranking stops before it so the notes and goal-log rows
|
|
77
|
+
* produced up to there match the CLI's error path. */
|
|
78
|
+
haltBefore?: RankStage;
|
|
79
|
+
}
|
|
80
|
+
/** Ranked results and what the caller needs to report and persist them. */
|
|
81
|
+
export interface RankRecallResult {
|
|
82
|
+
results: SearchResult[];
|
|
83
|
+
/** The candidate pools after the scope and bi-temporal filters. */
|
|
84
|
+
localEntries: MemoryEntry[];
|
|
85
|
+
globalEntries: MemoryEntry[];
|
|
86
|
+
/** Candidates loaded, before any filter. */
|
|
87
|
+
totalCandidates: number;
|
|
88
|
+
/** Rows dropped by a named filter (scope, bi-temporal, conflicts, outcome, layer). */
|
|
89
|
+
droppedPreRank: number;
|
|
90
|
+
/** Rows graph expansion surfaced that the lexical pool never held. */
|
|
91
|
+
graphAdded: number;
|
|
92
|
+
/** goal_recall_log rows the session goal boost earned; the caller writes them. */
|
|
93
|
+
goalRecallLog: GoalRecallLogRow[];
|
|
94
|
+
/** True when ranking stopped at `haltBefore`. */
|
|
95
|
+
halted: boolean;
|
|
96
|
+
}
|
|
97
|
+
/** Ranks memories for a query as `hippo recall` does, through the `limit` slice. Reads stores, embeddings and
|
|
98
|
+
* physics state; writes nothing and prints nothing (notes go to `ctx.note`). The caller supplies the cost
|
|
99
|
+
* function and reranker in `opts`, and persists the returned goal-log rows. */
|
|
100
|
+
export declare function rankRecall(ctx: RankRecallCtx, opts: RankRecallOpts): Promise<RankRecallResult>;
|
|
101
|
+
//# sourceMappingURL=recall-pipeline.d.ts.map
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
// Ranking core shared by `hippo recall` and `hippo explain`: load, search, expand, re-rank and filter, with no
|
|
2
|
+
// writes and no direct output. Callers own flag parsing, printing, budget fitting and persistence.
|
|
3
|
+
import { evalNow } from './ablation.js';
|
|
4
|
+
import { oneCopyPerMemory } from './api.js';
|
|
5
|
+
import { compareEntryIdentity } from './compare.js';
|
|
6
|
+
import { closeHippoDb, openHippoDb } from './db.js';
|
|
7
|
+
import { isEmbeddingAvailable } from './embeddings.js';
|
|
8
|
+
import { computeGoalStackBoost } from './goals.js';
|
|
9
|
+
import { graphExpandRecall } from './graph-recall.js';
|
|
10
|
+
import { DEFAULT_GRAPH_STREAM_WEIGHT } from './graph-stream.js';
|
|
11
|
+
import { Layer } from './memory.js';
|
|
12
|
+
import { multihopSearch } from './multihop.js';
|
|
13
|
+
import { passesCliRecallScopeFilter } from './recall-scope.js';
|
|
14
|
+
import { hybridSearch, physicsSearch, textOverlap } from './search.js';
|
|
15
|
+
import { searchBothHybrid } from './shared.js';
|
|
16
|
+
import { loadRecallSearchEntries } from './store.js';
|
|
17
|
+
import { tokenize as tokenizeQuery } from './tokenize.js';
|
|
18
|
+
/** Ranks memories for a query as `hippo recall` does, through the `limit` slice. Reads stores, embeddings and
|
|
19
|
+
* physics state; writes nothing and prints nothing (notes go to `ctx.note`). The caller supplies the cost
|
|
20
|
+
* function and reranker in `opts`, and persists the returned goal-log rows. */
|
|
21
|
+
export async function rankRecall(ctx, opts) {
|
|
22
|
+
const pool = loadRecallPool(ctx, opts);
|
|
23
|
+
const state = { results: [], droppedPreRank: pool.dropped, graphAdded: 0, goalRecallLog: [] };
|
|
24
|
+
const done = (halted) => ({
|
|
25
|
+
results: state.results,
|
|
26
|
+
localEntries: pool.local,
|
|
27
|
+
globalEntries: pool.global,
|
|
28
|
+
totalCandidates: pool.total,
|
|
29
|
+
droppedPreRank: state.droppedPreRank,
|
|
30
|
+
graphAdded: state.graphAdded,
|
|
31
|
+
goalRecallLog: state.goalRecallLog,
|
|
32
|
+
halted,
|
|
33
|
+
});
|
|
34
|
+
state.results = await searchPool(ctx, opts, pool);
|
|
35
|
+
if (opts.haltBefore === 'expand')
|
|
36
|
+
return done(true);
|
|
37
|
+
if (opts.graphHops)
|
|
38
|
+
expandGraph(ctx, opts, opts.graphHops, state);
|
|
39
|
+
applyPfcRerankers(opts, state);
|
|
40
|
+
if (opts.haltBefore === 'rerank')
|
|
41
|
+
return done(true);
|
|
42
|
+
if (opts.reranker)
|
|
43
|
+
state.results = await applyReranker(opts, opts.reranker, state.results);
|
|
44
|
+
applyGoalBoosts(ctx, opts, state);
|
|
45
|
+
if (opts.haltBefore === 'salience')
|
|
46
|
+
return done(true);
|
|
47
|
+
if (opts.salienceThreshold !== undefined)
|
|
48
|
+
state.results = applySalience(opts, opts.salienceThreshold, state.results);
|
|
49
|
+
if (opts.haltBefore === 'outcome')
|
|
50
|
+
return done(true);
|
|
51
|
+
if (opts.outcome)
|
|
52
|
+
dropUnless(state, (r) => r.entry.layer !== Layer.Trace || r.entry.trace_outcome === opts.outcome);
|
|
53
|
+
if (opts.haltBefore === 'layer')
|
|
54
|
+
return done(true);
|
|
55
|
+
if (opts.layer)
|
|
56
|
+
dropUnless(state, (r) => r.entry.layer === opts.layer);
|
|
57
|
+
if (opts.limit < state.results.length)
|
|
58
|
+
state.results = state.results.slice(0, opts.limit);
|
|
59
|
+
return done(false);
|
|
60
|
+
}
|
|
61
|
+
function loadRecallPool(ctx, opts) {
|
|
62
|
+
// An explicit --scope unlocks that envelope scope; the regex-only `<source>:private:*` deny is the JS half below.
|
|
63
|
+
const requested = opts.explicitScope || undefined;
|
|
64
|
+
const loadSuperseded = opts.includeSuperseded || Boolean(opts.asOf);
|
|
65
|
+
let local = loadRecallSearchEntries(ctx.hippoRoot, opts.query, undefined, ctx.tenantId, requested, 'additive', loadSuperseded);
|
|
66
|
+
let global = ctx.globalRoot
|
|
67
|
+
? loadRecallSearchEntries(ctx.globalRoot, opts.query, undefined, ctx.tenantId, requested, 'additive', loadSuperseded)
|
|
68
|
+
: [];
|
|
69
|
+
// SQL-excluded rows are pre-candidate, so the total is taken before the JS filters, which normally drop nothing.
|
|
70
|
+
const total = local.length + global.length;
|
|
71
|
+
const passes = (e) => passesCliRecallScopeFilter(e.scope ?? null, requested);
|
|
72
|
+
local = local.filter(passes);
|
|
73
|
+
global = global.filter(passes);
|
|
74
|
+
if (opts.asOf) {
|
|
75
|
+
local = currentAsOf(local, opts.asOf);
|
|
76
|
+
global = currentAsOf(global, opts.asOf);
|
|
77
|
+
}
|
|
78
|
+
else if (!opts.includeSuperseded) {
|
|
79
|
+
local = local.filter((e) => !e.superseded_by);
|
|
80
|
+
global = global.filter((e) => !e.superseded_by);
|
|
81
|
+
}
|
|
82
|
+
return { local, global, total, dropped: total - (local.length + global.length) };
|
|
83
|
+
}
|
|
84
|
+
/** Rows that were true at `asOf`: valid by then, and not yet replaced by a successor in the same pool. */
|
|
85
|
+
function currentAsOf(entries, asOf) {
|
|
86
|
+
const asOfDate = new Date(asOf);
|
|
87
|
+
const successorValidFrom = new Map();
|
|
88
|
+
for (const e of entries) {
|
|
89
|
+
if (e.superseded_by) {
|
|
90
|
+
const successor = entries.find(s => s.id === e.superseded_by);
|
|
91
|
+
if (successor)
|
|
92
|
+
successorValidFrom.set(e.id, successor.valid_from);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return entries.filter(e => {
|
|
96
|
+
if (new Date(e.valid_from) > asOfDate)
|
|
97
|
+
return false;
|
|
98
|
+
if (!e.superseded_by)
|
|
99
|
+
return true;
|
|
100
|
+
const succVf = successorValidFrom.get(e.id);
|
|
101
|
+
return succVf ? new Date(succVf) > asOfDate : true;
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
async function searchPool(ctx, opts, pool) {
|
|
105
|
+
const { query, budget, cost, includeSuperseded, asOf, activeScope: scope, search } = opts;
|
|
106
|
+
const { mmr, mmrLambda, minResults, explain } = search;
|
|
107
|
+
const globalRoot = pool.global.length > 0 ? ctx.globalRoot : undefined;
|
|
108
|
+
if (search.graphStream) {
|
|
109
|
+
// Without embeddings hybridSearch falls back to BM25-only and the stream is inert; say so rather than no-op.
|
|
110
|
+
if (!isEmbeddingAvailable()) {
|
|
111
|
+
ctx.note?.('[note] --graph-stream needs embeddings (rrf fusion); none available, so the graph stream is inert. Run `hippo embed` first.');
|
|
112
|
+
}
|
|
113
|
+
if (globalRoot)
|
|
114
|
+
ctx.note?.('[note] --graph-stream searches the local store only; global graph fusion is a follow-up.');
|
|
115
|
+
// With seedCount or fewer candidates every one is a seed and the stream degrades to the 2-list fusion.
|
|
116
|
+
return hybridSearch(query, pool.local, {
|
|
117
|
+
budget, cost, hippoRoot: ctx.hippoRoot, mmr, mmrLambda, minResults, scope, includeSuperseded, asOf,
|
|
118
|
+
scoring: 'rrf',
|
|
119
|
+
graphStream: { weight: DEFAULT_GRAPH_STREAM_WEIGHT, tenantId: ctx.tenantId, hops: search.graphStream.hops, seedCount: search.graphStream.seeds },
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
if (search.multihop) {
|
|
123
|
+
// Unlike searchBothHybrid below, multihop ranks one pooled list, so a shared memory's two copies both compete.
|
|
124
|
+
const allEntries = oneCopyPerMemory(pool.local, pool.global, evalNow()).flat();
|
|
125
|
+
return multihopSearch(query, allEntries, { budget, cost, hippoRoot: ctx.hippoRoot, minResults, includeSuperseded, asOf });
|
|
126
|
+
}
|
|
127
|
+
if (search.usePhysics && !globalRoot) {
|
|
128
|
+
// Explain has never passed the bi-temporal flags to physics; keeping that holds its output steady.
|
|
129
|
+
const temporal = explain ? {} : { minResults, includeSuperseded, asOf };
|
|
130
|
+
return physicsSearch(query, pool.local, { budget, cost, hippoRoot: ctx.hippoRoot, physicsConfig: search.physicsConfig, scope, explain, ...temporal });
|
|
131
|
+
}
|
|
132
|
+
if (globalRoot) {
|
|
133
|
+
// searchBothHybrid reloads candidates itself, so the scope rule is passed in rather than inherited from the pool.
|
|
134
|
+
return searchBothHybrid(query, ctx.hippoRoot, globalRoot, {
|
|
135
|
+
budget, cost, explain, mmr, mmrLambda, localBump: search.localBump, minResults, scope, tenantId: ctx.tenantId,
|
|
136
|
+
includeSuperseded, asOf,
|
|
137
|
+
recallScope: opts.explicitScope ? { requested: opts.explicitScope, additive: true } : {},
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
return hybridSearch(query, pool.local, { budget, cost, hippoRoot: ctx.hippoRoot, explain, mmr, mmrLambda, minResults, scope, includeSuperseded, asOf });
|
|
141
|
+
}
|
|
142
|
+
/** Adds memories reached by walking the entity graph out from the lexical seeds. */
|
|
143
|
+
function expandGraph(ctx, opts, graph, state) {
|
|
144
|
+
if (graph.hops <= 0)
|
|
145
|
+
return;
|
|
146
|
+
// Expansion both adds neighbours and evicts weak base rows, so compare id sets: a net count would hide both.
|
|
147
|
+
const beforeGraphIds = new Set(state.results.map((r) => r.entry.id));
|
|
148
|
+
state.results = graphExpandRecall(state.results, {
|
|
149
|
+
hops: graph.hops,
|
|
150
|
+
maxNeighbors: graph.maxNeighbors,
|
|
151
|
+
hippoRoot: ctx.hippoRoot,
|
|
152
|
+
globalRoot: ctx.globalRoot !== ctx.hippoRoot ? ctx.globalRoot : undefined,
|
|
153
|
+
tenantId: ctx.tenantId,
|
|
154
|
+
includeSuperseded: opts.includeSuperseded,
|
|
155
|
+
asOf: opts.asOf,
|
|
156
|
+
budget: opts.budget,
|
|
157
|
+
cost: opts.cost,
|
|
158
|
+
minResults: opts.search.minResults ?? 1,
|
|
159
|
+
recallScope: opts.explicitScope ? { requested: opts.explicitScope, additive: true } : {},
|
|
160
|
+
});
|
|
161
|
+
for (const r of state.results) {
|
|
162
|
+
if (!beforeGraphIds.has(r.entry.id))
|
|
163
|
+
state.graphAdded++;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
/** Appends one rerank step to a re-scored row when `why` is on. */
|
|
167
|
+
function traced(opts, prev, next, step) {
|
|
168
|
+
if (opts.why)
|
|
169
|
+
next.rerankTrace = [...(prev.rerankTrace ?? []), step];
|
|
170
|
+
return next;
|
|
171
|
+
}
|
|
172
|
+
/** Re-sort after a score change. Plain and stable on purpose: ties keep the prior rank, not a content order. */
|
|
173
|
+
function byScore(results) {
|
|
174
|
+
return results.sort((a, b) => b.score - a.score);
|
|
175
|
+
}
|
|
176
|
+
function applyPfcRerankers(opts, state) {
|
|
177
|
+
if (opts.evcAdaptive && state.results.length >= 2)
|
|
178
|
+
state.results = evcAdaptive(opts.query, state.results);
|
|
179
|
+
if (opts.filterConflicts) {
|
|
180
|
+
dropUnless(state, (r) => !r.entry.superseded_by);
|
|
181
|
+
// Recorded conflicts only: a lexical-overlap gate once wrecked benchmark recall. Down-rank, never delete.
|
|
182
|
+
const presentIds = new Set(state.results.map((r) => r.entry.id));
|
|
183
|
+
state.results = byScore(state.results.map((r) => {
|
|
184
|
+
const hasPeerInResults = (r.entry.conflicts_with || []).some((peerId) => presentIds.has(peerId));
|
|
185
|
+
if (!hasPeerInResults)
|
|
186
|
+
return r;
|
|
187
|
+
const next = { ...r, score: r.score * 0.3 };
|
|
188
|
+
return traced(opts, r, next, { stage: 'interference', multiplier: 0.3, scoreBefore: r.score, scoreAfter: next.score });
|
|
189
|
+
}));
|
|
190
|
+
}
|
|
191
|
+
if (opts.valueAware && state.results.length >= 1) {
|
|
192
|
+
// Wider clamp than the always-on outcome boost, so outcome history can decide the order.
|
|
193
|
+
state.results = byScore(state.results.map((r) => {
|
|
194
|
+
const pos = r.entry.outcome_positive ?? 0;
|
|
195
|
+
const neg = r.entry.outcome_negative ?? 0;
|
|
196
|
+
if (pos === 0 && neg === 0)
|
|
197
|
+
return r;
|
|
198
|
+
const valueMult = Math.max(0.7, Math.min(1.3, 1 + 0.3 * Math.tanh(pos - neg)));
|
|
199
|
+
const next = { ...r, score: r.score * valueMult };
|
|
200
|
+
return traced(opts, r, next, { stage: 'value', multiplier: valueMult, scoreBefore: r.score, scoreAfter: next.score });
|
|
201
|
+
}));
|
|
202
|
+
}
|
|
203
|
+
if (opts.rerankUtility) {
|
|
204
|
+
// utility = score * (0.5 + 0.5 * strength) * (1 - min(0.3, tokens / 10000)); long evidence-rich rows pay for length.
|
|
205
|
+
state.results = byScore(state.results.map((r) => {
|
|
206
|
+
const strength = typeof r.entry.strength === 'number' ? r.entry.strength : 1.0;
|
|
207
|
+
const utilityMult = (0.5 + 0.5 * strength) * (1 - Math.min(0.3, (r.tokens || 0) / 10000));
|
|
208
|
+
const utility = r.score * utilityMult;
|
|
209
|
+
return traced(opts, r, { ...r, score: utility }, { stage: 'utility', multiplier: utilityMult, scoreBefore: r.score, scoreAfter: utility });
|
|
210
|
+
}));
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
/** When the top hits are near-duplicates (same topic, different facts), surface the newest on-topic row first. */
|
|
214
|
+
function evcAdaptive(query, results) {
|
|
215
|
+
const slice = results.slice(0, Math.min(3, results.length));
|
|
216
|
+
let pairs = 0;
|
|
217
|
+
let overlapSum = 0;
|
|
218
|
+
for (let i = 0; i < slice.length; i++) {
|
|
219
|
+
for (let j = i + 1; j < slice.length; j++) {
|
|
220
|
+
overlapSum += textOverlap(slice[i].entry.content, slice[j].entry.content);
|
|
221
|
+
pairs++;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
if ((pairs > 0 ? overlapSum / pairs : 0) < 0.4)
|
|
225
|
+
return results;
|
|
226
|
+
const poolSize = Math.min(results.length, Math.max(slice.length * 3, 9));
|
|
227
|
+
const pool = results.slice(0, poolSize);
|
|
228
|
+
const scoreFloor = pool.reduce((m, r) => Math.max(m, r.score), 0) * 0.5;
|
|
229
|
+
// Query coverage catches the differently phrased update a score floor alone would miss.
|
|
230
|
+
const queryTokens = new Set(tokenizeQuery(query));
|
|
231
|
+
const onTopic = [];
|
|
232
|
+
const offTopic = [];
|
|
233
|
+
for (const r of pool) {
|
|
234
|
+
let hits = 0;
|
|
235
|
+
if (queryTokens.size > 0) {
|
|
236
|
+
const candTokens = new Set(tokenizeQuery(r.entry.content));
|
|
237
|
+
for (const t of queryTokens)
|
|
238
|
+
if (candTokens.has(t))
|
|
239
|
+
hits++;
|
|
240
|
+
}
|
|
241
|
+
const queryCoverage = queryTokens.size > 0 ? hits / queryTokens.size : 0;
|
|
242
|
+
(r.score >= scoreFloor || queryCoverage >= 0.6 ? onTopic : offTopic).push(r);
|
|
243
|
+
}
|
|
244
|
+
// Recency is the primary key; identity only breaks exact-timestamp ties.
|
|
245
|
+
onTopic.sort((a, b) => {
|
|
246
|
+
const ta = new Date(a.entry.created).getTime();
|
|
247
|
+
const tb = new Date(b.entry.created).getTime();
|
|
248
|
+
return tb !== ta ? tb - ta : compareEntryIdentity(a.entry, b.entry);
|
|
249
|
+
});
|
|
250
|
+
return [...onTopic, ...offTopic, ...results.slice(poolSize)];
|
|
251
|
+
}
|
|
252
|
+
async function applyReranker(opts, reranker, results) {
|
|
253
|
+
const { fn, topK } = reranker;
|
|
254
|
+
const rerankInput = results.slice(0, topK).map((r, i) => ({ ...r, preRerankRank: i + 1 }));
|
|
255
|
+
const reranked = await fn(opts.query, rerankInput, { topK });
|
|
256
|
+
// The reranker's score becomes `score` so later stages that sort by score keep its order.
|
|
257
|
+
const withPostRank = reranked.map((r, i) => traced(opts, r, { ...r, score: r.rerankScore, postRerankRank: i + 1 }, { stage: 'reranker', scoreBefore: r.score, scoreAfter: r.rerankScore }));
|
|
258
|
+
return [...withPostRank, ...results.slice(topK)];
|
|
259
|
+
}
|
|
260
|
+
function applyGoalBoosts(ctx, opts, state) {
|
|
261
|
+
const goalTag = opts.goalTag ?? '';
|
|
262
|
+
if (goalTag) {
|
|
263
|
+
// Its own trace stage: `goal` is the explicit flag, `goal-boost` the session stack it replaces.
|
|
264
|
+
state.results = byScore(state.results.map((r) => {
|
|
265
|
+
if (!r.entry.tags?.includes(goalTag))
|
|
266
|
+
return r;
|
|
267
|
+
const boosted = { ...r, score: r.score * 1.5 };
|
|
268
|
+
return traced(opts, r, boosted, { stage: 'goal', multiplier: 1.5, scoreBefore: r.score, scoreAfter: r.score * 1.5, note: `--goal ${goalTag}` });
|
|
269
|
+
}));
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
if (!opts.sessionId)
|
|
273
|
+
return;
|
|
274
|
+
const db = openHippoDb(ctx.hippoRoot);
|
|
275
|
+
// The helper re-spreads rows, so its steps come back in a map keyed by entry id.
|
|
276
|
+
const goalBoostTrace = opts.why ? new Map() : undefined;
|
|
277
|
+
try {
|
|
278
|
+
const boost = computeGoalStackBoost(db, state.results, {
|
|
279
|
+
sessionId: opts.sessionId,
|
|
280
|
+
tenantId: ctx.tenantId,
|
|
281
|
+
limit: opts.limit,
|
|
282
|
+
trace: goalBoostTrace,
|
|
283
|
+
});
|
|
284
|
+
state.results = boost.results;
|
|
285
|
+
state.goalRecallLog = boost.log;
|
|
286
|
+
}
|
|
287
|
+
finally {
|
|
288
|
+
closeHippoDb(db);
|
|
289
|
+
}
|
|
290
|
+
if (goalBoostTrace && goalBoostTrace.size > 0) {
|
|
291
|
+
state.results = state.results.map((r) => {
|
|
292
|
+
const step = goalBoostTrace.get(r.entry.id);
|
|
293
|
+
return step ? { ...r, rerankTrace: [...(r.rerankTrace ?? []), step] } : r;
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
/** Soft-demotes rarely recalled rows: score *= max(0.5, retrieval_count / T); never drops one. */
|
|
298
|
+
function applySalience(opts, threshold, results) {
|
|
299
|
+
return byScore(results.map((r) => {
|
|
300
|
+
const count = r.entry.retrieval_count ?? 0;
|
|
301
|
+
if (count >= threshold)
|
|
302
|
+
return r;
|
|
303
|
+
const mult = Math.max(0.5, count / threshold);
|
|
304
|
+
const next = { ...r, score: r.score * mult };
|
|
305
|
+
return traced(opts, r, next, { stage: 'retrieval-count-downweight', multiplier: mult, scoreBefore: r.score, scoreAfter: next.score });
|
|
306
|
+
}));
|
|
307
|
+
}
|
|
308
|
+
function dropUnless(state, keep) {
|
|
309
|
+
const before = state.results.length;
|
|
310
|
+
state.results = state.results.filter(keep);
|
|
311
|
+
state.droppedPreRank += before - state.results.length;
|
|
312
|
+
}
|
|
313
|
+
//# sourceMappingURL=recall-pipeline.js.map
|
package/dist/recall-scope.d.ts
CHANGED
|
@@ -6,6 +6,28 @@
|
|
|
6
6
|
* these for its own call sites AND re-exports them for back-compat
|
|
7
7
|
* (`api.isPrivateScope`, test imports of `passesScopeFilterForRecall`).
|
|
8
8
|
*/
|
|
9
|
+
/**
|
|
10
|
+
* Literal scopes excluded from recall by default-deny when the
|
|
11
|
+
* caller passes no `scope`. The SQL clause in `loadSearchRows` and the JS
|
|
12
|
+
* helper `passesScopeFilterForRecall` (src/api.ts) both read from this
|
|
13
|
+
* constant. Adding a deny scope is a one-place change.
|
|
14
|
+
*
|
|
15
|
+
* Regex-based denies (e.g. `<source>:private:*`) stay in
|
|
16
|
+
* `passesScopeFilterForRecall` as a separate JS step — they don't translate
|
|
17
|
+
* cleanly to SQL.
|
|
18
|
+
*
|
|
19
|
+
* Invariant: never empty. An empty array would silently allow quarantine
|
|
20
|
+
* scopes through both paths (SQL clause omitted, JS check vacuous). The
|
|
21
|
+
* module-load assertion below pins this loudly.
|
|
22
|
+
*/
|
|
23
|
+
export declare const RECALL_DEFAULT_DENY_SCOPES: readonly ["unknown:legacy"];
|
|
24
|
+
/**
|
|
25
|
+
* @internal Runtime guard against a future maintainer blanking a
|
|
26
|
+
* load-bearing literal array. Extracted from the inline guard so the throw
|
|
27
|
+
* path is directly testable. `as const` arrays widen via `readonly T[]` at
|
|
28
|
+
* the call site so the empty case is reachable at runtime.
|
|
29
|
+
*/
|
|
30
|
+
export declare function assertNonEmpty<T>(arr: readonly T[], name: string): void;
|
|
9
31
|
/**
|
|
10
32
|
* v1.2.1: source-agnostic private-scope detector. A scope string is treated
|
|
11
33
|
* as private when it has the shape `<lowercase-source>:private:<rest>`.
|
package/dist/recall-scope.js
CHANGED
|
@@ -6,7 +6,33 @@
|
|
|
6
6
|
* these for its own call sites AND re-exports them for back-compat
|
|
7
7
|
* (`api.isPrivateScope`, test imports of `passesScopeFilterForRecall`).
|
|
8
8
|
*/
|
|
9
|
-
|
|
9
|
+
/**
|
|
10
|
+
* Literal scopes excluded from recall by default-deny when the
|
|
11
|
+
* caller passes no `scope`. The SQL clause in `loadSearchRows` and the JS
|
|
12
|
+
* helper `passesScopeFilterForRecall` (src/api.ts) both read from this
|
|
13
|
+
* constant. Adding a deny scope is a one-place change.
|
|
14
|
+
*
|
|
15
|
+
* Regex-based denies (e.g. `<source>:private:*`) stay in
|
|
16
|
+
* `passesScopeFilterForRecall` as a separate JS step — they don't translate
|
|
17
|
+
* cleanly to SQL.
|
|
18
|
+
*
|
|
19
|
+
* Invariant: never empty. An empty array would silently allow quarantine
|
|
20
|
+
* scopes through both paths (SQL clause omitted, JS check vacuous). The
|
|
21
|
+
* module-load assertion below pins this loudly.
|
|
22
|
+
*/
|
|
23
|
+
export const RECALL_DEFAULT_DENY_SCOPES = ['unknown:legacy'];
|
|
24
|
+
/**
|
|
25
|
+
* @internal Runtime guard against a future maintainer blanking a
|
|
26
|
+
* load-bearing literal array. Extracted from the inline guard so the throw
|
|
27
|
+
* path is directly testable. `as const` arrays widen via `readonly T[]` at
|
|
28
|
+
* the call site so the empty case is reachable at runtime.
|
|
29
|
+
*/
|
|
30
|
+
export function assertNonEmpty(arr, name) {
|
|
31
|
+
if (arr.length === 0) {
|
|
32
|
+
throw new Error(`${name} cannot be empty — would silently allow quarantine scopes`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
assertNonEmpty(RECALL_DEFAULT_DENY_SCOPES, 'RECALL_DEFAULT_DENY_SCOPES');
|
|
10
36
|
/**
|
|
11
37
|
* v1.2.1: source-agnostic private-scope detector. A scope string is treated
|
|
12
38
|
* as private when it has the shape `<lowercase-source>:private:<rest>`.
|