hippo-memory 1.58.0 → 1.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/dist/agent-memories/apply.d.ts +1 -1
- package/dist/agent-memories/legacy.js +4 -1
- package/dist/api.d.ts +2 -0
- package/dist/api.js +25 -9
- package/dist/audit.d.ts +1 -1
- package/dist/audit.js +3 -0
- package/dist/autolearn.js +5 -2
- package/dist/capture.js +8 -6
- package/dist/cli/output.d.ts +3 -0
- package/dist/cli/output.js +7 -0
- package/dist/cli/projects.d.ts +4 -0
- package/dist/cli/projects.js +90 -0
- package/dist/cli/shared.js +14 -8
- package/dist/cli/sleep.js +5 -3
- package/dist/cli.d.ts +356 -0
- package/dist/cli.js +1587 -1404
- package/dist/compaction-record.js +7 -6
- package/dist/config.js +12 -11
- package/dist/connectors/github/cli-impl.js +1 -0
- package/dist/consolidate.js +16 -4
- package/dist/dag.js +11 -9
- package/dist/dashboard.js +4 -2
- package/dist/db.js +16 -3
- package/dist/dedupe.js +1 -1
- package/dist/delivery-recorder.js +1 -0
- package/dist/doctor.js +24 -0
- package/dist/dormant.d.ts +2 -2
- package/dist/dormant.js +1 -0
- package/dist/embedding-provider.js +3 -2
- package/dist/embeddings.js +10 -7
- package/dist/extract.js +20 -19
- package/dist/graph.js +3 -8
- package/dist/handoff.js +3 -0
- package/dist/hooks.js +3 -0
- package/dist/importers.js +6 -3
- package/dist/incidents.js +1 -0
- package/dist/judgment.js +5 -2
- package/dist/mcp/server.d.ts +5 -0
- package/dist/mcp/server.js +32 -13
- package/dist/memory.d.ts +5 -3
- package/dist/processes.js +1 -0
- package/dist/project-identity.d.ts +1 -1
- package/dist/project-identity.js +9 -4
- package/dist/project-merge.d.ts +52 -0
- package/dist/project-merge.js +168 -0
- package/dist/raw-archive-mirror-cleanup.js +2 -1
- package/dist/recall-scope.d.ts +3 -3
- package/dist/recall-scope.js +5 -4
- package/dist/recall-trace.d.ts +2 -2
- package/dist/recall-trace.js +10 -14
- package/dist/refine-llm.js +18 -10
- package/dist/rerankers/clef.d.ts +29 -0
- package/dist/rerankers/clef.js +223 -0
- package/dist/rerankers/cross-encoder.js +6 -4
- package/dist/rerankers/index.js +3 -0
- package/dist/rerankers/jev.d.ts +14 -1
- package/dist/rerankers/jev.js +30 -20
- package/dist/rerankers/llm.js +3 -3
- package/dist/rerankers/types.d.ts +16 -0
- package/dist/same-text.d.ts +2 -0
- package/dist/same-text.js +4 -0
- package/dist/scheduler.js +1 -0
- package/dist/search.js +4 -2
- package/dist/secret-detect.js +1 -0
- package/dist/server.d.ts +6 -1
- package/dist/server.js +38 -15
- package/dist/shared.js +21 -19
- package/dist/stdin.js +1 -0
- package/dist/store.d.ts +33 -1
- package/dist/store.js +123 -22
- package/dist/token-ledger.js +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +6 -2
package/dist/rerankers/index.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import { clefFlashReranker, clefReranker } from './clef.js';
|
|
1
2
|
import { crossEncoderReranker } from './cross-encoder.js';
|
|
2
3
|
import { jevReranker } from './jev.js';
|
|
3
4
|
import { llmReranker } from './llm.js';
|
|
4
5
|
const REGISTRY = {
|
|
6
|
+
clef: clefReranker,
|
|
7
|
+
'clef-flash': clefFlashReranker,
|
|
5
8
|
'cross-encoder': crossEncoderReranker,
|
|
6
9
|
jev: jevReranker,
|
|
7
10
|
llm: llmReranker,
|
package/dist/rerankers/jev.d.ts
CHANGED
|
@@ -1,7 +1,20 @@
|
|
|
1
|
-
import type { RerankerFn } from './types.js';
|
|
1
|
+
import type { RerankerFn, RerankResult } from './types.js';
|
|
2
|
+
import type { SearchResult } from '../search.js';
|
|
2
3
|
export declare const JEV_DEFAULT_TOP_K = 40;
|
|
4
|
+
/** The System One `state` and one `noul` question per candidate (`c1`..`cN`). */
|
|
5
|
+
export interface RelevanceRequest {
|
|
6
|
+
state: string;
|
|
7
|
+
questions: Record<string, {
|
|
8
|
+
type: 'noul';
|
|
9
|
+
instructions: string;
|
|
10
|
+
}>;
|
|
11
|
+
}
|
|
12
|
+
/** Redacted query plus numbered, redacted, truncated candidates. Shared with CLEF so both arms see matched input. */
|
|
13
|
+
export declare function buildRelevanceRequest(query: string, head: readonly SearchResult[]): RelevanceRequest;
|
|
3
14
|
/** Builds a Jev reranker around the reranker it degrades to. Exported so a test can pass a stand-in. */
|
|
4
15
|
export declare function createJevReranker(localFallback: RerankerFn): RerankerFn;
|
|
16
|
+
/** Orders `head` by `scores[i]`, keeping any upstream pre-rerank rank. Shared with CLEF. */
|
|
17
|
+
export declare function rankByScores(head: readonly SearchResult[], scores: readonly number[]): RerankResult[];
|
|
5
18
|
/** Track 4 reranker: hosted TypeSafe Jev, opt-in and paid (TYPESAFE_API_KEY), one batched call per recall.
|
|
6
19
|
* Any failure warns once and delegates to the local cross-encoder. Scores are not bit-stable run to run.
|
|
7
20
|
* Cost, env vars, evidence and limits: docs/evals/2026-09-19-jev-reranker.md. */
|
package/dist/rerankers/jev.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { crossEncoderReranker } from './cross-encoder.js';
|
|
2
|
-
import {
|
|
2
|
+
import { redactSecretsStrict } from '../secret-detect.js';
|
|
3
|
+
import { log } from '../log.js';
|
|
3
4
|
const ENDPOINT = 'https://api.typesafe.ai/v1/systemone';
|
|
4
5
|
const DEFAULT_TIMEOUT_MS = 5_000;
|
|
5
6
|
const TRUNCATE_CHARS = 1200;
|
|
@@ -28,13 +29,11 @@ function parseScores(answers, n) {
|
|
|
28
29
|
}
|
|
29
30
|
return out;
|
|
30
31
|
}
|
|
31
|
-
/**
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const lines = head.map((r, i) => `[${i + 1}] ${truncate(redactSecrets(r.entry.content), TRUNCATE_CHARS)}`);
|
|
37
|
-
const state = `Query: ${redactSecrets(query)}\n\nNumbered candidate memories from an AI coding agent's project store:\n\n${lines.join('\n\n')}`;
|
|
32
|
+
/** Redacted query plus numbered, redacted, truncated candidates. Shared with CLEF so both arms see matched input. */
|
|
33
|
+
export function buildRelevanceRequest(query, head) {
|
|
34
|
+
// Strict: this text leaves the machine, so Bearer, Basic-auth and JWT shapes go too.
|
|
35
|
+
const lines = head.map((r, i) => `[${i + 1}] ${truncate(redactSecretsStrict(r.entry.content), TRUNCATE_CHARS)}`);
|
|
36
|
+
const state = `Query: ${redactSecretsStrict(query)}\n\nNumbered candidate memories from an AI coding agent's project store:\n\n${lines.join('\n\n')}`;
|
|
38
37
|
const questions = {};
|
|
39
38
|
for (let i = 1; i <= head.length; i++) {
|
|
40
39
|
questions[`c${i}`] = {
|
|
@@ -42,6 +41,14 @@ async function requestScores(query, head) {
|
|
|
42
41
|
instructions: `Probability that candidate ${i} (numbered in the state above) helps answer the query.`,
|
|
43
42
|
};
|
|
44
43
|
}
|
|
44
|
+
return { state, questions };
|
|
45
|
+
}
|
|
46
|
+
/** One batched request for the whole candidate list. Rejects with the reason when there are no usable scores. */
|
|
47
|
+
async function requestScores(query, head) {
|
|
48
|
+
const key = process.env.TYPESAFE_API_KEY;
|
|
49
|
+
if (!key)
|
|
50
|
+
throw new Error('TYPESAFE_API_KEY not set');
|
|
51
|
+
const { state, questions } = buildRelevanceRequest(query, head);
|
|
45
52
|
const parsed = Number.parseInt(process.env.HIPPO_JEV_TIMEOUT_MS ?? '', 10);
|
|
46
53
|
const timeoutMs = parsed > 0 ? parsed : DEFAULT_TIMEOUT_MS;
|
|
47
54
|
const controller = new AbortController();
|
|
@@ -96,23 +103,26 @@ export function createJevReranker(localFallback) {
|
|
|
96
103
|
if (!warned) {
|
|
97
104
|
warned = true;
|
|
98
105
|
const reason = err instanceof Error ? err.message : 'unknown error';
|
|
99
|
-
|
|
100
|
-
console.warn(`[hippo] jev reranker unavailable (${reason}); falling back to the local cross-encoder. Subsequent calls will not repeat this warning.`);
|
|
106
|
+
log.warn(`jev reranker unavailable (${reason}); falling back to the local cross-encoder. Subsequent calls will not repeat this warning.`);
|
|
101
107
|
}
|
|
102
108
|
return localFallback(query, head, options);
|
|
103
109
|
}
|
|
104
|
-
|
|
105
|
-
...r,
|
|
106
|
-
rerankScore: scores[i],
|
|
107
|
-
preRerankRank: r.preRerankRank ?? i + 1,
|
|
108
|
-
postRerankRank: 0,
|
|
109
|
-
}));
|
|
110
|
-
// Stable sort: ties fall back to the prior relevance order.
|
|
111
|
-
scored.sort((a, b) => b.rerankScore - a.rerankScore);
|
|
112
|
-
scored.forEach((r, i) => (r.postRerankRank = i + 1));
|
|
113
|
-
return scored;
|
|
110
|
+
return rankByScores(head, scores);
|
|
114
111
|
};
|
|
115
112
|
}
|
|
113
|
+
/** Orders `head` by `scores[i]`, keeping any upstream pre-rerank rank. Shared with CLEF. */
|
|
114
|
+
export function rankByScores(head, scores) {
|
|
115
|
+
const scored = head.map((r, i) => ({
|
|
116
|
+
...r,
|
|
117
|
+
rerankScore: scores[i],
|
|
118
|
+
preRerankRank: r.preRerankRank ?? i + 1,
|
|
119
|
+
postRerankRank: 0,
|
|
120
|
+
}));
|
|
121
|
+
// Stable sort: ties fall back to the prior relevance order.
|
|
122
|
+
scored.sort((a, b) => b.rerankScore - a.rerankScore);
|
|
123
|
+
scored.forEach((r, i) => (r.postRerankRank = i + 1));
|
|
124
|
+
return scored;
|
|
125
|
+
}
|
|
116
126
|
/** Track 4 reranker: hosted TypeSafe Jev, opt-in and paid (TYPESAFE_API_KEY), one batched call per recall.
|
|
117
127
|
* Any failure warns once and delegates to the local cross-encoder. Scores are not bit-stable run to run.
|
|
118
128
|
* Cost, env vars, evidence and limits: docs/evals/2026-09-19-jev-reranker.md. */
|
package/dist/rerankers/llm.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { redactSecretsStrict } from '../secret-detect.js';
|
|
2
2
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
3
3
|
/**
|
|
4
4
|
* Track 3 reranker: listwise LLM rerank. Uses a customer-supplied
|
|
@@ -22,8 +22,8 @@ export const llmReranker = async (query, results, options) => {
|
|
|
22
22
|
const head = results.slice(0, topK);
|
|
23
23
|
const prompt = [
|
|
24
24
|
`Rerank the candidates below by relevance to the query. Output a JSON array of indices (zero-indexed) in best-first order.`,
|
|
25
|
-
`Query: ${
|
|
26
|
-
...head.map((r, i) => `[${i}] ${
|
|
25
|
+
`Query: ${redactSecretsStrict(query)}`,
|
|
26
|
+
...head.map((r, i) => `[${i}] ${redactSecretsStrict(r.entry.content)}`),
|
|
27
27
|
`Output format: [<int>, <int>, ...] with all ${head.length} indices.`,
|
|
28
28
|
].join('\n');
|
|
29
29
|
const timeoutMs = Number.parseInt(process.env.HIPPO_LLM_RERANKER_TIMEOUT_MS ?? '', 10) || DEFAULT_TIMEOUT_MS;
|
|
@@ -26,6 +26,20 @@ export interface RerankerOptions {
|
|
|
26
26
|
/** Per-track config blob; opaque to the seam. */
|
|
27
27
|
config?: Record<string, RerankerConfigValue>;
|
|
28
28
|
}
|
|
29
|
+
/** Which backend and model produced a rerank, or why it fell back. */
|
|
30
|
+
export interface RerankProvenance {
|
|
31
|
+
/** `cloudflare`, `private-endpoint`, or `native` when the input order was kept. */
|
|
32
|
+
backend: 'cloudflare' | 'private-endpoint' | 'native';
|
|
33
|
+
/** Model the caller asked for. */
|
|
34
|
+
requestedModel: string;
|
|
35
|
+
/** Model the provider says scored the request, when it reports one. */
|
|
36
|
+
actualModel?: string;
|
|
37
|
+
/** Set when the input order was kept instead of a model ranking. */
|
|
38
|
+
fallbackReason?: string;
|
|
39
|
+
/** Provider-reported token usage, when sent. */
|
|
40
|
+
inputTokens?: number;
|
|
41
|
+
outputTokens?: number;
|
|
42
|
+
}
|
|
29
43
|
export interface RerankResult extends SearchResult {
|
|
30
44
|
/** Score assigned by the reranker. Replaces `score` for downstream
|
|
31
45
|
* ordering; original `score` preserved on the SearchResult. */
|
|
@@ -34,5 +48,7 @@ export interface RerankResult extends SearchResult {
|
|
|
34
48
|
preRerankRank: number;
|
|
35
49
|
/** 1-indexed rank in the reranker output. */
|
|
36
50
|
postRerankRank: number;
|
|
51
|
+
/** Recorded by rerankers that track model identity (the CLEF rerankers). */
|
|
52
|
+
rerankProvenance?: RerankProvenance;
|
|
37
53
|
}
|
|
38
54
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/same-text.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export declare function mergedText(header: string, texts: readonly string[]): st
|
|
|
10
10
|
export declare function heldTexts(entry: Text): string[];
|
|
11
11
|
/** Keys of every text a row holds word for word: its own, plus each source text inside a sleep-merged row. */
|
|
12
12
|
export declare function heldTextKeys(entry: Text): string[];
|
|
13
|
+
/** The longest word of a text: every row holding the text word for word contains it, so a store lookup can filter on it. */
|
|
14
|
+
export declare function longestWord(text: string): string;
|
|
13
15
|
export declare function storedTextKeys(entries: readonly Text[]): Set<string>;
|
|
14
16
|
/** A final result list without copies: drops each unpinned row a sleep-merged row in the list holds word for word, and each later unpinned copy of a text. */
|
|
15
17
|
export declare function dropHeldCopies<T>(rows: readonly T[], textOf: (row: T) => Text): T[];
|
package/dist/same-text.js
CHANGED
|
@@ -19,6 +19,10 @@ export function heldTexts(entry) {
|
|
|
19
19
|
export function heldTextKeys(entry) {
|
|
20
20
|
return [duplicateKey(entry.content), ...heldTexts(entry).map(duplicateKey)];
|
|
21
21
|
}
|
|
22
|
+
/** The longest word of a text: every row holding the text word for word contains it, so a store lookup can filter on it. */
|
|
23
|
+
export function longestWord(text) {
|
|
24
|
+
return duplicateKey(text).split(' ').reduce((best, w) => (w.length > best.length ? w : best), '');
|
|
25
|
+
}
|
|
22
26
|
export function storedTextKeys(entries) {
|
|
23
27
|
return new Set(entries.flatMap(heldTextKeys));
|
|
24
28
|
}
|
package/dist/scheduler.js
CHANGED
package/dist/search.js
CHANGED
|
@@ -640,7 +640,8 @@ export async function physicsSearch(query, entries, options = {}) {
|
|
|
640
640
|
const [vec] = await provider.embed([query], 'query');
|
|
641
641
|
queryVector = vec ?? [];
|
|
642
642
|
}
|
|
643
|
-
catch {
|
|
643
|
+
catch (err) {
|
|
644
|
+
log.debug(`physics search: query embed failed, using hybrid: ${err instanceof Error ? err.message : String(err)}`);
|
|
644
645
|
return hybridSearch(query, entries, options);
|
|
645
646
|
}
|
|
646
647
|
if (queryVector.length === 0) {
|
|
@@ -658,7 +659,8 @@ export async function physicsSearch(query, entries, options = {}) {
|
|
|
658
659
|
closeHippoDb(db);
|
|
659
660
|
}
|
|
660
661
|
}
|
|
661
|
-
catch {
|
|
662
|
+
catch (err) {
|
|
663
|
+
log.debug(`physics search: state load failed, using hybrid: ${err instanceof Error ? err.message : String(err)}`);
|
|
662
664
|
return hybridSearch(query, entries, options);
|
|
663
665
|
}
|
|
664
666
|
// Split entries into physics-enabled and classic
|
package/dist/secret-detect.js
CHANGED
package/dist/server.d.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { type IncomingMessage } from 'node:http';
|
|
2
2
|
/** Test-only: reset the module-level recall-history Map. Call from beforeEach. */
|
|
3
3
|
export declare function __resetSessionRecallHistoryHttp(): void;
|
|
4
|
+
import { authRevoke, type Actor, type Context } from './api.js';
|
|
5
|
+
import { ForbiddenError } from './api-errors.js';
|
|
6
|
+
export { authRevoke, ForbiddenError, type Context, type Actor };
|
|
4
7
|
export interface ServerHandle {
|
|
5
8
|
port: number;
|
|
6
9
|
url: string;
|
|
@@ -15,7 +18,7 @@ export interface ServerHandle {
|
|
|
15
18
|
export interface ResolvedBearer {
|
|
16
19
|
tenantId: string;
|
|
17
20
|
subject: string;
|
|
18
|
-
/** Not 'admin' means 'member'. Admin is tenant-only, yet can mint member API keys (POST /v1/auth/keys)
|
|
21
|
+
/** Not 'admin' means 'member'. Admin is tenant-only, yet can mint member API keys (POST /v1/auth/keys); an add-on revokes them through the exported authRevoke when the IdP deprovisions the minter. */
|
|
19
22
|
role: 'admin' | 'member';
|
|
20
23
|
scopes?: readonly string[];
|
|
21
24
|
}
|
|
@@ -56,6 +59,8 @@ export declare function isCrossSite(req: IncomingMessage): boolean;
|
|
|
56
59
|
* bucket per request and bypass the limiter entirely.
|
|
57
60
|
*/
|
|
58
61
|
export declare function clientIpForRateLimit(req: IncomingMessage): string;
|
|
62
|
+
/** Add-ons call this to refuse a subject that would collide with a built-in actor. */
|
|
63
|
+
export declare function isReservedActor(subject: string): boolean;
|
|
59
64
|
/**
|
|
60
65
|
* Boot the HTTP daemon on host:port and write the pidfile under hippoRoot.
|
|
61
66
|
*
|
package/dist/server.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { createServer } from 'node:http';
|
|
2
2
|
import { createHash, randomUUID } from 'node:crypto';
|
|
3
|
+
import { existsSync } from 'node:fs';
|
|
3
4
|
import { dirname, resolve } from 'node:path';
|
|
4
5
|
import { resolveProjectIdentity } from './project-identity.js';
|
|
5
6
|
import { assembleCost, contextCost, drillCost } from './context-render.js';
|
|
6
7
|
import { detectServer, writePidfile, removePidfileIfOwned } from './server-detect.js';
|
|
7
8
|
import { resolveTenantId } from './tenant.js';
|
|
8
|
-
import { openHippoDb, closeHippoDb } from './db.js';
|
|
9
|
+
import { openHippoDb, closeHippoDb, getHippoDbPath } from './db.js';
|
|
9
10
|
import { updateStats } from './store.js';
|
|
10
11
|
import { buildSessionKey, getOrCreateRing, appendRecall, snapshotRing, hashQueryText, biasHintEnabled, } from './recall-history.js';
|
|
11
12
|
import { appendAuditEvent, auditQueryFields, auditWriteFailureCount, AUDIT_OPS } from './audit.js';
|
|
@@ -35,11 +36,13 @@ import { savePolicy, closePolicy, loadPolicyById, loadPolicies, loadPoliciesAsOf
|
|
|
35
36
|
import { saveSkill, closeSkill, loadSkillById, loadSkills, exportSkills, VALID_SKILL_STATES, } from './skills.js';
|
|
36
37
|
import { saveProjectBrief, closeProjectBrief, loadProjectBriefById, loadProjectBriefs, assembleBriefFromReceipts, refreshBrief, VALID_BRIEF_STATES, } from './project-briefs.js';
|
|
37
38
|
import { saveCustomerNote, closeCustomerNote, loadCustomerNoteById, loadCustomerNotes, VALID_NOTE_STATES, } from './customer-notes.js';
|
|
38
|
-
import { handleMcpRequest } from './mcp/server.js';
|
|
39
|
+
import { handleMcpRequest, mcpErrorResponse } from './mcp/server.js';
|
|
39
40
|
import { handleSlackEventsWebhook } from './connectors/slack/webhook.js';
|
|
40
41
|
import { handleGitHubEventsWebhook } from './connectors/github/webhook.js';
|
|
41
42
|
import { HttpError, JSON_HEADERS, BodyTooLargeError, isHeaderString, isJsonObjectRecord, mapApiError, readBody, sendJson, } from './http-util.js';
|
|
42
|
-
import { NotFoundError } from './api-errors.js';
|
|
43
|
+
import { ForbiddenError, NotFoundError } from './api-errors.js';
|
|
44
|
+
// Add-on packages revoke keys through these without importing the whole api surface.
|
|
45
|
+
export { authRevoke, ForbiddenError };
|
|
43
46
|
// Review patch #2: explicit allow-list for unauthenticated /v1/* routes.
|
|
44
47
|
// New unauth routes MUST be added here AND get a corresponding entry in
|
|
45
48
|
// tests/server-bearer-lockdown.test.ts. Do not gate auth elsewhere by
|
|
@@ -364,6 +367,11 @@ export function clientIpForRateLimit(req) {
|
|
|
364
367
|
const RESERVED_ACTOR_NAMES = [
|
|
365
368
|
'api_key', 'localhost', 'cli', 'system', 'mcp', 'connector', 'sleep', 'post-compact', 'recall', 'agent-memories',
|
|
366
369
|
];
|
|
370
|
+
/** Add-ons call this to refuse a subject that would collide with a built-in actor. */
|
|
371
|
+
export function isReservedActor(subject) {
|
|
372
|
+
const lower = subject.toLowerCase();
|
|
373
|
+
return RESERVED_ACTOR_NAMES.some((n) => lower === n || lower.startsWith(`${n}:`));
|
|
374
|
+
}
|
|
367
375
|
function hasControlChar(s) {
|
|
368
376
|
for (let i = 0; i < s.length; i++) {
|
|
369
377
|
const c = s.charCodeAt(i);
|
|
@@ -388,8 +396,7 @@ function sanitiseResolved(r) {
|
|
|
388
396
|
// Padding would let "system " pass the reserved-name check yet read as `system` in an audit log.
|
|
389
397
|
if (hasControlChar(subject) || subject !== subject.trim())
|
|
390
398
|
return null;
|
|
391
|
-
|
|
392
|
-
if (RESERVED_ACTOR_NAMES.some((n) => lower === n || lower.startsWith(`${n}:`)))
|
|
399
|
+
if (isReservedActor(subject))
|
|
393
400
|
return null;
|
|
394
401
|
const clean = { tenantId: tenant, subject, role: role === 'admin' ? 'admin' : 'member' };
|
|
395
402
|
if (Array.isArray(scopes))
|
|
@@ -398,8 +405,7 @@ function sanitiseResolved(r) {
|
|
|
398
405
|
}
|
|
399
406
|
function logResolverFailure(what, raw, token) {
|
|
400
407
|
// The plugin's message is logged, but never the token, even if the plugin echoed it.
|
|
401
|
-
|
|
402
|
-
process.stderr.write(`[hippo] auth resolver ${what}: ${msg}\n`);
|
|
408
|
+
log.error(`auth resolver ${what}: ${raw.split(token).join('[token]')}`);
|
|
403
409
|
}
|
|
404
410
|
/** 503 when upstream throws or misses the deadline, so a stream heartbeat can tell an outage from a revocation. */
|
|
405
411
|
async function askResolver(resolver, token, deadlineMs) {
|
|
@@ -2408,11 +2414,7 @@ async function handleRequest(req, res, opts, startedAt, limiter) {
|
|
|
2408
2414
|
});
|
|
2409
2415
|
}
|
|
2410
2416
|
catch (err) {
|
|
2411
|
-
mcpRes =
|
|
2412
|
-
jsonrpc: '2.0',
|
|
2413
|
-
id: mcpReq.id,
|
|
2414
|
-
error: { code: -32603, message: err instanceof Error ? err.message : 'internal error' },
|
|
2415
|
-
};
|
|
2417
|
+
mcpRes = mcpErrorResponse(rpcReq.id, err, requestIds.get(req));
|
|
2416
2418
|
}
|
|
2417
2419
|
if (mcpRes === null) {
|
|
2418
2420
|
// Notification — no body, 202 Accepted.
|
|
@@ -2487,7 +2489,7 @@ async function handleRequest(req, res, opts, startedAt, limiter) {
|
|
|
2487
2489
|
res.write(': ping\n\n');
|
|
2488
2490
|
}
|
|
2489
2491
|
catch {
|
|
2490
|
-
clearInterval(ping);
|
|
2492
|
+
clearInterval(ping); // the client hung up; stop pinging a dead socket
|
|
2491
2493
|
}
|
|
2492
2494
|
});
|
|
2493
2495
|
}, heartbeatMs);
|
|
@@ -2555,7 +2557,23 @@ export async function serve(opts) {
|
|
|
2555
2557
|
const limiter = Number.isFinite(v1Rps) && v1Rps > 0
|
|
2556
2558
|
? createRateLimiter({ ratePerSec: v1Rps, burst: v1Rps * 2, idleEvictMs: 60000, maxKeys: 10000 })
|
|
2557
2559
|
: undefined;
|
|
2560
|
+
// Handlers open and close their own connections; while this one is held, none of those closes is SQLite's last,
|
|
2561
|
+
// which checkpoints and deletes the WAL. It opens only once the store exists, so serving never creates one.
|
|
2562
|
+
let heldDb;
|
|
2563
|
+
let stopHolding = false;
|
|
2564
|
+
const holdStore = () => {
|
|
2565
|
+
if (heldDb || stopHolding || !existsSync(getHippoDbPath(opts.hippoRoot)))
|
|
2566
|
+
return;
|
|
2567
|
+
try {
|
|
2568
|
+
heldDb = openHippoDb(opts.hippoRoot);
|
|
2569
|
+
}
|
|
2570
|
+
catch (err) {
|
|
2571
|
+
stopHolding = true;
|
|
2572
|
+
log.warn(`serve: could not hold a store connection; requests still work, only slower: ${err instanceof Error ? err.message : String(err)}`);
|
|
2573
|
+
}
|
|
2574
|
+
};
|
|
2558
2575
|
const server = createServer((req, res) => {
|
|
2576
|
+
res.once('finish', holdStore);
|
|
2559
2577
|
const requestId = resolveRequestId(req.headers['x-request-id']);
|
|
2560
2578
|
requestIds.set(req, requestId);
|
|
2561
2579
|
res.setHeader('X-Request-Id', requestId);
|
|
@@ -2622,6 +2640,7 @@ export async function serve(opts) {
|
|
|
2622
2640
|
const actualPort = addressInfo.port;
|
|
2623
2641
|
const url = `http://${host}:${actualPort}`;
|
|
2624
2642
|
writePidfile(opts.hippoRoot, { port: actualPort, url, startedAt });
|
|
2643
|
+
holdStore();
|
|
2625
2644
|
let stopping = false;
|
|
2626
2645
|
const stop = async () => {
|
|
2627
2646
|
if (stopping)
|
|
@@ -2639,6 +2658,10 @@ export async function serve(opts) {
|
|
|
2639
2658
|
await new Promise((resolve) => {
|
|
2640
2659
|
server.close(() => resolve());
|
|
2641
2660
|
});
|
|
2661
|
+
stopHolding = true;
|
|
2662
|
+
if (heldDb)
|
|
2663
|
+
closeHippoDb(heldDb);
|
|
2664
|
+
heldDb = undefined;
|
|
2642
2665
|
};
|
|
2643
2666
|
if (opts.handleSignals) {
|
|
2644
2667
|
let shuttingDown = false;
|
|
@@ -2646,12 +2669,12 @@ export async function serve(opts) {
|
|
|
2646
2669
|
if (shuttingDown)
|
|
2647
2670
|
return;
|
|
2648
2671
|
shuttingDown = true;
|
|
2649
|
-
|
|
2672
|
+
log.warn(`received ${signal}, shutting down`);
|
|
2650
2673
|
try {
|
|
2651
2674
|
await stop();
|
|
2652
2675
|
}
|
|
2653
2676
|
catch (err) {
|
|
2654
|
-
|
|
2677
|
+
log.error(`error during stop: ${err instanceof Error ? err.message : String(err)}`);
|
|
2655
2678
|
}
|
|
2656
2679
|
finally {
|
|
2657
2680
|
process.exit(0);
|
package/dist/shared.js
CHANGED
|
@@ -9,7 +9,7 @@ import * as fs from 'fs';
|
|
|
9
9
|
import * as path from 'path';
|
|
10
10
|
import { generateId, COMPACTION_MEMORY_TAG } from './memory.js';
|
|
11
11
|
import { AGENT_MEMORY_SOURCE_PREFIX, AGENT_MEMORY_TAGS } from './agent-memories/tools.js';
|
|
12
|
-
import { initStore, loadAllEntries, loadIndex, loadSearchEntries, loadRecallSearchEntries, writeEntry, readEntry, } from './store.js';
|
|
12
|
+
import { initStore, loadAllEntries, loadIndex, loadSearchEntries, loadRecallSearchEntries, tallySources, writeEntry, readEntry, } from './store.js';
|
|
13
13
|
import { passesScopeFilterForRecall, passesCliRecallScopeFilter } from './recall-scope.js';
|
|
14
14
|
import { search, hybridSearch, fitBudget } from './search.js';
|
|
15
15
|
import { evalNow } from './ablation.js';
|
|
@@ -19,6 +19,11 @@ import { isQuarantineScope } from './quarantine.js';
|
|
|
19
19
|
import { RejectedValueError } from './rejection.js';
|
|
20
20
|
import { embedMemory, embedAll } from './embeddings.js';
|
|
21
21
|
import { duplicateKey, storedTextKeys } from './same-text.js';
|
|
22
|
+
import { log } from './log.js';
|
|
23
|
+
// The rows are already copied; a failed background embed only delays vectors, so it warns instead of throwing.
|
|
24
|
+
function logEmbedAllFailure(caller, err) {
|
|
25
|
+
log.warn(`${caller}: background embed failed (${err instanceof Error ? err.message : String(err)}); run 'hippo embed' to backfill`);
|
|
26
|
+
}
|
|
22
27
|
/**
|
|
23
28
|
* Returns the path to the global Hippo store.
|
|
24
29
|
* Resolution order: $HIPPO_HOME > $XDG_DATA_HOME/hippo > ~/.hippo/
|
|
@@ -330,32 +335,29 @@ export function listPeers(globalRoot, tenantId) {
|
|
|
330
335
|
return [];
|
|
331
336
|
// D4: tenant-scoped by default when tenantId provided. Host-wide when
|
|
332
337
|
// undefined (preserves back-compat).
|
|
333
|
-
const
|
|
334
|
-
const entries = tenantId !== undefined
|
|
335
|
-
? allEntries.filter((e) => e.tenantId === tenantId)
|
|
336
|
-
: allEntries;
|
|
338
|
+
const tallies = tallySources(root, tenantId).sort((a, b) => (a.first < b.first ? -1 : a.first > b.first ? 1 : 0));
|
|
337
339
|
const peerMap = new Map();
|
|
338
|
-
for (const
|
|
340
|
+
for (const tally of tallies) {
|
|
339
341
|
let project = 'unknown';
|
|
340
|
-
if (
|
|
341
|
-
const parts =
|
|
342
|
+
if (tally.source.startsWith('shared:')) {
|
|
343
|
+
const parts = tally.source.split(':');
|
|
342
344
|
project = parts[1] || 'unknown';
|
|
343
345
|
}
|
|
344
|
-
else if (
|
|
345
|
-
const promotedPath =
|
|
346
|
+
else if (tally.source.startsWith('promoted:')) {
|
|
347
|
+
const promotedPath = tally.source.slice('promoted:'.length);
|
|
346
348
|
project = path.basename(path.resolve(promotedPath, '..'));
|
|
347
349
|
}
|
|
348
|
-
else if (
|
|
350
|
+
else if (tally.source === 'cli-global') {
|
|
349
351
|
project = 'global-cli';
|
|
350
352
|
}
|
|
351
353
|
const existing = peerMap.get(project);
|
|
352
354
|
if (!existing) {
|
|
353
|
-
peerMap.set(project, { count:
|
|
355
|
+
peerMap.set(project, { count: tally.count, latest: tally.latest });
|
|
354
356
|
}
|
|
355
357
|
else {
|
|
356
|
-
existing.count
|
|
357
|
-
if (
|
|
358
|
-
existing.latest =
|
|
358
|
+
existing.count += tally.count;
|
|
359
|
+
if (tally.latest > existing.latest)
|
|
360
|
+
existing.latest = tally.latest;
|
|
359
361
|
}
|
|
360
362
|
}
|
|
361
363
|
return Array.from(peerMap.entries())
|
|
@@ -458,10 +460,10 @@ export function autoShare(localRoot, options = {}) {
|
|
|
458
460
|
}
|
|
459
461
|
}
|
|
460
462
|
if (rejectedSkipped > 0) {
|
|
461
|
-
|
|
463
|
+
log.warn(`autoShare: skipped ${rejectedSkipped} candidate(s) refused by the global store's rejection tombstone`);
|
|
462
464
|
}
|
|
463
465
|
if (shared.length > 0) {
|
|
464
|
-
void embedAll(globalRoot).catch(() =>
|
|
466
|
+
void embedAll(globalRoot).catch((err) => logEmbedAllFailure('autoShare', err));
|
|
465
467
|
}
|
|
466
468
|
return shared;
|
|
467
469
|
}
|
|
@@ -516,12 +518,12 @@ export function syncGlobalToLocal(localRoot, globalRoot, opts = {}) {
|
|
|
516
518
|
count++;
|
|
517
519
|
}
|
|
518
520
|
if (rejected > 0) {
|
|
519
|
-
|
|
521
|
+
log.warn(`syncGlobalToLocal: skipped ${rejected} rejected value(s) (run \`hippo unreject\` on the local store to allow).`);
|
|
520
522
|
}
|
|
521
523
|
// Batch producer: one embedAll() on the destination rather than an
|
|
522
524
|
// embedMemory() per copied row (same batching invariant as autoShare).
|
|
523
525
|
if (count > 0) {
|
|
524
|
-
void embedAll(localRoot).catch(() =>
|
|
526
|
+
void embedAll(localRoot).catch((err) => logEmbedAllFailure('syncGlobalToLocal', err));
|
|
525
527
|
}
|
|
526
528
|
return count;
|
|
527
529
|
}
|
package/dist/stdin.js
CHANGED
|
@@ -11,6 +11,7 @@ export function readStdinBounded(waitMs = defaultWaitMs()) {
|
|
|
11
11
|
stdin = process.stdin;
|
|
12
12
|
}
|
|
13
13
|
catch {
|
|
14
|
+
// Reading process.stdin can throw when the handle is closed; that is the same as no input.
|
|
14
15
|
return Promise.resolve({ timedOut: false });
|
|
15
16
|
}
|
|
16
17
|
if (stdin.isTTY)
|
package/dist/store.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* SQLite is the source of truth.
|
|
5
5
|
* Markdown + JSON files remain as human-readable compatibility mirrors.
|
|
6
6
|
*/
|
|
7
|
-
import { MemoryEntry, Layer } from './memory.js';
|
|
7
|
+
import { MemoryEntry, Layer, type StrengthInputs } from './memory.js';
|
|
8
8
|
import { openHippoDb, getHippoDbPath, type DatabaseSyncLike } from './db.js';
|
|
9
9
|
import { SessionHandoff, HandoffEvidence, HandoffOutcome } from './handoff.js';
|
|
10
10
|
import { type ResolveProjectIdentityOpts } from './project-identity.js';
|
|
@@ -76,6 +76,7 @@ export interface SessionEvent {
|
|
|
76
76
|
metadata: Record<string, JsonValue>;
|
|
77
77
|
created_at: string;
|
|
78
78
|
}
|
|
79
|
+
export declare const MEMORY_SELECT_COLUMNS = "id, created, last_retrieved, retrieval_count, strength, half_life_days, layer, tags_json, emotional_valence, schema_fit, source, outcome_score, outcome_positive, outcome_negative, conflicts_with_json, pinned, confidence, content, parents_json, starred, trace_outcome, source_session_id, valid_from, superseded_by, extracted_from, dag_level, dag_parent_id, kind, scope, owner, artifact_ref, tenant_id, origin_project, descendant_count, earliest_at, latest_at, summary_dirty, last_rebuilt_at, rebuild_count, dag_level_3_built_at";
|
|
79
80
|
/**
|
|
80
81
|
* Default candidate-pool size for `loadSearchEntries` when called with
|
|
81
82
|
* `limit === undefined`. Single source of truth; `api.recall` imports
|
|
@@ -413,7 +414,38 @@ export interface AmbientLoadResult {
|
|
|
413
414
|
entries: MemoryEntry[];
|
|
414
415
|
recall?: MemoryEntry[];
|
|
415
416
|
}
|
|
417
|
+
/** Exported so the plan test runs the exact SQL; idx_memories_pinned (db.ts v51) serves it. */
|
|
418
|
+
export declare const AMBIENT_PINNED_WHERE = "pinned = 1 AND superseded_by IS NULL AND tenant_id = ? ORDER BY created ASC, id ASC";
|
|
419
|
+
/** Exported so the plan test runs the exact SQL; idx_memories_created_drift (db.ts v51) serves it. */
|
|
420
|
+
export declare const AMBIENT_DRIFT_SQL = "SELECT 1 FROM memories WHERE superseded_by IS NULL AND tenant_id = ? AND (length(created) <> 24 OR created NOT LIKE '%Z') LIMIT 1";
|
|
416
421
|
export declare function loadAmbientCandidates(hippoRoot: string, tenantId: string, recentNeeded: number, admit: (e: MemoryEntry) => boolean, recall?: AmbientRecallRequest): AmbientLoadResult;
|
|
422
|
+
/** The rows a non-pinned context read may admit; each predicate is one getContext's admission applies again in JS. */
|
|
423
|
+
export interface ContextCandidateFilter {
|
|
424
|
+
/** Envelope scope asked for by name; absent applies recall's default deny. */
|
|
425
|
+
exactScope?: string;
|
|
426
|
+
/** Rows of this project and user-global rows pass; absent admits every origin. */
|
|
427
|
+
project?: string;
|
|
428
|
+
/** Most rows returned; past it, the rows decay has worn least win. */
|
|
429
|
+
cap: number;
|
|
430
|
+
now: Date;
|
|
431
|
+
}
|
|
432
|
+
/** Live tenant rows passing `filter`, at most `filter.cap`, in loadAllEntries' order; below the cap, every such row. */
|
|
433
|
+
export declare function loadContextCandidates(hippoRoot: string, tenantId: string, filter: ContextCandidateFilter): MemoryEntry[];
|
|
434
|
+
/** A row's strength inputs and tags, without its text or the JSON lists a full row parses. */
|
|
435
|
+
export type StrengthRow = StrengthInputs & Pick<MemoryEntry, 'tags'>;
|
|
436
|
+
/** Every tenant row as a StrengthRow, for whole-store health numbers; defaults match rowToEntry's. */
|
|
437
|
+
export declare function loadStrengthRows(hippoRoot: string, tenantId: string): StrengthRow[];
|
|
438
|
+
/** Text and source of tenant rows holding any of `words`; a row equal to a text apart from spacing holds its every word. */
|
|
439
|
+
export declare function loadTextsHoldingWords(hippoRoot: string, tenantId: string, words: readonly string[]): Array<Pick<MemoryEntry, 'content' | 'source'>>;
|
|
440
|
+
export interface SourceTally {
|
|
441
|
+
source: string;
|
|
442
|
+
count: number;
|
|
443
|
+
latest: string;
|
|
444
|
+
/** `created`, a unit separator, then `id` of the source's first row in loadAllEntries' order. */
|
|
445
|
+
first: string;
|
|
446
|
+
}
|
|
447
|
+
/** Row count and newest `created` per source, for peer listings that need no row; all tenants when `tenantId` is absent. */
|
|
448
|
+
export declare function tallySources(hippoRoot: string, tenantId?: string): SourceTally[];
|
|
417
449
|
/**
|
|
418
450
|
* Load likely search candidates directly from SQLite.
|
|
419
451
|
* Uses FTS5 when available, falls back to LIKE matching, then full-store fallback.
|