hippo-memory 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/api/detached.d.ts +6 -0
- package/dist/api/detached.js +16 -0
- package/dist/api/embeddings.d.ts +4 -3
- package/dist/api/embeddings.js +4 -4
- package/dist/api/memories/promote.js +2 -1
- package/dist/api/memories/remember-local.d.ts +5 -4
- package/dist/api/memories/remember-local.js +13 -10
- package/dist/api/session-end.js +1 -0
- package/dist/api/sharing.js +9 -7
- package/dist/api/sleep/auto-sleep.d.ts +1 -1
- package/dist/api/sleep/auto-sleep.js +5 -3
- package/dist/api/sleep/learn.js +5 -2
- package/dist/api/sleep/sleep-run.js +4 -2
- package/dist/api/types.d.ts +4 -0
- package/dist/base/util/detached-set.d.ts +8 -0
- package/dist/base/util/detached-set.js +35 -0
- package/dist/base/util/version.d.ts +1 -1
- package/dist/base/util/version.js +1 -1
- package/dist/db/store-runtime.d.ts +10 -3
- package/dist/db/store-runtime.js +19 -3
- package/dist/domain/search/detached-work.d.ts +6 -0
- package/dist/domain/search/detached-work.js +2 -0
- package/dist/domain/search/embed-memories.d.ts +4 -4
- package/dist/domain/search/embed-memories.js +4 -5
- package/dist/domain/sharing/global-sync.d.ts +8 -2
- package/dist/domain/sharing/global-sync.js +5 -5
- package/dist/domain/sharing/share.d.ts +4 -0
- package/dist/domain/sharing/share.js +2 -2
- package/dist/entry/eval-scripts.d.ts +1 -0
- package/dist/entry/sleep.d.ts +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/server.d.ts +1 -0
- package/dist/store/sqlite/runtime/server-runtime.d.ts +6 -1
- package/dist/store/sqlite/runtime/server-runtime.js +15 -3
- package/dist/store/sqlite/runtime/worker-jobs.js +2 -2
- package/dist/surface/capture/compact.d.ts +3 -3
- package/dist/surface/capture/compact.js +4 -4
- package/dist/surface/capture/session-texts.js +4 -6
- package/dist/surface/cli/api-context.d.ts +5 -0
- package/dist/surface/cli/api-context.js +12 -0
- package/dist/surface/cli/goals.js +6 -8
- package/dist/surface/cli/hook-runtime.js +2 -2
- package/dist/surface/cli/memory-import.js +20 -15
- package/dist/surface/mcp/protocol.d.ts +6 -0
- package/dist/surface/mcp/protocol.js +1 -0
- package/dist/surface/mcp/stdio.js +11 -5
- package/dist/surface/open-store.d.ts +2 -2
- package/dist/surface/open-store.js +11 -7
- package/dist/surface/server/auth.d.ts +1 -1
- package/dist/surface/server/auth.js +13 -2
- package/dist/surface/server/boot.js +3 -2
- package/dist/surface/server/dashboard-listener.js +2 -2
- package/dist/surface/server/lifecycle.d.ts +2 -0
- package/dist/surface/server/lifecycle.js +4 -2
- package/dist/surface/server/mcp-http.js +1 -0
- package/dist/surface/server/types.d.ts +3 -0
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
package/dist/entry/sleep.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ export type { SleepOptions, SleepResult } from '../api/sleep/sleep.js';
|
|
|
8
8
|
export type { SleepEvent } from '../api/sleep/events.js';
|
|
9
9
|
export type { RescueRank } from '../domain/consolidate/events.js';
|
|
10
10
|
export type { SleepContext, SleepStore };
|
|
11
|
+
export type { DetachedWork } from '../domain/search/detached-work.js';
|
|
11
12
|
/** Sleeps the whole store under `ctx`, whose store sets the groups a sleep pass works on. */
|
|
12
13
|
export declare function sleep(ctx: SleepContext, opts?: SleepOptions): Promise<SleepResult>;
|
|
13
14
|
/** consolidate's options as published: the LLM phases call the model ANTHROPIC_API_KEY opens,
|
package/dist/index.d.ts
CHANGED
|
@@ -4,6 +4,6 @@
|
|
|
4
4
|
* hippo-memory/importers and hippo-memory/eval.
|
|
5
5
|
*/
|
|
6
6
|
export { MemoryEntry, Layer, EmotionalValence, ConfidenceLevel, DecayOptions, calculateStrength, resolveConfidence, confidenceFacets, type ConfidenceFacets, applyOutcome, generateId, computeSchemaFit, createMemory, markRetrieved, estimateTokens, SessionHandoff, HandoffOutcome, HandoffEvidence, isHandoffOutcome, Card, CardStatus, CardRun, CardComment, CardTransitions, isCardStatus, CARD_TRANSITIONS, CARD_LEASE_MS, computeSalience, SalienceDecision, SalienceResult, SalienceOptions, computeAmbientState, renderAmbientSummary, formatAmbientVector, AmbientState, tokenize, textOverlap, isEmbeddingAvailable, getEmbedding, } from './entry/memory.js';
|
|
7
|
-
export { ConsolidationResult, type SleepOptions, type SleepResult, type SleepEvent, type RescueRank, type SleepContext, type SleepStore, sleep, consolidate, strengthBucket, } from './entry/sleep.js';
|
|
7
|
+
export { ConsolidationResult, type SleepOptions, type SleepResult, type SleepEvent, type RescueRank, type SleepContext, type SleepStore, type DetachedWork, sleep, consolidate, strengthBucket, } from './entry/sleep.js';
|
|
8
8
|
export { captureError, extractLessons, partitionLessons, getGlobalRoot, transferScore, } from './entry/learning.js';
|
|
9
9
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/server.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ export { clientIpForRateLimit } from './surface/server/client-ip.js';
|
|
|
14
14
|
export { isLoopback, isReservedActor } from './surface/server/auth.js';
|
|
15
15
|
export type { AddonCall, AddonRoute, AuthResolver, RateLimitSpec, ResolvedBearer, ServeOptions, ServerHandle, } from './surface/server/types.js';
|
|
16
16
|
export type { HippoStore } from './base/ports/store-index.js';
|
|
17
|
+
export type { DetachedWork } from './domain/search/detached-work.js';
|
|
17
18
|
export { HttpError } from './base/errors/http-errors.js';
|
|
18
19
|
export type { CallerProject } from './api/prompt-hook.js';
|
|
19
20
|
export type { JsonValue } from './base/util/json.js';
|
|
@@ -3,6 +3,7 @@ import type { HippoStore, StoreReach } from '../../../base/ports/store.js';
|
|
|
3
3
|
import { type ExecutorOptions } from './executor.js';
|
|
4
4
|
import { workerCounts } from './executor-counts.js';
|
|
5
5
|
import { type StoreHolder } from './held-connection.js';
|
|
6
|
+
import { StoreRuntime } from '../../../db/store-runtime.js';
|
|
6
7
|
import type { RequestStores } from '../../../db/index.js';
|
|
7
8
|
import type { RequestLimits } from '../../../base/util/request-scope.js';
|
|
8
9
|
export interface ServerRuntimeOptions extends ExecutorOptions {
|
|
@@ -30,7 +31,11 @@ export interface SqliteServerRuntime extends Omit<StoreHolder, 'release'> {
|
|
|
30
31
|
readonly runRequest: <T>(run: (session: RequestStores) => Promise<T>) => Promise<T>;
|
|
31
32
|
/** What the store workers of this process refused, expired and replaced; /health reports them. */
|
|
32
33
|
readonly workerCounts: Readonly<typeof workerCounts>;
|
|
33
|
-
|
|
34
|
+
/** Keeps the work a request starts and does not wait on, such as an embed; close() settles it. */
|
|
35
|
+
readonly detached: Pick<StoreRuntime, 'detach'>;
|
|
36
|
+
/** Waits up to `settleWithinMs` for the detached work, then closes the stores, the held
|
|
37
|
+
* connection last. */
|
|
38
|
+
readonly close: (settleWithinMs: number) => Promise<void>;
|
|
34
39
|
/** What the root's exit hook runs when the process exits before close(): the held writes and
|
|
35
40
|
* handles of the server's store runtime. */
|
|
36
41
|
readonly closeOnExit: () => void;
|
|
@@ -10,12 +10,22 @@ import { inSqliteSession, sqliteReach } from '../store.js';
|
|
|
10
10
|
import { runSqliteRequest } from './request-scope.js';
|
|
11
11
|
import { globalStoresFor, storeForRequest, workerSqliteStore } from './worker-store.js';
|
|
12
12
|
import { StoreRuntime } from '../../../db/store-runtime.js';
|
|
13
|
+
import { log } from '../../../base/util/log.js';
|
|
13
14
|
/** `own` as one request calls it: worker calls carry the request's limits, and an in-process
|
|
14
15
|
* store opens on its session. */
|
|
15
16
|
function boundToRequest(own, limits, session) {
|
|
16
17
|
const bound = storeForRequest(own, limits);
|
|
17
18
|
return session === undefined ? bound : inSqliteSession(bound, session);
|
|
18
19
|
}
|
|
20
|
+
/** Detached work still writes through the stores, so it settles before they close; one past the
|
|
21
|
+
* bound fails against the closed stores, and DetachedSet logs it. */
|
|
22
|
+
async function settleDetached(runtime, withinMs) {
|
|
23
|
+
const left = await runtime.settle(withinMs);
|
|
24
|
+
if (left === 0)
|
|
25
|
+
return;
|
|
26
|
+
const line = `shutdown: ${left} handed-off task(s) still running after ${withinMs} ms; closing the stores`;
|
|
27
|
+
log.warn(line, { left });
|
|
28
|
+
}
|
|
19
29
|
/** hippo.db under `hippoRoot` as a server runs it: `opts.store` when given, else a worker store
|
|
20
30
|
* on `opts`' executor. */
|
|
21
31
|
export function sqliteServerRuntime(hippoRoot, opts = {}) {
|
|
@@ -41,18 +51,20 @@ export function sqliteServerRuntime(hippoRoot, opts = {}) {
|
|
|
41
51
|
}),
|
|
42
52
|
runRequest: (run) => runSqliteRequest(run, 'server', runtime),
|
|
43
53
|
workerCounts,
|
|
54
|
+
detached: runtime,
|
|
44
55
|
hold: holder.hold,
|
|
45
56
|
afterResponse: holder.afterResponse,
|
|
46
|
-
close: async () => {
|
|
57
|
+
close: async (settleWithinMs) => {
|
|
58
|
+
await settleDetached(runtime, settleWithinMs);
|
|
47
59
|
// The store's worker threads and the pooled handles close first, so the held one is still
|
|
48
60
|
// SQLite's last.
|
|
49
61
|
if (given === undefined)
|
|
50
62
|
await store.close();
|
|
51
63
|
await globalStores.close();
|
|
52
|
-
runtime.
|
|
64
|
+
runtime.closeNow();
|
|
53
65
|
await holder.release();
|
|
54
66
|
},
|
|
55
|
-
closeOnExit: () => runtime.
|
|
67
|
+
closeOnExit: () => runtime.closeNow(),
|
|
56
68
|
};
|
|
57
69
|
}
|
|
58
70
|
//# sourceMappingURL=server-runtime.js.map
|
|
@@ -127,7 +127,7 @@ function post(port, reply, end) {
|
|
|
127
127
|
});
|
|
128
128
|
}
|
|
129
129
|
// Each step still closes what is left when the one before it throws, so a failed flush leaves no
|
|
130
|
-
// handle on the files. Held writes run first, then the thread's session closes, and
|
|
130
|
+
// handle on the files. Held writes run first, then the thread's session closes, and closeNow finds
|
|
131
131
|
// no session left to close twice.
|
|
132
132
|
function closeThread({ runtime, stores }, port) {
|
|
133
133
|
try {
|
|
@@ -136,7 +136,7 @@ function closeThread({ runtime, stores }, port) {
|
|
|
136
136
|
}
|
|
137
137
|
finally {
|
|
138
138
|
try {
|
|
139
|
-
runtime.
|
|
139
|
+
runtime.closeNow();
|
|
140
140
|
}
|
|
141
141
|
finally {
|
|
142
142
|
port.close();
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { type HookRuntime } from '../../base/session/capture-contract.js';
|
|
2
2
|
import type { StoreGroups } from '../../base/ports/store.js';
|
|
3
|
-
/** Log-forgery guard: strip
|
|
4
|
-
* so a crafted value cannot inject fake `[hippo]`
|
|
5
|
-
* for every
|
|
3
|
+
/** Log-forgery guard: strip control chars (U+0000-U+001F, DEL, U+0080-U+009F; newlines and NEL
|
|
4
|
+
* above all) so a crafted value cannot inject fake `[hippo]` lines or terminal escapes. The one
|
|
5
|
+
* shared guard for every log line and printed name that interpolates payload values. */
|
|
6
6
|
export declare function sanitizeLogMessage(message: string): string;
|
|
7
7
|
/** The folder's store as a compaction hook reaches it, built by the CLI root. */
|
|
8
8
|
export type CompactionStore = Pick<StoreGroups, 'compactionHooks' | 'sessions'>;
|
|
@@ -15,12 +15,12 @@ import { transcriptWorkingState } from './working-state.js';
|
|
|
15
15
|
import { PRIVATE_DIR } from '../../host/private-dir.js';
|
|
16
16
|
// Diagnostic-only log; a long-lived install must not grow it unbounded.
|
|
17
17
|
const PRE_COMPACT_LOG_MAX_BYTES = 256 * 1024;
|
|
18
|
-
/** Log-forgery guard: strip
|
|
19
|
-
* so a crafted value cannot inject fake `[hippo]`
|
|
20
|
-
* for every
|
|
18
|
+
/** Log-forgery guard: strip control chars (U+0000-U+001F, DEL, U+0080-U+009F; newlines and NEL
|
|
19
|
+
* above all) so a crafted value cannot inject fake `[hippo]` lines or terminal escapes. The one
|
|
20
|
+
* shared guard for every log line and printed name that interpolates payload values. */
|
|
21
21
|
export function sanitizeLogMessage(message) {
|
|
22
22
|
// eslint-disable-next-line no-control-regex
|
|
23
|
-
return message.replace(/[\x00-\x1f]/g, '');
|
|
23
|
+
return message.replace(/[\x00-\x1f\x7f-\x9f]/g, '');
|
|
24
24
|
}
|
|
25
25
|
function appendPreCompactLog(logFile, message, fields) {
|
|
26
26
|
try {
|
|
@@ -9,6 +9,7 @@ import { captureExtractedItems } from './command.js';
|
|
|
9
9
|
import { detachedEmbedder } from '../../api/embeddings.js';
|
|
10
10
|
import { extractFromTexts } from './extract.js';
|
|
11
11
|
import { withEntryInfra } from '../entry-infra.js';
|
|
12
|
+
import { withDetached } from '../../api/detached.js';
|
|
12
13
|
const MAX_TEXTS = 30;
|
|
13
14
|
const MAX_TEXT_BYTES = 32 * 1024;
|
|
14
15
|
const MAX_TOTAL_BYTES = 256 * 1024;
|
|
@@ -49,11 +50,8 @@ export async function captureSessionTexts(given, req) {
|
|
|
49
50
|
const dedup = { tenantId: ctx.tenantId, project: projectNames(req.project) };
|
|
50
51
|
const held = await ctx.store.captures.textsHolding(dedup.tenantId, items.map((i) => i.content), { project: dedup.project });
|
|
51
52
|
const { store, config } = ctx;
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
config,
|
|
55
|
-
embed: detachedEmbedder({ store, config, embedderFor: ctx.infra.embedderFor }, ctx.infra),
|
|
56
|
-
}, {
|
|
53
|
+
const folder = { store, config, embedderFor: ctx.infra.embedderFor };
|
|
54
|
+
const tally = await withDetached(ctx, (detached) => captureExtractedItems({ store, config, embed: detachedEmbedder(folder, ctx.infra, detached) }, {
|
|
57
55
|
tenantId: ctx.tenantId,
|
|
58
56
|
originProject: req.project,
|
|
59
57
|
sessionId: req.sessionId,
|
|
@@ -61,7 +59,7 @@ export async function captureSessionTexts(given, req) {
|
|
|
61
59
|
lean: true,
|
|
62
60
|
dedup,
|
|
63
61
|
now: ctx.infra.clock(),
|
|
64
|
-
}, items, storedTextKeys(held));
|
|
62
|
+
}, items, storedTextKeys(held)));
|
|
65
63
|
return { ...tally, skipped: tally.skipped + extracted.length - items.length };
|
|
66
64
|
}
|
|
67
65
|
//# sourceMappingURL=session-texts.js.map
|
|
@@ -11,6 +11,7 @@ import type { OptionalGroup } from '../../base/ports/store.js';
|
|
|
11
11
|
import { type StoreSession } from '../open-store.js';
|
|
12
12
|
import type { RerankerFn } from '../../base/ports/reranker.js';
|
|
13
13
|
import { type CallerState } from '../../api/callers.js';
|
|
14
|
+
import type { DetachedWork } from '../../domain/search/detached-work.js';
|
|
14
15
|
/** The CLI process's config reader: the stdio MCP server lives as long as its host, so each call
|
|
15
16
|
* parses an unchanged config.json once. */
|
|
16
17
|
export declare const cliConfig: ConfigReader;
|
|
@@ -49,6 +50,10 @@ export declare function cliStoreBatch(hippoRoot: string): <T>(run: (store: Hippo
|
|
|
49
50
|
/** The session a scoped CLI verb runs in, one handle per store; named here so no other surface
|
|
50
51
|
* opens one. */
|
|
51
52
|
export declare function cliRequest<T>(run: () => T | Promise<T>): Promise<T>;
|
|
53
|
+
/** Where a verb hands work it does not wait on, such as an embed: the command's runtime, which
|
|
54
|
+
* settles it before the command ends. Outside a command no runtime closes under it, so it runs on
|
|
55
|
+
* unowned, as before. */
|
|
56
|
+
export declare function cliDetached(): DetachedWork;
|
|
52
57
|
/** The CLI's store is hippo.db, which sets every optional group, so its verbs call the api
|
|
53
58
|
* unchecked; the folder rides beside the api's Context for the verbs that reopen it. */
|
|
54
59
|
export type CliContext = ContextWith<OptionalGroup> & {
|
|
@@ -15,6 +15,7 @@ import { createCallerState, newRowsOn, } from '../../api/callers.js';
|
|
|
15
15
|
import { rootInfra } from '../../infra/ports.js';
|
|
16
16
|
import { createHost } from '../../host/host.js';
|
|
17
17
|
import { serverNearestAsk } from './server-nearest.js';
|
|
18
|
+
import { DetachedSet } from '../../base/util/detached-set.js';
|
|
18
19
|
/** The CLI process's config reader: the stdio MCP server lives as long as its host, so each call
|
|
19
20
|
* parses an unchanged config.json once. */
|
|
20
21
|
export const cliConfig = configReader(envRuntimeSettings);
|
|
@@ -108,6 +109,12 @@ export function cliStoreBatch(hippoRoot) {
|
|
|
108
109
|
export function cliRequest(run) {
|
|
109
110
|
return runStoreRequest((session) => asCommand(session, run), 'cli', cliRuntime());
|
|
110
111
|
}
|
|
112
|
+
/** Where a verb hands work it does not wait on, such as an embed: the command's runtime, which
|
|
113
|
+
* settles it before the command ends. Outside a command no runtime closes under it, so it runs on
|
|
114
|
+
* unowned, as before. */
|
|
115
|
+
export function cliDetached() {
|
|
116
|
+
return cliRuntime() ?? new DetachedSet();
|
|
117
|
+
}
|
|
111
118
|
/** The stores a CLI call on `hippoRoot` reaches, each global one with its config and on the
|
|
112
119
|
* call's session. */
|
|
113
120
|
function cliReach(hippoRoot, store, embedderFor, session = commandSession) {
|
|
@@ -137,6 +144,7 @@ export function cliApiContext(hippoRoot, tenantId, callers = createCallerState()
|
|
|
137
144
|
config,
|
|
138
145
|
callers,
|
|
139
146
|
infra,
|
|
147
|
+
detached: cliRuntime(),
|
|
140
148
|
};
|
|
141
149
|
}
|
|
142
150
|
/** A second store a CLI read searches beside its own, with that store's config. */
|
|
@@ -167,6 +175,8 @@ export function cliStdioHost() {
|
|
|
167
175
|
const hostTenantId = envTenant();
|
|
168
176
|
const shared = createCallerState();
|
|
169
177
|
const tools = allMcpTools();
|
|
178
|
+
// The command's runtime under `hippo mcp`; a host built outside a command keeps its own.
|
|
179
|
+
const detached = cliRuntime() ?? new DetachedSet();
|
|
170
180
|
// The rings span the loop; each folder keeps an auto-sleep and its latch, as a write's context
|
|
171
181
|
// names no folder, and each frame counts on its own store.
|
|
172
182
|
const byRoot = new Map();
|
|
@@ -196,12 +206,14 @@ export function cliStdioHost() {
|
|
|
196
206
|
callers: callersOn(hippoRoot, store),
|
|
197
207
|
infra,
|
|
198
208
|
signal,
|
|
209
|
+
detached,
|
|
199
210
|
};
|
|
200
211
|
};
|
|
201
212
|
// Frames interleave, so each takes a session of its own and hands it to every store its context
|
|
202
213
|
// builds.
|
|
203
214
|
return {
|
|
204
215
|
runFrame: (signal, run) => runStoreRequest((session) => run(contextIn(session, signal)), 'cli', cliRuntime()),
|
|
216
|
+
settle: (withinMs) => detached.settle(withinMs),
|
|
205
217
|
};
|
|
206
218
|
}
|
|
207
219
|
//# sourceMappingURL=api-context.js.map
|
|
@@ -4,6 +4,7 @@ import { DEFAULT_TENANT_ID } from '../../base/util/tenant.js';
|
|
|
4
4
|
import * as api from '../../api/index.js';
|
|
5
5
|
import { cliApiContext } from './api-context.js';
|
|
6
6
|
import { printError } from './output.js';
|
|
7
|
+
import { sanitizeLogMessage } from '../capture/compact.js';
|
|
7
8
|
import { boolFlag, flagValueless, stringFlag, stringFlagOrExit, } from './flag-values.js';
|
|
8
9
|
import { fail } from './exit.js';
|
|
9
10
|
// `hippo goal <push|list|complete|suspend|resume>`
|
|
@@ -16,10 +17,6 @@ const GOAL_POLICY_TYPES = [
|
|
|
16
17
|
function isGoalPolicyType(value) {
|
|
17
18
|
return GOAL_POLICY_TYPES.some((type) => type === value);
|
|
18
19
|
}
|
|
19
|
-
function sanitizeGoalName(s) {
|
|
20
|
-
// Strip C0 control chars + DEL to prevent terminal escape injection.
|
|
21
|
-
return s.replace(/[^\x20-\x7e\u0080-\uffff]/g, '?');
|
|
22
|
-
}
|
|
23
20
|
function resolveGoalSession(flags, defaultTenantId) {
|
|
24
21
|
const sessionId = (stringFlagOrExit(flags, 'session-id') ?? envHippoSessionId() ?? '').trim();
|
|
25
22
|
if (!sessionId) {
|
|
@@ -55,11 +52,12 @@ function readGoalLevel(flags) {
|
|
|
55
52
|
}
|
|
56
53
|
async function pushGoal(hippoRoot, defaultTenantId, args, flags) {
|
|
57
54
|
const rawName = args.join(' ').trim();
|
|
58
|
-
|
|
55
|
+
// Sanitize at WRITE time so corrupt names never enter the DB, and before the empty check so a
|
|
56
|
+
// name of control characters alone is no name.
|
|
57
|
+
const name = sanitizeLogMessage(rawName).trim();
|
|
58
|
+
if (!name) {
|
|
59
59
|
fail('Usage: hippo goal push <name> [--policy <type>] [--success "<condition>"] [--level N] [--parent <goalId>]');
|
|
60
60
|
}
|
|
61
|
-
// Sanitize at WRITE time so corrupt names never enter the DB.
|
|
62
|
-
const name = sanitizeGoalName(rawName);
|
|
63
61
|
if (name !== rawName) {
|
|
64
62
|
printError('note: stripped control characters from goal name');
|
|
65
63
|
}
|
|
@@ -97,7 +95,7 @@ async function listGoals(hippoRoot, defaultTenantId, flags) {
|
|
|
97
95
|
const rows = goals.map((g) => ({
|
|
98
96
|
id: g.id,
|
|
99
97
|
status: g.status,
|
|
100
|
-
name:
|
|
98
|
+
name: sanitizeLogMessage(g.goalName),
|
|
101
99
|
outcome: g.outcomeScore !== undefined ? g.outcomeScore.toString() : '-',
|
|
102
100
|
}));
|
|
103
101
|
const widths = {
|
|
@@ -5,7 +5,7 @@ import * as path from 'node:path';
|
|
|
5
5
|
import * as fs from 'node:fs';
|
|
6
6
|
import { bookTokenUse } from '../../api/ledger-db.js';
|
|
7
7
|
import { sessionPilotArm } from '../../api/pilot-arm.js';
|
|
8
|
-
import { cliFolder, cliInfra, cliLedgerReach, cliSyncOptions } from './api-context.js';
|
|
8
|
+
import { cliDetached, cliFolder, cliInfra, cliLedgerReach, cliSyncOptions } from './api-context.js';
|
|
9
9
|
import { detachedEmbedder } from '../../api/embeddings.js';
|
|
10
10
|
import { hookPayloadSessionId, hookPayloadString, isSubagentPayload, } from '../../base/util/hook-payload.js';
|
|
11
11
|
import { importAtSessionEnd } from '../../api/agent-memories.js';
|
|
@@ -156,7 +156,7 @@ export function captureTarget(root) {
|
|
|
156
156
|
root,
|
|
157
157
|
store: folder.store,
|
|
158
158
|
config: folder.config,
|
|
159
|
-
embed: detachedEmbedder(folder, cliInfra()),
|
|
159
|
+
embed: detachedEmbedder(folder, cliInfra(), cliDetached()),
|
|
160
160
|
};
|
|
161
161
|
}
|
|
162
162
|
//# sourceMappingURL=hook-runtime.js.map
|
|
@@ -31,11 +31,15 @@ async function warnBackfillFailed(ctx, embedCommand, err) {
|
|
|
31
31
|
log.warn(`import: embedding backfill failed (${errorMessage(err)}); ${waiting} rows have no vector; ` +
|
|
32
32
|
`run '${embedCommand}' to backfill`, errorFields(err));
|
|
33
33
|
}
|
|
34
|
-
/** Batch-
|
|
35
|
-
*
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
34
|
+
/** Batch-embeds freshly imported rows before the command returns, since the command's store
|
|
35
|
+
* runtime closes then. */
|
|
36
|
+
async function backfillVectors(ctx, embedCommand) {
|
|
37
|
+
try {
|
|
38
|
+
await memoryEmbedMissing(ctx);
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
await warnBackfillFailed(ctx, embedCommand, err);
|
|
42
|
+
}
|
|
39
43
|
}
|
|
40
44
|
function warnRedacted(count) {
|
|
41
45
|
if (count) {
|
|
@@ -94,7 +98,7 @@ async function importIntoStore(run) {
|
|
|
94
98
|
dryRun,
|
|
95
99
|
});
|
|
96
100
|
}
|
|
97
|
-
importFromFile(targetRoot, args, flags, { importOptions, useGlobal, dryRun, tenantId });
|
|
101
|
+
await importFromFile(targetRoot, args, flags, { importOptions, useGlobal, dryRun, tenantId });
|
|
98
102
|
}
|
|
99
103
|
// Determine which importer to use based on flag
|
|
100
104
|
function pickImporter(args, flags) {
|
|
@@ -138,7 +142,7 @@ function pickImporter(args, flags) {
|
|
|
138
142
|
return { filePath: args[0], importer: importGenericFileInto, importerName: 'File' };
|
|
139
143
|
return { filePath: undefined, importer: undefined, importerName: '' };
|
|
140
144
|
}
|
|
141
|
-
function importFromFile(targetRoot, args, flags, options) {
|
|
145
|
+
async function importFromFile(targetRoot, args, flags, options) {
|
|
142
146
|
const { importOptions, useGlobal, dryRun, tenantId } = options;
|
|
143
147
|
const { filePath, importer, importerName } = pickImporter(args, flags);
|
|
144
148
|
if (!filePath || !importer) {
|
|
@@ -149,12 +153,12 @@ function importFromFile(targetRoot, args, flags, options) {
|
|
|
149
153
|
fail(`File not found: ${filePath}`);
|
|
150
154
|
}
|
|
151
155
|
const result = importer(filePath, importOptions);
|
|
156
|
+
const storeLabel = useGlobal ? `global (${hippoHomeDir()})` : targetRoot;
|
|
157
|
+
printFileImportSummary(result, { importerName, filePath, storeLabel, dryRun });
|
|
152
158
|
// The importers' writeEntry sites don't embed, so the new rows are embedded here in one pass.
|
|
153
159
|
if (!dryRun && result.imported >= 1) {
|
|
154
|
-
backfillVectors(cliApiContext(targetRoot, tenantId), useGlobal ? 'hippo embed --global' : 'hippo embed');
|
|
160
|
+
await backfillVectors(cliApiContext(targetRoot, tenantId), useGlobal ? 'hippo embed --global' : 'hippo embed');
|
|
155
161
|
}
|
|
156
|
-
const storeLabel = useGlobal ? `global (${hippoHomeDir()})` : targetRoot;
|
|
157
|
-
printFileImportSummary(result, { importerName, filePath, storeLabel, dryRun });
|
|
158
162
|
}
|
|
159
163
|
function printFileImportSummary(result, { importerName, filePath, storeLabel, dryRun }) {
|
|
160
164
|
console.log(`\nImport ${importerName}: ${filePath}`);
|
|
@@ -206,7 +210,12 @@ async function importVaultFolder(options) {
|
|
|
206
210
|
batch: cliStoreBatch(hippoRoot),
|
|
207
211
|
infra: cliInfra(),
|
|
208
212
|
});
|
|
209
|
-
|
|
213
|
+
const ctx = cliApiContext(hippoRoot, tenantId);
|
|
214
|
+
printVaultSummary(vaultResult, folderPath, ctx, dryRun);
|
|
215
|
+
// Vault rows write through api.remember, which never embeds, so backfill them as the single-file
|
|
216
|
+
// import does.
|
|
217
|
+
if (!dryRun && vaultResult.imported >= 1)
|
|
218
|
+
await backfillVectors(ctx, 'hippo embed');
|
|
210
219
|
}
|
|
211
220
|
function checkVaultArgs(folderPath, flags, useGlobal) {
|
|
212
221
|
if (!fs.existsSync(folderPath) || !fs.statSync(folderPath).isDirectory()) {
|
|
@@ -239,9 +248,5 @@ function printVaultSummary(vaultResult, folderPath, ctx, dryRun) {
|
|
|
239
248
|
warnRedacted(vaultResult.redacted);
|
|
240
249
|
console.log(` ${dryRun ? 'Would archive: ' : 'Archived (removed): '}${vaultResult.archived ?? 0}`);
|
|
241
250
|
console.log(` Store: ${ctx.hippoRoot}`);
|
|
242
|
-
// Vault rows write through api.remember, which never embeds, so backfill them as the single-file
|
|
243
|
-
// import does.
|
|
244
|
-
if (!dryRun && vaultResult.imported >= 1)
|
|
245
|
-
backfillVectors(ctx, 'hippo embed');
|
|
246
251
|
}
|
|
247
252
|
//# sourceMappingURL=memory-import.js.map
|
|
@@ -10,6 +10,7 @@ import type { ContextWith } from '../../api/types.js';
|
|
|
10
10
|
import type { ConfiguredFolder } from '../../base/config/configured-folder.js';
|
|
11
11
|
import type { CallerState } from '../../api/callers.js';
|
|
12
12
|
import type { InfraPorts } from '../../base/ports/infra.js';
|
|
13
|
+
import type { DetachedWork } from '../../domain/search/detached-work.js';
|
|
13
14
|
/** Same bounded walk as the CLI (ends at home, so HIPPO_HOME wins over ~/.hippo); cwd/opts are
|
|
14
15
|
* the test seam. */
|
|
15
16
|
export declare function findHippoRoot(cwd?: string, opts?: ResolveProjectIdentityOptions): string | null;
|
|
@@ -55,6 +56,8 @@ export interface McpCaller<S extends RequiredStore = RequiredStore> {
|
|
|
55
56
|
clientKey?: string;
|
|
56
57
|
/** Aborts when the call passes its deadline, so a tool stops starting work nobody waits for. */
|
|
57
58
|
signal?: AbortSignal;
|
|
59
|
+
/** The transport root's keeper of work a call does not wait on; unset, each call waits for it. */
|
|
60
|
+
detached?: DetachedWork;
|
|
58
61
|
}
|
|
59
62
|
/** A tool's run for one call's arguments. */
|
|
60
63
|
export type ToolRun = (args: Record<string, JsonValue>) => string | Promise<string>;
|
|
@@ -84,6 +87,9 @@ export type McpContextFor<S extends RequiredStore = RequiredStore> = (hippoRoot:
|
|
|
84
87
|
* src/surface/mcp does not. */
|
|
85
88
|
export interface McpHost<S extends RequiredStore = RequiredStore> {
|
|
86
89
|
runFrame<T>(signal: AbortSignal, run: (contextFor: McpContextFor<S>) => Promise<T>): Promise<T>;
|
|
90
|
+
/** Waits up to `withinMs` for the work the frames handed off, such as an embed; resolves with
|
|
91
|
+
* how many were still running. */
|
|
92
|
+
settle(withinMs: number): Promise<number>;
|
|
87
93
|
}
|
|
88
94
|
/** One tool call after the store, config and tenant are resolved; every handler reads the same
|
|
89
95
|
* five. */
|
|
@@ -96,7 +96,9 @@ function track(method, call) {
|
|
|
96
96
|
.finally(() => inFlight.delete(call))
|
|
97
97
|
.catch((err) => log.error(`mcp: ${method} failed after it was dispatched`, errorFields(err)));
|
|
98
98
|
}
|
|
99
|
-
|
|
99
|
+
/** Waits for the calls in flight, then for the work they handed off, one bound for both. */
|
|
100
|
+
async function drainInFlight(host) {
|
|
101
|
+
const started = performance.now();
|
|
100
102
|
if (inFlight.size > 0) {
|
|
101
103
|
let bound;
|
|
102
104
|
const timedOut = new Promise((resolve) => {
|
|
@@ -108,9 +110,13 @@ async function drainInFlight() {
|
|
|
108
110
|
for (const method of inFlight.values()) {
|
|
109
111
|
log.warn(`mcp: ${method} was still running ${STDIN_END_DRAIN_MS} ms after stdin closed; exiting without its reply`, { method });
|
|
110
112
|
}
|
|
113
|
+
const left = await host.settle(Math.max(0, STDIN_END_DRAIN_MS - (performance.now() - started)));
|
|
114
|
+
if (left > 0) {
|
|
115
|
+
log.warn(`mcp: ${left} handed-off task(s), such as an embed, still running ${STDIN_END_DRAIN_MS} ms after stdin closed`, { left });
|
|
116
|
+
}
|
|
111
117
|
}
|
|
112
|
-
async function exitWhenDrained() {
|
|
113
|
-
await drainInFlight();
|
|
118
|
+
async function exitWhenDrained(host) {
|
|
119
|
+
await drainInFlight(host);
|
|
114
120
|
exitAfterFlush(0);
|
|
115
121
|
}
|
|
116
122
|
/** Answers `req` with one timeout error at the deadline a served request has. The returned
|
|
@@ -222,7 +228,7 @@ export function startStdioLoop(host) {
|
|
|
222
228
|
runWithRequestId(randomUUID(), () => dispatch(body, abort, host));
|
|
223
229
|
}));
|
|
224
230
|
process.stdin.on('end', () => {
|
|
225
|
-
exitWhenDrained().catch((err) => {
|
|
231
|
+
exitWhenDrained(host).catch((err) => {
|
|
226
232
|
log.error('mcp: draining the calls in flight failed; exiting', errorFields(err));
|
|
227
233
|
exitAfterFlush(1);
|
|
228
234
|
});
|
|
@@ -231,7 +237,7 @@ export function startStdioLoop(host) {
|
|
|
231
237
|
// A crash waits for the calls in flight as stdin's end does, so a client is not left without its
|
|
232
238
|
// reply.
|
|
233
239
|
installCrashHandlers('mcp', {
|
|
234
|
-
run: drainInFlight,
|
|
240
|
+
run: () => drainInFlight(host),
|
|
235
241
|
boundMs: STDIN_END_DRAIN_MS + CRASH_DRAIN_SLACK_MS,
|
|
236
242
|
});
|
|
237
243
|
}
|
|
@@ -6,8 +6,8 @@ export { decodeVector, EMBEDDING_MODEL_META_KEY, encodeVector, OTHER_STORE_MARKE
|
|
|
6
6
|
/** The default store as ./store-kit publishes it, with only the rules an add-on picks: a runtime
|
|
7
7
|
* or a session belongs to a composition root, and the published entries export neither. */
|
|
8
8
|
export declare const publishedDefaultStore: (hippoRoot: string, rules?: StoreRules) => HippoStore & StoreGroups;
|
|
9
|
-
/** Runs `fn` on a store runtime of its own, disposed
|
|
10
|
-
* the process exits first. */
|
|
9
|
+
/** Runs `fn` on a store runtime of its own, disposed with the work `fn` detached on it once `fn`
|
|
10
|
+
* settles, or closed by its exit hook if the process exits first. */
|
|
11
11
|
export declare function withStoreRuntime<T>(fn: (runtime: StoreRuntime) => Promise<T>): Promise<T>;
|
|
12
12
|
/** hippo.db as a server serves it, with an exit hook that closes what the server opened if
|
|
13
13
|
* close() never runs. */
|
|
@@ -5,18 +5,22 @@ export { decodeVector, EMBEDDING_MODEL_META_KEY, encodeVector, OTHER_STORE_MARKE
|
|
|
5
5
|
/** The default store as ./store-kit publishes it, with only the rules an add-on picks: a runtime
|
|
6
6
|
* or a session belongs to a composition root, and the published entries export neither. */
|
|
7
7
|
export const publishedDefaultStore = sqliteStoreWith;
|
|
8
|
-
/** Runs `fn` on a store runtime of its own, disposed
|
|
9
|
-
* the process exits first. */
|
|
8
|
+
/** Runs `fn` on a store runtime of its own, disposed with the work `fn` detached on it once `fn`
|
|
9
|
+
* settles, or closed by its exit hook if the process exits first. */
|
|
10
10
|
export async function withStoreRuntime(fn) {
|
|
11
11
|
const runtime = new StoreRuntime();
|
|
12
|
-
const onExit = () => runtime.
|
|
12
|
+
const onExit = () => runtime.closeNow();
|
|
13
13
|
process.once('exit', onExit);
|
|
14
14
|
try {
|
|
15
15
|
return await fn(runtime);
|
|
16
16
|
}
|
|
17
17
|
finally {
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
try {
|
|
19
|
+
await runtime.dispose();
|
|
20
|
+
}
|
|
21
|
+
finally {
|
|
22
|
+
process.off('exit', onExit);
|
|
23
|
+
}
|
|
20
24
|
}
|
|
21
25
|
}
|
|
22
26
|
/** hippo.db as a server serves it, with an exit hook that closes what the server opened if
|
|
@@ -27,9 +31,9 @@ export function defaultServerRuntime(hippoRoot, opts = {}) {
|
|
|
27
31
|
process.once('exit', onExit);
|
|
28
32
|
return {
|
|
29
33
|
...runtime,
|
|
30
|
-
close: async () => {
|
|
34
|
+
close: async (settleWithinMs) => {
|
|
31
35
|
process.off('exit', onExit);
|
|
32
|
-
await runtime.close();
|
|
36
|
+
await runtime.close(settleWithinMs);
|
|
33
37
|
},
|
|
34
38
|
};
|
|
35
39
|
}
|
|
@@ -26,7 +26,7 @@ type AuthHeader = {
|
|
|
26
26
|
};
|
|
27
27
|
export declare function readAuthHeader(req: IncomingMessage): AuthHeader;
|
|
28
28
|
type AuthOptions = Pick<ResolvedServeOptions, 'hippoRoot' | 'authResolver' | 'authResolverTimeoutMs' | 'store' | 'reachOf' | 'callerLimiter' | 'localKey' | 'keyVerifier'> & Partial<Pick<ResolvedServeOptions, 'failedAuthLimiter'>>;
|
|
29
|
-
type ContextOptions = AuthOptions & Pick<ResolvedServeOptions, 'configAt' | 'onHippoDb' | 'infra'> & Partial<Pick<ResolvedServeOptions, 'callers' | 'signal' | 'ledgerStore'>>;
|
|
29
|
+
type ContextOptions = AuthOptions & Pick<ResolvedServeOptions, 'configAt' | 'onHippoDb' | 'infra' | 'detached'> & Partial<Pick<ResolvedServeOptions, 'callers' | 'signal' | 'ledgerStore'>>;
|
|
30
30
|
/** Add-ons call this to refuse a subject that would collide with a built-in actor. */
|
|
31
31
|
export declare function isReservedActor(subject: string): boolean;
|
|
32
32
|
/** Builds a per-request Context from the Authorization header and remote address; throws
|
|
@@ -323,8 +323,19 @@ function contextFor(id, opts, clientKey) {
|
|
|
323
323
|
const reach = contextReach(opts.reachOf(team), opts.configAt, infra.embedderFor, ledger);
|
|
324
324
|
const callers = opts.callers;
|
|
325
325
|
const actor = bearerActor(id);
|
|
326
|
-
const { store, signal } = opts;
|
|
327
|
-
return {
|
|
326
|
+
const { store, signal, detached } = opts;
|
|
327
|
+
return {
|
|
328
|
+
tenantId: id.tenantId,
|
|
329
|
+
actor,
|
|
330
|
+
store,
|
|
331
|
+
reach,
|
|
332
|
+
config,
|
|
333
|
+
callers,
|
|
334
|
+
clientKey,
|
|
335
|
+
infra,
|
|
336
|
+
signal,
|
|
337
|
+
detached,
|
|
338
|
+
};
|
|
328
339
|
}
|
|
329
340
|
/** Auth check for routes that need no tenant Context (e.g. the MCP transport, which resolves its
|
|
330
341
|
* own root): throws HttpError 401 like buildContextWithAuth. */
|
|
@@ -15,7 +15,7 @@ import { isLocalCaller } from './auth.js';
|
|
|
15
15
|
import { assertClientIpHeaderPinned, enforceRateLimit } from './client-ip.js';
|
|
16
16
|
import { createSleepQueue } from './sleep-offload.js';
|
|
17
17
|
import { createCallerState, newRowsOn } from '../../api/callers.js';
|
|
18
|
-
import { DEFAULT_SHUTDOWN_DRAIN_MS, drainAndClose, setKeepAliveTimeouts, shutdownBoundMs, } from './lifecycle.js';
|
|
18
|
+
import { DEFAULT_SHUTDOWN_DRAIN_MS, DETACHED_SETTLE_MS, drainAndClose, setKeepAliveTimeouts, shutdownBoundMs, } from './lifecycle.js';
|
|
19
19
|
import { createReadyProbe } from './ready.js';
|
|
20
20
|
import { DEFAULT_SERVER_HOST, DEFAULT_SERVER_PORT } from './defaults.js';
|
|
21
21
|
import { installCrashHandlers, installSignalHandlers } from '../../base/util/crash-handlers.js';
|
|
@@ -221,7 +221,7 @@ async function bindClaimed(opts, plan) {
|
|
|
221
221
|
// An auto-sleep answers no request, so draining skips it; its child must leave the store before
|
|
222
222
|
// the store closes.
|
|
223
223
|
await served.sleeps.settled();
|
|
224
|
-
await runtime.close();
|
|
224
|
+
await runtime.close(DETACHED_SETTLE_MS);
|
|
225
225
|
};
|
|
226
226
|
if (opts.handleSignals) {
|
|
227
227
|
exitOnSignalOrCrash(stop, opts.shutdownDrainMs ?? DEFAULT_SHUTDOWN_DRAIN_MS);
|
|
@@ -280,6 +280,7 @@ function servedOptsFor(opts, { routes, publicJsonBodies, limiters }, runtime, lo
|
|
|
280
280
|
callersFor: (storeFor) => callersOnRequest(callers, counted && storeFor(counted)),
|
|
281
281
|
ledgerStore: hasGroups(store, ['ledger', 'folderFiles']) ? store : undefined,
|
|
282
282
|
infra: rootInfra(runtime.host),
|
|
283
|
+
detached: runtime.detached,
|
|
283
284
|
sleeps,
|
|
284
285
|
counts,
|
|
285
286
|
};
|
|
@@ -13,7 +13,7 @@ import { answerRequest } from './failure-replies.js';
|
|
|
13
13
|
import { startDashboardReader } from './dashboard-reader.js';
|
|
14
14
|
import { answerDashboard } from './dashboard-routes.js';
|
|
15
15
|
import { dashboardTickets } from './dashboard-session.js';
|
|
16
|
-
import { DEFAULT_SHUTDOWN_DRAIN_MS, drainAndClose, setKeepAliveTimeouts, shutdownBoundMs, } from './lifecycle.js';
|
|
16
|
+
import { DEFAULT_SHUTDOWN_DRAIN_MS, DETACHED_SETTLE_MS, drainAndClose, setKeepAliveTimeouts, shutdownBoundMs, } from './lifecycle.js';
|
|
17
17
|
import { createListener } from './tls.js';
|
|
18
18
|
import { rootInfra } from '../../infra/ports.js';
|
|
19
19
|
// The audit log's actor for every dashboard write.
|
|
@@ -67,7 +67,7 @@ export async function serveDashboard(opts) {
|
|
|
67
67
|
// close is the last.
|
|
68
68
|
const release = async () => {
|
|
69
69
|
await serving.reader.close();
|
|
70
|
-
await runtime.close();
|
|
70
|
+
await runtime.close(DETACHED_SETTLE_MS);
|
|
71
71
|
};
|
|
72
72
|
try {
|
|
73
73
|
await listenOn(server, opts.port, DASHBOARD_HOST);
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import type { Server, ServerResponse } from 'node:http';
|
|
2
2
|
export declare function setKeepAliveTimeouts(server: Server): void;
|
|
3
3
|
export declare const DEFAULT_SHUTDOWN_DRAIN_MS = 5000;
|
|
4
|
+
/** How long a stop waits for work a request handed off, such as an embed, before the stores close. */
|
|
5
|
+
export declare const DETACHED_SETTLE_MS = 5000;
|
|
4
6
|
/** How long a signal or crash shutdown may run before the process is ended without it. */
|
|
5
7
|
export declare function shutdownBoundMs(drainMs: number): number;
|
|
6
8
|
/**
|
|
@@ -15,8 +15,10 @@ export function setKeepAliveTimeouts(server) {
|
|
|
15
15
|
server.requestTimeout = Math.max(REQUEST_RECEIVE_TIMEOUT_MS, server.headersTimeout + bodyDeadlineMs(envBodyTimeoutMs()));
|
|
16
16
|
}
|
|
17
17
|
export const DEFAULT_SHUTDOWN_DRAIN_MS = 5000;
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
/** How long a stop waits for work a request handed off, such as an embed, before the stores close. */
|
|
19
|
+
export const DETACHED_SETTLE_MS = 5000;
|
|
20
|
+
// Past the request drain, a stop waits up to DETACHED_SETTLE_MS for handed-off work, then for each
|
|
21
|
+
// store thread to end its statement and close: seconds at most, so longer means a stuck thread.
|
|
20
22
|
const STORE_CLOSE_GRACE_MS = 10_000;
|
|
21
23
|
/** How long a signal or crash shutdown may run before the process is ended without it. */
|
|
22
24
|
export function shutdownBoundMs(drainMs) {
|