bulkhead-runtime 0.1.0 → 2026.4.5-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +337 -234
- package/dist/cli.js +5 -1
- package/dist/cli.js.map +1 -1
- package/dist/config/index.d.ts +28 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +9 -6
- package/dist/config/index.js.map +1 -1
- package/dist/credentials/store.d.ts.map +1 -1
- package/dist/credentials/store.js +39 -15
- package/dist/credentials/store.js.map +1 -1
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +38 -1
- package/dist/index.js.map +1 -1
- package/dist/infra/warning-filter.js +1 -1
- package/dist/infra/warning-filter.js.map +1 -1
- package/dist/logging/subsystem.d.ts +15 -1
- package/dist/logging/subsystem.d.ts.map +1 -1
- package/dist/logging/subsystem.js +310 -45
- package/dist/logging/subsystem.js.map +1 -1
- package/dist/memory/embedding-batch.d.ts +38 -0
- package/dist/memory/embedding-batch.d.ts.map +1 -0
- package/dist/memory/embedding-batch.js +253 -0
- package/dist/memory/embedding-batch.js.map +1 -0
- package/dist/memory/embedding-cache.d.ts +16 -0
- package/dist/memory/embedding-cache.d.ts.map +1 -0
- package/dist/memory/embedding-cache.js +113 -0
- package/dist/memory/embedding-cache.js.map +1 -0
- package/dist/memory/embeddings-debug.js +1 -1
- package/dist/memory/embeddings.d.ts +1 -0
- package/dist/memory/embeddings.d.ts.map +1 -1
- package/dist/memory/embeddings.js +115 -92
- package/dist/memory/embeddings.js.map +1 -1
- package/dist/memory/file-indexer.d.ts +26 -0
- package/dist/memory/file-indexer.d.ts.map +1 -0
- package/dist/memory/file-indexer.js +245 -0
- package/dist/memory/file-indexer.js.map +1 -0
- package/dist/memory/hybrid.d.ts.map +1 -1
- package/dist/memory/hybrid.js +6 -2
- package/dist/memory/hybrid.js.map +1 -1
- package/dist/memory/index.d.ts +5 -0
- package/dist/memory/index.d.ts.map +1 -1
- package/dist/memory/index.js +5 -2
- package/dist/memory/index.js.map +1 -1
- package/dist/memory/session-indexer.d.ts +41 -0
- package/dist/memory/session-indexer.d.ts.map +1 -0
- package/dist/memory/session-indexer.js +341 -0
- package/dist/memory/session-indexer.js.map +1 -0
- package/dist/memory/simple-manager.d.ts +6 -0
- package/dist/memory/simple-manager.d.ts.map +1 -1
- package/dist/memory/simple-manager.js +35 -12
- package/dist/memory/simple-manager.js.map +1 -1
- package/dist/memory/ssrf.d.ts +18 -0
- package/dist/memory/ssrf.d.ts.map +1 -0
- package/dist/memory/ssrf.js +316 -0
- package/dist/memory/ssrf.js.map +1 -0
- package/dist/package.json +8 -5
- package/dist/platform/platform.d.ts.map +1 -1
- package/dist/platform/platform.js +30 -7
- package/dist/platform/platform.js.map +1 -1
- package/dist/platform/types.d.ts +2 -0
- package/dist/platform/types.d.ts.map +1 -1
- package/dist/runtime/agent.d.ts +8 -0
- package/dist/runtime/agent.d.ts.map +1 -1
- package/dist/runtime/agent.js +194 -46
- package/dist/runtime/agent.js.map +1 -1
- package/dist/runtime/api-key-rotation.d.ts +26 -0
- package/dist/runtime/api-key-rotation.d.ts.map +1 -0
- package/dist/runtime/api-key-rotation.js +174 -0
- package/dist/runtime/api-key-rotation.js.map +1 -0
- package/dist/runtime/context-guard.d.ts +32 -0
- package/dist/runtime/context-guard.d.ts.map +1 -0
- package/dist/runtime/context-guard.js +61 -0
- package/dist/runtime/context-guard.js.map +1 -0
- package/dist/runtime/failover-error.d.ts +62 -0
- package/dist/runtime/failover-error.d.ts.map +1 -0
- package/dist/runtime/failover-error.js +733 -0
- package/dist/runtime/failover-error.js.map +1 -0
- package/dist/runtime/failover-policy.d.ts +5 -0
- package/dist/runtime/failover-policy.d.ts.map +1 -0
- package/dist/runtime/failover-policy.js +18 -0
- package/dist/runtime/failover-policy.js.map +1 -0
- package/dist/runtime/index.d.ts +11 -0
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +11 -0
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/memory-flush.d.ts +24 -0
- package/dist/runtime/memory-flush.d.ts.map +1 -0
- package/dist/runtime/memory-flush.js +64 -0
- package/dist/runtime/memory-flush.js.map +1 -0
- package/dist/runtime/memory-tools.d.ts +14 -0
- package/dist/runtime/memory-tools.d.ts.map +1 -0
- package/dist/runtime/memory-tools.js +58 -0
- package/dist/runtime/memory-tools.js.map +1 -0
- package/dist/runtime/model-fallback.d.ts +56 -0
- package/dist/runtime/model-fallback.d.ts.map +1 -0
- package/dist/runtime/model-fallback.js +301 -0
- package/dist/runtime/model-fallback.js.map +1 -0
- package/dist/runtime/model-fallback.types.d.ts +14 -0
- package/dist/runtime/model-fallback.types.d.ts.map +1 -0
- package/dist/runtime/model-fallback.types.js +3 -0
- package/dist/runtime/model-fallback.types.js.map +1 -0
- package/dist/runtime/retry.d.ts +24 -0
- package/dist/runtime/retry.d.ts.map +1 -0
- package/dist/runtime/retry.js +102 -0
- package/dist/runtime/retry.js.map +1 -0
- package/dist/runtime/session-pruning.d.ts +22 -0
- package/dist/runtime/session-pruning.d.ts.map +1 -0
- package/dist/runtime/session-pruning.js +118 -0
- package/dist/runtime/session-pruning.js.map +1 -0
- package/dist/runtime/stream-adapters.d.ts +11 -0
- package/dist/runtime/stream-adapters.d.ts.map +1 -0
- package/dist/runtime/stream-adapters.js +46 -0
- package/dist/runtime/stream-adapters.js.map +1 -0
- package/dist/runtime/subagent.d.ts +83 -0
- package/dist/runtime/subagent.d.ts.map +1 -0
- package/dist/runtime/subagent.js +190 -0
- package/dist/runtime/subagent.js.map +1 -0
- package/dist/runtime/tool-result-truncation.d.ts +25 -0
- package/dist/runtime/tool-result-truncation.d.ts.map +1 -0
- package/dist/runtime/tool-result-truncation.js +115 -0
- package/dist/runtime/tool-result-truncation.js.map +1 -0
- package/dist/sandbox/cgroup.d.ts +4 -1
- package/dist/sandbox/cgroup.d.ts.map +1 -1
- package/dist/sandbox/cgroup.js +28 -15
- package/dist/sandbox/cgroup.js.map +1 -1
- package/dist/sandbox/index.d.ts +2 -1
- package/dist/sandbox/index.d.ts.map +1 -1
- package/dist/sandbox/index.js +2 -1
- package/dist/sandbox/index.js.map +1 -1
- package/dist/sandbox/ipc.d.ts +4 -1
- package/dist/sandbox/ipc.d.ts.map +1 -1
- package/dist/sandbox/ipc.js +33 -17
- package/dist/sandbox/ipc.js.map +1 -1
- package/dist/sandbox/manager.d.ts +1 -2
- package/dist/sandbox/manager.d.ts.map +1 -1
- package/dist/sandbox/manager.js +132 -130
- package/dist/sandbox/manager.js.map +1 -1
- package/dist/sandbox/namespace.d.ts +1 -1
- package/dist/sandbox/namespace.d.ts.map +1 -1
- package/dist/sandbox/namespace.js +36 -37
- package/dist/sandbox/namespace.js.map +1 -1
- package/dist/sandbox/rootfs.d.ts +6 -1
- package/dist/sandbox/rootfs.d.ts.map +1 -1
- package/dist/sandbox/rootfs.js +114 -30
- package/dist/sandbox/rootfs.js.map +1 -1
- package/dist/sandbox/seccomp-apply.d.ts +9 -0
- package/dist/sandbox/seccomp-apply.d.ts.map +1 -0
- package/dist/sandbox/seccomp-apply.js +227 -0
- package/dist/sandbox/seccomp-apply.js.map +1 -0
- package/dist/sandbox/seccomp.js +3 -3
- package/dist/sandbox/seccomp.js.map +1 -1
- package/dist/sandbox/types.d.ts +1 -3
- package/dist/sandbox/types.d.ts.map +1 -1
- package/dist/sandbox/types.js.map +1 -1
- package/dist/sandbox/worker.d.ts +3 -0
- package/dist/sandbox/worker.d.ts.map +1 -1
- package/dist/sandbox/worker.js +84 -17
- package/dist/sandbox/worker.js.map +1 -1
- package/dist/sessions/index.d.ts +1 -0
- package/dist/sessions/index.d.ts.map +1 -1
- package/dist/sessions/index.js +1 -0
- package/dist/sessions/index.js.map +1 -1
- package/dist/sessions/store.d.ts +2 -2
- package/dist/sessions/store.d.ts.map +1 -1
- package/dist/sessions/store.js +49 -27
- package/dist/sessions/store.js.map +1 -1
- package/dist/sessions/transcript-events.d.ts +11 -0
- package/dist/sessions/transcript-events.d.ts.map +1 -0
- package/dist/sessions/transcript-events.js +40 -0
- package/dist/sessions/transcript-events.js.map +1 -0
- package/dist/shared/agent-session.d.ts +10 -0
- package/dist/shared/agent-session.d.ts.map +1 -0
- package/dist/shared/agent-session.js +33 -0
- package/dist/shared/agent-session.js.map +1 -0
- package/dist/shared/constants.d.ts +6 -0
- package/dist/shared/constants.d.ts.map +1 -0
- package/dist/shared/constants.js +11 -0
- package/dist/shared/constants.js.map +1 -0
- package/dist/shared/fs.d.ts +7 -0
- package/dist/shared/fs.d.ts.map +1 -0
- package/dist/shared/fs.js +14 -0
- package/dist/shared/fs.js.map +1 -0
- package/dist/shared/index.d.ts +4 -0
- package/dist/shared/index.d.ts.map +1 -0
- package/dist/shared/index.js +4 -0
- package/dist/shared/index.js.map +1 -0
- package/dist/skills/enablement.d.ts.map +1 -1
- package/dist/skills/enablement.js +2 -2
- package/dist/skills/enablement.js.map +1 -1
- package/dist/workspace/runner.d.ts.map +1 -1
- package/dist/workspace/runner.js +353 -106
- package/dist/workspace/runner.js.map +1 -1
- package/dist/workspace/types.d.ts +1 -0
- package/dist/workspace/types.d.ts.map +1 -1
- package/dist/workspace/workspace.d.ts.map +1 -1
- package/dist/workspace/workspace.js +12 -3
- package/dist/workspace/workspace.js.map +1 -1
- package/package.json +1 -1
package/dist/memory/hybrid.js
CHANGED
|
@@ -14,8 +14,12 @@ export function buildFtsQuery(raw) {
|
|
|
14
14
|
return quoted.join(" AND ");
|
|
15
15
|
}
|
|
16
16
|
export function bm25RankToScore(rank) {
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
if (!Number.isFinite(rank))
|
|
18
|
+
return 0;
|
|
19
|
+
// bm25() returns values <= 0 where more negative = more relevant.
|
|
20
|
+
// Negate to get a positive relevance value, then normalize to [0, 1).
|
|
21
|
+
const positive = Math.max(0, -rank);
|
|
22
|
+
return positive / (1 + positive);
|
|
19
23
|
}
|
|
20
24
|
export async function mergeHybridResults(params) {
|
|
21
25
|
const byId = new Map();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid.js","sourceRoot":"","sources":["../../src/memory/hybrid.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAkB,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACvF,OAAO,EACL,iCAAiC,EAEjC,6BAA6B,GAC9B,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EAAkB,kBAAkB,EAAE,CAAC;AAC9C,OAAO,EAA4B,6BAA6B,EAAE,CAAC;AAsBnE,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,MAAM,MAAM,GACV,GAAG;SACA,KAAK,CAAC,kBAAkB,CAAC;QAC1B,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACrB,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAC/D,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,
|
|
1
|
+
{"version":3,"file":"hybrid.js","sourceRoot":"","sources":["../../src/memory/hybrid.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAkB,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACvF,OAAO,EACL,iCAAiC,EAEjC,6BAA6B,GAC9B,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EAAkB,kBAAkB,EAAE,CAAC;AAC9C,OAAO,EAA4B,6BAA6B,EAAE,CAAC;AAsBnE,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,MAAM,MAAM,GACV,GAAG;SACA,KAAK,CAAC,kBAAkB,CAAC;QAC1B,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACrB,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAC/D,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACrC,kEAAkE;IAClE,sEAAsE;IACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IACpC,OAAO,QAAQ,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAYxC;IAUC,MAAM,IAAI,GAAG,IAAI,GAAG,EAYjB,CAAC;IAEJ,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACb,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,SAAS,EAAE,CAAC;SACb,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;YACjC,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,QAAQ,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC;YAC/B,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACb,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACrD,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC;QAC5F,OAAO;YACL,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK;YACL,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,mBAAmB,GAAG,EAAE,GAAG,6BAA6B,EAAE,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;IAC1F,MAAM,OAAO,GAAG,MAAM,iCAAiC,CAAC;QACtD,OAAO,EAAE,MAAM;QACf,aAAa,EAAE,mBAAmB;QAClC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAE7D,kCAAkC;IAClC,MAAM,SAAS,GAAG,EAAE,GAAG,kBAAkB,EAAE,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;IAC3D,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;QACtB,OAAO,uBAAuB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/memory/index.d.ts
CHANGED
|
@@ -9,4 +9,9 @@ export { extractKeywords } from "./query-expansion.js";
|
|
|
9
9
|
export { buildFtsQuery, bm25RankToScore } from "./hybrid.js";
|
|
10
10
|
export { requireNodeSqlite } from "./sqlite.js";
|
|
11
11
|
export { createSimpleMemoryManager, type SimpleMemoryManager } from "./simple-manager.js";
|
|
12
|
+
export { createEmbeddingCache, hashContent, type EmbeddingCache, } from "./embedding-cache.js";
|
|
13
|
+
export { embedBatchWithCacheAndRetry, embedQueryWithTimeout, runBatchWithFallback, createBatchFailureState, enforceEmbeddingMaxInputTokens, type BatchEmbeddingOptions, type BatchEmbeddingResult, type BatchFailureState, } from "./embedding-batch.js";
|
|
14
|
+
export { createFileIndexer, type FileIndexer, type FileIndexerOptions, type FileIndexSyncResult, } from "./file-indexer.js";
|
|
15
|
+
export { createSessionIndexer, type SessionIndexer, type SessionIndexerOptions, type SessionIndexResult, } from "./session-indexer.js";
|
|
16
|
+
export { buildBaseUrlPolicy, validateUrl, fetchWithSsrfGuard, type SsrfPolicy, } from "./ssrf.js";
|
|
12
17
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/memory/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/memory/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,YAAY,EACZ,kBAAkB,EAClB,mBAAmB,EACnB,0BAA0B,EAC1B,wBAAwB,EACxB,oBAAoB,GACrB,MAAM,YAAY,CAAC;AAEpB,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEzD,OAAO,EACL,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,cAAc,EACd,KAAK,eAAe,EACpB,KAAK,WAAW,GACjB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAC9D,OAAO,EACL,iCAAiC,EACjC,yBAAyB,EACzB,gCAAgC,GACjC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,yBAAyB,EAAE,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1F,OAAO,EACL,oBAAoB,EACpB,WAAW,EACX,KAAK,cAAc,GACpB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,2BAA2B,EAC3B,qBAAqB,EACrB,oBAAoB,EACpB,uBAAuB,EACvB,8BAA8B,EAC9B,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,EACzB,KAAK,iBAAiB,GACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,iBAAiB,EACjB,KAAK,WAAW,EAChB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,GACzB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,oBAAoB,EACpB,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,GACxB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,kBAAkB,EAClB,WAAW,EACX,kBAAkB,EAClB,KAAK,UAAU,GAChB,MAAM,WAAW,CAAC"}
|
package/dist/memory/index.js
CHANGED
|
@@ -6,7 +6,10 @@ export { applyTemporalDecayToHybridResults, applyTemporalDecayToScore, calculate
|
|
|
6
6
|
export { extractKeywords } from "./query-expansion.js";
|
|
7
7
|
export { buildFtsQuery, bm25RankToScore } from "./hybrid.js";
|
|
8
8
|
export { requireNodeSqlite } from "./sqlite.js";
|
|
9
|
-
// Simple memory store for consumers that want store/search/delete/list
|
|
10
|
-
// without the full OpenClaw MemoryIndexManager infrastructure.
|
|
11
9
|
export { createSimpleMemoryManager } from "./simple-manager.js";
|
|
10
|
+
export { createEmbeddingCache, hashContent, } from "./embedding-cache.js";
|
|
11
|
+
export { embedBatchWithCacheAndRetry, embedQueryWithTimeout, runBatchWithFallback, createBatchFailureState, enforceEmbeddingMaxInputTokens, } from "./embedding-batch.js";
|
|
12
|
+
export { createFileIndexer, } from "./file-indexer.js";
|
|
13
|
+
export { createSessionIndexer, } from "./session-indexer.js";
|
|
14
|
+
export { buildBaseUrlPolicy, validateUrl, fetchWithSsrfGuard, } from "./ssrf.js";
|
|
12
15
|
//# sourceMappingURL=index.js.map
|
package/dist/memory/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/memory/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/memory/index.ts"],"names":[],"mappings":"AAWA,OAAO,EACL,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,cAAc,GAGf,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAC9D,OAAO,EACL,iCAAiC,EACjC,yBAAyB,EACzB,gCAAgC,GACjC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,yBAAyB,EAA4B,MAAM,qBAAqB,CAAC;AAE1F,OAAO,EACL,oBAAoB,EACpB,WAAW,GAEZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,2BAA2B,EAC3B,qBAAqB,EACrB,oBAAoB,EACpB,uBAAuB,EACvB,8BAA8B,GAI/B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,iBAAiB,GAIlB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,oBAAoB,GAIrB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,kBAAkB,EAClB,WAAW,EACX,kBAAkB,GAEnB,MAAM,WAAW,CAAC"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { SimpleMemoryManager } from "./simple-manager.js";
|
|
2
|
+
export type PostCompactionSyncMode = "off" | "async" | "await";
|
|
3
|
+
export interface MemoryIndexMeta {
|
|
4
|
+
model: string;
|
|
5
|
+
provider: string;
|
|
6
|
+
providerKey?: string;
|
|
7
|
+
chunkTokens: number;
|
|
8
|
+
chunkOverlap: number;
|
|
9
|
+
sources?: string[];
|
|
10
|
+
}
|
|
11
|
+
export interface SessionIndexerOptions {
|
|
12
|
+
sessionsDir: string;
|
|
13
|
+
memory: SimpleMemoryManager;
|
|
14
|
+
deltaBytes?: number;
|
|
15
|
+
deltaMessages?: number;
|
|
16
|
+
postCompactionSyncMode?: PostCompactionSyncMode;
|
|
17
|
+
embeddingProvider?: {
|
|
18
|
+
id: string;
|
|
19
|
+
model: string;
|
|
20
|
+
};
|
|
21
|
+
chunking?: {
|
|
22
|
+
tokens: number;
|
|
23
|
+
overlap: number;
|
|
24
|
+
};
|
|
25
|
+
onProviderError?: (error: unknown) => void;
|
|
26
|
+
}
|
|
27
|
+
export interface SessionIndexer {
|
|
28
|
+
indexSession(sessionFile: string): Promise<SessionIndexResult>;
|
|
29
|
+
indexAllSessions(): Promise<SessionIndexResult>;
|
|
30
|
+
onTranscriptUpdate(sessionFile: string): void;
|
|
31
|
+
startListener(): () => void;
|
|
32
|
+
runPostCompactionSideEffects(sessionFile: string): Promise<void>;
|
|
33
|
+
}
|
|
34
|
+
export interface SessionIndexResult {
|
|
35
|
+
sessionsProcessed: number;
|
|
36
|
+
chunksStored: number;
|
|
37
|
+
chunksRemoved: number;
|
|
38
|
+
errors: number;
|
|
39
|
+
}
|
|
40
|
+
export declare function createSessionIndexer(options: SessionIndexerOptions): SessionIndexer;
|
|
41
|
+
//# sourceMappingURL=session-indexer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-indexer.d.ts","sourceRoot":"","sources":["../../src/memory/session-indexer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAY/D,MAAM,MAAM,sBAAsB,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC;AAE/D,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,mBAAmB,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;IAChD,iBAAiB,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAClD,QAAQ,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;CAC5C;AAED,MAAM,WAAW,cAAc;IAC7B,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC/D,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAChD,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9C,aAAa,IAAI,MAAM,IAAI,CAAC;IAC5B,4BAA4B,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAClE;AAED,MAAM,WAAW,kBAAkB;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;CAChB;AAeD,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,qBAAqB,GAAG,cAAc,CAwVnF"}
|
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
// Ported from OpenClaw extensions/memory-core/src/memory/manager-sync-ops.ts
|
|
2
|
+
// Session listener, delta tracking, debounce, countNewlines, post-compaction support
|
|
3
|
+
import * as fs from "node:fs";
|
|
4
|
+
import * as path from "node:path";
|
|
5
|
+
import { createSubsystemLogger } from "../logging/subsystem.js";
|
|
6
|
+
import { hashText } from "./internal.js";
|
|
7
|
+
import { onSessionTranscriptUpdate, emitSessionTranscriptUpdate, } from "../sessions/transcript-events.js";
|
|
8
|
+
const log = createSubsystemLogger("memory/session-indexer");
|
|
9
|
+
// Constants from OpenClaw
|
|
10
|
+
const SESSION_DIRTY_DEBOUNCE_MS = 5000;
|
|
11
|
+
const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
|
|
12
|
+
export function createSessionIndexer(options) {
|
|
13
|
+
const { sessionsDir, memory, deltaBytes = 4096, deltaMessages = 10, postCompactionSyncMode = "async", embeddingProvider, chunking = { tokens: 500, overlap: 50 }, onProviderError, } = options;
|
|
14
|
+
let currentMeta = null;
|
|
15
|
+
function buildMeta() {
|
|
16
|
+
return {
|
|
17
|
+
model: embeddingProvider?.model ?? "fts-only",
|
|
18
|
+
provider: embeddingProvider?.id ?? "none",
|
|
19
|
+
chunkTokens: chunking.tokens,
|
|
20
|
+
chunkOverlap: chunking.overlap,
|
|
21
|
+
sources: ["sessions"],
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
function needsFullReindex() {
|
|
25
|
+
if (!currentMeta)
|
|
26
|
+
return true;
|
|
27
|
+
const expected = buildMeta();
|
|
28
|
+
return (currentMeta.model !== expected.model ||
|
|
29
|
+
currentMeta.provider !== expected.provider ||
|
|
30
|
+
currentMeta.chunkTokens !== expected.chunkTokens ||
|
|
31
|
+
currentMeta.chunkOverlap !== expected.chunkOverlap);
|
|
32
|
+
}
|
|
33
|
+
const sessionStates = new Map();
|
|
34
|
+
const sessionDeltas = new Map();
|
|
35
|
+
const pendingUpdates = new Map();
|
|
36
|
+
let sessionsDirty = false;
|
|
37
|
+
const sessionsDirtyFiles = new Set();
|
|
38
|
+
// --- countNewlines (from OpenClaw, 64KB chunk reads) ---
|
|
39
|
+
async function countNewlines(absPath, start, end) {
|
|
40
|
+
if (end <= start)
|
|
41
|
+
return 0;
|
|
42
|
+
let handle;
|
|
43
|
+
try {
|
|
44
|
+
handle = await fs.promises.open(absPath, "r");
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return 0;
|
|
48
|
+
}
|
|
49
|
+
try {
|
|
50
|
+
let offset = start;
|
|
51
|
+
let count = 0;
|
|
52
|
+
const buffer = Buffer.alloc(SESSION_DELTA_READ_CHUNK_BYTES);
|
|
53
|
+
while (offset < end) {
|
|
54
|
+
const toRead = Math.min(buffer.length, end - offset);
|
|
55
|
+
const { bytesRead } = await handle.read(buffer, 0, toRead, offset);
|
|
56
|
+
if (bytesRead <= 0)
|
|
57
|
+
break;
|
|
58
|
+
for (let i = 0; i < bytesRead; i += 1) {
|
|
59
|
+
if (buffer[i] === 10)
|
|
60
|
+
count += 1;
|
|
61
|
+
}
|
|
62
|
+
offset += bytesRead;
|
|
63
|
+
}
|
|
64
|
+
return count;
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
await handle.close();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// --- Delta tracking (from OpenClaw updateSessionDelta) ---
|
|
71
|
+
async function updateSessionDelta(sessionFile) {
|
|
72
|
+
let stat;
|
|
73
|
+
try {
|
|
74
|
+
stat = await fs.promises.stat(sessionFile);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return { shouldSync: false, state: { lastSize: 0, pendingBytes: 0, pendingMessages: 0 } };
|
|
78
|
+
}
|
|
79
|
+
const size = stat.size;
|
|
80
|
+
let state = sessionDeltas.get(sessionFile);
|
|
81
|
+
if (!state) {
|
|
82
|
+
state = { lastSize: 0, pendingBytes: 0, pendingMessages: 0 };
|
|
83
|
+
sessionDeltas.set(sessionFile, state);
|
|
84
|
+
}
|
|
85
|
+
const deltaB = Math.max(0, size - state.lastSize);
|
|
86
|
+
if (deltaB === 0 && size === state.lastSize) {
|
|
87
|
+
const bytesHit = deltaBytes <= 0 ? state.pendingBytes > 0 : state.pendingBytes >= deltaBytes;
|
|
88
|
+
const messagesHit = deltaMessages <= 0 ? state.pendingMessages > 0 : state.pendingMessages >= deltaMessages;
|
|
89
|
+
return { shouldSync: bytesHit || messagesHit, state };
|
|
90
|
+
}
|
|
91
|
+
if (size < state.lastSize) {
|
|
92
|
+
state.lastSize = size;
|
|
93
|
+
state.pendingBytes += size;
|
|
94
|
+
const shouldCountMessages = deltaMessages > 0 && (deltaBytes <= 0 || state.pendingBytes < deltaBytes);
|
|
95
|
+
if (shouldCountMessages) {
|
|
96
|
+
state.pendingMessages += await countNewlines(sessionFile, 0, size);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
state.pendingBytes += deltaB;
|
|
101
|
+
const shouldCountMessages = deltaMessages > 0 && (deltaBytes <= 0 || state.pendingBytes < deltaBytes);
|
|
102
|
+
if (shouldCountMessages) {
|
|
103
|
+
state.pendingMessages += await countNewlines(sessionFile, state.lastSize, size);
|
|
104
|
+
}
|
|
105
|
+
state.lastSize = size;
|
|
106
|
+
}
|
|
107
|
+
const bytesHit = deltaBytes <= 0 ? state.pendingBytes > 0 : state.pendingBytes >= deltaBytes;
|
|
108
|
+
const messagesHit = deltaMessages <= 0 ? state.pendingMessages > 0 : state.pendingMessages >= deltaMessages;
|
|
109
|
+
return { shouldSync: bytesHit || messagesHit, state };
|
|
110
|
+
}
|
|
111
|
+
function resetSessionDelta(absPath, size) {
|
|
112
|
+
const state = sessionDeltas.get(absPath);
|
|
113
|
+
if (!state)
|
|
114
|
+
return;
|
|
115
|
+
state.lastSize = size;
|
|
116
|
+
state.pendingBytes = 0;
|
|
117
|
+
state.pendingMessages = 0;
|
|
118
|
+
}
|
|
119
|
+
// --- Session file discovery ---
|
|
120
|
+
function findSessionFiles() {
|
|
121
|
+
const files = [];
|
|
122
|
+
try {
|
|
123
|
+
if (!fs.existsSync(sessionsDir))
|
|
124
|
+
return files;
|
|
125
|
+
const entries = fs.readdirSync(sessionsDir, { withFileTypes: true });
|
|
126
|
+
for (const entry of entries) {
|
|
127
|
+
if (entry.isDirectory()) {
|
|
128
|
+
const subDir = path.join(sessionsDir, entry.name);
|
|
129
|
+
try {
|
|
130
|
+
const subEntries = fs.readdirSync(subDir);
|
|
131
|
+
for (const subEntry of subEntries) {
|
|
132
|
+
if (subEntry.endsWith(".jsonl"))
|
|
133
|
+
files.push(path.join(subDir, subEntry));
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
catch { /* skip */ }
|
|
137
|
+
}
|
|
138
|
+
else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
|
|
139
|
+
files.push(path.join(sessionsDir, entry.name));
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
catch { /* sessions dir may not exist yet */ }
|
|
144
|
+
return files;
|
|
145
|
+
}
|
|
146
|
+
// --- Transcript parsing ---
|
|
147
|
+
function parseTranscript(filePath) {
|
|
148
|
+
const messages = [];
|
|
149
|
+
try {
|
|
150
|
+
const raw = fs.readFileSync(filePath, "utf-8");
|
|
151
|
+
for (const line of raw.split("\n")) {
|
|
152
|
+
if (!line.trim())
|
|
153
|
+
continue;
|
|
154
|
+
try {
|
|
155
|
+
const parsed = JSON.parse(line);
|
|
156
|
+
const role = String(parsed.role ?? "unknown");
|
|
157
|
+
let content = "";
|
|
158
|
+
if (typeof parsed.content === "string") {
|
|
159
|
+
content = parsed.content;
|
|
160
|
+
}
|
|
161
|
+
else if (Array.isArray(parsed.content)) {
|
|
162
|
+
content = parsed.content
|
|
163
|
+
.filter((p) => p.type === "text")
|
|
164
|
+
.map((p) => String(p.text ?? ""))
|
|
165
|
+
.join("\n");
|
|
166
|
+
}
|
|
167
|
+
if (content.trim())
|
|
168
|
+
messages.push({ role, content });
|
|
169
|
+
}
|
|
170
|
+
catch { /* skip malformed lines */ }
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
catch { /* file not readable */ }
|
|
174
|
+
return messages;
|
|
175
|
+
}
|
|
176
|
+
function summarizeTranscript(messages) {
|
|
177
|
+
const chunks = [];
|
|
178
|
+
for (const msg of messages) {
|
|
179
|
+
const prefix = msg.role === "assistant" ? "Assistant" : "User";
|
|
180
|
+
const truncated = msg.content.length > 500 ? msg.content.slice(0, 500) + "..." : msg.content;
|
|
181
|
+
chunks.push(`${prefix}: ${truncated}`);
|
|
182
|
+
}
|
|
183
|
+
return chunks.join("\n\n");
|
|
184
|
+
}
|
|
185
|
+
// --- Index a single session ---
|
|
186
|
+
async function indexSession(sessionFile) {
|
|
187
|
+
const result = { sessionsProcessed: 0, chunksStored: 0, chunksRemoved: 0, errors: 0 };
|
|
188
|
+
try {
|
|
189
|
+
if (!fs.existsSync(sessionFile))
|
|
190
|
+
return result;
|
|
191
|
+
const stat = fs.statSync(sessionFile);
|
|
192
|
+
const currentHash = hashText(fs.readFileSync(sessionFile, "utf-8"));
|
|
193
|
+
const existing = sessionStates.get(sessionFile);
|
|
194
|
+
if (existing && existing.hash === currentHash) {
|
|
195
|
+
result.sessionsProcessed++;
|
|
196
|
+
resetSessionDelta(sessionFile, stat.size);
|
|
197
|
+
return result;
|
|
198
|
+
}
|
|
199
|
+
if (existing) {
|
|
200
|
+
for (const id of existing.chunkIds) {
|
|
201
|
+
try {
|
|
202
|
+
await memory.delete(id);
|
|
203
|
+
result.chunksRemoved++;
|
|
204
|
+
}
|
|
205
|
+
catch {
|
|
206
|
+
result.errors++;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
const messages = parseTranscript(sessionFile);
|
|
211
|
+
if (messages.length < 2) {
|
|
212
|
+
result.sessionsProcessed++;
|
|
213
|
+
resetSessionDelta(sessionFile, stat.size);
|
|
214
|
+
return result;
|
|
215
|
+
}
|
|
216
|
+
const summary = summarizeTranscript(messages);
|
|
217
|
+
const relPath = path.relative(path.dirname(path.dirname(sessionFile)), sessionFile);
|
|
218
|
+
const CHUNK_SIZE = 2000;
|
|
219
|
+
const newChunkIds = [];
|
|
220
|
+
for (let i = 0; i < summary.length; i += CHUNK_SIZE) {
|
|
221
|
+
const chunk = summary.slice(i, i + CHUNK_SIZE);
|
|
222
|
+
try {
|
|
223
|
+
const id = await memory.store(chunk, { source: "session", path: relPath, messageCount: messages.length });
|
|
224
|
+
newChunkIds.push(id);
|
|
225
|
+
result.chunksStored++;
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
result.errors++;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
sessionStates.set(sessionFile, { hash: currentHash, bytesSeen: stat.size, chunkIds: newChunkIds });
|
|
232
|
+
resetSessionDelta(sessionFile, stat.size);
|
|
233
|
+
result.sessionsProcessed++;
|
|
234
|
+
}
|
|
235
|
+
catch (err) {
|
|
236
|
+
log.error(`failed to index session ${sessionFile}`, { error: String(err) });
|
|
237
|
+
result.errors++;
|
|
238
|
+
}
|
|
239
|
+
return result;
|
|
240
|
+
}
|
|
241
|
+
async function indexAllSessions() {
|
|
242
|
+
if (needsFullReindex()) {
|
|
243
|
+
log.info("config changed, clearing session index state and stale chunks");
|
|
244
|
+
// Clear stale chunks from DB before re-indexing (prevents duplicates)
|
|
245
|
+
for (const [, state] of sessionStates) {
|
|
246
|
+
for (const id of state.chunkIds) {
|
|
247
|
+
try {
|
|
248
|
+
await memory.delete(id);
|
|
249
|
+
}
|
|
250
|
+
catch { /* best effort */ }
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
sessionStates.clear();
|
|
254
|
+
sessionDeltas.clear();
|
|
255
|
+
currentMeta = buildMeta();
|
|
256
|
+
}
|
|
257
|
+
const totals = { sessionsProcessed: 0, chunksStored: 0, chunksRemoved: 0, errors: 0 };
|
|
258
|
+
const files = findSessionFiles();
|
|
259
|
+
for (const file of files) {
|
|
260
|
+
try {
|
|
261
|
+
const r = await indexSession(file);
|
|
262
|
+
totals.sessionsProcessed += r.sessionsProcessed;
|
|
263
|
+
totals.chunksStored += r.chunksStored;
|
|
264
|
+
totals.chunksRemoved += r.chunksRemoved;
|
|
265
|
+
totals.errors += r.errors;
|
|
266
|
+
}
|
|
267
|
+
catch (err) {
|
|
268
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
269
|
+
if (/embedding|embeddings|batch/i.test(message)) {
|
|
270
|
+
log.warn(`provider error during session indexing, notifying caller: ${message}`);
|
|
271
|
+
onProviderError?.(err);
|
|
272
|
+
}
|
|
273
|
+
totals.errors++;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
log.info("session indexing complete", {
|
|
277
|
+
sessions: totals.sessionsProcessed,
|
|
278
|
+
stored: totals.chunksStored,
|
|
279
|
+
removed: totals.chunksRemoved,
|
|
280
|
+
errors: totals.errors,
|
|
281
|
+
});
|
|
282
|
+
return totals;
|
|
283
|
+
}
|
|
284
|
+
// --- Debounced update (from OpenClaw scheduleSessionDirty + processSessionDeltaBatch) ---
|
|
285
|
+
function onTranscriptUpdate(sessionFile) {
|
|
286
|
+
const existing = pendingUpdates.get(sessionFile);
|
|
287
|
+
if (existing)
|
|
288
|
+
clearTimeout(existing);
|
|
289
|
+
pendingUpdates.set(sessionFile, setTimeout(async () => {
|
|
290
|
+
pendingUpdates.delete(sessionFile);
|
|
291
|
+
const delta = await updateSessionDelta(sessionFile);
|
|
292
|
+
if (delta.shouldSync) {
|
|
293
|
+
sessionsDirtyFiles.add(sessionFile);
|
|
294
|
+
sessionsDirty = true;
|
|
295
|
+
// Reset consumed delta
|
|
296
|
+
if (deltaBytes > 0)
|
|
297
|
+
delta.state.pendingBytes = Math.max(0, delta.state.pendingBytes - deltaBytes);
|
|
298
|
+
if (deltaMessages > 0)
|
|
299
|
+
delta.state.pendingMessages = Math.max(0, delta.state.pendingMessages - deltaMessages);
|
|
300
|
+
indexSession(sessionFile).catch((err) => log.error("session index update failed", { error: String(err) }));
|
|
301
|
+
}
|
|
302
|
+
}, SESSION_DIRTY_DEBOUNCE_MS));
|
|
303
|
+
}
|
|
304
|
+
// --- Listener (from OpenClaw ensureSessionListener) ---
|
|
305
|
+
function startListener() {
|
|
306
|
+
return onSessionTranscriptUpdate((update) => {
|
|
307
|
+
const sessionFile = update.sessionFile;
|
|
308
|
+
if (!sessionFile)
|
|
309
|
+
return;
|
|
310
|
+
const resolvedFile = path.resolve(sessionFile);
|
|
311
|
+
const resolvedDir = path.resolve(sessionsDir);
|
|
312
|
+
if (!resolvedFile.startsWith(`${resolvedDir}${path.sep}`))
|
|
313
|
+
return;
|
|
314
|
+
onTranscriptUpdate(resolvedFile);
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
// --- Post-compaction (from OpenClaw compaction-hooks.ts) ---
|
|
318
|
+
async function runPostCompactionSideEffects(sessionFile) {
|
|
319
|
+
const trimmed = sessionFile.trim();
|
|
320
|
+
if (!trimmed)
|
|
321
|
+
return;
|
|
322
|
+
emitSessionTranscriptUpdate(trimmed);
|
|
323
|
+
if (postCompactionSyncMode === "off")
|
|
324
|
+
return;
|
|
325
|
+
const syncTask = indexSession(trimmed);
|
|
326
|
+
if (postCompactionSyncMode === "await") {
|
|
327
|
+
await syncTask;
|
|
328
|
+
}
|
|
329
|
+
else {
|
|
330
|
+
void syncTask;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
return {
|
|
334
|
+
indexSession,
|
|
335
|
+
indexAllSessions,
|
|
336
|
+
onTranscriptUpdate,
|
|
337
|
+
startListener,
|
|
338
|
+
runPostCompactionSideEffects,
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
//# sourceMappingURL=session-indexer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-indexer.js","sourceRoot":"","sources":["../../src/memory/session-indexer.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,qFAAqF;AAErF,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EACL,yBAAyB,EACzB,2BAA2B,GAC5B,MAAM,kCAAkC,CAAC;AAE1C,MAAM,GAAG,GAAG,qBAAqB,CAAC,wBAAwB,CAAC,CAAC;AAE5D,0BAA0B;AAC1B,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,8BAA8B,GAAG,EAAE,GAAG,IAAI,CAAC;AAoDjD,MAAM,UAAU,oBAAoB,CAAC,OAA8B;IACjE,MAAM,EACJ,WAAW,EACX,MAAM,EACN,UAAU,GAAG,IAAI,EACjB,aAAa,GAAG,EAAE,EAClB,sBAAsB,GAAG,OAAO,EAChC,iBAAiB,EACjB,QAAQ,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,EACvC,eAAe,GAChB,GAAG,OAAO,CAAC;IAEZ,IAAI,WAAW,GAA2B,IAAI,CAAC;IAE/C,SAAS,SAAS;QAChB,OAAO;YACL,KAAK,EAAE,iBAAiB,EAAE,KAAK,IAAI,UAAU;YAC7C,QAAQ,EAAE,iBAAiB,EAAE,EAAE,IAAI,MAAM;YACzC,WAAW,EAAE,QAAQ,CAAC,MAAM;YAC5B,YAAY,EAAE,QAAQ,CAAC,OAAO;YAC9B,OAAO,EAAE,CAAC,UAAU,CAAC;SACtB,CAAC;IACJ,CAAC;IAED,SAAS,gBAAgB;QACvB,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAC9B,MAAM,QAAQ,GAAG,SAAS,EAAE,CAAC;QAC7B,OAAO,CACL,WAAW,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK;YACpC,WAAW,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ;YAC1C,WAAW,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;YAChD,WAAW,CAAC,YAAY,KAAK,QAAQ,CAAC,YAAY,CACnD,CAAC;IACJ,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAwB,CAAC;IACtD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAwB,CAAC;IACtD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAyC,CAAC;IACxE,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE7C,0DAA0D;IAE1D,KAAK,UAAU,aAAa,CAAC,OAAe,EAAE,KAAa,EAAE,GAAW;QACtE,IAAI,GAAG,IAAI,KAAK;YAAE,OAAO,CAAC,CAAC;QAC3B,IAAI,MAA8B,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,CAAC;YACH,IAAI,MAAM,GAAG,KAAK,CAAC;YACnB,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAC5D,OAAO,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,GAAG,MAAM,CAAC,CAAC;gBACrD,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;gBACnE,IAAI,SAAS,IAAI,CAAC;oBAAE,MAAM;gBAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;oBACtC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE;wBAAE,KAAK,IAAI,CAAC,CAAC;gBACnC,CAAC;gBACD,MAAM,IAAI,SAAS,CAAC;YACtB,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,4DAA4D;IAE5D,KAAK,UAAU,kBAAkB,CAAC,WAAmB;QAInD,IAAI,IAAsB,CAAC;QAC3B,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5F,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;YAC7D,aAAa,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,MAAM,KAAK,CAAC,IAAI,IAAI,KAAK,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,UAAU,CAAC;YAC7F,MAAM,WAAW,GAAG,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,IAAI,aAAa,CAAC;YAC5G,OAAO,EAAE,UAAU,EAAE,QAAQ,IAAI,WAAW,EAAE,KAAK,EAAE,CAAC;QACxD,CAAC;QAED,IAAI,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC1B,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;YACtB,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC;YAC3B,MAAM,mBAAmB,GAAG,aAAa,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,KAAK,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC;YACtG,IAAI,mBAAmB,EAAE,CAAC;gBACxB,KAAK,CAAC,eAAe,IAAI,MAAM,aAAa,CAAC,WAAW,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;YACrE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,YAAY,IAAI,MAAM,CAAC;YAC7B,MAAM,mBAAmB,GAAG,aAAa,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,KAAK,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC;YACtG,IAAI,mBAAmB,EAAE,CAAC;gBACxB,KAAK,CAAC,eAAe,IAAI,MAAM,aAAa,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAClF,CAAC;YACD,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,UAAU,CAAC;QAC7F,MAAM,WAAW,GAAG,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,IAAI,aAAa,CAAC;QAC5G,OAAO,EAAE,UAAU,EAAE,QAAQ,IAAI,WAAW,EAAE,KAAK,EAAE,CAAC;IACxD,CAAC;IAED,SAAS,iBAAiB,CAAC,OAAe,EAAE,IAAY;QACtD,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;QACtB,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC;QACvB,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,iCAAiC;IAEjC,SAAS,gBAAgB;QACvB,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC9C,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YACrE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;oBACxB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;oBAClD,IAAI,CAAC;wBACH,MAAM,UAAU,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;wBAC1C,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;4BAClC,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC;gCAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;wBAC3E,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;gBACxB,CAAC;qBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC3D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,oCAAoC,CAAC,CAAC;QAChD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,6BAA6B;IAE7B,SAAS,eAAe,CAAC,QAAgB;QACvC,MAAM,QAAQ,GAA6C,EAAE,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC/C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBAAE,SAAS;gBAC3B,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;oBAC3D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;oBAC9C,IAAI,OAAO,GAAG,EAAE,CAAC;oBACjB,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;wBACvC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;oBAC3B,CAAC;yBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;wBACzC,OAAO,GAAI,MAAM,CAAC,OAA0C;6BACzD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;6BAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;6BAChC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAChB,CAAC;oBACD,IAAI,OAAO,CAAC,IAAI,EAAE;wBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBACvD,CAAC;gBAAC,MAAM,CAAC,CAAC,0BAA0B,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;QACnC,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,SAAS,mBAAmB,CAAC,QAAkD;QAC7E,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;YAC/D,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;YAC7F,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,KAAK,SAAS,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7B,CAAC;IAED,iCAAiC;IAEjC,KAAK,UAAU,YAAY,CAAC,WAAmB;QAC7C,MAAM,MAAM,GAAuB,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QAC1G,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC;gBAAE,OAAO,MAAM,CAAC;YAC/C,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YACtC,MAAM,WAAW,GAAG,QAAQ,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;YACpE,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAEhD,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC9C,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC1C,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,IAAI,QAAQ,EAAE,CAAC;gBACb,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBACnC,IAAI,CAAC;wBAAC,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;wBAAC,MAAM,CAAC,aAAa,EAAE,CAAC;oBAAC,CAAC;oBAAC,MAAM,CAAC;wBAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBAAC,CAAC;gBACrF,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;YAC9C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC1C,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,MAAM,OAAO,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;YACpF,MAAM,UAAU,GAAG,IAAI,CAAC;YACxB,MAAM,WAAW,GAAa,EAAE,CAAC;YAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;gBAC/C,IAAI,CAAC;oBACH,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;oBAC1G,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACrB,MAAM,CAAC,YAAY,EAAE,CAAC;gBACxB,CAAC;gBAAC,MAAM,CAAC;oBAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBAAC,CAAC;YAC9B,CAAC;YAED,aAAa,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;YACnG,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,2BAA2B,WAAW,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5E,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,UAAU,gBAAgB;QAC7B,IAAI,gBAAgB,EAAE,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;YAC1E,sEAAsE;YACtE,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;gBACtC,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;oBAChC,IAAI,CAAC;wBAAC,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;oBAAC,CAAC;oBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC9D,CAAC;YACH,CAAC;YACD,aAAa,CAAC,KAAK,EAAE,CAAC;YACtB,aAAa,CAAC,KAAK,EAAE,CAAC;YACtB,WAAW,GAAG,SAAS,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,MAAM,GAAuB,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QAC1G,MAAM,KAAK,GAAG,gBAAgB,EAAE,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;gBACnC,MAAM,CAAC,iBAAiB,IAAI,CAAC,CAAC,iBAAiB,CAAC;gBAChD,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC,YAAY,CAAC;gBACtC,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC,aAAa,CAAC;gBACxC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;YAC5B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,IAAI,6BAA6B,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBAChD,GAAG,CAAC,IAAI,CAAC,6DAA6D,OAAO,EAAE,CAAC,CAAC;oBACjF,eAAe,EAAE,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;gBACD,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACpC,QAAQ,EAAE,MAAM,CAAC,iBAAiB;YAClC,MAAM,EAAE,MAAM,CAAC,YAAY;YAC3B,OAAO,EAAE,MAAM,CAAC,aAAa;YAC7B,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,2FAA2F;IAE3F,SAAS,kBAAkB,CAAC,WAAmB;QAC7C,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACjD,IAAI,QAAQ;YAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;QAErC,cAAc,CAAC,GAAG,CAChB,WAAW,EACX,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,WAAW,CAAC,CAAC;YACpD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACrB,kBAAkB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;gBACpC,aAAa,GAAG,IAAI,CAAC;gBACrB,uBAAuB;gBACvB,IAAI,UAAU,GAAG,CAAC;oBAAE,KAAK,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC;gBAClG,IAAI,aAAa,GAAG,CAAC;oBAAE,KAAK,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,eAAe,GAAG,aAAa,CAAC,CAAC;gBAC9G,YAAY,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CACtC,GAAG,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CACjE,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,yBAAyB,CAAC,CAC9B,CAAC;IACJ,CAAC;IAED,yDAAyD;IAEzD,SAAS,aAAa;QACpB,OAAO,yBAAyB,CAAC,CAAC,MAAM,EAAE,EAAE;YAC1C,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;YACvC,IAAI,CAAC,WAAW;gBAAE,OAAO;YACzB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAC9C,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAAE,OAAO;YAClE,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,8DAA8D;IAE9D,KAAK,UAAU,4BAA4B,CAAC,WAAmB;QAC7D,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAErC,IAAI,sBAAsB,KAAK,KAAK;YAAE,OAAO;QAE7C,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,sBAAsB,KAAK,OAAO,EAAE,CAAC;YACvC,MAAM,QAAQ,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,KAAK,QAAQ,CAAC;QAChB,CAAC;IACH,CAAC;IAED,OAAO;QACL,YAAY;QACZ,gBAAgB;QAChB,kBAAkB;QAClB,aAAa;QACb,4BAA4B;KAC7B,CAAC;AACJ,CAAC"}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import type { DatabaseSync } from "node:sqlite";
|
|
1
2
|
import type { EmbeddingProvider } from "./embeddings.js";
|
|
2
3
|
import type { MemorySearchResult } from "./types.js";
|
|
4
|
+
import { type EmbeddingCache } from "./embedding-cache.js";
|
|
3
5
|
export interface SimpleMemoryManager {
|
|
4
6
|
store(content: string, metadata?: Record<string, unknown>): Promise<string>;
|
|
5
7
|
search(query: string, options?: {
|
|
@@ -14,10 +16,14 @@ export interface SimpleMemoryManager {
|
|
|
14
16
|
createdAt: string;
|
|
15
17
|
}>>;
|
|
16
18
|
close(): Promise<void>;
|
|
19
|
+
readonly db: DatabaseSync;
|
|
20
|
+
readonly embeddingCache: EmbeddingCache | null;
|
|
17
21
|
}
|
|
18
22
|
export interface SimpleMemoryManagerOptions {
|
|
19
23
|
dbDir: string;
|
|
20
24
|
embeddingProvider?: EmbeddingProvider;
|
|
25
|
+
enableEmbeddingCache?: boolean;
|
|
26
|
+
maxCacheEntries?: number;
|
|
21
27
|
}
|
|
22
28
|
export declare function createSimpleMemoryManager(options: SimpleMemoryManagerOptions): SimpleMemoryManager;
|
|
23
29
|
//# sourceMappingURL=simple-manager.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"simple-manager.d.ts","sourceRoot":"","sources":["../../src/memory/simple-manager.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"simple-manager.d.ts","sourceRoot":"","sources":["../../src/memory/simple-manager.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAKhD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAErD,OAAO,EAAqC,KAAK,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAK9F,MAAM,WAAW,mBAAmB;IAClC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5E,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC3G,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;IAC/G,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,QAAQ,CAAC,EAAE,EAAE,YAAY,CAAC;IAC1B,QAAQ,CAAC,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;CAChD;AAED,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAMD,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,0BAA0B,GAAG,mBAAmB,CA+NlG"}
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
// Simple memory manager that wraps OpenClaw's search/indexing components
|
|
2
|
-
// with a user-friendly store/search/delete/list API.
|
|
3
1
|
import * as fs from "node:fs";
|
|
4
2
|
import * as path from "node:path";
|
|
3
|
+
import * as crypto from "node:crypto";
|
|
5
4
|
import { requireNodeSqlite } from "./sqlite.js";
|
|
6
5
|
import { cosineSimilarity, parseEmbedding, hashText } from "./internal.js";
|
|
7
6
|
import { extractKeywords } from "./query-expansion.js";
|
|
8
|
-
import { buildFtsQuery, bm25RankToScore } from "./hybrid.js";
|
|
9
|
-
import { mergeHybridResults } from "./hybrid.js";
|
|
7
|
+
import { buildFtsQuery, bm25RankToScore, mergeHybridResults } from "./hybrid.js";
|
|
10
8
|
import { debugEmbeddingsLog } from "./embeddings-debug.js";
|
|
9
|
+
import { createEmbeddingCache, hashContent } from "./embedding-cache.js";
|
|
11
10
|
const SNIPPET_MAX_CHARS = 700;
|
|
11
|
+
const MAX_VECTOR_SEARCH_CHUNKS = 10_000;
|
|
12
12
|
function generateId() {
|
|
13
|
-
return `mem_${Date.now()}_${
|
|
13
|
+
return `mem_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
|
|
14
14
|
}
|
|
15
15
|
export function createSimpleMemoryManager(options) {
|
|
16
16
|
fs.mkdirSync(options.dbDir, { recursive: true });
|
|
@@ -36,11 +36,15 @@ export function createSimpleMemoryManager(options) {
|
|
|
36
36
|
)
|
|
37
37
|
`);
|
|
38
38
|
let ftsAvailable = false;
|
|
39
|
+
let stmtFtsInsert = null;
|
|
40
|
+
let stmtFtsDelete = null;
|
|
39
41
|
try {
|
|
40
42
|
db.exec(`
|
|
41
43
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts
|
|
42
44
|
USING fts5(id, content, source, model)
|
|
43
45
|
`);
|
|
46
|
+
stmtFtsInsert = db.prepare("INSERT INTO chunks_fts(id, content, source, model) VALUES (?, ?, ?, ?)");
|
|
47
|
+
stmtFtsDelete = db.prepare("DELETE FROM chunks_fts WHERE id = ?");
|
|
44
48
|
ftsAvailable = true;
|
|
45
49
|
}
|
|
46
50
|
catch {
|
|
@@ -48,6 +52,10 @@ export function createSimpleMemoryManager(options) {
|
|
|
48
52
|
}
|
|
49
53
|
const provider = options.embeddingProvider ?? null;
|
|
50
54
|
const providerModel = provider ? `${provider.id}/${provider.model}` : "none";
|
|
55
|
+
const embeddingCache = (options.enableEmbeddingCache !== false && provider)
|
|
56
|
+
? createEmbeddingCache(db)
|
|
57
|
+
: null;
|
|
58
|
+
const maxCacheEntries = options.maxCacheEntries ?? 50_000;
|
|
51
59
|
const stmtInsert = db.prepare(`
|
|
52
60
|
INSERT OR REPLACE INTO chunks (id, content, embedding, metadata, source, path, model, start_line, end_line, hash, created_at)
|
|
53
61
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
@@ -63,18 +71,28 @@ export function createSimpleMemoryManager(options) {
|
|
|
63
71
|
let embeddingJson = null;
|
|
64
72
|
if (provider) {
|
|
65
73
|
try {
|
|
66
|
-
const
|
|
74
|
+
const contentHash = hashContent(content);
|
|
75
|
+
const cached = embeddingCache?.get(provider.id, provider.model, contentHash);
|
|
76
|
+
let vec;
|
|
77
|
+
if (cached) {
|
|
78
|
+
vec = cached;
|
|
79
|
+
debugEmbeddingsLog("embedding from cache", { id, dims: vec.length });
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
vec = await provider.embedQuery(content);
|
|
83
|
+
embeddingCache?.set(provider.id, provider.model, "", contentHash, vec, vec.length);
|
|
84
|
+
debugEmbeddingsLog("embedded chunk", { id, dims: vec.length });
|
|
85
|
+
}
|
|
67
86
|
embeddingJson = JSON.stringify(vec);
|
|
68
|
-
debugEmbeddingsLog("embedded chunk", { id, dims: vec.length });
|
|
69
87
|
}
|
|
70
88
|
catch (err) {
|
|
71
89
|
debugEmbeddingsLog("embedding failed", { id, error: String(err) });
|
|
72
90
|
}
|
|
73
91
|
}
|
|
74
92
|
stmtInsert.run(id, content, embeddingJson, metadata ? JSON.stringify(metadata) : null, "memory", null, providerModel, 1, 1, hash, now);
|
|
75
|
-
if (
|
|
93
|
+
if (stmtFtsInsert) {
|
|
76
94
|
try {
|
|
77
|
-
|
|
95
|
+
stmtFtsInsert.run(id, content, "memory", providerModel);
|
|
78
96
|
}
|
|
79
97
|
catch {
|
|
80
98
|
// FTS insert failed, keyword search won't find this chunk
|
|
@@ -91,7 +109,7 @@ export function createSimpleMemoryManager(options) {
|
|
|
91
109
|
if (provider) {
|
|
92
110
|
try {
|
|
93
111
|
const queryVec = await provider.embedQuery(query);
|
|
94
|
-
const rows = db.prepare("SELECT * FROM chunks WHERE embedding IS NOT NULL AND model = ?").all(providerModel);
|
|
112
|
+
const rows = db.prepare("SELECT * FROM chunks WHERE embedding IS NOT NULL AND model = ? LIMIT ?").all(providerModel, MAX_VECTOR_SEARCH_CHUNKS);
|
|
95
113
|
const scored = [];
|
|
96
114
|
for (const row of rows) {
|
|
97
115
|
const embedding = parseEmbedding(row.embedding);
|
|
@@ -168,9 +186,9 @@ export function createSimpleMemoryManager(options) {
|
|
|
168
186
|
if (!row)
|
|
169
187
|
return false;
|
|
170
188
|
stmtDelete.run(id);
|
|
171
|
-
if (
|
|
189
|
+
if (stmtFtsDelete) {
|
|
172
190
|
try {
|
|
173
|
-
|
|
191
|
+
stmtFtsDelete.run(id);
|
|
174
192
|
}
|
|
175
193
|
catch { }
|
|
176
194
|
}
|
|
@@ -186,8 +204,13 @@ export function createSimpleMemoryManager(options) {
|
|
|
186
204
|
}));
|
|
187
205
|
},
|
|
188
206
|
async close() {
|
|
207
|
+
if (embeddingCache) {
|
|
208
|
+
embeddingCache.prune(maxCacheEntries);
|
|
209
|
+
}
|
|
189
210
|
db.close();
|
|
190
211
|
},
|
|
212
|
+
get db() { return db; },
|
|
213
|
+
get embeddingCache() { return embeddingCache; },
|
|
191
214
|
};
|
|
192
215
|
}
|
|
193
216
|
//# sourceMappingURL=simple-manager.js.map
|