@theokit/sdk-memory 0.3.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/dist/index.cjs +165 -344
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +52 -326
- package/dist/index.js.map +1 -1
- package/dist/internal/active-memory/active-memory.d.ts.map +1 -1
- package/dist/internal/dreaming/dreaming-diary.d.ts +6 -61
- package/dist/internal/dreaming/dreaming-diary.d.ts.map +1 -1
- package/dist/internal/index/index-db.d.ts +0 -7
- package/dist/internal/index/index-db.d.ts.map +1 -1
- package/dist/internal/index/index-manager.d.ts +9 -1
- package/dist/internal/index/index-manager.d.ts.map +1 -1
- package/dist/internal/index/lance-index.d.ts +3 -6
- package/dist/internal/index/lance-index.d.ts.map +1 -1
- package/dist/internal/index/migrate-sqlite-to-lance.d.ts +5 -0
- package/dist/internal/index/migrate-sqlite-to-lance.d.ts.map +1 -1
- package/dist/internal/store/markdown-store.d.ts +24 -32
- package/dist/internal/store/markdown-store.d.ts.map +1 -1
- package/dist/internal/store/session-loader.d.ts +3 -27
- package/dist/internal/store/session-loader.d.ts.map +1 -1
- package/dist/internal/store/session-summary-writer.d.ts +6 -53
- package/dist/internal/store/session-summary-writer.d.ts.map +1 -1
- package/dist/internal/store/transcript-store.d.ts +3 -45
- package/dist/internal/store/transcript-store.d.ts.map +1 -1
- package/dist/internal/store/wiki-loader.d.ts +3 -32
- package/dist/internal/store/wiki-loader.d.ts.map +1 -1
- package/dist/internal/tools.d.ts +3 -1
- package/dist/internal/tools.d.ts.map +1 -1
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
|
-
import { mkdir,
|
|
2
|
-
import { join, resolve
|
|
1
|
+
import { mkdir, readdir, readFile, stat, access, unlink } from 'fs/promises';
|
|
2
|
+
import { join, resolve } from 'path';
|
|
3
3
|
import { Security } from '@theokit/sdk';
|
|
4
|
-
import { replaceFileAtomic, withCwdMutex,
|
|
5
|
-
import {
|
|
4
|
+
import { replaceFileAtomic, withCwdMutex, openSqliteResilient, sanitizeFts5Query } from '@theokit/sdk/persistence';
|
|
5
|
+
import { notesDir, resolveMemoryRoot, persistActiveMemoryTranscript, readFactsFromMarkdown, appendDiaryEntry, lanceStoragePath, defaultIndexPath, projectMemoryDir, collectMarkdownFiles, readAllSqliteFacts, memoryMdPath, appendFactToMarkdown } from '@theokit/sdk/internal/memory-store';
|
|
6
|
+
export { MEMORY_INDEX_MAX_BYTES, MEMORY_INDEX_MAX_LINES, appendDiaryEntry, appendFact, appendFactToMarkdown, asMemoryRoot, claudeProjectMemoryDir, collectMarkdownFiles, defaultIndexPath, diaryPath, discoverSessionFiles, discoverWikiFiles, entryHash, indexBudgetWarning, lanceStoragePath, memoryMdPath, memoryReadRoots, notesDir, persistActiveMemoryTranscript, projectMemoryDir, readAllSqliteFacts, readFacts, readFactsFromMarkdown, renderDiaryEntry, resolveMemoryRoot, sessionSummaryPath, sessionsDir, wikiDir, writeSessionSummary } from '@theokit/sdk/internal/memory-store';
|
|
6
7
|
import { createHash } from 'crypto';
|
|
7
8
|
import { createOpenAiCompatibleRuntime } from '@theokit/sdk/internal/memory-adapters';
|
|
8
9
|
export { createOpenAiCompatibleRuntime } from '@theokit/sdk/internal/memory-adapters';
|
|
9
10
|
import { AuthenticationError, RateLimitError, ConfigurationError, NetworkError, UnknownAgentError } from '@theokit/sdk/errors';
|
|
11
|
+
import { sanitizeIdentifier, safePathJoin } from '@theokit/sdk/path-safety';
|
|
10
12
|
import { mkdirSync, existsSync, rmSync, renameSync } from 'fs';
|
|
11
13
|
import { createRequire } from 'module';
|
|
12
14
|
|
|
@@ -238,121 +240,17 @@ function createInMemoryMarkdownProvider() {
|
|
|
238
240
|
}
|
|
239
241
|
};
|
|
240
242
|
}
|
|
241
|
-
function redactSecrets(text, opts) {
|
|
242
|
-
return Security.redact(text, opts);
|
|
243
|
-
}
|
|
244
|
-
function legacyMemoryJsonPath(cwd, config) {
|
|
245
|
-
if (config.storePath !== void 0) {
|
|
246
|
-
return resolve(cwd, config.storePath);
|
|
247
|
-
}
|
|
248
|
-
const namespace = sanitizeIdentifier(config.namespace ?? "default");
|
|
249
|
-
const scope = sanitizeIdentifier(config.scope ?? "agent", { maxLen: 16 });
|
|
250
|
-
const userId = sanitizeIdentifier(config.userId ?? "default");
|
|
251
|
-
return safePathJoin(cwd, ".theokit", "memory", namespace, `${scope}-${userId}.json`);
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
// src/internal/store/markdown-store.ts
|
|
255
|
-
var MEMORY_MD_HEADER = "# Memory\n\n> Auto-managed by @theokit/sdk. Edit freely \u2014 the SDK reads from here.\n";
|
|
256
|
-
var FACTS_HEADING = "## Facts";
|
|
257
|
-
function memoryDir(cwd) {
|
|
258
|
-
return join(cwd, ".theokit", "memory");
|
|
259
|
-
}
|
|
260
|
-
function memoryMdPath(cwd) {
|
|
261
|
-
return join(memoryDir(cwd), "MEMORY.md");
|
|
262
|
-
}
|
|
263
|
-
function notesDir(cwd) {
|
|
264
|
-
return join(memoryDir(cwd), "notes");
|
|
265
|
-
}
|
|
266
|
-
async function readFactsFromMarkdown(cwd) {
|
|
267
|
-
let raw;
|
|
268
|
-
try {
|
|
269
|
-
raw = await readFile(memoryMdPath(cwd), "utf8");
|
|
270
|
-
} catch {
|
|
271
|
-
return [];
|
|
272
|
-
}
|
|
273
|
-
return parseFactsSection(raw);
|
|
274
|
-
}
|
|
275
|
-
function appendFactToMarkdown(cwd, fact) {
|
|
276
|
-
return withCwdMutex(memoryDir(cwd), async () => {
|
|
277
|
-
const path = memoryMdPath(cwd);
|
|
278
|
-
let raw = "";
|
|
279
|
-
try {
|
|
280
|
-
raw = await readFile(path, "utf8");
|
|
281
|
-
} catch {
|
|
282
|
-
raw = "";
|
|
283
|
-
}
|
|
284
|
-
const sanitized = redactSecrets(fact.text);
|
|
285
|
-
const next = insertFactBullet(raw, sanitized);
|
|
286
|
-
await mkdir(memoryDir(cwd), { recursive: true });
|
|
287
|
-
await replaceFileAtomic(path, next);
|
|
288
|
-
});
|
|
289
|
-
}
|
|
290
243
|
async function listNotes(cwd) {
|
|
291
244
|
let entries = [];
|
|
292
245
|
try {
|
|
293
|
-
entries = await readdir(notesDir(cwd));
|
|
246
|
+
entries = await readdir(notesDir(resolveMemoryRoot(cwd)));
|
|
294
247
|
} catch {
|
|
295
248
|
return [];
|
|
296
249
|
}
|
|
297
|
-
return entries.filter((name) => name.endsWith(".md")).map((name) => ({
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
if (idx === -1) return [];
|
|
302
|
-
const tail = raw.slice(idx + FACTS_HEADING.length);
|
|
303
|
-
const nextHeading = tail.search(/\n#{1,2}\s/);
|
|
304
|
-
const block = nextHeading === -1 ? tail : tail.slice(0, nextHeading);
|
|
305
|
-
return block.split("\n").map((line) => line.trim()).filter((line) => line.startsWith("- ")).map((line) => ({ text: line.slice(2).trim() }));
|
|
306
|
-
}
|
|
307
|
-
function insertFactBullet(raw, fact) {
|
|
308
|
-
const bullet = `- ${fact}`;
|
|
309
|
-
if (raw.length === 0) {
|
|
310
|
-
return `${MEMORY_MD_HEADER}
|
|
311
|
-
${FACTS_HEADING}
|
|
312
|
-
|
|
313
|
-
${bullet}
|
|
314
|
-
`;
|
|
315
|
-
}
|
|
316
|
-
const idx = raw.indexOf(FACTS_HEADING);
|
|
317
|
-
if (idx === -1) {
|
|
318
|
-
const sep = raw.endsWith("\n") ? "" : "\n";
|
|
319
|
-
return `${raw}${sep}
|
|
320
|
-
${FACTS_HEADING}
|
|
321
|
-
|
|
322
|
-
${bullet}
|
|
323
|
-
`;
|
|
324
|
-
}
|
|
325
|
-
const after = idx + FACTS_HEADING.length;
|
|
326
|
-
const nextHeading = raw.slice(after).search(/\n#{1,2}\s/);
|
|
327
|
-
if (nextHeading === -1) {
|
|
328
|
-
const trailing = raw.endsWith("\n") ? "" : "\n";
|
|
329
|
-
return `${raw}${trailing}${bullet}
|
|
330
|
-
`;
|
|
331
|
-
}
|
|
332
|
-
const insertAt = after + nextHeading;
|
|
333
|
-
return `${raw.slice(0, insertAt)}
|
|
334
|
-
${bullet}${raw.slice(insertAt)}`;
|
|
335
|
-
}
|
|
336
|
-
async function readFacts(cwd, config) {
|
|
337
|
-
if (!config.enabled) return [];
|
|
338
|
-
return readFactsFromMarkdown(cwd);
|
|
339
|
-
}
|
|
340
|
-
async function appendFact(cwd, config, fact) {
|
|
341
|
-
if (!config.enabled) return;
|
|
342
|
-
await appendFactToMarkdown(cwd, fact);
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
// src/internal/store/transcript-store.ts
|
|
346
|
-
async function persistActiveMemoryTranscript(cwd, transcript) {
|
|
347
|
-
try {
|
|
348
|
-
const dir = join(memoryDir(cwd), "transcripts", "active-memory");
|
|
349
|
-
const file = join(dir, `${transcript.runId}.json`);
|
|
350
|
-
await atomicWriteJson(file, transcript);
|
|
351
|
-
} catch (cause) {
|
|
352
|
-
const message = cause instanceof Error ? cause.message : String(cause);
|
|
353
|
-
process.stderr.write(`[theokit-sdk] active-memory transcript persist failed: ${message}
|
|
354
|
-
`);
|
|
355
|
-
}
|
|
250
|
+
return entries.filter((name) => name.endsWith(".md")).map((name) => ({
|
|
251
|
+
slug: name.replace(/\.md$/, ""),
|
|
252
|
+
path: join(notesDir(resolveMemoryRoot(cwd)), name)
|
|
253
|
+
}));
|
|
356
254
|
}
|
|
357
255
|
|
|
358
256
|
// src/internal/active-memory/active-memory.ts
|
|
@@ -495,7 +393,7 @@ async function finalize(args, queryMode, result) {
|
|
|
495
393
|
};
|
|
496
394
|
args.cache?.set(args.userText, queryMode, result, tenantCtx);
|
|
497
395
|
if (args.persistTranscripts === true && args.cwd !== void 0) {
|
|
498
|
-
await persistActiveMemoryTranscript(args.cwd, {
|
|
396
|
+
await persistActiveMemoryTranscript(resolveMemoryRoot(args.cwd), {
|
|
499
397
|
runId: args.runId ?? `run-${Date.now()}`,
|
|
500
398
|
startedAtMs: Date.now() - result.durationMs,
|
|
501
399
|
userText: args.userText,
|
|
@@ -949,6 +847,20 @@ var MEMORY_EMBEDDING_ADAPTERS = {
|
|
|
949
847
|
jina: jinaMemoryEmbeddingProviderAdapter,
|
|
950
848
|
gemini: geminiMemoryEmbeddingProviderAdapter
|
|
951
849
|
};
|
|
850
|
+
function redactSecrets(text, opts) {
|
|
851
|
+
return Security.redact(text, opts);
|
|
852
|
+
}
|
|
853
|
+
function legacyMemoryJsonPath(cwd, config) {
|
|
854
|
+
if (config.storePath !== void 0) {
|
|
855
|
+
return resolve(cwd, config.storePath);
|
|
856
|
+
}
|
|
857
|
+
const namespace = sanitizeIdentifier(config.namespace ?? "default");
|
|
858
|
+
const scope = sanitizeIdentifier(config.scope ?? "agent", { maxLen: 16 });
|
|
859
|
+
const userId = sanitizeIdentifier(config.userId ?? "default");
|
|
860
|
+
return safePathJoin(cwd, ".theokit", "memory", namespace, `${scope}-${userId}.json`);
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// src/internal/adapter-http-error.ts
|
|
952
864
|
var RAW_MAX_BYTES = 2048;
|
|
953
865
|
function parseRetryAfter(headers) {
|
|
954
866
|
if (headers === void 0) return void 0;
|
|
@@ -1037,7 +949,7 @@ function mapOpenAiStatusToCode(status, body) {
|
|
|
1037
949
|
function formatMessage(providerId, status, code) {
|
|
1038
950
|
return `${providerId} API error: ${code} (HTTP ${status})`;
|
|
1039
951
|
}
|
|
1040
|
-
var
|
|
952
|
+
var FACTS_HEADING = "## Facts";
|
|
1041
953
|
function createCategorizedMemory(options) {
|
|
1042
954
|
const { root, categories } = options;
|
|
1043
955
|
validateCategories(categories);
|
|
@@ -1111,7 +1023,7 @@ function header(category) {
|
|
|
1111
1023
|
category: ${category}
|
|
1112
1024
|
---
|
|
1113
1025
|
|
|
1114
|
-
${
|
|
1026
|
+
${FACTS_HEADING}
|
|
1115
1027
|
`;
|
|
1116
1028
|
}
|
|
1117
1029
|
async function readFileOrEmpty(path) {
|
|
@@ -1135,24 +1047,24 @@ function decodeFact(text) {
|
|
|
1135
1047
|
);
|
|
1136
1048
|
}
|
|
1137
1049
|
function parseFactBullets(raw) {
|
|
1138
|
-
const idx = raw.indexOf(
|
|
1050
|
+
const idx = raw.indexOf(FACTS_HEADING);
|
|
1139
1051
|
if (idx === -1) return [];
|
|
1140
|
-
const tail = raw.slice(idx +
|
|
1052
|
+
const tail = raw.slice(idx + FACTS_HEADING.length);
|
|
1141
1053
|
const nextHeading = tail.search(/\n#{1,2}\s/);
|
|
1142
1054
|
const block = nextHeading === -1 ? tail : tail.slice(0, nextHeading);
|
|
1143
1055
|
return block.split("\n").map((line) => line.trim()).filter((line) => line.startsWith("- ")).map((line) => line.slice(2).trim());
|
|
1144
1056
|
}
|
|
1145
1057
|
function appendBullet(raw, text) {
|
|
1146
1058
|
const bullet = `- ${text}`;
|
|
1147
|
-
const idx = raw.indexOf(
|
|
1059
|
+
const idx = raw.indexOf(FACTS_HEADING);
|
|
1148
1060
|
if (idx === -1) {
|
|
1149
1061
|
const sep = raw.endsWith("\n") ? "" : "\n";
|
|
1150
|
-
return `${raw}${sep}${
|
|
1062
|
+
return `${raw}${sep}${FACTS_HEADING}
|
|
1151
1063
|
|
|
1152
1064
|
${bullet}
|
|
1153
1065
|
`;
|
|
1154
1066
|
}
|
|
1155
|
-
const after = idx +
|
|
1067
|
+
const after = idx + FACTS_HEADING.length;
|
|
1156
1068
|
const nextHeading = raw.slice(after).search(/\n#{1,2}\s/);
|
|
1157
1069
|
if (nextHeading === -1) {
|
|
1158
1070
|
const trailing = raw.endsWith("\n") ? "" : "\n";
|
|
@@ -1163,47 +1075,6 @@ ${bullet}
|
|
|
1163
1075
|
return `${raw.slice(0, insertAt)}
|
|
1164
1076
|
${bullet}${raw.slice(insertAt)}`;
|
|
1165
1077
|
}
|
|
1166
|
-
function diaryPath(cwd) {
|
|
1167
|
-
return join(memoryDir(cwd), "dream-diary.md");
|
|
1168
|
-
}
|
|
1169
|
-
function renderDiaryEntry(entry) {
|
|
1170
|
-
const stamp = new Date(entry.timestampMs).toISOString();
|
|
1171
|
-
const hash = entryHash(entry).slice(0, 8);
|
|
1172
|
-
return [
|
|
1173
|
-
`## ${stamp}`,
|
|
1174
|
-
"",
|
|
1175
|
-
`- entry-hash: ${hash}`,
|
|
1176
|
-
`- facts before: ${entry.factsBefore}`,
|
|
1177
|
-
`- facts after: ${entry.factsAfter}`,
|
|
1178
|
-
`- duplicates removed: ${entry.duplicatesRemoved}`,
|
|
1179
|
-
`- clusters created: ${entry.clustersCreated}`,
|
|
1180
|
-
`- notes written: ${entry.notesWritten}`,
|
|
1181
|
-
""
|
|
1182
|
-
].join("\n");
|
|
1183
|
-
}
|
|
1184
|
-
async function appendDiaryEntry(cwd, entry) {
|
|
1185
|
-
const path = diaryPath(cwd);
|
|
1186
|
-
let raw = "";
|
|
1187
|
-
try {
|
|
1188
|
-
raw = await readFile(path, "utf8");
|
|
1189
|
-
} catch {
|
|
1190
|
-
raw = "# Dream Diary\n\n";
|
|
1191
|
-
}
|
|
1192
|
-
const next = `${raw.endsWith("\n") ? raw : `${raw}
|
|
1193
|
-
`}${renderDiaryEntry(entry)}`;
|
|
1194
|
-
await replaceFileAtomic(path, next);
|
|
1195
|
-
}
|
|
1196
|
-
function entryHash(entry) {
|
|
1197
|
-
return createHash("sha256").update(
|
|
1198
|
-
[
|
|
1199
|
-
entry.factsBefore,
|
|
1200
|
-
entry.factsAfter,
|
|
1201
|
-
entry.duplicatesRemoved,
|
|
1202
|
-
entry.clustersCreated,
|
|
1203
|
-
entry.notesWritten
|
|
1204
|
-
].join("|")
|
|
1205
|
-
).digest("hex");
|
|
1206
|
-
}
|
|
1207
1078
|
|
|
1208
1079
|
// src/internal/dreaming/dreaming-phases.ts
|
|
1209
1080
|
var DEFAULT_DEDUP_THRESHOLD = 0.95;
|
|
@@ -1325,7 +1196,7 @@ async function runInner(options) {
|
|
|
1325
1196
|
notesWritten,
|
|
1326
1197
|
diaryEntryHash: void 0
|
|
1327
1198
|
};
|
|
1328
|
-
await appendDiaryEntry(options.cwd, {
|
|
1199
|
+
await appendDiaryEntry(resolveMemoryRoot(options.cwd), {
|
|
1329
1200
|
timestampMs,
|
|
1330
1201
|
factsBefore: result.factsBefore,
|
|
1331
1202
|
factsAfter: result.factsAfter,
|
|
@@ -1343,10 +1214,10 @@ async function runInner(options) {
|
|
|
1343
1214
|
}
|
|
1344
1215
|
async function writeConsolidatedNotes(cwd, clusters, timestampMs) {
|
|
1345
1216
|
if (clusters.length === 0) return 0;
|
|
1346
|
-
const
|
|
1347
|
-
await mkdir(
|
|
1217
|
+
const notesDir3 = join(resolveMemoryRoot(cwd), "notes");
|
|
1218
|
+
await mkdir(notesDir3, { recursive: true });
|
|
1348
1219
|
const isoSlug = new Date(timestampMs).toISOString().replace(/[^\dT]/g, "-");
|
|
1349
|
-
const file = join(
|
|
1220
|
+
const file = join(notesDir3, `dreamed-${isoSlug}.md`);
|
|
1350
1221
|
const body = deepPhase(clusters, timestampMs);
|
|
1351
1222
|
await replaceFileAtomic(file, body);
|
|
1352
1223
|
return 1;
|
|
@@ -1416,9 +1287,6 @@ async function openMemoryDb(opts) {
|
|
|
1416
1287
|
}
|
|
1417
1288
|
});
|
|
1418
1289
|
}
|
|
1419
|
-
function defaultIndexPath(cwd) {
|
|
1420
|
-
return join(cwd, ".theokit", "memory", ".index", "memory.sqlite");
|
|
1421
|
-
}
|
|
1422
1290
|
var HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
|
|
1423
1291
|
function chunkMarkdown(text, options = {}) {
|
|
1424
1292
|
const maxChars = options.maxChars ?? 800;
|
|
@@ -1494,91 +1362,6 @@ function findWordBoundarySplit(text, maxChars) {
|
|
|
1494
1362
|
}
|
|
1495
1363
|
return maxChars;
|
|
1496
1364
|
}
|
|
1497
|
-
var MAX_TURN_CHARS = 2e3;
|
|
1498
|
-
function sessionsDir(cwd) {
|
|
1499
|
-
return join(memoryDir(cwd), "sessions");
|
|
1500
|
-
}
|
|
1501
|
-
function sessionSummaryPath(cwd, runId) {
|
|
1502
|
-
return join(sessionsDir(cwd), `${sanitizeRunId2(runId)}.md`);
|
|
1503
|
-
}
|
|
1504
|
-
function sanitizeRunId2(runId) {
|
|
1505
|
-
return runId.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 128);
|
|
1506
|
-
}
|
|
1507
|
-
function truncate2(text) {
|
|
1508
|
-
if (text.length <= MAX_TURN_CHARS) return text;
|
|
1509
|
-
return `${text.slice(0, MAX_TURN_CHARS)}\u2026`;
|
|
1510
|
-
}
|
|
1511
|
-
async function writeSessionSummary(input) {
|
|
1512
|
-
if (input.status !== "finished") return;
|
|
1513
|
-
const path = sessionSummaryPath(input.cwd, input.runId);
|
|
1514
|
-
await mkdir(sessionsDir(input.cwd), { recursive: true });
|
|
1515
|
-
const safeUser = redactSecrets(truncate2(input.userText));
|
|
1516
|
-
const safeAssistant = redactSecrets(truncate2(input.assistantText));
|
|
1517
|
-
const iso = new Date(input.at).toISOString();
|
|
1518
|
-
const body = [
|
|
1519
|
-
"---",
|
|
1520
|
-
`runId: ${input.runId}`,
|
|
1521
|
-
`agentId: ${input.agentId}`,
|
|
1522
|
-
`at: ${iso}`,
|
|
1523
|
-
`status: ${input.status}`,
|
|
1524
|
-
"---",
|
|
1525
|
-
"",
|
|
1526
|
-
"## User",
|
|
1527
|
-
"",
|
|
1528
|
-
safeUser,
|
|
1529
|
-
"",
|
|
1530
|
-
"## Assistant",
|
|
1531
|
-
"",
|
|
1532
|
-
safeAssistant,
|
|
1533
|
-
""
|
|
1534
|
-
].join("\n");
|
|
1535
|
-
await replaceFileAtomic(path, body);
|
|
1536
|
-
}
|
|
1537
|
-
|
|
1538
|
-
// src/internal/store/session-loader.ts
|
|
1539
|
-
async function discoverSessionFiles(cwd) {
|
|
1540
|
-
let entries;
|
|
1541
|
-
try {
|
|
1542
|
-
entries = await readdir(sessionsDir(cwd));
|
|
1543
|
-
} catch {
|
|
1544
|
-
return [];
|
|
1545
|
-
}
|
|
1546
|
-
const root = memoryDir(cwd);
|
|
1547
|
-
return entries.filter((entry) => entry.endsWith(".md")).map((entry) => {
|
|
1548
|
-
const absolutePath = join(sessionsDir(cwd), entry);
|
|
1549
|
-
return {
|
|
1550
|
-
absolutePath,
|
|
1551
|
-
relPath: relativeToRoot(root, absolutePath)
|
|
1552
|
-
};
|
|
1553
|
-
});
|
|
1554
|
-
}
|
|
1555
|
-
function relativeToRoot(root, absolutePath) {
|
|
1556
|
-
if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
|
|
1557
|
-
return absolutePath;
|
|
1558
|
-
}
|
|
1559
|
-
function wikiDir(cwd) {
|
|
1560
|
-
return join(memoryDir(cwd), "wiki");
|
|
1561
|
-
}
|
|
1562
|
-
async function discoverWikiFiles(cwd) {
|
|
1563
|
-
let entries;
|
|
1564
|
-
try {
|
|
1565
|
-
entries = await readdir(wikiDir(cwd));
|
|
1566
|
-
} catch {
|
|
1567
|
-
return [];
|
|
1568
|
-
}
|
|
1569
|
-
const root = memoryDir(cwd);
|
|
1570
|
-
return entries.filter((entry) => entry.endsWith(".md")).map((entry) => ({
|
|
1571
|
-
absolutePath: join(wikiDir(cwd), entry),
|
|
1572
|
-
relPath: join("wiki", entry)
|
|
1573
|
-
})).map((file) => ({
|
|
1574
|
-
absolutePath: file.absolutePath,
|
|
1575
|
-
relPath: relativeToRoot2(root, file.absolutePath)
|
|
1576
|
-
}));
|
|
1577
|
-
}
|
|
1578
|
-
function relativeToRoot2(root, absolutePath) {
|
|
1579
|
-
if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
|
|
1580
|
-
return absolutePath;
|
|
1581
|
-
}
|
|
1582
1365
|
function requireLance() {
|
|
1583
1366
|
try {
|
|
1584
1367
|
const r = createRequire(import.meta.url);
|
|
@@ -1601,7 +1384,7 @@ var LanceIndex = class _LanceIndex {
|
|
|
1601
1384
|
embeddingDim;
|
|
1602
1385
|
static async open(opts) {
|
|
1603
1386
|
const lance = requireLance();
|
|
1604
|
-
const storagePath = opts.storagePath ??
|
|
1387
|
+
const storagePath = opts.storagePath ?? lanceStoragePath(opts.memoryRoot ?? resolveMemoryRoot(opts.cwd));
|
|
1605
1388
|
mkdirSync(storagePath, { recursive: true });
|
|
1606
1389
|
const conn = await lance.connect(storagePath);
|
|
1607
1390
|
const dim = opts.embedding.dimension;
|
|
@@ -1703,9 +1486,6 @@ function isLanceAvailable() {
|
|
|
1703
1486
|
return false;
|
|
1704
1487
|
}
|
|
1705
1488
|
}
|
|
1706
|
-
function lanceStoragePath(cwd) {
|
|
1707
|
-
return join(cwd, ".theokit", "memory", "lance");
|
|
1708
|
-
}
|
|
1709
1489
|
|
|
1710
1490
|
// src/internal/index/memory-index.ts
|
|
1711
1491
|
function parseSearchOptions(options = {}) {
|
|
@@ -1905,12 +1685,12 @@ async function embedMissingChunks(args) {
|
|
|
1905
1685
|
|
|
1906
1686
|
// src/internal/index/index-manager.ts
|
|
1907
1687
|
var IndexManager = class _IndexManager {
|
|
1908
|
-
constructor(
|
|
1909
|
-
this.
|
|
1688
|
+
constructor(memoryRoot, db, embedding) {
|
|
1689
|
+
this.memoryRoot = memoryRoot;
|
|
1910
1690
|
this.db = db;
|
|
1911
1691
|
this.embedding = embedding;
|
|
1912
1692
|
}
|
|
1913
|
-
|
|
1693
|
+
memoryRoot;
|
|
1914
1694
|
db;
|
|
1915
1695
|
embedding;
|
|
1916
1696
|
lastSyncMs;
|
|
@@ -1923,9 +1703,10 @@ var IndexManager = class _IndexManager {
|
|
|
1923
1703
|
}
|
|
1924
1704
|
/** Internal SQLite-path open. Renamed from previous public `open`. */
|
|
1925
1705
|
static async openSqliteInternal(opts) {
|
|
1926
|
-
const
|
|
1706
|
+
const memoryRoot = opts.memoryRoot ?? resolveMemoryRoot(opts.cwd);
|
|
1707
|
+
const filePath = opts.filePath ?? defaultIndexPath(projectMemoryDir(opts.cwd));
|
|
1927
1708
|
const db = await openMemoryDb({ filePath });
|
|
1928
|
-
const manager = new _IndexManager(
|
|
1709
|
+
const manager = new _IndexManager(memoryRoot, db, opts.embedding);
|
|
1929
1710
|
if (opts.embedding !== void 0) await manager.initVectorBackend(opts.embedding);
|
|
1930
1711
|
return manager;
|
|
1931
1712
|
}
|
|
@@ -1946,7 +1727,7 @@ var IndexManager = class _IndexManager {
|
|
|
1946
1727
|
}
|
|
1947
1728
|
/** Walk the memory corpus + (re)index changed files. */
|
|
1948
1729
|
async sync() {
|
|
1949
|
-
const files = await collectMarkdownFiles(this.
|
|
1730
|
+
const files = await collectMarkdownFiles(this.memoryRoot);
|
|
1950
1731
|
let filesUpdated = 0;
|
|
1951
1732
|
let chunksWritten = 0;
|
|
1952
1733
|
const existingByPath = this.loadFilesIndex();
|
|
@@ -2143,45 +1924,6 @@ function blendScores(hit, vectorScore, weights) {
|
|
|
2143
1924
|
...vectorScore > 0 ? { vectorScore } : {}
|
|
2144
1925
|
};
|
|
2145
1926
|
}
|
|
2146
|
-
async function collectMarkdownFiles(cwd) {
|
|
2147
|
-
const root = memoryDir(cwd);
|
|
2148
|
-
const results = [];
|
|
2149
|
-
try {
|
|
2150
|
-
await stat(memoryMdPath(cwd));
|
|
2151
|
-
results.push({
|
|
2152
|
-
absolutePath: memoryMdPath(cwd),
|
|
2153
|
-
relPath: relative(root, memoryMdPath(cwd)),
|
|
2154
|
-
source: "memory"
|
|
2155
|
-
});
|
|
2156
|
-
} catch {
|
|
2157
|
-
}
|
|
2158
|
-
try {
|
|
2159
|
-
const entries = await readdir(notesDir(cwd));
|
|
2160
|
-
for (const entry of entries) {
|
|
2161
|
-
if (!entry.endsWith(".md")) continue;
|
|
2162
|
-
const abs = join(notesDir(cwd), entry);
|
|
2163
|
-
results.push({ absolutePath: abs, relPath: relative(root, abs), source: "memory" });
|
|
2164
|
-
}
|
|
2165
|
-
} catch {
|
|
2166
|
-
}
|
|
2167
|
-
const wikiFiles = await discoverWikiFiles(cwd);
|
|
2168
|
-
for (const wiki of wikiFiles) {
|
|
2169
|
-
results.push({
|
|
2170
|
-
absolutePath: wiki.absolutePath,
|
|
2171
|
-
relPath: wiki.relPath,
|
|
2172
|
-
source: "wiki"
|
|
2173
|
-
});
|
|
2174
|
-
}
|
|
2175
|
-
const sessionFiles = await discoverSessionFiles(cwd);
|
|
2176
|
-
for (const session of sessionFiles) {
|
|
2177
|
-
results.push({
|
|
2178
|
-
absolutePath: session.absolutePath,
|
|
2179
|
-
relPath: session.relPath,
|
|
2180
|
-
source: "sessions"
|
|
2181
|
-
});
|
|
2182
|
-
}
|
|
2183
|
-
return results;
|
|
2184
|
-
}
|
|
2185
1927
|
function sha256(text) {
|
|
2186
1928
|
return createHash("sha256").update(text).digest("hex");
|
|
2187
1929
|
}
|
|
@@ -2189,30 +1931,14 @@ function truncateSnippet(text) {
|
|
|
2189
1931
|
const max = 500;
|
|
2190
1932
|
return text.length <= max ? text : `${text.slice(0, max)}\u2026`;
|
|
2191
1933
|
}
|
|
2192
|
-
async function readAllSqliteFacts(cwd) {
|
|
2193
|
-
const dbPath = defaultIndexPath(cwd);
|
|
2194
|
-
if (!existsSync(dbPath)) return [];
|
|
2195
|
-
const db = await openMemoryDb({ filePath: dbPath });
|
|
2196
|
-
try {
|
|
2197
|
-
const stmt = db.prepare("SELECT id, path, source, start_line, end_line, text FROM chunks");
|
|
2198
|
-
const rows = stmt.all();
|
|
2199
|
-
return rows.map((r) => ({
|
|
2200
|
-
...r,
|
|
2201
|
-
namespace: "default",
|
|
2202
|
-
scope: "agent",
|
|
2203
|
-
user_id: "default"
|
|
2204
|
-
}));
|
|
2205
|
-
} finally {
|
|
2206
|
-
db.close();
|
|
2207
|
-
}
|
|
2208
|
-
}
|
|
2209
1934
|
function nfcEqual(a, b) {
|
|
2210
1935
|
return a.normalize("NFC") === b.normalize("NFC");
|
|
2211
1936
|
}
|
|
2212
1937
|
async function migrateSqliteToLance(opts) {
|
|
2213
1938
|
const cwd = opts.cwd;
|
|
2214
|
-
const
|
|
2215
|
-
const
|
|
1939
|
+
const memoryRoot = resolveMemoryRoot(cwd, { directory: opts.directory });
|
|
1940
|
+
const finalPath = lanceStoragePath(memoryRoot);
|
|
1941
|
+
const newPath = join(memoryRoot, "lance-new");
|
|
2216
1942
|
const rawLog = opts.logger ?? ((m) => console.log(m));
|
|
2217
1943
|
const log = (m) => rawLog(redactSecrets(m));
|
|
2218
1944
|
if (existsSync(finalPath)) {
|
|
@@ -2226,7 +1952,7 @@ async function migrateSqliteToLance(opts) {
|
|
|
2226
1952
|
rmSync(newPath, { recursive: true, force: true });
|
|
2227
1953
|
}
|
|
2228
1954
|
log(`Reading SQLite facts from ${cwd}/.theokit/memory/index.sqlite ...`);
|
|
2229
|
-
const sqliteFacts = await readAllSqliteFacts(
|
|
1955
|
+
const sqliteFacts = await readAllSqliteFacts(memoryRoot);
|
|
2230
1956
|
log(`SQLite has ${sqliteFacts.length} facts.`);
|
|
2231
1957
|
if (sqliteFacts.length === 0) {
|
|
2232
1958
|
return {
|
|
@@ -2370,7 +2096,7 @@ async function migrateLegacyJson(cwd, config) {
|
|
|
2370
2096
|
if (!await fileExists(jsonPath)) {
|
|
2371
2097
|
return { migrated: false, factCount: 0, reason: "no-legacy-json" };
|
|
2372
2098
|
}
|
|
2373
|
-
if (await fileExists(memoryMdPath(cwd))) {
|
|
2099
|
+
if (await fileExists(memoryMdPath(resolveMemoryRoot(cwd)))) {
|
|
2374
2100
|
process.stderr.write(
|
|
2375
2101
|
`[theokit-sdk] memory migration skipped: both MEMORY.md and legacy JSON exist at ${jsonPath}; leaving both intact
|
|
2376
2102
|
`
|
|
@@ -2456,7 +2182,7 @@ function createMemorySearchTool(opts) {
|
|
|
2456
2182
|
};
|
|
2457
2183
|
}
|
|
2458
2184
|
function createMemoryGetTool(opts) {
|
|
2459
|
-
const memoryRoot = resolve(
|
|
2185
|
+
const memoryRoot = resolve(opts.root);
|
|
2460
2186
|
return {
|
|
2461
2187
|
name: "memory_get",
|
|
2462
2188
|
description: GET_DESCRIPTION,
|
|
@@ -2525,6 +2251,6 @@ function isPathInside(root, candidate) {
|
|
|
2525
2251
|
return candidate === root || candidate.startsWith(normalizedRoot);
|
|
2526
2252
|
}
|
|
2527
2253
|
|
|
2528
|
-
export { ActiveMemoryCache, CircuitBreaker, DEFAULT_AZURE_OPENAI_EMBEDDING_MODEL, DEFAULT_COHERE_EMBEDDING_MODEL, DEFAULT_DEEPINFRA_EMBEDDING_MODEL, DEFAULT_GEMINI_EMBEDDING_MODEL, DEFAULT_JINA_EMBEDDING_MODEL, DEFAULT_MEMORY_READ_LINES, DEFAULT_MISTRAL_EMBEDDING_MODEL, DEFAULT_OLLAMA_EMBEDDING_MODEL, DEFAULT_OPENAI_EMBEDDING_MODEL, DEFAULT_OPENROUTER_EMBEDDING_MODEL, DEFAULT_VOYAGE_EMBEDDING_MODEL, IndexManager, LanceIndex, LanceMemoryAdapter, MEMORY_EMBEDDING_ADAPTERS, META_KEY_DIMENSION, META_KEY_MODEL, META_KEY_PROVIDER_ID, PRAGMA_STATEMENTS, SCHEMA_STATEMENTS, VALID_BACKENDS,
|
|
2254
|
+
export { ActiveMemoryCache, CircuitBreaker, DEFAULT_AZURE_OPENAI_EMBEDDING_MODEL, DEFAULT_COHERE_EMBEDDING_MODEL, DEFAULT_DEEPINFRA_EMBEDDING_MODEL, DEFAULT_GEMINI_EMBEDDING_MODEL, DEFAULT_JINA_EMBEDDING_MODEL, DEFAULT_MEMORY_READ_LINES, DEFAULT_MISTRAL_EMBEDDING_MODEL, DEFAULT_OLLAMA_EMBEDDING_MODEL, DEFAULT_OPENAI_EMBEDDING_MODEL, DEFAULT_OPENROUTER_EMBEDDING_MODEL, DEFAULT_VOYAGE_EMBEDDING_MODEL, IndexManager, LanceIndex, LanceMemoryAdapter, MEMORY_EMBEDDING_ADAPTERS, META_KEY_DIMENSION, META_KEY_MODEL, META_KEY_PROVIDER_ID, PRAGMA_STATEMENTS, SCHEMA_STATEMENTS, VALID_BACKENDS, assertValidBackend, azureOpenAiMemoryEmbeddingProviderAdapter, buildErrorMetadata, chunkMarkdown, cohereMemoryEmbeddingProviderAdapter, createCategorizedMemory, createInMemoryMarkdownProvider, createMemoryGetTool, createMemorySearchTool, createVectorIndex, deepPhase, deepinfraMemoryEmbeddingProviderAdapter, dropVectorIndex, embedMissingChunks, geminiMemoryEmbeddingProviderAdapter, identityMatches, isLanceAvailable, isSqliteVecLoaded, jinaMemoryEmbeddingProviderAdapter, legacyMemoryJsonPath, lightPhase, listNotes, loadSqliteVecExtension, mapOpenAICompatibleError, migrateLegacyJson, migrateSqliteToLance, mistralMemoryEmbeddingProviderAdapter, ollamaMemoryEmbeddingProviderAdapter, openAiMemoryEmbeddingProviderAdapter, openLanceIndex, openMemoryDb, openRouterMemoryEmbeddingProviderAdapter, packVector, parseRetryAfter, parseSearchOptions, readEmbeddingIdentity, readMemoryFileBounded, redactSecrets, remPhase, resetMigrationStateForTests, runActiveMemory, runDreamingSweep, truncateRaw, upsertEmbedding, vectorSearch, voyageMemoryEmbeddingProviderAdapter, writeEmbeddingIdentity };
|
|
2529
2255
|
//# sourceMappingURL=index.js.map
|
|
2530
2256
|
//# sourceMappingURL=index.js.map
|