@theokit/sdk-memory 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/CHANGELOG.md +31 -1
  2. package/dist/index.cjs +309 -209
  3. package/dist/index.cjs.map +1 -1
  4. package/dist/index.d.cts +34 -33
  5. package/dist/index.d.ts +34 -33
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js +312 -213
  8. package/dist/index.js.map +1 -1
  9. package/dist/internal/active-memory/active-memory-cache.d.ts.map +1 -0
  10. package/dist/internal/{active-memory-types.d.ts → active-memory/active-memory-types.d.ts} +1 -1
  11. package/dist/internal/active-memory/active-memory-types.d.ts.map +1 -0
  12. package/dist/internal/{active-memory.d.ts → active-memory/active-memory.d.ts} +2 -2
  13. package/dist/internal/active-memory/active-memory.d.ts.map +1 -0
  14. package/dist/internal/active-memory/composite-scorer.d.ts.map +1 -0
  15. package/dist/internal/active-memory/query-analyzer.d.ts.map +1 -0
  16. package/dist/internal/adapter-catalog.d.ts +6 -6
  17. package/dist/internal/categorized-memory.d.ts +56 -0
  18. package/dist/internal/categorized-memory.d.ts.map +1 -0
  19. package/dist/internal/dreaming/dreaming-diary.d.ts.map +1 -0
  20. package/dist/internal/{dreaming-phases.d.ts → dreaming/dreaming-phases.d.ts} +2 -2
  21. package/dist/internal/dreaming/dreaming-phases.d.ts.map +1 -0
  22. package/dist/internal/{dreaming-run.d.ts → dreaming/dreaming-run.d.ts} +1 -1
  23. package/dist/internal/dreaming/dreaming-run.d.ts.map +1 -0
  24. package/dist/internal/embedding/batch-encoder.d.ts.map +1 -0
  25. package/dist/internal/embedding/deepinfra-embedding.d.ts.map +1 -0
  26. package/dist/internal/embedding/embedding-adapter.d.ts.map +1 -0
  27. package/dist/internal/embedding/embedding-cache.d.ts.map +1 -0
  28. package/dist/internal/embedding/mistral-embedding.d.ts.map +1 -0
  29. package/dist/internal/embedding/ollama-embedding.d.ts.map +1 -0
  30. package/dist/internal/embedding/openai-compatible.d.ts.map +1 -0
  31. package/dist/internal/embedding/openai-embedding.d.ts.map +1 -0
  32. package/dist/internal/embedding/openrouter-embedding.d.ts.map +1 -0
  33. package/dist/internal/embedding/voyage-embedding.d.ts.map +1 -0
  34. package/dist/internal/index/index-db.d.ts.map +1 -0
  35. package/dist/internal/{index-manager-contract.d.ts → index/index-manager-contract.d.ts} +1 -1
  36. package/dist/internal/index/index-manager-contract.d.ts.map +1 -0
  37. package/dist/internal/index/index-manager-dispatch.d.ts.map +1 -0
  38. package/dist/internal/{index-manager.d.ts → index/index-manager.d.ts} +1 -1
  39. package/dist/internal/index/index-manager.d.ts.map +1 -0
  40. package/dist/internal/index/index-schema.d.ts.map +1 -0
  41. package/dist/internal/{lance-index.d.ts → index/lance-index.d.ts} +1 -1
  42. package/dist/internal/index/lance-index.d.ts.map +1 -0
  43. package/dist/internal/index/lance-memory-adapter.d.ts.map +1 -0
  44. package/dist/internal/index/memory-index.d.ts.map +1 -0
  45. package/dist/internal/index/migrate-sqlite-to-lance.d.ts.map +1 -0
  46. package/dist/internal/{migration.d.ts → index/migration.d.ts} +1 -1
  47. package/dist/internal/index/migration.d.ts.map +1 -0
  48. package/dist/internal/index/sqlite-vec-loader.d.ts.map +1 -0
  49. package/dist/internal/{vec-index.d.ts → index/vec-index.d.ts} +1 -1
  50. package/dist/internal/index/vec-index.d.ts.map +1 -0
  51. package/dist/internal/memory-types.d.ts +5 -0
  52. package/dist/internal/memory-types.d.ts.map +1 -1
  53. package/dist/internal/store/chunk-markdown.d.ts.map +1 -0
  54. package/dist/internal/{markdown-store.d.ts → store/markdown-store.d.ts} +1 -1
  55. package/dist/internal/store/markdown-store.d.ts.map +1 -0
  56. package/dist/internal/store/reader.d.ts.map +1 -0
  57. package/dist/internal/store/session-loader.d.ts.map +1 -0
  58. package/dist/internal/store/session-summary-writer.d.ts.map +1 -0
  59. package/dist/internal/store/transcript-store.d.ts.map +1 -0
  60. package/dist/internal/store/wiki-loader.d.ts.map +1 -0
  61. package/dist/internal/tools.d.ts +1 -1
  62. package/dist/internal/tools.d.ts.map +1 -1
  63. package/package.json +9 -9
  64. package/dist/internal/active-memory-cache.d.ts.map +0 -1
  65. package/dist/internal/active-memory-types.d.ts.map +0 -1
  66. package/dist/internal/active-memory.d.ts.map +0 -1
  67. package/dist/internal/batch-encoder.d.ts.map +0 -1
  68. package/dist/internal/chunk-markdown.d.ts.map +0 -1
  69. package/dist/internal/composite-scorer.d.ts.map +0 -1
  70. package/dist/internal/deepinfra-embedding.d.ts.map +0 -1
  71. package/dist/internal/dreaming-diary.d.ts.map +0 -1
  72. package/dist/internal/dreaming-phases.d.ts.map +0 -1
  73. package/dist/internal/dreaming-run.d.ts.map +0 -1
  74. package/dist/internal/embedding-adapter.d.ts.map +0 -1
  75. package/dist/internal/embedding-cache.d.ts.map +0 -1
  76. package/dist/internal/index-db.d.ts.map +0 -1
  77. package/dist/internal/index-manager-contract.d.ts.map +0 -1
  78. package/dist/internal/index-manager-dispatch.d.ts.map +0 -1
  79. package/dist/internal/index-manager.d.ts.map +0 -1
  80. package/dist/internal/index-schema.d.ts.map +0 -1
  81. package/dist/internal/lance-index.d.ts.map +0 -1
  82. package/dist/internal/lance-memory-adapter.d.ts.map +0 -1
  83. package/dist/internal/markdown-store.d.ts.map +0 -1
  84. package/dist/internal/memory-index.d.ts.map +0 -1
  85. package/dist/internal/migrate-sqlite-to-lance.d.ts.map +0 -1
  86. package/dist/internal/migration.d.ts.map +0 -1
  87. package/dist/internal/mistral-embedding.d.ts.map +0 -1
  88. package/dist/internal/ollama-embedding.d.ts.map +0 -1
  89. package/dist/internal/openai-compatible.d.ts.map +0 -1
  90. package/dist/internal/openai-embedding.d.ts.map +0 -1
  91. package/dist/internal/openrouter-embedding.d.ts.map +0 -1
  92. package/dist/internal/query-analyzer.d.ts.map +0 -1
  93. package/dist/internal/reader.d.ts.map +0 -1
  94. package/dist/internal/session-loader.d.ts.map +0 -1
  95. package/dist/internal/session-summary-writer.d.ts.map +0 -1
  96. package/dist/internal/sqlite-vec-loader.d.ts.map +0 -1
  97. package/dist/internal/transcript-store.d.ts.map +0 -1
  98. package/dist/internal/vec-index.d.ts.map +0 -1
  99. package/dist/internal/voyage-embedding.d.ts.map +0 -1
  100. package/dist/internal/wiki-loader.d.ts.map +0 -1
  101. /package/dist/internal/{active-memory-cache.d.ts → active-memory/active-memory-cache.d.ts} +0 -0
  102. /package/dist/internal/{composite-scorer.d.ts → active-memory/composite-scorer.d.ts} +0 -0
  103. /package/dist/internal/{query-analyzer.d.ts → active-memory/query-analyzer.d.ts} +0 -0
  104. /package/dist/internal/{dreaming-diary.d.ts → dreaming/dreaming-diary.d.ts} +0 -0
  105. /package/dist/internal/{batch-encoder.d.ts → embedding/batch-encoder.d.ts} +0 -0
  106. /package/dist/internal/{deepinfra-embedding.d.ts → embedding/deepinfra-embedding.d.ts} +0 -0
  107. /package/dist/internal/{embedding-adapter.d.ts → embedding/embedding-adapter.d.ts} +0 -0
  108. /package/dist/internal/{embedding-cache.d.ts → embedding/embedding-cache.d.ts} +0 -0
  109. /package/dist/internal/{mistral-embedding.d.ts → embedding/mistral-embedding.d.ts} +0 -0
  110. /package/dist/internal/{ollama-embedding.d.ts → embedding/ollama-embedding.d.ts} +0 -0
  111. /package/dist/internal/{openai-compatible.d.ts → embedding/openai-compatible.d.ts} +0 -0
  112. /package/dist/internal/{openai-embedding.d.ts → embedding/openai-embedding.d.ts} +0 -0
  113. /package/dist/internal/{openrouter-embedding.d.ts → embedding/openrouter-embedding.d.ts} +0 -0
  114. /package/dist/internal/{voyage-embedding.d.ts → embedding/voyage-embedding.d.ts} +0 -0
  115. /package/dist/internal/{index-db.d.ts → index/index-db.d.ts} +0 -0
  116. /package/dist/internal/{index-manager-dispatch.d.ts → index/index-manager-dispatch.d.ts} +0 -0
  117. /package/dist/internal/{index-schema.d.ts → index/index-schema.d.ts} +0 -0
  118. /package/dist/internal/{lance-memory-adapter.d.ts → index/lance-memory-adapter.d.ts} +0 -0
  119. /package/dist/internal/{memory-index.d.ts → index/memory-index.d.ts} +0 -0
  120. /package/dist/internal/{migrate-sqlite-to-lance.d.ts → index/migrate-sqlite-to-lance.d.ts} +0 -0
  121. /package/dist/internal/{sqlite-vec-loader.d.ts → index/sqlite-vec-loader.d.ts} +0 -0
  122. /package/dist/internal/{chunk-markdown.d.ts → store/chunk-markdown.d.ts} +0 -0
  123. /package/dist/internal/{reader.d.ts → store/reader.d.ts} +0 -0
  124. /package/dist/internal/{session-loader.d.ts → store/session-loader.d.ts} +0 -0
  125. /package/dist/internal/{session-summary-writer.d.ts → store/session-summary-writer.d.ts} +0 -0
  126. /package/dist/internal/{transcript-store.d.ts → store/transcript-store.d.ts} +0 -0
  127. /package/dist/internal/{wiki-loader.d.ts → store/wiki-loader.d.ts} +0 -0
package/dist/index.cjs CHANGED
@@ -252,7 +252,7 @@ function legacyMemoryJsonPath(cwd, config) {
252
252
  return pathSafety.safePathJoin(cwd, ".theokit", "memory", namespace, `${scope}-${userId}.json`);
253
253
  }
254
254
 
255
- // src/internal/markdown-store.ts
255
+ // src/internal/store/markdown-store.ts
256
256
  var MEMORY_MD_HEADER = "# Memory\n\n> Auto-managed by @theokit/sdk. Edit freely \u2014 the SDK reads from here.\n";
257
257
  var FACTS_HEADING = "## Facts";
258
258
  function memoryDir(cwd) {
@@ -343,7 +343,7 @@ async function appendFact(cwd, config, fact) {
343
343
  await appendFactToMarkdown(cwd, fact);
344
344
  }
345
345
 
346
- // src/internal/transcript-store.ts
346
+ // src/internal/store/transcript-store.ts
347
347
  async function persistActiveMemoryTranscript(cwd, transcript) {
348
348
  try {
349
349
  const dir = path.join(memoryDir(cwd), "transcripts", "active-memory");
@@ -356,7 +356,7 @@ async function persistActiveMemoryTranscript(cwd, transcript) {
356
356
  }
357
357
  }
358
358
 
359
- // src/internal/active-memory.ts
359
+ // src/internal/active-memory/active-memory.ts
360
360
  var SPAN_MEMORY_RECALL = "memory.recall";
361
361
  var HISTOGRAM_MEMORY_RECALL_DURATION_MS = "theokit_memory_recall_duration_ms";
362
362
  var DEFAULTS = {
@@ -387,7 +387,12 @@ async function runActiveMemory(args) {
387
387
  hits: []
388
388
  });
389
389
  }
390
- const cached = args.cache?.get(args.userText, cfg.queryMode);
390
+ const tenantCtx = {
391
+ namespace: args.namespace,
392
+ userId: args.userId,
393
+ scope: args.scope
394
+ };
395
+ const cached = args.cache?.get(args.userText, cfg.queryMode, tenantCtx);
391
396
  if (cached !== void 0) return endRecallSpan(span, args, cached);
392
397
  const query = buildQuery(args.userText, args.priorMessages, cfg.queryMode, cfg.recentUserTurns);
393
398
  if (query.trim().length === 0) {
@@ -484,7 +489,12 @@ function notifyBreaker(breaker, key, status) {
484
489
  else if (status === "ok" || status === "no-recall") breaker.recordSuccess(key);
485
490
  }
486
491
  async function finalize(args, queryMode, result) {
487
- args.cache?.set(args.userText, queryMode, result);
492
+ const tenantCtx = {
493
+ namespace: args.namespace,
494
+ userId: args.userId,
495
+ scope: args.scope
496
+ };
497
+ args.cache?.set(args.userText, queryMode, result, tenantCtx);
488
498
  if (args.persistTranscripts === true && args.cwd !== void 0) {
489
499
  await persistActiveMemoryTranscript(args.cwd, {
490
500
  runId: args.runId ?? `run-${Date.now()}`,
@@ -705,7 +715,7 @@ function formatMessage(providerId, status, code) {
705
715
  return `${providerId} API error: ${code} (HTTP ${status})`;
706
716
  }
707
717
 
708
- // src/internal/embedding-cache.ts
718
+ // src/internal/embedding/embedding-cache.ts
709
719
  var LruEmbeddingCache = class {
710
720
  constructor(capacity = 5e3) {
711
721
  this.capacity = capacity;
@@ -732,7 +742,7 @@ var LruEmbeddingCache = class {
732
742
  }
733
743
  };
734
744
 
735
- // src/internal/openai-compatible.ts
745
+ // src/internal/embedding/openai-compatible.ts
736
746
  var MAX_BATCH = 100;
737
747
  var MAX_RETRIES = 2;
738
748
  async function createOpenAiCompatibleRuntime(cfg, options) {
@@ -903,7 +913,7 @@ function sleep(ms) {
903
913
  return new Promise((resolve) => setTimeout(resolve, ms));
904
914
  }
905
915
 
906
- // src/internal/deepinfra-embedding.ts
916
+ // src/internal/embedding/deepinfra-embedding.ts
907
917
  var DEFAULT_DEEPINFRA_EMBEDDING_MODEL = "BAAI/bge-large-en-v1.5";
908
918
  var DIMENSION_BY_MODEL = {
909
919
  "BAAI/bge-large-en-v1.5": 1024,
@@ -937,7 +947,7 @@ var deepinfraMemoryEmbeddingProviderAdapter = {
937
947
  )
938
948
  };
939
949
 
940
- // src/internal/mistral-embedding.ts
950
+ // src/internal/embedding/mistral-embedding.ts
941
951
  var DEFAULT_MISTRAL_EMBEDDING_MODEL = "mistral-embed";
942
952
  var DIMENSION_BY_MODEL2 = {
943
953
  "mistral-embed": 1024
@@ -961,7 +971,7 @@ var mistralMemoryEmbeddingProviderAdapter = {
961
971
  )
962
972
  };
963
973
 
964
- // src/internal/ollama-embedding.ts
974
+ // src/internal/embedding/ollama-embedding.ts
965
975
  var DEFAULT_OLLAMA_EMBEDDING_MODEL = "nomic-embed-text";
966
976
  var OLLAMA_LOCAL_SENTINEL_KEY = "ollama-local";
967
977
  var DIMENSION_BY_MODEL3 = {
@@ -1004,7 +1014,7 @@ var ollamaMemoryEmbeddingProviderAdapter = {
1004
1014
  }
1005
1015
  };
1006
1016
 
1007
- // src/internal/openai-embedding.ts
1017
+ // src/internal/embedding/openai-embedding.ts
1008
1018
  var DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
1009
1019
  var DIMENSION_BY_MODEL4 = {
1010
1020
  "text-embedding-3-small": 1536,
@@ -1030,7 +1040,7 @@ var openAiMemoryEmbeddingProviderAdapter = {
1030
1040
  )
1031
1041
  };
1032
1042
 
1033
- // src/internal/openrouter-embedding.ts
1043
+ // src/internal/embedding/openrouter-embedding.ts
1034
1044
  var DEFAULT_OPENROUTER_EMBEDDING_MODEL = "openai/text-embedding-3-small";
1035
1045
  var DIMENSION_BY_MODEL5 = {
1036
1046
  "openai/text-embedding-3-small": 1536,
@@ -1057,7 +1067,7 @@ var openRouterMemoryEmbeddingProviderAdapter = {
1057
1067
  )
1058
1068
  };
1059
1069
 
1060
- // src/internal/voyage-embedding.ts
1070
+ // src/internal/embedding/voyage-embedding.ts
1061
1071
  var DEFAULT_VOYAGE_EMBEDDING_MODEL = "voyage-3-lite";
1062
1072
  var DIMENSION_BY_MODEL6 = {
1063
1073
  "voyage-3-lite": 512,
@@ -1094,80 +1104,131 @@ var MEMORY_EMBEDDING_ADAPTERS = {
1094
1104
  deepinfra: deepinfraMemoryEmbeddingProviderAdapter,
1095
1105
  ollama: ollamaMemoryEmbeddingProviderAdapter
1096
1106
  };
1097
- var HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
1098
- function chunkMarkdown(text, options = {}) {
1099
- const maxChars = options.maxChars ?? 800;
1100
- if (text.length === 0) return [];
1101
- const lines = text.split("\n");
1102
- const acc = { chunks: [], lines: [], start: 1, heading: void 0 };
1103
- for (let i = 0; i < lines.length; i++) {
1104
- const line = lines[i] ?? "";
1105
- const lineNumber = i + 1;
1106
- processLine(acc, line, lineNumber, maxChars);
1107
+ var FACTS_HEADING2 = "## Facts";
1108
+ function createCategorizedMemory(options) {
1109
+ const { root, categories } = options;
1110
+ validateCategories(categories);
1111
+ const allowed = new Set(categories);
1112
+ function assertKnown(category) {
1113
+ if (!allowed.has(category)) {
1114
+ throw new errors.ConfigurationError(
1115
+ `createCategorizedMemory: unknown category ${JSON.stringify(category)} (declared: ${categories.join(", ")})`,
1116
+ { code: "unknown_category" }
1117
+ );
1118
+ }
1107
1119
  }
1108
- flushAccumulator(acc, lines.length, maxChars);
1109
- return acc.chunks;
1110
- }
1111
- function processLine(acc, line, lineNumber, maxChars) {
1112
- const headingMatch = HEADING_RE.exec(line);
1113
- const isBlank = line.trim().length === 0;
1114
- if (headingMatch !== null && acc.lines.length > 0) {
1115
- flushAccumulator(acc, lineNumber - 1, maxChars);
1120
+ function categoryPath(category) {
1121
+ return pathSafety.safePathJoin(root, `${pathSafety.sanitizeIdentifier(category)}.md`);
1122
+ }
1123
+ async function add(category, text) {
1124
+ assertKnown(category);
1125
+ const encoded = encodeFact(redactSecrets(text));
1126
+ const sane = pathSafety.sanitizeIdentifier(category);
1127
+ await persistence.withCwdMutex(`catmem:${root}:${sane}`, async () => {
1128
+ const path = pathSafety.safePathJoin(root, `${sane}.md`);
1129
+ const raw = await readFileOrEmpty(path) || header(category);
1130
+ await promises.mkdir(root, { recursive: true });
1131
+ await persistence.replaceFileAtomic(path, appendBullet(raw, encoded));
1132
+ });
1116
1133
  }
1117
- if (headingMatch !== null) acc.heading = headingMatch[2];
1118
- acc.lines.push(line);
1119
- if (isBlank && acc.lines.length > 1) {
1120
- const joined = acc.lines.join("\n");
1121
- if (joined.trim().length > 0) flushAccumulator(acc, lineNumber, maxChars);
1134
+ async function list(category) {
1135
+ if (category !== void 0) {
1136
+ assertKnown(category);
1137
+ return readCategory(categoryPath(category), category);
1138
+ }
1139
+ const out = [];
1140
+ for (const c of categories) {
1141
+ out.push(...await readCategory(categoryPath(c), c));
1142
+ }
1143
+ return out;
1122
1144
  }
1145
+ return { categories: [...categories], add, list };
1123
1146
  }
1124
- function flushAccumulator(acc, endLineExclusive, maxChars) {
1125
- if (acc.lines.length === 0) return;
1126
- const chunkText = acc.lines.join("\n");
1127
- if (chunkText.trim().length === 0) {
1128
- acc.lines = [];
1129
- acc.start = endLineExclusive + 1;
1130
- return;
1147
+ function validateCategories(categories) {
1148
+ if (categories.length === 0) {
1149
+ throw new errors.ConfigurationError("createCategorizedMemory: categories must be non-empty", {
1150
+ code: "invalid_categories"
1151
+ });
1131
1152
  }
1132
- if (chunkText.length > maxChars) {
1133
- pushOversizedSlices(acc.chunks, chunkText, acc.start, acc.heading, maxChars);
1134
- } else {
1135
- acc.chunks.push(buildChunk(chunkText, acc.start, endLineExclusive, acc.heading));
1153
+ if (new Set(categories).size !== categories.length) {
1154
+ throw new errors.ConfigurationError("createCategorizedMemory: categories must be unique", {
1155
+ code: "invalid_categories"
1156
+ });
1157
+ }
1158
+ const sanitized = categories.map((c) => trySanitize(c));
1159
+ if (new Set(sanitized).size !== categories.length) {
1160
+ throw new errors.ConfigurationError(
1161
+ "createCategorizedMemory: categories must map to distinct filenames after sanitization",
1162
+ { code: "invalid_categories" }
1163
+ );
1136
1164
  }
1137
- acc.lines = [];
1138
- acc.start = endLineExclusive + 1;
1139
- }
1140
- function buildChunk(text, startLine, endLine, heading) {
1141
- return {
1142
- startLine,
1143
- endLine,
1144
- text,
1145
- hash: crypto.createHash("sha256").update(text).digest("hex"),
1146
- ...heading !== void 0 ? { heading } : {}
1147
- };
1148
1165
  }
1149
- function pushOversizedSlices(chunks, text, startLine, heading, maxChars) {
1150
- let remaining = text;
1151
- let lineCursor = startLine;
1152
- while (remaining.length > maxChars) {
1153
- const splitAt = findWordBoundarySplit(remaining, maxChars);
1154
- const slice = remaining.slice(0, splitAt);
1155
- const sliceLines = slice.split("\n").length;
1156
- chunks.push(buildChunk(slice, lineCursor, lineCursor + sliceLines - 1, heading));
1157
- remaining = remaining.slice(splitAt).replace(/^\s+/, "");
1158
- lineCursor += sliceLines - 1;
1166
+ function trySanitize(category) {
1167
+ try {
1168
+ return pathSafety.sanitizeIdentifier(category);
1169
+ } catch (cause) {
1170
+ throw new errors.ConfigurationError(
1171
+ `createCategorizedMemory: category ${JSON.stringify(category)} is not a valid identifier`,
1172
+ { code: "invalid_categories", cause }
1173
+ );
1159
1174
  }
1160
- if (remaining.length > 0) {
1161
- const sliceLines = remaining.split("\n").length;
1162
- chunks.push(buildChunk(remaining, lineCursor, lineCursor + sliceLines - 1, heading));
1175
+ }
1176
+ function header(category) {
1177
+ return `---
1178
+ category: ${category}
1179
+ ---
1180
+
1181
+ ${FACTS_HEADING2}
1182
+ `;
1183
+ }
1184
+ async function readFileOrEmpty(path) {
1185
+ try {
1186
+ return await promises.readFile(path, "utf8");
1187
+ } catch {
1188
+ return "";
1163
1189
  }
1164
1190
  }
1165
- function findWordBoundarySplit(text, maxChars) {
1166
- if (text.length <= maxChars) return text.length;
1167
- for (let i = maxChars; i > Math.max(0, maxChars - 200); i--) {
1168
- if (/\s/.test(text[i] ?? "x")) return i;
1191
+ async function readCategory(path, category) {
1192
+ const raw = await readFileOrEmpty(path);
1193
+ return parseFactBullets(raw).map((text) => ({ text: decodeFact(text), category }));
1194
+ }
1195
+ function encodeFact(text) {
1196
+ return text.replace(/\\/g, "\\\\").replace(/\r/g, "\\r").replace(/\n/g, "\\n");
1197
+ }
1198
+ function decodeFact(text) {
1199
+ return text.replace(
1200
+ /\\(.)/g,
1201
+ (_match, char) => char === "n" ? "\n" : char === "r" ? "\r" : char
1202
+ );
1203
+ }
1204
+ function parseFactBullets(raw) {
1205
+ const idx = raw.indexOf(FACTS_HEADING2);
1206
+ if (idx === -1) return [];
1207
+ const tail = raw.slice(idx + FACTS_HEADING2.length);
1208
+ const nextHeading = tail.search(/\n#{1,2}\s/);
1209
+ const block = nextHeading === -1 ? tail : tail.slice(0, nextHeading);
1210
+ return block.split("\n").map((line) => line.trim()).filter((line) => line.startsWith("- ")).map((line) => line.slice(2).trim());
1211
+ }
1212
+ function appendBullet(raw, text) {
1213
+ const bullet = `- ${text}`;
1214
+ const idx = raw.indexOf(FACTS_HEADING2);
1215
+ if (idx === -1) {
1216
+ const sep = raw.endsWith("\n") ? "" : "\n";
1217
+ return `${raw}${sep}${FACTS_HEADING2}
1218
+
1219
+ ${bullet}
1220
+ `;
1169
1221
  }
1170
- return maxChars;
1222
+ const after = idx + FACTS_HEADING2.length;
1223
+ const nextHeading = raw.slice(after).search(/\n#{1,2}\s/);
1224
+ if (nextHeading === -1) {
1225
+ const trailing = raw.endsWith("\n") ? "" : "\n";
1226
+ return `${raw}${trailing}${bullet}
1227
+ `;
1228
+ }
1229
+ const insertAt = after + nextHeading;
1230
+ return `${raw.slice(0, insertAt)}
1231
+ ${bullet}${raw.slice(insertAt)}`;
1171
1232
  }
1172
1233
  function diaryPath(cwd) {
1173
1234
  return path.join(memoryDir(cwd), "dream-diary.md");
@@ -1211,7 +1272,7 @@ function entryHash(entry) {
1211
1272
  ).digest("hex");
1212
1273
  }
1213
1274
 
1214
- // src/internal/dreaming-phases.ts
1275
+ // src/internal/dreaming/dreaming-phases.ts
1215
1276
  var DEFAULT_DEDUP_THRESHOLD = 0.95;
1216
1277
  var DEFAULT_CLUSTER_THRESHOLD = 0.75;
1217
1278
  async function lightPhase(facts, embedding, threshold = DEFAULT_DEDUP_THRESHOLD) {
@@ -1369,7 +1430,7 @@ function emptyResult(status) {
1369
1430
  };
1370
1431
  }
1371
1432
 
1372
- // src/internal/index-schema.ts
1433
+ // src/internal/index/index-schema.ts
1373
1434
  var SCHEMA_STATEMENTS = [
1374
1435
  `CREATE TABLE IF NOT EXISTS files (
1375
1436
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -1415,57 +1476,180 @@ var PRAGMA_STATEMENTS = [
1415
1476
  "PRAGMA foreign_keys=ON"
1416
1477
  ];
1417
1478
 
1418
- // src/internal/index-db.ts
1479
+ // src/internal/index/index-db.ts
1419
1480
  async function openMemoryDb(opts) {
1420
- await promises.mkdir(path.dirname(opts.filePath), { recursive: true });
1421
- try {
1422
- return await openConcrete(opts.filePath);
1423
- } catch (cause) {
1424
- if (opts.recoverCorrupt !== false && isCorruptionError(cause)) {
1425
- await renameAside(opts.filePath);
1426
- return await openConcrete(opts.filePath);
1481
+ return persistence.openSqliteResilient({
1482
+ filePath: opts.filePath,
1483
+ label: "memory-index",
1484
+ recoverCorrupt: opts.recoverCorrupt,
1485
+ onOpen: (db) => {
1486
+ for (const pragma of PRAGMA_STATEMENTS) db.exec(pragma);
1487
+ for (const stmt of SCHEMA_STATEMENTS) db.exec(stmt);
1427
1488
  }
1428
- throw cause;
1489
+ });
1490
+ }
1491
+ function defaultIndexPath(cwd) {
1492
+ return path.join(cwd, ".theokit", "memory", ".index", "memory.sqlite");
1493
+ }
1494
+ var HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
1495
+ function chunkMarkdown(text, options = {}) {
1496
+ const maxChars = options.maxChars ?? 800;
1497
+ if (text.length === 0) return [];
1498
+ const lines = text.split("\n");
1499
+ const acc = { chunks: [], lines: [], start: 1, heading: void 0 };
1500
+ for (let i = 0; i < lines.length; i++) {
1501
+ const line = lines[i] ?? "";
1502
+ const lineNumber = i + 1;
1503
+ processLine(acc, line, lineNumber, maxChars);
1504
+ }
1505
+ flushAccumulator(acc, lines.length, maxChars);
1506
+ return acc.chunks;
1507
+ }
1508
+ function processLine(acc, line, lineNumber, maxChars) {
1509
+ const headingMatch = HEADING_RE.exec(line);
1510
+ const isBlank = line.trim().length === 0;
1511
+ if (headingMatch !== null && acc.lines.length > 0) {
1512
+ flushAccumulator(acc, lineNumber - 1, maxChars);
1513
+ }
1514
+ if (headingMatch !== null) acc.heading = headingMatch[2];
1515
+ acc.lines.push(line);
1516
+ if (isBlank && acc.lines.length > 1) {
1517
+ const joined = acc.lines.join("\n");
1518
+ if (joined.trim().length > 0) flushAccumulator(acc, lineNumber, maxChars);
1429
1519
  }
1430
1520
  }
1431
- async function openConcrete(filePath) {
1432
- const db = await loadDriver(filePath);
1433
- persistence.applyWalWithFallback(db, "memory-index");
1434
- for (const pragma of PRAGMA_STATEMENTS) db.exec(pragma);
1435
- for (const stmt of SCHEMA_STATEMENTS) db.exec(stmt);
1436
- return db;
1521
+ function flushAccumulator(acc, endLineExclusive, maxChars) {
1522
+ if (acc.lines.length === 0) return;
1523
+ const chunkText = acc.lines.join("\n");
1524
+ if (chunkText.trim().length === 0) {
1525
+ acc.lines = [];
1526
+ acc.start = endLineExclusive + 1;
1527
+ return;
1528
+ }
1529
+ if (chunkText.length > maxChars) {
1530
+ pushOversizedSlices(acc.chunks, chunkText, acc.start, acc.heading, maxChars);
1531
+ } else {
1532
+ acc.chunks.push(buildChunk(chunkText, acc.start, endLineExclusive, acc.heading));
1533
+ }
1534
+ acc.lines = [];
1535
+ acc.start = endLineExclusive + 1;
1437
1536
  }
1438
- async function loadDriver(filePath) {
1537
+ function buildChunk(text, startLine, endLine, heading) {
1538
+ return {
1539
+ startLine,
1540
+ endLine,
1541
+ text,
1542
+ hash: crypto.createHash("sha256").update(text).digest("hex"),
1543
+ ...heading !== void 0 ? { heading } : {}
1544
+ };
1545
+ }
1546
+ function pushOversizedSlices(chunks, text, startLine, heading, maxChars) {
1547
+ let remaining = text;
1548
+ let lineCursor = startLine;
1549
+ while (remaining.length > maxChars) {
1550
+ const splitAt = findWordBoundarySplit(remaining, maxChars);
1551
+ const slice = remaining.slice(0, splitAt);
1552
+ const sliceLines = slice.split("\n").length;
1553
+ chunks.push(buildChunk(slice, lineCursor, lineCursor + sliceLines - 1, heading));
1554
+ remaining = remaining.slice(splitAt).replace(/^\s+/, "");
1555
+ lineCursor += sliceLines - 1;
1556
+ }
1557
+ if (remaining.length > 0) {
1558
+ const sliceLines = remaining.split("\n").length;
1559
+ chunks.push(buildChunk(remaining, lineCursor, lineCursor + sliceLines - 1, heading));
1560
+ }
1561
+ }
1562
+ function findWordBoundarySplit(text, maxChars) {
1563
+ if (text.length <= maxChars) return text.length;
1564
+ for (let i = maxChars; i > Math.max(0, maxChars - 200); i--) {
1565
+ if (/\s/.test(text[i] ?? "x")) return i;
1566
+ }
1567
+ return maxChars;
1568
+ }
1569
+ var MAX_TURN_CHARS = 2e3;
1570
+ function sessionsDir(cwd) {
1571
+ return path.join(memoryDir(cwd), "sessions");
1572
+ }
1573
+ function sessionSummaryPath(cwd, runId) {
1574
+ return path.join(sessionsDir(cwd), `${sanitizeRunId2(runId)}.md`);
1575
+ }
1576
+ function sanitizeRunId2(runId) {
1577
+ return runId.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 128);
1578
+ }
1579
+ function truncate2(text) {
1580
+ if (text.length <= MAX_TURN_CHARS) return text;
1581
+ return `${text.slice(0, MAX_TURN_CHARS)}\u2026`;
1582
+ }
1583
+ async function writeSessionSummary(input) {
1584
+ if (input.status !== "finished") return;
1585
+ const path = sessionSummaryPath(input.cwd, input.runId);
1586
+ await promises.mkdir(sessionsDir(input.cwd), { recursive: true });
1587
+ const safeUser = redactSecrets(truncate2(input.userText));
1588
+ const safeAssistant = redactSecrets(truncate2(input.assistantText));
1589
+ const iso = new Date(input.at).toISOString();
1590
+ const body = [
1591
+ "---",
1592
+ `runId: ${input.runId}`,
1593
+ `agentId: ${input.agentId}`,
1594
+ `at: ${iso}`,
1595
+ `status: ${input.status}`,
1596
+ "---",
1597
+ "",
1598
+ "## User",
1599
+ "",
1600
+ safeUser,
1601
+ "",
1602
+ "## Assistant",
1603
+ "",
1604
+ safeAssistant,
1605
+ ""
1606
+ ].join("\n");
1607
+ await persistence.replaceFileAtomic(path, body);
1608
+ }
1609
+
1610
+ // src/internal/store/session-loader.ts
1611
+ async function discoverSessionFiles(cwd) {
1612
+ let entries;
1439
1613
  try {
1440
- const mod = await import('better-sqlite3');
1441
- const Ctor = mod.default ?? mod;
1442
- const db = new Ctor(filePath);
1443
- return db;
1444
- } catch (cause) {
1445
- const message = cause instanceof Error ? cause.message : String(cause);
1446
- throw new errors.ConfigurationError(
1447
- `Failed to load SQLite driver. Install \`better-sqlite3\` or run on Node 22.5+ for built-in \`node:sqlite\`. Cause: ${message}`,
1448
- { code: "sqlite_driver_unavailable", cause }
1449
- );
1614
+ entries = await promises.readdir(sessionsDir(cwd));
1615
+ } catch {
1616
+ return [];
1450
1617
  }
1618
+ const root = memoryDir(cwd);
1619
+ return entries.filter((entry) => entry.endsWith(".md")).map((entry) => {
1620
+ const absolutePath = path.join(sessionsDir(cwd), entry);
1621
+ return {
1622
+ absolutePath,
1623
+ relPath: relativeToRoot(root, absolutePath)
1624
+ };
1625
+ });
1451
1626
  }
1452
- function isCorruptionError(cause) {
1453
- if (!(cause instanceof Error)) return false;
1454
- const msg = cause.message.toLowerCase();
1455
- return msg.includes("malformed") || msg.includes("not a database") || msg.includes("encrypted") || msg.includes("disk image is malformed");
1627
+ function relativeToRoot(root, absolutePath) {
1628
+ if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
1629
+ return absolutePath;
1456
1630
  }
1457
- async function renameAside(filePath) {
1458
- const asidePath = `${filePath}.corrupt-${Date.now()}`;
1459
- await promises.rename(filePath, asidePath).catch(() => void 0);
1460
- await promises.rename(`${filePath}-wal`, `${asidePath}-wal`).catch(() => void 0);
1461
- await promises.rename(`${filePath}-shm`, `${asidePath}-shm`).catch(() => void 0);
1462
- process.stderr.write(
1463
- `[theokit-sdk] memory index corrupt; renamed aside to ${asidePath} and rebuilt schema
1464
- `
1465
- );
1631
+ function wikiDir(cwd) {
1632
+ return path.join(memoryDir(cwd), "wiki");
1466
1633
  }
1467
- function defaultIndexPath(cwd) {
1468
- return path.join(cwd, ".theokit", "memory", ".index", "memory.sqlite");
1634
+ async function discoverWikiFiles(cwd) {
1635
+ let entries;
1636
+ try {
1637
+ entries = await promises.readdir(wikiDir(cwd));
1638
+ } catch {
1639
+ return [];
1640
+ }
1641
+ const root = memoryDir(cwd);
1642
+ return entries.filter((entry) => entry.endsWith(".md")).map((entry) => ({
1643
+ absolutePath: path.join(wikiDir(cwd), entry),
1644
+ relPath: path.join("wiki", entry)
1645
+ })).map((file) => ({
1646
+ absolutePath: file.absolutePath,
1647
+ relPath: relativeToRoot2(root, file.absolutePath)
1648
+ }));
1649
+ }
1650
+ function relativeToRoot2(root, absolutePath) {
1651
+ if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
1652
+ return absolutePath;
1469
1653
  }
1470
1654
  function requireLance() {
1471
1655
  try {
@@ -1595,7 +1779,7 @@ function lanceStoragePath(cwd) {
1595
1779
  return path.join(cwd, ".theokit", "memory", "lance");
1596
1780
  }
1597
1781
 
1598
- // src/internal/memory-index.ts
1782
+ // src/internal/index/memory-index.ts
1599
1783
  function parseSearchOptions(options = {}) {
1600
1784
  return {
1601
1785
  maxResults: Math.max(1, options.maxResults ?? 10),
@@ -1603,7 +1787,7 @@ function parseSearchOptions(options = {}) {
1603
1787
  };
1604
1788
  }
1605
1789
 
1606
- // src/internal/lance-memory-adapter.ts
1790
+ // src/internal/index/lance-memory-adapter.ts
1607
1791
  var DEFAULT_NAMESPACE = "default";
1608
1792
  var EMPTY_SYNC_RESULT = Object.freeze({
1609
1793
  filesScanned: 0,
@@ -1668,7 +1852,7 @@ function translateLanceHit(hit) {
1668
1852
  };
1669
1853
  }
1670
1854
 
1671
- // src/internal/index-manager-dispatch.ts
1855
+ // src/internal/index/index-manager-dispatch.ts
1672
1856
  var VALID_BACKENDS = ["sqlite-vec", "lance"];
1673
1857
  function assertValidBackend(backend) {
1674
1858
  if (!VALID_BACKENDS.includes(backend)) {
@@ -1692,68 +1876,6 @@ async function openLanceIndex(opts) {
1692
1876
  });
1693
1877
  return new LanceMemoryAdapter(lance);
1694
1878
  }
1695
- var MAX_TURN_CHARS = 2e3;
1696
- function sessionsDir(cwd) {
1697
- return path.join(memoryDir(cwd), "sessions");
1698
- }
1699
- function sessionSummaryPath(cwd, runId) {
1700
- return path.join(sessionsDir(cwd), `${sanitizeRunId2(runId)}.md`);
1701
- }
1702
- function sanitizeRunId2(runId) {
1703
- return runId.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 128);
1704
- }
1705
- function truncate2(text) {
1706
- if (text.length <= MAX_TURN_CHARS) return text;
1707
- return `${text.slice(0, MAX_TURN_CHARS)}\u2026`;
1708
- }
1709
- async function writeSessionSummary(input) {
1710
- if (input.status !== "finished") return;
1711
- const path = sessionSummaryPath(input.cwd, input.runId);
1712
- await promises.mkdir(sessionsDir(input.cwd), { recursive: true });
1713
- const safeUser = redactSecrets(truncate2(input.userText));
1714
- const safeAssistant = redactSecrets(truncate2(input.assistantText));
1715
- const iso = new Date(input.at).toISOString();
1716
- const body = [
1717
- "---",
1718
- `runId: ${input.runId}`,
1719
- `agentId: ${input.agentId}`,
1720
- `at: ${iso}`,
1721
- `status: ${input.status}`,
1722
- "---",
1723
- "",
1724
- "## User",
1725
- "",
1726
- safeUser,
1727
- "",
1728
- "## Assistant",
1729
- "",
1730
- safeAssistant,
1731
- ""
1732
- ].join("\n");
1733
- await persistence.replaceFileAtomic(path, body);
1734
- }
1735
-
1736
- // src/internal/session-loader.ts
1737
- async function discoverSessionFiles(cwd) {
1738
- let entries;
1739
- try {
1740
- entries = await promises.readdir(sessionsDir(cwd));
1741
- } catch {
1742
- return [];
1743
- }
1744
- const root = memoryDir(cwd);
1745
- return entries.filter((entry) => entry.endsWith(".md")).map((entry) => {
1746
- const absolutePath = path.join(sessionsDir(cwd), entry);
1747
- return {
1748
- absolutePath,
1749
- relPath: relativeToRoot(root, absolutePath)
1750
- };
1751
- });
1752
- }
1753
- function relativeToRoot(root, absolutePath) {
1754
- if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
1755
- return absolutePath;
1756
- }
1757
1879
  async function loadSqliteVecExtension(db) {
1758
1880
  try {
1759
1881
  const mod = await import('sqlite-vec');
@@ -1779,7 +1901,7 @@ function isSqliteVecLoaded(db) {
1779
1901
  }
1780
1902
  }
1781
1903
 
1782
- // src/internal/vec-index.ts
1904
+ // src/internal/index/vec-index.ts
1783
1905
  var META_KEY_PROVIDER_ID = "embedding.providerId";
1784
1906
  var META_KEY_MODEL = "embedding.model";
1785
1907
  var META_KEY_DIMENSION = "embedding.dimension";
@@ -1852,31 +1974,8 @@ async function embedMissingChunks(args) {
1852
1974
  }
1853
1975
  return normalized.length;
1854
1976
  }
1855
- function wikiDir(cwd) {
1856
- return path.join(memoryDir(cwd), "wiki");
1857
- }
1858
- async function discoverWikiFiles(cwd) {
1859
- let entries;
1860
- try {
1861
- entries = await promises.readdir(wikiDir(cwd));
1862
- } catch {
1863
- return [];
1864
- }
1865
- const root = memoryDir(cwd);
1866
- return entries.filter((entry) => entry.endsWith(".md")).map((entry) => ({
1867
- absolutePath: path.join(wikiDir(cwd), entry),
1868
- relPath: path.join("wiki", entry)
1869
- })).map((file) => ({
1870
- absolutePath: file.absolutePath,
1871
- relPath: relativeToRoot2(root, file.absolutePath)
1872
- }));
1873
- }
1874
- function relativeToRoot2(root, absolutePath) {
1875
- if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
1876
- return absolutePath;
1877
- }
1878
1977
 
1879
- // src/internal/index-manager.ts
1978
+ // src/internal/index/index-manager.ts
1880
1979
  var IndexManager = class _IndexManager {
1881
1980
  constructor(cwd, db, embedding) {
1882
1981
  this.cwd = cwd;
@@ -2524,6 +2623,7 @@ exports.appendFactToMarkdown = appendFactToMarkdown;
2524
2623
  exports.assertValidBackend = assertValidBackend;
2525
2624
  exports.buildErrorMetadata = buildErrorMetadata;
2526
2625
  exports.chunkMarkdown = chunkMarkdown;
2626
+ exports.createCategorizedMemory = createCategorizedMemory;
2527
2627
  exports.createInMemoryMarkdownProvider = createInMemoryMarkdownProvider;
2528
2628
  exports.createMemoryGetTool = createMemoryGetTool;
2529
2629
  exports.createMemorySearchTool = createMemorySearchTool;