@theokit/sdk-memory 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/CHANGELOG.md +22 -1
  2. package/dist/index.cjs +297 -207
  3. package/dist/index.cjs.map +1 -1
  4. package/dist/index.d.cts +34 -33
  5. package/dist/index.d.ts +34 -33
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js +300 -211
  8. package/dist/index.js.map +1 -1
  9. package/dist/internal/active-memory/active-memory-cache.d.ts.map +1 -0
  10. package/dist/internal/{active-memory-types.d.ts → active-memory/active-memory-types.d.ts} +1 -1
  11. package/dist/internal/active-memory/active-memory-types.d.ts.map +1 -0
  12. package/dist/internal/{active-memory.d.ts → active-memory/active-memory.d.ts} +2 -2
  13. package/dist/internal/active-memory/active-memory.d.ts.map +1 -0
  14. package/dist/internal/active-memory/composite-scorer.d.ts.map +1 -0
  15. package/dist/internal/active-memory/query-analyzer.d.ts.map +1 -0
  16. package/dist/internal/adapter-catalog.d.ts +6 -6
  17. package/dist/internal/categorized-memory.d.ts +56 -0
  18. package/dist/internal/categorized-memory.d.ts.map +1 -0
  19. package/dist/internal/dreaming/dreaming-diary.d.ts.map +1 -0
  20. package/dist/internal/{dreaming-phases.d.ts → dreaming/dreaming-phases.d.ts} +2 -2
  21. package/dist/internal/dreaming/dreaming-phases.d.ts.map +1 -0
  22. package/dist/internal/{dreaming-run.d.ts → dreaming/dreaming-run.d.ts} +1 -1
  23. package/dist/internal/dreaming/dreaming-run.d.ts.map +1 -0
  24. package/dist/internal/embedding/batch-encoder.d.ts.map +1 -0
  25. package/dist/internal/embedding/deepinfra-embedding.d.ts.map +1 -0
  26. package/dist/internal/embedding/embedding-adapter.d.ts.map +1 -0
  27. package/dist/internal/embedding/embedding-cache.d.ts.map +1 -0
  28. package/dist/internal/embedding/mistral-embedding.d.ts.map +1 -0
  29. package/dist/internal/embedding/ollama-embedding.d.ts.map +1 -0
  30. package/dist/internal/embedding/openai-compatible.d.ts.map +1 -0
  31. package/dist/internal/embedding/openai-embedding.d.ts.map +1 -0
  32. package/dist/internal/embedding/openrouter-embedding.d.ts.map +1 -0
  33. package/dist/internal/embedding/voyage-embedding.d.ts.map +1 -0
  34. package/dist/internal/index/index-db.d.ts.map +1 -0
  35. package/dist/internal/{index-manager-contract.d.ts → index/index-manager-contract.d.ts} +1 -1
  36. package/dist/internal/index/index-manager-contract.d.ts.map +1 -0
  37. package/dist/internal/index/index-manager-dispatch.d.ts.map +1 -0
  38. package/dist/internal/{index-manager.d.ts → index/index-manager.d.ts} +1 -1
  39. package/dist/internal/index/index-manager.d.ts.map +1 -0
  40. package/dist/internal/index/index-schema.d.ts.map +1 -0
  41. package/dist/internal/{lance-index.d.ts → index/lance-index.d.ts} +1 -1
  42. package/dist/internal/index/lance-index.d.ts.map +1 -0
  43. package/dist/internal/index/lance-memory-adapter.d.ts.map +1 -0
  44. package/dist/internal/index/memory-index.d.ts.map +1 -0
  45. package/dist/internal/index/migrate-sqlite-to-lance.d.ts.map +1 -0
  46. package/dist/internal/{migration.d.ts → index/migration.d.ts} +1 -1
  47. package/dist/internal/index/migration.d.ts.map +1 -0
  48. package/dist/internal/index/sqlite-vec-loader.d.ts.map +1 -0
  49. package/dist/internal/{vec-index.d.ts → index/vec-index.d.ts} +1 -1
  50. package/dist/internal/index/vec-index.d.ts.map +1 -0
  51. package/dist/internal/memory-types.d.ts +5 -0
  52. package/dist/internal/memory-types.d.ts.map +1 -1
  53. package/dist/internal/store/chunk-markdown.d.ts.map +1 -0
  54. package/dist/internal/{markdown-store.d.ts → store/markdown-store.d.ts} +1 -1
  55. package/dist/internal/store/markdown-store.d.ts.map +1 -0
  56. package/dist/internal/store/reader.d.ts.map +1 -0
  57. package/dist/internal/store/session-loader.d.ts.map +1 -0
  58. package/dist/internal/store/session-summary-writer.d.ts.map +1 -0
  59. package/dist/internal/store/transcript-store.d.ts.map +1 -0
  60. package/dist/internal/store/wiki-loader.d.ts.map +1 -0
  61. package/dist/internal/tools.d.ts +1 -1
  62. package/dist/internal/tools.d.ts.map +1 -1
  63. package/package.json +9 -9
  64. package/dist/internal/active-memory-cache.d.ts.map +0 -1
  65. package/dist/internal/active-memory-types.d.ts.map +0 -1
  66. package/dist/internal/active-memory.d.ts.map +0 -1
  67. package/dist/internal/batch-encoder.d.ts.map +0 -1
  68. package/dist/internal/chunk-markdown.d.ts.map +0 -1
  69. package/dist/internal/composite-scorer.d.ts.map +0 -1
  70. package/dist/internal/deepinfra-embedding.d.ts.map +0 -1
  71. package/dist/internal/dreaming-diary.d.ts.map +0 -1
  72. package/dist/internal/dreaming-phases.d.ts.map +0 -1
  73. package/dist/internal/dreaming-run.d.ts.map +0 -1
  74. package/dist/internal/embedding-adapter.d.ts.map +0 -1
  75. package/dist/internal/embedding-cache.d.ts.map +0 -1
  76. package/dist/internal/index-db.d.ts.map +0 -1
  77. package/dist/internal/index-manager-contract.d.ts.map +0 -1
  78. package/dist/internal/index-manager-dispatch.d.ts.map +0 -1
  79. package/dist/internal/index-manager.d.ts.map +0 -1
  80. package/dist/internal/index-schema.d.ts.map +0 -1
  81. package/dist/internal/lance-index.d.ts.map +0 -1
  82. package/dist/internal/lance-memory-adapter.d.ts.map +0 -1
  83. package/dist/internal/markdown-store.d.ts.map +0 -1
  84. package/dist/internal/memory-index.d.ts.map +0 -1
  85. package/dist/internal/migrate-sqlite-to-lance.d.ts.map +0 -1
  86. package/dist/internal/migration.d.ts.map +0 -1
  87. package/dist/internal/mistral-embedding.d.ts.map +0 -1
  88. package/dist/internal/ollama-embedding.d.ts.map +0 -1
  89. package/dist/internal/openai-compatible.d.ts.map +0 -1
  90. package/dist/internal/openai-embedding.d.ts.map +0 -1
  91. package/dist/internal/openrouter-embedding.d.ts.map +0 -1
  92. package/dist/internal/query-analyzer.d.ts.map +0 -1
  93. package/dist/internal/reader.d.ts.map +0 -1
  94. package/dist/internal/session-loader.d.ts.map +0 -1
  95. package/dist/internal/session-summary-writer.d.ts.map +0 -1
  96. package/dist/internal/sqlite-vec-loader.d.ts.map +0 -1
  97. package/dist/internal/transcript-store.d.ts.map +0 -1
  98. package/dist/internal/vec-index.d.ts.map +0 -1
  99. package/dist/internal/voyage-embedding.d.ts.map +0 -1
  100. package/dist/internal/wiki-loader.d.ts.map +0 -1
  101. /package/dist/internal/{active-memory-cache.d.ts → active-memory/active-memory-cache.d.ts} +0 -0
  102. /package/dist/internal/{composite-scorer.d.ts → active-memory/composite-scorer.d.ts} +0 -0
  103. /package/dist/internal/{query-analyzer.d.ts → active-memory/query-analyzer.d.ts} +0 -0
  104. /package/dist/internal/{dreaming-diary.d.ts → dreaming/dreaming-diary.d.ts} +0 -0
  105. /package/dist/internal/{batch-encoder.d.ts → embedding/batch-encoder.d.ts} +0 -0
  106. /package/dist/internal/{deepinfra-embedding.d.ts → embedding/deepinfra-embedding.d.ts} +0 -0
  107. /package/dist/internal/{embedding-adapter.d.ts → embedding/embedding-adapter.d.ts} +0 -0
  108. /package/dist/internal/{embedding-cache.d.ts → embedding/embedding-cache.d.ts} +0 -0
  109. /package/dist/internal/{mistral-embedding.d.ts → embedding/mistral-embedding.d.ts} +0 -0
  110. /package/dist/internal/{ollama-embedding.d.ts → embedding/ollama-embedding.d.ts} +0 -0
  111. /package/dist/internal/{openai-compatible.d.ts → embedding/openai-compatible.d.ts} +0 -0
  112. /package/dist/internal/{openai-embedding.d.ts → embedding/openai-embedding.d.ts} +0 -0
  113. /package/dist/internal/{openrouter-embedding.d.ts → embedding/openrouter-embedding.d.ts} +0 -0
  114. /package/dist/internal/{voyage-embedding.d.ts → embedding/voyage-embedding.d.ts} +0 -0
  115. /package/dist/internal/{index-db.d.ts → index/index-db.d.ts} +0 -0
  116. /package/dist/internal/{index-manager-dispatch.d.ts → index/index-manager-dispatch.d.ts} +0 -0
  117. /package/dist/internal/{index-schema.d.ts → index/index-schema.d.ts} +0 -0
  118. /package/dist/internal/{lance-memory-adapter.d.ts → index/lance-memory-adapter.d.ts} +0 -0
  119. /package/dist/internal/{memory-index.d.ts → index/memory-index.d.ts} +0 -0
  120. /package/dist/internal/{migrate-sqlite-to-lance.d.ts → index/migrate-sqlite-to-lance.d.ts} +0 -0
  121. /package/dist/internal/{sqlite-vec-loader.d.ts → index/sqlite-vec-loader.d.ts} +0 -0
  122. /package/dist/internal/{chunk-markdown.d.ts → store/chunk-markdown.d.ts} +0 -0
  123. /package/dist/internal/{reader.d.ts → store/reader.d.ts} +0 -0
  124. /package/dist/internal/{session-loader.d.ts → store/session-loader.d.ts} +0 -0
  125. /package/dist/internal/{session-summary-writer.d.ts → store/session-summary-writer.d.ts} +0 -0
  126. /package/dist/internal/{transcript-store.d.ts → store/transcript-store.d.ts} +0 -0
  127. /package/dist/internal/{wiki-loader.d.ts → store/wiki-loader.d.ts} +0 -0
package/dist/index.cjs CHANGED
@@ -252,7 +252,7 @@ function legacyMemoryJsonPath(cwd, config) {
252
252
  return pathSafety.safePathJoin(cwd, ".theokit", "memory", namespace, `${scope}-${userId}.json`);
253
253
  }
254
254
 
255
- // src/internal/markdown-store.ts
255
+ // src/internal/store/markdown-store.ts
256
256
  var MEMORY_MD_HEADER = "# Memory\n\n> Auto-managed by @theokit/sdk. Edit freely \u2014 the SDK reads from here.\n";
257
257
  var FACTS_HEADING = "## Facts";
258
258
  function memoryDir(cwd) {
@@ -343,7 +343,7 @@ async function appendFact(cwd, config, fact) {
343
343
  await appendFactToMarkdown(cwd, fact);
344
344
  }
345
345
 
346
- // src/internal/transcript-store.ts
346
+ // src/internal/store/transcript-store.ts
347
347
  async function persistActiveMemoryTranscript(cwd, transcript) {
348
348
  try {
349
349
  const dir = path.join(memoryDir(cwd), "transcripts", "active-memory");
@@ -356,7 +356,7 @@ async function persistActiveMemoryTranscript(cwd, transcript) {
356
356
  }
357
357
  }
358
358
 
359
- // src/internal/active-memory.ts
359
+ // src/internal/active-memory/active-memory.ts
360
360
  var SPAN_MEMORY_RECALL = "memory.recall";
361
361
  var HISTOGRAM_MEMORY_RECALL_DURATION_MS = "theokit_memory_recall_duration_ms";
362
362
  var DEFAULTS = {
@@ -705,7 +705,7 @@ function formatMessage(providerId, status, code) {
705
705
  return `${providerId} API error: ${code} (HTTP ${status})`;
706
706
  }
707
707
 
708
- // src/internal/embedding-cache.ts
708
+ // src/internal/embedding/embedding-cache.ts
709
709
  var LruEmbeddingCache = class {
710
710
  constructor(capacity = 5e3) {
711
711
  this.capacity = capacity;
@@ -732,7 +732,7 @@ var LruEmbeddingCache = class {
732
732
  }
733
733
  };
734
734
 
735
- // src/internal/openai-compatible.ts
735
+ // src/internal/embedding/openai-compatible.ts
736
736
  var MAX_BATCH = 100;
737
737
  var MAX_RETRIES = 2;
738
738
  async function createOpenAiCompatibleRuntime(cfg, options) {
@@ -903,7 +903,7 @@ function sleep(ms) {
903
903
  return new Promise((resolve) => setTimeout(resolve, ms));
904
904
  }
905
905
 
906
- // src/internal/deepinfra-embedding.ts
906
+ // src/internal/embedding/deepinfra-embedding.ts
907
907
  var DEFAULT_DEEPINFRA_EMBEDDING_MODEL = "BAAI/bge-large-en-v1.5";
908
908
  var DIMENSION_BY_MODEL = {
909
909
  "BAAI/bge-large-en-v1.5": 1024,
@@ -937,7 +937,7 @@ var deepinfraMemoryEmbeddingProviderAdapter = {
937
937
  )
938
938
  };
939
939
 
940
- // src/internal/mistral-embedding.ts
940
+ // src/internal/embedding/mistral-embedding.ts
941
941
  var DEFAULT_MISTRAL_EMBEDDING_MODEL = "mistral-embed";
942
942
  var DIMENSION_BY_MODEL2 = {
943
943
  "mistral-embed": 1024
@@ -961,7 +961,7 @@ var mistralMemoryEmbeddingProviderAdapter = {
961
961
  )
962
962
  };
963
963
 
964
- // src/internal/ollama-embedding.ts
964
+ // src/internal/embedding/ollama-embedding.ts
965
965
  var DEFAULT_OLLAMA_EMBEDDING_MODEL = "nomic-embed-text";
966
966
  var OLLAMA_LOCAL_SENTINEL_KEY = "ollama-local";
967
967
  var DIMENSION_BY_MODEL3 = {
@@ -1004,7 +1004,7 @@ var ollamaMemoryEmbeddingProviderAdapter = {
1004
1004
  }
1005
1005
  };
1006
1006
 
1007
- // src/internal/openai-embedding.ts
1007
+ // src/internal/embedding/openai-embedding.ts
1008
1008
  var DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
1009
1009
  var DIMENSION_BY_MODEL4 = {
1010
1010
  "text-embedding-3-small": 1536,
@@ -1030,7 +1030,7 @@ var openAiMemoryEmbeddingProviderAdapter = {
1030
1030
  )
1031
1031
  };
1032
1032
 
1033
- // src/internal/openrouter-embedding.ts
1033
+ // src/internal/embedding/openrouter-embedding.ts
1034
1034
  var DEFAULT_OPENROUTER_EMBEDDING_MODEL = "openai/text-embedding-3-small";
1035
1035
  var DIMENSION_BY_MODEL5 = {
1036
1036
  "openai/text-embedding-3-small": 1536,
@@ -1057,7 +1057,7 @@ var openRouterMemoryEmbeddingProviderAdapter = {
1057
1057
  )
1058
1058
  };
1059
1059
 
1060
- // src/internal/voyage-embedding.ts
1060
+ // src/internal/embedding/voyage-embedding.ts
1061
1061
  var DEFAULT_VOYAGE_EMBEDDING_MODEL = "voyage-3-lite";
1062
1062
  var DIMENSION_BY_MODEL6 = {
1063
1063
  "voyage-3-lite": 512,
@@ -1094,80 +1094,131 @@ var MEMORY_EMBEDDING_ADAPTERS = {
1094
1094
  deepinfra: deepinfraMemoryEmbeddingProviderAdapter,
1095
1095
  ollama: ollamaMemoryEmbeddingProviderAdapter
1096
1096
  };
1097
- var HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
1098
- function chunkMarkdown(text, options = {}) {
1099
- const maxChars = options.maxChars ?? 800;
1100
- if (text.length === 0) return [];
1101
- const lines = text.split("\n");
1102
- const acc = { chunks: [], lines: [], start: 1, heading: void 0 };
1103
- for (let i = 0; i < lines.length; i++) {
1104
- const line = lines[i] ?? "";
1105
- const lineNumber = i + 1;
1106
- processLine(acc, line, lineNumber, maxChars);
1097
+ var FACTS_HEADING2 = "## Facts";
1098
+ function createCategorizedMemory(options) {
1099
+ const { root, categories } = options;
1100
+ validateCategories(categories);
1101
+ const allowed = new Set(categories);
1102
+ function assertKnown(category) {
1103
+ if (!allowed.has(category)) {
1104
+ throw new errors.ConfigurationError(
1105
+ `createCategorizedMemory: unknown category ${JSON.stringify(category)} (declared: ${categories.join(", ")})`,
1106
+ { code: "unknown_category" }
1107
+ );
1108
+ }
1107
1109
  }
1108
- flushAccumulator(acc, lines.length, maxChars);
1109
- return acc.chunks;
1110
- }
1111
- function processLine(acc, line, lineNumber, maxChars) {
1112
- const headingMatch = HEADING_RE.exec(line);
1113
- const isBlank = line.trim().length === 0;
1114
- if (headingMatch !== null && acc.lines.length > 0) {
1115
- flushAccumulator(acc, lineNumber - 1, maxChars);
1110
+ function categoryPath(category) {
1111
+ return pathSafety.safePathJoin(root, `${pathSafety.sanitizeIdentifier(category)}.md`);
1112
+ }
1113
+ async function add(category, text) {
1114
+ assertKnown(category);
1115
+ const encoded = encodeFact(redactSecrets(text));
1116
+ const sane = pathSafety.sanitizeIdentifier(category);
1117
+ await persistence.withCwdMutex(`catmem:${root}:${sane}`, async () => {
1118
+ const path = pathSafety.safePathJoin(root, `${sane}.md`);
1119
+ const raw = await readFileOrEmpty(path) || header(category);
1120
+ await promises.mkdir(root, { recursive: true });
1121
+ await persistence.replaceFileAtomic(path, appendBullet(raw, encoded));
1122
+ });
1116
1123
  }
1117
- if (headingMatch !== null) acc.heading = headingMatch[2];
1118
- acc.lines.push(line);
1119
- if (isBlank && acc.lines.length > 1) {
1120
- const joined = acc.lines.join("\n");
1121
- if (joined.trim().length > 0) flushAccumulator(acc, lineNumber, maxChars);
1124
+ async function list(category) {
1125
+ if (category !== void 0) {
1126
+ assertKnown(category);
1127
+ return readCategory(categoryPath(category), category);
1128
+ }
1129
+ const out = [];
1130
+ for (const c of categories) {
1131
+ out.push(...await readCategory(categoryPath(c), c));
1132
+ }
1133
+ return out;
1122
1134
  }
1135
+ return { categories: [...categories], add, list };
1123
1136
  }
1124
- function flushAccumulator(acc, endLineExclusive, maxChars) {
1125
- if (acc.lines.length === 0) return;
1126
- const chunkText = acc.lines.join("\n");
1127
- if (chunkText.trim().length === 0) {
1128
- acc.lines = [];
1129
- acc.start = endLineExclusive + 1;
1130
- return;
1137
+ function validateCategories(categories) {
1138
+ if (categories.length === 0) {
1139
+ throw new errors.ConfigurationError("createCategorizedMemory: categories must be non-empty", {
1140
+ code: "invalid_categories"
1141
+ });
1131
1142
  }
1132
- if (chunkText.length > maxChars) {
1133
- pushOversizedSlices(acc.chunks, chunkText, acc.start, acc.heading, maxChars);
1134
- } else {
1135
- acc.chunks.push(buildChunk(chunkText, acc.start, endLineExclusive, acc.heading));
1143
+ if (new Set(categories).size !== categories.length) {
1144
+ throw new errors.ConfigurationError("createCategorizedMemory: categories must be unique", {
1145
+ code: "invalid_categories"
1146
+ });
1147
+ }
1148
+ const sanitized = categories.map((c) => trySanitize(c));
1149
+ if (new Set(sanitized).size !== categories.length) {
1150
+ throw new errors.ConfigurationError(
1151
+ "createCategorizedMemory: categories must map to distinct filenames after sanitization",
1152
+ { code: "invalid_categories" }
1153
+ );
1136
1154
  }
1137
- acc.lines = [];
1138
- acc.start = endLineExclusive + 1;
1139
- }
1140
- function buildChunk(text, startLine, endLine, heading) {
1141
- return {
1142
- startLine,
1143
- endLine,
1144
- text,
1145
- hash: crypto.createHash("sha256").update(text).digest("hex"),
1146
- ...heading !== void 0 ? { heading } : {}
1147
- };
1148
1155
  }
1149
- function pushOversizedSlices(chunks, text, startLine, heading, maxChars) {
1150
- let remaining = text;
1151
- let lineCursor = startLine;
1152
- while (remaining.length > maxChars) {
1153
- const splitAt = findWordBoundarySplit(remaining, maxChars);
1154
- const slice = remaining.slice(0, splitAt);
1155
- const sliceLines = slice.split("\n").length;
1156
- chunks.push(buildChunk(slice, lineCursor, lineCursor + sliceLines - 1, heading));
1157
- remaining = remaining.slice(splitAt).replace(/^\s+/, "");
1158
- lineCursor += sliceLines - 1;
1156
+ function trySanitize(category) {
1157
+ try {
1158
+ return pathSafety.sanitizeIdentifier(category);
1159
+ } catch (cause) {
1160
+ throw new errors.ConfigurationError(
1161
+ `createCategorizedMemory: category ${JSON.stringify(category)} is not a valid identifier`,
1162
+ { code: "invalid_categories", cause }
1163
+ );
1159
1164
  }
1160
- if (remaining.length > 0) {
1161
- const sliceLines = remaining.split("\n").length;
1162
- chunks.push(buildChunk(remaining, lineCursor, lineCursor + sliceLines - 1, heading));
1165
+ }
1166
+ function header(category) {
1167
+ return `---
1168
+ category: ${category}
1169
+ ---
1170
+
1171
+ ${FACTS_HEADING2}
1172
+ `;
1173
+ }
1174
+ async function readFileOrEmpty(path) {
1175
+ try {
1176
+ return await promises.readFile(path, "utf8");
1177
+ } catch {
1178
+ return "";
1163
1179
  }
1164
1180
  }
1165
- function findWordBoundarySplit(text, maxChars) {
1166
- if (text.length <= maxChars) return text.length;
1167
- for (let i = maxChars; i > Math.max(0, maxChars - 200); i--) {
1168
- if (/\s/.test(text[i] ?? "x")) return i;
1181
+ async function readCategory(path, category) {
1182
+ const raw = await readFileOrEmpty(path);
1183
+ return parseFactBullets(raw).map((text) => ({ text: decodeFact(text), category }));
1184
+ }
1185
+ function encodeFact(text) {
1186
+ return text.replace(/\\/g, "\\\\").replace(/\r/g, "\\r").replace(/\n/g, "\\n");
1187
+ }
1188
+ function decodeFact(text) {
1189
+ return text.replace(
1190
+ /\\(.)/g,
1191
+ (_match, char) => char === "n" ? "\n" : char === "r" ? "\r" : char
1192
+ );
1193
+ }
1194
+ function parseFactBullets(raw) {
1195
+ const idx = raw.indexOf(FACTS_HEADING2);
1196
+ if (idx === -1) return [];
1197
+ const tail = raw.slice(idx + FACTS_HEADING2.length);
1198
+ const nextHeading = tail.search(/\n#{1,2}\s/);
1199
+ const block = nextHeading === -1 ? tail : tail.slice(0, nextHeading);
1200
+ return block.split("\n").map((line) => line.trim()).filter((line) => line.startsWith("- ")).map((line) => line.slice(2).trim());
1201
+ }
1202
+ function appendBullet(raw, text) {
1203
+ const bullet = `- ${text}`;
1204
+ const idx = raw.indexOf(FACTS_HEADING2);
1205
+ if (idx === -1) {
1206
+ const sep = raw.endsWith("\n") ? "" : "\n";
1207
+ return `${raw}${sep}${FACTS_HEADING2}
1208
+
1209
+ ${bullet}
1210
+ `;
1169
1211
  }
1170
- return maxChars;
1212
+ const after = idx + FACTS_HEADING2.length;
1213
+ const nextHeading = raw.slice(after).search(/\n#{1,2}\s/);
1214
+ if (nextHeading === -1) {
1215
+ const trailing = raw.endsWith("\n") ? "" : "\n";
1216
+ return `${raw}${trailing}${bullet}
1217
+ `;
1218
+ }
1219
+ const insertAt = after + nextHeading;
1220
+ return `${raw.slice(0, insertAt)}
1221
+ ${bullet}${raw.slice(insertAt)}`;
1171
1222
  }
1172
1223
  function diaryPath(cwd) {
1173
1224
  return path.join(memoryDir(cwd), "dream-diary.md");
@@ -1211,7 +1262,7 @@ function entryHash(entry) {
1211
1262
  ).digest("hex");
1212
1263
  }
1213
1264
 
1214
- // src/internal/dreaming-phases.ts
1265
+ // src/internal/dreaming/dreaming-phases.ts
1215
1266
  var DEFAULT_DEDUP_THRESHOLD = 0.95;
1216
1267
  var DEFAULT_CLUSTER_THRESHOLD = 0.75;
1217
1268
  async function lightPhase(facts, embedding, threshold = DEFAULT_DEDUP_THRESHOLD) {
@@ -1369,7 +1420,7 @@ function emptyResult(status) {
1369
1420
  };
1370
1421
  }
1371
1422
 
1372
- // src/internal/index-schema.ts
1423
+ // src/internal/index/index-schema.ts
1373
1424
  var SCHEMA_STATEMENTS = [
1374
1425
  `CREATE TABLE IF NOT EXISTS files (
1375
1426
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -1415,57 +1466,180 @@ var PRAGMA_STATEMENTS = [
1415
1466
  "PRAGMA foreign_keys=ON"
1416
1467
  ];
1417
1468
 
1418
- // src/internal/index-db.ts
1469
+ // src/internal/index/index-db.ts
1419
1470
  async function openMemoryDb(opts) {
1420
- await promises.mkdir(path.dirname(opts.filePath), { recursive: true });
1421
- try {
1422
- return await openConcrete(opts.filePath);
1423
- } catch (cause) {
1424
- if (opts.recoverCorrupt !== false && isCorruptionError(cause)) {
1425
- await renameAside(opts.filePath);
1426
- return await openConcrete(opts.filePath);
1471
+ return persistence.openSqliteResilient({
1472
+ filePath: opts.filePath,
1473
+ label: "memory-index",
1474
+ recoverCorrupt: opts.recoverCorrupt,
1475
+ onOpen: (db) => {
1476
+ for (const pragma of PRAGMA_STATEMENTS) db.exec(pragma);
1477
+ for (const stmt of SCHEMA_STATEMENTS) db.exec(stmt);
1427
1478
  }
1428
- throw cause;
1479
+ });
1480
+ }
1481
+ function defaultIndexPath(cwd) {
1482
+ return path.join(cwd, ".theokit", "memory", ".index", "memory.sqlite");
1483
+ }
1484
+ var HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
1485
+ function chunkMarkdown(text, options = {}) {
1486
+ const maxChars = options.maxChars ?? 800;
1487
+ if (text.length === 0) return [];
1488
+ const lines = text.split("\n");
1489
+ const acc = { chunks: [], lines: [], start: 1, heading: void 0 };
1490
+ for (let i = 0; i < lines.length; i++) {
1491
+ const line = lines[i] ?? "";
1492
+ const lineNumber = i + 1;
1493
+ processLine(acc, line, lineNumber, maxChars);
1429
1494
  }
1495
+ flushAccumulator(acc, lines.length, maxChars);
1496
+ return acc.chunks;
1430
1497
  }
1431
- async function openConcrete(filePath) {
1432
- const db = await loadDriver(filePath);
1433
- persistence.applyWalWithFallback(db, "memory-index");
1434
- for (const pragma of PRAGMA_STATEMENTS) db.exec(pragma);
1435
- for (const stmt of SCHEMA_STATEMENTS) db.exec(stmt);
1436
- return db;
1498
+ function processLine(acc, line, lineNumber, maxChars) {
1499
+ const headingMatch = HEADING_RE.exec(line);
1500
+ const isBlank = line.trim().length === 0;
1501
+ if (headingMatch !== null && acc.lines.length > 0) {
1502
+ flushAccumulator(acc, lineNumber - 1, maxChars);
1503
+ }
1504
+ if (headingMatch !== null) acc.heading = headingMatch[2];
1505
+ acc.lines.push(line);
1506
+ if (isBlank && acc.lines.length > 1) {
1507
+ const joined = acc.lines.join("\n");
1508
+ if (joined.trim().length > 0) flushAccumulator(acc, lineNumber, maxChars);
1509
+ }
1510
+ }
1511
+ function flushAccumulator(acc, endLineExclusive, maxChars) {
1512
+ if (acc.lines.length === 0) return;
1513
+ const chunkText = acc.lines.join("\n");
1514
+ if (chunkText.trim().length === 0) {
1515
+ acc.lines = [];
1516
+ acc.start = endLineExclusive + 1;
1517
+ return;
1518
+ }
1519
+ if (chunkText.length > maxChars) {
1520
+ pushOversizedSlices(acc.chunks, chunkText, acc.start, acc.heading, maxChars);
1521
+ } else {
1522
+ acc.chunks.push(buildChunk(chunkText, acc.start, endLineExclusive, acc.heading));
1523
+ }
1524
+ acc.lines = [];
1525
+ acc.start = endLineExclusive + 1;
1526
+ }
1527
+ function buildChunk(text, startLine, endLine, heading) {
1528
+ return {
1529
+ startLine,
1530
+ endLine,
1531
+ text,
1532
+ hash: crypto.createHash("sha256").update(text).digest("hex"),
1533
+ ...heading !== void 0 ? { heading } : {}
1534
+ };
1437
1535
  }
1438
- async function loadDriver(filePath) {
1536
+ function pushOversizedSlices(chunks, text, startLine, heading, maxChars) {
1537
+ let remaining = text;
1538
+ let lineCursor = startLine;
1539
+ while (remaining.length > maxChars) {
1540
+ const splitAt = findWordBoundarySplit(remaining, maxChars);
1541
+ const slice = remaining.slice(0, splitAt);
1542
+ const sliceLines = slice.split("\n").length;
1543
+ chunks.push(buildChunk(slice, lineCursor, lineCursor + sliceLines - 1, heading));
1544
+ remaining = remaining.slice(splitAt).replace(/^\s+/, "");
1545
+ lineCursor += sliceLines - 1;
1546
+ }
1547
+ if (remaining.length > 0) {
1548
+ const sliceLines = remaining.split("\n").length;
1549
+ chunks.push(buildChunk(remaining, lineCursor, lineCursor + sliceLines - 1, heading));
1550
+ }
1551
+ }
1552
+ function findWordBoundarySplit(text, maxChars) {
1553
+ if (text.length <= maxChars) return text.length;
1554
+ for (let i = maxChars; i > Math.max(0, maxChars - 200); i--) {
1555
+ if (/\s/.test(text[i] ?? "x")) return i;
1556
+ }
1557
+ return maxChars;
1558
+ }
1559
+ var MAX_TURN_CHARS = 2e3;
1560
+ function sessionsDir(cwd) {
1561
+ return path.join(memoryDir(cwd), "sessions");
1562
+ }
1563
+ function sessionSummaryPath(cwd, runId) {
1564
+ return path.join(sessionsDir(cwd), `${sanitizeRunId2(runId)}.md`);
1565
+ }
1566
+ function sanitizeRunId2(runId) {
1567
+ return runId.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 128);
1568
+ }
1569
+ function truncate2(text) {
1570
+ if (text.length <= MAX_TURN_CHARS) return text;
1571
+ return `${text.slice(0, MAX_TURN_CHARS)}\u2026`;
1572
+ }
1573
+ async function writeSessionSummary(input) {
1574
+ if (input.status !== "finished") return;
1575
+ const path = sessionSummaryPath(input.cwd, input.runId);
1576
+ await promises.mkdir(sessionsDir(input.cwd), { recursive: true });
1577
+ const safeUser = redactSecrets(truncate2(input.userText));
1578
+ const safeAssistant = redactSecrets(truncate2(input.assistantText));
1579
+ const iso = new Date(input.at).toISOString();
1580
+ const body = [
1581
+ "---",
1582
+ `runId: ${input.runId}`,
1583
+ `agentId: ${input.agentId}`,
1584
+ `at: ${iso}`,
1585
+ `status: ${input.status}`,
1586
+ "---",
1587
+ "",
1588
+ "## User",
1589
+ "",
1590
+ safeUser,
1591
+ "",
1592
+ "## Assistant",
1593
+ "",
1594
+ safeAssistant,
1595
+ ""
1596
+ ].join("\n");
1597
+ await persistence.replaceFileAtomic(path, body);
1598
+ }
1599
+
1600
+ // src/internal/store/session-loader.ts
1601
+ async function discoverSessionFiles(cwd) {
1602
+ let entries;
1439
1603
  try {
1440
- const mod = await import('better-sqlite3');
1441
- const Ctor = mod.default ?? mod;
1442
- const db = new Ctor(filePath);
1443
- return db;
1444
- } catch (cause) {
1445
- const message = cause instanceof Error ? cause.message : String(cause);
1446
- throw new errors.ConfigurationError(
1447
- `Failed to load SQLite driver. Install \`better-sqlite3\` or run on Node 22.5+ for built-in \`node:sqlite\`. Cause: ${message}`,
1448
- { code: "sqlite_driver_unavailable", cause }
1449
- );
1604
+ entries = await promises.readdir(sessionsDir(cwd));
1605
+ } catch {
1606
+ return [];
1450
1607
  }
1608
+ const root = memoryDir(cwd);
1609
+ return entries.filter((entry) => entry.endsWith(".md")).map((entry) => {
1610
+ const absolutePath = path.join(sessionsDir(cwd), entry);
1611
+ return {
1612
+ absolutePath,
1613
+ relPath: relativeToRoot(root, absolutePath)
1614
+ };
1615
+ });
1451
1616
  }
1452
- function isCorruptionError(cause) {
1453
- if (!(cause instanceof Error)) return false;
1454
- const msg = cause.message.toLowerCase();
1455
- return msg.includes("malformed") || msg.includes("not a database") || msg.includes("encrypted") || msg.includes("disk image is malformed");
1617
+ function relativeToRoot(root, absolutePath) {
1618
+ if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
1619
+ return absolutePath;
1456
1620
  }
1457
- async function renameAside(filePath) {
1458
- const asidePath = `${filePath}.corrupt-${Date.now()}`;
1459
- await promises.rename(filePath, asidePath).catch(() => void 0);
1460
- await promises.rename(`${filePath}-wal`, `${asidePath}-wal`).catch(() => void 0);
1461
- await promises.rename(`${filePath}-shm`, `${asidePath}-shm`).catch(() => void 0);
1462
- process.stderr.write(
1463
- `[theokit-sdk] memory index corrupt; renamed aside to ${asidePath} and rebuilt schema
1464
- `
1465
- );
1621
+ function wikiDir(cwd) {
1622
+ return path.join(memoryDir(cwd), "wiki");
1466
1623
  }
1467
- function defaultIndexPath(cwd) {
1468
- return path.join(cwd, ".theokit", "memory", ".index", "memory.sqlite");
1624
+ async function discoverWikiFiles(cwd) {
1625
+ let entries;
1626
+ try {
1627
+ entries = await promises.readdir(wikiDir(cwd));
1628
+ } catch {
1629
+ return [];
1630
+ }
1631
+ const root = memoryDir(cwd);
1632
+ return entries.filter((entry) => entry.endsWith(".md")).map((entry) => ({
1633
+ absolutePath: path.join(wikiDir(cwd), entry),
1634
+ relPath: path.join("wiki", entry)
1635
+ })).map((file) => ({
1636
+ absolutePath: file.absolutePath,
1637
+ relPath: relativeToRoot2(root, file.absolutePath)
1638
+ }));
1639
+ }
1640
+ function relativeToRoot2(root, absolutePath) {
1641
+ if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
1642
+ return absolutePath;
1469
1643
  }
1470
1644
  function requireLance() {
1471
1645
  try {
@@ -1595,7 +1769,7 @@ function lanceStoragePath(cwd) {
1595
1769
  return path.join(cwd, ".theokit", "memory", "lance");
1596
1770
  }
1597
1771
 
1598
- // src/internal/memory-index.ts
1772
+ // src/internal/index/memory-index.ts
1599
1773
  function parseSearchOptions(options = {}) {
1600
1774
  return {
1601
1775
  maxResults: Math.max(1, options.maxResults ?? 10),
@@ -1603,7 +1777,7 @@ function parseSearchOptions(options = {}) {
1603
1777
  };
1604
1778
  }
1605
1779
 
1606
- // src/internal/lance-memory-adapter.ts
1780
+ // src/internal/index/lance-memory-adapter.ts
1607
1781
  var DEFAULT_NAMESPACE = "default";
1608
1782
  var EMPTY_SYNC_RESULT = Object.freeze({
1609
1783
  filesScanned: 0,
@@ -1668,7 +1842,7 @@ function translateLanceHit(hit) {
1668
1842
  };
1669
1843
  }
1670
1844
 
1671
- // src/internal/index-manager-dispatch.ts
1845
+ // src/internal/index/index-manager-dispatch.ts
1672
1846
  var VALID_BACKENDS = ["sqlite-vec", "lance"];
1673
1847
  function assertValidBackend(backend) {
1674
1848
  if (!VALID_BACKENDS.includes(backend)) {
@@ -1692,68 +1866,6 @@ async function openLanceIndex(opts) {
1692
1866
  });
1693
1867
  return new LanceMemoryAdapter(lance);
1694
1868
  }
1695
- var MAX_TURN_CHARS = 2e3;
1696
- function sessionsDir(cwd) {
1697
- return path.join(memoryDir(cwd), "sessions");
1698
- }
1699
- function sessionSummaryPath(cwd, runId) {
1700
- return path.join(sessionsDir(cwd), `${sanitizeRunId2(runId)}.md`);
1701
- }
1702
- function sanitizeRunId2(runId) {
1703
- return runId.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 128);
1704
- }
1705
- function truncate2(text) {
1706
- if (text.length <= MAX_TURN_CHARS) return text;
1707
- return `${text.slice(0, MAX_TURN_CHARS)}\u2026`;
1708
- }
1709
- async function writeSessionSummary(input) {
1710
- if (input.status !== "finished") return;
1711
- const path = sessionSummaryPath(input.cwd, input.runId);
1712
- await promises.mkdir(sessionsDir(input.cwd), { recursive: true });
1713
- const safeUser = redactSecrets(truncate2(input.userText));
1714
- const safeAssistant = redactSecrets(truncate2(input.assistantText));
1715
- const iso = new Date(input.at).toISOString();
1716
- const body = [
1717
- "---",
1718
- `runId: ${input.runId}`,
1719
- `agentId: ${input.agentId}`,
1720
- `at: ${iso}`,
1721
- `status: ${input.status}`,
1722
- "---",
1723
- "",
1724
- "## User",
1725
- "",
1726
- safeUser,
1727
- "",
1728
- "## Assistant",
1729
- "",
1730
- safeAssistant,
1731
- ""
1732
- ].join("\n");
1733
- await persistence.replaceFileAtomic(path, body);
1734
- }
1735
-
1736
- // src/internal/session-loader.ts
1737
- async function discoverSessionFiles(cwd) {
1738
- let entries;
1739
- try {
1740
- entries = await promises.readdir(sessionsDir(cwd));
1741
- } catch {
1742
- return [];
1743
- }
1744
- const root = memoryDir(cwd);
1745
- return entries.filter((entry) => entry.endsWith(".md")).map((entry) => {
1746
- const absolutePath = path.join(sessionsDir(cwd), entry);
1747
- return {
1748
- absolutePath,
1749
- relPath: relativeToRoot(root, absolutePath)
1750
- };
1751
- });
1752
- }
1753
- function relativeToRoot(root, absolutePath) {
1754
- if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
1755
- return absolutePath;
1756
- }
1757
1869
  async function loadSqliteVecExtension(db) {
1758
1870
  try {
1759
1871
  const mod = await import('sqlite-vec');
@@ -1779,7 +1891,7 @@ function isSqliteVecLoaded(db) {
1779
1891
  }
1780
1892
  }
1781
1893
 
1782
- // src/internal/vec-index.ts
1894
+ // src/internal/index/vec-index.ts
1783
1895
  var META_KEY_PROVIDER_ID = "embedding.providerId";
1784
1896
  var META_KEY_MODEL = "embedding.model";
1785
1897
  var META_KEY_DIMENSION = "embedding.dimension";
@@ -1852,31 +1964,8 @@ async function embedMissingChunks(args) {
1852
1964
  }
1853
1965
  return normalized.length;
1854
1966
  }
1855
- function wikiDir(cwd) {
1856
- return path.join(memoryDir(cwd), "wiki");
1857
- }
1858
- async function discoverWikiFiles(cwd) {
1859
- let entries;
1860
- try {
1861
- entries = await promises.readdir(wikiDir(cwd));
1862
- } catch {
1863
- return [];
1864
- }
1865
- const root = memoryDir(cwd);
1866
- return entries.filter((entry) => entry.endsWith(".md")).map((entry) => ({
1867
- absolutePath: path.join(wikiDir(cwd), entry),
1868
- relPath: path.join("wiki", entry)
1869
- })).map((file) => ({
1870
- absolutePath: file.absolutePath,
1871
- relPath: relativeToRoot2(root, file.absolutePath)
1872
- }));
1873
- }
1874
- function relativeToRoot2(root, absolutePath) {
1875
- if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
1876
- return absolutePath;
1877
- }
1878
1967
 
1879
- // src/internal/index-manager.ts
1968
+ // src/internal/index/index-manager.ts
1880
1969
  var IndexManager = class _IndexManager {
1881
1970
  constructor(cwd, db, embedding) {
1882
1971
  this.cwd = cwd;
@@ -2524,6 +2613,7 @@ exports.appendFactToMarkdown = appendFactToMarkdown;
2524
2613
  exports.assertValidBackend = assertValidBackend;
2525
2614
  exports.buildErrorMetadata = buildErrorMetadata;
2526
2615
  exports.chunkMarkdown = chunkMarkdown;
2616
+ exports.createCategorizedMemory = createCategorizedMemory;
2527
2617
  exports.createInMemoryMarkdownProvider = createInMemoryMarkdownProvider;
2528
2618
  exports.createMemoryGetTool = createMemoryGetTool;
2529
2619
  exports.createMemorySearchTool = createMemorySearchTool;