engrams 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -45,12 +45,14 @@ var init_schema = __esm({
45
45
  hasPiiFlag: integer("has_pii_flag").notNull().default(0),
46
46
  entityType: text("entity_type"),
47
47
  entityName: text("entity_name"),
48
- structuredData: text("structured_data")
48
+ structuredData: text("structured_data"),
49
+ updatedAt: text("updated_at")
49
50
  });
50
51
  memoryConnections = sqliteTable("memory_connections", {
51
52
  sourceMemoryId: text("source_memory_id").notNull().references(() => memories.id),
52
53
  targetMemoryId: text("target_memory_id").notNull().references(() => memories.id),
53
- relationship: text("relationship").notNull()
54
+ relationship: text("relationship").notNull(),
55
+ updatedAt: text("updated_at")
54
56
  });
55
57
  memoryEvents = sqliteTable("memory_events", {
56
58
  id: text("id").primaryKey(),
@@ -239,6 +241,38 @@ function runMigrations(sqlite) {
239
241
  sqlite.exec(`CREATE INDEX IF NOT EXISTS idx_memories_entity_type ON memories(entity_type) WHERE deleted_at IS NULL`);
240
242
  sqlite.exec(`CREATE INDEX IF NOT EXISTS idx_memories_entity_name ON memories(entity_name) WHERE deleted_at IS NULL`);
241
243
  });
244
+ runMigration(sqlite, "add_updated_at", () => {
245
+ sqlite.exec(`ALTER TABLE memories ADD COLUMN updated_at TEXT`);
246
+ sqlite.exec(`UPDATE memories SET updated_at = COALESCE(confirmed_at, learned_at, datetime('now')) WHERE updated_at IS NULL`);
247
+ sqlite.exec(`ALTER TABLE memory_connections ADD COLUMN updated_at TEXT`);
248
+ sqlite.exec(`UPDATE memory_connections SET updated_at = datetime('now') WHERE updated_at IS NULL`);
249
+ sqlite.exec(`
250
+ CREATE TRIGGER IF NOT EXISTS memories_updated_at_insert
251
+ AFTER INSERT ON memories
252
+ BEGIN
253
+ UPDATE memories SET updated_at = datetime('now') WHERE id = NEW.id AND updated_at IS NULL;
254
+ END;
255
+ `);
256
+ sqlite.exec(`
257
+ CREATE TRIGGER IF NOT EXISTS memories_updated_at_update
258
+ AFTER UPDATE ON memories
259
+ WHEN NEW.updated_at IS OLD.updated_at OR NEW.updated_at IS NULL
260
+ BEGIN
261
+ UPDATE memories SET updated_at = datetime('now') WHERE id = NEW.id;
262
+ END;
263
+ `);
264
+ sqlite.exec(`
265
+ CREATE TRIGGER IF NOT EXISTS connections_updated_at_insert
266
+ AFTER INSERT ON memory_connections
267
+ BEGIN
268
+ UPDATE memory_connections SET updated_at = datetime('now')
269
+ WHERE source_memory_id = NEW.source_memory_id
270
+ AND target_memory_id = NEW.target_memory_id
271
+ AND relationship = NEW.relationship
272
+ AND updated_at IS NULL;
273
+ END;
274
+ `);
275
+ });
242
276
  runMigration(sqlite, "fts_add_entity_name", () => {
243
277
  sqlite.exec(`DROP TRIGGER IF EXISTS memory_fts_insert`);
244
278
  sqlite.exec(`DROP TRIGGER IF EXISTS memory_fts_delete`);
@@ -660,21 +694,24 @@ var init_pii = __esm({
660
694
  }
661
695
  });
662
696
 
697
+ // ../core/dist/llm-utils.js
698
+ function parseLLMJson(text2) {
699
+ const cleaned = text2.replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trim();
700
+ return JSON.parse(cleaned);
701
+ }
702
+ var init_llm_utils = __esm({
703
+ "../core/dist/llm-utils.js"() {
704
+ "use strict";
705
+ }
706
+ });
707
+
663
708
  // ../core/dist/entity-extraction.js
664
- async function extractEntity(content, detail, existingEntityNames) {
665
- const { default: Anthropic2 } = await import("@anthropic-ai/sdk");
666
- const client = new Anthropic2();
709
+ async function extractEntity(provider, content, detail, existingEntityNames) {
667
710
  const existingNamesHint = existingEntityNames && existingEntityNames.length > 0 ? `
668
711
 
669
712
  Existing entity names in the system (prefer matching these over creating new ones):
670
713
  ${existingEntityNames.slice(0, 50).map((n) => `- ${n}`).join("\n")}` : "";
671
- const response = await client.messages.create({
672
- model: "claude-sonnet-4-5-20250514",
673
- max_tokens: 500,
674
- messages: [
675
- {
676
- role: "user",
677
- content: `Classify this memory and extract structured data.
714
+ const prompt = `Classify this memory and extract structured data.
678
715
 
679
716
  Memory: ${content}${detail ? `
680
717
  Detail: ${detail}` : ""}${existingNamesHint}
@@ -709,17 +746,528 @@ For structured_data, include relevant fields:
709
746
  - preference: category, strength (strong/mild/contextual)
710
747
  - event: what, when, who
711
748
  - goal: what, timeline, status (active/achieved/abandoned)
712
- - fact: category`
713
- }
714
- ]
715
- });
716
- const text2 = response.content[0].type === "text" ? response.content[0].text : "";
717
- const cleaned = text2.replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trim();
718
- return JSON.parse(cleaned);
749
+ - fact: category`;
750
+ const text2 = await provider.complete(prompt, { maxTokens: 500, json: true });
751
+ return parseLLMJson(text2);
719
752
  }
720
753
  var init_entity_extraction = __esm({
721
754
  "../core/dist/entity-extraction.js"() {
722
755
  "use strict";
756
+ init_llm_utils();
757
+ }
758
+ });
759
+
760
+ // ../core/dist/crypto.js
761
+ import { randomBytes, scryptSync, createCipheriv, createDecipheriv } from "crypto";
762
+ function deriveKeys(passphrase, salt) {
763
+ const derived = scryptSync(passphrase, salt, KEY_LENGTH * 2, {
764
+ N: SCRYPT_N,
765
+ r: SCRYPT_R,
766
+ p: SCRYPT_P,
767
+ maxmem: 256 * 1024 * 1024
768
+ // 256 MB — N=131072,r=8 needs ~128 MB
769
+ });
770
+ return {
771
+ encryptionKey: derived.subarray(0, KEY_LENGTH),
772
+ hmacKey: derived.subarray(KEY_LENGTH)
773
+ };
774
+ }
775
+ function encrypt(plaintext, key) {
776
+ const iv = randomBytes(IV_LENGTH);
777
+ const cipher = createCipheriv(ALGORITHM, key, iv);
778
+ const encrypted = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
779
+ const authTag = cipher.getAuthTag();
780
+ return Buffer.concat([iv, encrypted, authTag]).toString("base64");
781
+ }
782
+ function decrypt(encoded, key) {
783
+ const data = Buffer.from(encoded, "base64");
784
+ const iv = data.subarray(0, IV_LENGTH);
785
+ const authTag = data.subarray(data.length - AUTH_TAG_LENGTH);
786
+ const ciphertext = data.subarray(IV_LENGTH, data.length - AUTH_TAG_LENGTH);
787
+ const decipher = createDecipheriv(ALGORITHM, key, iv);
788
+ decipher.setAuthTag(authTag);
789
+ return decipher.update(ciphertext) + decipher.final("utf8");
790
+ }
791
+ function encryptMemory(memory, key) {
792
+ return {
793
+ content: encrypt(memory.content, key),
794
+ detail: memory.detail ? encrypt(memory.detail, key) : null,
795
+ structured_data: memory.structured_data ? encrypt(memory.structured_data, key) : null
796
+ };
797
+ }
798
+ function decryptMemory(memory, key) {
799
+ return {
800
+ content: decrypt(memory.content, key),
801
+ detail: memory.detail ? decrypt(memory.detail, key) : null,
802
+ structured_data: memory.structured_data ? decrypt(memory.structured_data, key) : null
803
+ };
804
+ }
805
+ var ALGORITHM, IV_LENGTH, AUTH_TAG_LENGTH, SCRYPT_N, SCRYPT_R, SCRYPT_P, KEY_LENGTH;
806
+ var init_crypto = __esm({
807
+ "../core/dist/crypto.js"() {
808
+ "use strict";
809
+ ALGORITHM = "aes-256-gcm";
810
+ IV_LENGTH = 12;
811
+ AUTH_TAG_LENGTH = 16;
812
+ SCRYPT_N = 131072;
813
+ SCRYPT_R = 8;
814
+ SCRYPT_P = 1;
815
+ KEY_LENGTH = 32;
816
+ }
817
+ });
818
+
819
+ // ../core/dist/credentials.js
820
+ import { resolve as resolve3 } from "path";
821
+ import { homedir as homedir3 } from "os";
822
+ import { readFileSync, writeFileSync, existsSync, chmodSync as chmodSync2 } from "fs";
823
+ function loadCredentials() {
824
+ if (!existsSync(CRED_PATH))
825
+ return null;
826
+ try {
827
+ return JSON.parse(readFileSync(CRED_PATH, "utf8"));
828
+ } catch {
829
+ return null;
830
+ }
831
+ }
832
+ function loadConfig() {
833
+ if (!existsSync(CONFIG_PATH))
834
+ return {};
835
+ try {
836
+ return JSON.parse(readFileSync(CONFIG_PATH, "utf8"));
837
+ } catch {
838
+ return {};
839
+ }
840
+ }
841
+ function saveConfig(config) {
842
+ writeFileSync(CONFIG_PATH, JSON.stringify(config, null, 2), "utf8");
843
+ try {
844
+ chmodSync2(CONFIG_PATH, 384);
845
+ } catch {
846
+ }
847
+ }
848
+ var ENGRAMS_DIR, CRED_PATH, CONFIG_PATH;
849
+ var init_credentials = __esm({
850
+ "../core/dist/credentials.js"() {
851
+ "use strict";
852
+ ENGRAMS_DIR = resolve3(homedir3(), ".engrams");
853
+ CRED_PATH = resolve3(ENGRAMS_DIR, "credentials.json");
854
+ CONFIG_PATH = resolve3(ENGRAMS_DIR, "config.json");
855
+ }
856
+ });
857
+
858
+ // ../core/dist/sync.js
859
+ import { createClient } from "@libsql/client";
860
+ async function initRemoteSchema(client) {
861
+ await client.executeMultiple(`
862
+ CREATE TABLE IF NOT EXISTS memories (
863
+ id TEXT PRIMARY KEY,
864
+ content TEXT NOT NULL,
865
+ detail TEXT,
866
+ domain TEXT NOT NULL DEFAULT 'general',
867
+ source_agent_id TEXT NOT NULL,
868
+ source_agent_name TEXT NOT NULL,
869
+ cross_agent_id TEXT,
870
+ cross_agent_name TEXT,
871
+ source_type TEXT NOT NULL,
872
+ source_description TEXT,
873
+ confidence REAL NOT NULL DEFAULT 0.7,
874
+ confirmed_count INTEGER NOT NULL DEFAULT 0,
875
+ corrected_count INTEGER NOT NULL DEFAULT 0,
876
+ mistake_count INTEGER NOT NULL DEFAULT 0,
877
+ used_count INTEGER NOT NULL DEFAULT 0,
878
+ learned_at TEXT,
879
+ confirmed_at TEXT,
880
+ last_used_at TEXT,
881
+ deleted_at TEXT,
882
+ has_pii_flag INTEGER NOT NULL DEFAULT 0,
883
+ entity_type TEXT,
884
+ entity_name TEXT,
885
+ structured_data TEXT,
886
+ device_id TEXT,
887
+ updated_at TEXT NOT NULL DEFAULT (datetime('now'))
888
+ );
889
+
890
+ CREATE TABLE IF NOT EXISTS memory_connections (
891
+ source_memory_id TEXT NOT NULL,
892
+ target_memory_id TEXT NOT NULL,
893
+ relationship TEXT NOT NULL,
894
+ device_id TEXT,
895
+ updated_at TEXT NOT NULL DEFAULT (datetime('now'))
896
+ );
897
+
898
+ CREATE TABLE IF NOT EXISTS memory_events (
899
+ id TEXT PRIMARY KEY,
900
+ memory_id TEXT NOT NULL,
901
+ event_type TEXT NOT NULL,
902
+ agent_id TEXT,
903
+ agent_name TEXT,
904
+ old_value TEXT,
905
+ new_value TEXT,
906
+ timestamp TEXT NOT NULL,
907
+ device_id TEXT
908
+ );
909
+
910
+ CREATE TABLE IF NOT EXISTS sync_log (
911
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
912
+ device_id TEXT NOT NULL,
913
+ synced_at TEXT NOT NULL DEFAULT (datetime('now')),
914
+ pushed INTEGER NOT NULL DEFAULT 0,
915
+ pulled INTEGER NOT NULL DEFAULT 0
916
+ );
917
+ `);
918
+ }
919
+ async function pushChanges(sqlite, client, config, deviceId) {
920
+ const lastSync = await getLastSyncTime(client, deviceId);
921
+ const changedMemories = sqlite.prepare(`
922
+ SELECT * FROM memories WHERE updated_at > ? OR (updated_at IS NULL AND learned_at > ?)
923
+ `).all(lastSync, lastSync);
924
+ let pushed = 0;
925
+ for (const mem of changedMemories) {
926
+ const encrypted = encryptMemory({
927
+ content: mem.content,
928
+ detail: mem.detail,
929
+ structured_data: mem.structured_data
930
+ }, config.keys.encryptionKey);
931
+ await client.execute({
932
+ sql: `INSERT OR REPLACE INTO memories
933
+ (id, content, detail, domain, source_agent_id, source_agent_name,
934
+ cross_agent_id, cross_agent_name, source_type, source_description,
935
+ confidence, confirmed_count, corrected_count, mistake_count, used_count,
936
+ learned_at, confirmed_at, last_used_at, deleted_at,
937
+ has_pii_flag, entity_type, entity_name, structured_data,
938
+ device_id, updated_at)
939
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`,
940
+ args: [
941
+ mem.id,
942
+ encrypted.content,
943
+ encrypted.detail,
944
+ mem.domain,
945
+ mem.source_agent_id,
946
+ mem.source_agent_name,
947
+ mem.cross_agent_id ?? null,
948
+ mem.cross_agent_name ?? null,
949
+ mem.source_type,
950
+ mem.source_description ?? null,
951
+ mem.confidence,
952
+ mem.confirmed_count,
953
+ mem.corrected_count,
954
+ mem.mistake_count,
955
+ mem.used_count,
956
+ mem.learned_at ?? null,
957
+ mem.confirmed_at ?? null,
958
+ mem.last_used_at ?? null,
959
+ mem.deleted_at ?? null,
960
+ mem.has_pii_flag ?? 0,
961
+ mem.entity_type ?? null,
962
+ mem.entity_name ?? null,
963
+ encrypted.structured_data,
964
+ deviceId
965
+ ]
966
+ });
967
+ pushed++;
968
+ }
969
+ const changedConnections = sqlite.prepare(`
970
+ SELECT * FROM memory_connections WHERE updated_at > ? OR updated_at IS NULL
971
+ `).all(lastSync);
972
+ for (const conn of changedConnections) {
973
+ await client.execute({
974
+ sql: `INSERT OR REPLACE INTO memory_connections
975
+ (source_memory_id, target_memory_id, relationship, device_id, updated_at)
976
+ VALUES (?, ?, ?, ?, datetime('now'))`,
977
+ args: [
978
+ conn.source_memory_id,
979
+ conn.target_memory_id,
980
+ conn.relationship,
981
+ deviceId
982
+ ]
983
+ });
984
+ }
985
+ const changedEvents = sqlite.prepare(`
986
+ SELECT * FROM memory_events WHERE timestamp > ?
987
+ `).all(lastSync);
988
+ for (const evt of changedEvents) {
989
+ await client.execute({
990
+ sql: `INSERT OR IGNORE INTO memory_events
991
+ (id, memory_id, event_type, agent_id, agent_name, old_value, new_value, timestamp, device_id)
992
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
993
+ args: [
994
+ evt.id,
995
+ evt.memory_id,
996
+ evt.event_type,
997
+ evt.agent_id ?? null,
998
+ evt.agent_name ?? null,
999
+ evt.old_value ?? null,
1000
+ evt.new_value ?? null,
1001
+ evt.timestamp,
1002
+ deviceId
1003
+ ]
1004
+ });
1005
+ }
1006
+ return pushed + changedConnections.length + changedEvents.length;
1007
+ }
1008
+ async function pullChanges(sqlite, client, config, deviceId) {
1009
+ const lastSync = await getLastSyncTime(client, deviceId);
1010
+ const result = await client.execute({
1011
+ sql: `SELECT * FROM memories WHERE device_id != ? AND updated_at > ?`,
1012
+ args: [deviceId, lastSync]
1013
+ });
1014
+ let pulled = 0;
1015
+ for (const row of result.rows) {
1016
+ const decrypted = decryptMemory({
1017
+ content: row.content,
1018
+ detail: row.detail,
1019
+ structured_data: row.structured_data
1020
+ }, config.keys.encryptionKey);
1021
+ const local = sqlite.prepare(`SELECT updated_at FROM memories WHERE id = ?`).get(row.id);
1022
+ if (local && local.updated_at >= row.updated_at)
1023
+ continue;
1024
+ sqlite.prepare(`
1025
+ INSERT OR REPLACE INTO memories
1026
+ (id, content, detail, domain, source_agent_id, source_agent_name,
1027
+ cross_agent_id, cross_agent_name, source_type, source_description,
1028
+ confidence, confirmed_count, corrected_count, mistake_count, used_count,
1029
+ learned_at, confirmed_at, last_used_at, deleted_at,
1030
+ has_pii_flag, entity_type, entity_name, structured_data, updated_at)
1031
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1032
+ `).run(row.id, decrypted.content, decrypted.detail, row.domain, row.source_agent_id, row.source_agent_name, row.cross_agent_id, row.cross_agent_name, row.source_type, row.source_description, row.confidence, row.confirmed_count, row.corrected_count, row.mistake_count, row.used_count, row.learned_at, row.confirmed_at, row.last_used_at, row.deleted_at, row.has_pii_flag, row.entity_type, row.entity_name, decrypted.structured_data, row.updated_at);
1033
+ pulled++;
1034
+ }
1035
+ const connResult = await client.execute({
1036
+ sql: `SELECT * FROM memory_connections WHERE device_id != ? AND updated_at > ?`,
1037
+ args: [deviceId, lastSync]
1038
+ });
1039
+ for (const row of connResult.rows) {
1040
+ sqlite.prepare(`
1041
+ INSERT OR IGNORE INTO memory_connections (source_memory_id, target_memory_id, relationship, updated_at)
1042
+ VALUES (?, ?, ?, ?)
1043
+ `).run(row.source_memory_id, row.target_memory_id, row.relationship, row.updated_at);
1044
+ pulled++;
1045
+ }
1046
+ const evtResult = await client.execute({
1047
+ sql: `SELECT * FROM memory_events WHERE device_id != ? AND timestamp > ?`,
1048
+ args: [deviceId, lastSync]
1049
+ });
1050
+ for (const row of evtResult.rows) {
1051
+ sqlite.prepare(`
1052
+ INSERT OR IGNORE INTO memory_events (id, memory_id, event_type, agent_id, agent_name, old_value, new_value, timestamp)
1053
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
1054
+ `).run(row.id, row.memory_id, row.event_type, row.agent_id, row.agent_name, row.old_value, row.new_value, row.timestamp);
1055
+ pulled++;
1056
+ }
1057
+ return pulled;
1058
+ }
1059
+ async function sync(sqlite, config, deviceId) {
1060
+ const client = createClient({
1061
+ url: config.tursoUrl,
1062
+ authToken: config.tursoAuthToken
1063
+ });
1064
+ try {
1065
+ await initRemoteSchema(client);
1066
+ const pushed = await pushChanges(sqlite, client, config, deviceId);
1067
+ const pulled = await pullChanges(sqlite, client, config, deviceId);
1068
+ await client.execute({
1069
+ sql: `INSERT INTO sync_log (device_id, pushed, pulled) VALUES (?, ?, ?)`,
1070
+ args: [deviceId, pushed, pulled]
1071
+ });
1072
+ sqlite.prepare(`INSERT OR REPLACE INTO engrams_meta (key, value) VALUES ('last_modified', datetime('now'))`).run();
1073
+ return { pushed, pulled, conflicts: 0 };
1074
+ } finally {
1075
+ client.close();
1076
+ }
1077
+ }
1078
+ async function getLastSyncTime(client, deviceId) {
1079
+ const result = await client.execute({
1080
+ sql: `SELECT synced_at FROM sync_log WHERE device_id = ? ORDER BY synced_at DESC LIMIT 1`,
1081
+ args: [deviceId]
1082
+ });
1083
+ return result.rows.length > 0 ? result.rows[0].synced_at : "1970-01-01T00:00:00Z";
1084
+ }
1085
+ var init_sync = __esm({
1086
+ "../core/dist/sync.js"() {
1087
+ "use strict";
1088
+ init_crypto();
1089
+ }
1090
+ });
1091
+
1092
+ // ../core/dist/llm.js
1093
+ function createLLMProvider(config, task) {
1094
+ const model = config.model || task && DEFAULT_TASK_MODELS[config.provider]?.[task] || DEFAULT_MODELS[config.provider] || "gpt-4o";
1095
+ switch (config.provider) {
1096
+ case "anthropic":
1097
+ return new AnthropicProvider(config.apiKey, model);
1098
+ case "openai":
1099
+ return new OpenAICompatibleProvider(config.apiKey, model, config.baseUrl);
1100
+ case "ollama":
1101
+ return new OpenAICompatibleProvider(void 0, model, config.baseUrl || "http://localhost:11434/v1");
1102
+ default:
1103
+ throw new Error(`Unknown LLM provider: ${config.provider}`);
1104
+ }
1105
+ }
1106
+ var DEFAULT_MODELS, DEFAULT_TASK_MODELS, AnthropicProvider, OpenAICompatibleProvider;
1107
+ var init_llm = __esm({
1108
+ "../core/dist/llm.js"() {
1109
+ "use strict";
1110
+ DEFAULT_MODELS = {
1111
+ anthropic: "claude-sonnet-4-5-20250514",
1112
+ openai: "gpt-4o",
1113
+ ollama: "llama3.2"
1114
+ };
1115
+ DEFAULT_TASK_MODELS = {
1116
+ anthropic: {
1117
+ extraction: "claude-haiku-4-5-20251001",
1118
+ analysis: "claude-sonnet-4-5-20250514"
1119
+ },
1120
+ openai: {
1121
+ extraction: "gpt-4o-mini",
1122
+ analysis: "gpt-4o"
1123
+ },
1124
+ ollama: {
1125
+ extraction: "llama3.2",
1126
+ analysis: "llama3.2"
1127
+ }
1128
+ };
1129
+ AnthropicProvider = class {
1130
+ client = null;
1131
+ apiKey;
1132
+ model;
1133
+ constructor(apiKey, model) {
1134
+ this.apiKey = apiKey;
1135
+ this.model = model;
1136
+ }
1137
+ async getClient() {
1138
+ if (!this.client) {
1139
+ const { default: Anthropic } = await import("@anthropic-ai/sdk");
1140
+ this.client = new Anthropic({ apiKey: this.apiKey });
1141
+ }
1142
+ return this.client;
1143
+ }
1144
+ async complete(prompt, options) {
1145
+ const client = await this.getClient();
1146
+ const response = await client.messages.create({
1147
+ model: this.model,
1148
+ max_tokens: options?.maxTokens ?? 1024,
1149
+ ...options?.system ? { system: options.system } : {},
1150
+ messages: [{ role: "user", content: prompt }]
1151
+ });
1152
+ const text2 = response.content[0].type === "text" ? response.content[0].text : "";
1153
+ return text2;
1154
+ }
1155
+ };
1156
+ OpenAICompatibleProvider = class {
1157
+ apiKey;
1158
+ model;
1159
+ baseUrl;
1160
+ constructor(apiKey, model, baseUrl) {
1161
+ this.apiKey = apiKey;
1162
+ this.model = model;
1163
+ this.baseUrl = baseUrl;
1164
+ }
1165
+ async complete(prompt, options) {
1166
+ const url = `${this.baseUrl || "https://api.openai.com/v1"}/chat/completions`;
1167
+ const body = {
1168
+ model: this.model,
1169
+ max_tokens: options?.maxTokens ?? 1024,
1170
+ messages: [
1171
+ ...options?.system ? [{ role: "system", content: options.system }] : [],
1172
+ { role: "user", content: prompt }
1173
+ ]
1174
+ };
1175
+ if (options?.json) {
1176
+ body.response_format = { type: "json_object" };
1177
+ }
1178
+ const response = await fetch(url, {
1179
+ method: "POST",
1180
+ headers: {
1181
+ "Content-Type": "application/json",
1182
+ ...this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}
1183
+ },
1184
+ body: JSON.stringify(body)
1185
+ });
1186
+ if (!response.ok) {
1187
+ const error = await response.text();
1188
+ throw new Error(`LLM request failed (${response.status}): ${error}`);
1189
+ }
1190
+ const data = await response.json();
1191
+ return data.choices[0]?.message?.content ?? "";
1192
+ }
1193
+ };
1194
+ }
1195
+ });
1196
+
1197
+ // ../core/dist/llm-config.js
1198
+ function resolveLLMProvider(task) {
1199
+ const config = loadConfig();
1200
+ if (config.llm?.provider) {
1201
+ const apiKey = config.llm.apiKey || process.env.ENGRAMS_API_KEY || (config.llm.provider === "anthropic" ? process.env.ANTHROPIC_API_KEY : void 0) || (config.llm.provider === "openai" ? process.env.OPENAI_API_KEY : void 0);
1202
+ const taskModel = task && config.llm.models?.[task];
1203
+ return createLLMProvider({
1204
+ provider: config.llm.provider,
1205
+ model: taskModel || config.llm.model || void 0,
1206
+ apiKey: apiKey || void 0,
1207
+ baseUrl: config.llm.baseUrl
1208
+ }, task);
1209
+ }
1210
+ const engramsProvider = process.env.ENGRAMS_LLM_PROVIDER;
1211
+ if (engramsProvider) {
1212
+ return createLLMProvider({
1213
+ provider: engramsProvider,
1214
+ model: process.env.ENGRAMS_LLM_MODEL || void 0,
1215
+ apiKey: process.env.ENGRAMS_API_KEY,
1216
+ baseUrl: process.env.ENGRAMS_LLM_BASE_URL
1217
+ }, task);
1218
+ }
1219
+ if (process.env.ANTHROPIC_API_KEY) {
1220
+ return createLLMProvider({
1221
+ provider: "anthropic",
1222
+ apiKey: process.env.ANTHROPIC_API_KEY
1223
+ }, task);
1224
+ }
1225
+ if (process.env.OPENAI_API_KEY) {
1226
+ return createLLMProvider({
1227
+ provider: "openai",
1228
+ apiKey: process.env.OPENAI_API_KEY
1229
+ }, task);
1230
+ }
1231
+ return null;
1232
+ }
1233
+ var init_llm_config = __esm({
1234
+ "../core/dist/llm-config.js"() {
1235
+ "use strict";
1236
+ init_credentials();
1237
+ init_llm();
1238
+ }
1239
+ });
1240
+
1241
+ // ../core/dist/llm-validation.js
1242
+ function validateExtraction(result) {
1243
+ const warnings = [];
1244
+ if (!result || typeof result !== "object") {
1245
+ return { valid: false, error: "LLM did not return a valid object", warnings, qualityScore: 0 };
1246
+ }
1247
+ const r = result;
1248
+ if (!r.entity_type || !VALID_ENTITY_TYPES.includes(r.entity_type)) {
1249
+ return {
1250
+ valid: false,
1251
+ error: `Invalid entity_type "${r.entity_type}". Model may not be capable enough for extraction. Recommended: claude-haiku-4-5, gpt-4o-mini, or better.`,
1252
+ warnings,
1253
+ qualityScore: 0
1254
+ };
1255
+ }
1256
+ if (r.entity_name && typeof r.entity_name === "string" && r.entity_name.length > 100) {
1257
+ warnings.push("entity_name is unusually long \u2014 model may have used full content instead of a canonical name");
1258
+ }
1259
+ if (r.suggested_connections && !Array.isArray(r.suggested_connections)) {
1260
+ warnings.push("suggested_connections is not an array \u2014 ignoring");
1261
+ }
1262
+ const score = 1 - warnings.length * 0.1 - (r.entity_name ? 0 : 0.1);
1263
+ return { valid: true, warnings, qualityScore: Math.max(0, score) };
1264
+ }
1265
+ var VALID_ENTITY_TYPES;
1266
+ var init_llm_validation = __esm({
1267
+ "../core/dist/llm-validation.js"() {
1268
+ "use strict";
1269
+ init_embeddings();
1270
+ VALID_ENTITY_TYPES = ["person", "organization", "place", "project", "preference", "event", "goal", "fact"];
723
1271
  }
724
1272
  });
725
1273
 
@@ -738,6 +1286,13 @@ var init_dist = __esm({
738
1286
  init_db();
739
1287
  init_pii();
740
1288
  init_entity_extraction();
1289
+ init_crypto();
1290
+ init_credentials();
1291
+ init_sync();
1292
+ init_llm();
1293
+ init_llm_utils();
1294
+ init_llm_config();
1295
+ init_llm_validation();
741
1296
  }
742
1297
  });
743
1298
 
@@ -747,10 +1302,10 @@ __export(http_exports, {
747
1302
  startHttpApi: () => startHttpApi
748
1303
  });
749
1304
  import { createServer } from "http";
750
- import { randomBytes } from "crypto";
1305
+ import { randomBytes as randomBytes2 } from "crypto";
751
1306
  import { eq, and, isNull } from "drizzle-orm";
752
1307
  function generateId() {
753
- return randomBytes(16).toString("hex");
1308
+ return randomBytes2(16).toString("hex");
754
1309
  }
755
1310
  function now() {
756
1311
  return (/* @__PURE__ */ new Date()).toISOString();
@@ -760,12 +1315,12 @@ function json(res, data, status = 200) {
760
1315
  res.end(JSON.stringify(data));
761
1316
  }
762
1317
  function parseBody(req) {
763
- return new Promise((resolve3, reject) => {
1318
+ return new Promise((resolve4, reject) => {
764
1319
  let body = "";
765
1320
  req.on("data", (chunk) => body += chunk);
766
1321
  req.on("end", () => {
767
1322
  try {
768
- resolve3(body ? JSON.parse(body) : {});
1323
+ resolve4(body ? JSON.parse(body) : {});
769
1324
  } catch {
770
1325
  reject(new Error("Invalid JSON"));
771
1326
  }
@@ -936,10 +1491,9 @@ import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mc
936
1491
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
937
1492
  import { z } from "zod";
938
1493
  import { eq as eq2, and as and2, isNull as isNull2 } from "drizzle-orm";
939
- import { randomBytes as randomBytes2 } from "crypto";
940
- import Anthropic from "@anthropic-ai/sdk";
1494
+ import { randomBytes as randomBytes3 } from "crypto";
941
1495
  function generateId2() {
942
- return randomBytes2(16).toString("hex");
1496
+ return randomBytes3(16).toString("hex");
943
1497
  }
944
1498
  function now2() {
945
1499
  return (/* @__PURE__ */ new Date()).toISOString();
@@ -964,6 +1518,8 @@ async function startServer() {
964
1518
  lastDecayRun = now3;
965
1519
  }
966
1520
  }
1521
+ let extractionProvider = resolveLLMProvider("extraction");
1522
+ let analysisProvider = resolveLLMProvider("analysis");
967
1523
  if (vecAvailable) {
968
1524
  backfillEmbeddings(sqlite).then((count) => {
969
1525
  if (count > 0) process.stderr.write(`[engrams] Backfilled embeddings for ${count} memories
@@ -1259,9 +1815,9 @@ Organize memories by life domain: general, work, health, finance, relationships,
1259
1815
  }
1260
1816
  }
1261
1817
  }
1262
- const VALID_ENTITY_TYPES = ["person", "organization", "place", "project", "preference", "event", "goal", "fact"];
1263
- if (params.entityType && !VALID_ENTITY_TYPES.includes(params.entityType)) {
1264
- return textResult({ error: `Invalid entity_type: "${params.entityType}". Must be one of: ${VALID_ENTITY_TYPES.join(", ")}` });
1818
+ const VALID_ENTITY_TYPES2 = ["person", "organization", "place", "project", "preference", "event", "goal", "fact"];
1819
+ if (params.entityType && !VALID_ENTITY_TYPES2.includes(params.entityType)) {
1820
+ return textResult({ error: `Invalid entity_type: "${params.entityType}". Must be one of: ${VALID_ENTITY_TYPES2.join(", ")}` });
1265
1821
  }
1266
1822
  const id = generateId2();
1267
1823
  const confidence = getInitialConfidence(params.sourceType);
@@ -1304,15 +1860,22 @@ Organize memories by life domain: general, work, health, finance, relationships,
1304
1860
  newValue: JSON.stringify({ content: params.content, domain: params.domain ?? "general" }),
1305
1861
  timestamp
1306
1862
  }).run();
1307
- if (!params.entityType && process.env.ANTHROPIC_API_KEY) {
1863
+ if (!params.entityType && extractionProvider) {
1308
1864
  (async () => {
1309
1865
  try {
1310
1866
  const existingNames = sqlite.prepare(`SELECT DISTINCT entity_name FROM memories WHERE entity_name IS NOT NULL AND deleted_at IS NULL`).all();
1311
1867
  const extraction = await extractEntity(
1868
+ extractionProvider,
1312
1869
  params.content,
1313
1870
  params.detail ?? null,
1314
1871
  existingNames.map((r) => r.entity_name)
1315
1872
  );
1873
+ const validation = validateExtraction(extraction);
1874
+ if (!validation.valid) {
1875
+ process.stderr.write(`[engrams] Entity extraction failed validation: ${validation.error}
1876
+ `);
1877
+ return;
1878
+ }
1316
1879
  const current = sqlite.prepare(`SELECT entity_type FROM memories WHERE id = ? AND deleted_at IS NULL`).get(id);
1317
1880
  if (!current || current.entity_type) return;
1318
1881
  sqlite.transaction(() => {
@@ -1341,16 +1904,9 @@ Organize memories by life domain: general, work, health, finance, relationships,
1341
1904
  const fullText = params.content + (params.detail ? " " + params.detail : "");
1342
1905
  const sentences = fullText.split(/(?<=[.!?])\s+/).filter((s) => s.length > 10);
1343
1906
  let splitSuggestion = null;
1344
- if (sentences.length >= 3 && process.env.ANTHROPIC_API_KEY) {
1907
+ if (sentences.length >= 3 && extractionProvider) {
1345
1908
  try {
1346
- const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
1347
- const resp = await client.messages.create({
1348
- model: "claude-haiku-4-5-20251001",
1349
- max_tokens: 512,
1350
- messages: [
1351
- {
1352
- role: "user",
1353
- content: `Analyze this memory and determine if it contains multiple distinct topics that should be stored separately. Only suggest splitting if the topics are genuinely independent (would be searched for separately).
1909
+ const splitPrompt = `Analyze this memory and determine if it contains multiple distinct topics that should be stored separately. Only suggest splitting if the topics are genuinely independent (would be searched for separately).
1354
1910
 
1355
1911
  Memory content: ${JSON.stringify(params.content)}
1356
1912
  Memory detail: ${JSON.stringify(params.detail ?? null)}
@@ -1359,12 +1915,9 @@ Respond with ONLY valid JSON:
1359
1915
  - If it should NOT be split: {"should_split": false}
1360
1916
  - If it SHOULD be split: {"should_split": true, "parts": [{"content": "...", "detail": "..."}, ...]}
1361
1917
 
1362
- Each part should have a concise "content" (one sentence) and optional "detail". Do not split if the content is a single coherent topic.`
1363
- }
1364
- ]
1365
- });
1366
- const text2 = resp.content[0].type === "text" ? resp.content[0].text : "";
1367
- splitSuggestion = JSON.parse(text2);
1918
+ Each part should have a concise "content" (one sentence) and optional "detail". Do not split if the content is a single coherent topic.`;
1919
+ const text2 = await extractionProvider.complete(splitPrompt, { maxTokens: 512, json: true });
1920
+ splitSuggestion = parseLLMJson(text2);
1368
1921
  } catch {
1369
1922
  }
1370
1923
  }
@@ -1377,7 +1930,7 @@ Each part should have a concise "content" (one sentence) and optional "detail".
1377
1930
  entityType: params.entityType ?? null,
1378
1931
  entityName: params.entityName ?? null
1379
1932
  };
1380
- if (!params.entityType && process.env.ANTHROPIC_API_KEY) {
1933
+ if (!params.entityType && extractionProvider) {
1381
1934
  result._background_classification = "running";
1382
1935
  }
1383
1936
  if (hasPii) {
@@ -1419,7 +1972,9 @@ Each part should have a concise "content" (one sentence) and optional "detail".
1419
1972
  similarityThreshold: params.similarityThreshold
1420
1973
  });
1421
1974
  if (searchResults.length === 0) {
1422
- return textResult({ memories: [], count: 0, totalConnected: 0, cached: wasCached });
1975
+ const totalCount = sqlite.prepare("SELECT COUNT(*) as count FROM memories WHERE deleted_at IS NULL").get().count;
1976
+ const onboarding_hint = totalCount < 5 ? "Your memory database is nearly empty. Call memory_onboard with your list of available tools to run a guided setup." : void 0;
1977
+ return textResult({ memories: [], count: 0, totalConnected: 0, cached: wasCached, ...onboarding_hint ? { onboarding_hint } : {} });
1423
1978
  }
1424
1979
  const timestamp = now2();
1425
1980
  const updateStmt = sqlite.prepare(
@@ -1913,8 +2468,8 @@ Each part should have a concise "content" (one sentence) and optional "detail".
1913
2468
  domain: z.string().optional().describe("Only classify memories in this domain")
1914
2469
  },
1915
2470
  async (params) => {
1916
- if (!process.env.ANTHROPIC_API_KEY) {
1917
- return textResult({ error: "ANTHROPIC_API_KEY not set \u2014 entity extraction requires an API key" });
2471
+ if (!extractionProvider) {
2472
+ return textResult({ error: "No LLM provider configured. Set ANTHROPIC_API_KEY, OPENAI_API_KEY, or configure ~/.engrams/config.json" });
1918
2473
  }
1919
2474
  const classifyLimit = params.limit ?? 50;
1920
2475
  let query = `SELECT id, content, detail FROM memories WHERE entity_type IS NULL AND deleted_at IS NULL`;
@@ -1935,7 +2490,12 @@ Each part should have a concise "content" (one sentence) and optional "detail".
1935
2490
  let errors = 0;
1936
2491
  for (const mem of untyped) {
1937
2492
  try {
1938
- const extraction = await extractEntity(mem.content, mem.detail, nameList);
2493
+ const extraction = await extractEntity(extractionProvider, mem.content, mem.detail, nameList);
2494
+ const validation = validateExtraction(extraction);
2495
+ if (!validation.valid) {
2496
+ errors++;
2497
+ continue;
2498
+ }
1939
2499
  sqlite.transaction(() => {
1940
2500
  sqlite.prepare(
1941
2501
  `UPDATE memories SET entity_type = ?, entity_name = ?, structured_data = ? WHERE id = ? AND entity_type IS NULL AND deleted_at IS NULL`
@@ -1960,7 +2520,7 @@ Each part should have a concise "content" (one sentence) and optional "detail".
1960
2520
  errors++;
1961
2521
  }
1962
2522
  if (classified + errors < untyped.length) {
1963
- await new Promise((resolve3) => setTimeout(resolve3, 200));
2523
+ await new Promise((resolve4) => setTimeout(resolve4, 200));
1964
2524
  }
1965
2525
  }
1966
2526
  if (classified > 0) bumpLastModified(sqlite);
@@ -2043,6 +2603,531 @@ Each part should have a concise "content" (one sentence) and optional "detail".
2043
2603
  });
2044
2604
  }
2045
2605
  );
2606
+ server.tool(
2607
+ "memory_configure",
2608
+ "Configure Engrams settings. Currently supports LLM provider setup for entity extraction, correction, and splitting.",
2609
+ {
2610
+ llm_provider: z.enum(["anthropic", "openai", "ollama"]).describe("LLM provider to use"),
2611
+ llm_api_key: z.string().optional().describe("API key for the provider. Not needed for Ollama."),
2612
+ llm_base_url: z.string().optional().describe("Custom base URL for OpenAI-compatible endpoints or Ollama."),
2613
+ llm_extraction_model: z.string().optional().describe("Model for entity extraction (high-volume, cheap). Defaults: anthropic=claude-haiku-4-5, openai=gpt-4o-mini, ollama=llama3.2"),
2614
+ llm_analysis_model: z.string().optional().describe("Model for correction/splitting (user-initiated, capable). Defaults: anthropic=claude-sonnet-4-5, openai=gpt-4o, ollama=llama3.2")
2615
+ },
2616
+ async (params) => {
2617
+ const config = loadConfig();
2618
+ config.llm = {
2619
+ provider: params.llm_provider,
2620
+ apiKey: params.llm_api_key || void 0,
2621
+ baseUrl: params.llm_base_url || void 0,
2622
+ models: {
2623
+ extraction: params.llm_extraction_model || void 0,
2624
+ analysis: params.llm_analysis_model || void 0
2625
+ }
2626
+ };
2627
+ saveConfig(config);
2628
+ extractionProvider = resolveLLMProvider("extraction");
2629
+ analysisProvider = resolveLLMProvider("analysis");
2630
+ try {
2631
+ if (extractionProvider) {
2632
+ await extractionProvider.complete("Say ok", { maxTokens: 10 });
2633
+ }
2634
+ return textResult({
2635
+ status: "configured",
2636
+ provider: params.llm_provider,
2637
+ extraction_model: params.llm_extraction_model || "(default)",
2638
+ analysis_model: params.llm_analysis_model || "(default)"
2639
+ });
2640
+ } catch (err) {
2641
+ return textResult({
2642
+ status: "configured_with_error",
2643
+ provider: params.llm_provider,
2644
+ error: `Config saved but connection test failed: ${err instanceof Error ? err.message : "Unknown error"}`
2645
+ });
2646
+ }
2647
+ }
2648
+ );
2649
+ server.tool(
2650
+ "memory_onboard",
2651
+ "Get a personalized onboarding plan to seed your memory database. Returns a structured action plan based on your current memory state. Call this when the user is new or asks to set up their memory. The plan tells you which connected tools to scan and what interview questions to ask \u2014 execute each step using the tools available to you.",
2652
+ {
2653
+ available_tools: z.array(z.string()).optional().describe("List of MCP tool names you have access to (e.g. ['gcal_list_events', 'gmail_search_messages', 'github_list_repos']). This helps generate a targeted plan."),
2654
+ skip_scan: z.boolean().optional().describe("Skip the tool scan phase and go straight to interview. Default false."),
2655
+ skip_interview: z.boolean().optional().describe("Skip the interview phase. Useful if re-running just the scan. Default false.")
2656
+ },
2657
+ async (params) => {
2658
+ const memoryCount = sqlite.prepare("SELECT COUNT(*) as count FROM memories WHERE deleted_at IS NULL").get();
2659
+ const entityCounts = sqlite.prepare(`
2660
+ SELECT entity_type, COUNT(*) as count
2661
+ FROM memories
2662
+ WHERE entity_type IS NOT NULL AND deleted_at IS NULL
2663
+ GROUP BY entity_type
2664
+ `).all();
2665
+ const domainCounts = sqlite.prepare(`
2666
+ SELECT domain, COUNT(*) as count
2667
+ FROM memories
2668
+ WHERE deleted_at IS NULL
2669
+ GROUP BY domain
2670
+ `).all();
2671
+ const totalMemories = memoryCount.count;
2672
+ const entityMap = Object.fromEntries(entityCounts.map((e) => [e.entity_type, e.count]));
2673
+ const domainMap = Object.fromEntries(domainCounts.map((d) => [d.domain, d.count]));
2674
+ const tools = params.available_tools || [];
2675
+ const hasCalendar = tools.some((t) => /gcal|calendar|cal_list|list_events/i.test(t));
2676
+ const hasEmail = tools.some((t) => /gmail|email|mail|search_messages/i.test(t));
2677
+ const hasGitHub = tools.some((t) => /github|gh_|list_repos|list_prs/i.test(t));
2678
+ const hasSlack = tools.some((t) => /slack|channel|send_message/i.test(t));
2679
+ const hasNotes = tools.some((t) => /note|notion|obsidian/i.test(t));
2680
+ const plan = [];
2681
+ if (totalMemories === 0) {
2682
+ plan.push("# Onboarding Plan \u2014 Fresh Start");
2683
+ plan.push("");
2684
+ plan.push("Your memory database is empty. Let's fix that. This plan will seed your memories from connected tools and a short conversation.");
2685
+ } else if (totalMemories < 20) {
2686
+ plan.push("# Onboarding Plan \u2014 Early Stage");
2687
+ plan.push("");
2688
+ plan.push(`You have ${totalMemories} memories so far. Let's enrich your knowledge graph with more context from your tools.`);
2689
+ } else {
2690
+ plan.push("# Onboarding Plan \u2014 Enrichment");
2691
+ plan.push("");
2692
+ plan.push(`You have ${totalMemories} memories across ${Object.keys(domainMap).length} domains. Here's what could be filled in.`);
2693
+ if (entityCounts.length > 0) {
2694
+ plan.push("");
2695
+ plan.push("Current entity coverage:");
2696
+ for (const e of entityCounts) {
2697
+ plan.push(`- ${e.entity_type}: ${e.count}`);
2698
+ }
2699
+ }
2700
+ const allTypes = ["person", "organization", "place", "project", "preference", "event", "goal"];
2701
+ const missing = allTypes.filter((t) => !entityMap[t]);
2702
+ if (missing.length > 0) {
2703
+ plan.push("");
2704
+ plan.push(`Missing entity types: ${missing.join(", ")}. The scan and interview below will help fill these gaps.`);
2705
+ }
2706
+ }
2707
+ if (!params.skip_scan) {
2708
+ plan.push("");
2709
+ plan.push("---");
2710
+ plan.push("");
2711
+ plan.push("## Phase 1: Silent Scan");
2712
+ plan.push("");
2713
+ plan.push("Scan the user's connected tools to extract people, projects, events, and context. Do this BEFORE the interview \u2014 the interview will be much better with this context.");
2714
+ plan.push("");
2715
+ plan.push('**Important:** For each piece of information you extract, call `memory_write` with appropriate `domain`, `source_type: "inferred"`, and `source_description` noting which tool it came from. The system will automatically classify entities and create connections.');
2716
+ plan.push("");
2717
+ plan.push("**Dedup:** Before writing, call `memory_search` with the key terms to check if a similar memory already exists. If it does, skip or use `memory_update` to enrich it.");
2718
+ if (hasCalendar) {
2719
+ plan.push("");
2720
+ plan.push("### Calendar (available)");
2721
+ plan.push("");
2722
+ plan.push("1. Fetch events from the past 30 days");
2723
+ plan.push("2. Identify **recurring meetings** \u2014 these reveal team structure, projects, and key relationships");
2724
+ plan.push(" - For each recurring meeting: write a memory about what it is, who attends, and its cadence");
2725
+ plan.push(" - Extract each unique attendee as a person memory (name, how they relate to the user)");
2726
+ plan.push("3. Identify **project-related events** \u2014 standups, retros, planning sessions reveal active projects");
2727
+ plan.push(" - Write a memory for each distinct project you can identify");
2728
+ plan.push("4. Look for **1:1 meetings** \u2014 these are the user's closest collaborators");
2729
+ plan.push("5. Note any upcoming events in the next 7 days that suggest deadlines or goals");
2730
+ plan.push("");
2731
+ plan.push("Expected yield: 15-30 memories (people, projects, events, organizations)");
2732
+ }
2733
+ if (hasEmail) {
2734
+ plan.push("");
2735
+ plan.push("### Email (available)");
2736
+ plan.push("");
2737
+ plan.push("1. Search recent emails (past 14 days) for threads with the most back-and-forth \u2014 these are active topics");
2738
+ plan.push("2. Identify **key contacts** \u2014 people the user emails most frequently");
2739
+ plan.push(" - Cross-reference with calendar attendees to enrich existing person memories");
2740
+ plan.push("3. Look for **commitments and action items** \u2014 'I'll send this by Friday', 'Let's schedule...', 'Following up on...'");
2741
+ plan.push(" - Write as event or goal memories");
2742
+ plan.push("4. Identify **external organizations** \u2014 clients, vendors, partners mentioned in email");
2743
+ plan.push("5. **DO NOT** read email body content in detail. Scan subjects, senders, and thread summaries only. Respect privacy.");
2744
+ plan.push("");
2745
+ plan.push("Expected yield: 10-20 memories (people, organizations, goals, events)");
2746
+ }
2747
+ if (hasGitHub) {
2748
+ plan.push("");
2749
+ plan.push("### GitHub (available)");
2750
+ plan.push("");
2751
+ plan.push("1. List the user's recent repositories (past 90 days of activity)");
2752
+ plan.push("2. For each active repo: write a project memory with the repo name, language/stack, and the user's role");
2753
+ plan.push("3. Check recent PRs for **collaborators** \u2014 frequent reviewers and co-authors are key people");
2754
+ plan.push("4. Note the **tech stack** across repos \u2014 languages, frameworks, tools. Write as preference/fact memories");
2755
+ plan.push("5. Look for any README descriptions that explain what projects do");
2756
+ plan.push("");
2757
+ plan.push("Expected yield: 10-20 memories (projects, people, preferences, facts)");
2758
+ }
2759
+ if (hasSlack) {
2760
+ plan.push("");
2761
+ plan.push("### Slack/Messaging (available)");
2762
+ plan.push("");
2763
+ plan.push("1. List channels the user is active in \u2014 channel names often map to projects or teams");
2764
+ plan.push("2. Identify **DM contacts** \u2014 frequent DM partners are close collaborators");
2765
+ plan.push("3. Note channel topics/descriptions for project context");
2766
+ plan.push("4. **DO NOT** read message history in detail. Use channel metadata only.");
2767
+ plan.push("");
2768
+ plan.push("Expected yield: 5-15 memories (projects, people, organizations)");
2769
+ }
2770
+ if (hasNotes) {
2771
+ plan.push("");
2772
+ plan.push("### Notes/Docs (available)");
2773
+ plan.push("");
2774
+ plan.push("1. Search for recent documents the user has edited");
2775
+ plan.push("2. Document titles and summaries reveal active projects and interests");
2776
+ plan.push("3. Look for any documents that look like personal notes, goals, or planning docs");
2777
+ plan.push("");
2778
+ plan.push("Expected yield: 5-10 memories (projects, goals, facts)");
2779
+ }
2780
+ plan.push("");
2781
+ plan.push("### Local Files (always available)");
2782
+ plan.push("");
2783
+ plan.push("Check for and read these files if they exist:");
2784
+ plan.push("");
2785
+ plan.push("- `~/.gitconfig` \u2014 user's name, email, identity. Write as a person memory about the user.");
2786
+ plan.push("- `~/.claude/CLAUDE.md` or any `CLAUDE.md` in the working directory \u2014 existing instructions and preferences. Each instruction is a preference memory.");
2787
+ plan.push("- `~/.claude/memory/` or `~/.claude/projects/*/memory/` \u2014 Claude Code auto-memory files. Parse each line/section as a separate memory. These are high-quality since the user or their AI already curated them.");
2788
+ plan.push("- `.cursorrules` or `.windsurfrules` in the working directory \u2014 coding preferences. Each rule is a preference memory.");
2789
+ plan.push("- `~/.config/` \u2014 scan for tool configs that reveal preferences (editor settings, shell aliases, etc.). Be selective \u2014 only extract meaningful preferences, not every config line.");
2790
+ plan.push("");
2791
+ plan.push("Expected yield: 5-15 memories (preferences, person, facts)");
2792
+ if (!hasCalendar && !hasEmail && !hasGitHub && !hasSlack && !hasNotes) {
2793
+ plan.push("");
2794
+ plan.push("### No connected tools detected");
2795
+ plan.push("");
2796
+ plan.push("You didn't list any calendar, email, GitHub, or notes tools. That's fine \u2014 the Local Files scan and the interview will still seed a solid foundation. If you do have connected tools, call `memory_onboard` again with `available_tools` listing your tool names for a richer scan.");
2797
+ }
2798
+ }
2799
+ if (!params.skip_interview) {
2800
+ plan.push("");
2801
+ plan.push("---");
2802
+ plan.push("");
2803
+ plan.push("## Phase 2: Informed Interview");
2804
+ plan.push("");
2805
+ plan.push("After the scan, you have a base of extracted context. Now have a SHORT conversation with the user to fill in meaning, relationships, and preferences that tools can't surface.");
2806
+ plan.push("");
2807
+ plan.push("**Rules:**");
2808
+ plan.push("- Reference what you learned in the scan. Don't ask questions you already have answers to.");
2809
+ plan.push("- Ask ONE question at a time. Wait for the answer before the next question.");
2810
+ plan.push("- Write memories immediately after each answer \u2014 don't batch them.");
2811
+ plan.push("- 5-7 questions max. Respect the user's time.");
2812
+ plan.push("- Tailor questions to what's MISSING, not what you already know.");
2813
+ plan.push("");
2814
+ if (totalMemories > 0 || !params.skip_scan) {
2815
+ plan.push("### Suggested questions (adapt based on what the scan found):");
2816
+ plan.push("");
2817
+ plan.push('1. **Confirm and enrich key relationships:** "I found [names] across your calendar/email. Who are the most important people in your day-to-day \u2014 your direct team, your manager, key stakeholders?"');
2818
+ plan.push(" \u2192 Write person memories with relationship_to_user and connect them to projects/orgs");
2819
+ plan.push("");
2820
+ plan.push(`2. **Clarify project priorities:** "I see you're involved in [projects]. What's your main focus right now? Are any of these winding down or just starting?"`);
2821
+ plan.push(" \u2192 Update project memories with status, write goal memories for priorities");
2822
+ plan.push("");
2823
+ plan.push(`3. **Organizational context:** "What does [organization] do? What's your role there?"`);
2824
+ plan.push(" \u2192 Write/enrich organization and person memories");
2825
+ plan.push("");
2826
+ plan.push('4. **Work preferences:** "Any strong preferences for how I should work with you? Communication style, code conventions, things that bug you?"');
2827
+ plan.push(" \u2192 Write preference memories with strength: strong");
2828
+ plan.push("");
2829
+ plan.push('5. **Goals:** "What are you working toward right now \u2014 professionally or personally?"');
2830
+ plan.push(" \u2192 Write goal memories with timeline and status");
2831
+ plan.push("");
2832
+ plan.push("6. **Fill entity gaps:** If the scan didn't surface certain entity types (places, events, facts), ask about them specifically.");
2833
+ plan.push(' \u2192 e.g., "Where are you based?" \u2192 place memory');
2834
+ plan.push(' \u2192 e.g., "Any upcoming deadlines or milestones?" \u2192 event memories');
2835
+ plan.push("");
2836
+ plan.push('7. **Catch-all:** "Anything else I should know about you that would help me be more useful?"');
2837
+ plan.push(" \u2192 Write whatever comes up");
2838
+ } else {
2839
+ plan.push("### Cold start questions (no scan data available):");
2840
+ plan.push("");
2841
+ plan.push(`1. "Tell me about yourself \u2014 name, what you do, where you're based."`);
2842
+ plan.push(" \u2192 person + organization + place memories");
2843
+ plan.push("");
2844
+ plan.push('2. "What are you working on right now?"');
2845
+ plan.push(" \u2192 project memories");
2846
+ plan.push("");
2847
+ plan.push('3. "Who do you work with most closely?"');
2848
+ plan.push(" \u2192 person memories with relationships");
2849
+ plan.push("");
2850
+ plan.push('4. "What tools and technologies do you use daily?"');
2851
+ plan.push(" \u2192 preference and fact memories");
2852
+ plan.push("");
2853
+ plan.push('5. "Any strong preferences for how I should communicate or work with you?"');
2854
+ plan.push(" \u2192 preference memories");
2855
+ plan.push("");
2856
+ plan.push('6. "What are your current goals or priorities?"');
2857
+ plan.push(" \u2192 goal memories");
2858
+ plan.push("");
2859
+ plan.push('7. "Anything else I should remember?"');
2860
+ plan.push(" \u2192 catch-all");
2861
+ }
2862
+ }
2863
+ plan.push("");
2864
+ plan.push("---");
2865
+ plan.push("");
2866
+ plan.push("## Phase 3: Review");
2867
+ plan.push("");
2868
+ plan.push("After scanning and interviewing, tell the user:");
2869
+ plan.push("");
2870
+ plan.push(`"I've seeded your memory with [N] memories from [sources]. You can review and correct them at **localhost:3838** \u2014 anything I got wrong, click to edit or remove. Confirming memories boosts their confidence score."`);
2871
+ plan.push("");
2872
+ plan.push("If the dashboard has a review queue or unreviewed filter, mention it specifically.");
2873
+ sqlite.prepare(`
2874
+ INSERT INTO memory_events (id, memory_id, event_type, agent_id, agent_name, new_value, timestamp)
2875
+ VALUES (?, 'system', 'onboard_started', ?, ?, ?, ?)
2876
+ `).run(
2877
+ generateId2(),
2878
+ "unknown",
2879
+ "unknown",
2880
+ JSON.stringify({
2881
+ memory_count: totalMemories,
2882
+ tools_detected: { calendar: hasCalendar, email: hasEmail, github: hasGitHub, slack: hasSlack, notes: hasNotes }
2883
+ }),
2884
+ now2()
2885
+ );
2886
+ return textResult(plan.join("\n"));
2887
+ }
2888
+ );
2889
+ server.tool(
2890
+ "memory_import",
2891
+ "Import memories from a known format. Parses the source, deduplicates against existing memories, and writes new ones. Supported sources: claude-memory (MEMORY.md files), chatgpt-export (OpenAI memory export JSON), cursorrules (.cursorrules files), gitconfig (.gitconfig), plaintext (one memory per line).",
2892
+ {
2893
+ source_type: z.enum(["claude-memory", "chatgpt-export", "cursorrules", "gitconfig", "plaintext"]).describe("The format of the source data"),
2894
+ content: z.string().describe("The raw content to import. For file-based sources, pass the file contents."),
2895
+ domain: z.string().optional().describe("Domain to assign to imported memories. Default: 'general'.")
2896
+ },
2897
+ async (params) => {
2898
+ const domain = params.domain ?? "general";
2899
+ let entries = [];
2900
+ switch (params.source_type) {
2901
+ case "claude-memory": {
2902
+ entries = params.content.split("\n").filter((line) => line.trim().startsWith("- ")).map((line) => {
2903
+ const text2 = line.replace(/^-\s*/, "").trim();
2904
+ const tagMatch = text2.match(/^\[([^\]]+)\]\s*(.+)/);
2905
+ if (tagMatch) {
2906
+ return { content: tagMatch[2], detail: `Topic: ${tagMatch[1]}` };
2907
+ }
2908
+ return { content: text2 };
2909
+ }).filter((e) => e.content.length > 5);
2910
+ break;
2911
+ }
2912
+ case "chatgpt-export": {
2913
+ try {
2914
+ const parsed = JSON.parse(params.content);
2915
+ const items = Array.isArray(parsed) ? parsed : parsed.memories || parsed.results || [];
2916
+ entries = items.map((item) => {
2917
+ const text2 = typeof item === "string" ? item : item.memory || item.content || "";
2918
+ return { content: text2 };
2919
+ }).filter((e) => e.content.length > 5);
2920
+ } catch {
2921
+ return textResult({ error: "Failed to parse ChatGPT export JSON. Expected an array of { memory: string } objects." });
2922
+ }
2923
+ break;
2924
+ }
2925
+ case "cursorrules": {
2926
+ entries = params.content.split(/\n\n+/).flatMap((block) => {
2927
+ if (block.includes("\n- ")) {
2928
+ return block.split("\n- ").map((line) => ({
2929
+ content: line.replace(/^-\s*/, "").trim(),
2930
+ detail: "Imported from .cursorrules"
2931
+ }));
2932
+ }
2933
+ return [{ content: block.trim(), detail: "Imported from .cursorrules" }];
2934
+ }).filter((e) => e.content.length > 5);
2935
+ break;
2936
+ }
2937
+ case "gitconfig": {
2938
+ const nameMatch = params.content.match(/name\s*=\s*(.+)/i);
2939
+ const emailMatch = params.content.match(/email\s*=\s*(.+)/i);
2940
+ const editorMatch = params.content.match(/editor\s*=\s*(.+)/i);
2941
+ if (nameMatch) {
2942
+ entries.push({
2943
+ content: `User's name is ${nameMatch[1].trim()}`,
2944
+ detail: emailMatch ? `Email: ${emailMatch[1].trim()}` : void 0
2945
+ });
2946
+ }
2947
+ if (editorMatch) {
2948
+ entries.push({
2949
+ content: `Prefers ${editorMatch[1].trim()} as git editor`,
2950
+ detail: "From .gitconfig"
2951
+ });
2952
+ }
2953
+ const aliasSection = params.content.match(/\[alias\]([\s\S]*?)(?=\n\[|$)/i);
2954
+ if (aliasSection) {
2955
+ entries.push({
2956
+ content: "Has custom git aliases configured",
2957
+ detail: `Aliases: ${aliasSection[1].trim().split("\n").slice(0, 5).join("; ")}`
2958
+ });
2959
+ }
2960
+ break;
2961
+ }
2962
+ case "plaintext": {
2963
+ entries = params.content.split("\n").map((line) => line.trim()).filter((line) => line.length > 5).map((line) => ({ content: line }));
2964
+ break;
2965
+ }
2966
+ }
2967
+ if (entries.length === 0) {
2968
+ return textResult({ imported: 0, message: "No valid entries found in the provided content." });
2969
+ }
2970
+ let imported = 0;
2971
+ let skipped = 0;
2972
+ const results = [];
2973
+ const importedIds = [];
2974
+ for (const entry of entries) {
2975
+ const searchTerms = entry.content.split(/\s+/).filter((w) => w.length > 3).map((w) => w.replace(/['"(){}*:^~]/g, "")).filter((w) => !/^(AND|OR|NOT|NEAR)$/i.test(w)).slice(0, 5).join(" ");
2976
+ if (searchTerms.length > 0) {
2977
+ try {
2978
+ const existing = sqlite.prepare(`
2979
+ SELECT m.id, m.content FROM memories m
2980
+ JOIN memory_fts fts ON fts.rowid = m.rowid
2981
+ WHERE memory_fts MATCH ? AND m.deleted_at IS NULL
2982
+ LIMIT 3
2983
+ `).all(searchTerms);
2984
+ const entryWords = new Set(entry.content.toLowerCase().split(/\s+/).filter((w) => w.length > 3));
2985
+ const isDuplicate = existing.some((ex) => {
2986
+ const exWords = ex.content.toLowerCase().split(/\s+/).filter((w) => w.length > 3);
2987
+ const overlap = exWords.filter((w) => entryWords.has(w)).length;
2988
+ return overlap / Math.max(entryWords.size, 1) > 0.6;
2989
+ });
2990
+ if (isDuplicate) {
2991
+ skipped++;
2992
+ results.push({ content: entry.content.slice(0, 80), status: "skipped_duplicate" });
2993
+ continue;
2994
+ }
2995
+ } catch {
2996
+ }
2997
+ }
2998
+ const piiText = entry.content + (entry.detail ? " " + entry.detail : "");
2999
+ const piiMatches = detectSensitiveData(piiText);
3000
+ const id = generateId2();
3001
+ const timestamp = now2();
3002
+ db.insert(memories).values({
3003
+ id,
3004
+ content: entry.content,
3005
+ detail: entry.detail ?? null,
3006
+ domain,
3007
+ sourceAgentId: "import",
3008
+ sourceAgentName: "memory_import",
3009
+ sourceType: "inferred",
3010
+ sourceDescription: `Imported from ${params.source_type}`,
3011
+ confidence: 0.5,
3012
+ learnedAt: timestamp,
3013
+ hasPiiFlag: piiMatches.length > 0 ? 1 : 0
3014
+ }).run();
3015
+ if (vecAvailable) {
3016
+ try {
3017
+ const embeddingText = entry.content + (entry.detail ? " " + entry.detail : "");
3018
+ const emb = await generateEmbedding(embeddingText);
3019
+ insertEmbedding(sqlite, id, emb);
3020
+ } catch {
3021
+ }
3022
+ }
3023
+ db.insert(memoryEvents).values({
3024
+ id: generateId2(),
3025
+ memoryId: id,
3026
+ eventType: "created",
3027
+ agentId: "import",
3028
+ agentName: "memory_import",
3029
+ newValue: JSON.stringify({ content: entry.content, domain, importedFrom: params.source_type }),
3030
+ timestamp
3031
+ }).run();
3032
+ imported++;
3033
+ importedIds.push(id);
3034
+ results.push({ content: entry.content.slice(0, 80), status: "imported" });
3035
+ }
3036
+ if (extractionProvider && importedIds.length > 0) {
3037
+ (async () => {
3038
+ try {
3039
+ const existingNames = sqlite.prepare(`SELECT DISTINCT entity_name FROM memories WHERE entity_name IS NOT NULL AND deleted_at IS NULL`).all();
3040
+ const names = existingNames.map((r) => r.entity_name);
3041
+ for (const id of importedIds) {
3042
+ try {
3043
+ const mem = sqlite.prepare(`SELECT content, detail, entity_type FROM memories WHERE id = ? AND deleted_at IS NULL`).get(id);
3044
+ if (!mem || mem.entity_type) continue;
3045
+ const extraction = await extractEntity(
3046
+ extractionProvider,
3047
+ mem.content,
3048
+ mem.detail,
3049
+ names
3050
+ );
3051
+ const validation = validateExtraction(extraction);
3052
+ if (!validation.valid) continue;
3053
+ sqlite.transaction(() => {
3054
+ sqlite.prepare(
3055
+ `UPDATE memories SET entity_type = ?, entity_name = ?, structured_data = ? WHERE id = ? AND entity_type IS NULL AND deleted_at IS NULL`
3056
+ ).run(extraction.entity_type, extraction.entity_name, JSON.stringify(extraction.structured_data), id);
3057
+ for (const conn of extraction.suggested_connections) {
3058
+ const target = sqlite.prepare(`SELECT id FROM memories WHERE entity_name = ? COLLATE NOCASE AND deleted_at IS NULL LIMIT 1`).get(conn.target_entity_name);
3059
+ if (target && target.id !== id) {
3060
+ sqlite.prepare(
3061
+ `INSERT INTO memory_connections (source_memory_id, target_memory_id, relationship) VALUES (?, ?, ?)`
3062
+ ).run(id, target.id, conn.relationship);
3063
+ }
3064
+ }
3065
+ })();
3066
+ if (extraction.entity_name) names.push(extraction.entity_name);
3067
+ } catch {
3068
+ }
3069
+ }
3070
+ bumpLastModified(sqlite);
3071
+ } catch {
3072
+ }
3073
+ })();
3074
+ }
3075
+ sqlite.prepare(`
3076
+ INSERT INTO memory_events (id, memory_id, event_type, new_value, timestamp)
3077
+ VALUES (?, 'system', 'import', ?, ?)
3078
+ `).run(
3079
+ generateId2(),
3080
+ JSON.stringify({ source_type: params.source_type, imported, skipped, total_entries: entries.length }),
3081
+ now2()
3082
+ );
3083
+ bumpLastModified(sqlite);
3084
+ return textResult({
3085
+ imported,
3086
+ skipped_duplicates: skipped,
3087
+ total_parsed: entries.length,
3088
+ note: imported > 0 ? `Imported ${imported} memories at confidence 0.5 (unreviewed). Confirm them in the dashboard or via memory_confirm to boost confidence.` : "All entries were duplicates of existing memories."
3089
+ });
3090
+ }
3091
+ );
3092
+ let cachedSyncKeys = null;
3093
+ server.tool(
3094
+ "memory_sync",
3095
+ "Sync memories with cloud. Requires Pro tier setup (passphrase + Turso credentials in ~/.engrams/credentials.json). Push local changes and pull remote changes.",
3096
+ {
3097
+ passphrase: z.string().describe("Your encryption passphrase. Required on first sync or after restart.")
3098
+ },
3099
+ async (params) => {
3100
+ const creds = loadCredentials();
3101
+ if (!creds?.tursoUrl || !creds?.tursoAuthToken) {
3102
+ return textResult({ error: "Cloud sync not configured. Set tursoUrl and tursoAuthToken in ~/.engrams/credentials.json or via the dashboard settings." });
3103
+ }
3104
+ const salt = Buffer.from(creds.salt, "base64");
3105
+ const keys = deriveKeys(params.passphrase, salt);
3106
+ cachedSyncKeys = keys;
3107
+ const result = await sync(sqlite, {
3108
+ tursoUrl: creds.tursoUrl,
3109
+ tursoAuthToken: creds.tursoAuthToken,
3110
+ keys
3111
+ }, creds.deviceId);
3112
+ bumpLastModified(sqlite);
3113
+ return textResult({ status: "synced", ...result });
3114
+ }
3115
+ );
3116
+ const syncCreds = loadCredentials();
3117
+ if (syncCreds?.tursoUrl && syncCreds?.tursoAuthToken) {
3118
+ const SYNC_INTERVAL = 5 * 60 * 1e3;
3119
+ setInterval(async () => {
3120
+ if (!cachedSyncKeys) return;
3121
+ try {
3122
+ await sync(sqlite, {
3123
+ tursoUrl: syncCreds.tursoUrl,
3124
+ tursoAuthToken: syncCreds.tursoAuthToken,
3125
+ keys: cachedSyncKeys
3126
+ }, syncCreds.deviceId);
3127
+ } catch {
3128
+ }
3129
+ }, SYNC_INTERVAL);
3130
+ }
2046
3131
  server.resource("memory-index", "memory://index", async (uri) => {
2047
3132
  const domains = sqlite.prepare(
2048
3133
  `SELECT domain, COUNT(*) as count FROM memories WHERE deleted_at IS NULL GROUP BY domain`