engrams 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1136 -51
- package/dist/http.js +16 -4
- package/dist/index.js +1136 -51
- package/package.json +4 -2
package/dist/cli.js
CHANGED
|
@@ -45,12 +45,14 @@ var init_schema = __esm({
|
|
|
45
45
|
hasPiiFlag: integer("has_pii_flag").notNull().default(0),
|
|
46
46
|
entityType: text("entity_type"),
|
|
47
47
|
entityName: text("entity_name"),
|
|
48
|
-
structuredData: text("structured_data")
|
|
48
|
+
structuredData: text("structured_data"),
|
|
49
|
+
updatedAt: text("updated_at")
|
|
49
50
|
});
|
|
50
51
|
memoryConnections = sqliteTable("memory_connections", {
|
|
51
52
|
sourceMemoryId: text("source_memory_id").notNull().references(() => memories.id),
|
|
52
53
|
targetMemoryId: text("target_memory_id").notNull().references(() => memories.id),
|
|
53
|
-
relationship: text("relationship").notNull()
|
|
54
|
+
relationship: text("relationship").notNull(),
|
|
55
|
+
updatedAt: text("updated_at")
|
|
54
56
|
});
|
|
55
57
|
memoryEvents = sqliteTable("memory_events", {
|
|
56
58
|
id: text("id").primaryKey(),
|
|
@@ -239,6 +241,38 @@ function runMigrations(sqlite) {
|
|
|
239
241
|
sqlite.exec(`CREATE INDEX IF NOT EXISTS idx_memories_entity_type ON memories(entity_type) WHERE deleted_at IS NULL`);
|
|
240
242
|
sqlite.exec(`CREATE INDEX IF NOT EXISTS idx_memories_entity_name ON memories(entity_name) WHERE deleted_at IS NULL`);
|
|
241
243
|
});
|
|
244
|
+
runMigration(sqlite, "add_updated_at", () => {
|
|
245
|
+
sqlite.exec(`ALTER TABLE memories ADD COLUMN updated_at TEXT`);
|
|
246
|
+
sqlite.exec(`UPDATE memories SET updated_at = COALESCE(confirmed_at, learned_at, datetime('now')) WHERE updated_at IS NULL`);
|
|
247
|
+
sqlite.exec(`ALTER TABLE memory_connections ADD COLUMN updated_at TEXT`);
|
|
248
|
+
sqlite.exec(`UPDATE memory_connections SET updated_at = datetime('now') WHERE updated_at IS NULL`);
|
|
249
|
+
sqlite.exec(`
|
|
250
|
+
CREATE TRIGGER IF NOT EXISTS memories_updated_at_insert
|
|
251
|
+
AFTER INSERT ON memories
|
|
252
|
+
BEGIN
|
|
253
|
+
UPDATE memories SET updated_at = datetime('now') WHERE id = NEW.id AND updated_at IS NULL;
|
|
254
|
+
END;
|
|
255
|
+
`);
|
|
256
|
+
sqlite.exec(`
|
|
257
|
+
CREATE TRIGGER IF NOT EXISTS memories_updated_at_update
|
|
258
|
+
AFTER UPDATE ON memories
|
|
259
|
+
WHEN NEW.updated_at IS OLD.updated_at OR NEW.updated_at IS NULL
|
|
260
|
+
BEGIN
|
|
261
|
+
UPDATE memories SET updated_at = datetime('now') WHERE id = NEW.id;
|
|
262
|
+
END;
|
|
263
|
+
`);
|
|
264
|
+
sqlite.exec(`
|
|
265
|
+
CREATE TRIGGER IF NOT EXISTS connections_updated_at_insert
|
|
266
|
+
AFTER INSERT ON memory_connections
|
|
267
|
+
BEGIN
|
|
268
|
+
UPDATE memory_connections SET updated_at = datetime('now')
|
|
269
|
+
WHERE source_memory_id = NEW.source_memory_id
|
|
270
|
+
AND target_memory_id = NEW.target_memory_id
|
|
271
|
+
AND relationship = NEW.relationship
|
|
272
|
+
AND updated_at IS NULL;
|
|
273
|
+
END;
|
|
274
|
+
`);
|
|
275
|
+
});
|
|
242
276
|
runMigration(sqlite, "fts_add_entity_name", () => {
|
|
243
277
|
sqlite.exec(`DROP TRIGGER IF EXISTS memory_fts_insert`);
|
|
244
278
|
sqlite.exec(`DROP TRIGGER IF EXISTS memory_fts_delete`);
|
|
@@ -660,21 +694,24 @@ var init_pii = __esm({
|
|
|
660
694
|
}
|
|
661
695
|
});
|
|
662
696
|
|
|
697
|
+
// ../core/dist/llm-utils.js
|
|
698
|
+
function parseLLMJson(text2) {
|
|
699
|
+
const cleaned = text2.replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trim();
|
|
700
|
+
return JSON.parse(cleaned);
|
|
701
|
+
}
|
|
702
|
+
var init_llm_utils = __esm({
|
|
703
|
+
"../core/dist/llm-utils.js"() {
|
|
704
|
+
"use strict";
|
|
705
|
+
}
|
|
706
|
+
});
|
|
707
|
+
|
|
663
708
|
// ../core/dist/entity-extraction.js
|
|
664
|
-
async function extractEntity(content, detail, existingEntityNames) {
|
|
665
|
-
const { default: Anthropic2 } = await import("@anthropic-ai/sdk");
|
|
666
|
-
const client = new Anthropic2();
|
|
709
|
+
async function extractEntity(provider, content, detail, existingEntityNames) {
|
|
667
710
|
const existingNamesHint = existingEntityNames && existingEntityNames.length > 0 ? `
|
|
668
711
|
|
|
669
712
|
Existing entity names in the system (prefer matching these over creating new ones):
|
|
670
713
|
${existingEntityNames.slice(0, 50).map((n) => `- ${n}`).join("\n")}` : "";
|
|
671
|
-
const
|
|
672
|
-
model: "claude-sonnet-4-5-20250514",
|
|
673
|
-
max_tokens: 500,
|
|
674
|
-
messages: [
|
|
675
|
-
{
|
|
676
|
-
role: "user",
|
|
677
|
-
content: `Classify this memory and extract structured data.
|
|
714
|
+
const prompt = `Classify this memory and extract structured data.
|
|
678
715
|
|
|
679
716
|
Memory: ${content}${detail ? `
|
|
680
717
|
Detail: ${detail}` : ""}${existingNamesHint}
|
|
@@ -709,17 +746,528 @@ For structured_data, include relevant fields:
|
|
|
709
746
|
- preference: category, strength (strong/mild/contextual)
|
|
710
747
|
- event: what, when, who
|
|
711
748
|
- goal: what, timeline, status (active/achieved/abandoned)
|
|
712
|
-
- fact: category
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
});
|
|
716
|
-
const text2 = response.content[0].type === "text" ? response.content[0].text : "";
|
|
717
|
-
const cleaned = text2.replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trim();
|
|
718
|
-
return JSON.parse(cleaned);
|
|
749
|
+
- fact: category`;
|
|
750
|
+
const text2 = await provider.complete(prompt, { maxTokens: 500, json: true });
|
|
751
|
+
return parseLLMJson(text2);
|
|
719
752
|
}
|
|
720
753
|
var init_entity_extraction = __esm({
|
|
721
754
|
"../core/dist/entity-extraction.js"() {
|
|
722
755
|
"use strict";
|
|
756
|
+
init_llm_utils();
|
|
757
|
+
}
|
|
758
|
+
});
|
|
759
|
+
|
|
760
|
+
// ../core/dist/crypto.js
|
|
761
|
+
import { randomBytes, scryptSync, createCipheriv, createDecipheriv } from "crypto";
|
|
762
|
+
function deriveKeys(passphrase, salt) {
|
|
763
|
+
const derived = scryptSync(passphrase, salt, KEY_LENGTH * 2, {
|
|
764
|
+
N: SCRYPT_N,
|
|
765
|
+
r: SCRYPT_R,
|
|
766
|
+
p: SCRYPT_P,
|
|
767
|
+
maxmem: 256 * 1024 * 1024
|
|
768
|
+
// 256 MB — N=131072,r=8 needs ~128 MB
|
|
769
|
+
});
|
|
770
|
+
return {
|
|
771
|
+
encryptionKey: derived.subarray(0, KEY_LENGTH),
|
|
772
|
+
hmacKey: derived.subarray(KEY_LENGTH)
|
|
773
|
+
};
|
|
774
|
+
}
|
|
775
|
+
function encrypt(plaintext, key) {
|
|
776
|
+
const iv = randomBytes(IV_LENGTH);
|
|
777
|
+
const cipher = createCipheriv(ALGORITHM, key, iv);
|
|
778
|
+
const encrypted = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
|
|
779
|
+
const authTag = cipher.getAuthTag();
|
|
780
|
+
return Buffer.concat([iv, encrypted, authTag]).toString("base64");
|
|
781
|
+
}
|
|
782
|
+
function decrypt(encoded, key) {
|
|
783
|
+
const data = Buffer.from(encoded, "base64");
|
|
784
|
+
const iv = data.subarray(0, IV_LENGTH);
|
|
785
|
+
const authTag = data.subarray(data.length - AUTH_TAG_LENGTH);
|
|
786
|
+
const ciphertext = data.subarray(IV_LENGTH, data.length - AUTH_TAG_LENGTH);
|
|
787
|
+
const decipher = createDecipheriv(ALGORITHM, key, iv);
|
|
788
|
+
decipher.setAuthTag(authTag);
|
|
789
|
+
return decipher.update(ciphertext) + decipher.final("utf8");
|
|
790
|
+
}
|
|
791
|
+
function encryptMemory(memory, key) {
|
|
792
|
+
return {
|
|
793
|
+
content: encrypt(memory.content, key),
|
|
794
|
+
detail: memory.detail ? encrypt(memory.detail, key) : null,
|
|
795
|
+
structured_data: memory.structured_data ? encrypt(memory.structured_data, key) : null
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
function decryptMemory(memory, key) {
|
|
799
|
+
return {
|
|
800
|
+
content: decrypt(memory.content, key),
|
|
801
|
+
detail: memory.detail ? decrypt(memory.detail, key) : null,
|
|
802
|
+
structured_data: memory.structured_data ? decrypt(memory.structured_data, key) : null
|
|
803
|
+
};
|
|
804
|
+
}
|
|
805
|
+
var ALGORITHM, IV_LENGTH, AUTH_TAG_LENGTH, SCRYPT_N, SCRYPT_R, SCRYPT_P, KEY_LENGTH;
|
|
806
|
+
var init_crypto = __esm({
|
|
807
|
+
"../core/dist/crypto.js"() {
|
|
808
|
+
"use strict";
|
|
809
|
+
ALGORITHM = "aes-256-gcm";
|
|
810
|
+
IV_LENGTH = 12;
|
|
811
|
+
AUTH_TAG_LENGTH = 16;
|
|
812
|
+
SCRYPT_N = 131072;
|
|
813
|
+
SCRYPT_R = 8;
|
|
814
|
+
SCRYPT_P = 1;
|
|
815
|
+
KEY_LENGTH = 32;
|
|
816
|
+
}
|
|
817
|
+
});
|
|
818
|
+
|
|
819
|
+
// ../core/dist/credentials.js
|
|
820
|
+
import { resolve as resolve3 } from "path";
|
|
821
|
+
import { homedir as homedir3 } from "os";
|
|
822
|
+
import { readFileSync, writeFileSync, existsSync, chmodSync as chmodSync2 } from "fs";
|
|
823
|
+
function loadCredentials() {
|
|
824
|
+
if (!existsSync(CRED_PATH))
|
|
825
|
+
return null;
|
|
826
|
+
try {
|
|
827
|
+
return JSON.parse(readFileSync(CRED_PATH, "utf8"));
|
|
828
|
+
} catch {
|
|
829
|
+
return null;
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
function loadConfig() {
|
|
833
|
+
if (!existsSync(CONFIG_PATH))
|
|
834
|
+
return {};
|
|
835
|
+
try {
|
|
836
|
+
return JSON.parse(readFileSync(CONFIG_PATH, "utf8"));
|
|
837
|
+
} catch {
|
|
838
|
+
return {};
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
function saveConfig(config) {
|
|
842
|
+
writeFileSync(CONFIG_PATH, JSON.stringify(config, null, 2), "utf8");
|
|
843
|
+
try {
|
|
844
|
+
chmodSync2(CONFIG_PATH, 384);
|
|
845
|
+
} catch {
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
var ENGRAMS_DIR, CRED_PATH, CONFIG_PATH;
|
|
849
|
+
var init_credentials = __esm({
|
|
850
|
+
"../core/dist/credentials.js"() {
|
|
851
|
+
"use strict";
|
|
852
|
+
ENGRAMS_DIR = resolve3(homedir3(), ".engrams");
|
|
853
|
+
CRED_PATH = resolve3(ENGRAMS_DIR, "credentials.json");
|
|
854
|
+
CONFIG_PATH = resolve3(ENGRAMS_DIR, "config.json");
|
|
855
|
+
}
|
|
856
|
+
});
|
|
857
|
+
|
|
858
|
+
// ../core/dist/sync.js
|
|
859
|
+
import { createClient } from "@libsql/client";
|
|
860
|
+
async function initRemoteSchema(client) {
|
|
861
|
+
await client.executeMultiple(`
|
|
862
|
+
CREATE TABLE IF NOT EXISTS memories (
|
|
863
|
+
id TEXT PRIMARY KEY,
|
|
864
|
+
content TEXT NOT NULL,
|
|
865
|
+
detail TEXT,
|
|
866
|
+
domain TEXT NOT NULL DEFAULT 'general',
|
|
867
|
+
source_agent_id TEXT NOT NULL,
|
|
868
|
+
source_agent_name TEXT NOT NULL,
|
|
869
|
+
cross_agent_id TEXT,
|
|
870
|
+
cross_agent_name TEXT,
|
|
871
|
+
source_type TEXT NOT NULL,
|
|
872
|
+
source_description TEXT,
|
|
873
|
+
confidence REAL NOT NULL DEFAULT 0.7,
|
|
874
|
+
confirmed_count INTEGER NOT NULL DEFAULT 0,
|
|
875
|
+
corrected_count INTEGER NOT NULL DEFAULT 0,
|
|
876
|
+
mistake_count INTEGER NOT NULL DEFAULT 0,
|
|
877
|
+
used_count INTEGER NOT NULL DEFAULT 0,
|
|
878
|
+
learned_at TEXT,
|
|
879
|
+
confirmed_at TEXT,
|
|
880
|
+
last_used_at TEXT,
|
|
881
|
+
deleted_at TEXT,
|
|
882
|
+
has_pii_flag INTEGER NOT NULL DEFAULT 0,
|
|
883
|
+
entity_type TEXT,
|
|
884
|
+
entity_name TEXT,
|
|
885
|
+
structured_data TEXT,
|
|
886
|
+
device_id TEXT,
|
|
887
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
888
|
+
);
|
|
889
|
+
|
|
890
|
+
CREATE TABLE IF NOT EXISTS memory_connections (
|
|
891
|
+
source_memory_id TEXT NOT NULL,
|
|
892
|
+
target_memory_id TEXT NOT NULL,
|
|
893
|
+
relationship TEXT NOT NULL,
|
|
894
|
+
device_id TEXT,
|
|
895
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
896
|
+
);
|
|
897
|
+
|
|
898
|
+
CREATE TABLE IF NOT EXISTS memory_events (
|
|
899
|
+
id TEXT PRIMARY KEY,
|
|
900
|
+
memory_id TEXT NOT NULL,
|
|
901
|
+
event_type TEXT NOT NULL,
|
|
902
|
+
agent_id TEXT,
|
|
903
|
+
agent_name TEXT,
|
|
904
|
+
old_value TEXT,
|
|
905
|
+
new_value TEXT,
|
|
906
|
+
timestamp TEXT NOT NULL,
|
|
907
|
+
device_id TEXT
|
|
908
|
+
);
|
|
909
|
+
|
|
910
|
+
CREATE TABLE IF NOT EXISTS sync_log (
|
|
911
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
912
|
+
device_id TEXT NOT NULL,
|
|
913
|
+
synced_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
914
|
+
pushed INTEGER NOT NULL DEFAULT 0,
|
|
915
|
+
pulled INTEGER NOT NULL DEFAULT 0
|
|
916
|
+
);
|
|
917
|
+
`);
|
|
918
|
+
}
|
|
919
|
+
async function pushChanges(sqlite, client, config, deviceId) {
|
|
920
|
+
const lastSync = await getLastSyncTime(client, deviceId);
|
|
921
|
+
const changedMemories = sqlite.prepare(`
|
|
922
|
+
SELECT * FROM memories WHERE updated_at > ? OR (updated_at IS NULL AND learned_at > ?)
|
|
923
|
+
`).all(lastSync, lastSync);
|
|
924
|
+
let pushed = 0;
|
|
925
|
+
for (const mem of changedMemories) {
|
|
926
|
+
const encrypted = encryptMemory({
|
|
927
|
+
content: mem.content,
|
|
928
|
+
detail: mem.detail,
|
|
929
|
+
structured_data: mem.structured_data
|
|
930
|
+
}, config.keys.encryptionKey);
|
|
931
|
+
await client.execute({
|
|
932
|
+
sql: `INSERT OR REPLACE INTO memories
|
|
933
|
+
(id, content, detail, domain, source_agent_id, source_agent_name,
|
|
934
|
+
cross_agent_id, cross_agent_name, source_type, source_description,
|
|
935
|
+
confidence, confirmed_count, corrected_count, mistake_count, used_count,
|
|
936
|
+
learned_at, confirmed_at, last_used_at, deleted_at,
|
|
937
|
+
has_pii_flag, entity_type, entity_name, structured_data,
|
|
938
|
+
device_id, updated_at)
|
|
939
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`,
|
|
940
|
+
args: [
|
|
941
|
+
mem.id,
|
|
942
|
+
encrypted.content,
|
|
943
|
+
encrypted.detail,
|
|
944
|
+
mem.domain,
|
|
945
|
+
mem.source_agent_id,
|
|
946
|
+
mem.source_agent_name,
|
|
947
|
+
mem.cross_agent_id ?? null,
|
|
948
|
+
mem.cross_agent_name ?? null,
|
|
949
|
+
mem.source_type,
|
|
950
|
+
mem.source_description ?? null,
|
|
951
|
+
mem.confidence,
|
|
952
|
+
mem.confirmed_count,
|
|
953
|
+
mem.corrected_count,
|
|
954
|
+
mem.mistake_count,
|
|
955
|
+
mem.used_count,
|
|
956
|
+
mem.learned_at ?? null,
|
|
957
|
+
mem.confirmed_at ?? null,
|
|
958
|
+
mem.last_used_at ?? null,
|
|
959
|
+
mem.deleted_at ?? null,
|
|
960
|
+
mem.has_pii_flag ?? 0,
|
|
961
|
+
mem.entity_type ?? null,
|
|
962
|
+
mem.entity_name ?? null,
|
|
963
|
+
encrypted.structured_data,
|
|
964
|
+
deviceId
|
|
965
|
+
]
|
|
966
|
+
});
|
|
967
|
+
pushed++;
|
|
968
|
+
}
|
|
969
|
+
const changedConnections = sqlite.prepare(`
|
|
970
|
+
SELECT * FROM memory_connections WHERE updated_at > ? OR updated_at IS NULL
|
|
971
|
+
`).all(lastSync);
|
|
972
|
+
for (const conn of changedConnections) {
|
|
973
|
+
await client.execute({
|
|
974
|
+
sql: `INSERT OR REPLACE INTO memory_connections
|
|
975
|
+
(source_memory_id, target_memory_id, relationship, device_id, updated_at)
|
|
976
|
+
VALUES (?, ?, ?, ?, datetime('now'))`,
|
|
977
|
+
args: [
|
|
978
|
+
conn.source_memory_id,
|
|
979
|
+
conn.target_memory_id,
|
|
980
|
+
conn.relationship,
|
|
981
|
+
deviceId
|
|
982
|
+
]
|
|
983
|
+
});
|
|
984
|
+
}
|
|
985
|
+
const changedEvents = sqlite.prepare(`
|
|
986
|
+
SELECT * FROM memory_events WHERE timestamp > ?
|
|
987
|
+
`).all(lastSync);
|
|
988
|
+
for (const evt of changedEvents) {
|
|
989
|
+
await client.execute({
|
|
990
|
+
sql: `INSERT OR IGNORE INTO memory_events
|
|
991
|
+
(id, memory_id, event_type, agent_id, agent_name, old_value, new_value, timestamp, device_id)
|
|
992
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
993
|
+
args: [
|
|
994
|
+
evt.id,
|
|
995
|
+
evt.memory_id,
|
|
996
|
+
evt.event_type,
|
|
997
|
+
evt.agent_id ?? null,
|
|
998
|
+
evt.agent_name ?? null,
|
|
999
|
+
evt.old_value ?? null,
|
|
1000
|
+
evt.new_value ?? null,
|
|
1001
|
+
evt.timestamp,
|
|
1002
|
+
deviceId
|
|
1003
|
+
]
|
|
1004
|
+
});
|
|
1005
|
+
}
|
|
1006
|
+
return pushed + changedConnections.length + changedEvents.length;
|
|
1007
|
+
}
|
|
1008
|
+
async function pullChanges(sqlite, client, config, deviceId) {
|
|
1009
|
+
const lastSync = await getLastSyncTime(client, deviceId);
|
|
1010
|
+
const result = await client.execute({
|
|
1011
|
+
sql: `SELECT * FROM memories WHERE device_id != ? AND updated_at > ?`,
|
|
1012
|
+
args: [deviceId, lastSync]
|
|
1013
|
+
});
|
|
1014
|
+
let pulled = 0;
|
|
1015
|
+
for (const row of result.rows) {
|
|
1016
|
+
const decrypted = decryptMemory({
|
|
1017
|
+
content: row.content,
|
|
1018
|
+
detail: row.detail,
|
|
1019
|
+
structured_data: row.structured_data
|
|
1020
|
+
}, config.keys.encryptionKey);
|
|
1021
|
+
const local = sqlite.prepare(`SELECT updated_at FROM memories WHERE id = ?`).get(row.id);
|
|
1022
|
+
if (local && local.updated_at >= row.updated_at)
|
|
1023
|
+
continue;
|
|
1024
|
+
sqlite.prepare(`
|
|
1025
|
+
INSERT OR REPLACE INTO memories
|
|
1026
|
+
(id, content, detail, domain, source_agent_id, source_agent_name,
|
|
1027
|
+
cross_agent_id, cross_agent_name, source_type, source_description,
|
|
1028
|
+
confidence, confirmed_count, corrected_count, mistake_count, used_count,
|
|
1029
|
+
learned_at, confirmed_at, last_used_at, deleted_at,
|
|
1030
|
+
has_pii_flag, entity_type, entity_name, structured_data, updated_at)
|
|
1031
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1032
|
+
`).run(row.id, decrypted.content, decrypted.detail, row.domain, row.source_agent_id, row.source_agent_name, row.cross_agent_id, row.cross_agent_name, row.source_type, row.source_description, row.confidence, row.confirmed_count, row.corrected_count, row.mistake_count, row.used_count, row.learned_at, row.confirmed_at, row.last_used_at, row.deleted_at, row.has_pii_flag, row.entity_type, row.entity_name, decrypted.structured_data, row.updated_at);
|
|
1033
|
+
pulled++;
|
|
1034
|
+
}
|
|
1035
|
+
const connResult = await client.execute({
|
|
1036
|
+
sql: `SELECT * FROM memory_connections WHERE device_id != ? AND updated_at > ?`,
|
|
1037
|
+
args: [deviceId, lastSync]
|
|
1038
|
+
});
|
|
1039
|
+
for (const row of connResult.rows) {
|
|
1040
|
+
sqlite.prepare(`
|
|
1041
|
+
INSERT OR IGNORE INTO memory_connections (source_memory_id, target_memory_id, relationship, updated_at)
|
|
1042
|
+
VALUES (?, ?, ?, ?)
|
|
1043
|
+
`).run(row.source_memory_id, row.target_memory_id, row.relationship, row.updated_at);
|
|
1044
|
+
pulled++;
|
|
1045
|
+
}
|
|
1046
|
+
const evtResult = await client.execute({
|
|
1047
|
+
sql: `SELECT * FROM memory_events WHERE device_id != ? AND timestamp > ?`,
|
|
1048
|
+
args: [deviceId, lastSync]
|
|
1049
|
+
});
|
|
1050
|
+
for (const row of evtResult.rows) {
|
|
1051
|
+
sqlite.prepare(`
|
|
1052
|
+
INSERT OR IGNORE INTO memory_events (id, memory_id, event_type, agent_id, agent_name, old_value, new_value, timestamp)
|
|
1053
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1054
|
+
`).run(row.id, row.memory_id, row.event_type, row.agent_id, row.agent_name, row.old_value, row.new_value, row.timestamp);
|
|
1055
|
+
pulled++;
|
|
1056
|
+
}
|
|
1057
|
+
return pulled;
|
|
1058
|
+
}
|
|
1059
|
+
async function sync(sqlite, config, deviceId) {
|
|
1060
|
+
const client = createClient({
|
|
1061
|
+
url: config.tursoUrl,
|
|
1062
|
+
authToken: config.tursoAuthToken
|
|
1063
|
+
});
|
|
1064
|
+
try {
|
|
1065
|
+
await initRemoteSchema(client);
|
|
1066
|
+
const pushed = await pushChanges(sqlite, client, config, deviceId);
|
|
1067
|
+
const pulled = await pullChanges(sqlite, client, config, deviceId);
|
|
1068
|
+
await client.execute({
|
|
1069
|
+
sql: `INSERT INTO sync_log (device_id, pushed, pulled) VALUES (?, ?, ?)`,
|
|
1070
|
+
args: [deviceId, pushed, pulled]
|
|
1071
|
+
});
|
|
1072
|
+
sqlite.prepare(`INSERT OR REPLACE INTO engrams_meta (key, value) VALUES ('last_modified', datetime('now'))`).run();
|
|
1073
|
+
return { pushed, pulled, conflicts: 0 };
|
|
1074
|
+
} finally {
|
|
1075
|
+
client.close();
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
async function getLastSyncTime(client, deviceId) {
|
|
1079
|
+
const result = await client.execute({
|
|
1080
|
+
sql: `SELECT synced_at FROM sync_log WHERE device_id = ? ORDER BY synced_at DESC LIMIT 1`,
|
|
1081
|
+
args: [deviceId]
|
|
1082
|
+
});
|
|
1083
|
+
return result.rows.length > 0 ? result.rows[0].synced_at : "1970-01-01T00:00:00Z";
|
|
1084
|
+
}
|
|
1085
|
+
var init_sync = __esm({
|
|
1086
|
+
"../core/dist/sync.js"() {
|
|
1087
|
+
"use strict";
|
|
1088
|
+
init_crypto();
|
|
1089
|
+
}
|
|
1090
|
+
});
|
|
1091
|
+
|
|
1092
|
+
// ../core/dist/llm.js
|
|
1093
|
+
function createLLMProvider(config, task) {
|
|
1094
|
+
const model = config.model || task && DEFAULT_TASK_MODELS[config.provider]?.[task] || DEFAULT_MODELS[config.provider] || "gpt-4o";
|
|
1095
|
+
switch (config.provider) {
|
|
1096
|
+
case "anthropic":
|
|
1097
|
+
return new AnthropicProvider(config.apiKey, model);
|
|
1098
|
+
case "openai":
|
|
1099
|
+
return new OpenAICompatibleProvider(config.apiKey, model, config.baseUrl);
|
|
1100
|
+
case "ollama":
|
|
1101
|
+
return new OpenAICompatibleProvider(void 0, model, config.baseUrl || "http://localhost:11434/v1");
|
|
1102
|
+
default:
|
|
1103
|
+
throw new Error(`Unknown LLM provider: ${config.provider}`);
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
var DEFAULT_MODELS, DEFAULT_TASK_MODELS, AnthropicProvider, OpenAICompatibleProvider;
|
|
1107
|
+
var init_llm = __esm({
|
|
1108
|
+
"../core/dist/llm.js"() {
|
|
1109
|
+
"use strict";
|
|
1110
|
+
DEFAULT_MODELS = {
|
|
1111
|
+
anthropic: "claude-sonnet-4-5-20250514",
|
|
1112
|
+
openai: "gpt-4o",
|
|
1113
|
+
ollama: "llama3.2"
|
|
1114
|
+
};
|
|
1115
|
+
DEFAULT_TASK_MODELS = {
|
|
1116
|
+
anthropic: {
|
|
1117
|
+
extraction: "claude-haiku-4-5-20251001",
|
|
1118
|
+
analysis: "claude-sonnet-4-5-20250514"
|
|
1119
|
+
},
|
|
1120
|
+
openai: {
|
|
1121
|
+
extraction: "gpt-4o-mini",
|
|
1122
|
+
analysis: "gpt-4o"
|
|
1123
|
+
},
|
|
1124
|
+
ollama: {
|
|
1125
|
+
extraction: "llama3.2",
|
|
1126
|
+
analysis: "llama3.2"
|
|
1127
|
+
}
|
|
1128
|
+
};
|
|
1129
|
+
AnthropicProvider = class {
|
|
1130
|
+
client = null;
|
|
1131
|
+
apiKey;
|
|
1132
|
+
model;
|
|
1133
|
+
constructor(apiKey, model) {
|
|
1134
|
+
this.apiKey = apiKey;
|
|
1135
|
+
this.model = model;
|
|
1136
|
+
}
|
|
1137
|
+
async getClient() {
|
|
1138
|
+
if (!this.client) {
|
|
1139
|
+
const { default: Anthropic } = await import("@anthropic-ai/sdk");
|
|
1140
|
+
this.client = new Anthropic({ apiKey: this.apiKey });
|
|
1141
|
+
}
|
|
1142
|
+
return this.client;
|
|
1143
|
+
}
|
|
1144
|
+
async complete(prompt, options) {
|
|
1145
|
+
const client = await this.getClient();
|
|
1146
|
+
const response = await client.messages.create({
|
|
1147
|
+
model: this.model,
|
|
1148
|
+
max_tokens: options?.maxTokens ?? 1024,
|
|
1149
|
+
...options?.system ? { system: options.system } : {},
|
|
1150
|
+
messages: [{ role: "user", content: prompt }]
|
|
1151
|
+
});
|
|
1152
|
+
const text2 = response.content[0].type === "text" ? response.content[0].text : "";
|
|
1153
|
+
return text2;
|
|
1154
|
+
}
|
|
1155
|
+
};
|
|
1156
|
+
OpenAICompatibleProvider = class {
|
|
1157
|
+
apiKey;
|
|
1158
|
+
model;
|
|
1159
|
+
baseUrl;
|
|
1160
|
+
constructor(apiKey, model, baseUrl) {
|
|
1161
|
+
this.apiKey = apiKey;
|
|
1162
|
+
this.model = model;
|
|
1163
|
+
this.baseUrl = baseUrl;
|
|
1164
|
+
}
|
|
1165
|
+
async complete(prompt, options) {
|
|
1166
|
+
const url = `${this.baseUrl || "https://api.openai.com/v1"}/chat/completions`;
|
|
1167
|
+
const body = {
|
|
1168
|
+
model: this.model,
|
|
1169
|
+
max_tokens: options?.maxTokens ?? 1024,
|
|
1170
|
+
messages: [
|
|
1171
|
+
...options?.system ? [{ role: "system", content: options.system }] : [],
|
|
1172
|
+
{ role: "user", content: prompt }
|
|
1173
|
+
]
|
|
1174
|
+
};
|
|
1175
|
+
if (options?.json) {
|
|
1176
|
+
body.response_format = { type: "json_object" };
|
|
1177
|
+
}
|
|
1178
|
+
const response = await fetch(url, {
|
|
1179
|
+
method: "POST",
|
|
1180
|
+
headers: {
|
|
1181
|
+
"Content-Type": "application/json",
|
|
1182
|
+
...this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}
|
|
1183
|
+
},
|
|
1184
|
+
body: JSON.stringify(body)
|
|
1185
|
+
});
|
|
1186
|
+
if (!response.ok) {
|
|
1187
|
+
const error = await response.text();
|
|
1188
|
+
throw new Error(`LLM request failed (${response.status}): ${error}`);
|
|
1189
|
+
}
|
|
1190
|
+
const data = await response.json();
|
|
1191
|
+
return data.choices[0]?.message?.content ?? "";
|
|
1192
|
+
}
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
});
|
|
1196
|
+
|
|
1197
|
+
// ../core/dist/llm-config.js
|
|
1198
|
+
function resolveLLMProvider(task) {
|
|
1199
|
+
const config = loadConfig();
|
|
1200
|
+
if (config.llm?.provider) {
|
|
1201
|
+
const apiKey = config.llm.apiKey || process.env.ENGRAMS_API_KEY || (config.llm.provider === "anthropic" ? process.env.ANTHROPIC_API_KEY : void 0) || (config.llm.provider === "openai" ? process.env.OPENAI_API_KEY : void 0);
|
|
1202
|
+
const taskModel = task && config.llm.models?.[task];
|
|
1203
|
+
return createLLMProvider({
|
|
1204
|
+
provider: config.llm.provider,
|
|
1205
|
+
model: taskModel || config.llm.model || void 0,
|
|
1206
|
+
apiKey: apiKey || void 0,
|
|
1207
|
+
baseUrl: config.llm.baseUrl
|
|
1208
|
+
}, task);
|
|
1209
|
+
}
|
|
1210
|
+
const engramsProvider = process.env.ENGRAMS_LLM_PROVIDER;
|
|
1211
|
+
if (engramsProvider) {
|
|
1212
|
+
return createLLMProvider({
|
|
1213
|
+
provider: engramsProvider,
|
|
1214
|
+
model: process.env.ENGRAMS_LLM_MODEL || void 0,
|
|
1215
|
+
apiKey: process.env.ENGRAMS_API_KEY,
|
|
1216
|
+
baseUrl: process.env.ENGRAMS_LLM_BASE_URL
|
|
1217
|
+
}, task);
|
|
1218
|
+
}
|
|
1219
|
+
if (process.env.ANTHROPIC_API_KEY) {
|
|
1220
|
+
return createLLMProvider({
|
|
1221
|
+
provider: "anthropic",
|
|
1222
|
+
apiKey: process.env.ANTHROPIC_API_KEY
|
|
1223
|
+
}, task);
|
|
1224
|
+
}
|
|
1225
|
+
if (process.env.OPENAI_API_KEY) {
|
|
1226
|
+
return createLLMProvider({
|
|
1227
|
+
provider: "openai",
|
|
1228
|
+
apiKey: process.env.OPENAI_API_KEY
|
|
1229
|
+
}, task);
|
|
1230
|
+
}
|
|
1231
|
+
return null;
|
|
1232
|
+
}
|
|
1233
|
+
var init_llm_config = __esm({
|
|
1234
|
+
"../core/dist/llm-config.js"() {
|
|
1235
|
+
"use strict";
|
|
1236
|
+
init_credentials();
|
|
1237
|
+
init_llm();
|
|
1238
|
+
}
|
|
1239
|
+
});
|
|
1240
|
+
|
|
1241
|
+
// ../core/dist/llm-validation.js
|
|
1242
|
+
function validateExtraction(result) {
|
|
1243
|
+
const warnings = [];
|
|
1244
|
+
if (!result || typeof result !== "object") {
|
|
1245
|
+
return { valid: false, error: "LLM did not return a valid object", warnings, qualityScore: 0 };
|
|
1246
|
+
}
|
|
1247
|
+
const r = result;
|
|
1248
|
+
if (!r.entity_type || !VALID_ENTITY_TYPES.includes(r.entity_type)) {
|
|
1249
|
+
return {
|
|
1250
|
+
valid: false,
|
|
1251
|
+
error: `Invalid entity_type "${r.entity_type}". Model may not be capable enough for extraction. Recommended: claude-haiku-4-5, gpt-4o-mini, or better.`,
|
|
1252
|
+
warnings,
|
|
1253
|
+
qualityScore: 0
|
|
1254
|
+
};
|
|
1255
|
+
}
|
|
1256
|
+
if (r.entity_name && typeof r.entity_name === "string" && r.entity_name.length > 100) {
|
|
1257
|
+
warnings.push("entity_name is unusually long \u2014 model may have used full content instead of a canonical name");
|
|
1258
|
+
}
|
|
1259
|
+
if (r.suggested_connections && !Array.isArray(r.suggested_connections)) {
|
|
1260
|
+
warnings.push("suggested_connections is not an array \u2014 ignoring");
|
|
1261
|
+
}
|
|
1262
|
+
const score = 1 - warnings.length * 0.1 - (r.entity_name ? 0 : 0.1);
|
|
1263
|
+
return { valid: true, warnings, qualityScore: Math.max(0, score) };
|
|
1264
|
+
}
|
|
1265
|
+
var VALID_ENTITY_TYPES;
|
|
1266
|
+
var init_llm_validation = __esm({
|
|
1267
|
+
"../core/dist/llm-validation.js"() {
|
|
1268
|
+
"use strict";
|
|
1269
|
+
init_embeddings();
|
|
1270
|
+
VALID_ENTITY_TYPES = ["person", "organization", "place", "project", "preference", "event", "goal", "fact"];
|
|
723
1271
|
}
|
|
724
1272
|
});
|
|
725
1273
|
|
|
@@ -738,6 +1286,13 @@ var init_dist = __esm({
|
|
|
738
1286
|
init_db();
|
|
739
1287
|
init_pii();
|
|
740
1288
|
init_entity_extraction();
|
|
1289
|
+
init_crypto();
|
|
1290
|
+
init_credentials();
|
|
1291
|
+
init_sync();
|
|
1292
|
+
init_llm();
|
|
1293
|
+
init_llm_utils();
|
|
1294
|
+
init_llm_config();
|
|
1295
|
+
init_llm_validation();
|
|
741
1296
|
}
|
|
742
1297
|
});
|
|
743
1298
|
|
|
@@ -747,10 +1302,10 @@ __export(http_exports, {
|
|
|
747
1302
|
startHttpApi: () => startHttpApi
|
|
748
1303
|
});
|
|
749
1304
|
import { createServer } from "http";
|
|
750
|
-
import { randomBytes } from "crypto";
|
|
1305
|
+
import { randomBytes as randomBytes2 } from "crypto";
|
|
751
1306
|
import { eq, and, isNull } from "drizzle-orm";
|
|
752
1307
|
function generateId() {
|
|
753
|
-
return
|
|
1308
|
+
return randomBytes2(16).toString("hex");
|
|
754
1309
|
}
|
|
755
1310
|
function now() {
|
|
756
1311
|
return (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -760,12 +1315,12 @@ function json(res, data, status = 200) {
|
|
|
760
1315
|
res.end(JSON.stringify(data));
|
|
761
1316
|
}
|
|
762
1317
|
function parseBody(req) {
|
|
763
|
-
return new Promise((
|
|
1318
|
+
return new Promise((resolve4, reject) => {
|
|
764
1319
|
let body = "";
|
|
765
1320
|
req.on("data", (chunk) => body += chunk);
|
|
766
1321
|
req.on("end", () => {
|
|
767
1322
|
try {
|
|
768
|
-
|
|
1323
|
+
resolve4(body ? JSON.parse(body) : {});
|
|
769
1324
|
} catch {
|
|
770
1325
|
reject(new Error("Invalid JSON"));
|
|
771
1326
|
}
|
|
@@ -936,10 +1491,9 @@ import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mc
|
|
|
936
1491
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
937
1492
|
import { z } from "zod";
|
|
938
1493
|
import { eq as eq2, and as and2, isNull as isNull2 } from "drizzle-orm";
|
|
939
|
-
import { randomBytes as
|
|
940
|
-
import Anthropic from "@anthropic-ai/sdk";
|
|
1494
|
+
import { randomBytes as randomBytes3 } from "crypto";
|
|
941
1495
|
function generateId2() {
|
|
942
|
-
return
|
|
1496
|
+
return randomBytes3(16).toString("hex");
|
|
943
1497
|
}
|
|
944
1498
|
function now2() {
|
|
945
1499
|
return (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -964,6 +1518,8 @@ async function startServer() {
|
|
|
964
1518
|
lastDecayRun = now3;
|
|
965
1519
|
}
|
|
966
1520
|
}
|
|
1521
|
+
let extractionProvider = resolveLLMProvider("extraction");
|
|
1522
|
+
let analysisProvider = resolveLLMProvider("analysis");
|
|
967
1523
|
if (vecAvailable) {
|
|
968
1524
|
backfillEmbeddings(sqlite).then((count) => {
|
|
969
1525
|
if (count > 0) process.stderr.write(`[engrams] Backfilled embeddings for ${count} memories
|
|
@@ -1259,9 +1815,9 @@ Organize memories by life domain: general, work, health, finance, relationships,
|
|
|
1259
1815
|
}
|
|
1260
1816
|
}
|
|
1261
1817
|
}
|
|
1262
|
-
const
|
|
1263
|
-
if (params.entityType && !
|
|
1264
|
-
return textResult({ error: `Invalid entity_type: "${params.entityType}". Must be one of: ${
|
|
1818
|
+
const VALID_ENTITY_TYPES2 = ["person", "organization", "place", "project", "preference", "event", "goal", "fact"];
|
|
1819
|
+
if (params.entityType && !VALID_ENTITY_TYPES2.includes(params.entityType)) {
|
|
1820
|
+
return textResult({ error: `Invalid entity_type: "${params.entityType}". Must be one of: ${VALID_ENTITY_TYPES2.join(", ")}` });
|
|
1265
1821
|
}
|
|
1266
1822
|
const id = generateId2();
|
|
1267
1823
|
const confidence = getInitialConfidence(params.sourceType);
|
|
@@ -1304,15 +1860,22 @@ Organize memories by life domain: general, work, health, finance, relationships,
|
|
|
1304
1860
|
newValue: JSON.stringify({ content: params.content, domain: params.domain ?? "general" }),
|
|
1305
1861
|
timestamp
|
|
1306
1862
|
}).run();
|
|
1307
|
-
if (!params.entityType &&
|
|
1863
|
+
if (!params.entityType && extractionProvider) {
|
|
1308
1864
|
(async () => {
|
|
1309
1865
|
try {
|
|
1310
1866
|
const existingNames = sqlite.prepare(`SELECT DISTINCT entity_name FROM memories WHERE entity_name IS NOT NULL AND deleted_at IS NULL`).all();
|
|
1311
1867
|
const extraction = await extractEntity(
|
|
1868
|
+
extractionProvider,
|
|
1312
1869
|
params.content,
|
|
1313
1870
|
params.detail ?? null,
|
|
1314
1871
|
existingNames.map((r) => r.entity_name)
|
|
1315
1872
|
);
|
|
1873
|
+
const validation = validateExtraction(extraction);
|
|
1874
|
+
if (!validation.valid) {
|
|
1875
|
+
process.stderr.write(`[engrams] Entity extraction failed validation: ${validation.error}
|
|
1876
|
+
`);
|
|
1877
|
+
return;
|
|
1878
|
+
}
|
|
1316
1879
|
const current = sqlite.prepare(`SELECT entity_type FROM memories WHERE id = ? AND deleted_at IS NULL`).get(id);
|
|
1317
1880
|
if (!current || current.entity_type) return;
|
|
1318
1881
|
sqlite.transaction(() => {
|
|
@@ -1341,16 +1904,9 @@ Organize memories by life domain: general, work, health, finance, relationships,
|
|
|
1341
1904
|
const fullText = params.content + (params.detail ? " " + params.detail : "");
|
|
1342
1905
|
const sentences = fullText.split(/(?<=[.!?])\s+/).filter((s) => s.length > 10);
|
|
1343
1906
|
let splitSuggestion = null;
|
|
1344
|
-
if (sentences.length >= 3 &&
|
|
1907
|
+
if (sentences.length >= 3 && extractionProvider) {
|
|
1345
1908
|
try {
|
|
1346
|
-
const
|
|
1347
|
-
const resp = await client.messages.create({
|
|
1348
|
-
model: "claude-haiku-4-5-20251001",
|
|
1349
|
-
max_tokens: 512,
|
|
1350
|
-
messages: [
|
|
1351
|
-
{
|
|
1352
|
-
role: "user",
|
|
1353
|
-
content: `Analyze this memory and determine if it contains multiple distinct topics that should be stored separately. Only suggest splitting if the topics are genuinely independent (would be searched for separately).
|
|
1909
|
+
const splitPrompt = `Analyze this memory and determine if it contains multiple distinct topics that should be stored separately. Only suggest splitting if the topics are genuinely independent (would be searched for separately).
|
|
1354
1910
|
|
|
1355
1911
|
Memory content: ${JSON.stringify(params.content)}
|
|
1356
1912
|
Memory detail: ${JSON.stringify(params.detail ?? null)}
|
|
@@ -1359,12 +1915,9 @@ Respond with ONLY valid JSON:
|
|
|
1359
1915
|
- If it should NOT be split: {"should_split": false}
|
|
1360
1916
|
- If it SHOULD be split: {"should_split": true, "parts": [{"content": "...", "detail": "..."}, ...]}
|
|
1361
1917
|
|
|
1362
|
-
Each part should have a concise "content" (one sentence) and optional "detail". Do not split if the content is a single coherent topic
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
});
|
|
1366
|
-
const text2 = resp.content[0].type === "text" ? resp.content[0].text : "";
|
|
1367
|
-
splitSuggestion = JSON.parse(text2);
|
|
1918
|
+
Each part should have a concise "content" (one sentence) and optional "detail". Do not split if the content is a single coherent topic.`;
|
|
1919
|
+
const text2 = await extractionProvider.complete(splitPrompt, { maxTokens: 512, json: true });
|
|
1920
|
+
splitSuggestion = parseLLMJson(text2);
|
|
1368
1921
|
} catch {
|
|
1369
1922
|
}
|
|
1370
1923
|
}
|
|
@@ -1377,7 +1930,7 @@ Each part should have a concise "content" (one sentence) and optional "detail".
|
|
|
1377
1930
|
entityType: params.entityType ?? null,
|
|
1378
1931
|
entityName: params.entityName ?? null
|
|
1379
1932
|
};
|
|
1380
|
-
if (!params.entityType &&
|
|
1933
|
+
if (!params.entityType && extractionProvider) {
|
|
1381
1934
|
result._background_classification = "running";
|
|
1382
1935
|
}
|
|
1383
1936
|
if (hasPii) {
|
|
@@ -1419,7 +1972,9 @@ Each part should have a concise "content" (one sentence) and optional "detail".
|
|
|
1419
1972
|
similarityThreshold: params.similarityThreshold
|
|
1420
1973
|
});
|
|
1421
1974
|
if (searchResults.length === 0) {
|
|
1422
|
-
|
|
1975
|
+
const totalCount = sqlite.prepare("SELECT COUNT(*) as count FROM memories WHERE deleted_at IS NULL").get().count;
|
|
1976
|
+
const onboarding_hint = totalCount < 5 ? "Your memory database is nearly empty. Call memory_onboard with your list of available tools to run a guided setup." : void 0;
|
|
1977
|
+
return textResult({ memories: [], count: 0, totalConnected: 0, cached: wasCached, ...onboarding_hint ? { onboarding_hint } : {} });
|
|
1423
1978
|
}
|
|
1424
1979
|
const timestamp = now2();
|
|
1425
1980
|
const updateStmt = sqlite.prepare(
|
|
@@ -1913,8 +2468,8 @@ Each part should have a concise "content" (one sentence) and optional "detail".
|
|
|
1913
2468
|
domain: z.string().optional().describe("Only classify memories in this domain")
|
|
1914
2469
|
},
|
|
1915
2470
|
async (params) => {
|
|
1916
|
-
if (!
|
|
1917
|
-
return textResult({ error: "
|
|
2471
|
+
if (!extractionProvider) {
|
|
2472
|
+
return textResult({ error: "No LLM provider configured. Set ANTHROPIC_API_KEY, OPENAI_API_KEY, or configure ~/.engrams/config.json" });
|
|
1918
2473
|
}
|
|
1919
2474
|
const classifyLimit = params.limit ?? 50;
|
|
1920
2475
|
let query = `SELECT id, content, detail FROM memories WHERE entity_type IS NULL AND deleted_at IS NULL`;
|
|
@@ -1935,7 +2490,12 @@ Each part should have a concise "content" (one sentence) and optional "detail".
|
|
|
1935
2490
|
let errors = 0;
|
|
1936
2491
|
for (const mem of untyped) {
|
|
1937
2492
|
try {
|
|
1938
|
-
const extraction = await extractEntity(mem.content, mem.detail, nameList);
|
|
2493
|
+
const extraction = await extractEntity(extractionProvider, mem.content, mem.detail, nameList);
|
|
2494
|
+
const validation = validateExtraction(extraction);
|
|
2495
|
+
if (!validation.valid) {
|
|
2496
|
+
errors++;
|
|
2497
|
+
continue;
|
|
2498
|
+
}
|
|
1939
2499
|
sqlite.transaction(() => {
|
|
1940
2500
|
sqlite.prepare(
|
|
1941
2501
|
`UPDATE memories SET entity_type = ?, entity_name = ?, structured_data = ? WHERE id = ? AND entity_type IS NULL AND deleted_at IS NULL`
|
|
@@ -1960,7 +2520,7 @@ Each part should have a concise "content" (one sentence) and optional "detail".
|
|
|
1960
2520
|
errors++;
|
|
1961
2521
|
}
|
|
1962
2522
|
if (classified + errors < untyped.length) {
|
|
1963
|
-
await new Promise((
|
|
2523
|
+
await new Promise((resolve4) => setTimeout(resolve4, 200));
|
|
1964
2524
|
}
|
|
1965
2525
|
}
|
|
1966
2526
|
if (classified > 0) bumpLastModified(sqlite);
|
|
@@ -2043,6 +2603,531 @@ Each part should have a concise "content" (one sentence) and optional "detail".
|
|
|
2043
2603
|
});
|
|
2044
2604
|
}
|
|
2045
2605
|
);
|
|
2606
|
+
server.tool(
|
|
2607
|
+
"memory_configure",
|
|
2608
|
+
"Configure Engrams settings. Currently supports LLM provider setup for entity extraction, correction, and splitting.",
|
|
2609
|
+
{
|
|
2610
|
+
llm_provider: z.enum(["anthropic", "openai", "ollama"]).describe("LLM provider to use"),
|
|
2611
|
+
llm_api_key: z.string().optional().describe("API key for the provider. Not needed for Ollama."),
|
|
2612
|
+
llm_base_url: z.string().optional().describe("Custom base URL for OpenAI-compatible endpoints or Ollama."),
|
|
2613
|
+
llm_extraction_model: z.string().optional().describe("Model for entity extraction (high-volume, cheap). Defaults: anthropic=claude-haiku-4-5, openai=gpt-4o-mini, ollama=llama3.2"),
|
|
2614
|
+
llm_analysis_model: z.string().optional().describe("Model for correction/splitting (user-initiated, capable). Defaults: anthropic=claude-sonnet-4-5, openai=gpt-4o, ollama=llama3.2")
|
|
2615
|
+
},
|
|
2616
|
+
async (params) => {
|
|
2617
|
+
const config = loadConfig();
|
|
2618
|
+
config.llm = {
|
|
2619
|
+
provider: params.llm_provider,
|
|
2620
|
+
apiKey: params.llm_api_key || void 0,
|
|
2621
|
+
baseUrl: params.llm_base_url || void 0,
|
|
2622
|
+
models: {
|
|
2623
|
+
extraction: params.llm_extraction_model || void 0,
|
|
2624
|
+
analysis: params.llm_analysis_model || void 0
|
|
2625
|
+
}
|
|
2626
|
+
};
|
|
2627
|
+
saveConfig(config);
|
|
2628
|
+
extractionProvider = resolveLLMProvider("extraction");
|
|
2629
|
+
analysisProvider = resolveLLMProvider("analysis");
|
|
2630
|
+
try {
|
|
2631
|
+
if (extractionProvider) {
|
|
2632
|
+
await extractionProvider.complete("Say ok", { maxTokens: 10 });
|
|
2633
|
+
}
|
|
2634
|
+
return textResult({
|
|
2635
|
+
status: "configured",
|
|
2636
|
+
provider: params.llm_provider,
|
|
2637
|
+
extraction_model: params.llm_extraction_model || "(default)",
|
|
2638
|
+
analysis_model: params.llm_analysis_model || "(default)"
|
|
2639
|
+
});
|
|
2640
|
+
} catch (err) {
|
|
2641
|
+
return textResult({
|
|
2642
|
+
status: "configured_with_error",
|
|
2643
|
+
provider: params.llm_provider,
|
|
2644
|
+
error: `Config saved but connection test failed: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
2645
|
+
});
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
);
|
|
2649
|
+
server.tool(
|
|
2650
|
+
"memory_onboard",
|
|
2651
|
+
"Get a personalized onboarding plan to seed your memory database. Returns a structured action plan based on your current memory state. Call this when the user is new or asks to set up their memory. The plan tells you which connected tools to scan and what interview questions to ask \u2014 execute each step using the tools available to you.",
|
|
2652
|
+
{
|
|
2653
|
+
available_tools: z.array(z.string()).optional().describe("List of MCP tool names you have access to (e.g. ['gcal_list_events', 'gmail_search_messages', 'github_list_repos']). This helps generate a targeted plan."),
|
|
2654
|
+
skip_scan: z.boolean().optional().describe("Skip the tool scan phase and go straight to interview. Default false."),
|
|
2655
|
+
skip_interview: z.boolean().optional().describe("Skip the interview phase. Useful if re-running just the scan. Default false.")
|
|
2656
|
+
},
|
|
2657
|
+
async (params) => {
|
|
2658
|
+
const memoryCount = sqlite.prepare("SELECT COUNT(*) as count FROM memories WHERE deleted_at IS NULL").get();
|
|
2659
|
+
const entityCounts = sqlite.prepare(`
|
|
2660
|
+
SELECT entity_type, COUNT(*) as count
|
|
2661
|
+
FROM memories
|
|
2662
|
+
WHERE entity_type IS NOT NULL AND deleted_at IS NULL
|
|
2663
|
+
GROUP BY entity_type
|
|
2664
|
+
`).all();
|
|
2665
|
+
const domainCounts = sqlite.prepare(`
|
|
2666
|
+
SELECT domain, COUNT(*) as count
|
|
2667
|
+
FROM memories
|
|
2668
|
+
WHERE deleted_at IS NULL
|
|
2669
|
+
GROUP BY domain
|
|
2670
|
+
`).all();
|
|
2671
|
+
const totalMemories = memoryCount.count;
|
|
2672
|
+
const entityMap = Object.fromEntries(entityCounts.map((e) => [e.entity_type, e.count]));
|
|
2673
|
+
const domainMap = Object.fromEntries(domainCounts.map((d) => [d.domain, d.count]));
|
|
2674
|
+
const tools = params.available_tools || [];
|
|
2675
|
+
const hasCalendar = tools.some((t) => /gcal|calendar|cal_list|list_events/i.test(t));
|
|
2676
|
+
const hasEmail = tools.some((t) => /gmail|email|mail|search_messages/i.test(t));
|
|
2677
|
+
const hasGitHub = tools.some((t) => /github|gh_|list_repos|list_prs/i.test(t));
|
|
2678
|
+
const hasSlack = tools.some((t) => /slack|channel|send_message/i.test(t));
|
|
2679
|
+
const hasNotes = tools.some((t) => /note|notion|obsidian/i.test(t));
|
|
2680
|
+
const plan = [];
|
|
2681
|
+
if (totalMemories === 0) {
|
|
2682
|
+
plan.push("# Onboarding Plan \u2014 Fresh Start");
|
|
2683
|
+
plan.push("");
|
|
2684
|
+
plan.push("Your memory database is empty. Let's fix that. This plan will seed your memories from connected tools and a short conversation.");
|
|
2685
|
+
} else if (totalMemories < 20) {
|
|
2686
|
+
plan.push("# Onboarding Plan \u2014 Early Stage");
|
|
2687
|
+
plan.push("");
|
|
2688
|
+
plan.push(`You have ${totalMemories} memories so far. Let's enrich your knowledge graph with more context from your tools.`);
|
|
2689
|
+
} else {
|
|
2690
|
+
plan.push("# Onboarding Plan \u2014 Enrichment");
|
|
2691
|
+
plan.push("");
|
|
2692
|
+
plan.push(`You have ${totalMemories} memories across ${Object.keys(domainMap).length} domains. Here's what could be filled in.`);
|
|
2693
|
+
if (entityCounts.length > 0) {
|
|
2694
|
+
plan.push("");
|
|
2695
|
+
plan.push("Current entity coverage:");
|
|
2696
|
+
for (const e of entityCounts) {
|
|
2697
|
+
plan.push(`- ${e.entity_type}: ${e.count}`);
|
|
2698
|
+
}
|
|
2699
|
+
}
|
|
2700
|
+
const allTypes = ["person", "organization", "place", "project", "preference", "event", "goal"];
|
|
2701
|
+
const missing = allTypes.filter((t) => !entityMap[t]);
|
|
2702
|
+
if (missing.length > 0) {
|
|
2703
|
+
plan.push("");
|
|
2704
|
+
plan.push(`Missing entity types: ${missing.join(", ")}. The scan and interview below will help fill these gaps.`);
|
|
2705
|
+
}
|
|
2706
|
+
}
|
|
2707
|
+
if (!params.skip_scan) {
|
|
2708
|
+
plan.push("");
|
|
2709
|
+
plan.push("---");
|
|
2710
|
+
plan.push("");
|
|
2711
|
+
plan.push("## Phase 1: Silent Scan");
|
|
2712
|
+
plan.push("");
|
|
2713
|
+
plan.push("Scan the user's connected tools to extract people, projects, events, and context. Do this BEFORE the interview \u2014 the interview will be much better with this context.");
|
|
2714
|
+
plan.push("");
|
|
2715
|
+
plan.push('**Important:** For each piece of information you extract, call `memory_write` with appropriate `domain`, `source_type: "inferred"`, and `source_description` noting which tool it came from. The system will automatically classify entities and create connections.');
|
|
2716
|
+
plan.push("");
|
|
2717
|
+
plan.push("**Dedup:** Before writing, call `memory_search` with the key terms to check if a similar memory already exists. If it does, skip or use `memory_update` to enrich it.");
|
|
2718
|
+
if (hasCalendar) {
|
|
2719
|
+
plan.push("");
|
|
2720
|
+
plan.push("### Calendar (available)");
|
|
2721
|
+
plan.push("");
|
|
2722
|
+
plan.push("1. Fetch events from the past 30 days");
|
|
2723
|
+
plan.push("2. Identify **recurring meetings** \u2014 these reveal team structure, projects, and key relationships");
|
|
2724
|
+
plan.push(" - For each recurring meeting: write a memory about what it is, who attends, and its cadence");
|
|
2725
|
+
plan.push(" - Extract each unique attendee as a person memory (name, how they relate to the user)");
|
|
2726
|
+
plan.push("3. Identify **project-related events** \u2014 standups, retros, planning sessions reveal active projects");
|
|
2727
|
+
plan.push(" - Write a memory for each distinct project you can identify");
|
|
2728
|
+
plan.push("4. Look for **1:1 meetings** \u2014 these are the user's closest collaborators");
|
|
2729
|
+
plan.push("5. Note any upcoming events in the next 7 days that suggest deadlines or goals");
|
|
2730
|
+
plan.push("");
|
|
2731
|
+
plan.push("Expected yield: 15-30 memories (people, projects, events, organizations)");
|
|
2732
|
+
}
|
|
2733
|
+
if (hasEmail) {
|
|
2734
|
+
plan.push("");
|
|
2735
|
+
plan.push("### Email (available)");
|
|
2736
|
+
plan.push("");
|
|
2737
|
+
plan.push("1. Search recent emails (past 14 days) for threads with the most back-and-forth \u2014 these are active topics");
|
|
2738
|
+
plan.push("2. Identify **key contacts** \u2014 people the user emails most frequently");
|
|
2739
|
+
plan.push(" - Cross-reference with calendar attendees to enrich existing person memories");
|
|
2740
|
+
plan.push("3. Look for **commitments and action items** \u2014 'I'll send this by Friday', 'Let's schedule...', 'Following up on...'");
|
|
2741
|
+
plan.push(" - Write as event or goal memories");
|
|
2742
|
+
plan.push("4. Identify **external organizations** \u2014 clients, vendors, partners mentioned in email");
|
|
2743
|
+
plan.push("5. **DO NOT** read email body content in detail. Scan subjects, senders, and thread summaries only. Respect privacy.");
|
|
2744
|
+
plan.push("");
|
|
2745
|
+
plan.push("Expected yield: 10-20 memories (people, organizations, goals, events)");
|
|
2746
|
+
}
|
|
2747
|
+
if (hasGitHub) {
|
|
2748
|
+
plan.push("");
|
|
2749
|
+
plan.push("### GitHub (available)");
|
|
2750
|
+
plan.push("");
|
|
2751
|
+
plan.push("1. List the user's recent repositories (past 90 days of activity)");
|
|
2752
|
+
plan.push("2. For each active repo: write a project memory with the repo name, language/stack, and the user's role");
|
|
2753
|
+
plan.push("3. Check recent PRs for **collaborators** \u2014 frequent reviewers and co-authors are key people");
|
|
2754
|
+
plan.push("4. Note the **tech stack** across repos \u2014 languages, frameworks, tools. Write as preference/fact memories");
|
|
2755
|
+
plan.push("5. Look for any README descriptions that explain what projects do");
|
|
2756
|
+
plan.push("");
|
|
2757
|
+
plan.push("Expected yield: 10-20 memories (projects, people, preferences, facts)");
|
|
2758
|
+
}
|
|
2759
|
+
if (hasSlack) {
|
|
2760
|
+
plan.push("");
|
|
2761
|
+
plan.push("### Slack/Messaging (available)");
|
|
2762
|
+
plan.push("");
|
|
2763
|
+
plan.push("1. List channels the user is active in \u2014 channel names often map to projects or teams");
|
|
2764
|
+
plan.push("2. Identify **DM contacts** \u2014 frequent DM partners are close collaborators");
|
|
2765
|
+
plan.push("3. Note channel topics/descriptions for project context");
|
|
2766
|
+
plan.push("4. **DO NOT** read message history in detail. Use channel metadata only.");
|
|
2767
|
+
plan.push("");
|
|
2768
|
+
plan.push("Expected yield: 5-15 memories (projects, people, organizations)");
|
|
2769
|
+
}
|
|
2770
|
+
if (hasNotes) {
|
|
2771
|
+
plan.push("");
|
|
2772
|
+
plan.push("### Notes/Docs (available)");
|
|
2773
|
+
plan.push("");
|
|
2774
|
+
plan.push("1. Search for recent documents the user has edited");
|
|
2775
|
+
plan.push("2. Document titles and summaries reveal active projects and interests");
|
|
2776
|
+
plan.push("3. Look for any documents that look like personal notes, goals, or planning docs");
|
|
2777
|
+
plan.push("");
|
|
2778
|
+
plan.push("Expected yield: 5-10 memories (projects, goals, facts)");
|
|
2779
|
+
}
|
|
2780
|
+
plan.push("");
|
|
2781
|
+
plan.push("### Local Files (always available)");
|
|
2782
|
+
plan.push("");
|
|
2783
|
+
plan.push("Check for and read these files if they exist:");
|
|
2784
|
+
plan.push("");
|
|
2785
|
+
plan.push("- `~/.gitconfig` \u2014 user's name, email, identity. Write as a person memory about the user.");
|
|
2786
|
+
plan.push("- `~/.claude/CLAUDE.md` or any `CLAUDE.md` in the working directory \u2014 existing instructions and preferences. Each instruction is a preference memory.");
|
|
2787
|
+
plan.push("- `~/.claude/memory/` or `~/.claude/projects/*/memory/` \u2014 Claude Code auto-memory files. Parse each line/section as a separate memory. These are high-quality since the user or their AI already curated them.");
|
|
2788
|
+
plan.push("- `.cursorrules` or `.windsurfrules` in the working directory \u2014 coding preferences. Each rule is a preference memory.");
|
|
2789
|
+
plan.push("- `~/.config/` \u2014 scan for tool configs that reveal preferences (editor settings, shell aliases, etc.). Be selective \u2014 only extract meaningful preferences, not every config line.");
|
|
2790
|
+
plan.push("");
|
|
2791
|
+
plan.push("Expected yield: 5-15 memories (preferences, person, facts)");
|
|
2792
|
+
if (!hasCalendar && !hasEmail && !hasGitHub && !hasSlack && !hasNotes) {
|
|
2793
|
+
plan.push("");
|
|
2794
|
+
plan.push("### No connected tools detected");
|
|
2795
|
+
plan.push("");
|
|
2796
|
+
plan.push("You didn't list any calendar, email, GitHub, or notes tools. That's fine \u2014 the Local Files scan and the interview will still seed a solid foundation. If you do have connected tools, call `memory_onboard` again with `available_tools` listing your tool names for a richer scan.");
|
|
2797
|
+
}
|
|
2798
|
+
}
|
|
2799
|
+
if (!params.skip_interview) {
|
|
2800
|
+
plan.push("");
|
|
2801
|
+
plan.push("---");
|
|
2802
|
+
plan.push("");
|
|
2803
|
+
plan.push("## Phase 2: Informed Interview");
|
|
2804
|
+
plan.push("");
|
|
2805
|
+
plan.push("After the scan, you have a base of extracted context. Now have a SHORT conversation with the user to fill in meaning, relationships, and preferences that tools can't surface.");
|
|
2806
|
+
plan.push("");
|
|
2807
|
+
plan.push("**Rules:**");
|
|
2808
|
+
plan.push("- Reference what you learned in the scan. Don't ask questions you already have answers to.");
|
|
2809
|
+
plan.push("- Ask ONE question at a time. Wait for the answer before the next question.");
|
|
2810
|
+
plan.push("- Write memories immediately after each answer \u2014 don't batch them.");
|
|
2811
|
+
plan.push("- 5-7 questions max. Respect the user's time.");
|
|
2812
|
+
plan.push("- Tailor questions to what's MISSING, not what you already know.");
|
|
2813
|
+
plan.push("");
|
|
2814
|
+
if (totalMemories > 0 || !params.skip_scan) {
|
|
2815
|
+
plan.push("### Suggested questions (adapt based on what the scan found):");
|
|
2816
|
+
plan.push("");
|
|
2817
|
+
plan.push('1. **Confirm and enrich key relationships:** "I found [names] across your calendar/email. Who are the most important people in your day-to-day \u2014 your direct team, your manager, key stakeholders?"');
|
|
2818
|
+
plan.push(" \u2192 Write person memories with relationship_to_user and connect them to projects/orgs");
|
|
2819
|
+
plan.push("");
|
|
2820
|
+
plan.push(`2. **Clarify project priorities:** "I see you're involved in [projects]. What's your main focus right now? Are any of these winding down or just starting?"`);
|
|
2821
|
+
plan.push(" \u2192 Update project memories with status, write goal memories for priorities");
|
|
2822
|
+
plan.push("");
|
|
2823
|
+
plan.push(`3. **Organizational context:** "What does [organization] do? What's your role there?"`);
|
|
2824
|
+
plan.push(" \u2192 Write/enrich organization and person memories");
|
|
2825
|
+
plan.push("");
|
|
2826
|
+
plan.push('4. **Work preferences:** "Any strong preferences for how I should work with you? Communication style, code conventions, things that bug you?"');
|
|
2827
|
+
plan.push(" \u2192 Write preference memories with strength: strong");
|
|
2828
|
+
plan.push("");
|
|
2829
|
+
plan.push('5. **Goals:** "What are you working toward right now \u2014 professionally or personally?"');
|
|
2830
|
+
plan.push(" \u2192 Write goal memories with timeline and status");
|
|
2831
|
+
plan.push("");
|
|
2832
|
+
plan.push("6. **Fill entity gaps:** If the scan didn't surface certain entity types (places, events, facts), ask about them specifically.");
|
|
2833
|
+
plan.push(' \u2192 e.g., "Where are you based?" \u2192 place memory');
|
|
2834
|
+
plan.push(' \u2192 e.g., "Any upcoming deadlines or milestones?" \u2192 event memories');
|
|
2835
|
+
plan.push("");
|
|
2836
|
+
plan.push('7. **Catch-all:** "Anything else I should know about you that would help me be more useful?"');
|
|
2837
|
+
plan.push(" \u2192 Write whatever comes up");
|
|
2838
|
+
} else {
|
|
2839
|
+
plan.push("### Cold start questions (no scan data available):");
|
|
2840
|
+
plan.push("");
|
|
2841
|
+
plan.push(`1. "Tell me about yourself \u2014 name, what you do, where you're based."`);
|
|
2842
|
+
plan.push(" \u2192 person + organization + place memories");
|
|
2843
|
+
plan.push("");
|
|
2844
|
+
plan.push('2. "What are you working on right now?"');
|
|
2845
|
+
plan.push(" \u2192 project memories");
|
|
2846
|
+
plan.push("");
|
|
2847
|
+
plan.push('3. "Who do you work with most closely?"');
|
|
2848
|
+
plan.push(" \u2192 person memories with relationships");
|
|
2849
|
+
plan.push("");
|
|
2850
|
+
plan.push('4. "What tools and technologies do you use daily?"');
|
|
2851
|
+
plan.push(" \u2192 preference and fact memories");
|
|
2852
|
+
plan.push("");
|
|
2853
|
+
plan.push('5. "Any strong preferences for how I should communicate or work with you?"');
|
|
2854
|
+
plan.push(" \u2192 preference memories");
|
|
2855
|
+
plan.push("");
|
|
2856
|
+
plan.push('6. "What are your current goals or priorities?"');
|
|
2857
|
+
plan.push(" \u2192 goal memories");
|
|
2858
|
+
plan.push("");
|
|
2859
|
+
plan.push('7. "Anything else I should remember?"');
|
|
2860
|
+
plan.push(" \u2192 catch-all");
|
|
2861
|
+
}
|
|
2862
|
+
}
|
|
2863
|
+
plan.push("");
|
|
2864
|
+
plan.push("---");
|
|
2865
|
+
plan.push("");
|
|
2866
|
+
plan.push("## Phase 3: Review");
|
|
2867
|
+
plan.push("");
|
|
2868
|
+
plan.push("After scanning and interviewing, tell the user:");
|
|
2869
|
+
plan.push("");
|
|
2870
|
+
plan.push(`"I've seeded your memory with [N] memories from [sources]. You can review and correct them at **localhost:3838** \u2014 anything I got wrong, click to edit or remove. Confirming memories boosts their confidence score."`);
|
|
2871
|
+
plan.push("");
|
|
2872
|
+
plan.push("If the dashboard has a review queue or unreviewed filter, mention it specifically.");
|
|
2873
|
+
sqlite.prepare(`
|
|
2874
|
+
INSERT INTO memory_events (id, memory_id, event_type, agent_id, agent_name, new_value, timestamp)
|
|
2875
|
+
VALUES (?, 'system', 'onboard_started', ?, ?, ?, ?)
|
|
2876
|
+
`).run(
|
|
2877
|
+
generateId2(),
|
|
2878
|
+
"unknown",
|
|
2879
|
+
"unknown",
|
|
2880
|
+
JSON.stringify({
|
|
2881
|
+
memory_count: totalMemories,
|
|
2882
|
+
tools_detected: { calendar: hasCalendar, email: hasEmail, github: hasGitHub, slack: hasSlack, notes: hasNotes }
|
|
2883
|
+
}),
|
|
2884
|
+
now2()
|
|
2885
|
+
);
|
|
2886
|
+
return textResult(plan.join("\n"));
|
|
2887
|
+
}
|
|
2888
|
+
);
|
|
2889
|
+
server.tool(
|
|
2890
|
+
"memory_import",
|
|
2891
|
+
"Import memories from a known format. Parses the source, deduplicates against existing memories, and writes new ones. Supported sources: claude-memory (MEMORY.md files), chatgpt-export (OpenAI memory export JSON), cursorrules (.cursorrules files), gitconfig (.gitconfig), plaintext (one memory per line).",
|
|
2892
|
+
{
|
|
2893
|
+
source_type: z.enum(["claude-memory", "chatgpt-export", "cursorrules", "gitconfig", "plaintext"]).describe("The format of the source data"),
|
|
2894
|
+
content: z.string().describe("The raw content to import. For file-based sources, pass the file contents."),
|
|
2895
|
+
domain: z.string().optional().describe("Domain to assign to imported memories. Default: 'general'.")
|
|
2896
|
+
},
|
|
2897
|
+
async (params) => {
|
|
2898
|
+
const domain = params.domain ?? "general";
|
|
2899
|
+
let entries = [];
|
|
2900
|
+
switch (params.source_type) {
|
|
2901
|
+
case "claude-memory": {
|
|
2902
|
+
entries = params.content.split("\n").filter((line) => line.trim().startsWith("- ")).map((line) => {
|
|
2903
|
+
const text2 = line.replace(/^-\s*/, "").trim();
|
|
2904
|
+
const tagMatch = text2.match(/^\[([^\]]+)\]\s*(.+)/);
|
|
2905
|
+
if (tagMatch) {
|
|
2906
|
+
return { content: tagMatch[2], detail: `Topic: ${tagMatch[1]}` };
|
|
2907
|
+
}
|
|
2908
|
+
return { content: text2 };
|
|
2909
|
+
}).filter((e) => e.content.length > 5);
|
|
2910
|
+
break;
|
|
2911
|
+
}
|
|
2912
|
+
case "chatgpt-export": {
|
|
2913
|
+
try {
|
|
2914
|
+
const parsed = JSON.parse(params.content);
|
|
2915
|
+
const items = Array.isArray(parsed) ? parsed : parsed.memories || parsed.results || [];
|
|
2916
|
+
entries = items.map((item) => {
|
|
2917
|
+
const text2 = typeof item === "string" ? item : item.memory || item.content || "";
|
|
2918
|
+
return { content: text2 };
|
|
2919
|
+
}).filter((e) => e.content.length > 5);
|
|
2920
|
+
} catch {
|
|
2921
|
+
return textResult({ error: "Failed to parse ChatGPT export JSON. Expected an array of { memory: string } objects." });
|
|
2922
|
+
}
|
|
2923
|
+
break;
|
|
2924
|
+
}
|
|
2925
|
+
case "cursorrules": {
|
|
2926
|
+
entries = params.content.split(/\n\n+/).flatMap((block) => {
|
|
2927
|
+
if (block.includes("\n- ")) {
|
|
2928
|
+
return block.split("\n- ").map((line) => ({
|
|
2929
|
+
content: line.replace(/^-\s*/, "").trim(),
|
|
2930
|
+
detail: "Imported from .cursorrules"
|
|
2931
|
+
}));
|
|
2932
|
+
}
|
|
2933
|
+
return [{ content: block.trim(), detail: "Imported from .cursorrules" }];
|
|
2934
|
+
}).filter((e) => e.content.length > 5);
|
|
2935
|
+
break;
|
|
2936
|
+
}
|
|
2937
|
+
case "gitconfig": {
|
|
2938
|
+
const nameMatch = params.content.match(/name\s*=\s*(.+)/i);
|
|
2939
|
+
const emailMatch = params.content.match(/email\s*=\s*(.+)/i);
|
|
2940
|
+
const editorMatch = params.content.match(/editor\s*=\s*(.+)/i);
|
|
2941
|
+
if (nameMatch) {
|
|
2942
|
+
entries.push({
|
|
2943
|
+
content: `User's name is ${nameMatch[1].trim()}`,
|
|
2944
|
+
detail: emailMatch ? `Email: ${emailMatch[1].trim()}` : void 0
|
|
2945
|
+
});
|
|
2946
|
+
}
|
|
2947
|
+
if (editorMatch) {
|
|
2948
|
+
entries.push({
|
|
2949
|
+
content: `Prefers ${editorMatch[1].trim()} as git editor`,
|
|
2950
|
+
detail: "From .gitconfig"
|
|
2951
|
+
});
|
|
2952
|
+
}
|
|
2953
|
+
const aliasSection = params.content.match(/\[alias\]([\s\S]*?)(?=\n\[|$)/i);
|
|
2954
|
+
if (aliasSection) {
|
|
2955
|
+
entries.push({
|
|
2956
|
+
content: "Has custom git aliases configured",
|
|
2957
|
+
detail: `Aliases: ${aliasSection[1].trim().split("\n").slice(0, 5).join("; ")}`
|
|
2958
|
+
});
|
|
2959
|
+
}
|
|
2960
|
+
break;
|
|
2961
|
+
}
|
|
2962
|
+
case "plaintext": {
|
|
2963
|
+
entries = params.content.split("\n").map((line) => line.trim()).filter((line) => line.length > 5).map((line) => ({ content: line }));
|
|
2964
|
+
break;
|
|
2965
|
+
}
|
|
2966
|
+
}
|
|
2967
|
+
if (entries.length === 0) {
|
|
2968
|
+
return textResult({ imported: 0, message: "No valid entries found in the provided content." });
|
|
2969
|
+
}
|
|
2970
|
+
let imported = 0;
|
|
2971
|
+
let skipped = 0;
|
|
2972
|
+
const results = [];
|
|
2973
|
+
const importedIds = [];
|
|
2974
|
+
for (const entry of entries) {
|
|
2975
|
+
const searchTerms = entry.content.split(/\s+/).filter((w) => w.length > 3).map((w) => w.replace(/['"(){}*:^~]/g, "")).filter((w) => !/^(AND|OR|NOT|NEAR)$/i.test(w)).slice(0, 5).join(" ");
|
|
2976
|
+
if (searchTerms.length > 0) {
|
|
2977
|
+
try {
|
|
2978
|
+
const existing = sqlite.prepare(`
|
|
2979
|
+
SELECT m.id, m.content FROM memories m
|
|
2980
|
+
JOIN memory_fts fts ON fts.rowid = m.rowid
|
|
2981
|
+
WHERE memory_fts MATCH ? AND m.deleted_at IS NULL
|
|
2982
|
+
LIMIT 3
|
|
2983
|
+
`).all(searchTerms);
|
|
2984
|
+
const entryWords = new Set(entry.content.toLowerCase().split(/\s+/).filter((w) => w.length > 3));
|
|
2985
|
+
const isDuplicate = existing.some((ex) => {
|
|
2986
|
+
const exWords = ex.content.toLowerCase().split(/\s+/).filter((w) => w.length > 3);
|
|
2987
|
+
const overlap = exWords.filter((w) => entryWords.has(w)).length;
|
|
2988
|
+
return overlap / Math.max(entryWords.size, 1) > 0.6;
|
|
2989
|
+
});
|
|
2990
|
+
if (isDuplicate) {
|
|
2991
|
+
skipped++;
|
|
2992
|
+
results.push({ content: entry.content.slice(0, 80), status: "skipped_duplicate" });
|
|
2993
|
+
continue;
|
|
2994
|
+
}
|
|
2995
|
+
} catch {
|
|
2996
|
+
}
|
|
2997
|
+
}
|
|
2998
|
+
const piiText = entry.content + (entry.detail ? " " + entry.detail : "");
|
|
2999
|
+
const piiMatches = detectSensitiveData(piiText);
|
|
3000
|
+
const id = generateId2();
|
|
3001
|
+
const timestamp = now2();
|
|
3002
|
+
db.insert(memories).values({
|
|
3003
|
+
id,
|
|
3004
|
+
content: entry.content,
|
|
3005
|
+
detail: entry.detail ?? null,
|
|
3006
|
+
domain,
|
|
3007
|
+
sourceAgentId: "import",
|
|
3008
|
+
sourceAgentName: "memory_import",
|
|
3009
|
+
sourceType: "inferred",
|
|
3010
|
+
sourceDescription: `Imported from ${params.source_type}`,
|
|
3011
|
+
confidence: 0.5,
|
|
3012
|
+
learnedAt: timestamp,
|
|
3013
|
+
hasPiiFlag: piiMatches.length > 0 ? 1 : 0
|
|
3014
|
+
}).run();
|
|
3015
|
+
if (vecAvailable) {
|
|
3016
|
+
try {
|
|
3017
|
+
const embeddingText = entry.content + (entry.detail ? " " + entry.detail : "");
|
|
3018
|
+
const emb = await generateEmbedding(embeddingText);
|
|
3019
|
+
insertEmbedding(sqlite, id, emb);
|
|
3020
|
+
} catch {
|
|
3021
|
+
}
|
|
3022
|
+
}
|
|
3023
|
+
db.insert(memoryEvents).values({
|
|
3024
|
+
id: generateId2(),
|
|
3025
|
+
memoryId: id,
|
|
3026
|
+
eventType: "created",
|
|
3027
|
+
agentId: "import",
|
|
3028
|
+
agentName: "memory_import",
|
|
3029
|
+
newValue: JSON.stringify({ content: entry.content, domain, importedFrom: params.source_type }),
|
|
3030
|
+
timestamp
|
|
3031
|
+
}).run();
|
|
3032
|
+
imported++;
|
|
3033
|
+
importedIds.push(id);
|
|
3034
|
+
results.push({ content: entry.content.slice(0, 80), status: "imported" });
|
|
3035
|
+
}
|
|
3036
|
+
if (extractionProvider && importedIds.length > 0) {
|
|
3037
|
+
(async () => {
|
|
3038
|
+
try {
|
|
3039
|
+
const existingNames = sqlite.prepare(`SELECT DISTINCT entity_name FROM memories WHERE entity_name IS NOT NULL AND deleted_at IS NULL`).all();
|
|
3040
|
+
const names = existingNames.map((r) => r.entity_name);
|
|
3041
|
+
for (const id of importedIds) {
|
|
3042
|
+
try {
|
|
3043
|
+
const mem = sqlite.prepare(`SELECT content, detail, entity_type FROM memories WHERE id = ? AND deleted_at IS NULL`).get(id);
|
|
3044
|
+
if (!mem || mem.entity_type) continue;
|
|
3045
|
+
const extraction = await extractEntity(
|
|
3046
|
+
extractionProvider,
|
|
3047
|
+
mem.content,
|
|
3048
|
+
mem.detail,
|
|
3049
|
+
names
|
|
3050
|
+
);
|
|
3051
|
+
const validation = validateExtraction(extraction);
|
|
3052
|
+
if (!validation.valid) continue;
|
|
3053
|
+
sqlite.transaction(() => {
|
|
3054
|
+
sqlite.prepare(
|
|
3055
|
+
`UPDATE memories SET entity_type = ?, entity_name = ?, structured_data = ? WHERE id = ? AND entity_type IS NULL AND deleted_at IS NULL`
|
|
3056
|
+
).run(extraction.entity_type, extraction.entity_name, JSON.stringify(extraction.structured_data), id);
|
|
3057
|
+
for (const conn of extraction.suggested_connections) {
|
|
3058
|
+
const target = sqlite.prepare(`SELECT id FROM memories WHERE entity_name = ? COLLATE NOCASE AND deleted_at IS NULL LIMIT 1`).get(conn.target_entity_name);
|
|
3059
|
+
if (target && target.id !== id) {
|
|
3060
|
+
sqlite.prepare(
|
|
3061
|
+
`INSERT INTO memory_connections (source_memory_id, target_memory_id, relationship) VALUES (?, ?, ?)`
|
|
3062
|
+
).run(id, target.id, conn.relationship);
|
|
3063
|
+
}
|
|
3064
|
+
}
|
|
3065
|
+
})();
|
|
3066
|
+
if (extraction.entity_name) names.push(extraction.entity_name);
|
|
3067
|
+
} catch {
|
|
3068
|
+
}
|
|
3069
|
+
}
|
|
3070
|
+
bumpLastModified(sqlite);
|
|
3071
|
+
} catch {
|
|
3072
|
+
}
|
|
3073
|
+
})();
|
|
3074
|
+
}
|
|
3075
|
+
sqlite.prepare(`
|
|
3076
|
+
INSERT INTO memory_events (id, memory_id, event_type, new_value, timestamp)
|
|
3077
|
+
VALUES (?, 'system', 'import', ?, ?)
|
|
3078
|
+
`).run(
|
|
3079
|
+
generateId2(),
|
|
3080
|
+
JSON.stringify({ source_type: params.source_type, imported, skipped, total_entries: entries.length }),
|
|
3081
|
+
now2()
|
|
3082
|
+
);
|
|
3083
|
+
bumpLastModified(sqlite);
|
|
3084
|
+
return textResult({
|
|
3085
|
+
imported,
|
|
3086
|
+
skipped_duplicates: skipped,
|
|
3087
|
+
total_parsed: entries.length,
|
|
3088
|
+
note: imported > 0 ? `Imported ${imported} memories at confidence 0.5 (unreviewed). Confirm them in the dashboard or via memory_confirm to boost confidence.` : "All entries were duplicates of existing memories."
|
|
3089
|
+
});
|
|
3090
|
+
}
|
|
3091
|
+
);
|
|
3092
|
+
let cachedSyncKeys = null;
|
|
3093
|
+
server.tool(
|
|
3094
|
+
"memory_sync",
|
|
3095
|
+
"Sync memories with cloud. Requires Pro tier setup (passphrase + Turso credentials in ~/.engrams/credentials.json). Push local changes and pull remote changes.",
|
|
3096
|
+
{
|
|
3097
|
+
passphrase: z.string().describe("Your encryption passphrase. Required on first sync or after restart.")
|
|
3098
|
+
},
|
|
3099
|
+
async (params) => {
|
|
3100
|
+
const creds = loadCredentials();
|
|
3101
|
+
if (!creds?.tursoUrl || !creds?.tursoAuthToken) {
|
|
3102
|
+
return textResult({ error: "Cloud sync not configured. Set tursoUrl and tursoAuthToken in ~/.engrams/credentials.json or via the dashboard settings." });
|
|
3103
|
+
}
|
|
3104
|
+
const salt = Buffer.from(creds.salt, "base64");
|
|
3105
|
+
const keys = deriveKeys(params.passphrase, salt);
|
|
3106
|
+
cachedSyncKeys = keys;
|
|
3107
|
+
const result = await sync(sqlite, {
|
|
3108
|
+
tursoUrl: creds.tursoUrl,
|
|
3109
|
+
tursoAuthToken: creds.tursoAuthToken,
|
|
3110
|
+
keys
|
|
3111
|
+
}, creds.deviceId);
|
|
3112
|
+
bumpLastModified(sqlite);
|
|
3113
|
+
return textResult({ status: "synced", ...result });
|
|
3114
|
+
}
|
|
3115
|
+
);
|
|
3116
|
+
const syncCreds = loadCredentials();
|
|
3117
|
+
if (syncCreds?.tursoUrl && syncCreds?.tursoAuthToken) {
|
|
3118
|
+
const SYNC_INTERVAL = 5 * 60 * 1e3;
|
|
3119
|
+
setInterval(async () => {
|
|
3120
|
+
if (!cachedSyncKeys) return;
|
|
3121
|
+
try {
|
|
3122
|
+
await sync(sqlite, {
|
|
3123
|
+
tursoUrl: syncCreds.tursoUrl,
|
|
3124
|
+
tursoAuthToken: syncCreds.tursoAuthToken,
|
|
3125
|
+
keys: cachedSyncKeys
|
|
3126
|
+
}, syncCreds.deviceId);
|
|
3127
|
+
} catch {
|
|
3128
|
+
}
|
|
3129
|
+
}, SYNC_INTERVAL);
|
|
3130
|
+
}
|
|
2046
3131
|
server.resource("memory-index", "memory://index", async (uri) => {
|
|
2047
3132
|
const domains = sqlite.prepare(
|
|
2048
3133
|
`SELECT domain, COUNT(*) as count FROM memories WHERE deleted_at IS NULL GROUP BY domain`
|