@slashml/memory 1.0.11 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +857 -13
- package/package.json +1 -1
package/bin/cli.js
CHANGED
|
@@ -10,7 +10,8 @@ const readline = require("readline");
|
|
|
10
10
|
|
|
11
11
|
const DEFAULT_BASE_URL = "https://api.slashml.ai";
|
|
12
12
|
const DEFAULT_CONSOLE_URL = "https://app.slashml.ai";
|
|
13
|
-
const PACKAGE_VERSION = "1.0.
|
|
13
|
+
const PACKAGE_VERSION = "1.0.13";
|
|
14
|
+
const MAX_STOP_BLOCKS_PER_PROMPT = 3;
|
|
14
15
|
|
|
15
16
|
function printHelp() {
|
|
16
17
|
console.log(`@slashml/memory v${PACKAGE_VERSION} -- Official CLI & MCP Stdio Server for / ml (SlashML · https://slashml.ai)
|
|
@@ -43,7 +44,14 @@ Options:
|
|
|
43
44
|
--once Run a single watch-sandbox transcript scan pass and exit
|
|
44
45
|
--transcript-dir <dir> Additional directory to scan for .jsonl agent transcripts
|
|
45
46
|
status, --status Show ambient memory ROI telemetry (tokens processed, % noise filtered, rules assimilated, amnesia intercepted)
|
|
46
|
-
--
|
|
47
|
+
migrate, --migrate Migrate memories from Mem0, Zep, Supermemory, Honcho, Hindsight, Letta, or a JSON/JSONL/MD file
|
|
48
|
+
--from <provider> Source provider for migrate: mem0 | zep | supermemory | honcho | hindsight | letta | file
|
|
49
|
+
--source-key <key> Source provider API key (or MEM0_API_KEY / ZEP_API_KEY / SUPERMEMORY_API_KEY / HONCHO_API_KEY / LETTA_API_KEY)
|
|
50
|
+
--source-url <url> Optional custom base URL for self-hosted source provider
|
|
51
|
+
--file <path> Import memories from a local JSON, JSONL, CSV, or Markdown export file
|
|
52
|
+
import-local, --import-local Import local coding-agent memories (~/.claude/projects/*/memory/*.md, CLAUDE.md, .cursorrules, .windsurfrules, .cursor/rules)
|
|
53
|
+
--dry-run Preview discovered memories during migrate or import-local without writing to / ml
|
|
54
|
+
--json Format status/sync/migrate output as JSON
|
|
47
55
|
--signup <work-email> Request a 6-digit OTP to provision a / ml free API key (requires work/custom domain @company.com; for @gmail.com / @hotmail.com sign in with Google/GitHub at https://app.slashml.ai)
|
|
48
56
|
--verify <work-email> <code> Verify 6-digit code and print the new SLASHML_API_KEY to stdout`);
|
|
49
57
|
}
|
|
@@ -195,6 +203,13 @@ function parseCliArgs(argv) {
|
|
|
195
203
|
let intervalMs = 1500;
|
|
196
204
|
let transcriptDir = process.env.SLASHML_TRANSCRIPT_DIR || null;
|
|
197
205
|
let forgetId = null;
|
|
206
|
+
let isMigrate = false;
|
|
207
|
+
let migrateFrom = null;
|
|
208
|
+
let sourceKey = null;
|
|
209
|
+
let sourceUrl = null;
|
|
210
|
+
let importFile = null;
|
|
211
|
+
let isImportLocal = false;
|
|
212
|
+
let isDryRun = false;
|
|
198
213
|
|
|
199
214
|
for (let i = 0; i < argv.length; i++) {
|
|
200
215
|
const arg = argv[i];
|
|
@@ -214,6 +229,22 @@ function parseCliArgs(argv) {
|
|
|
214
229
|
isSyncRules = true;
|
|
215
230
|
} else if (arg === "--watch-sandbox" || arg === "watch-sandbox") {
|
|
216
231
|
isWatchSandbox = true;
|
|
232
|
+
} else if (arg === "migrate" || arg === "--migrate") {
|
|
233
|
+
isMigrate = true;
|
|
234
|
+
} else if (arg === "--from" && i + 1 < argv.length) {
|
|
235
|
+
migrateFrom = argv[++i].trim().toLowerCase();
|
|
236
|
+
isMigrate = true;
|
|
237
|
+
} else if ((arg === "--source-key" || arg === "--from-key") && i + 1 < argv.length) {
|
|
238
|
+
sourceKey = argv[++i].trim();
|
|
239
|
+
} else if ((arg === "--source-url" || arg === "--from-url") && i + 1 < argv.length) {
|
|
240
|
+
sourceUrl = argv[++i].trim().replace(/\/+$/, "");
|
|
241
|
+
} else if (arg === "--file" && i + 1 < argv.length) {
|
|
242
|
+
importFile = argv[++i];
|
|
243
|
+
isMigrate = true;
|
|
244
|
+
} else if (arg === "import-local" || arg === "--import-local") {
|
|
245
|
+
isImportLocal = true;
|
|
246
|
+
} else if (arg === "--dry-run") {
|
|
247
|
+
isDryRun = true;
|
|
217
248
|
} else if ((arg === "--forget" || arg === "forget") && i + 1 < argv.length) {
|
|
218
249
|
forgetId = argv[++i];
|
|
219
250
|
} else if (arg === "--daemon") {
|
|
@@ -291,6 +322,13 @@ function parseCliArgs(argv) {
|
|
|
291
322
|
intervalMs,
|
|
292
323
|
transcriptDir,
|
|
293
324
|
forgetId,
|
|
325
|
+
isMigrate,
|
|
326
|
+
migrateFrom,
|
|
327
|
+
sourceKey,
|
|
328
|
+
sourceUrl,
|
|
329
|
+
importFile,
|
|
330
|
+
isImportLocal,
|
|
331
|
+
isDryRun,
|
|
294
332
|
};
|
|
295
333
|
}
|
|
296
334
|
|
|
@@ -1171,6 +1209,22 @@ async function runStatusCommand(opts) {
|
|
|
1171
1209
|
noiseFilteredPct = Number((Math.max(tokRatio, evRatio) * 100.0).toFixed(1));
|
|
1172
1210
|
}
|
|
1173
1211
|
|
|
1212
|
+
const objectionsRepeatSuppressed = Number(
|
|
1213
|
+
(serverStatus && serverStatus.objections_repeat_suppressed) || 0
|
|
1214
|
+
);
|
|
1215
|
+
const objectionsChainedBlocks = Number(
|
|
1216
|
+
(serverStatus && serverStatus.objections_chained_distinct_blocks) || 0
|
|
1217
|
+
);
|
|
1218
|
+
const objectionsResolvedOnRetry = Number(
|
|
1219
|
+
(serverStatus && serverStatus.objections_resolved_on_retry) || 0
|
|
1220
|
+
);
|
|
1221
|
+
const memoriesAutoDowngraded = Number(
|
|
1222
|
+
(serverStatus && serverStatus.memories_auto_downgraded) || 0
|
|
1223
|
+
);
|
|
1224
|
+
const flaggedNoisyMemories = Array.isArray(serverStatus && serverStatus.flagged_noisy_memories)
|
|
1225
|
+
? serverStatus.flagged_noisy_memories
|
|
1226
|
+
: [];
|
|
1227
|
+
|
|
1174
1228
|
const summaryLine =
|
|
1175
1229
|
(serverStatus && serverStatus.summary_line) ||
|
|
1176
1230
|
`${memoriesAssimilated} rules learned passively • ${amnesiaIntercepted} amnesia episodes intercepted • ${Math.round(noiseFilteredPct)}% stream noise filtered`;
|
|
@@ -1180,6 +1234,11 @@ async function runStatusCommand(opts) {
|
|
|
1180
1234
|
memories_assimilated: memoriesAssimilated,
|
|
1181
1235
|
memories_superseded: memoriesSuperseded,
|
|
1182
1236
|
amnesia_episodes_intercepted: amnesiaIntercepted,
|
|
1237
|
+
objections_chained_distinct_blocks: objectionsChainedBlocks,
|
|
1238
|
+
objections_repeat_suppressed: objectionsRepeatSuppressed,
|
|
1239
|
+
objections_resolved_on_retry: objectionsResolvedOnRetry,
|
|
1240
|
+
memories_auto_downgraded: memoriesAutoDowngraded,
|
|
1241
|
+
flagged_noisy_memories: flaggedNoisyMemories,
|
|
1183
1242
|
events_processed: eventsProcessed,
|
|
1184
1243
|
events_filtered_noise: eventsFiltered,
|
|
1185
1244
|
summary_line: summaryLine,
|
|
@@ -1200,6 +1259,18 @@ async function runStatusCommand(opts) {
|
|
|
1200
1259
|
` # New Memories Assimilated : ${memoriesAssimilated}${memoriesSuperseded > 0 ? ` (${memoriesSuperseded} superseded)` : ""}`
|
|
1201
1260
|
);
|
|
1202
1261
|
console.log(` # Amnesia Episodes Intercepted : ${amnesiaIntercepted}`);
|
|
1262
|
+
if (
|
|
1263
|
+
objectionsRepeatSuppressed > 0 ||
|
|
1264
|
+
objectionsResolvedOnRetry > 0 ||
|
|
1265
|
+
memoriesAutoDowngraded > 0
|
|
1266
|
+
) {
|
|
1267
|
+
console.log(
|
|
1268
|
+
` # Objections Resolved on Retry : ${objectionsResolvedOnRetry}${objectionsChainedBlocks > 0 ? ` (${objectionsChainedBlocks} chained distinct)` : ""}`
|
|
1269
|
+
);
|
|
1270
|
+
console.log(
|
|
1271
|
+
` # Repeat Objections Suppressed : ${objectionsRepeatSuppressed}${memoriesAutoDowngraded > 0 ? ` (${memoriesAutoDowngraded} noisy memories auto-downgraded)` : ""}`
|
|
1272
|
+
);
|
|
1273
|
+
}
|
|
1203
1274
|
}
|
|
1204
1275
|
|
|
1205
1276
|
function extractLastPlannerResponseFromTranscript(transcriptPath) {
|
|
@@ -1400,7 +1471,9 @@ async function runAmbientHookMode(opts) {
|
|
|
1400
1471
|
payload.terminationReason !== undefined ||
|
|
1401
1472
|
payload.workspacePaths !== undefined ||
|
|
1402
1473
|
payload.artifactDirectoryPath !== undefined ||
|
|
1403
|
-
payload.conversationId !== undefined
|
|
1474
|
+
payload.conversationId !== undefined ||
|
|
1475
|
+
(payload.conversation_id !== undefined &&
|
|
1476
|
+
(cliEvent === "Stop" || payload.transcript_path !== undefined))
|
|
1404
1477
|
);
|
|
1405
1478
|
|
|
1406
1479
|
let rawEventName = String(
|
|
@@ -1458,8 +1531,8 @@ async function runAmbientHookMode(opts) {
|
|
|
1458
1531
|
}
|
|
1459
1532
|
}
|
|
1460
1533
|
|
|
1461
|
-
//
|
|
1462
|
-
if (
|
|
1534
|
+
// Enforce hard 3-retry safety ceiling locally if harness passes an explicit loop_count >= 3
|
|
1535
|
+
if (hookEvent === "Stop" && Number(payload.loop_count || 0) >= MAX_STOP_BLOCKS_PER_PROMPT) {
|
|
1463
1536
|
writeDefaultHookOutput(isGeminiHook, rawEventName, isCursorHook);
|
|
1464
1537
|
return;
|
|
1465
1538
|
}
|
|
@@ -1655,6 +1728,30 @@ async function runAmbientHookMode(opts) {
|
|
|
1655
1728
|
.trim()
|
|
1656
1729
|
.slice(0, 1024);
|
|
1657
1730
|
|
|
1731
|
+
const whisperKey = sessionKey || scopeStateKey;
|
|
1732
|
+
const isMidTurnContinuation =
|
|
1733
|
+
rawEventName === "PreInvocation" &&
|
|
1734
|
+
(Number(payload.invocationNum || 1) > 1 || Number(payload.executionNum || 1) > 1);
|
|
1735
|
+
if (!dedupState.stop_turn_by_session || typeof dedupState.stop_turn_by_session !== "object") {
|
|
1736
|
+
dedupState.stop_turn_by_session = {};
|
|
1737
|
+
}
|
|
1738
|
+
if (hookEvent === "UserPromptSubmit" && !isMidTurnContinuation && whisperKey) {
|
|
1739
|
+
const existingStopTurn = dedupState.stop_turn_by_session[whisperKey];
|
|
1740
|
+
if (
|
|
1741
|
+
existingStopTurn &&
|
|
1742
|
+
((Array.isArray(existingStopTurn.blocked_ids) && existingStopTurn.blocked_ids.length > 0) ||
|
|
1743
|
+
Number(existingStopTurn.retry_count || 0) > 0)
|
|
1744
|
+
) {
|
|
1745
|
+
dedupState.stop_turn_by_session[whisperKey] = { blocked_ids: [], retry_count: 0 };
|
|
1746
|
+
try {
|
|
1747
|
+
fs.mkdirSync(path.dirname(dedupPath), { recursive: true });
|
|
1748
|
+
fs.writeFileSync(dedupPath, JSON.stringify(dedupState), "utf8");
|
|
1749
|
+
} catch {
|
|
1750
|
+
// ignore state persistence errors
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1658
1755
|
if (
|
|
1659
1756
|
!deltaText ||
|
|
1660
1757
|
deltaText.length < 4 ||
|
|
@@ -1665,6 +1762,21 @@ async function runAmbientHookMode(opts) {
|
|
|
1665
1762
|
return;
|
|
1666
1763
|
}
|
|
1667
1764
|
|
|
1765
|
+
const activeStopTurn =
|
|
1766
|
+
whisperKey &&
|
|
1767
|
+
dedupState.stop_turn_by_session[whisperKey] &&
|
|
1768
|
+
typeof dedupState.stop_turn_by_session[whisperKey] === "object"
|
|
1769
|
+
? dedupState.stop_turn_by_session[whisperKey]
|
|
1770
|
+
: { blocked_ids: [], retry_count: 0 };
|
|
1771
|
+
const turnBlockedIds = Array.isArray(activeStopTurn.blocked_ids)
|
|
1772
|
+
? activeStopTurn.blocked_ids.map(String).filter(Boolean)
|
|
1773
|
+
: [];
|
|
1774
|
+
const turnStopRetries = Number(activeStopTurn.retry_count || 0);
|
|
1775
|
+
if (hookEvent === "Stop" && turnStopRetries >= MAX_STOP_BLOCKS_PER_PROMPT) {
|
|
1776
|
+
writeDefaultHookOutput(isGeminiHook, rawEventName, isCursorHook);
|
|
1777
|
+
return;
|
|
1778
|
+
}
|
|
1779
|
+
|
|
1668
1780
|
// Event deduplication (scoped to project/session/step and only persisted for non-actionable noise)
|
|
1669
1781
|
const stepKey = String(
|
|
1670
1782
|
payload.stepIdx ?? payload.invocationNum ?? payload.executionNum ?? ""
|
|
@@ -1706,9 +1818,6 @@ async function runAmbientHookMode(opts) {
|
|
|
1706
1818
|
if (!dedupState.blocked_actions_by_session || typeof dedupState.blocked_actions_by_session !== "object") {
|
|
1707
1819
|
dedupState.blocked_actions_by_session = {};
|
|
1708
1820
|
}
|
|
1709
|
-
const whisperKey = sessionKey || scopeStateKey;
|
|
1710
|
-
const isMidTurnContinuation =
|
|
1711
|
-
rawEventName === "PreInvocation" && Number(payload.invocationNum || 1) > 1;
|
|
1712
1821
|
const targetSig = toolIn
|
|
1713
1822
|
? String(
|
|
1714
1823
|
toolIn.file_path ||
|
|
@@ -1801,8 +1910,11 @@ async function runAmbientHookMode(opts) {
|
|
|
1801
1910
|
? {
|
|
1802
1911
|
last_assistant_message: deltaText,
|
|
1803
1912
|
stop_hook_active: Boolean(payload.stop_hook_active),
|
|
1913
|
+
turn_blocked_memory_ids: turnBlockedIds,
|
|
1914
|
+
stop_retry_count: turnStopRetries,
|
|
1804
1915
|
}
|
|
1805
1916
|
: {}),
|
|
1917
|
+
...(isMidTurnContinuation ? { is_mid_turn_continuation: true } : {}),
|
|
1806
1918
|
tool_name: toolName,
|
|
1807
1919
|
tool_input: effectiveToolIn,
|
|
1808
1920
|
tool_response: payload.tool_response || payload.toolResponse || null,
|
|
@@ -1812,11 +1924,46 @@ async function runAmbientHookMode(opts) {
|
|
|
1812
1924
|
model: process.env.SLASHML_MODEL || undefined,
|
|
1813
1925
|
user_id: opts.userId,
|
|
1814
1926
|
...(projectId ? { project_id: projectId } : {}),
|
|
1815
|
-
|
|
1927
|
+
session_id: whisperKey,
|
|
1816
1928
|
client_raw_tokens_delta: (telem.pending_raw_tokens_delta || 0) + strippedTokens,
|
|
1817
1929
|
client_dropped_tokens_delta: (telem.pending_dropped_tokens_delta || 0) + strippedTokens,
|
|
1818
1930
|
client_dropped_events_delta: telem.pending_dropped_events_delta || 0,
|
|
1819
1931
|
});
|
|
1932
|
+
// Client-side per-memory Stop circuit breaker (defense-in-depth across server restarts/versions):
|
|
1933
|
+
// at most 1 Stop block per memory_id per user prompt, and at most MAX_STOP_BLOCKS_PER_PROMPT total.
|
|
1934
|
+
if (
|
|
1935
|
+
hookEvent === "Stop" &&
|
|
1936
|
+
resp &&
|
|
1937
|
+
resp.should_intervene &&
|
|
1938
|
+
(resp.intervention_severity === "BLOCK" || resp.action === "BLOCK")
|
|
1939
|
+
) {
|
|
1940
|
+
const respTriggered = Array.from(
|
|
1941
|
+
new Set(
|
|
1942
|
+
[
|
|
1943
|
+
...(Array.isArray(resp.violated_memory_ids) ? resp.violated_memory_ids : []),
|
|
1944
|
+
...(resp.violated_memory_id ? [resp.violated_memory_id] : []),
|
|
1945
|
+
...(resp.triggered_memory_id ? [resp.triggered_memory_id] : []),
|
|
1946
|
+
]
|
|
1947
|
+
.map(String)
|
|
1948
|
+
.filter(Boolean)
|
|
1949
|
+
)
|
|
1950
|
+
);
|
|
1951
|
+
const novelStopIds = respTriggered.filter((id) => !turnBlockedIds.includes(id));
|
|
1952
|
+
if (
|
|
1953
|
+
turnStopRetries >= MAX_STOP_BLOCKS_PER_PROMPT ||
|
|
1954
|
+
(respTriggered.length > 0 && novelStopIds.length === 0)
|
|
1955
|
+
) {
|
|
1956
|
+
resp.should_intervene = false;
|
|
1957
|
+
resp.intervention_severity = "NONE";
|
|
1958
|
+
resp.intervention_message = null;
|
|
1959
|
+
resp.hook_output = {};
|
|
1960
|
+
} else if (whisperKey) {
|
|
1961
|
+
dedupState.stop_turn_by_session[whisperKey] = {
|
|
1962
|
+
blocked_ids: Array.from(new Set([...turnBlockedIds, ...novelStopIds])).slice(-16),
|
|
1963
|
+
retry_count: turnStopRetries + 1,
|
|
1964
|
+
};
|
|
1965
|
+
}
|
|
1966
|
+
}
|
|
1820
1967
|
telem.pending_raw_tokens_delta = 0;
|
|
1821
1968
|
telem.pending_dropped_tokens_delta = 0;
|
|
1822
1969
|
telem.pending_dropped_events_delta = 0;
|
|
@@ -1879,10 +2026,18 @@ async function runAmbientHookMode(opts) {
|
|
|
1879
2026
|
} else {
|
|
1880
2027
|
telem.noise_tokens_filtered = (telem.noise_tokens_filtered || 0) + rawEventTokens;
|
|
1881
2028
|
telem.events_filtered_noise = (telem.events_filtered_noise || 0) + 1;
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
2029
|
+
const wasRepeatSuppressed = Boolean(
|
|
2030
|
+
resp &&
|
|
2031
|
+
((Array.isArray(resp.repeat_suppressed_memory_ids) &&
|
|
2032
|
+
resp.repeat_suppressed_memory_ids.length > 0) ||
|
|
2033
|
+
resp.objection_outcome === "SAFETY_CEILING_SUPPRESSED")
|
|
2034
|
+
);
|
|
2035
|
+
if (!wasRepeatSuppressed && hookEvent !== "Stop" && hookEvent !== "PreToolUse") {
|
|
2036
|
+
dedupState.hashes = [
|
|
2037
|
+
...(Array.isArray(dedupState.hashes) ? dedupState.hashes.slice(-63) : []),
|
|
2038
|
+
deltaHash,
|
|
2039
|
+
];
|
|
2040
|
+
}
|
|
1886
2041
|
}
|
|
1887
2042
|
try {
|
|
1888
2043
|
fs.mkdirSync(path.dirname(dedupPath), { recursive: true });
|
|
@@ -2637,6 +2792,695 @@ function drainPendingSandboxIntervention() {
|
|
|
2637
2792
|
return null;
|
|
2638
2793
|
}
|
|
2639
2794
|
|
|
2795
|
+
function makeMigrationDocId(prefix, text, userId = "default") {
|
|
2796
|
+
const crypto = require("crypto");
|
|
2797
|
+
const digest = crypto
|
|
2798
|
+
.createHash("sha1")
|
|
2799
|
+
.update(`${prefix}:${userId}:${String(text || "").trim().toLowerCase()}`)
|
|
2800
|
+
.digest("hex")
|
|
2801
|
+
.slice(0, 12);
|
|
2802
|
+
return `mig_${prefix}_${digest}`;
|
|
2803
|
+
}
|
|
2804
|
+
|
|
2805
|
+
function stripCommentMarkerBlock(input, startSuffix, endSuffix) {
|
|
2806
|
+
let out = String(input || "");
|
|
2807
|
+
while (true) {
|
|
2808
|
+
const mid = out.indexOf(startSuffix);
|
|
2809
|
+
if (mid === -1) break;
|
|
2810
|
+
const blockStart = out.lastIndexOf("<!--", mid);
|
|
2811
|
+
if (blockStart === -1) break;
|
|
2812
|
+
const endMid = out.indexOf(endSuffix, mid + startSuffix.length);
|
|
2813
|
+
if (endMid === -1) break;
|
|
2814
|
+
const blockEnd = out.indexOf("-->", endMid + endSuffix.length);
|
|
2815
|
+
if (blockEnd === -1) break;
|
|
2816
|
+
out = out.slice(0, blockStart) + out.slice(blockEnd + 3);
|
|
2817
|
+
}
|
|
2818
|
+
return out;
|
|
2819
|
+
}
|
|
2820
|
+
|
|
2821
|
+
function stripSlashmlInjectedBlocks(rawMarkdown) {
|
|
2822
|
+
const withoutRules = stripCommentMarkerBlock(
|
|
2823
|
+
rawMarkdown,
|
|
2824
|
+
"-MEMORY-RULE:START",
|
|
2825
|
+
"-MEMORY-RULE:END"
|
|
2826
|
+
);
|
|
2827
|
+
const withoutAmbient = stripCommentMarkerBlock(
|
|
2828
|
+
withoutRules,
|
|
2829
|
+
"SLASHML-AMBIENT-MEMORY:START",
|
|
2830
|
+
"SLASHML-AMBIENT-MEMORY:END"
|
|
2831
|
+
);
|
|
2832
|
+
return withoutAmbient
|
|
2833
|
+
.replace(/^---[\r\n]+[\s\S]*?[\r\n]+---[\r\n]*/m, "")
|
|
2834
|
+
.trim();
|
|
2835
|
+
}
|
|
2836
|
+
|
|
2837
|
+
function extractEntriesFromMarkdown(rawMarkdown, defaultUserId, baseMetadata = {}) {
|
|
2838
|
+
const cleaned = stripSlashmlInjectedBlocks(rawMarkdown);
|
|
2839
|
+
if (!cleaned) return [];
|
|
2840
|
+
const entries = [];
|
|
2841
|
+
const lines = cleaned.split(/\r?\n/);
|
|
2842
|
+
let currentHeading = "";
|
|
2843
|
+
let paragraphBuf = [];
|
|
2844
|
+
|
|
2845
|
+
function flushParagraph() {
|
|
2846
|
+
if (paragraphBuf.length === 0) return;
|
|
2847
|
+
const text = paragraphBuf.join(" ").replace(/\s+/g, " ").trim();
|
|
2848
|
+
paragraphBuf = [];
|
|
2849
|
+
if (text.length >= 12) {
|
|
2850
|
+
const fullText = currentHeading ? `[${currentHeading}] ${text}` : text;
|
|
2851
|
+
entries.push({
|
|
2852
|
+
text: fullText,
|
|
2853
|
+
user_id: defaultUserId,
|
|
2854
|
+
metadata: { ...baseMetadata, ...(currentHeading ? { section: currentHeading } : {}) },
|
|
2855
|
+
});
|
|
2856
|
+
}
|
|
2857
|
+
}
|
|
2858
|
+
|
|
2859
|
+
for (const rawLine of lines) {
|
|
2860
|
+
const line = rawLine.trim();
|
|
2861
|
+
if (!line) {
|
|
2862
|
+
flushParagraph();
|
|
2863
|
+
continue;
|
|
2864
|
+
}
|
|
2865
|
+
const headingMatch = line.match(/^#{1,4}\s+(.+)$/);
|
|
2866
|
+
if (headingMatch) {
|
|
2867
|
+
flushParagraph();
|
|
2868
|
+
currentHeading = headingMatch[1].trim();
|
|
2869
|
+
continue;
|
|
2870
|
+
}
|
|
2871
|
+
const bulletMatch = line.match(/^(?:[-*+]|\d+\.)\s+(.+)$/);
|
|
2872
|
+
if (bulletMatch) {
|
|
2873
|
+
flushParagraph();
|
|
2874
|
+
const itemText = bulletMatch[1].replace(/\s+/g, " ").trim();
|
|
2875
|
+
if (itemText.length >= 8) {
|
|
2876
|
+
const fullText = currentHeading ? `[${currentHeading}] ${itemText}` : itemText;
|
|
2877
|
+
entries.push({
|
|
2878
|
+
text: fullText,
|
|
2879
|
+
user_id: defaultUserId,
|
|
2880
|
+
metadata: { ...baseMetadata, ...(currentHeading ? { section: currentHeading } : {}) },
|
|
2881
|
+
});
|
|
2882
|
+
}
|
|
2883
|
+
continue;
|
|
2884
|
+
}
|
|
2885
|
+
paragraphBuf.push(line);
|
|
2886
|
+
}
|
|
2887
|
+
flushParagraph();
|
|
2888
|
+
return entries;
|
|
2889
|
+
}
|
|
2890
|
+
|
|
2891
|
+
function normalizeImportedRecord(item, provider, defaultUserId, extraMeta = {}) {
|
|
2892
|
+
if (!item) return null;
|
|
2893
|
+
if (typeof item === "string") {
|
|
2894
|
+
const t = item.trim();
|
|
2895
|
+
if (!t) return null;
|
|
2896
|
+
return {
|
|
2897
|
+
id: makeMigrationDocId(provider, t, defaultUserId),
|
|
2898
|
+
text: t,
|
|
2899
|
+
user_id: defaultUserId,
|
|
2900
|
+
metadata: { source_provider: provider, ...extraMeta },
|
|
2901
|
+
};
|
|
2902
|
+
}
|
|
2903
|
+
if (typeof item !== "object") return null;
|
|
2904
|
+
const text = String(
|
|
2905
|
+
item.memory ||
|
|
2906
|
+
item.text ||
|
|
2907
|
+
item.fact ||
|
|
2908
|
+
item.content ||
|
|
2909
|
+
item.summary ||
|
|
2910
|
+
item.value ||
|
|
2911
|
+
(Array.isArray(item.messages)
|
|
2912
|
+
? item.messages
|
|
2913
|
+
.map((m) => (typeof m === "string" ? m : `${m.role || "user"}: ${m.content || m.text || ""}`))
|
|
2914
|
+
.join("\n")
|
|
2915
|
+
: "") ||
|
|
2916
|
+
""
|
|
2917
|
+
).trim();
|
|
2918
|
+
if (!text) return null;
|
|
2919
|
+
const resolvedUser = String(
|
|
2920
|
+
item.user_id ||
|
|
2921
|
+
item.userId ||
|
|
2922
|
+
item.agent_id ||
|
|
2923
|
+
item.agentId ||
|
|
2924
|
+
(item.metadata && (item.metadata.user_id || item.metadata.agent_id)) ||
|
|
2925
|
+
defaultUserId ||
|
|
2926
|
+
"default"
|
|
2927
|
+
).trim();
|
|
2928
|
+
const baseMeta = item.metadata && typeof item.metadata === "object" ? item.metadata : {};
|
|
2929
|
+
const categories = Array.isArray(item.categories) ? item.categories : undefined;
|
|
2930
|
+
return {
|
|
2931
|
+
id: String(item.id || item.memory_id || item.uuid || makeMigrationDocId(provider, text, resolvedUser)),
|
|
2932
|
+
text,
|
|
2933
|
+
user_id: resolvedUser,
|
|
2934
|
+
metadata: {
|
|
2935
|
+
...baseMeta,
|
|
2936
|
+
...extraMeta,
|
|
2937
|
+
source_provider: provider,
|
|
2938
|
+
user_id: resolvedUser,
|
|
2939
|
+
...(item.agent_id ? { agent_id: item.agent_id } : {}),
|
|
2940
|
+
...(item.app_id ? { app_id: item.app_id } : {}),
|
|
2941
|
+
...(item.run_id || item.session_id ? { run_id: item.run_id || item.session_id } : {}),
|
|
2942
|
+
...(categories ? { categories } : {}),
|
|
2943
|
+
...(item.created_at ? { source_created_at: item.created_at } : {}),
|
|
2944
|
+
},
|
|
2945
|
+
};
|
|
2946
|
+
}
|
|
2947
|
+
|
|
2948
|
+
function parseMemoriesFromFileContent(rawContent, filePath, defaultUserId) {
|
|
2949
|
+
const ext = path.extname(filePath || "").toLowerCase();
|
|
2950
|
+
const trimmed = String(rawContent || "").trim();
|
|
2951
|
+
if (!trimmed) return [];
|
|
2952
|
+
|
|
2953
|
+
if (ext === ".md" || ext === ".mdc" || ext === ".txt" || filePath.endsWith("rules")) {
|
|
2954
|
+
if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) {
|
|
2955
|
+
return extractEntriesFromMarkdown(trimmed, defaultUserId, {
|
|
2956
|
+
source_file: path.basename(filePath),
|
|
2957
|
+
}).map((e) => normalizeImportedRecord(e, "file", defaultUserId, e.metadata));
|
|
2958
|
+
}
|
|
2959
|
+
}
|
|
2960
|
+
|
|
2961
|
+
// Try JSON
|
|
2962
|
+
try {
|
|
2963
|
+
const parsedJson = JSON.parse(trimmed);
|
|
2964
|
+
const list = Array.isArray(parsedJson)
|
|
2965
|
+
? parsedJson
|
|
2966
|
+
: Array.isArray(parsedJson.results)
|
|
2967
|
+
? parsedJson.results
|
|
2968
|
+
: Array.isArray(parsedJson.memories)
|
|
2969
|
+
? parsedJson.memories
|
|
2970
|
+
: Array.isArray(parsedJson.documents)
|
|
2971
|
+
? parsedJson.documents
|
|
2972
|
+
: Array.isArray(parsedJson.items)
|
|
2973
|
+
? parsedJson.items
|
|
2974
|
+
: Array.isArray(parsedJson.facts)
|
|
2975
|
+
? parsedJson.facts
|
|
2976
|
+
: parsedJson.vault && Array.isArray(parsedJson.vault.documents)
|
|
2977
|
+
? parsedJson.vault.documents
|
|
2978
|
+
: [parsedJson];
|
|
2979
|
+
return list
|
|
2980
|
+
.map((item) => normalizeImportedRecord(item, "file", defaultUserId, { source_file: path.basename(filePath) }))
|
|
2981
|
+
.filter(Boolean);
|
|
2982
|
+
} catch {
|
|
2983
|
+
// Try JSONL
|
|
2984
|
+
const lines = trimmed.split(/\r?\n/);
|
|
2985
|
+
const jsonlItems = [];
|
|
2986
|
+
let isJsonl = false;
|
|
2987
|
+
for (const l of lines) {
|
|
2988
|
+
const s = l.trim();
|
|
2989
|
+
if (!s) continue;
|
|
2990
|
+
if (s.startsWith("{")) {
|
|
2991
|
+
try {
|
|
2992
|
+
const obj = JSON.parse(s);
|
|
2993
|
+
isJsonl = true;
|
|
2994
|
+
const norm = normalizeImportedRecord(obj, "file", defaultUserId, {
|
|
2995
|
+
source_file: path.basename(filePath),
|
|
2996
|
+
});
|
|
2997
|
+
if (norm) jsonlItems.push(norm);
|
|
2998
|
+
} catch {
|
|
2999
|
+
// ignore
|
|
3000
|
+
}
|
|
3001
|
+
}
|
|
3002
|
+
}
|
|
3003
|
+
if (isJsonl && jsonlItems.length > 0) {
|
|
3004
|
+
return jsonlItems;
|
|
3005
|
+
}
|
|
3006
|
+
}
|
|
3007
|
+
|
|
3008
|
+
return extractEntriesFromMarkdown(trimmed, defaultUserId, {
|
|
3009
|
+
source_file: path.basename(filePath),
|
|
3010
|
+
})
|
|
3011
|
+
.map((e) => normalizeImportedRecord(e, "file", defaultUserId, e.metadata))
|
|
3012
|
+
.filter(Boolean);
|
|
3013
|
+
}
|
|
3014
|
+
|
|
3015
|
+
async function fetchProviderMemories(provider, sourceKey, sourceUrl, defaultUserId) {
|
|
3016
|
+
const normProvider = String(provider || "").trim().toLowerCase();
|
|
3017
|
+
const collected = [];
|
|
3018
|
+
|
|
3019
|
+
if (normProvider === "mem0") {
|
|
3020
|
+
const base = (sourceUrl || "https://api.mem0.ai").replace(/\/+$/, "");
|
|
3021
|
+
const headers = {
|
|
3022
|
+
Authorization: sourceKey.startsWith("Token ") || sourceKey.startsWith("Bearer ") ? sourceKey : `Token ${sourceKey}`,
|
|
3023
|
+
"Content-Type": "application/json",
|
|
3024
|
+
Accept: "application/json",
|
|
3025
|
+
};
|
|
3026
|
+
for (let page = 1; page <= 50; page++) {
|
|
3027
|
+
const qs = new URLSearchParams({ page: String(page), page_size: "100" });
|
|
3028
|
+
if (defaultUserId && defaultUserId !== "default") {
|
|
3029
|
+
qs.set("user_id", defaultUserId);
|
|
3030
|
+
}
|
|
3031
|
+
const res = await fetch(`${base}/v1/memories/?${qs.toString()}`, { headers });
|
|
3032
|
+
if (!res.ok) {
|
|
3033
|
+
if (page === 1) {
|
|
3034
|
+
const v2Res = await fetch(`${base}/v2/memories/`, {
|
|
3035
|
+
method: "POST",
|
|
3036
|
+
headers,
|
|
3037
|
+
body: JSON.stringify({
|
|
3038
|
+
filters: defaultUserId && defaultUserId !== "default" ? { user_id: defaultUserId } : {},
|
|
3039
|
+
page: 1,
|
|
3040
|
+
page_size: 200,
|
|
3041
|
+
}),
|
|
3042
|
+
});
|
|
3043
|
+
if (!v2Res.ok) {
|
|
3044
|
+
const errTxt = await res.text().catch(() => "");
|
|
3045
|
+
throw new Error(`Mem0 API returned HTTP ${res.status}: ${errTxt.slice(0, 200)}`);
|
|
3046
|
+
}
|
|
3047
|
+
const v2Data = await v2Res.json();
|
|
3048
|
+
const v2Items = Array.isArray(v2Data) ? v2Data : v2Data.results || v2Data.memories || [];
|
|
3049
|
+
for (const it of v2Items) {
|
|
3050
|
+
const n = normalizeImportedRecord(it, "mem0", defaultUserId);
|
|
3051
|
+
if (n) collected.push(n);
|
|
3052
|
+
}
|
|
3053
|
+
}
|
|
3054
|
+
break;
|
|
3055
|
+
}
|
|
3056
|
+
const data = await res.json();
|
|
3057
|
+
const items = Array.isArray(data) ? data : data.results || data.memories || [];
|
|
3058
|
+
if (items.length === 0) break;
|
|
3059
|
+
for (const it of items) {
|
|
3060
|
+
const n = normalizeImportedRecord(it, "mem0", defaultUserId);
|
|
3061
|
+
if (n) collected.push(n);
|
|
3062
|
+
}
|
|
3063
|
+
if (items.length < 100 || (!Array.isArray(data) && !data.next)) break;
|
|
3064
|
+
}
|
|
3065
|
+
return collected;
|
|
3066
|
+
}
|
|
3067
|
+
|
|
3068
|
+
if (normProvider === "zep") {
|
|
3069
|
+
const base = (sourceUrl || "https://api.getzep.com").replace(/\/+$/, "");
|
|
3070
|
+
const authHeader = sourceKey.startsWith("z_") ? `Api-Key ${sourceKey}` : `Bearer ${sourceKey}`;
|
|
3071
|
+
const headers = { Authorization: authHeader, Accept: "application/json" };
|
|
3072
|
+
|
|
3073
|
+
if (defaultUserId && defaultUserId !== "default") {
|
|
3074
|
+
const uRes = await fetch(`${base}/api/v2/users/${encodeURIComponent(defaultUserId)}/facts`, { headers });
|
|
3075
|
+
if (uRes.ok) {
|
|
3076
|
+
const uData = await uRes.json();
|
|
3077
|
+
const facts = Array.isArray(uData) ? uData : uData.facts || uData.results || [];
|
|
3078
|
+
for (const f of facts) {
|
|
3079
|
+
const n = normalizeImportedRecord(f, "zep", defaultUserId);
|
|
3080
|
+
if (n) collected.push(n);
|
|
3081
|
+
}
|
|
3082
|
+
}
|
|
3083
|
+
}
|
|
3084
|
+
|
|
3085
|
+
const sRes = await fetch(`${base}/api/v2/sessions-ordered?pageSize=100&pageNumber=1`, { headers });
|
|
3086
|
+
if (!sRes.ok && collected.length === 0) {
|
|
3087
|
+
const errTxt = await sRes.text().catch(() => "");
|
|
3088
|
+
throw new Error(`Zep API returned HTTP ${sRes.status}: ${errTxt.slice(0, 200)}`);
|
|
3089
|
+
}
|
|
3090
|
+
if (sRes.ok) {
|
|
3091
|
+
const sData = await sRes.json();
|
|
3092
|
+
const sessions = Array.isArray(sData) ? sData : sData.sessions || sData.results || [];
|
|
3093
|
+
for (const sess of sessions.slice(0, 50)) {
|
|
3094
|
+
const sid = sess.session_id || sess.sessionId || sess.id;
|
|
3095
|
+
if (!sid) continue;
|
|
3096
|
+
const mRes = await fetch(`${base}/api/v2/sessions/${encodeURIComponent(sid)}/memory`, { headers });
|
|
3097
|
+
if (!mRes.ok) continue;
|
|
3098
|
+
const mData = await mRes.json();
|
|
3099
|
+
const facts = Array.isArray(mData.facts)
|
|
3100
|
+
? mData.facts
|
|
3101
|
+
: Array.isArray(mData.relevant_facts)
|
|
3102
|
+
? mData.relevant_facts
|
|
3103
|
+
: [];
|
|
3104
|
+
for (const f of facts) {
|
|
3105
|
+
const n = normalizeImportedRecord(f, "zep", sess.user_id || defaultUserId, { session_id: sid });
|
|
3106
|
+
if (n) collected.push(n);
|
|
3107
|
+
}
|
|
3108
|
+
if (mData.summary && mData.summary.content) {
|
|
3109
|
+
const n = normalizeImportedRecord(
|
|
3110
|
+
{ text: mData.summary.content, user_id: sess.user_id || defaultUserId },
|
|
3111
|
+
"zep",
|
|
3112
|
+
sess.user_id || defaultUserId,
|
|
3113
|
+
{ session_id: sid, kind: "session_summary" }
|
|
3114
|
+
);
|
|
3115
|
+
if (n) collected.push(n);
|
|
3116
|
+
}
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
return collected;
|
|
3120
|
+
}
|
|
3121
|
+
|
|
3122
|
+
if (normProvider === "supermemory") {
|
|
3123
|
+
const base = (sourceUrl || "https://api.supermemory.ai").replace(/\/+$/, "");
|
|
3124
|
+
const headers = {
|
|
3125
|
+
Authorization: sourceKey.startsWith("Bearer ") ? sourceKey : `Bearer ${sourceKey}`,
|
|
3126
|
+
"Content-Type": "application/json",
|
|
3127
|
+
Accept: "application/json",
|
|
3128
|
+
};
|
|
3129
|
+
let res = await fetch(`${base}/v3/documents/list`, {
|
|
3130
|
+
method: "POST",
|
|
3131
|
+
headers,
|
|
3132
|
+
body: JSON.stringify({ page: 1, limit: 200 }),
|
|
3133
|
+
});
|
|
3134
|
+
if (!res.ok) {
|
|
3135
|
+
res = await fetch(`${base}/v3/memories`, { headers });
|
|
3136
|
+
}
|
|
3137
|
+
if (!res.ok) {
|
|
3138
|
+
const errTxt = await res.text().catch(() => "");
|
|
3139
|
+
throw new Error(`Supermemory API returned HTTP ${res.status}: ${errTxt.slice(0, 200)}`);
|
|
3140
|
+
}
|
|
3141
|
+
const data = await res.json();
|
|
3142
|
+
const items = Array.isArray(data) ? data : data.memories || data.documents || data.results || [];
|
|
3143
|
+
for (const it of items) {
|
|
3144
|
+
const n = normalizeImportedRecord(it, "supermemory", defaultUserId);
|
|
3145
|
+
if (n) collected.push(n);
|
|
3146
|
+
}
|
|
3147
|
+
return collected;
|
|
3148
|
+
}
|
|
3149
|
+
|
|
3150
|
+
if (normProvider === "honcho") {
|
|
3151
|
+
const base = (sourceUrl || "https://api.honcho.dev").replace(/\/+$/, "");
|
|
3152
|
+
const headers = {
|
|
3153
|
+
Authorization: sourceKey.startsWith("Bearer ") ? sourceKey : `Bearer ${sourceKey}`,
|
|
3154
|
+
Accept: "application/json",
|
|
3155
|
+
};
|
|
3156
|
+
const appsRes = await fetch(`${base}/v1/apps`, { headers });
|
|
3157
|
+
if (!appsRes.ok) {
|
|
3158
|
+
const errTxt = await appsRes.text().catch(() => "");
|
|
3159
|
+
throw new Error(`Honcho API returned HTTP ${appsRes.status}: ${errTxt.slice(0, 200)}`);
|
|
3160
|
+
}
|
|
3161
|
+
const appsData = await appsRes.json();
|
|
3162
|
+
const apps = Array.isArray(appsData) ? appsData : appsData.apps || appsData.items || [];
|
|
3163
|
+
for (const app of apps.slice(0, 10)) {
|
|
3164
|
+
const appId = app.id || app.app_id;
|
|
3165
|
+
if (!appId) continue;
|
|
3166
|
+
const uRes = await fetch(`${base}/v1/apps/${encodeURIComponent(appId)}/users?size=100`, { headers });
|
|
3167
|
+
if (!uRes.ok) continue;
|
|
3168
|
+
const uData = await uRes.json();
|
|
3169
|
+
const users = Array.isArray(uData) ? uData : uData.items || uData.users || [];
|
|
3170
|
+
for (const u of users.slice(0, 50)) {
|
|
3171
|
+
const uid = u.id || u.user_id;
|
|
3172
|
+
if (!uid) continue;
|
|
3173
|
+
const mmRes = await fetch(
|
|
3174
|
+
`${base}/v1/apps/${encodeURIComponent(appId)}/users/${encodeURIComponent(uid)}/metamessages?size=100`,
|
|
3175
|
+
{ headers }
|
|
3176
|
+
);
|
|
3177
|
+
if (!mmRes.ok) continue;
|
|
3178
|
+
const mmData = await mmRes.json();
|
|
3179
|
+
const mms = Array.isArray(mmData) ? mmData : mmData.items || mmData.metamessages || [];
|
|
3180
|
+
for (const mm of mms) {
|
|
3181
|
+
const n = normalizeImportedRecord(mm, "honcho", u.name || uid, { app_id: appId });
|
|
3182
|
+
if (n) collected.push(n);
|
|
3183
|
+
}
|
|
3184
|
+
}
|
|
3185
|
+
}
|
|
3186
|
+
return collected;
|
|
3187
|
+
}
|
|
3188
|
+
|
|
3189
|
+
if (normProvider === "hindsight") {
|
|
3190
|
+
const base = (sourceUrl || "https://api.hindsight.vectorize.io").replace(/\/+$/, "");
|
|
3191
|
+
const headers = {
|
|
3192
|
+
Authorization: sourceKey.startsWith("Bearer ") ? sourceKey : `Bearer ${sourceKey}`,
|
|
3193
|
+
Accept: "application/json",
|
|
3194
|
+
};
|
|
3195
|
+
const res = await fetch(`${base}/v1/memories`, { headers });
|
|
3196
|
+
if (!res.ok) {
|
|
3197
|
+
const errTxt = await res.text().catch(() => "");
|
|
3198
|
+
throw new Error(`Hindsight API returned HTTP ${res.status}: ${errTxt.slice(0, 200)}`);
|
|
3199
|
+
}
|
|
3200
|
+
const data = await res.json();
|
|
3201
|
+
const items = Array.isArray(data) ? data : data.memories || data.results || data.items || [];
|
|
3202
|
+
for (const it of items) {
|
|
3203
|
+
const n = normalizeImportedRecord(it, "hindsight", defaultUserId);
|
|
3204
|
+
if (n) collected.push(n);
|
|
3205
|
+
}
|
|
3206
|
+
return collected;
|
|
3207
|
+
}
|
|
3208
|
+
|
|
3209
|
+
if (normProvider === "letta" || normProvider === "memgpt") {
|
|
3210
|
+
const base = (sourceUrl || "https://api.letta.com").replace(/\/+$/, "");
|
|
3211
|
+
const headers = {
|
|
3212
|
+
Authorization: sourceKey.startsWith("Bearer ") ? sourceKey : `Bearer ${sourceKey}`,
|
|
3213
|
+
Accept: "application/json",
|
|
3214
|
+
};
|
|
3215
|
+
const aRes = await fetch(`${base}/v1/agents/`, { headers });
|
|
3216
|
+
if (!aRes.ok) {
|
|
3217
|
+
const errTxt = await aRes.text().catch(() => "");
|
|
3218
|
+
throw new Error(`Letta API returned HTTP ${aRes.status}: ${errTxt.slice(0, 200)}`);
|
|
3219
|
+
}
|
|
3220
|
+
const agents = await aRes.json();
|
|
3221
|
+
const agentList = Array.isArray(agents) ? agents : agents.agents || [];
|
|
3222
|
+
for (const ag of agentList.slice(0, 30)) {
|
|
3223
|
+
const aid = ag.id;
|
|
3224
|
+
if (!aid) continue;
|
|
3225
|
+
const pRes = await fetch(`${base}/v1/agents/${encodeURIComponent(aid)}/archival-memory?limit=100`, {
|
|
3226
|
+
headers,
|
|
3227
|
+
});
|
|
3228
|
+
if (!pRes.ok) continue;
|
|
3229
|
+
const passages = await pRes.json();
|
|
3230
|
+
const pList = Array.isArray(passages) ? passages : passages.passages || passages.results || [];
|
|
3231
|
+
for (const p of pList) {
|
|
3232
|
+
const n = normalizeImportedRecord(p, "letta", ag.name || defaultUserId, { agent_id: aid });
|
|
3233
|
+
if (n) collected.push(n);
|
|
3234
|
+
}
|
|
3235
|
+
}
|
|
3236
|
+
return collected;
|
|
3237
|
+
}
|
|
3238
|
+
|
|
3239
|
+
throw new Error(
|
|
3240
|
+
`Unsupported provider "${provider}". Supported providers: mem0, zep, supermemory, honcho, hindsight, letta, file.`
|
|
3241
|
+
);
|
|
3242
|
+
}
|
|
3243
|
+
|
|
3244
|
+
async function batchIngestMigratedRecords(opts, records) {
|
|
3245
|
+
const seenIds = new Set();
|
|
3246
|
+
const deduped = [];
|
|
3247
|
+
for (const r of records) {
|
|
3248
|
+
if (!r || !r.text) continue;
|
|
3249
|
+
const id = r.id || makeMigrationDocId("import", r.text, r.user_id || opts.userId);
|
|
3250
|
+
if (seenIds.has(id)) continue;
|
|
3251
|
+
seenIds.add(id);
|
|
3252
|
+
deduped.push({ ...r, id });
|
|
3253
|
+
}
|
|
3254
|
+
|
|
3255
|
+
if (opts.isDryRun || deduped.length === 0) {
|
|
3256
|
+
return { total_discovered: deduped.length, ingested_count: 0, dry_run: Boolean(opts.isDryRun), sample: deduped.slice(0, 5) };
|
|
3257
|
+
}
|
|
3258
|
+
|
|
3259
|
+
if (!opts.apiKey) {
|
|
3260
|
+
throw new Error(
|
|
3261
|
+
"Missing SLASHML_API_KEY. Set SLASHML_API_KEY or pass --api-key <sml_live_...> (or sign in at https://app.slashml.ai)."
|
|
3262
|
+
);
|
|
3263
|
+
}
|
|
3264
|
+
|
|
3265
|
+
let ingestedCount = 0;
|
|
3266
|
+
const batchSize = 50;
|
|
3267
|
+
for (let i = 0; i < deduped.length; i += batchSize) {
|
|
3268
|
+
const chunk = deduped.slice(i, i + batchSize);
|
|
3269
|
+
const documents = chunk.map((r) => ({
|
|
3270
|
+
id: r.id,
|
|
3271
|
+
text: r.text,
|
|
3272
|
+
metadata: {
|
|
3273
|
+
...(r.metadata || {}),
|
|
3274
|
+
user_id: r.user_id || opts.userId || "default",
|
|
3275
|
+
},
|
|
3276
|
+
}));
|
|
3277
|
+
await apiRequest(opts, "POST", "/v1/memory/ingest", { documents });
|
|
3278
|
+
ingestedCount += chunk.length;
|
|
3279
|
+
}
|
|
3280
|
+
|
|
3281
|
+
return {
|
|
3282
|
+
total_discovered: deduped.length,
|
|
3283
|
+
ingested_count: ingestedCount,
|
|
3284
|
+
dry_run: false,
|
|
3285
|
+
sample: deduped.slice(0, 5),
|
|
3286
|
+
};
|
|
3287
|
+
}
|
|
3288
|
+
|
|
3289
|
+
async function runMigrateCommand(opts) {
|
|
3290
|
+
const provider = (opts.migrateFrom || (opts.importFile ? "file" : "mem0")).toLowerCase();
|
|
3291
|
+
let records = [];
|
|
3292
|
+
|
|
3293
|
+
if (opts.importFile || provider === "file") {
|
|
3294
|
+
if (!opts.importFile) {
|
|
3295
|
+
throw new Error("Missing --file <path> for file migration. Example: npx -y @slashml/memory migrate --file ./memories.json");
|
|
3296
|
+
}
|
|
3297
|
+
const resolvedPath = path.resolve(opts.importFile);
|
|
3298
|
+
if (!fs.existsSync(resolvedPath)) {
|
|
3299
|
+
throw new Error(`File not found: ${resolvedPath}`);
|
|
3300
|
+
}
|
|
3301
|
+
const content = fs.readFileSync(resolvedPath, "utf8");
|
|
3302
|
+
records = parseMemoriesFromFileContent(content, resolvedPath, opts.userId || "default");
|
|
3303
|
+
} else {
|
|
3304
|
+
const envKeyMap = {
|
|
3305
|
+
mem0: process.env.MEM0_API_KEY,
|
|
3306
|
+
zep: process.env.ZEP_API_KEY,
|
|
3307
|
+
supermemory: process.env.SUPERMEMORY_API_KEY,
|
|
3308
|
+
honcho: process.env.HONCHO_API_KEY,
|
|
3309
|
+
hindsight: process.env.HINDSIGHT_API_KEY,
|
|
3310
|
+
letta: process.env.LETTA_API_KEY,
|
|
3311
|
+
memgpt: process.env.LETTA_API_KEY,
|
|
3312
|
+
};
|
|
3313
|
+
const resolvedSourceKey = opts.sourceKey || envKeyMap[provider];
|
|
3314
|
+
if (!resolvedSourceKey) {
|
|
3315
|
+
throw new Error(
|
|
3316
|
+
`Missing source API key for "${provider}". Pass --source-key <key> or set ${provider.toUpperCase()}_API_KEY.`
|
|
3317
|
+
);
|
|
3318
|
+
}
|
|
3319
|
+
records = await fetchProviderMemories(provider, resolvedSourceKey, opts.sourceUrl, opts.userId || "default");
|
|
3320
|
+
}
|
|
3321
|
+
|
|
3322
|
+
const res = await batchIngestMigratedRecords(opts, records);
|
|
3323
|
+
const payload = {
|
|
3324
|
+
status: opts.isDryRun ? "dry_run" : "migrated",
|
|
3325
|
+
source: opts.importFile ? `file:${opts.importFile}` : provider,
|
|
3326
|
+
user_id: opts.userId || "default",
|
|
3327
|
+
...res,
|
|
3328
|
+
};
|
|
3329
|
+
|
|
3330
|
+
if (opts.jsonOutput) {
|
|
3331
|
+
process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
|
|
3332
|
+
} else {
|
|
3333
|
+
console.log(
|
|
3334
|
+
`[/ ml] ${opts.isDryRun ? "[DRY RUN] Discovered" : "Migrated"} ${res.ingested_count || res.total_discovered} memories from ${payload.source} into / ml (user_id=${payload.user_id}).`
|
|
3335
|
+
);
|
|
3336
|
+
}
|
|
3337
|
+
}
|
|
3338
|
+
|
|
3339
|
+
function discoverLocalAgentMemoryFiles(workspaceRoot) {
|
|
3340
|
+
const home = os.homedir();
|
|
3341
|
+
const cwd = workspaceRoot ? path.resolve(workspaceRoot) : process.cwd();
|
|
3342
|
+
const candidates = [];
|
|
3343
|
+
|
|
3344
|
+
function addIfFile(p, kind) {
|
|
3345
|
+
try {
|
|
3346
|
+
if (fs.existsSync(p) && fs.statSync(p).isFile()) {
|
|
3347
|
+
candidates.push({ path: p, kind });
|
|
3348
|
+
}
|
|
3349
|
+
} catch {
|
|
3350
|
+
// ignore
|
|
3351
|
+
}
|
|
3352
|
+
}
|
|
3353
|
+
|
|
3354
|
+
// 1. Claude Code global & per-project auto-memory (~/.claude/CLAUDE.md, ~/.claude/projects/*/memory/*.md)
|
|
3355
|
+
addIfFile(path.join(home, ".claude", "CLAUDE.md"), "claude_global");
|
|
3356
|
+
const claudeProjectsDir = path.join(home, ".claude", "projects");
|
|
3357
|
+
if (fs.existsSync(claudeProjectsDir)) {
|
|
3358
|
+
try {
|
|
3359
|
+
for (const projEntry of fs.readdirSync(claudeProjectsDir, { withFileTypes: true })) {
|
|
3360
|
+
if (!projEntry.isDirectory()) continue;
|
|
3361
|
+
const projMemoryDir = path.join(claudeProjectsDir, projEntry.name, "memory");
|
|
3362
|
+
if (fs.existsSync(projMemoryDir)) {
|
|
3363
|
+
for (const memFile of fs.readdirSync(projMemoryDir, { withFileTypes: true })) {
|
|
3364
|
+
if (memFile.isFile() && memFile.name.endsWith(".md") && memFile.name.toLowerCase() !== "memory.md") {
|
|
3365
|
+
addIfFile(path.join(projMemoryDir, memFile.name), `claude_project:${projEntry.name}`);
|
|
3366
|
+
} else if (memFile.isFile() && memFile.name.toLowerCase() === "memory.md") {
|
|
3367
|
+
addIfFile(path.join(projMemoryDir, memFile.name), `claude_project_index:${projEntry.name}`);
|
|
3368
|
+
}
|
|
3369
|
+
}
|
|
3370
|
+
}
|
|
3371
|
+
}
|
|
3372
|
+
} catch {
|
|
3373
|
+
// ignore
|
|
3374
|
+
}
|
|
3375
|
+
}
|
|
3376
|
+
|
|
3377
|
+
// 2. Workspace-level agent rule & memory files
|
|
3378
|
+
for (const rel of [
|
|
3379
|
+
"CLAUDE.md",
|
|
3380
|
+
"AGENTS.md",
|
|
3381
|
+
".cursorrules",
|
|
3382
|
+
".windsurfrules",
|
|
3383
|
+
".clinerules",
|
|
3384
|
+
".github/copilot-instructions.md",
|
|
3385
|
+
]) {
|
|
3386
|
+
addIfFile(path.join(cwd, rel), "workspace_rules");
|
|
3387
|
+
}
|
|
3388
|
+
|
|
3389
|
+
// 3. Cursor .cursor/rules/*.{mdc,md}
|
|
3390
|
+
const cursorRulesDir = path.join(cwd, ".cursor", "rules");
|
|
3391
|
+
if (fs.existsSync(cursorRulesDir)) {
|
|
3392
|
+
try {
|
|
3393
|
+
for (const e of fs.readdirSync(cursorRulesDir, { withFileTypes: true })) {
|
|
3394
|
+
if (e.isFile() && (e.name.endsWith(".mdc") || e.name.endsWith(".md"))) {
|
|
3395
|
+
addIfFile(path.join(cursorRulesDir, e.name), "cursor_rule");
|
|
3396
|
+
}
|
|
3397
|
+
}
|
|
3398
|
+
} catch {
|
|
3399
|
+
// ignore
|
|
3400
|
+
}
|
|
3401
|
+
}
|
|
3402
|
+
|
|
3403
|
+
// 4. Local OpenMemory / Mem0 JSON dumps (~/.openmemory, ~/.mem0)
|
|
3404
|
+
for (const localDir of [path.join(home, ".openmemory"), path.join(home, ".mem0")]) {
|
|
3405
|
+
if (fs.existsSync(localDir)) {
|
|
3406
|
+
try {
|
|
3407
|
+
for (const e of fs.readdirSync(localDir, { withFileTypes: true })) {
|
|
3408
|
+
if (e.isFile() && (e.name.endsWith(".json") || e.name.endsWith(".jsonl"))) {
|
|
3409
|
+
addIfFile(path.join(localDir, e.name), "local_memory_dump");
|
|
3410
|
+
}
|
|
3411
|
+
}
|
|
3412
|
+
} catch {
|
|
3413
|
+
// ignore
|
|
3414
|
+
}
|
|
3415
|
+
}
|
|
3416
|
+
}
|
|
3417
|
+
|
|
3418
|
+
return candidates;
|
|
3419
|
+
}
|
|
3420
|
+
|
|
3421
|
+
async function runImportLocalCommand(opts) {
|
|
3422
|
+
const workspaceDir = opts.repo ? path.resolve(opts.repo) : process.cwd();
|
|
3423
|
+
const projectId = path.basename(workspaceDir.replace(/[\\/]+$/, "")) || "default";
|
|
3424
|
+
const discoveredFiles = discoverLocalAgentMemoryFiles(workspaceDir);
|
|
3425
|
+
const allRecords = [];
|
|
3426
|
+
|
|
3427
|
+
for (const item of discoveredFiles) {
|
|
3428
|
+
try {
|
|
3429
|
+
const raw = fs.readFileSync(item.path, "utf8");
|
|
3430
|
+
const parsedRecords = parseMemoriesFromFileContent(raw, item.path, opts.userId || "default").map((r) => ({
|
|
3431
|
+
...r,
|
|
3432
|
+
id: makeMigrationDocId("local", r.text, opts.userId || "default"),
|
|
3433
|
+
metadata: {
|
|
3434
|
+
...(r.metadata || {}),
|
|
3435
|
+
source_kind: item.kind,
|
|
3436
|
+
source_file: path.basename(item.path),
|
|
3437
|
+
project_id: projectId,
|
|
3438
|
+
},
|
|
3439
|
+
}));
|
|
3440
|
+
allRecords.push(...parsedRecords);
|
|
3441
|
+
} catch {
|
|
3442
|
+
// ignore unreadable file
|
|
3443
|
+
}
|
|
3444
|
+
}
|
|
3445
|
+
|
|
3446
|
+
const res = await batchIngestMigratedRecords(opts, allRecords);
|
|
3447
|
+
const payload = {
|
|
3448
|
+
status: opts.isDryRun ? "dry_run" : "imported_local",
|
|
3449
|
+
scanned_files: discoveredFiles.map((f) => f.path),
|
|
3450
|
+
project_id: projectId,
|
|
3451
|
+
user_id: opts.userId || "default",
|
|
3452
|
+
...res,
|
|
3453
|
+
};
|
|
3454
|
+
|
|
3455
|
+
if (opts.jsonOutput) {
|
|
3456
|
+
process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
|
|
3457
|
+
} else {
|
|
3458
|
+
console.log(
|
|
3459
|
+
`[/ ml] ${opts.isDryRun ? "[DRY RUN] Discovered" : "Imported"} ${res.ingested_count || res.total_discovered} local coding-agent memories across ${discoveredFiles.length} file(s) into / ml (project=${projectId}, user_id=${payload.user_id}).`
|
|
3460
|
+
);
|
|
3461
|
+
}
|
|
3462
|
+
}
|
|
3463
|
+
|
|
3464
|
+
if (parsed.isMigrate) {
|
|
3465
|
+
runMigrateCommand(parsed)
|
|
3466
|
+
.then(() => process.exit(0))
|
|
3467
|
+
.catch((err) => {
|
|
3468
|
+
console.error(`Error: ${err.message || err}`);
|
|
3469
|
+
process.exit(1);
|
|
3470
|
+
});
|
|
3471
|
+
return;
|
|
3472
|
+
}
|
|
3473
|
+
|
|
3474
|
+
if (parsed.isImportLocal) {
|
|
3475
|
+
runImportLocalCommand(parsed)
|
|
3476
|
+
.then(() => process.exit(0))
|
|
3477
|
+
.catch((err) => {
|
|
3478
|
+
console.error(`Error: ${err.message || err}`);
|
|
3479
|
+
process.exit(1);
|
|
3480
|
+
});
|
|
3481
|
+
return;
|
|
3482
|
+
}
|
|
3483
|
+
|
|
2640
3484
|
if (parsed.isInstallHooks) {
|
|
2641
3485
|
installAmbientHooks(parsed.apiKey);
|
|
2642
3486
|
process.exit(0);
|