nexusmem 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +3 -2
- package/dist/cli/index.js +606 -548
- package/dist/cli/index.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -764,46 +764,6 @@ import { mkdirSync } from "fs";
|
|
|
764
764
|
import { dirname as dirname3 } from "path";
|
|
765
765
|
import * as sqliteVec from "sqlite-vec";
|
|
766
766
|
|
|
767
|
-
// src/core/types.ts
|
|
768
|
-
function defaultProvenanceForKind(kind) {
|
|
769
|
-
switch (kind) {
|
|
770
|
-
case "git_commit":
|
|
771
|
-
case "code_diff":
|
|
772
|
-
case "shell_command":
|
|
773
|
-
return "observed";
|
|
774
|
-
case "conversation_turn":
|
|
775
|
-
case "session_summary":
|
|
776
|
-
case "doc_section":
|
|
777
|
-
case "note":
|
|
778
|
-
return "inferred";
|
|
779
|
-
}
|
|
780
|
-
}
|
|
781
|
-
|
|
782
|
-
// src/core/ids.ts
|
|
783
|
-
import { createHash } from "crypto";
|
|
784
|
-
var KEY_SEP = "\0";
|
|
785
|
-
function sha256Hex(input) {
|
|
786
|
-
return createHash("sha256").update(input, "utf8").digest("hex");
|
|
787
|
-
}
|
|
788
|
-
function makeNodeId(projectId, kind, naturalKey) {
|
|
789
|
-
return sha256Hex([projectId, kind, naturalKey].join(KEY_SEP)).slice(0, 24);
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
// src/store/fts.ts
|
|
793
|
-
var FTS_SYNTAX = /["'()*:^{}[\]-]/g;
|
|
794
|
-
var LOW_SIGNAL_TOKENS = /* @__PURE__ */ new Set(["id"]);
|
|
795
|
-
function significantTokens(input) {
|
|
796
|
-
const tokens = input.replace(FTS_SYNTAX, " ").split(/\s+/).map((t) => t.trim()).filter((t) => t.length > 0);
|
|
797
|
-
if (tokens.length === 0) return [];
|
|
798
|
-
const signal = tokens.filter((t) => !LOW_SIGNAL_TOKENS.has(t.toLowerCase()));
|
|
799
|
-
return signal.length > 0 ? signal : tokens;
|
|
800
|
-
}
|
|
801
|
-
function toMatchQuery(input) {
|
|
802
|
-
const kept = significantTokens(input);
|
|
803
|
-
if (kept.length === 0) return null;
|
|
804
|
-
return kept.map((t) => `"${t}"*`).join(" OR ");
|
|
805
|
-
}
|
|
806
|
-
|
|
807
767
|
// src/store/schema.ts
|
|
808
768
|
var V1 = `
|
|
809
769
|
CREATE TABLE meta (
|
|
@@ -1008,7 +968,227 @@ function migrate(db) {
|
|
|
1008
968
|
return { from, to: currentSchemaVersion(db) };
|
|
1009
969
|
}
|
|
1010
970
|
|
|
1011
|
-
// src/store/
|
|
971
|
+
// src/store/projects.ts
|
|
972
|
+
function upsertProject(db, project) {
|
|
973
|
+
db.prepare(
|
|
974
|
+
`INSERT INTO projects (id, root, origin_url, created_at)
|
|
975
|
+
VALUES (@id, @root, @originUrl, @now)
|
|
976
|
+
ON CONFLICT(id) DO UPDATE SET root = excluded.root, origin_url = excluded.origin_url`
|
|
977
|
+
).run({ ...project, now: Date.now() });
|
|
978
|
+
}
|
|
979
|
+
function markSynced(db, projectId) {
|
|
980
|
+
db.prepare("UPDATE projects SET last_synced_at = ? WHERE id = ?").run(Date.now(), projectId);
|
|
981
|
+
}
|
|
982
|
+
function listOtherProjectIds(db, currentProjectId) {
|
|
983
|
+
return db.prepare("SELECT id FROM projects WHERE id != ?").all(currentProjectId).map(
|
|
984
|
+
(r) => r.id
|
|
985
|
+
);
|
|
986
|
+
}
|
|
987
|
+
function countProjectNodes(db, projectIds) {
|
|
988
|
+
if (projectIds.length === 0) return 0;
|
|
989
|
+
const placeholders = projectIds.map(() => "?").join(", ");
|
|
990
|
+
const row = db.prepare(`SELECT COUNT(*) AS n FROM nodes WHERE project_id IN (${placeholders})`).get(...projectIds);
|
|
991
|
+
return row.n;
|
|
992
|
+
}
|
|
993
|
+
function getSyncCursor(db, projectId, source) {
|
|
994
|
+
const row = db.prepare("SELECT cursor FROM sync_state WHERE project_id = ? AND source = ?").get(projectId, source);
|
|
995
|
+
return row?.cursor ?? null;
|
|
996
|
+
}
|
|
997
|
+
function setSyncCursor(db, projectId, source, cursor) {
|
|
998
|
+
db.prepare(
|
|
999
|
+
`INSERT INTO sync_state (project_id, source, cursor, last_run_at)
|
|
1000
|
+
VALUES (?, ?, ?, ?)
|
|
1001
|
+
ON CONFLICT(project_id, source) DO UPDATE SET cursor = excluded.cursor, last_run_at = excluded.last_run_at`
|
|
1002
|
+
).run(projectId, source, cursor, Date.now());
|
|
1003
|
+
}
|
|
1004
|
+
function listSyncState(db, projectId) {
|
|
1005
|
+
return db.prepare("SELECT source, cursor, last_run_at AS lastRunAt FROM sync_state WHERE project_id = ? ORDER BY last_run_at DESC").all(projectId);
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
// src/core/types.ts
|
|
1009
|
+
function defaultProvenanceForKind(kind) {
|
|
1010
|
+
switch (kind) {
|
|
1011
|
+
case "git_commit":
|
|
1012
|
+
case "code_diff":
|
|
1013
|
+
case "shell_command":
|
|
1014
|
+
return "observed";
|
|
1015
|
+
case "conversation_turn":
|
|
1016
|
+
case "session_summary":
|
|
1017
|
+
case "doc_section":
|
|
1018
|
+
case "note":
|
|
1019
|
+
return "inferred";
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
// src/store/nodes.ts
|
|
1024
|
+
function epochOf(ts) {
|
|
1025
|
+
const parsed = Date.parse(ts);
|
|
1026
|
+
return Number.isNaN(parsed) ? Date.now() : parsed;
|
|
1027
|
+
}
|
|
1028
|
+
function upsertNodes(db, nodes) {
|
|
1029
|
+
const exists = db.prepare("SELECT body, signal, title FROM nodes WHERE id = ?");
|
|
1030
|
+
const dropStaleEmbedding = db.prepare("DELETE FROM nodes_vec WHERE rowid = (SELECT rowid FROM nodes WHERE id = ?)");
|
|
1031
|
+
const insertNode = db.prepare(
|
|
1032
|
+
`INSERT INTO nodes (id, kind, project_id, ts, ts_epoch, source, title, body, signal, meta, provenance, supersedes, created_at)
|
|
1033
|
+
VALUES (@id, @kind, @projectId, @ts, @tsEpoch, @source, @title, @body, @signal, @meta, @provenance, @supersedes, @now)
|
|
1034
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1035
|
+
ts = excluded.ts, ts_epoch = excluded.ts_epoch, source = excluded.source,
|
|
1036
|
+
title = excluded.title, body = excluded.body, signal = excluded.signal, meta = excluded.meta,
|
|
1037
|
+
provenance = excluded.provenance`
|
|
1038
|
+
);
|
|
1039
|
+
const clearFiles = db.prepare("DELETE FROM node_files WHERE node_id = ?");
|
|
1040
|
+
const insertFile = db.prepare(
|
|
1041
|
+
`INSERT INTO node_files (node_id, path, previous_path, insertions, deletions, is_binary)
|
|
1042
|
+
VALUES (@nodeId, @path, @previousPath, @insertions, @deletions, @isBinary)
|
|
1043
|
+
ON CONFLICT(node_id, path) DO UPDATE SET
|
|
1044
|
+
previous_path = excluded.previous_path, insertions = excluded.insertions,
|
|
1045
|
+
deletions = excluded.deletions, is_binary = excluded.is_binary`
|
|
1046
|
+
);
|
|
1047
|
+
const stats2 = { inserted: 0, updated: 0, unchanged: 0, denied: 0 };
|
|
1048
|
+
const denyEntriesByProject = /* @__PURE__ */ new Map();
|
|
1049
|
+
const run = db.transaction((batch) => {
|
|
1050
|
+
const now = Date.now();
|
|
1051
|
+
for (const node of batch) {
|
|
1052
|
+
let denyEntries = denyEntriesByProject.get(node.projectId);
|
|
1053
|
+
if (!denyEntries) {
|
|
1054
|
+
denyEntries = listDenyListEntries(db, node.projectId);
|
|
1055
|
+
denyEntriesByProject.set(node.projectId, denyEntries);
|
|
1056
|
+
}
|
|
1057
|
+
if (firstMatchingEntry(denyEntries, node)) {
|
|
1058
|
+
stats2.denied += 1;
|
|
1059
|
+
continue;
|
|
1060
|
+
}
|
|
1061
|
+
const prior = exists.get(node.id);
|
|
1062
|
+
if (prior) {
|
|
1063
|
+
if (prior.body === node.body && prior.signal === node.signal && prior.title === node.title) {
|
|
1064
|
+
stats2.unchanged += 1;
|
|
1065
|
+
continue;
|
|
1066
|
+
}
|
|
1067
|
+
stats2.updated += 1;
|
|
1068
|
+
dropStaleEmbedding.run(node.id);
|
|
1069
|
+
} else {
|
|
1070
|
+
stats2.inserted += 1;
|
|
1071
|
+
}
|
|
1072
|
+
insertNode.run({
|
|
1073
|
+
id: node.id,
|
|
1074
|
+
kind: node.kind,
|
|
1075
|
+
projectId: node.projectId,
|
|
1076
|
+
ts: node.ts,
|
|
1077
|
+
tsEpoch: epochOf(node.ts),
|
|
1078
|
+
source: node.source,
|
|
1079
|
+
title: node.title,
|
|
1080
|
+
body: node.body,
|
|
1081
|
+
signal: node.signal,
|
|
1082
|
+
meta: JSON.stringify(node.meta),
|
|
1083
|
+
provenance: node.provenance ?? defaultProvenanceForKind(node.kind),
|
|
1084
|
+
supersedes: node.supersedes ?? null,
|
|
1085
|
+
now
|
|
1086
|
+
});
|
|
1087
|
+
clearFiles.run(node.id);
|
|
1088
|
+
for (const file of node.files) {
|
|
1089
|
+
insertFile.run({
|
|
1090
|
+
nodeId: node.id,
|
|
1091
|
+
path: file.path,
|
|
1092
|
+
previousPath: file.previousPath ?? null,
|
|
1093
|
+
insertions: file.insertions,
|
|
1094
|
+
deletions: file.deletions,
|
|
1095
|
+
isBinary: file.binary ? 1 : 0
|
|
1096
|
+
});
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
});
|
|
1100
|
+
run(nodes);
|
|
1101
|
+
return stats2;
|
|
1102
|
+
}
|
|
1103
|
+
function getNodeMeta(db, id) {
|
|
1104
|
+
const row = db.prepare("SELECT meta FROM nodes WHERE id = ?").get(id);
|
|
1105
|
+
if (!row) return null;
|
|
1106
|
+
try {
|
|
1107
|
+
return JSON.parse(row.meta);
|
|
1108
|
+
} catch {
|
|
1109
|
+
return null;
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
function clearProject(db, projectId) {
|
|
1113
|
+
db.prepare("DELETE FROM nodes_vec WHERE rowid IN (SELECT rowid FROM nodes WHERE project_id = ?)").run(projectId);
|
|
1114
|
+
const info = db.prepare("DELETE FROM nodes WHERE project_id = ?").run(projectId);
|
|
1115
|
+
db.prepare("DELETE FROM sync_state WHERE project_id = ?").run(projectId);
|
|
1116
|
+
return info.changes;
|
|
1117
|
+
}
|
|
1118
|
+
function getNodesByIds(db, ids) {
|
|
1119
|
+
if (ids.length === 0) return [];
|
|
1120
|
+
return db.prepare(
|
|
1121
|
+
`SELECT id, kind, project_id AS projectId, ts, title, body, signal, provenance
|
|
1122
|
+
FROM nodes WHERE id IN (SELECT value FROM json_each(?))`
|
|
1123
|
+
).all(JSON.stringify(ids));
|
|
1124
|
+
}
|
|
1125
|
+
function listRecentNodes(db, projectId, limit = 20) {
|
|
1126
|
+
return db.prepare(
|
|
1127
|
+
`SELECT id, kind, ts, source, title, signal, provenance
|
|
1128
|
+
FROM nodes
|
|
1129
|
+
WHERE project_id = ?
|
|
1130
|
+
ORDER BY ts_epoch DESC
|
|
1131
|
+
LIMIT ?`
|
|
1132
|
+
).all(projectId, limit);
|
|
1133
|
+
}
|
|
1134
|
+
function countSourceNodes(db, projectId, source) {
|
|
1135
|
+
const row = db.prepare("SELECT COUNT(*) AS count FROM nodes WHERE project_id = ? AND source = ?").get(
|
|
1136
|
+
projectId,
|
|
1137
|
+
source
|
|
1138
|
+
);
|
|
1139
|
+
return row.count;
|
|
1140
|
+
}
|
|
1141
|
+
function pruneSourceNodes(db, projectId, source, keepIds, opts = {}) {
|
|
1142
|
+
const scope = `project_id = @projectId AND source = @source
|
|
1143
|
+
AND id NOT IN (SELECT value FROM json_each(@keepIds))
|
|
1144
|
+
AND id NOT IN (SELECT node_id FROM node_files WHERE path IN (SELECT value FROM json_each(@keepPaths)))`;
|
|
1145
|
+
const params = {
|
|
1146
|
+
projectId,
|
|
1147
|
+
source,
|
|
1148
|
+
keepIds: JSON.stringify(keepIds),
|
|
1149
|
+
keepPaths: JSON.stringify(opts.keepPaths ?? [])
|
|
1150
|
+
};
|
|
1151
|
+
return db.transaction(() => {
|
|
1152
|
+
db.prepare(`DELETE FROM nodes_vec WHERE rowid IN (SELECT rowid FROM nodes WHERE ${scope})`).run(params);
|
|
1153
|
+
return db.prepare(`DELETE FROM nodes WHERE ${scope}`).run(params).changes;
|
|
1154
|
+
})();
|
|
1155
|
+
}
|
|
1156
|
+
function getNodeProjectId(db, id) {
|
|
1157
|
+
const row = db.prepare("SELECT project_id FROM nodes WHERE id = ?").get(id);
|
|
1158
|
+
return row?.project_id ?? null;
|
|
1159
|
+
}
|
|
1160
|
+
function getSupersededIds(db, projectId) {
|
|
1161
|
+
const rows = db.prepare("SELECT DISTINCT supersedes AS id FROM nodes WHERE project_id = ? AND supersedes IS NOT NULL").all(projectId);
|
|
1162
|
+
return new Set(rows.map((r) => r.id));
|
|
1163
|
+
}
|
|
1164
|
+
function setSupersedes(db, newNodeId, staleNodeId) {
|
|
1165
|
+
db.prepare("UPDATE nodes SET supersedes = ? WHERE id = ?").run(staleNodeId, newNodeId);
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
// src/store/links.ts
|
|
1169
|
+
function linkNodes(db, fromNodeId, toNodeId, relation) {
|
|
1170
|
+
db.prepare("INSERT OR IGNORE INTO node_links (from_node_id, to_node_id, relation, created_at) VALUES (?, ?, ?, ?)").run(
|
|
1171
|
+
fromNodeId,
|
|
1172
|
+
toNodeId,
|
|
1173
|
+
relation,
|
|
1174
|
+
Date.now()
|
|
1175
|
+
);
|
|
1176
|
+
}
|
|
1177
|
+
function getLinkedNodeIds(db, fromNodeId, relation) {
|
|
1178
|
+
return db.prepare("SELECT to_node_id FROM node_links WHERE from_node_id = ? AND relation = ? ORDER BY created_at DESC").all(fromNodeId, relation).map((row) => row.to_node_id);
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
// src/core/ids.ts
|
|
1182
|
+
import { createHash } from "crypto";
|
|
1183
|
+
var KEY_SEP = "\0";
|
|
1184
|
+
function sha256Hex(input) {
|
|
1185
|
+
return createHash("sha256").update(input, "utf8").digest("hex");
|
|
1186
|
+
}
|
|
1187
|
+
function makeNodeId(projectId, kind, naturalKey) {
|
|
1188
|
+
return sha256Hex([projectId, kind, naturalKey].join(KEY_SEP)).slice(0, 24);
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
// src/store/forget.ts
|
|
1012
1192
|
function parseMeta(raw) {
|
|
1013
1193
|
try {
|
|
1014
1194
|
return JSON.parse(raw);
|
|
@@ -1016,10 +1196,228 @@ function parseMeta(raw) {
|
|
|
1016
1196
|
return {};
|
|
1017
1197
|
}
|
|
1018
1198
|
}
|
|
1019
|
-
function
|
|
1020
|
-
|
|
1021
|
-
|
|
1199
|
+
function previewForget(db, projectId, otherProjectIds, input) {
|
|
1200
|
+
validatePattern(input);
|
|
1201
|
+
const probeEntry = { id: -1, projectId, createdAt: 0, ...input };
|
|
1202
|
+
const select = db.prepare("SELECT project_id, source, title, body, meta FROM nodes WHERE project_id = ?");
|
|
1203
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1204
|
+
for (const scopeId of [projectId, ...otherProjectIds]) {
|
|
1205
|
+
const rows = select.all(scopeId);
|
|
1206
|
+
for (const row of rows) {
|
|
1207
|
+
if (!firstMatchingEntry([probeEntry], { title: row.title, body: row.body, meta: parseMeta(row.meta) })) continue;
|
|
1208
|
+
const key = `${row.project_id} ${row.source}`;
|
|
1209
|
+
const existing = counts.get(key);
|
|
1210
|
+
if (existing) existing.count += 1;
|
|
1211
|
+
else counts.set(key, { projectId: row.project_id, source: row.source, count: 1 });
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
return [...counts.values()];
|
|
1215
|
+
}
|
|
1216
|
+
function forget(db, projectId, otherProjectIds, input) {
|
|
1217
|
+
return db.transaction(() => {
|
|
1218
|
+
const entry = insertDenyListEntry(db, { ...input, projectId });
|
|
1219
|
+
const startedAt = Date.now();
|
|
1220
|
+
const auditId = Number(
|
|
1221
|
+
db.prepare(
|
|
1222
|
+
`INSERT INTO mutation_audit (action, project_id, detail, affected_count, succeeded, error, started_at, finished_at)
|
|
1223
|
+
VALUES ('forget', @projectId, @detail, 0, 1, NULL, @startedAt, @startedAt)`
|
|
1224
|
+
).run({
|
|
1225
|
+
projectId,
|
|
1226
|
+
detail: JSON.stringify({
|
|
1227
|
+
pattern: input.pattern,
|
|
1228
|
+
matchType: input.matchType,
|
|
1229
|
+
ignoreCase: input.ignoreCase,
|
|
1230
|
+
reason: input.reason,
|
|
1231
|
+
scopeProjectIds: [projectId, ...otherProjectIds]
|
|
1232
|
+
}),
|
|
1233
|
+
startedAt
|
|
1234
|
+
}).lastInsertRowid
|
|
1235
|
+
);
|
|
1236
|
+
const select = db.prepare(
|
|
1237
|
+
"SELECT id, kind, project_id, source, ts, signal, title, body, meta FROM nodes WHERE project_id = ?"
|
|
1238
|
+
);
|
|
1239
|
+
const dropEmbedding = db.prepare("DELETE FROM nodes_vec WHERE rowid = (SELECT rowid FROM nodes WHERE id = ?)");
|
|
1240
|
+
const deleteNode = db.prepare("DELETE FROM nodes WHERE id = ?");
|
|
1241
|
+
const insertTombstone = db.prepare(
|
|
1242
|
+
`INSERT INTO tombstones
|
|
1243
|
+
(node_id, project_id, kind, source, ts, signal, body_sha256, title_sha256, body_length, deny_list_id, mutation_audit_id, removed_at)
|
|
1244
|
+
VALUES (@nodeId, @projectId, @kind, @source, @ts, @signal, @bodySha256, @titleSha256, @bodyLength, @denyListId, @mutationAuditId, @removedAt)`
|
|
1245
|
+
);
|
|
1246
|
+
let removed = 0;
|
|
1247
|
+
for (const scopeId of [projectId, ...otherProjectIds]) {
|
|
1248
|
+
const rows = select.all(scopeId);
|
|
1249
|
+
for (const row of rows) {
|
|
1250
|
+
if (!firstMatchingEntry([entry], { title: row.title, body: row.body, meta: parseMeta(row.meta) })) continue;
|
|
1251
|
+
dropEmbedding.run(row.id);
|
|
1252
|
+
insertTombstone.run({
|
|
1253
|
+
nodeId: row.id,
|
|
1254
|
+
projectId: row.project_id,
|
|
1255
|
+
kind: row.kind,
|
|
1256
|
+
source: row.source,
|
|
1257
|
+
ts: row.ts,
|
|
1258
|
+
signal: row.signal,
|
|
1259
|
+
bodySha256: sha256Hex(row.body),
|
|
1260
|
+
titleSha256: sha256Hex(row.title),
|
|
1261
|
+
bodyLength: row.body.length,
|
|
1262
|
+
denyListId: entry.id,
|
|
1263
|
+
mutationAuditId: auditId,
|
|
1264
|
+
removedAt: Date.now()
|
|
1265
|
+
});
|
|
1266
|
+
deleteNode.run(row.id);
|
|
1267
|
+
removed += 1;
|
|
1268
|
+
}
|
|
1269
|
+
}
|
|
1270
|
+
db.prepare("UPDATE mutation_audit SET affected_count = ?, finished_at = ? WHERE id = ?").run(removed, Date.now(), auditId);
|
|
1271
|
+
return { removed, entryId: entry.id, auditId };
|
|
1272
|
+
})();
|
|
1273
|
+
}
|
|
1274
|
+
function listDenyList(db, projectId) {
|
|
1275
|
+
return listDenyListEntries(db, projectId);
|
|
1276
|
+
}
|
|
1277
|
+
function previewImportDenyList(db, projectId, otherProjectIds, entries) {
|
|
1278
|
+
return entries.map((input) => {
|
|
1279
|
+
validatePattern(input);
|
|
1280
|
+
if (denyListEntryExists(db, projectId, input)) {
|
|
1281
|
+
return { matchType: input.matchType, pattern: input.pattern, alreadyPresent: true, wouldRemove: 0 };
|
|
1282
|
+
}
|
|
1283
|
+
const preview = previewForget(db, projectId, otherProjectIds, input);
|
|
1284
|
+
return {
|
|
1285
|
+
matchType: input.matchType,
|
|
1286
|
+
pattern: input.pattern,
|
|
1287
|
+
alreadyPresent: false,
|
|
1288
|
+
wouldRemove: preview.reduce((sum, p) => sum + p.count, 0)
|
|
1289
|
+
};
|
|
1290
|
+
});
|
|
1291
|
+
}
|
|
1292
|
+
function importDenyList(db, projectId, otherProjectIds, entries) {
|
|
1293
|
+
let imported = 0;
|
|
1294
|
+
let skipped = 0;
|
|
1295
|
+
let removedNodes = 0;
|
|
1296
|
+
for (const input of entries) {
|
|
1297
|
+
validatePattern(input);
|
|
1298
|
+
if (denyListEntryExists(db, projectId, input)) {
|
|
1299
|
+
skipped += 1;
|
|
1300
|
+
continue;
|
|
1301
|
+
}
|
|
1302
|
+
const result = forget(db, projectId, otherProjectIds, input);
|
|
1303
|
+
imported += 1;
|
|
1304
|
+
removedNodes += result.removed;
|
|
1305
|
+
}
|
|
1306
|
+
return { imported, skipped, removedNodes };
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
// src/store/file-edges.ts
|
|
1310
|
+
function replaceFileEdges(db, projectId, edges) {
|
|
1311
|
+
db.transaction(() => {
|
|
1312
|
+
db.prepare("DELETE FROM file_edges WHERE project_id = ?").run(projectId);
|
|
1313
|
+
const insert = db.prepare("INSERT OR IGNORE INTO file_edges (project_id, from_path, to_path, kind) VALUES (?, ?, ?, ?)");
|
|
1314
|
+
for (const edge of edges) insert.run(projectId, edge.fromPath, edge.toPath, edge.kind);
|
|
1315
|
+
})();
|
|
1316
|
+
}
|
|
1317
|
+
function fileEdgeStats(db, projectId) {
|
|
1318
|
+
const edges = db.prepare("SELECT COUNT(*) AS c FROM file_edges WHERE project_id = ?").get(projectId).c;
|
|
1319
|
+
const files = db.prepare("SELECT COUNT(DISTINCT from_path) AS c FROM file_edges WHERE project_id = ?").get(projectId).c;
|
|
1320
|
+
return { edges, files };
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
// src/store/embeddings.ts
|
|
1324
|
+
function findNodesNeedingEmbedding(db, projectId, limit = 200, afterRowid = 0) {
|
|
1325
|
+
return db.prepare(
|
|
1326
|
+
`SELECT n.rowid AS rowid, n.id AS id, n.title AS title, n.body AS body
|
|
1327
|
+
FROM nodes n
|
|
1328
|
+
LEFT JOIN nodes_vec v ON v.rowid = n.rowid
|
|
1329
|
+
WHERE n.project_id = ? AND v.rowid IS NULL AND n.rowid > ?
|
|
1330
|
+
ORDER BY n.rowid
|
|
1331
|
+
LIMIT ?`
|
|
1332
|
+
).all(projectId, afterRowid, limit);
|
|
1333
|
+
}
|
|
1334
|
+
function countNodesNeedingEmbedding(db, projectId) {
|
|
1335
|
+
const row = db.prepare(
|
|
1336
|
+
`SELECT COUNT(*) AS n
|
|
1337
|
+
FROM nodes n
|
|
1338
|
+
LEFT JOIN nodes_vec v ON v.rowid = n.rowid
|
|
1339
|
+
WHERE n.project_id = ? AND v.rowid IS NULL`
|
|
1340
|
+
).get(projectId);
|
|
1341
|
+
return row.n;
|
|
1342
|
+
}
|
|
1343
|
+
function upsertEmbedding(db, rowid, embedding) {
|
|
1344
|
+
db.prepare("INSERT OR REPLACE INTO nodes_vec (rowid, embedding) VALUES (?, ?)").run(BigInt(rowid), embedding);
|
|
1345
|
+
}
|
|
1346
|
+
function dropAllEmbeddings(db) {
|
|
1347
|
+
return db.prepare("DELETE FROM nodes_vec").run().changes;
|
|
1348
|
+
}
|
|
1349
|
+
function vectorSearch(db, projectId, embedding, limit = 20) {
|
|
1350
|
+
const overfetch = Math.max(limit * 8, 50);
|
|
1351
|
+
return db.prepare(
|
|
1352
|
+
`SELECT n.id, n.kind, n.ts, n.title, n.body, n.signal, n.provenance, v.distance AS distance
|
|
1353
|
+
FROM nodes_vec v
|
|
1354
|
+
JOIN nodes n ON n.rowid = v.rowid
|
|
1355
|
+
WHERE v.embedding MATCH ? AND k = ? AND n.project_id = ?
|
|
1356
|
+
ORDER BY v.distance
|
|
1357
|
+
LIMIT ?`
|
|
1358
|
+
).all(embedding, overfetch, projectId, limit);
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
// src/store/fts.ts
|
|
1362
|
+
var FTS_SYNTAX = /["'()*:^{}[\]-]/g;
|
|
1363
|
+
var LOW_SIGNAL_TOKENS = /* @__PURE__ */ new Set(["id"]);
|
|
1364
|
+
function significantTokens(input) {
|
|
1365
|
+
const tokens = input.replace(FTS_SYNTAX, " ").split(/\s+/).map((t) => t.trim()).filter((t) => t.length > 0);
|
|
1366
|
+
if (tokens.length === 0) return [];
|
|
1367
|
+
const signal = tokens.filter((t) => !LOW_SIGNAL_TOKENS.has(t.toLowerCase()));
|
|
1368
|
+
return signal.length > 0 ? signal : tokens;
|
|
1022
1369
|
}
|
|
1370
|
+
function toMatchQuery(input) {
|
|
1371
|
+
const kept = significantTokens(input);
|
|
1372
|
+
if (kept.length === 0) return null;
|
|
1373
|
+
return kept.map((t) => `"${t}"*`).join(" OR ");
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
// src/store/search.ts
|
|
1377
|
+
function search(db, projectId, query, limit = 20) {
|
|
1378
|
+
const match = toMatchQuery(query);
|
|
1379
|
+
if (!match) return [];
|
|
1380
|
+
const rows = db.prepare(
|
|
1381
|
+
`SELECT n.id, n.kind, n.ts, n.title, n.body, n.signal, n.provenance,
|
|
1382
|
+
bm25(nodes_fts, 10.0, 1.0) AS rank
|
|
1383
|
+
FROM nodes_fts
|
|
1384
|
+
JOIN nodes n ON n.rowid = nodes_fts.rowid
|
|
1385
|
+
WHERE nodes_fts MATCH ? AND n.project_id = ?
|
|
1386
|
+
ORDER BY rank
|
|
1387
|
+
LIMIT ?`
|
|
1388
|
+
).all(match, projectId, limit);
|
|
1389
|
+
return rows;
|
|
1390
|
+
}
|
|
1391
|
+
function stats(db, projectId) {
|
|
1392
|
+
const kinds = db.prepare("SELECT kind, COUNT(*) AS n FROM nodes WHERE project_id = ? GROUP BY kind").all(projectId);
|
|
1393
|
+
const range = db.prepare("SELECT MIN(ts) AS oldest, MAX(ts) AS newest FROM nodes WHERE project_id = ?").get(projectId);
|
|
1394
|
+
const files = db.prepare(
|
|
1395
|
+
`SELECT COUNT(DISTINCT f.path) AS n
|
|
1396
|
+
FROM node_files f JOIN nodes n ON n.id = f.node_id
|
|
1397
|
+
WHERE n.project_id = ?`
|
|
1398
|
+
).get(projectId);
|
|
1399
|
+
return {
|
|
1400
|
+
total: kinds.reduce((sum, k) => sum + k.n, 0),
|
|
1401
|
+
byKind: Object.fromEntries(kinds.map((k) => [k.kind, k.n])),
|
|
1402
|
+
oldest: range.oldest,
|
|
1403
|
+
newest: range.newest,
|
|
1404
|
+
distinctFiles: files.n
|
|
1405
|
+
};
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
// src/store/meta.ts
|
|
1409
|
+
function getMeta(db, key) {
|
|
1410
|
+
const row = db.prepare("SELECT value FROM meta WHERE key = ?").get(key);
|
|
1411
|
+
return row?.value ?? null;
|
|
1412
|
+
}
|
|
1413
|
+
function setMeta(db, key, value) {
|
|
1414
|
+
db.prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(
|
|
1415
|
+
key,
|
|
1416
|
+
value
|
|
1417
|
+
);
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
// src/store/store.ts
|
|
1023
1421
|
var MemoryStore = class _MemoryStore {
|
|
1024
1422
|
constructor(db) {
|
|
1025
1423
|
this.db = db;
|
|
@@ -1039,35 +1437,17 @@ var MemoryStore = class _MemoryStore {
|
|
|
1039
1437
|
this.db.close();
|
|
1040
1438
|
}
|
|
1041
1439
|
upsertProject(project) {
|
|
1042
|
-
this.db
|
|
1043
|
-
`INSERT INTO projects (id, root, origin_url, created_at)
|
|
1044
|
-
VALUES (@id, @root, @originUrl, @now)
|
|
1045
|
-
ON CONFLICT(id) DO UPDATE SET root = excluded.root, origin_url = excluded.origin_url`
|
|
1046
|
-
).run({ ...project, now: Date.now() });
|
|
1440
|
+
upsertProject(this.db, project);
|
|
1047
1441
|
}
|
|
1048
1442
|
markSynced(projectId) {
|
|
1049
|
-
this.db
|
|
1443
|
+
markSynced(this.db, projectId);
|
|
1050
1444
|
}
|
|
1051
|
-
/**
|
|
1052
|
-
* Every other project id ever recorded in THIS repo's own database.
|
|
1053
|
-
*
|
|
1054
|
-
* A repo's `.nexusmem/memory.db` is never shared with another repo (each
|
|
1055
|
-
* gets its own, gitignored), so any id here besides `currentProjectId` is
|
|
1056
|
-
* evidence of a prior identity for this same repo -- typically its git
|
|
1057
|
-
* remote URL changed since the last sync. See `reconcileProjectId` in
|
|
1058
|
-
* `store/reconcile.ts`.
|
|
1059
|
-
*/
|
|
1060
1445
|
listOtherProjectIds(currentProjectId) {
|
|
1061
|
-
return this.db
|
|
1062
|
-
(r) => r.id
|
|
1063
|
-
);
|
|
1446
|
+
return listOtherProjectIds(this.db, currentProjectId);
|
|
1064
1447
|
}
|
|
1065
1448
|
/** Total nodes held under the given project identities. */
|
|
1066
1449
|
countProjectNodes(projectIds) {
|
|
1067
|
-
|
|
1068
|
-
const placeholders = projectIds.map(() => "?").join(", ");
|
|
1069
|
-
const row = this.db.prepare(`SELECT COUNT(*) AS n FROM nodes WHERE project_id IN (${placeholders})`).get(...projectIds);
|
|
1070
|
-
return row.n;
|
|
1450
|
+
return countProjectNodes(this.db, projectIds);
|
|
1071
1451
|
}
|
|
1072
1452
|
/**
|
|
1073
1453
|
* Write a batch of nodes in one transaction.
|
|
@@ -1077,81 +1457,7 @@ var MemoryStore = class _MemoryStore {
|
|
|
1077
1457
|
* happens when scoring or body composition is improved between releases).
|
|
1078
1458
|
*/
|
|
1079
1459
|
upsertNodes(nodes) {
|
|
1080
|
-
|
|
1081
|
-
const dropStaleEmbedding = this.db.prepare(
|
|
1082
|
-
"DELETE FROM nodes_vec WHERE rowid = (SELECT rowid FROM nodes WHERE id = ?)"
|
|
1083
|
-
);
|
|
1084
|
-
const insertNode = this.db.prepare(
|
|
1085
|
-
`INSERT INTO nodes (id, kind, project_id, ts, ts_epoch, source, title, body, signal, meta, provenance, supersedes, created_at)
|
|
1086
|
-
VALUES (@id, @kind, @projectId, @ts, @tsEpoch, @source, @title, @body, @signal, @meta, @provenance, @supersedes, @now)
|
|
1087
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
1088
|
-
ts = excluded.ts, ts_epoch = excluded.ts_epoch, source = excluded.source,
|
|
1089
|
-
title = excluded.title, body = excluded.body, signal = excluded.signal, meta = excluded.meta,
|
|
1090
|
-
provenance = excluded.provenance`
|
|
1091
|
-
);
|
|
1092
|
-
const clearFiles = this.db.prepare("DELETE FROM node_files WHERE node_id = ?");
|
|
1093
|
-
const insertFile = this.db.prepare(
|
|
1094
|
-
`INSERT INTO node_files (node_id, path, previous_path, insertions, deletions, is_binary)
|
|
1095
|
-
VALUES (@nodeId, @path, @previousPath, @insertions, @deletions, @isBinary)
|
|
1096
|
-
ON CONFLICT(node_id, path) DO UPDATE SET
|
|
1097
|
-
previous_path = excluded.previous_path, insertions = excluded.insertions,
|
|
1098
|
-
deletions = excluded.deletions, is_binary = excluded.is_binary`
|
|
1099
|
-
);
|
|
1100
|
-
const stats = { inserted: 0, updated: 0, unchanged: 0, denied: 0 };
|
|
1101
|
-
const denyEntriesByProject = /* @__PURE__ */ new Map();
|
|
1102
|
-
const run = this.db.transaction((batch) => {
|
|
1103
|
-
const now = Date.now();
|
|
1104
|
-
for (const node of batch) {
|
|
1105
|
-
let denyEntries = denyEntriesByProject.get(node.projectId);
|
|
1106
|
-
if (!denyEntries) {
|
|
1107
|
-
denyEntries = listDenyListEntries(this.db, node.projectId);
|
|
1108
|
-
denyEntriesByProject.set(node.projectId, denyEntries);
|
|
1109
|
-
}
|
|
1110
|
-
if (firstMatchingEntry(denyEntries, node)) {
|
|
1111
|
-
stats.denied += 1;
|
|
1112
|
-
continue;
|
|
1113
|
-
}
|
|
1114
|
-
const prior = exists.get(node.id);
|
|
1115
|
-
if (prior) {
|
|
1116
|
-
if (prior.body === node.body && prior.signal === node.signal && prior.title === node.title) {
|
|
1117
|
-
stats.unchanged += 1;
|
|
1118
|
-
continue;
|
|
1119
|
-
}
|
|
1120
|
-
stats.updated += 1;
|
|
1121
|
-
dropStaleEmbedding.run(node.id);
|
|
1122
|
-
} else {
|
|
1123
|
-
stats.inserted += 1;
|
|
1124
|
-
}
|
|
1125
|
-
insertNode.run({
|
|
1126
|
-
id: node.id,
|
|
1127
|
-
kind: node.kind,
|
|
1128
|
-
projectId: node.projectId,
|
|
1129
|
-
ts: node.ts,
|
|
1130
|
-
tsEpoch: epochOf(node.ts),
|
|
1131
|
-
source: node.source,
|
|
1132
|
-
title: node.title,
|
|
1133
|
-
body: node.body,
|
|
1134
|
-
signal: node.signal,
|
|
1135
|
-
meta: JSON.stringify(node.meta),
|
|
1136
|
-
provenance: node.provenance ?? defaultProvenanceForKind(node.kind),
|
|
1137
|
-
supersedes: node.supersedes ?? null,
|
|
1138
|
-
now
|
|
1139
|
-
});
|
|
1140
|
-
clearFiles.run(node.id);
|
|
1141
|
-
for (const file of node.files) {
|
|
1142
|
-
insertFile.run({
|
|
1143
|
-
nodeId: node.id,
|
|
1144
|
-
path: file.path,
|
|
1145
|
-
previousPath: file.previousPath ?? null,
|
|
1146
|
-
insertions: file.insertions,
|
|
1147
|
-
deletions: file.deletions,
|
|
1148
|
-
isBinary: file.binary ? 1 : 0
|
|
1149
|
-
});
|
|
1150
|
-
}
|
|
1151
|
-
}
|
|
1152
|
-
});
|
|
1153
|
-
run(nodes);
|
|
1154
|
-
return stats;
|
|
1460
|
+
return upsertNodes(this.db, nodes);
|
|
1155
1461
|
}
|
|
1156
1462
|
/**
|
|
1157
1463
|
* The stored `meta` blob for one node, or null if it has never been
|
|
@@ -1159,53 +1465,28 @@ var MemoryStore = class _MemoryStore {
|
|
|
1159
1465
|
* already done without re-reading the node's whole body.
|
|
1160
1466
|
*/
|
|
1161
1467
|
getNodeMeta(id) {
|
|
1162
|
-
|
|
1163
|
-
if (!row) return null;
|
|
1164
|
-
try {
|
|
1165
|
-
return JSON.parse(row.meta);
|
|
1166
|
-
} catch {
|
|
1167
|
-
return null;
|
|
1168
|
-
}
|
|
1468
|
+
return getNodeMeta(this.db, id);
|
|
1169
1469
|
}
|
|
1170
1470
|
getSyncCursor(projectId, source) {
|
|
1171
|
-
|
|
1172
|
-
return row?.cursor ?? null;
|
|
1471
|
+
return getSyncCursor(this.db, projectId, source);
|
|
1173
1472
|
}
|
|
1174
1473
|
setSyncCursor(projectId, source, cursor) {
|
|
1175
|
-
this.db
|
|
1176
|
-
`INSERT INTO sync_state (project_id, source, cursor, last_run_at)
|
|
1177
|
-
VALUES (?, ?, ?, ?)
|
|
1178
|
-
ON CONFLICT(project_id, source) DO UPDATE SET cursor = excluded.cursor, last_run_at = excluded.last_run_at`
|
|
1179
|
-
).run(projectId, source, cursor, Date.now());
|
|
1474
|
+
setSyncCursor(this.db, projectId, source, cursor);
|
|
1180
1475
|
}
|
|
1181
1476
|
/** Every source that has ever synced for this project, most recently run first. */
|
|
1182
1477
|
listSyncState(projectId) {
|
|
1183
|
-
return this.db
|
|
1478
|
+
return listSyncState(this.db, projectId);
|
|
1184
1479
|
}
|
|
1185
1480
|
/** Drop every node for a project. Used by `sync --rebuild`. */
|
|
1186
1481
|
clearProject(projectId) {
|
|
1187
|
-
this.db
|
|
1188
|
-
const info = this.db.prepare("DELETE FROM nodes WHERE project_id = ?").run(projectId);
|
|
1189
|
-
this.db.prepare("DELETE FROM sync_state WHERE project_id = ?").run(projectId);
|
|
1190
|
-
return info.changes;
|
|
1482
|
+
return clearProject(this.db, projectId);
|
|
1191
1483
|
}
|
|
1192
|
-
/**
|
|
1193
|
-
* Record a directed relationship between two existing nodes -- e.g. a
|
|
1194
|
-
* failed `shell_command` and whatever node later resolved it
|
|
1195
|
-
* (`relation = 'resolved_by'`). A relation, not a new content node: the
|
|
1196
|
-
* correlation *is* the relationship, and duplicating either side's content
|
|
1197
|
-
* into a third node would just be another independently-ranked candidate.
|
|
1198
|
-
*
|
|
1199
|
-
* Idempotent by design (`INSERT OR IGNORE` against the table's own primary
|
|
1200
|
-
* key) so re-running a correlation pass over already-linked nodes is a
|
|
1201
|
-
* no-op, not a duplicate-row error.
|
|
1202
|
-
*/
|
|
1203
1484
|
linkNodes(fromNodeId, toNodeId, relation) {
|
|
1204
|
-
this.db
|
|
1485
|
+
linkNodes(this.db, fromNodeId, toNodeId, relation);
|
|
1205
1486
|
}
|
|
1206
1487
|
/** Ids linked from `fromNodeId` under one relation, most recently linked first. Empty if none exist. */
|
|
1207
1488
|
getLinkedNodeIds(fromNodeId, relation) {
|
|
1208
|
-
return this.db
|
|
1489
|
+
return getLinkedNodeIds(this.db, fromNodeId, relation);
|
|
1209
1490
|
}
|
|
1210
1491
|
/**
|
|
1211
1492
|
* Hydrate full content for a set of node ids, e.g. to pack a linked
|
|
@@ -1219,11 +1500,7 @@ var MemoryStore = class _MemoryStore {
|
|
|
1219
1500
|
* that gap is not addressed here.
|
|
1220
1501
|
*/
|
|
1221
1502
|
getNodesByIds(ids) {
|
|
1222
|
-
|
|
1223
|
-
return this.db.prepare(
|
|
1224
|
-
`SELECT id, kind, project_id AS projectId, ts, title, body, signal, provenance
|
|
1225
|
-
FROM nodes WHERE id IN (SELECT value FROM json_each(?))`
|
|
1226
|
-
).all(JSON.stringify(ids));
|
|
1503
|
+
return getNodesByIds(this.db, ids);
|
|
1227
1504
|
}
|
|
1228
1505
|
/**
|
|
1229
1506
|
* The most recently-remembered nodes for a project, newest event first --
|
|
@@ -1232,18 +1509,11 @@ var MemoryStore = class _MemoryStore {
|
|
|
1232
1509
|
* `idx_nodes_project_ts` already exists for exactly this access pattern.
|
|
1233
1510
|
*/
|
|
1234
1511
|
listRecentNodes(projectId, limit = 20) {
|
|
1235
|
-
return this.db
|
|
1236
|
-
`SELECT id, kind, ts, source, title, signal, provenance
|
|
1237
|
-
FROM nodes
|
|
1238
|
-
WHERE project_id = ?
|
|
1239
|
-
ORDER BY ts_epoch DESC
|
|
1240
|
-
LIMIT ?`
|
|
1241
|
-
).all(projectId, limit);
|
|
1512
|
+
return listRecentNodes(this.db, projectId, limit);
|
|
1242
1513
|
}
|
|
1243
1514
|
/** How many nodes of one source exist for a project. Used to preview a `pruneSourceNodes` wipe before running it. */
|
|
1244
1515
|
countSourceNodes(projectId, source) {
|
|
1245
|
-
|
|
1246
|
-
return row.count;
|
|
1516
|
+
return countSourceNodes(this.db, projectId, source);
|
|
1247
1517
|
}
|
|
1248
1518
|
/**
|
|
1249
1519
|
* Delete the nodes of one source that its latest full scan did not produce.
|
|
@@ -1271,314 +1541,70 @@ var MemoryStore = class _MemoryStore {
|
|
|
1271
1541
|
* real history.
|
|
1272
1542
|
*/
|
|
1273
1543
|
pruneSourceNodes(projectId, source, keepIds, opts = {}) {
|
|
1274
|
-
|
|
1275
|
-
AND id NOT IN (SELECT value FROM json_each(@keepIds))
|
|
1276
|
-
AND id NOT IN (SELECT node_id FROM node_files WHERE path IN (SELECT value FROM json_each(@keepPaths)))`;
|
|
1277
|
-
const params = {
|
|
1278
|
-
projectId,
|
|
1279
|
-
source,
|
|
1280
|
-
keepIds: JSON.stringify(keepIds),
|
|
1281
|
-
keepPaths: JSON.stringify(opts.keepPaths ?? [])
|
|
1282
|
-
};
|
|
1283
|
-
return this.db.transaction(() => {
|
|
1284
|
-
this.db.prepare(`DELETE FROM nodes_vec WHERE rowid IN (SELECT rowid FROM nodes WHERE ${scope})`).run(params);
|
|
1285
|
-
return this.db.prepare(`DELETE FROM nodes WHERE ${scope}`).run(params).changes;
|
|
1286
|
-
})();
|
|
1544
|
+
return pruneSourceNodes(this.db, projectId, source, keepIds, opts);
|
|
1287
1545
|
}
|
|
1288
|
-
/**
|
|
1289
|
-
* What `forget(projectId, otherProjectIds, input)` with these same
|
|
1290
|
-
* arguments would remove, without writing anything -- the `forget` CLI
|
|
1291
|
-
* command's dry-run default.
|
|
1292
|
-
*/
|
|
1293
1546
|
previewForget(projectId, otherProjectIds, input) {
|
|
1294
|
-
|
|
1295
|
-
const probeEntry = { id: -1, projectId, createdAt: 0, ...input };
|
|
1296
|
-
const select = this.db.prepare("SELECT project_id, source, title, body, meta FROM nodes WHERE project_id = ?");
|
|
1297
|
-
const counts = /* @__PURE__ */ new Map();
|
|
1298
|
-
for (const scopeId of [projectId, ...otherProjectIds]) {
|
|
1299
|
-
const rows = select.all(scopeId);
|
|
1300
|
-
for (const row of rows) {
|
|
1301
|
-
if (!firstMatchingEntry([probeEntry], { title: row.title, body: row.body, meta: parseMeta(row.meta) })) continue;
|
|
1302
|
-
const key = `${row.project_id} ${row.source}`;
|
|
1303
|
-
const existing = counts.get(key);
|
|
1304
|
-
if (existing) existing.count += 1;
|
|
1305
|
-
else counts.set(key, { projectId: row.project_id, source: row.source, count: 1 });
|
|
1306
|
-
}
|
|
1307
|
-
}
|
|
1308
|
-
return [...counts.values()];
|
|
1547
|
+
return previewForget(this.db, projectId, otherProjectIds, input);
|
|
1309
1548
|
}
|
|
1310
|
-
/**
|
|
1311
|
-
* Permanently deny-list a value and delete every node it currently matches
|
|
1312
|
-
* across `projectId` + `otherProjectIds` (the same sweep `pruneSourceNodes`
|
|
1313
|
-
* uses for a repo's stale prior identities).
|
|
1314
|
-
*
|
|
1315
|
-
* Unlike `pruneSourceNodes`, this doesn't just delete: the deny-list entry
|
|
1316
|
-
* written here is consulted by `upsertNodes` and `reconcile.ts` on every
|
|
1317
|
-
* future write, so a value forgotten today cannot be re-derived from an
|
|
1318
|
-
* append-only source (the shell-hook log, a full transcript re-read) on a
|
|
1319
|
-
* later `sync --rebuild`. Each removed node leaves a hash-only tombstone
|
|
1320
|
-
* (never the content itself) and the whole operation writes one
|
|
1321
|
-
* `mutation_audit` row, whether or not anything matched -- pre-emptively
|
|
1322
|
-
* blocking a value that hasn't appeared yet is a valid, auditable call.
|
|
1323
|
-
*/
|
|
1324
1549
|
forget(projectId, otherProjectIds, input) {
|
|
1325
|
-
return this.db
|
|
1326
|
-
const entry = insertDenyListEntry(this.db, { ...input, projectId });
|
|
1327
|
-
const startedAt = Date.now();
|
|
1328
|
-
const auditId = Number(
|
|
1329
|
-
this.db.prepare(
|
|
1330
|
-
`INSERT INTO mutation_audit (action, project_id, detail, affected_count, succeeded, error, started_at, finished_at)
|
|
1331
|
-
VALUES ('forget', @projectId, @detail, 0, 1, NULL, @startedAt, @startedAt)`
|
|
1332
|
-
).run({
|
|
1333
|
-
projectId,
|
|
1334
|
-
detail: JSON.stringify({
|
|
1335
|
-
pattern: input.pattern,
|
|
1336
|
-
matchType: input.matchType,
|
|
1337
|
-
ignoreCase: input.ignoreCase,
|
|
1338
|
-
reason: input.reason,
|
|
1339
|
-
scopeProjectIds: [projectId, ...otherProjectIds]
|
|
1340
|
-
}),
|
|
1341
|
-
startedAt
|
|
1342
|
-
}).lastInsertRowid
|
|
1343
|
-
);
|
|
1344
|
-
const select = this.db.prepare(
|
|
1345
|
-
"SELECT id, kind, project_id, source, ts, signal, title, body, meta FROM nodes WHERE project_id = ?"
|
|
1346
|
-
);
|
|
1347
|
-
const dropEmbedding = this.db.prepare("DELETE FROM nodes_vec WHERE rowid = (SELECT rowid FROM nodes WHERE id = ?)");
|
|
1348
|
-
const deleteNode = this.db.prepare("DELETE FROM nodes WHERE id = ?");
|
|
1349
|
-
const insertTombstone = this.db.prepare(
|
|
1350
|
-
`INSERT INTO tombstones
|
|
1351
|
-
(node_id, project_id, kind, source, ts, signal, body_sha256, title_sha256, body_length, deny_list_id, mutation_audit_id, removed_at)
|
|
1352
|
-
VALUES (@nodeId, @projectId, @kind, @source, @ts, @signal, @bodySha256, @titleSha256, @bodyLength, @denyListId, @mutationAuditId, @removedAt)`
|
|
1353
|
-
);
|
|
1354
|
-
let removed = 0;
|
|
1355
|
-
for (const scopeId of [projectId, ...otherProjectIds]) {
|
|
1356
|
-
const rows = select.all(scopeId);
|
|
1357
|
-
for (const row of rows) {
|
|
1358
|
-
if (!firstMatchingEntry([entry], { title: row.title, body: row.body, meta: parseMeta(row.meta) })) continue;
|
|
1359
|
-
dropEmbedding.run(row.id);
|
|
1360
|
-
insertTombstone.run({
|
|
1361
|
-
nodeId: row.id,
|
|
1362
|
-
projectId: row.project_id,
|
|
1363
|
-
kind: row.kind,
|
|
1364
|
-
source: row.source,
|
|
1365
|
-
ts: row.ts,
|
|
1366
|
-
signal: row.signal,
|
|
1367
|
-
bodySha256: sha256Hex(row.body),
|
|
1368
|
-
titleSha256: sha256Hex(row.title),
|
|
1369
|
-
bodyLength: row.body.length,
|
|
1370
|
-
denyListId: entry.id,
|
|
1371
|
-
mutationAuditId: auditId,
|
|
1372
|
-
removedAt: Date.now()
|
|
1373
|
-
});
|
|
1374
|
-
deleteNode.run(row.id);
|
|
1375
|
-
removed += 1;
|
|
1376
|
-
}
|
|
1377
|
-
}
|
|
1378
|
-
this.db.prepare("UPDATE mutation_audit SET affected_count = ?, finished_at = ? WHERE id = ?").run(removed, Date.now(), auditId);
|
|
1379
|
-
return { removed, entryId: entry.id, auditId };
|
|
1380
|
-
})();
|
|
1550
|
+
return forget(this.db, projectId, otherProjectIds, input);
|
|
1381
1551
|
}
|
|
1382
1552
|
/** Active deny-list entries for one project, oldest first. */
|
|
1383
1553
|
listDenyList(projectId) {
|
|
1384
|
-
return
|
|
1554
|
+
return listDenyList(this.db, projectId);
|
|
1385
1555
|
}
|
|
1386
|
-
/**
|
|
1387
|
-
* What `importDenyList` with these same entries would do, without writing
|
|
1388
|
-
* anything -- `forget --import`'s dry-run default, same convention as
|
|
1389
|
-
* `previewForget`.
|
|
1390
|
-
*/
|
|
1391
1556
|
previewImportDenyList(projectId, otherProjectIds, entries) {
|
|
1392
|
-
return
|
|
1393
|
-
validatePattern(input);
|
|
1394
|
-
if (denyListEntryExists(this.db, projectId, input)) {
|
|
1395
|
-
return { matchType: input.matchType, pattern: input.pattern, alreadyPresent: true, wouldRemove: 0 };
|
|
1396
|
-
}
|
|
1397
|
-
const preview = this.previewForget(projectId, otherProjectIds, input);
|
|
1398
|
-
return {
|
|
1399
|
-
matchType: input.matchType,
|
|
1400
|
-
pattern: input.pattern,
|
|
1401
|
-
alreadyPresent: false,
|
|
1402
|
-
wouldRemove: preview.reduce((sum, p) => sum + p.count, 0)
|
|
1403
|
-
};
|
|
1404
|
-
});
|
|
1557
|
+
return previewImportDenyList(this.db, projectId, otherProjectIds, entries);
|
|
1405
1558
|
}
|
|
1406
|
-
/**
|
|
1407
|
-
* Re-apply a previously-exported deny-list against this project.
|
|
1408
|
-
*
|
|
1409
|
-
* This is the fix for `forget`'s per-checkout gap: `deny_list` lives in
|
|
1410
|
-
* `.nexusmem/memory.db`, which is gitignored and never travels with `git
|
|
1411
|
-
* clone`/`git push`, while the things a fresh `sync` re-derives from --
|
|
1412
|
-
* git history and the user-home shell-hook log -- both travel or persist
|
|
1413
|
-
* independently of any one checkout. A fresh clone or a restored backup
|
|
1414
|
-
* starts with an empty deny_list and no memory of what was forgotten. See
|
|
1415
|
-
* docs/forget-mechanism.md.
|
|
1416
|
-
*
|
|
1417
|
-
* Entries already active (same matchType+pattern+ignoreCase) are left
|
|
1418
|
-
* untouched. Every new one goes through `forget` itself, so an imported
|
|
1419
|
-
* value is deleted from this checkout's nodes too, not just blocked going
|
|
1420
|
-
* forward -- exactly what running `nexusmem forget <value>` fresh in this
|
|
1421
|
-
* checkout would have done.
|
|
1422
|
-
*/
|
|
1423
1559
|
importDenyList(projectId, otherProjectIds, entries) {
|
|
1424
|
-
|
|
1425
|
-
let skipped = 0;
|
|
1426
|
-
let removedNodes = 0;
|
|
1427
|
-
for (const input of entries) {
|
|
1428
|
-
validatePattern(input);
|
|
1429
|
-
if (denyListEntryExists(this.db, projectId, input)) {
|
|
1430
|
-
skipped += 1;
|
|
1431
|
-
continue;
|
|
1432
|
-
}
|
|
1433
|
-
const result = this.forget(projectId, otherProjectIds, input);
|
|
1434
|
-
imported += 1;
|
|
1435
|
-
removedNodes += result.removed;
|
|
1436
|
-
}
|
|
1437
|
-
return { imported, skipped, removedNodes };
|
|
1560
|
+
return importDenyList(this.db, projectId, otherProjectIds, entries);
|
|
1438
1561
|
}
|
|
1439
|
-
/**
|
|
1440
|
-
* Replace this project's entire `file_edges` snapshot in one transaction.
|
|
1441
|
-
*
|
|
1442
|
-
* Edges describe the current working tree, not history -- unlike
|
|
1443
|
-
* `pruneSourceNodes`'s incremental diff-against-a-scan, there is no cursor
|
|
1444
|
-
* to walk, so every `scan-structure`/sync run is a full rescan and this is
|
|
1445
|
-
* always a delete-then-insert of the whole set, never a partial update.
|
|
1446
|
-
*/
|
|
1447
1562
|
replaceFileEdges(projectId, edges) {
|
|
1448
|
-
this.db
|
|
1449
|
-
this.db.prepare("DELETE FROM file_edges WHERE project_id = ?").run(projectId);
|
|
1450
|
-
const insert = this.db.prepare(
|
|
1451
|
-
"INSERT OR IGNORE INTO file_edges (project_id, from_path, to_path, kind) VALUES (?, ?, ?, ?)"
|
|
1452
|
-
);
|
|
1453
|
-
for (const edge of edges) insert.run(projectId, edge.fromPath, edge.toPath, edge.kind);
|
|
1454
|
-
})();
|
|
1563
|
+
replaceFileEdges(this.db, projectId, edges);
|
|
1455
1564
|
}
|
|
1456
1565
|
/** Edge count + distinct source-file count, for `nexusmem status`'s `structure` line. */
|
|
1457
1566
|
fileEdgeStats(projectId) {
|
|
1458
|
-
|
|
1459
|
-
const files = this.db.prepare("SELECT COUNT(DISTINCT from_path) AS c FROM file_edges WHERE project_id = ?").get(projectId).c;
|
|
1460
|
-
return { edges, files };
|
|
1567
|
+
return fileEdgeStats(this.db, projectId);
|
|
1461
1568
|
}
|
|
1462
|
-
/**
|
|
1463
|
-
* Nodes for this project that have no embedding yet (new, or invalidated
|
|
1464
|
-
* by a content change).
|
|
1465
|
-
*
|
|
1466
|
-
* `afterRowid` makes paging monotonic: the pass walks rowids strictly
|
|
1467
|
-
* upward instead of re-reading "the first N still pending". That matters
|
|
1468
|
-
* because a node the provider *failed* on stays pending -- an offset-free
|
|
1469
|
-
* loop would fetch the same failures forever, which is exactly the shape
|
|
1470
|
-
* of an infinite sync.
|
|
1471
|
-
*/
|
|
1472
1569
|
findNodesNeedingEmbedding(projectId, limit = 200, afterRowid = 0) {
|
|
1473
|
-
return this.db
|
|
1474
|
-
`SELECT n.rowid AS rowid, n.id AS id, n.title AS title, n.body AS body
|
|
1475
|
-
FROM nodes n
|
|
1476
|
-
LEFT JOIN nodes_vec v ON v.rowid = n.rowid
|
|
1477
|
-
WHERE n.project_id = ? AND v.rowid IS NULL AND n.rowid > ?
|
|
1478
|
-
ORDER BY n.rowid
|
|
1479
|
-
LIMIT ?`
|
|
1480
|
-
).all(projectId, afterRowid, limit);
|
|
1570
|
+
return findNodesNeedingEmbedding(this.db, projectId, limit, afterRowid);
|
|
1481
1571
|
}
|
|
1482
1572
|
/** How many of this project's nodes still need a vector. For progress reporting. */
|
|
1483
1573
|
countNodesNeedingEmbedding(projectId) {
|
|
1484
|
-
|
|
1485
|
-
`SELECT COUNT(*) AS n
|
|
1486
|
-
FROM nodes n
|
|
1487
|
-
LEFT JOIN nodes_vec v ON v.rowid = n.rowid
|
|
1488
|
-
WHERE n.project_id = ? AND v.rowid IS NULL`
|
|
1489
|
-
).get(projectId);
|
|
1490
|
-
return row.n;
|
|
1574
|
+
return countNodesNeedingEmbedding(this.db, projectId);
|
|
1491
1575
|
}
|
|
1492
1576
|
upsertEmbedding(rowid, embedding) {
|
|
1493
|
-
this.db
|
|
1577
|
+
upsertEmbedding(this.db, rowid, embedding);
|
|
1494
1578
|
}
|
|
1495
|
-
/**
|
|
1496
|
-
* Drop every vector in this database, across all projects.
|
|
1497
|
-
*
|
|
1498
|
-
* Whole-database on purpose: `nodes_vec` is shared and holds no
|
|
1499
|
-
* provenance, so once the vectors in it stopped being comparable there is
|
|
1500
|
-
* no subset that is still trustworthy. Nodes are untouched, so the next
|
|
1501
|
-
* embedding pass simply rebuilds them.
|
|
1502
|
-
*/
|
|
1503
1579
|
dropAllEmbeddings() {
|
|
1504
|
-
return this.db
|
|
1580
|
+
return dropAllEmbeddings(this.db);
|
|
1505
1581
|
}
|
|
1506
1582
|
getMeta(key) {
|
|
1507
|
-
|
|
1508
|
-
return row?.value ?? null;
|
|
1583
|
+
return getMeta(this.db, key);
|
|
1509
1584
|
}
|
|
1510
1585
|
setMeta(key, value) {
|
|
1511
|
-
this.db
|
|
1586
|
+
setMeta(this.db, key, value);
|
|
1512
1587
|
}
|
|
1513
|
-
/**
|
|
1514
|
-
* Nearest-neighbour search over the corpus.
|
|
1515
|
-
*
|
|
1516
|
-
* `nodes_vec` has no `project_id` column of its own (embeddings are
|
|
1517
|
-
* generic; project scoping lives on `nodes`), so this over-fetches `k`
|
|
1518
|
-
* before joining and filtering, then caps to `limit`. Simple and correct;
|
|
1519
|
-
* not the efficient way to do this at a scale this project isn't at yet.
|
|
1520
|
-
*/
|
|
1521
1588
|
vectorSearch(projectId, embedding, limit = 20) {
|
|
1522
|
-
|
|
1523
|
-
return this.db.prepare(
|
|
1524
|
-
`SELECT n.id, n.kind, n.ts, n.title, n.body, n.signal, n.provenance, v.distance AS distance
|
|
1525
|
-
FROM nodes_vec v
|
|
1526
|
-
JOIN nodes n ON n.rowid = v.rowid
|
|
1527
|
-
WHERE v.embedding MATCH ? AND k = ? AND n.project_id = ?
|
|
1528
|
-
ORDER BY v.distance
|
|
1529
|
-
LIMIT ?`
|
|
1530
|
-
).all(embedding, overfetch, projectId, limit);
|
|
1589
|
+
return vectorSearch(this.db, projectId, embedding, limit);
|
|
1531
1590
|
}
|
|
1532
1591
|
stats(projectId) {
|
|
1533
|
-
|
|
1534
|
-
const range = this.db.prepare("SELECT MIN(ts) AS oldest, MAX(ts) AS newest FROM nodes WHERE project_id = ?").get(projectId);
|
|
1535
|
-
const files = this.db.prepare(
|
|
1536
|
-
`SELECT COUNT(DISTINCT f.path) AS n
|
|
1537
|
-
FROM node_files f JOIN nodes n ON n.id = f.node_id
|
|
1538
|
-
WHERE n.project_id = ?`
|
|
1539
|
-
).get(projectId);
|
|
1540
|
-
return {
|
|
1541
|
-
total: kinds.reduce((sum, k) => sum + k.n, 0),
|
|
1542
|
-
byKind: Object.fromEntries(kinds.map((k) => [k.kind, k.n])),
|
|
1543
|
-
oldest: range.oldest,
|
|
1544
|
-
newest: range.newest,
|
|
1545
|
-
distinctFiles: files.n
|
|
1546
|
-
};
|
|
1592
|
+
return stats(this.db, projectId);
|
|
1547
1593
|
}
|
|
1548
|
-
/**
|
|
1549
|
-
* Lexical search over the corpus.
|
|
1550
|
-
*
|
|
1551
|
-
* Title is weighted 10x body: a commit subject that names the thing you asked
|
|
1552
|
-
* about is far stronger evidence than the same word buried in a file list.
|
|
1553
|
-
* Ranking by `relevance x signal` happens a layer up, in retrieval.
|
|
1554
|
-
*/
|
|
1555
1594
|
search(projectId, query, limit = 20) {
|
|
1556
|
-
|
|
1557
|
-
if (!match) return [];
|
|
1558
|
-
const rows = this.db.prepare(
|
|
1559
|
-
`SELECT n.id, n.kind, n.ts, n.title, n.body, n.signal, n.provenance,
|
|
1560
|
-
bm25(nodes_fts, 10.0, 1.0) AS rank
|
|
1561
|
-
FROM nodes_fts
|
|
1562
|
-
JOIN nodes n ON n.rowid = nodes_fts.rowid
|
|
1563
|
-
WHERE nodes_fts MATCH ? AND n.project_id = ?
|
|
1564
|
-
ORDER BY rank
|
|
1565
|
-
LIMIT ?`
|
|
1566
|
-
).all(match, projectId, limit);
|
|
1567
|
-
return rows;
|
|
1595
|
+
return search(this.db, projectId, query, limit);
|
|
1568
1596
|
}
|
|
1569
1597
|
/** The project a node belongs to, or null if no node has this id. Used by `mark-stale` to validate both ids. */
|
|
1570
1598
|
getNodeProjectId(id) {
|
|
1571
|
-
|
|
1572
|
-
return row?.project_id ?? null;
|
|
1599
|
+
return getNodeProjectId(this.db, id);
|
|
1573
1600
|
}
|
|
1574
1601
|
/** Every node id some other node's `supersedes` points at, for one project -- what the ranker should down-weight. */
|
|
1575
1602
|
getSupersededIds(projectId) {
|
|
1576
|
-
|
|
1577
|
-
return new Set(rows.map((r) => r.id));
|
|
1603
|
+
return getSupersededIds(this.db, projectId);
|
|
1578
1604
|
}
|
|
1579
1605
|
/** Record that `newNodeId` supersedes `staleNodeId` -- the write behind `nexusmem mark-stale`. Caller validates both ids first. */
|
|
1580
1606
|
setSupersedes(newNodeId, staleNodeId) {
|
|
1581
|
-
this.db
|
|
1607
|
+
setSupersedes(this.db, newNodeId, staleNodeId);
|
|
1582
1608
|
}
|
|
1583
1609
|
/** Escape hatch for tests and future modules. */
|
|
1584
1610
|
get raw() {
|
|
@@ -3915,8 +3941,8 @@ function hookEntryToRaw(e) {
|
|
|
3915
3941
|
}
|
|
3916
3942
|
async function tryReadScrapeSource(path, parse, tailLines) {
|
|
3917
3943
|
if (!existsSync4(path)) return null;
|
|
3918
|
-
const [raw,
|
|
3919
|
-
return parse(raw,
|
|
3944
|
+
const [raw, stats2] = await Promise.all([readFile9(path, "utf8"), stat2(path)]);
|
|
3945
|
+
return parse(raw, stats2.mtimeMs, { tailLines });
|
|
3920
3946
|
}
|
|
3921
3947
|
async function collectAvailableShellHistory(opts = {}) {
|
|
3922
3948
|
const results = [];
|
|
@@ -4575,7 +4601,7 @@ async function runSync(opts) {
|
|
|
4575
4601
|
addStats(totals, conversation.totals);
|
|
4576
4602
|
addStats(totals, sessions.totals);
|
|
4577
4603
|
addStats(totals, docs.totals);
|
|
4578
|
-
const
|
|
4604
|
+
const stats2 = store.stats(projectId);
|
|
4579
4605
|
const elapsed = ((Date.now() - started) / 1e3).toFixed(2);
|
|
4580
4606
|
const conversationPart = conversationEnabled ? `, ${conversation.seen} conversation exchange(s)` : "";
|
|
4581
4607
|
const sessionPart = config.sources.session.enabled ? `, ${sessions.seen} session summar${sessions.seen === 1 ? "y" : "ies"}` : "";
|
|
@@ -4587,7 +4613,7 @@ async function runSync(opts) {
|
|
|
4587
4613
|
[
|
|
4588
4614
|
`${pc7.green("synced")} ${git2.seen} commit(s)${diffPart}, ${shell.seen} shell entr${shell.seen === 1 ? "y" : "ies"}${conversationPart}${sessionPart}${docsPart}${structurePart} in ${elapsed}s`,
|
|
4589
4615
|
` ${pc7.green(`+${totals.inserted} new`)} ${pc7.yellow(`~${totals.updated} updated`)} ${pc7.dim(`=${totals.unchanged} unchanged`)}${deniedPart}`,
|
|
4590
|
-
` ${pc7.dim(`${
|
|
4616
|
+
` ${pc7.dim(`${stats2.total} node(s) total across ${stats2.distinctFiles} file path(s)`)}`,
|
|
4591
4617
|
""
|
|
4592
4618
|
].join("\n") + embedLine + linkLine
|
|
4593
4619
|
);
|
|
@@ -4682,8 +4708,8 @@ async function getStatus(input) {
|
|
|
4682
4708
|
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
4683
4709
|
const store = MemoryStore.open(ws.dbPath);
|
|
4684
4710
|
try {
|
|
4685
|
-
const
|
|
4686
|
-
return { total:
|
|
4711
|
+
const stats2 = store.stats(projectId);
|
|
4712
|
+
return { total: stats2.total, byKind: stats2.byKind, sources: store.listSyncState(projectId) };
|
|
4687
4713
|
} finally {
|
|
4688
4714
|
store.close();
|
|
4689
4715
|
}
|
|
@@ -4995,6 +5021,26 @@ var BAND_COLOR = {
|
|
|
4995
5021
|
function formatSignal(signal, bands) {
|
|
4996
5022
|
return BAND_COLOR[signalBand(signal, bands)](signal.toFixed(2));
|
|
4997
5023
|
}
|
|
5024
|
+
function approxTotalTokens(nodes) {
|
|
5025
|
+
return nodes.reduce((n, x) => n + approxTokens(x.body), 0);
|
|
5026
|
+
}
|
|
5027
|
+
function summarize2(nodes) {
|
|
5028
|
+
if (nodes.length === 0) return pc10.yellow("no commits matched");
|
|
5029
|
+
const timestamps = nodes.map((n) => n.ts).sort();
|
|
5030
|
+
const avgSignal = nodes.reduce((n, x) => n + x.signal, 0) / nodes.length;
|
|
5031
|
+
const totalTokens = approxTotalTokens(nodes);
|
|
5032
|
+
const fileHits = /* @__PURE__ */ new Map();
|
|
5033
|
+
for (const node of nodes) {
|
|
5034
|
+
for (const f of node.files) fileHits.set(f.path, (fileHits.get(f.path) ?? 0) + 1);
|
|
5035
|
+
}
|
|
5036
|
+
const hottest = [...fileHits.entries()].sort((a, b) => b[1] - a[1]).slice(0, 5).map(([path, count]) => ` ${String(count).padStart(3)}x ${path}`);
|
|
5037
|
+
return [
|
|
5038
|
+
`${pc10.bold(String(nodes.length))} nodes ${pc10.dim(`${timestamps[0]?.slice(0, 10)} .. ${timestamps.at(-1)?.slice(0, 10)}`)}`,
|
|
5039
|
+
` avg signal ${avgSignal.toFixed(3)} ~${totalTokens.toLocaleString()} tokens if sent raw`,
|
|
5040
|
+
hottest.length ? ` hottest files:
|
|
5041
|
+
${hottest.join("\n")}` : ""
|
|
5042
|
+
].filter(Boolean).join("\n");
|
|
5043
|
+
}
|
|
4998
5044
|
|
|
4999
5045
|
// src/cli/commands/scan-conversation.ts
|
|
5000
5046
|
async function runScanConversation(opts) {
|
|
@@ -5019,7 +5065,7 @@ async function runScanConversation(opts) {
|
|
|
5019
5065
|
for (const node of nodes) process.stdout.write(`${formatNode(node)}
|
|
5020
5066
|
`);
|
|
5021
5067
|
const redactedTotal = nodes.reduce((n, x) => n + (Number(x.meta.redactedCount) || 0), 0);
|
|
5022
|
-
const approxTotal = nodes
|
|
5068
|
+
const approxTotal = approxTotalTokens(nodes);
|
|
5023
5069
|
process.stderr.write(
|
|
5024
5070
|
`
|
|
5025
5071
|
${pc11.bold(String(nodes.length))} of ${turns.length} exchange(s) above threshold ${pc11.dim(`~${approxTotal.toLocaleString()} tokens if sent raw`)}` + (redactedTotal > 0 ? ` ${pc11.yellow(`${redactedTotal} secret-like value(s) redacted`)}` : "") + "\n"
|
|
@@ -5031,11 +5077,9 @@ function formatNode(node) {
|
|
|
5031
5077
|
}
|
|
5032
5078
|
|
|
5033
5079
|
// src/cli/commands/scan-diff.ts
|
|
5034
|
-
import pc13 from "picocolors";
|
|
5035
|
-
|
|
5036
|
-
// src/cli/commands/scan-git.ts
|
|
5037
5080
|
import pc12 from "picocolors";
|
|
5038
|
-
|
|
5081
|
+
var DEFAULT_SCAN_COMMITS = 50;
|
|
5082
|
+
async function runScanDiff(opts) {
|
|
5039
5083
|
const repo = await readRepoInfo(opts.cwd);
|
|
5040
5084
|
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
5041
5085
|
if (!opts.json) {
|
|
@@ -5043,19 +5087,16 @@ async function runScanGit(opts) {
|
|
|
5043
5087
|
[
|
|
5044
5088
|
`${pc12.dim("repo ")} ${repo.root}`,
|
|
5045
5089
|
`${pc12.dim("branch ")} ${repo.branch ?? pc12.yellow("(detached)")}`,
|
|
5046
|
-
`${pc12.dim("origin ")} ${repo.originUrl ?? pc12.dim("(none)")}`,
|
|
5047
5090
|
`${pc12.dim("project")} ${pc12.cyan(projectId)}`,
|
|
5048
5091
|
""
|
|
5049
5092
|
].join("\n")
|
|
5050
5093
|
);
|
|
5051
5094
|
}
|
|
5052
5095
|
const nodes = [];
|
|
5053
|
-
const
|
|
5096
|
+
for await (const node of collectCommitDiffs(repo.root, projectId, {
|
|
5054
5097
|
since: opts.since ?? null,
|
|
5055
|
-
maxCount: opts.limit ??
|
|
5056
|
-
|
|
5057
|
-
};
|
|
5058
|
-
for await (const node of collectGitCommits(repo.root, projectId, collectOpts)) {
|
|
5098
|
+
maxCount: opts.limit ?? DEFAULT_SCAN_COMMITS
|
|
5099
|
+
})) {
|
|
5059
5100
|
if (node.signal < opts.minSignal) continue;
|
|
5060
5101
|
nodes.push(node);
|
|
5061
5102
|
if (!opts.json) process.stdout.write(`${formatNode2(node)}
|
|
@@ -5073,119 +5114,105 @@ ${summarize2(nodes)}
|
|
|
5073
5114
|
}
|
|
5074
5115
|
function formatNode2(node) {
|
|
5075
5116
|
const sha = String(node.meta.shortSha ?? "").padEnd(9);
|
|
5076
|
-
const date = node.ts.slice(0, 10);
|
|
5077
|
-
const files = Number(node.meta.filesChanged ?? 0);
|
|
5078
5117
|
const churn = `+${node.meta.insertions ?? 0}/-${node.meta.deletions ?? 0}`;
|
|
5079
5118
|
return [
|
|
5080
|
-
formatSignal(node.signal,
|
|
5081
|
-
pc12.dim(
|
|
5119
|
+
formatSignal(node.signal, DIFF_SIGNAL_BANDS),
|
|
5120
|
+
pc12.dim(node.ts.slice(0, 10)),
|
|
5082
5121
|
pc12.magenta(sha),
|
|
5083
|
-
node.
|
|
5084
|
-
pc12.dim(`(${
|
|
5122
|
+
String(node.meta.path ?? ""),
|
|
5123
|
+
pc12.dim(`(${churn}, ${node.meta.hunkCount ?? 0} hunk(s))`)
|
|
5085
5124
|
].join(" ");
|
|
5086
5125
|
}
|
|
5087
|
-
function summarize2(nodes) {
|
|
5088
|
-
if (nodes.length === 0) return pc12.yellow("no commits matched");
|
|
5089
|
-
const timestamps = nodes.map((n) => n.ts).sort();
|
|
5090
|
-
const avgSignal = nodes.reduce((n, x) => n + x.signal, 0) / nodes.length;
|
|
5091
|
-
const totalTokens = nodes.reduce((n, x) => n + approxTokens(x.body), 0);
|
|
5092
|
-
const fileHits = /* @__PURE__ */ new Map();
|
|
5093
|
-
for (const node of nodes) {
|
|
5094
|
-
for (const f of node.files) fileHits.set(f.path, (fileHits.get(f.path) ?? 0) + 1);
|
|
5095
|
-
}
|
|
5096
|
-
const hottest = [...fileHits.entries()].sort((a, b) => b[1] - a[1]).slice(0, 5).map(([path, count]) => ` ${String(count).padStart(3)}x ${path}`);
|
|
5097
|
-
return [
|
|
5098
|
-
`${pc12.bold(String(nodes.length))} nodes ${pc12.dim(`${timestamps[0]?.slice(0, 10)} .. ${timestamps.at(-1)?.slice(0, 10)}`)}`,
|
|
5099
|
-
` avg signal ${avgSignal.toFixed(3)} ~${totalTokens.toLocaleString()} tokens if sent raw`,
|
|
5100
|
-
hottest.length ? ` hottest files:
|
|
5101
|
-
${hottest.join("\n")}` : ""
|
|
5102
|
-
].filter(Boolean).join("\n");
|
|
5103
|
-
}
|
|
5104
5126
|
|
|
5105
|
-
// src/cli/commands/scan-
|
|
5106
|
-
|
|
5107
|
-
async function
|
|
5127
|
+
// src/cli/commands/scan-docs.ts
|
|
5128
|
+
import pc13 from "picocolors";
|
|
5129
|
+
async function runScanDocs(opts) {
|
|
5108
5130
|
const repo = await readRepoInfo(opts.cwd);
|
|
5109
5131
|
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
5132
|
+
const { files, unreadable } = await readDocFiles(repo.root);
|
|
5110
5133
|
if (!opts.json) {
|
|
5111
5134
|
process.stderr.write(
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
5116
|
-
""
|
|
5117
|
-
].join("\n")
|
|
5135
|
+
files.length ? `${pc13.dim("tracked .md files")} ${files.map((f) => f.path).join(", ")}
|
|
5136
|
+
|
|
5137
|
+
` : `${pc13.yellow("no tracked .md files found")}
|
|
5138
|
+
`
|
|
5118
5139
|
);
|
|
5119
|
-
|
|
5120
|
-
|
|
5121
|
-
|
|
5122
|
-
since: opts.since ?? null,
|
|
5123
|
-
maxCount: opts.limit ?? DEFAULT_SCAN_COMMITS
|
|
5124
|
-
})) {
|
|
5125
|
-
if (node.signal < opts.minSignal) continue;
|
|
5126
|
-
nodes.push(node);
|
|
5127
|
-
if (!opts.json) process.stdout.write(`${formatNode3(node)}
|
|
5140
|
+
if (unreadable.length > 0) {
|
|
5141
|
+
process.stderr.write(`${pc13.yellow("unreadable")} ${unreadable.join(", ")}
|
|
5142
|
+
|
|
5128
5143
|
`);
|
|
5144
|
+
}
|
|
5129
5145
|
}
|
|
5146
|
+
const nodes = collectDocFiles(files, projectId).filter((n) => n.signal >= opts.minSignal);
|
|
5130
5147
|
if (opts.json) {
|
|
5131
5148
|
process.stdout.write(`${JSON.stringify(nodes, null, 2)}
|
|
5132
5149
|
`);
|
|
5133
5150
|
return 0;
|
|
5134
5151
|
}
|
|
5135
|
-
process.
|
|
5136
|
-
${summarize2(nodes)}
|
|
5152
|
+
for (const node of nodes) process.stdout.write(`${formatNode3(node)}
|
|
5137
5153
|
`);
|
|
5154
|
+
const approxTotal = approxTotalTokens(nodes);
|
|
5155
|
+
process.stderr.write(
|
|
5156
|
+
`
|
|
5157
|
+
${pc13.bold(String(nodes.length))} section(s) from ${files.length} file(s) above threshold ${pc13.dim(`~${approxTotal.toLocaleString()} tokens if sent raw`)}
|
|
5158
|
+
`
|
|
5159
|
+
);
|
|
5138
5160
|
return 0;
|
|
5139
5161
|
}
|
|
5140
5162
|
function formatNode3(node) {
|
|
5141
|
-
|
|
5142
|
-
const churn = `+${node.meta.insertions ?? 0}/-${node.meta.deletions ?? 0}`;
|
|
5143
|
-
return [
|
|
5144
|
-
formatSignal(node.signal, DIFF_SIGNAL_BANDS),
|
|
5145
|
-
pc13.dim(node.ts.slice(0, 10)),
|
|
5146
|
-
pc13.magenta(sha),
|
|
5147
|
-
String(node.meta.path ?? ""),
|
|
5148
|
-
pc13.dim(`(${churn}, ${node.meta.hunkCount ?? 0} hunk(s))`)
|
|
5149
|
-
].join(" ");
|
|
5163
|
+
return [formatSignal(node.signal, DOCS_SIGNAL_BANDS), node.title].join(" ");
|
|
5150
5164
|
}
|
|
5151
5165
|
|
|
5152
|
-
// src/cli/commands/scan-
|
|
5166
|
+
// src/cli/commands/scan-git.ts
|
|
5153
5167
|
import pc14 from "picocolors";
|
|
5154
|
-
async function
|
|
5168
|
+
async function runScanGit(opts) {
|
|
5155
5169
|
const repo = await readRepoInfo(opts.cwd);
|
|
5156
5170
|
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
5157
|
-
const { files, unreadable } = await readDocFiles(repo.root);
|
|
5158
5171
|
if (!opts.json) {
|
|
5159
5172
|
process.stderr.write(
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5173
|
+
[
|
|
5174
|
+
`${pc14.dim("repo ")} ${repo.root}`,
|
|
5175
|
+
`${pc14.dim("branch ")} ${repo.branch ?? pc14.yellow("(detached)")}`,
|
|
5176
|
+
`${pc14.dim("origin ")} ${repo.originUrl ?? pc14.dim("(none)")}`,
|
|
5177
|
+
`${pc14.dim("project")} ${pc14.cyan(projectId)}`,
|
|
5178
|
+
""
|
|
5179
|
+
].join("\n")
|
|
5164
5180
|
);
|
|
5165
|
-
|
|
5166
|
-
|
|
5167
|
-
|
|
5181
|
+
}
|
|
5182
|
+
const nodes = [];
|
|
5183
|
+
const collectOpts = {
|
|
5184
|
+
since: opts.since ?? null,
|
|
5185
|
+
maxCount: opts.limit ?? null,
|
|
5186
|
+
includeMerges: opts.merges
|
|
5187
|
+
};
|
|
5188
|
+
for await (const node of collectGitCommits(repo.root, projectId, collectOpts)) {
|
|
5189
|
+
if (node.signal < opts.minSignal) continue;
|
|
5190
|
+
nodes.push(node);
|
|
5191
|
+
if (!opts.json) process.stdout.write(`${formatNode4(node)}
|
|
5168
5192
|
`);
|
|
5169
|
-
}
|
|
5170
5193
|
}
|
|
5171
|
-
const nodes = collectDocFiles(files, projectId).filter((n) => n.signal >= opts.minSignal);
|
|
5172
5194
|
if (opts.json) {
|
|
5173
5195
|
process.stdout.write(`${JSON.stringify(nodes, null, 2)}
|
|
5174
5196
|
`);
|
|
5175
5197
|
return 0;
|
|
5176
5198
|
}
|
|
5177
|
-
|
|
5199
|
+
process.stderr.write(`
|
|
5200
|
+
${summarize2(nodes)}
|
|
5178
5201
|
`);
|
|
5179
|
-
const approxTotal = nodes.reduce((n, x) => n + approxTokens(x.body), 0);
|
|
5180
|
-
process.stderr.write(
|
|
5181
|
-
`
|
|
5182
|
-
${pc14.bold(String(nodes.length))} section(s) from ${files.length} file(s) above threshold ${pc14.dim(`~${approxTotal.toLocaleString()} tokens if sent raw`)}
|
|
5183
|
-
`
|
|
5184
|
-
);
|
|
5185
5202
|
return 0;
|
|
5186
5203
|
}
|
|
5187
5204
|
function formatNode4(node) {
|
|
5188
|
-
|
|
5205
|
+
const sha = String(node.meta.shortSha ?? "").padEnd(9);
|
|
5206
|
+
const date = node.ts.slice(0, 10);
|
|
5207
|
+
const files = Number(node.meta.filesChanged ?? 0);
|
|
5208
|
+
const churn = `+${node.meta.insertions ?? 0}/-${node.meta.deletions ?? 0}`;
|
|
5209
|
+
return [
|
|
5210
|
+
formatSignal(node.signal, GIT_SIGNAL_BANDS),
|
|
5211
|
+
pc14.dim(date),
|
|
5212
|
+
pc14.magenta(sha),
|
|
5213
|
+
node.title,
|
|
5214
|
+
pc14.dim(`(${files} file${files === 1 ? "" : "s"}, ${churn})`)
|
|
5215
|
+
].join(" ");
|
|
5189
5216
|
}
|
|
5190
5217
|
|
|
5191
5218
|
// src/cli/commands/scan-session.ts
|
|
@@ -5303,7 +5330,7 @@ async function runScanShell(opts) {
|
|
|
5303
5330
|
`);
|
|
5304
5331
|
return 0;
|
|
5305
5332
|
}
|
|
5306
|
-
const approxTotal = allNodes
|
|
5333
|
+
const approxTotal = approxTotalTokens(allNodes);
|
|
5307
5334
|
process.stderr.write(`${pc16.bold(String(allNodes.length))} node(s) total ${pc16.dim(`~${approxTotal.toLocaleString()} tokens if sent raw`)}
|
|
5308
5335
|
`);
|
|
5309
5336
|
return 0;
|
|
@@ -5343,8 +5370,35 @@ ${pc17.bold(String(edges.length))} edge(s) from ${filesScanned} tracked .ts/.tsx
|
|
|
5343
5370
|
}
|
|
5344
5371
|
|
|
5345
5372
|
// src/cli/commands/status.ts
|
|
5373
|
+
import { basename as basename4 } from "path";
|
|
5346
5374
|
import { statSync } from "fs";
|
|
5347
5375
|
import pc18 from "picocolors";
|
|
5376
|
+
function daySpan(oldest, newest) {
|
|
5377
|
+
const oldestDay = Date.parse(oldest.slice(0, 10));
|
|
5378
|
+
const newestDay = Date.parse(newest.slice(0, 10));
|
|
5379
|
+
return Math.round((newestDay - oldestDay) / 864e5) + 1;
|
|
5380
|
+
}
|
|
5381
|
+
function buildShareText(repo, stats2, chains) {
|
|
5382
|
+
if (stats2.total === 0) {
|
|
5383
|
+
return "Nothing synced yet in this repo -- run `nexusmem sync` first, then `nexusmem status --share`.\n";
|
|
5384
|
+
}
|
|
5385
|
+
const days = daySpan(stats2.oldest ?? stats2.newest ?? "", stats2.newest ?? stats2.oldest ?? "");
|
|
5386
|
+
const commits = stats2.byKind.git_commit ?? 0;
|
|
5387
|
+
const shellCommands = stats2.byKind.shell_command ?? 0;
|
|
5388
|
+
const docs = stats2.byKind.doc_section ?? 0;
|
|
5389
|
+
const parts = [`${commits} commit(s)`, `${shellCommands} shell command(s)`, `${docs} doc section(s)`].filter(
|
|
5390
|
+
(p) => !p.startsWith("0 ")
|
|
5391
|
+
);
|
|
5392
|
+
const lines = [
|
|
5393
|
+
`NexusMem has been watching ${basename4(repo.root)} for ${days} day(s):`,
|
|
5394
|
+
` ${stats2.total} memories${parts.length ? ` (${parts.join(", ")})` : ""}`
|
|
5395
|
+
];
|
|
5396
|
+
if (chains.failuresTotal) {
|
|
5397
|
+
lines.push(` ${chains.resolvedTotal}/${chains.failuresTotal} failure -> fix chain(s) linked`);
|
|
5398
|
+
}
|
|
5399
|
+
lines.push("", "Local-only SQLite, no cloud, no telemetry.", "https://github.com/yaminbkk/NexusMem");
|
|
5400
|
+
return lines.join("\n").concat("\n");
|
|
5401
|
+
}
|
|
5348
5402
|
function humanBytes(bytes) {
|
|
5349
5403
|
if (bytes < 1024) return `${bytes} B`;
|
|
5350
5404
|
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
@@ -5362,16 +5416,20 @@ async function runStatus(opts) {
|
|
|
5362
5416
|
const { repo, ws, projectId } = await loadContext(opts.cwd);
|
|
5363
5417
|
const store = MemoryStore.open(ws.dbPath);
|
|
5364
5418
|
try {
|
|
5365
|
-
const
|
|
5419
|
+
const stats2 = store.stats(projectId);
|
|
5420
|
+
const chains = getChainStats(store, projectId);
|
|
5421
|
+
if (opts.share) {
|
|
5422
|
+
out(buildShareText(repo, stats2, chains));
|
|
5423
|
+
return 0;
|
|
5424
|
+
}
|
|
5366
5425
|
const sources = store.listSyncState(projectId);
|
|
5367
5426
|
const gitCursor = sources.find((s) => s.source === "git")?.cursor ?? null;
|
|
5368
5427
|
const schema = currentSchemaVersion(store.raw);
|
|
5369
|
-
const chains = getChainStats(store, projectId);
|
|
5370
5428
|
const otherProjectIds = store.listOtherProjectIds(projectId);
|
|
5371
5429
|
const otherProjectNodes = store.countProjectNodes(otherProjectIds);
|
|
5372
5430
|
const structure = store.fileEdgeStats(projectId);
|
|
5373
5431
|
const dbBytes = fileSize(ws.dbPath) + fileSize(`${ws.dbPath}-wal`);
|
|
5374
|
-
const kinds = Object.entries(
|
|
5432
|
+
const kinds = Object.entries(stats2.byKind).sort((a, b) => b[1] - a[1]).map(([kind, n]) => ` ${String(n).padStart(6)} ${kind}`);
|
|
5375
5433
|
const staleProjectWarning = otherProjectIds.length ? `${pc18.yellow("stale ")} ${otherProjectIds.length} prior project ${otherProjectIds.length === 1 ? "identity holds" : "identities hold"} ${otherProjectNodes} node(s) \u2014 run ${pc18.bold(
|
|
5376
5434
|
"nexusmem sync --prune-source <name>"
|
|
5377
5435
|
)} to remove stale source data` : "";
|
|
@@ -5384,9 +5442,9 @@ async function runStatus(opts) {
|
|
|
5384
5442
|
`${pc18.dim("database")} ${ws.dbPath} ${pc18.dim(`(${humanBytes(dbBytes)})`)}`,
|
|
5385
5443
|
staleProjectWarning,
|
|
5386
5444
|
"",
|
|
5387
|
-
`${pc18.bold(String(
|
|
5445
|
+
`${pc18.bold(String(stats2.total))} node(s)${stats2.total ? ` ${pc18.dim(`${stats2.oldest?.slice(0, 10)} .. ${stats2.newest?.slice(0, 10)}`)}` : ""}`,
|
|
5388
5446
|
...kinds,
|
|
5389
|
-
|
|
5447
|
+
stats2.total ? ` ${pc18.dim(`${stats2.distinctFiles} distinct file path(s)`)}` : "",
|
|
5390
5448
|
"",
|
|
5391
5449
|
sources.length ? pc18.dim("sources") : pc18.yellow("no sources synced yet"),
|
|
5392
5450
|
...sources.map((s) => {
|
|
@@ -5394,8 +5452,8 @@ async function runStatus(opts) {
|
|
|
5394
5452
|
const cursorLabel = s.source === "git" ? s.cursor?.slice(0, 7) ?? "-" : s.cursor ?? "-";
|
|
5395
5453
|
return ` ${s.source.padEnd(14)} ${pc18.dim(`last run ${when}`)} ${pc18.dim(`cursor ${cursorLabel}`)}`;
|
|
5396
5454
|
}),
|
|
5397
|
-
gitCursor && gitCursor !== repo.head ? `${pc18.yellow("git behind HEAD")} \u2014 run ${pc18.bold("nexusmem sync")}` : "",
|
|
5398
5455
|
"",
|
|
5456
|
+
gitCursor && gitCursor !== repo.head ? `${pc18.yellow("git behind HEAD")} \u2014 run ${pc18.bold("nexusmem sync")}` : "",
|
|
5399
5457
|
chains.failuresTotal ? `${pc18.dim("chains ")} ${pc18.bold(String(chains.resolvedTotal))}/${chains.failuresTotal} failure(s) resolved ${pc18.dim(`(${chains.resolvedByRetry} retry, ${chains.resolvedByDiscussion} discussion)`)}${chains.resolvedTotal < chains.failuresTotal ? ` \u2014 run ${pc18.bold("nexusmem sync --link-failures")} to link more` : ""}` : "",
|
|
5400
5458
|
structure.edges ? `${pc18.dim("structure")} ${pc18.bold(String(structure.edges))} import edge(s) across ${structure.files} file(s)` : ""
|
|
5401
5459
|
].filter((line) => line !== "").join("\n").concat("\n")
|
|
@@ -5482,7 +5540,7 @@ program.command("hook").description("Manage the opt-in PowerShell hook that logs
|
|
|
5482
5540
|
new Command("status").description("Show whether the git pre-commit hook is installed").option("-C, --cwd <path>", "repository path", process.cwd()).action((options) => guard(() => runHookGitStatus({ cwd: options.cwd }))())
|
|
5483
5541
|
)
|
|
5484
5542
|
);
|
|
5485
|
-
program.command("status").description("Show what is currently remembered for this repository").option("-C, --cwd <path>", "repository path", process.cwd()).action((options) => guard(() => runStatus({ cwd: options.cwd }))());
|
|
5543
|
+
program.command("status").description("Show what is currently remembered for this repository").option("-C, --cwd <path>", "repository path", process.cwd()).option("--share", "print a plain-text summary formatted for sharing, e.g. on X or Reddit").action((options) => guard(() => runStatus({ cwd: options.cwd, share: options.share }))());
|
|
5486
5544
|
program.command("query").description("Search remembered history and print a token-budgeted context block").argument("<text>", "free-text query").option("-C, --cwd <path>", "repository path", process.cwd()).option("-b, --budget <tokens>", "max tokens in the packed context", (v) => Number.parseInt(v, 10), 2e3).option("-n, --candidates <count>", "how many search hits to rank before packing", (v) => Number.parseInt(v, 10), 30).option("--half-life <days>", "days for a node's recency weight to halve", (v) => Number.parseFloat(v)).option("--no-vector", "BM25 only -- skip embedding the query and vector search").option("-a, --all-projects", "search every registered repository, not just this one", false).option("--json", "emit the packed result as JSON on stdout", false).action(
|
|
5487
5545
|
(text, options) => guard(
|
|
5488
5546
|
() => runQuery({
|