devlensio 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +674 -0
- package/dist/clustering/index.d.ts +27 -0
- package/dist/clustering/index.js +149 -0
- package/dist/config/index.d.ts +10 -0
- package/dist/config/index.js +78 -0
- package/dist/config/providers/file.d.ts +19 -0
- package/dist/config/providers/file.js +215 -0
- package/dist/config/providers/request.d.ts +2 -0
- package/dist/config/providers/request.js +72 -0
- package/dist/config/types.d.ts +46 -0
- package/dist/config/types.js +81 -0
- package/dist/config/writer.d.ts +29 -0
- package/dist/config/writer.js +103 -0
- package/dist/filesystem/appRouter.d.ts +2 -0
- package/dist/filesystem/appRouter.js +126 -0
- package/dist/filesystem/backendRoutes.d.ts +2 -0
- package/dist/filesystem/backendRoutes.js +161 -0
- package/dist/filesystem/index.d.ts +2 -0
- package/dist/filesystem/index.js +28 -0
- package/dist/filesystem/index.test.d.ts +1 -0
- package/dist/filesystem/index.test.js +178 -0
- package/dist/filesystem/pagesRouter.d.ts +2 -0
- package/dist/filesystem/pagesRouter.js +109 -0
- package/dist/fingerprint/detectors.d.ts +8 -0
- package/dist/fingerprint/detectors.js +174 -0
- package/dist/fingerprint/index.d.ts +2 -0
- package/dist/fingerprint/index.js +41 -0
- package/dist/fingerprint/index.test.d.ts +1 -0
- package/dist/fingerprint/index.test.js +148 -0
- package/dist/graph/buildLookup.d.ts +10 -0
- package/dist/graph/buildLookup.js +32 -0
- package/dist/graph/edges/callEdges.d.ts +7 -0
- package/dist/graph/edges/callEdges.js +145 -0
- package/dist/graph/edges/eventEdges.d.ts +7 -0
- package/dist/graph/edges/eventEdges.js +203 -0
- package/dist/graph/edges/guardEdges.d.ts +3 -0
- package/dist/graph/edges/guardEdges.js +232 -0
- package/dist/graph/edges/hookEdges.d.ts +3 -0
- package/dist/graph/edges/hookEdges.js +54 -0
- package/dist/graph/edges/importEdges.d.ts +8 -0
- package/dist/graph/edges/importEdges.js +224 -0
- package/dist/graph/edges/propEdges.d.ts +3 -0
- package/dist/graph/edges/propEdges.js +142 -0
- package/dist/graph/edges/routeEdge.d.ts +3 -0
- package/dist/graph/edges/routeEdge.js +124 -0
- package/dist/graph/edges/stateEdges.d.ts +3 -0
- package/dist/graph/edges/stateEdges.js +206 -0
- package/dist/graph/edges/testEdges.d.ts +3 -0
- package/dist/graph/edges/testEdges.js +143 -0
- package/dist/graph/edges/utils.d.ts +2 -0
- package/dist/graph/edges/utils.js +25 -0
- package/dist/graph/index.d.ts +6 -0
- package/dist/graph/index.js +65 -0
- package/dist/graph/index.test.d.ts +1 -0
- package/dist/graph/index.test.js +542 -0
- package/dist/graph/thirdPartyLibs.d.ts +8 -0
- package/dist/graph/thirdPartyLibs.js +162 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +15 -0
- package/dist/jobs/index.d.ts +5 -0
- package/dist/jobs/index.js +11 -0
- package/dist/jobs/queue/interface.d.ts +13 -0
- package/dist/jobs/queue/interface.js +1 -0
- package/dist/jobs/queue/memory.d.ts +24 -0
- package/dist/jobs/queue/memory.js +291 -0
- package/dist/jobs/runner.d.ts +3 -0
- package/dist/jobs/runner.js +136 -0
- package/dist/jobs/types.d.ts +112 -0
- package/dist/jobs/types.js +33 -0
- package/dist/parser/directives.d.ts +4 -0
- package/dist/parser/directives.js +31 -0
- package/dist/parser/extractors/components.d.ts +5 -0
- package/dist/parser/extractors/components.js +240 -0
- package/dist/parser/extractors/functions.d.ts +4 -0
- package/dist/parser/extractors/functions.js +240 -0
- package/dist/parser/extractors/hooks.d.ts +4 -0
- package/dist/parser/extractors/hooks.js +128 -0
- package/dist/parser/extractors/stores.d.ts +3 -0
- package/dist/parser/extractors/stores.js +181 -0
- package/dist/parser/index.d.ts +14 -0
- package/dist/parser/index.js +168 -0
- package/dist/parser/index.test.d.ts +1 -0
- package/dist/parser/index.test.js +319 -0
- package/dist/parser/typeUtils.d.ts +9 -0
- package/dist/parser/typeUtils.js +46 -0
- package/dist/pipeline/index.d.ts +50 -0
- package/dist/pipeline/index.js +249 -0
- package/dist/scoring/connectionCounter.d.ts +28 -0
- package/dist/scoring/connectionCounter.js +134 -0
- package/dist/scoring/fileScorer.d.ts +2 -0
- package/dist/scoring/fileScorer.js +44 -0
- package/dist/scoring/index.d.ts +22 -0
- package/dist/scoring/index.js +130 -0
- package/dist/scoring/index.test.d.ts +1 -0
- package/dist/scoring/index.test.js +453 -0
- package/dist/scoring/nodeScorer.d.ts +3 -0
- package/dist/scoring/nodeScorer.js +108 -0
- package/dist/scoring/noiseFilter.d.ts +18 -0
- package/dist/scoring/noiseFilter.js +92 -0
- package/dist/storage/fileStorage.d.ts +117 -0
- package/dist/storage/fileStorage.js +616 -0
- package/dist/storage/index.d.ts +4 -0
- package/dist/storage/index.js +2 -0
- package/dist/storage/interface.d.ts +27 -0
- package/dist/storage/interface.js +1 -0
- package/dist/summarizer/checkpoint.d.ts +15 -0
- package/dist/summarizer/checkpoint.js +110 -0
- package/dist/summarizer/index.d.ts +2 -0
- package/dist/summarizer/index.js +281 -0
- package/dist/summarizer/mapreduce.d.ts +4 -0
- package/dist/summarizer/mapreduce.js +87 -0
- package/dist/summarizer/prompts.d.ts +22 -0
- package/dist/summarizer/prompts.js +205 -0
- package/dist/summarizer/providers/anthropic.d.ts +9 -0
- package/dist/summarizer/providers/anthropic.js +78 -0
- package/dist/summarizer/providers/gemini.d.ts +9 -0
- package/dist/summarizer/providers/gemini.js +79 -0
- package/dist/summarizer/providers/index.d.ts +3 -0
- package/dist/summarizer/providers/index.js +43 -0
- package/dist/summarizer/providers/ollama.d.ts +9 -0
- package/dist/summarizer/providers/ollama.js +23 -0
- package/dist/summarizer/providers/openRouter.d.ts +9 -0
- package/dist/summarizer/providers/openRouter.js +19 -0
- package/dist/summarizer/providers/openai.d.ts +9 -0
- package/dist/summarizer/providers/openai.js +72 -0
- package/dist/summarizer/providers/types.d.ts +32 -0
- package/dist/summarizer/providers/types.js +1 -0
- package/dist/summarizer/retry.d.ts +7 -0
- package/dist/summarizer/retry.js +51 -0
- package/dist/summarizer/topological.d.ts +3 -0
- package/dist/summarizer/topological.js +105 -0
- package/dist/summarizer/types.d.ts +57 -0
- package/dist/summarizer/types.js +17 -0
- package/dist/types.d.ts +78 -0
- package/dist/types.js +1 -0
- package/package.json +48 -0
|
@@ -0,0 +1,616 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import os from "os";
|
|
4
|
+
import { simpleGit } from "simple-git";
|
|
5
|
+
// ─── Storage layout ───────────────────────────────────────────────────────────
|
|
6
|
+
//
|
|
7
|
+
// ~/.devlens/
|
|
8
|
+
// ├── index.json lightweight list of all repos
|
|
9
|
+
// └── graphs/
|
|
10
|
+
// └── {graphId}/ one folder per repo (stable hash of path)
|
|
11
|
+
// ├── meta.json fingerprint, routes, commit history
|
|
12
|
+
// └── commits/
|
|
13
|
+
// └── {hash}.json full node/edge data per commit
|
|
14
|
+
//
|
|
15
|
+
// Diffs are computed on demand — not stored.
|
|
16
|
+
// GitHub scope is reserved in meta.json for future cloud repo support.
|
|
17
|
+
const STORAGE_DIR = path.join(os.homedir(), ".devlens");
|
|
18
|
+
const GRAPHS_DIR = path.join(STORAGE_DIR, "graphs");
|
|
19
|
+
const INDEX_FILE = path.join(STORAGE_DIR, "index.json");
|
|
20
|
+
const SCHEMA_VERSION = "1.0";
|
|
21
|
+
// ─── Path helpers ─────────────────────────────────────────────────────────────
|
|
22
|
+
function graphDir(graphId) {
|
|
23
|
+
return path.join(GRAPHS_DIR, graphId);
|
|
24
|
+
}
|
|
25
|
+
function metaFile(graphId) {
|
|
26
|
+
return path.join(graphDir(graphId), "meta.json");
|
|
27
|
+
}
|
|
28
|
+
function commitsDir(graphId) {
|
|
29
|
+
return path.join(graphDir(graphId), "commits");
|
|
30
|
+
}
|
|
31
|
+
function commitFile(graphId, commitHash) {
|
|
32
|
+
return path.join(commitsDir(graphId), `${commitHash}.json`);
|
|
33
|
+
}
|
|
34
|
+
// Checkpoint file for summarization progress — one per commit
|
|
35
|
+
// Purely a progress tracker — summaries live on nodes in commitFile()
|
|
36
|
+
function checkpointFile(graphId, commitHash) {
|
|
37
|
+
return path.join(commitsDir(graphId), `${commitHash}.summaries.json`);
|
|
38
|
+
}
|
|
39
|
+
// Exported so checkpoint.ts knows where to read/write
|
|
40
|
+
export function getCheckpointPath(graphId, commitHash) {
|
|
41
|
+
return checkpointFile(graphId, commitHash);
|
|
42
|
+
}
|
|
43
|
+
// ─── Init ─────────────────────────────────────────────────────────────────────
|
|
44
|
+
function ensureStorageExists() {
|
|
45
|
+
if (!fs.existsSync(STORAGE_DIR))
|
|
46
|
+
fs.mkdirSync(STORAGE_DIR, { recursive: true });
|
|
47
|
+
if (!fs.existsSync(GRAPHS_DIR))
|
|
48
|
+
fs.mkdirSync(GRAPHS_DIR, { recursive: true });
|
|
49
|
+
if (!fs.existsSync(INDEX_FILE))
|
|
50
|
+
writeIndex({ version: SCHEMA_VERSION, graphs: [] });
|
|
51
|
+
}
|
|
52
|
+
function ensureGraphDirExists(graphId) {
|
|
53
|
+
const dir = graphDir(graphId);
|
|
54
|
+
if (!fs.existsSync(dir))
|
|
55
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
56
|
+
const cDir = commitsDir(graphId);
|
|
57
|
+
if (!fs.existsSync(cDir))
|
|
58
|
+
fs.mkdirSync(cDir, { recursive: true });
|
|
59
|
+
}
|
|
60
|
+
function readIndex() {
|
|
61
|
+
try {
|
|
62
|
+
return JSON.parse(fs.readFileSync(INDEX_FILE, "utf-8"));
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return { version: SCHEMA_VERSION, graphs: [] };
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function writeIndex(index) {
|
|
69
|
+
fs.writeFileSync(INDEX_FILE, JSON.stringify(index, null, 2), "utf-8");
|
|
70
|
+
}
|
|
71
|
+
// ─── Meta helpers ─────────────────────────────────────────────────────────────
|
|
72
|
+
function readMeta(graphId) {
|
|
73
|
+
const file = metaFile(graphId);
|
|
74
|
+
if (!fs.existsSync(file))
|
|
75
|
+
return undefined;
|
|
76
|
+
try {
|
|
77
|
+
return JSON.parse(fs.readFileSync(file, "utf-8"));
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
return undefined;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
function writeMeta(meta) {
|
|
84
|
+
fs.writeFileSync(metaFile(meta.graphId), JSON.stringify(meta, null, 2), "utf-8");
|
|
85
|
+
}
|
|
86
|
+
// ─── Build helpers ────────────────────────────────────────────────────────────
|
|
87
|
+
function buildIndexEntry(result, commitCount) {
|
|
88
|
+
return {
|
|
89
|
+
graphId: result.graphId,
|
|
90
|
+
repoPath: result.repoPath,
|
|
91
|
+
isGithubRepo: result.isGithubRepo,
|
|
92
|
+
githubUrl: null, // populated when GitHub support is added
|
|
93
|
+
framework: result.fingerprint.framework,
|
|
94
|
+
language: result.fingerprint.language,
|
|
95
|
+
latestCommit: result.gitInfo.commitHash,
|
|
96
|
+
latestAnalyzedAt: result.analyzedAt,
|
|
97
|
+
commitCount,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
function buildCommitSummary(result) {
|
|
101
|
+
return {
|
|
102
|
+
commitHash: result.gitInfo.commitHash,
|
|
103
|
+
branch: result.gitInfo.branch,
|
|
104
|
+
message: result.gitInfo.message,
|
|
105
|
+
analyzedAt: result.analyzedAt,
|
|
106
|
+
nodeCount: result.nodes.length,
|
|
107
|
+
edgeCount: result.edges.length,
|
|
108
|
+
hasGit: result.gitInfo.hasGit,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
function buildCommitData(result) {
|
|
112
|
+
return {
|
|
113
|
+
commitHash: result.gitInfo.commitHash,
|
|
114
|
+
analyzedAt: result.analyzedAt,
|
|
115
|
+
nodes: result.nodes,
|
|
116
|
+
edges: result.edges,
|
|
117
|
+
allNodes: result.allNodes,
|
|
118
|
+
allEdges: result.allEdges,
|
|
119
|
+
nodeScores: result.nodeScores,
|
|
120
|
+
stats: result.stats,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
// ─── Public API ───────────────────────────────────────────────────────────────
|
|
124
|
+
export function saveGraph(result, options) {
|
|
125
|
+
ensureStorageExists();
|
|
126
|
+
ensureGraphDirExists(result.graphId);
|
|
127
|
+
const commitHash = result.gitInfo.commitHash;
|
|
128
|
+
// ── 1. Write commit data file ────────────────────────────────
|
|
129
|
+
const cFile = commitFile(result.graphId, commitHash);
|
|
130
|
+
const commitData = buildCommitData(result);
|
|
131
|
+
// If a commit file already exists and force is not set, preserve any
|
|
132
|
+
// summaries already written onto nodes. This handles the crash-mid-
|
|
133
|
+
// summarization case: server restarts, Phase 1 re-runs, saveGraph is
|
|
134
|
+
// called again — without this merge, summaries written before the crash
|
|
135
|
+
// are lost even though the checkpoint says those levels are done.
|
|
136
|
+
if (!options?.force && fs.existsSync(cFile)) {
|
|
137
|
+
try {
|
|
138
|
+
const existing = JSON.parse(fs.readFileSync(cFile, "utf-8"));
|
|
139
|
+
const summaryCache = new Map();
|
|
140
|
+
for (const node of existing.allNodes) {
|
|
141
|
+
if (node.technicalSummary) {
|
|
142
|
+
summaryCache.set(node.id, {
|
|
143
|
+
technicalSummary: node.technicalSummary,
|
|
144
|
+
businessSummary: node.businessSummary,
|
|
145
|
+
security: node.security,
|
|
146
|
+
summaryModel: node.summaryModel,
|
|
147
|
+
summarizedAt: node.summarizedAt,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
if (summaryCache.size > 0) {
|
|
152
|
+
for (const node of commitData.allNodes) {
|
|
153
|
+
const s = summaryCache.get(node.id);
|
|
154
|
+
if (s)
|
|
155
|
+
Object.assign(node, s);
|
|
156
|
+
}
|
|
157
|
+
for (const node of commitData.nodes) {
|
|
158
|
+
const s = summaryCache.get(node.id);
|
|
159
|
+
if (s)
|
|
160
|
+
Object.assign(node, s);
|
|
161
|
+
}
|
|
162
|
+
console.log(`♻️ Preserved ${summaryCache.size} existing summaries for ${commitHash}`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
console.warn(`⚠️ Could not read existing commit file for ${commitHash} — writing fresh`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
fs.writeFileSync(cFile, JSON.stringify(commitData, null, 2), "utf-8");
|
|
170
|
+
// ── 2. Update meta.json ──────────────────────────────────────
|
|
171
|
+
const existingMeta = readMeta(result.graphId);
|
|
172
|
+
const newSummary = buildCommitSummary(result);
|
|
173
|
+
const meta = existingMeta ?? {
|
|
174
|
+
graphId: result.graphId,
|
|
175
|
+
repoPath: result.repoPath,
|
|
176
|
+
isGithubRepo: result.isGithubRepo,
|
|
177
|
+
githubUrl: null,
|
|
178
|
+
githubOwner: null,
|
|
179
|
+
githubRepo: null,
|
|
180
|
+
fingerprint: result.fingerprint,
|
|
181
|
+
routes: result.routes,
|
|
182
|
+
commits: [],
|
|
183
|
+
summarizedCommits: [],
|
|
184
|
+
};
|
|
185
|
+
// Replace if same commit already exists, else append
|
|
186
|
+
const existingCommit = meta.commits.findIndex((c) => c.commitHash === commitHash);
|
|
187
|
+
if (existingCommit >= 0) {
|
|
188
|
+
meta.commits[existingCommit] = newSummary;
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
meta.commits.push(newSummary);
|
|
192
|
+
}
|
|
193
|
+
// Keep commits sorted newest first
|
|
194
|
+
meta.commits.sort((a, b) => new Date(b.analyzedAt).getTime() - new Date(a.analyzedAt).getTime());
|
|
195
|
+
writeMeta(meta);
|
|
196
|
+
// ── 3. Update index.json ─────────────────────────────────────
|
|
197
|
+
const index = readIndex();
|
|
198
|
+
const indexEntry = buildIndexEntry(result, meta.commits.length);
|
|
199
|
+
const existing = index.graphs.findIndex((g) => g.graphId === result.graphId);
|
|
200
|
+
if (existing >= 0) {
|
|
201
|
+
index.graphs[existing] = indexEntry;
|
|
202
|
+
}
|
|
203
|
+
else {
|
|
204
|
+
index.graphs.push(indexEntry);
|
|
205
|
+
}
|
|
206
|
+
index.graphs.sort((a, b) => new Date(b.latestAnalyzedAt).getTime() - new Date(a.latestAnalyzedAt).getTime());
|
|
207
|
+
writeIndex(index);
|
|
208
|
+
console.log(`\n💾 Saved: ${cFile}`);
|
|
209
|
+
}
|
|
210
|
+
// Returns latest commit data merged with meta — reconstructs PipelineResult shape
|
|
211
|
+
export function getGraph(graphId, commitHash // defaults to latest commit
|
|
212
|
+
) {
|
|
213
|
+
ensureStorageExists();
|
|
214
|
+
const meta = readMeta(graphId);
|
|
215
|
+
if (!meta || meta.commits.length === 0)
|
|
216
|
+
return undefined;
|
|
217
|
+
// Use provided commitHash or fall back to latest
|
|
218
|
+
const targetHash = commitHash ?? meta.commits[0].commitHash;
|
|
219
|
+
const cFile = commitFile(graphId, targetHash);
|
|
220
|
+
if (!fs.existsSync(cFile))
|
|
221
|
+
return undefined;
|
|
222
|
+
try {
|
|
223
|
+
const data = JSON.parse(fs.readFileSync(cFile, "utf-8"));
|
|
224
|
+
// Reconstruct full PipelineResult from meta + commit data
|
|
225
|
+
return {
|
|
226
|
+
graphId: meta.graphId,
|
|
227
|
+
repoPath: meta.repoPath,
|
|
228
|
+
analyzedAt: data.analyzedAt,
|
|
229
|
+
fingerprint: meta.fingerprint,
|
|
230
|
+
routes: meta.routes,
|
|
231
|
+
nodes: data.nodes,
|
|
232
|
+
edges: data.edges,
|
|
233
|
+
allNodes: data.allNodes,
|
|
234
|
+
allEdges: data.allEdges,
|
|
235
|
+
nodeScores: data.nodeScores,
|
|
236
|
+
stats: data.stats,
|
|
237
|
+
isGithubRepo: meta.isGithubRepo,
|
|
238
|
+
gitInfo: {
|
|
239
|
+
commitHash: data.commitHash,
|
|
240
|
+
branch: meta.commits.find(c => c.commitHash === targetHash)?.branch ?? "unknown",
|
|
241
|
+
message: meta.commits.find(c => c.commitHash === targetHash)?.message ?? "",
|
|
242
|
+
hasGit: meta.commits.find(c => c.commitHash === targetHash)?.hasGit ?? false,
|
|
243
|
+
},
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
catch (err) {
|
|
247
|
+
console.error(`Failed to read commit ${targetHash} for graph ${graphId}:`, err);
|
|
248
|
+
return undefined;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
export function getNodeCode(graphId, commitHash, nodeId) {
|
|
252
|
+
ensureStorageExists();
|
|
253
|
+
const cFile = commitFile(graphId, commitHash);
|
|
254
|
+
if (!fs.existsSync(cFile))
|
|
255
|
+
return undefined;
|
|
256
|
+
try {
|
|
257
|
+
const data = JSON.parse(fs.readFileSync(cFile, "utf-8"));
|
|
258
|
+
return data.allNodes.find(n => n.id === nodeId);
|
|
259
|
+
}
|
|
260
|
+
catch (err) {
|
|
261
|
+
console.error(`Failed to read node ${nodeId} from ${graphId}/${commitHash}:`, err);
|
|
262
|
+
return undefined;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
export function listGraphs() {
|
|
266
|
+
ensureStorageExists();
|
|
267
|
+
return readIndex().graphs;
|
|
268
|
+
}
|
|
269
|
+
export function getGraphMeta(graphId) {
|
|
270
|
+
ensureStorageExists();
|
|
271
|
+
return readMeta(graphId);
|
|
272
|
+
}
|
|
273
|
+
export function deleteGraph(graphId) {
|
|
274
|
+
ensureStorageExists();
|
|
275
|
+
const dir = graphDir(graphId);
|
|
276
|
+
if (!fs.existsSync(dir))
|
|
277
|
+
return false;
|
|
278
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
279
|
+
const index = readIndex();
|
|
280
|
+
index.graphs = index.graphs.filter((g) => g.graphId !== graphId);
|
|
281
|
+
writeIndex(index);
|
|
282
|
+
console.log(`🗑️ Deleted graph: ${graphId}`);
|
|
283
|
+
return true;
|
|
284
|
+
}
|
|
285
|
+
export function deleteCommit(graphId, commitHash) {
|
|
286
|
+
ensureStorageExists();
|
|
287
|
+
const cFile = commitFile(graphId, commitHash);
|
|
288
|
+
if (!fs.existsSync(cFile))
|
|
289
|
+
return false;
|
|
290
|
+
fs.unlinkSync(cFile);
|
|
291
|
+
const meta = readMeta(graphId);
|
|
292
|
+
if (meta) {
|
|
293
|
+
meta.commits = meta.commits.filter((c) => c.commitHash !== commitHash);
|
|
294
|
+
writeMeta(meta);
|
|
295
|
+
// Update index entry
|
|
296
|
+
const index = readIndex();
|
|
297
|
+
const entry = index.graphs.find((g) => g.graphId === graphId);
|
|
298
|
+
if (entry) {
|
|
299
|
+
entry.commitCount = meta.commits.length;
|
|
300
|
+
entry.latestCommit = meta.commits[0]?.commitHash ?? "";
|
|
301
|
+
entry.latestAnalyzedAt = meta.commits[0]?.analyzedAt ?? "";
|
|
302
|
+
writeIndex(index);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
console.log(`🗑️ Deleted commit ${commitHash} from graph ${graphId}`);
|
|
306
|
+
return true;
|
|
307
|
+
}
|
|
308
|
+
// ─── Diff — computed on demand, never stored ──────────────────────────────────
|
|
309
|
+
const SCORE_CHANGE_THRESHOLD = 0.5;
|
|
310
|
+
export function diffCommits(graphId, fromHash, toHash) {
|
|
311
|
+
ensureStorageExists();
|
|
312
|
+
const fileA = commitFile(graphId, fromHash);
|
|
313
|
+
const fileB = commitFile(graphId, toHash);
|
|
314
|
+
if (!fs.existsSync(fileA) || !fs.existsSync(fileB))
|
|
315
|
+
return undefined;
|
|
316
|
+
const dataA = JSON.parse(fs.readFileSync(fileA, "utf-8"));
|
|
317
|
+
const dataB = JSON.parse(fs.readFileSync(fileB, "utf-8"));
|
|
318
|
+
// Build id → node maps — O(n)
|
|
319
|
+
const nodesA = new Map(dataA.allNodes.map((n) => [n.id, n]));
|
|
320
|
+
const nodesB = new Map(dataB.allNodes.map((n) => [n.id, n]));
|
|
321
|
+
// Build name+type → node map for A — only for globally unique names
|
|
322
|
+
// Functions with the same name in multiple files are excluded — they can't
|
|
323
|
+
// be reliably detected as moved vs added (e.g. addFilesRecursively in multiple edge detectors)
|
|
324
|
+
const nameCount = new Map();
|
|
325
|
+
for (const node of dataA.allNodes) {
|
|
326
|
+
const key = `${node.name}::${node.type}`;
|
|
327
|
+
nameCount.set(key, (nameCount.get(key) ?? 0) + 1);
|
|
328
|
+
}
|
|
329
|
+
const byNameA = new Map();
|
|
330
|
+
for (const node of dataA.allNodes) {
|
|
331
|
+
const key = `${node.name}::${node.type}`;
|
|
332
|
+
if (nameCount.get(key) === 1) {
|
|
333
|
+
byNameA.set(key, node);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
// Build edge maps — nodeId → Set of "type::targetId"
|
|
337
|
+
const edgesA = new Map();
|
|
338
|
+
const edgesB = new Map();
|
|
339
|
+
for (const edge of dataA.allEdges) {
|
|
340
|
+
if (!edgesA.has(edge.from))
|
|
341
|
+
edgesA.set(edge.from, new Set());
|
|
342
|
+
edgesA.get(edge.from).add(`${edge.type}::${edge.to}`);
|
|
343
|
+
}
|
|
344
|
+
for (const edge of dataB.allEdges) {
|
|
345
|
+
if (!edgesB.has(edge.from))
|
|
346
|
+
edgesB.set(edge.from, new Set());
|
|
347
|
+
edgesB.get(edge.from).add(`${edge.type}::${edge.to}`);
|
|
348
|
+
}
|
|
349
|
+
const added = [];
|
|
350
|
+
const removed = [];
|
|
351
|
+
const scoreChanged = [];
|
|
352
|
+
const codeChanged = [];
|
|
353
|
+
const edgesChanged = [];
|
|
354
|
+
const moved = [];
|
|
355
|
+
const movedIds = new Set(); // track moved nodes to exclude from added/removed
|
|
356
|
+
const codeChangedIds = new Set(); // track code-changed nodes to exclude from unchanged
|
|
357
|
+
let unchanged = 0;
|
|
358
|
+
// ── Find moved nodes first ───────────────────────────────────
|
|
359
|
+
// Moved = same name+type (unique in A), different filePath, original gone from B
|
|
360
|
+
for (const [idB, nodeB] of nodesB) {
|
|
361
|
+
if (nodesA.has(idB))
|
|
362
|
+
continue;
|
|
363
|
+
const key = `${nodeB.name}::${nodeB.type}`;
|
|
364
|
+
const nodeA = byNameA.get(key);
|
|
365
|
+
if (!nodeA)
|
|
366
|
+
continue;
|
|
367
|
+
if (nodeA.filePath === nodeB.filePath)
|
|
368
|
+
continue;
|
|
369
|
+
if (nodesB.has(nodeA.id))
|
|
370
|
+
continue; // original still exists this means its possible that it might be added.
|
|
371
|
+
moved.push({
|
|
372
|
+
nodeId: idB,
|
|
373
|
+
name: nodeB.name,
|
|
374
|
+
fromFile: nodeA.filePath,
|
|
375
|
+
toFile: nodeB.filePath,
|
|
376
|
+
scoreBefore: dataA.nodeScores[nodeA.id] ?? 0,
|
|
377
|
+
scoreAfter: dataB.nodeScores[idB] ?? 0,
|
|
378
|
+
});
|
|
379
|
+
movedIds.add(idB);
|
|
380
|
+
movedIds.add(nodeA.id);
|
|
381
|
+
}
|
|
382
|
+
// ── Find code-changed nodes ──────────────────────────────────
|
|
383
|
+
// Code changed = same ID, same file, but codeHash differs.
|
|
384
|
+
// Must run before added/removed so codeChangedIds is populated.
|
|
385
|
+
for (const [idA, nodeA] of nodesA) {
|
|
386
|
+
const nodeB = nodesB.get(idA);
|
|
387
|
+
if (!nodeB)
|
|
388
|
+
continue; // not in both commits
|
|
389
|
+
if (movedIds.has(idA))
|
|
390
|
+
continue; // already counted as moved
|
|
391
|
+
if (!nodeA.codeHash || !nodeB.codeHash)
|
|
392
|
+
continue; // no hash — skip
|
|
393
|
+
if (nodeA.codeHash === nodeB.codeHash)
|
|
394
|
+
continue; // code unchanged
|
|
395
|
+
codeChanged.push({
|
|
396
|
+
nodeId: idA,
|
|
397
|
+
name: nodeA.name,
|
|
398
|
+
type: nodeA.type,
|
|
399
|
+
filePath: nodeA.filePath,
|
|
400
|
+
score: dataB.nodeScores[idA] ?? 0,
|
|
401
|
+
scoreBefore: dataA.nodeScores[idA] ?? 0,
|
|
402
|
+
scoreAfter: dataB.nodeScores[idA] ?? 0,
|
|
403
|
+
});
|
|
404
|
+
codeChangedIds.add(idA);
|
|
405
|
+
}
|
|
406
|
+
// ── Find added nodes ─────────────────────────────────────────
|
|
407
|
+
for (const [idB, nodeB] of nodesB) {
|
|
408
|
+
if (nodesA.has(idB) || movedIds.has(idB))
|
|
409
|
+
continue;
|
|
410
|
+
added.push({
|
|
411
|
+
nodeId: idB,
|
|
412
|
+
name: nodeB.name,
|
|
413
|
+
type: nodeB.type,
|
|
414
|
+
score: dataB.nodeScores[idB] ?? 0,
|
|
415
|
+
filePath: nodeB.filePath,
|
|
416
|
+
});
|
|
417
|
+
}
|
|
418
|
+
// ── Find removed nodes ───────────────────────────────────────
|
|
419
|
+
for (const [idA, nodeA] of nodesA) {
|
|
420
|
+
if (nodesB.has(idA) || movedIds.has(idA))
|
|
421
|
+
continue;
|
|
422
|
+
removed.push({
|
|
423
|
+
nodeId: idA,
|
|
424
|
+
name: nodeA.name,
|
|
425
|
+
type: nodeA.type,
|
|
426
|
+
score: dataA.nodeScores[idA] ?? 0,
|
|
427
|
+
filePath: nodeA.filePath,
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
// ── Find score and edge changes for nodes in both commits ────
|
|
431
|
+
for (const [idA, nodeA] of nodesA) {
|
|
432
|
+
const nodeB = nodesB.get(idA);
|
|
433
|
+
if (!nodeB || movedIds.has(idA))
|
|
434
|
+
continue;
|
|
435
|
+
const scoreA = dataA.nodeScores[idA] ?? 0;
|
|
436
|
+
const scoreB = dataB.nodeScores[idA] ?? 0;
|
|
437
|
+
const delta = scoreB - scoreA;
|
|
438
|
+
if (Math.abs(delta) >= SCORE_CHANGE_THRESHOLD) {
|
|
439
|
+
scoreChanged.push({
|
|
440
|
+
nodeId: idA,
|
|
441
|
+
name: nodeA.name,
|
|
442
|
+
type: nodeA.type,
|
|
443
|
+
scoreBefore: scoreA,
|
|
444
|
+
scoreAfter: scoreB,
|
|
445
|
+
delta: parseFloat(delta.toFixed(2)),
|
|
446
|
+
});
|
|
447
|
+
}
|
|
448
|
+
// Edge diff for this node
|
|
449
|
+
const eA = edgesA.get(idA) ?? new Set();
|
|
450
|
+
const eB = edgesB.get(idA) ?? new Set();
|
|
451
|
+
const addedEdges = [...eB].filter((e) => !eA.has(e)).map((e) => {
|
|
452
|
+
const [type, to] = e.split("::");
|
|
453
|
+
return { type, to };
|
|
454
|
+
});
|
|
455
|
+
const removedEdges = [...eA].filter((e) => !eB.has(e)).map((e) => {
|
|
456
|
+
const [type, to] = e.split("::");
|
|
457
|
+
return { type, to };
|
|
458
|
+
});
|
|
459
|
+
if (addedEdges.length > 0 || removedEdges.length > 0) {
|
|
460
|
+
edgesChanged.push({
|
|
461
|
+
nodeId: idA,
|
|
462
|
+
name: nodeA.name,
|
|
463
|
+
addedEdges,
|
|
464
|
+
removedEdges,
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
else if (Math.abs(delta) < SCORE_CHANGE_THRESHOLD && !codeChangedIds.has(idA)) {
|
|
468
|
+
unchanged++;
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
// Sort score changes by absolute delta descending
|
|
472
|
+
scoreChanged.sort((a, b) => Math.abs(b.delta) - Math.abs(a.delta));
|
|
473
|
+
return { added, removed, scoreChanged, codeChanged, edgesChanged, moved, unchanged };
|
|
474
|
+
}
|
|
475
|
+
// ─── Summarization helpers ────────────────────────────────────────────────────
|
|
476
|
+
// Marks a commit as fully summarized in meta.json.
|
|
477
|
+
// Called by the summarizer after all nodes in a commit have been summarized.
|
|
478
|
+
// Uses commit hash as key — globally unique, works across branches.
|
|
479
|
+
export function markCommitSummarized(graphId, commitHash) {
|
|
480
|
+
ensureStorageExists();
|
|
481
|
+
const meta = readMeta(graphId);
|
|
482
|
+
if (!meta)
|
|
483
|
+
return;
|
|
484
|
+
// Add to summarizedCommits set if not already there
|
|
485
|
+
if (!meta.summarizedCommits?.includes(commitHash)) {
|
|
486
|
+
meta.summarizedCommits?.push(commitHash);
|
|
487
|
+
}
|
|
488
|
+
// Also update the isSummarized flag on the CommitSummary entry
|
|
489
|
+
const commit = meta.commits.find(c => c.commitHash === commitHash);
|
|
490
|
+
if (commit)
|
|
491
|
+
commit.isSummarized = true;
|
|
492
|
+
writeMeta(meta);
|
|
493
|
+
console.log(`✅ Marked commit ${commitHash} as summarized`);
|
|
494
|
+
}
|
|
495
|
+
// Checks if a commit has already been summarized.
|
|
496
|
+
// Fast lookup — O(n) but summarizedCommits list is small.
|
|
497
|
+
export function isCommitSummarized(graphId, commitHash) {
|
|
498
|
+
const meta = readMeta(graphId);
|
|
499
|
+
if (!meta)
|
|
500
|
+
return false;
|
|
501
|
+
return meta.summarizedCommits?.includes(commitHash);
|
|
502
|
+
}
|
|
503
|
+
// Finds the most recent ancestor commit that has been summarized.
|
|
504
|
+
// Uses simple-git to walk the commit history of the repo.
|
|
505
|
+
// Returns undefined if no summarized ancestor exists (= full summarization needed).
|
|
506
|
+
//
|
|
507
|
+
// This works correctly across branches because commit hashes are globally unique.
|
|
508
|
+
// If branch A and branch B both point to commit C, and C is summarized,
|
|
509
|
+
// both branches benefit — no re-summarization needed.
|
|
510
|
+
export async function findLastSummarizedAncestor(graphId, commitHash, repoPath) {
|
|
511
|
+
const meta = readMeta(graphId);
|
|
512
|
+
if (!meta || meta.summarizedCommits?.length === 0)
|
|
513
|
+
return undefined;
|
|
514
|
+
const summarizedSet = new Set(meta.summarizedCommits);
|
|
515
|
+
const commitEntry = meta.commits.find(c => c.commitHash === commitHash);
|
|
516
|
+
if (commitEntry && !commitEntry.hasGit)
|
|
517
|
+
return undefined;
|
|
518
|
+
try {
|
|
519
|
+
const git = simpleGit(repoPath);
|
|
520
|
+
const raw = await git.raw(["log", "--format=%H", commitHash]);
|
|
521
|
+
const hashes = raw.trim().split("\n").filter(Boolean);
|
|
522
|
+
for (const hash of hashes) {
|
|
523
|
+
// exact match — full hash stored
|
|
524
|
+
if (summarizedSet.has(hash))
|
|
525
|
+
return hash;
|
|
526
|
+
// startsWith handles transition: stored short hash vs full hash from git
|
|
527
|
+
// or vice versa — works regardless of which side is shorter
|
|
528
|
+
for (const stored of summarizedSet) {
|
|
529
|
+
if (hash.startsWith(stored) || stored.startsWith(hash))
|
|
530
|
+
return stored; //comparing startswith only for the case where user have already summareized. Earlier in the Pipeline the in getGitInfo method I was using only first 8 characters, but in large monoRepos it is possible that git may use full 40 characters long commit, so Its better to just compare that.
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
return undefined;
|
|
534
|
+
}
|
|
535
|
+
catch (err) {
|
|
536
|
+
console.warn(`Could not walk git history for ${repoPath}:`, err);
|
|
537
|
+
return undefined;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
// Saves summaries back onto nodes in the commit file after each batch.
|
|
541
|
+
// Called by the summarizer after each batch completes.
|
|
542
|
+
// Merges summary fields onto existing nodes — never replaces the whole file.
|
|
543
|
+
//
|
|
544
|
+
// nodeUpdates: Map<nodeId, NodeSummary> — only the nodes summarized in this batch
|
|
545
|
+
export function saveNodeSummaries(graphId, commitHash, nodeUpdates) {
|
|
546
|
+
ensureStorageExists();
|
|
547
|
+
const cFile = commitFile(graphId, commitHash);
|
|
548
|
+
if (!fs.existsSync(cFile)) {
|
|
549
|
+
console.error(`Cannot save summaries — commit file not found: ${cFile}`);
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
const data = JSON.parse(fs.readFileSync(cFile, "utf-8"));
|
|
553
|
+
// Build id → node maps once — O(n)
|
|
554
|
+
// JSON.parse gives independent copies of allNodes and nodes (filtered copy from allNodes),
|
|
555
|
+
// so both arrays need updating. Maps let us apply the batch
|
|
556
|
+
// in O(b) instead of scanning all nodes per update.
|
|
557
|
+
const allNodesById = new Map(data.allNodes.map(n => [n.id, n]));
|
|
558
|
+
const nodesById = new Map(data.nodes.map(n => [n.id, n]));
|
|
559
|
+
// Apply updates — O(b) where b = batch size
|
|
560
|
+
let updatedCount = 0;
|
|
561
|
+
for (const [nodeId, summary] of nodeUpdates) {
|
|
562
|
+
const apply = (node) => {
|
|
563
|
+
node.technicalSummary = summary.technicalSummary;
|
|
564
|
+
node.businessSummary = summary.businessSummary;
|
|
565
|
+
node.security = summary.security;
|
|
566
|
+
node.summaryModel = summary.summaryModel;
|
|
567
|
+
node.summarizedAt = summary.summarizedAt;
|
|
568
|
+
};
|
|
569
|
+
const allNode = allNodesById.get(nodeId);
|
|
570
|
+
if (allNode) {
|
|
571
|
+
apply(allNode);
|
|
572
|
+
updatedCount++;
|
|
573
|
+
}
|
|
574
|
+
const node = nodesById.get(nodeId);
|
|
575
|
+
if (node)
|
|
576
|
+
apply(node);
|
|
577
|
+
}
|
|
578
|
+
// Write back atomically
|
|
579
|
+
const tmp = `${cFile}.tmp`;
|
|
580
|
+
fs.writeFileSync(tmp, JSON.stringify(data, null, 2), "utf-8");
|
|
581
|
+
fs.renameSync(tmp, cFile);
|
|
582
|
+
console.log(`💾 Saved ${updatedCount} node summaries to ${commitHash}`);
|
|
583
|
+
}
|
|
584
|
+
export function removeFromSummarizedCommits(graphId, commitHash) {
|
|
585
|
+
const meta = readMeta(graphId);
|
|
586
|
+
if (!meta)
|
|
587
|
+
return;
|
|
588
|
+
meta.summarizedCommits = (meta.summarizedCommits ?? []).filter(h => h !== commitHash);
|
|
589
|
+
// Also reset isSummarized flag on the commit entry
|
|
590
|
+
const commit = meta.commits.find(c => c.commitHash === commitHash);
|
|
591
|
+
if (commit)
|
|
592
|
+
commit.isSummarized = false;
|
|
593
|
+
writeMeta(meta);
|
|
594
|
+
}
|
|
595
|
+
export const fileStorage = {
|
|
596
|
+
saveGraph,
|
|
597
|
+
getGraph,
|
|
598
|
+
listGraphs,
|
|
599
|
+
getGraphMeta,
|
|
600
|
+
deleteGraph,
|
|
601
|
+
diffCommits,
|
|
602
|
+
markCommitSummarized,
|
|
603
|
+
isCommitSummarized,
|
|
604
|
+
findLastSummarizedAncestor,
|
|
605
|
+
saveNodeSummaries,
|
|
606
|
+
getCheckpointPath,
|
|
607
|
+
removeFromSummarizedCommits
|
|
608
|
+
};
|
|
609
|
+
// What each function does:
|
|
610
|
+
// saveGraph() → writes 3 things: commit file, meta.json, index.json
|
|
611
|
+
// getGraph() → reads meta + commit file, reconstructs PipelineResult
|
|
612
|
+
// listGraphs() → reads index.json only, never touches graph folders
|
|
613
|
+
// getGraphMeta() → reads meta.json — returns commit history for a repo
|
|
614
|
+
// deleteGraph() → deletes entire folder + removes from index
|
|
615
|
+
// deleteCommit() → deletes one commit file + updates meta + index
|
|
616
|
+
// diffCommits() → loads two commit files, computes O(n) diff, returns result
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { PipelineResult } from "../pipeline/index.js";
|
|
2
|
+
import type { GraphIndexEntry, GraphMeta, NodeDiff } from "./fileStorage.js";
|
|
3
|
+
export interface GraphStorage {
|
|
4
|
+
saveGraph(result: PipelineResult, options?: {
|
|
5
|
+
force?: boolean;
|
|
6
|
+
}): void;
|
|
7
|
+
getGraph(graphId: string, commitHash?: string): PipelineResult | undefined;
|
|
8
|
+
listGraphs(): GraphIndexEntry[];
|
|
9
|
+
getGraphMeta(graphId: string): GraphMeta | undefined;
|
|
10
|
+
deleteGraph(graphId: string): boolean;
|
|
11
|
+
diffCommits(graphId: string, fromHash: string, toHash: string): NodeDiff | undefined;
|
|
12
|
+
markCommitSummarized(graphId: string, commitHash: string): void;
|
|
13
|
+
isCommitSummarized(graphId: string, commitHash: string): boolean;
|
|
14
|
+
findLastSummarizedAncestor(graphId: string, commitHash: string, repoPath: string): Promise<string | undefined>;
|
|
15
|
+
saveNodeSummaries(graphId: string, commitHash: string, nodeUpdates: Map<string, {
|
|
16
|
+
technicalSummary: string;
|
|
17
|
+
businessSummary: string;
|
|
18
|
+
security: {
|
|
19
|
+
severity: "none" | "low" | "medium" | "high";
|
|
20
|
+
summary: string;
|
|
21
|
+
};
|
|
22
|
+
summaryModel: string;
|
|
23
|
+
summarizedAt: string;
|
|
24
|
+
}>): void;
|
|
25
|
+
getCheckpointPath(graphId: string, commitHash: string): string;
|
|
26
|
+
removeFromSummarizedCommits(graphId: string, commitHash: string): void;
|
|
27
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { SummaryCheckpoint } from "./types.js";
|
|
2
|
+
export declare function loadCheckpoint(graphId: string, commitHash: string): SummaryCheckpoint | undefined;
|
|
3
|
+
export declare function saveCheckpoint(checkpoint: SummaryCheckpoint): void;
|
|
4
|
+
export declare function deleteCheckpoint(graphId: string, commitHash: string): void;
|
|
5
|
+
export declare function createCheckpoint(graphId: string, commitHash: string, nodeOrder: string[][], cycleGroups: SummaryCheckpoint["cycleGroups"], fileNodes: string[]): SummaryCheckpoint;
|
|
6
|
+
export type ResumePhase = "nodes" | "cycles" | "files" | "done";
|
|
7
|
+
export interface ResumePoint {
|
|
8
|
+
phase: ResumePhase;
|
|
9
|
+
index: number;
|
|
10
|
+
}
|
|
11
|
+
export declare function getResumePoint(checkpoint: SummaryCheckpoint): ResumePoint;
|
|
12
|
+
export declare function markLevelCompleted(checkpoint: SummaryCheckpoint, levelIndex: number): void;
|
|
13
|
+
export declare function markCycleGroupCompleted(checkpoint: SummaryCheckpoint, groupIndex: number): void;
|
|
14
|
+
export declare function markFileNodeCompleted(checkpoint: SummaryCheckpoint, index: number): void;
|
|
15
|
+
export declare function markFileNodeBatchCompleted(checkpoint: SummaryCheckpoint, batchEnd: number, count: number): void;
|