memorix 1.8.6 → 1.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +1 -1
- package/README.zh-CN.md +1 -1
- package/dist/cli/index.js +2208 -1160
- package/dist/dashboard/static/app.js +51 -3
- package/dist/dashboard/static/style.css +88 -0
- package/dist/index.js +1726 -736
- package/dist/maintenance-runner.js +855 -218
- package/dist/media-audio-runner.js +856 -219
- package/dist/media-video-runner.js +854 -217
- package/dist/memcode-runtime/CHANGELOG.md +33 -0
- package/dist/sdk.js +1716 -726
- package/dist/vector-backfill-runner.js +853 -216
- package/docs/1.8.7-CODEGRAPH.md +166 -0
- package/docs/API_REFERENCE.md +23 -2
- package/docs/ARCHITECTURE.md +13 -0
- package/docs/README.md +4 -1
- package/package.json +3 -2
- package/plugins/antigravity/memorix/plugin.json +1 -1
- package/plugins/claude/memorix/.claude-plugin/plugin.json +1 -1
- package/plugins/codebuddy/memorix/.codebuddy-plugin/plugin.json +1 -1
- package/plugins/codex/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/copilot/memorix/plugin.json +1 -1
- package/plugins/gemini/memorix/gemini-extension.json +1 -1
- package/plugins/hermes/memorix/plugin.yaml +1 -1
- package/plugins/omp/memorix/package.json +1 -1
- package/plugins/openclaw/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/pi/memorix/package.json +1 -1
- package/server.json +2 -2
- package/src/cli/commands/codegraph.ts +25 -11
- package/src/cli/commands/doctor.ts +11 -2
- package/src/cli/commands/serve-http.ts +27 -1
- package/src/codegraph/auto-context.ts +19 -10
- package/src/codegraph/context-pack.ts +20 -6
- package/src/codegraph/dashboard.ts +82 -0
- package/src/codegraph/external-provider.ts +10 -1
- package/src/codegraph/freshness.ts +1 -1
- package/src/codegraph/lite-provider.ts +81 -0
- package/src/codegraph/project-context.ts +18 -5
- package/src/codegraph/scip-provider.ts +181 -0
- package/src/codegraph/semantic-context.ts +139 -0
- package/src/codegraph/semantic-provider.ts +404 -0
- package/src/codegraph/store.ts +253 -18
- package/src/codegraph/types.ts +25 -3
- package/src/dashboard/server.ts +6 -0
- package/src/dashboard/static/style.css +88 -0
- package/src/knowledge/workset.ts +46 -44
- package/src/server/mcp-binding-store.ts +17 -0
- package/src/store/sqlite-db.ts +13 -0
|
@@ -19,7 +19,8 @@ import {
|
|
|
19
19
|
inspectExternalCodeGraph,
|
|
20
20
|
type ExternalCodeGraphRunner,
|
|
21
21
|
} from './external-provider.js';
|
|
22
|
-
import type {
|
|
22
|
+
import type { CodeGraphOutline, CodeGraphProviderQuality } from './types.js';
|
|
23
|
+
import { buildSemanticContext, qualityWithPersistedSemantic } from './semantic-context.js';
|
|
23
24
|
import {
|
|
24
25
|
buildProjectContextExplain,
|
|
25
26
|
type ProjectContextExplain,
|
|
@@ -341,10 +342,18 @@ export async function buildAutoProjectContext(input: {
|
|
|
341
342
|
}
|
|
342
343
|
}
|
|
343
344
|
const sourceSets = lensSourceSets({ task, lens, explain });
|
|
344
|
-
let
|
|
345
|
+
let semanticOutline: CodeGraphOutline | undefined;
|
|
345
346
|
let externalCaution: string | undefined;
|
|
346
347
|
let providerQuality: CodeGraphProviderQuality;
|
|
347
348
|
if (task) {
|
|
349
|
+
const internalOutline = buildSemanticContext({
|
|
350
|
+
store,
|
|
351
|
+
projectId: input.project.id,
|
|
352
|
+
task,
|
|
353
|
+
preferredPaths: overview.suggestedReads,
|
|
354
|
+
maxNodes: 8,
|
|
355
|
+
maxEdges: 12,
|
|
356
|
+
});
|
|
348
357
|
const external = await getExternalCodeGraphContext({
|
|
349
358
|
projectRoot: input.project.rootPath,
|
|
350
359
|
task,
|
|
@@ -354,9 +363,9 @@ export async function buildAutoProjectContext(input: {
|
|
|
354
363
|
timeoutMs: codegraphConfig.externalTimeoutMs,
|
|
355
364
|
...(input.externalRunner ? { runner: input.externalRunner } : {}),
|
|
356
365
|
});
|
|
357
|
-
|
|
358
|
-
externalCaution = external.caution;
|
|
359
|
-
providerQuality = external.quality;
|
|
366
|
+
semanticOutline = external.outline ?? internalOutline;
|
|
367
|
+
externalCaution = external.outline ? external.caution : internalOutline ? undefined : external.caution;
|
|
368
|
+
providerQuality = qualityWithPersistedSemantic(external.quality, store.status(input.project.id));
|
|
360
369
|
} else {
|
|
361
370
|
const external = await inspectExternalCodeGraph({
|
|
362
371
|
projectRoot: input.project.rootPath,
|
|
@@ -365,17 +374,17 @@ export async function buildAutoProjectContext(input: {
|
|
|
365
374
|
timeoutMs: codegraphConfig.externalTimeoutMs,
|
|
366
375
|
...(input.externalRunner ? { runner: input.externalRunner } : {}),
|
|
367
376
|
});
|
|
368
|
-
providerQuality = external.quality;
|
|
377
|
+
providerQuality = qualityWithPersistedSemantic(external.quality, store.status(input.project.id));
|
|
369
378
|
}
|
|
370
|
-
const externalStartHere =
|
|
371
|
-
?
|
|
379
|
+
const externalStartHere = semanticOutline
|
|
380
|
+
? semanticOutline.entryPoints.map(entry => entry.path)
|
|
372
381
|
: [];
|
|
373
382
|
const startHere = [...new Set([
|
|
374
|
-
...externalStartHere,
|
|
375
383
|
...rankLensPaths([
|
|
376
384
|
...existingLensCandidates(input.project.rootPath, lens),
|
|
377
385
|
...overview.suggestedReads,
|
|
378
386
|
], lens, task),
|
|
387
|
+
...externalStartHere,
|
|
379
388
|
])].slice(0, 5);
|
|
380
389
|
const runtimeCautions: WorksetCaution[] = [];
|
|
381
390
|
if (refresh.reason === 'queued') {
|
|
@@ -434,7 +443,7 @@ export async function buildAutoProjectContext(input: {
|
|
|
434
443
|
...(alwaysOn.profile.length > 0 || alwaysOn.state || alwaysOn.durable.length > 0
|
|
435
444
|
? { alwaysOn }
|
|
436
445
|
: {}),
|
|
437
|
-
...(
|
|
446
|
+
...(semanticOutline ? { semanticCode: semanticOutline } : {}),
|
|
438
447
|
providerQuality,
|
|
439
448
|
currentFacts: worksetFactLines(currentFacts),
|
|
440
449
|
...(continuation ? { continuation } : {}),
|
|
@@ -3,13 +3,14 @@ import { buildTaskWorkset, type TaskWorkset, type WorksetCaution } from '../know
|
|
|
3
3
|
import type {
|
|
4
4
|
CodeFile,
|
|
5
5
|
CodeGraphProviderQuality,
|
|
6
|
+
CodeGraphOutline,
|
|
6
7
|
CodeRefStatus,
|
|
7
8
|
CodeSymbol,
|
|
8
|
-
ExternalCodeGraphOutline,
|
|
9
9
|
ObservationCodeRef,
|
|
10
10
|
} from './types.js';
|
|
11
11
|
import type { CodeGraphStore } from './store.js';
|
|
12
12
|
import { isCodeGraphExcludedPath } from './exclude.js';
|
|
13
|
+
import { buildSemanticContext } from './semantic-context.js';
|
|
13
14
|
import type { AgentTarget, ObservationReader } from '../types.js';
|
|
14
15
|
|
|
15
16
|
export interface ContextPackMemory {
|
|
@@ -43,6 +44,7 @@ export interface ContextPack {
|
|
|
43
44
|
warnings: ContextPackWarning[];
|
|
44
45
|
suggestedReads: string[];
|
|
45
46
|
suggestedVerification: string[];
|
|
47
|
+
semanticCode?: CodeGraphOutline;
|
|
46
48
|
/** 1.2 bounded task selection, added without removing legacy pack fields. */
|
|
47
49
|
workset?: TaskWorkset;
|
|
48
50
|
}
|
|
@@ -67,6 +69,7 @@ export interface AssembleContextPackInput {
|
|
|
67
69
|
symbols: CodeSymbol[];
|
|
68
70
|
suggestedVerification?: string[];
|
|
69
71
|
exclude?: string[];
|
|
72
|
+
semanticCode?: CodeGraphOutline;
|
|
70
73
|
}
|
|
71
74
|
|
|
72
75
|
function uniq<T>(items: T[]): T[] {
|
|
@@ -151,6 +154,14 @@ export function assembleContextPackForTask(input: {
|
|
|
151
154
|
const fileIds = new Set(refs.map(ref => ref.fileId).filter(Boolean));
|
|
152
155
|
const files = input.store.listFiles(input.projectId).filter(file => fileIds.has(file.id));
|
|
153
156
|
const symbols = files.flatMap(file => input.store.listSymbolsForFile(file.id));
|
|
157
|
+
const semanticCode = buildSemanticContext({
|
|
158
|
+
store: input.store,
|
|
159
|
+
projectId: input.projectId,
|
|
160
|
+
task: input.task,
|
|
161
|
+
preferredPaths: files.map(file => file.path),
|
|
162
|
+
maxNodes: 8,
|
|
163
|
+
maxEdges: 12,
|
|
164
|
+
});
|
|
154
165
|
|
|
155
166
|
return assembleContextPack({
|
|
156
167
|
task: input.task,
|
|
@@ -160,6 +171,7 @@ export function assembleContextPackForTask(input: {
|
|
|
160
171
|
symbols,
|
|
161
172
|
suggestedVerification: input.suggestedVerification,
|
|
162
173
|
exclude: input.exclude,
|
|
174
|
+
...(semanticCode ? { semanticCode } : {}),
|
|
163
175
|
});
|
|
164
176
|
}
|
|
165
177
|
|
|
@@ -252,6 +264,7 @@ export function assembleContextPack(input: AssembleContextPackInput): ContextPac
|
|
|
252
264
|
hasWarnings: warnings.length > 0,
|
|
253
265
|
hasUnboundMemories: unboundMemories.length > 0,
|
|
254
266
|
}),
|
|
267
|
+
...(input.semanticCode ? { semanticCode: input.semanticCode } : {}),
|
|
255
268
|
};
|
|
256
269
|
}
|
|
257
270
|
|
|
@@ -274,14 +287,15 @@ export async function attachTaskWorkset(input: {
|
|
|
274
287
|
worktreeState?: 'clean' | 'dirty' | 'unavailable';
|
|
275
288
|
incomplete?: boolean;
|
|
276
289
|
};
|
|
277
|
-
semanticCode?:
|
|
290
|
+
semanticCode?: CodeGraphOutline;
|
|
278
291
|
providerQuality?: CodeGraphProviderQuality;
|
|
279
292
|
runtimeCautions?: WorksetCaution[];
|
|
280
293
|
reader?: ObservationReader;
|
|
281
294
|
agent?: AgentTarget;
|
|
282
295
|
}): Promise<ContextPack> {
|
|
283
|
-
const
|
|
284
|
-
|
|
296
|
+
const semanticCode = input.semanticCode ?? input.pack.semanticCode;
|
|
297
|
+
const semanticStartHere = semanticCode
|
|
298
|
+
? semanticCode.entryPoints.map(entry => entry.path)
|
|
285
299
|
: [];
|
|
286
300
|
const workset = await buildTaskWorkset({
|
|
287
301
|
projectId: input.projectId,
|
|
@@ -292,7 +306,7 @@ export async function attachTaskWorkset(input: {
|
|
|
292
306
|
currentFacts: input.currentFacts ?? [],
|
|
293
307
|
...(input.codeState ? { codeState: input.codeState } : {}),
|
|
294
308
|
startHere: uniq([...semanticStartHere, ...input.pack.suggestedReads]),
|
|
295
|
-
...(
|
|
309
|
+
...(semanticCode ? { semanticCode } : {}),
|
|
296
310
|
...(input.providerQuality ? { providerQuality: input.providerQuality } : {}),
|
|
297
311
|
reliableMemory: input.pack.memories
|
|
298
312
|
.filter(memory => memory.status === 'current')
|
|
@@ -323,7 +337,7 @@ export async function attachTaskWorkset(input: {
|
|
|
323
337
|
...(input.reader ? { reader: input.reader } : {}),
|
|
324
338
|
deliveryTarget: 'context-pack',
|
|
325
339
|
});
|
|
326
|
-
return { ...input.pack, workset };
|
|
340
|
+
return { ...input.pack, ...(semanticCode ? { semanticCode } : {}), workset };
|
|
327
341
|
}
|
|
328
342
|
|
|
329
343
|
export function buildContextPackPrompt(pack: ContextPack): string {
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { CodeGraphStore } from './store.js';
|
|
2
|
+
|
|
3
|
+
export interface CodeGraphDashboardStatus {
|
|
4
|
+
projectId: string;
|
|
5
|
+
provider: 'lite';
|
|
6
|
+
state: 'not-indexed' | 'ready' | 'partial';
|
|
7
|
+
files: number;
|
|
8
|
+
symbols: number;
|
|
9
|
+
edges: number;
|
|
10
|
+
refs: number;
|
|
11
|
+
indexedAt?: string;
|
|
12
|
+
semantic: {
|
|
13
|
+
files: number;
|
|
14
|
+
symbols: number;
|
|
15
|
+
edges: number;
|
|
16
|
+
parserErrors: number;
|
|
17
|
+
};
|
|
18
|
+
lite: {
|
|
19
|
+
files: number;
|
|
20
|
+
symbols: number;
|
|
21
|
+
edges: number;
|
|
22
|
+
};
|
|
23
|
+
parserErrorPaths: string[];
|
|
24
|
+
snapshot?: {
|
|
25
|
+
id: string;
|
|
26
|
+
sourceEpoch: number;
|
|
27
|
+
worktreeState: string;
|
|
28
|
+
incomplete: boolean;
|
|
29
|
+
changedPathCount: number;
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Read-only dashboard projection; it never refreshes or mutates the index. */
|
|
34
|
+
export async function getCodeGraphDashboardStatus(
|
|
35
|
+
dataDir: string,
|
|
36
|
+
projectId: string,
|
|
37
|
+
): Promise<CodeGraphDashboardStatus> {
|
|
38
|
+
const store = new CodeGraphStore();
|
|
39
|
+
await store.init(dataDir);
|
|
40
|
+
const status = store.status(projectId);
|
|
41
|
+
const files = store.listFiles(projectId);
|
|
42
|
+
const semanticFiles = files.filter(file => file.source === 'typescript-compiler');
|
|
43
|
+
const parserErrorPaths = files
|
|
44
|
+
.filter(file => Boolean(file.parserError))
|
|
45
|
+
.map(file => file.path)
|
|
46
|
+
.slice(0, 8);
|
|
47
|
+
const semanticIndexed = status.semanticSymbols > 0 || status.semanticEdges > 0;
|
|
48
|
+
const snapshot = status.latestSnapshot;
|
|
49
|
+
return {
|
|
50
|
+
projectId,
|
|
51
|
+
provider: 'lite',
|
|
52
|
+
state: status.parserErrors > 0 ? 'partial' : semanticIndexed ? 'ready' : 'not-indexed',
|
|
53
|
+
files: status.files,
|
|
54
|
+
symbols: status.symbols,
|
|
55
|
+
edges: status.edges,
|
|
56
|
+
refs: status.refs,
|
|
57
|
+
...(status.indexedAt ? { indexedAt: status.indexedAt } : {}),
|
|
58
|
+
semantic: {
|
|
59
|
+
files: semanticFiles.length,
|
|
60
|
+
symbols: status.semanticSymbols,
|
|
61
|
+
edges: status.semanticEdges,
|
|
62
|
+
parserErrors: status.parserErrors,
|
|
63
|
+
},
|
|
64
|
+
lite: {
|
|
65
|
+
files: files.length - semanticFiles.length,
|
|
66
|
+
symbols: Math.max(0, status.symbols - status.semanticSymbols),
|
|
67
|
+
edges: Math.max(0, status.edges - status.semanticEdges),
|
|
68
|
+
},
|
|
69
|
+
parserErrorPaths,
|
|
70
|
+
...(snapshot ? {
|
|
71
|
+
snapshot: {
|
|
72
|
+
id: snapshot.id,
|
|
73
|
+
sourceEpoch: snapshot.sourceEpoch,
|
|
74
|
+
worktreeState: snapshot.worktreeState,
|
|
75
|
+
incomplete: snapshot.completeness.skippedOversizedFiles > 0
|
|
76
|
+
|| (snapshot.completeness.unreadableFiles ?? 0) > 0
|
|
77
|
+
|| snapshot.completeness.removalScanDeferred,
|
|
78
|
+
changedPathCount: snapshot.changedPathCount,
|
|
79
|
+
},
|
|
80
|
+
} : {}),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
@@ -4,6 +4,7 @@ import { delimiter, dirname, extname, isAbsolute, join, relative, resolve, sep }
|
|
|
4
4
|
import { sanitizeCredentials } from '../memory/secret-filter.js';
|
|
5
5
|
import { isCodeGraphExcludedPath } from './exclude.js';
|
|
6
6
|
import { normalizeCodePath } from './ids.js';
|
|
7
|
+
import { normalizeScipOutline } from './scip-provider.js';
|
|
7
8
|
import type {
|
|
8
9
|
CodeGraphExternalMode,
|
|
9
10
|
CodeGraphProviderQuality,
|
|
@@ -327,7 +328,7 @@ const defaultRunner: ExternalCodeGraphRunner = {
|
|
|
327
328
|
function providerQuality(input: {
|
|
328
329
|
mode: CodeGraphExternalMode;
|
|
329
330
|
health: ExternalCodeGraphHealth;
|
|
330
|
-
selected?: 'lite' | 'external';
|
|
331
|
+
selected?: 'lite' | 'semantic' | 'external';
|
|
331
332
|
}): CodeGraphProviderQuality {
|
|
332
333
|
const selected = input.selected ?? 'lite';
|
|
333
334
|
return {
|
|
@@ -346,6 +347,12 @@ function providerQuality(input: {
|
|
|
346
347
|
supportedLanguages: LITE_SUPPORTED_LANGUAGES,
|
|
347
348
|
},
|
|
348
349
|
external: input.health,
|
|
350
|
+
semantic: {
|
|
351
|
+
state: 'not-indexed',
|
|
352
|
+
symbols: 0,
|
|
353
|
+
edges: 0,
|
|
354
|
+
parserErrors: 0,
|
|
355
|
+
},
|
|
349
356
|
};
|
|
350
357
|
}
|
|
351
358
|
|
|
@@ -637,6 +644,8 @@ function externalSymbol(raw: RawExternalNode): ExternalCodeGraphSymbol {
|
|
|
637
644
|
}
|
|
638
645
|
|
|
639
646
|
function parseOutline(value: unknown, projectRoot: string, exclude?: string[]): ExternalCodeGraphOutline | undefined {
|
|
647
|
+
const scipOutline = normalizeScipOutline(value, { projectRoot, exclude });
|
|
648
|
+
if (scipOutline) return scipOutline as ExternalCodeGraphOutline;
|
|
640
649
|
if (!isRecord(value) || !Array.isArray(value.entryPoints) || !Array.isArray(value.nodes)
|
|
641
650
|
|| !Array.isArray(value.edges) || !Array.isArray(value.relatedFiles)) return undefined;
|
|
642
651
|
if ('codeBlocks' in value && (!Array.isArray(value.codeBlocks) || value.codeBlocks.length > 0)) return undefined;
|
|
@@ -20,7 +20,7 @@ export function evaluateCodeRefFreshness(
|
|
|
20
20
|
|
|
21
21
|
if (ref.capturedSymbolHash && symbol?.contentHash) {
|
|
22
22
|
if (symbol.contentHash !== ref.capturedSymbolHash) {
|
|
23
|
-
return { status: '
|
|
23
|
+
return { status: 'suspect', reason: 'referenced symbol content changed' };
|
|
24
24
|
}
|
|
25
25
|
return { status: 'current', reason: 'referenced symbol content still matches' };
|
|
26
26
|
}
|
|
@@ -6,6 +6,7 @@ import { collectCodeStateSnapshot } from './code-state.js';
|
|
|
6
6
|
import { makeCodeEdgeId, makeCodeFileId, makeCodeSymbolId, normalizeCodePath } from './ids.js';
|
|
7
7
|
import { isCodeGraphExcludedPath, normalizeCodeGraphExcludePatterns } from './exclude.js';
|
|
8
8
|
import { CodeGraphStore, type CodeGraphFileDelta } from './store.js';
|
|
9
|
+
import { buildTypeScriptSemanticIndex, isSemanticLanguage } from './semantic-provider.js';
|
|
9
10
|
|
|
10
11
|
export interface LiteIndexOptions {
|
|
11
12
|
projectId: string;
|
|
@@ -13,6 +14,8 @@ export interface LiteIndexOptions {
|
|
|
13
14
|
exclude?: string[];
|
|
14
15
|
maxFiles?: number;
|
|
15
16
|
maxFileBytes?: number;
|
|
17
|
+
/** Injectable only for deterministic failure-path tests. */
|
|
18
|
+
semanticIndexer?: typeof buildTypeScriptSemanticIndex;
|
|
16
19
|
}
|
|
17
20
|
|
|
18
21
|
export interface LiteIndexResult {
|
|
@@ -35,6 +38,17 @@ export interface LiteRefreshResult {
|
|
|
35
38
|
unreadableFiles: number;
|
|
36
39
|
removalScanDeferred: boolean;
|
|
37
40
|
snapshot: CodeStateSnapshot;
|
|
41
|
+
semantic?: {
|
|
42
|
+
state: 'ready' | 'partial' | 'failed';
|
|
43
|
+
parsedFiles: number;
|
|
44
|
+
affectedFiles: number;
|
|
45
|
+
symbols: number;
|
|
46
|
+
edges: number;
|
|
47
|
+
parserErrors: number;
|
|
48
|
+
unresolvedCalls: number;
|
|
49
|
+
durationMs: number;
|
|
50
|
+
error?: string;
|
|
51
|
+
};
|
|
38
52
|
}
|
|
39
53
|
|
|
40
54
|
export const DEFAULT_CODEGRAPH_MAX_FILE_BYTES = 2 * 1024 * 1024;
|
|
@@ -257,6 +271,7 @@ function extractSymbols(projectId: string, file: CodeFile, text: string, indexed
|
|
|
257
271
|
signature: match[0].slice(0, 160),
|
|
258
272
|
contentHash: hashText(match[0]),
|
|
259
273
|
indexedAt,
|
|
274
|
+
source: 'lite-regex',
|
|
260
275
|
});
|
|
261
276
|
}
|
|
262
277
|
}
|
|
@@ -284,6 +299,7 @@ function extractImportEdges(projectId: string, file: CodeFile, text: string, ind
|
|
|
284
299
|
confidence: 0.7,
|
|
285
300
|
evidence: line,
|
|
286
301
|
indexedAt,
|
|
302
|
+
source: 'lite-regex',
|
|
287
303
|
});
|
|
288
304
|
}
|
|
289
305
|
continue;
|
|
@@ -297,6 +313,7 @@ function extractImportEdges(projectId: string, file: CodeFile, text: string, ind
|
|
|
297
313
|
confidence: 0.7,
|
|
298
314
|
evidence: target,
|
|
299
315
|
indexedAt,
|
|
316
|
+
source: 'lite-regex',
|
|
300
317
|
});
|
|
301
318
|
}
|
|
302
319
|
}
|
|
@@ -330,6 +347,7 @@ function indexFileLite(
|
|
|
330
347
|
mtimeMs: Number(stat.mtimeMs),
|
|
331
348
|
sizeBytes: Number(stat.size),
|
|
332
349
|
indexedAt,
|
|
350
|
+
source: 'lite-regex',
|
|
333
351
|
};
|
|
334
352
|
return {
|
|
335
353
|
delta: {
|
|
@@ -395,6 +413,7 @@ export async function refreshProjectLite(
|
|
|
395
413
|
const indexedAt = new Date().toISOString();
|
|
396
414
|
const paths = walk(options.projectRoot, exclude, maxFiles);
|
|
397
415
|
const existingByPath = new Map(store.listFiles(options.projectId).map((file) => [file.path, file]));
|
|
416
|
+
const changedPaths = new Set<string>();
|
|
398
417
|
const changed: CodeGraphFileDelta[] = [];
|
|
399
418
|
const metadataOnly: CodeFile[] = [];
|
|
400
419
|
const seenPaths = new Set<string>();
|
|
@@ -420,6 +439,7 @@ export async function refreshProjectLite(
|
|
|
420
439
|
if (sizeBytes > maxFileBytes) {
|
|
421
440
|
skippedOversizedFiles++;
|
|
422
441
|
if (existing) oversizedExistingFileIds.add(existing.id);
|
|
442
|
+
changedPaths.add(rel);
|
|
423
443
|
continue;
|
|
424
444
|
}
|
|
425
445
|
if (existing && existing.mtimeMs === mtimeMs && existing.sizeBytes === sizeBytes) {
|
|
@@ -438,6 +458,7 @@ export async function refreshProjectLite(
|
|
|
438
458
|
unchangedFiles++;
|
|
439
459
|
} else {
|
|
440
460
|
changed.push(delta);
|
|
461
|
+
changedPaths.add(rel);
|
|
441
462
|
}
|
|
442
463
|
}
|
|
443
464
|
|
|
@@ -451,11 +472,70 @@ export async function refreshProjectLite(
|
|
|
451
472
|
.filter((file) => !seenPaths.has(file.path))
|
|
452
473
|
.map((file) => file.id)),
|
|
453
474
|
])];
|
|
475
|
+
for (const file of existingByPath.values()) {
|
|
476
|
+
if (removedFileIds.includes(file.id)) changedPaths.add(file.path);
|
|
477
|
+
}
|
|
478
|
+
// An export or symbol change can invalidate the semantic edges of every
|
|
479
|
+
// local importer. Capture those paths before applyFileDeltas clears them.
|
|
480
|
+
const affectedSemanticPaths = new Set([
|
|
481
|
+
...changedPaths,
|
|
482
|
+
...store.listImportDependentPaths(
|
|
483
|
+
options.projectId,
|
|
484
|
+
[...changedPaths],
|
|
485
|
+
changed.map(delta => delta.file.path),
|
|
486
|
+
),
|
|
487
|
+
]);
|
|
454
488
|
store.applyFileDeltas(options.projectId, {
|
|
455
489
|
changed,
|
|
456
490
|
metadataOnly,
|
|
457
491
|
removedFileIds,
|
|
458
492
|
});
|
|
493
|
+
const currentFiles = store.listFiles(options.projectId);
|
|
494
|
+
const beforeSemantic = store.status(options.projectId).semanticSymbols;
|
|
495
|
+
const semanticPaths = beforeSemantic === 0
|
|
496
|
+
? currentFiles.filter(file => isSemanticLanguage(file.language)).map(file => file.path)
|
|
497
|
+
: currentFiles
|
|
498
|
+
.filter(file => isSemanticLanguage(file.language) && affectedSemanticPaths.has(file.path))
|
|
499
|
+
.map(file => file.path);
|
|
500
|
+
let semanticSummary: LiteRefreshResult['semantic'];
|
|
501
|
+
if (semanticPaths.length > 0) {
|
|
502
|
+
const semanticIndexer = options.semanticIndexer ?? buildTypeScriptSemanticIndex;
|
|
503
|
+
const semanticInput = {
|
|
504
|
+
projectId: options.projectId,
|
|
505
|
+
projectRoot: options.projectRoot,
|
|
506
|
+
files: currentFiles,
|
|
507
|
+
exclude,
|
|
508
|
+
includePaths: semanticPaths,
|
|
509
|
+
};
|
|
510
|
+
try {
|
|
511
|
+
const semantic = semanticIndexer(semanticInput);
|
|
512
|
+
store.replaceSemanticSlice(options.projectId, semanticPaths, semantic);
|
|
513
|
+
semanticSummary = {
|
|
514
|
+
state: semantic.parserErrors > 0 ? 'partial' : 'ready',
|
|
515
|
+
parsedFiles: semantic.parsedFiles,
|
|
516
|
+
affectedFiles: semanticPaths.length,
|
|
517
|
+
symbols: semantic.symbols.length,
|
|
518
|
+
edges: semantic.edges.length,
|
|
519
|
+
parserErrors: semantic.parserErrors,
|
|
520
|
+
unresolvedCalls: semantic.unresolvedCalls,
|
|
521
|
+
durationMs: semantic.durationMs,
|
|
522
|
+
};
|
|
523
|
+
} catch (error) {
|
|
524
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
525
|
+
store.markSemanticSliceFailed(options.projectId, semanticPaths, message);
|
|
526
|
+
semanticSummary = {
|
|
527
|
+
state: 'failed',
|
|
528
|
+
parsedFiles: 0,
|
|
529
|
+
affectedFiles: semanticPaths.length,
|
|
530
|
+
symbols: 0,
|
|
531
|
+
edges: 0,
|
|
532
|
+
parserErrors: semanticPaths.length,
|
|
533
|
+
unresolvedCalls: 0,
|
|
534
|
+
durationMs: 0,
|
|
535
|
+
error: message.slice(0, 500),
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
}
|
|
459
539
|
const completeness = {
|
|
460
540
|
scannedFiles: paths.length,
|
|
461
541
|
maxFiles,
|
|
@@ -486,5 +566,6 @@ export async function refreshProjectLite(
|
|
|
486
566
|
unreadableFiles,
|
|
487
567
|
removalScanDeferred,
|
|
488
568
|
snapshot,
|
|
569
|
+
...(semanticSummary ? { semantic: semanticSummary } : {}),
|
|
489
570
|
};
|
|
490
571
|
}
|
|
@@ -134,17 +134,30 @@ function collectGraph(
|
|
|
134
134
|
const sources: ProjectContextSource[] = [];
|
|
135
135
|
const suggestedReads: string[] = [];
|
|
136
136
|
|
|
137
|
+
const severity: Record<CodeRefStatus, number> = { current: 1, suspect: 2, stale: 3, unbound: 0 };
|
|
138
|
+
const grouped = new Map<number, {
|
|
139
|
+
observation: ProjectContextObservation;
|
|
140
|
+
file?: CodeFile;
|
|
141
|
+
symbol?: CodeSymbol;
|
|
142
|
+
result: ReturnType<typeof evaluateCodeRefFreshness>;
|
|
143
|
+
}>();
|
|
137
144
|
for (const ref of refs) {
|
|
138
145
|
const file = ref.fileId ? filesById.get(ref.fileId) : undefined;
|
|
139
146
|
const symbol = ref.symbolId ? symbolsById.get(ref.symbolId) : undefined;
|
|
140
147
|
const result = evaluateCodeRefFreshness(ref, file, symbol);
|
|
141
|
-
freshness[result.status] += 1;
|
|
142
|
-
|
|
143
148
|
const observation = observationsById.get(ref.observationId);
|
|
144
149
|
if (!observation) continue;
|
|
145
|
-
const
|
|
146
|
-
if (result.status
|
|
147
|
-
|
|
150
|
+
const existing = grouped.get(ref.observationId);
|
|
151
|
+
if (!existing || severity[result.status] > severity[existing.result.status]
|
|
152
|
+
|| (severity[result.status] === severity[existing.result.status] && symbol && !existing.symbol)) {
|
|
153
|
+
grouped.set(ref.observationId, { observation, file, symbol, result });
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
for (const { observation, file, symbol, result } of grouped.values()) {
|
|
158
|
+
freshness[result.status] += 1;
|
|
159
|
+
if (file && isCodeGraphExcludedPath(file.path, exclude)) continue;
|
|
160
|
+
if (result.status === 'current' && file) suggestedReads.push(file.path);
|
|
148
161
|
sources.push({
|
|
149
162
|
observationId: observation.id,
|
|
150
163
|
title: observation.title,
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { isAbsolute, relative, resolve } from 'node:path';
|
|
2
|
+
import { sanitizeCredentials } from '../memory/secret-filter.js';
|
|
3
|
+
import { isCodeGraphExcludedPath } from './exclude.js';
|
|
4
|
+
import { normalizeCodePath } from './ids.js';
|
|
5
|
+
import type { CodeGraphOutline, ExternalCodeGraphRelation, ExternalCodeGraphSymbol } from './types.js';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* SCIP is intentionally an input boundary here, not a second persistence
|
|
9
|
+
* format. The normalized outline is bounded and remains separate from the
|
|
10
|
+
* canonical local CodeGraph tables.
|
|
11
|
+
*/
|
|
12
|
+
export interface ScipOutlineOptions {
|
|
13
|
+
projectRoot: string;
|
|
14
|
+
exclude?: string[];
|
|
15
|
+
maxNodes?: number;
|
|
16
|
+
maxEdges?: number;
|
|
17
|
+
maxFiles?: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const DEFAULT_MAX_NODES = 16;
|
|
21
|
+
const DEFAULT_MAX_EDGES = 24;
|
|
22
|
+
const DEFAULT_MAX_FILES = 12;
|
|
23
|
+
|
|
24
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
25
|
+
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function text(value: unknown, max = 240): string | undefined {
|
|
29
|
+
if (typeof value !== 'string') return undefined;
|
|
30
|
+
const normalized = sanitizeCredentials(value).replace(/[\u0000-\u001f]/g, ' ').replace(/\s+/g, ' ').trim();
|
|
31
|
+
return normalized && normalized.length <= max ? normalized : undefined;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function safeRelativePath(projectRoot: string, value: unknown, exclude?: string[]): string | undefined {
|
|
35
|
+
const raw = text(value, 1_024);
|
|
36
|
+
if (!raw || isAbsolute(raw)) return undefined;
|
|
37
|
+
const absolute = resolve(projectRoot, raw);
|
|
38
|
+
const fromRoot = relative(resolve(projectRoot), absolute);
|
|
39
|
+
if (!fromRoot || fromRoot === '..' || fromRoot.startsWith('..\\') || fromRoot.startsWith('../') || isAbsolute(fromRoot)) return undefined;
|
|
40
|
+
const normalized = normalizeCodePath(fromRoot);
|
|
41
|
+
return isCodeGraphExcludedPath(normalized, exclude) ? undefined : normalized;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function integer(value: unknown): number | undefined {
|
|
45
|
+
return typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : undefined;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function lineFromRange(value: unknown): number | undefined {
|
|
49
|
+
if (!Array.isArray(value) || value.length === 0) return undefined;
|
|
50
|
+
const line = integer(value[0]);
|
|
51
|
+
return line === undefined ? undefined : line + 1;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function symbolKind(value: unknown): string {
|
|
55
|
+
if (typeof value === 'string' && value.trim()) return value.trim().slice(0, 80);
|
|
56
|
+
if (typeof value === 'number' && Number.isInteger(value)) return `scip-kind-${value}`;
|
|
57
|
+
return 'symbol';
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function symbolName(symbolId: string, value: unknown): string {
|
|
61
|
+
const displayName = text(value, 180);
|
|
62
|
+
if (displayName) return displayName;
|
|
63
|
+
const pieces = symbolId.split(/[/#.:]/).filter(Boolean);
|
|
64
|
+
return (pieces[pieces.length - 1] || symbolId).slice(0, 180);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function relationKind(relationship: Record<string, unknown>): string {
|
|
68
|
+
if (relationship.isImplementation === true || relationship.is_implementation === true) return 'implements';
|
|
69
|
+
if (relationship.isDefinition === true || relationship.is_definition === true) return 'defines';
|
|
70
|
+
if (relationship.isReference === true || relationship.is_reference === true) return 'references';
|
|
71
|
+
return 'references';
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
interface ScipNodeInfo {
|
|
75
|
+
symbolId: string;
|
|
76
|
+
node: ExternalCodeGraphSymbol;
|
|
77
|
+
defined: boolean;
|
|
78
|
+
line?: number;
|
|
79
|
+
relationships: Array<{ target: string; kind: string }>;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Normalize a bounded `scip print --json` payload into Memorix's outline. */
|
|
83
|
+
export function normalizeScipOutline(value: unknown, options: ScipOutlineOptions): CodeGraphOutline | undefined {
|
|
84
|
+
if (!isRecord(value) || !Array.isArray(value.documents)) return undefined;
|
|
85
|
+
const maxNodes = Math.max(2, Math.min(64, Math.floor(options.maxNodes ?? DEFAULT_MAX_NODES)));
|
|
86
|
+
const maxEdges = Math.max(1, Math.min(96, Math.floor(options.maxEdges ?? DEFAULT_MAX_EDGES)));
|
|
87
|
+
const maxFiles = Math.max(1, Math.min(32, Math.floor(options.maxFiles ?? DEFAULT_MAX_FILES)));
|
|
88
|
+
const nodes = new Map<string, ScipNodeInfo>();
|
|
89
|
+
const relatedFiles = new Set<string>();
|
|
90
|
+
const definitions = new Set<string>();
|
|
91
|
+
|
|
92
|
+
for (const document of value.documents.slice(0, maxFiles * 4)) {
|
|
93
|
+
if (!isRecord(document)) continue;
|
|
94
|
+
const filePath = safeRelativePath(options.projectRoot, document.relativePath ?? document.relative_path, options.exclude);
|
|
95
|
+
if (!filePath || relatedFiles.size >= maxFiles && !relatedFiles.has(filePath)) continue;
|
|
96
|
+
relatedFiles.add(filePath);
|
|
97
|
+
const occurrences = Array.isArray(document.occurrences) ? document.occurrences : [];
|
|
98
|
+
const definitionLines = new Map<string, number>();
|
|
99
|
+
for (const occurrence of occurrences) {
|
|
100
|
+
if (!isRecord(occurrence)) continue;
|
|
101
|
+
const id = text(occurrence.symbol, 240);
|
|
102
|
+
const isDefinition = occurrence.isDefinition === true || occurrence.is_definition === true;
|
|
103
|
+
const rawRoles = occurrence.symbolRoles ?? occurrence.symbol_roles;
|
|
104
|
+
if (!id || (rawRoles === undefined && !isDefinition)) continue;
|
|
105
|
+
const roles = integer(rawRoles) ?? 0;
|
|
106
|
+
// SCIP SymbolRole.Definition is bit 1. Accept the explicit boolean used
|
|
107
|
+
// by a few JSON emitters as well, without trusting arbitrary roles.
|
|
108
|
+
if ((roles & 1) !== 0 || isDefinition) {
|
|
109
|
+
definitions.add(id);
|
|
110
|
+
const line = lineFromRange(occurrence.range);
|
|
111
|
+
if (line !== undefined) definitionLines.set(id, line);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const symbolInfos = Array.isArray(document.symbols) ? document.symbols : [];
|
|
116
|
+
for (const rawSymbol of symbolInfos) {
|
|
117
|
+
if (!isRecord(rawSymbol)) continue;
|
|
118
|
+
const id = text(rawSymbol.symbol, 240);
|
|
119
|
+
if (!id || nodes.has(id)) continue;
|
|
120
|
+
const relationships: Array<{ target: string; kind: string }> = [];
|
|
121
|
+
if (Array.isArray(rawSymbol.relationships)) {
|
|
122
|
+
for (const rawRelationship of rawSymbol.relationships.slice(0, 32)) {
|
|
123
|
+
if (!isRecord(rawRelationship)) continue;
|
|
124
|
+
const target = text(rawRelationship.symbol, 240);
|
|
125
|
+
if (target) relationships.push({ target, kind: relationKind(rawRelationship) });
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
nodes.set(id, {
|
|
129
|
+
symbolId: id,
|
|
130
|
+
node: {
|
|
131
|
+
id,
|
|
132
|
+
name: symbolName(id, rawSymbol.displayName ?? rawSymbol.display_name),
|
|
133
|
+
qualifiedName: text(rawSymbol.displayName ?? rawSymbol.display_name, 220),
|
|
134
|
+
kind: symbolKind(rawSymbol.kind),
|
|
135
|
+
path: filePath,
|
|
136
|
+
language: text(document.language, 80),
|
|
137
|
+
...(definitionLines.has(id) ? { startLine: definitionLines.get(id) } : {}),
|
|
138
|
+
},
|
|
139
|
+
defined: definitions.has(id),
|
|
140
|
+
...(definitionLines.has(id) ? { line: definitionLines.get(id) } : {}),
|
|
141
|
+
relationships,
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (nodes.size === 0) return undefined;
|
|
147
|
+
|
|
148
|
+
const selectedNodes = [...nodes.values()]
|
|
149
|
+
.sort((left, right) => Number(right.defined) - Number(left.defined) || left.node.path.localeCompare(right.node.path) || left.node.name.localeCompare(right.node.name))
|
|
150
|
+
.slice(0, maxNodes);
|
|
151
|
+
const selectedById = new Map(selectedNodes.map(item => [item.symbolId, item]));
|
|
152
|
+
const relations: ExternalCodeGraphRelation[] = [];
|
|
153
|
+
for (const source of selectedNodes) {
|
|
154
|
+
for (const relationship of source.relationships) {
|
|
155
|
+
const target = selectedById.get(relationship.target);
|
|
156
|
+
if (!target || relations.length >= maxEdges) continue;
|
|
157
|
+
relations.push({
|
|
158
|
+
from: source.node,
|
|
159
|
+
to: target.node,
|
|
160
|
+
kind: relationship.kind,
|
|
161
|
+
...(source.line ? { line: source.line } : {}),
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const entryPoints = selectedNodes
|
|
167
|
+
.filter(item => item.defined)
|
|
168
|
+
.slice(0, 5)
|
|
169
|
+
.map(item => item.node);
|
|
170
|
+
return {
|
|
171
|
+
provider: 'external',
|
|
172
|
+
entryPoints: entryPoints.length > 0 ? entryPoints : selectedNodes.slice(0, 5).map(item => item.node),
|
|
173
|
+
relations,
|
|
174
|
+
relatedFiles: [...new Set(selectedNodes.map(item => item.node.path))].slice(0, maxFiles),
|
|
175
|
+
stats: {
|
|
176
|
+
nodes: nodes.size,
|
|
177
|
+
edges: relations.length,
|
|
178
|
+
files: relatedFiles.size,
|
|
179
|
+
},
|
|
180
|
+
};
|
|
181
|
+
}
|