knodin 0.8.7 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cli.js +341 -91
- package/dist/fixtures/update-verification/config/application.xml +4 -0
- package/dist/fixtures/update-verification/dbt/manifest.json +17 -0
- package/dist/fixtures/update-verification/dbt/models/accounts.sql +1 -0
- package/dist/fixtures/update-verification/dbt/models/contacts.sql +1 -0
- package/dist/fixtures/update-verification/fixture.json +147 -0
- package/dist/fixtures/update-verification/force-app/main/default/classes/UpdateFixture.cls +14 -0
- package/dist/fixtures/update-verification/force-app/main/default/flows/Update_Verification.flow-meta.xml +9 -0
- package/dist/fixtures/update-verification/package.json +5 -0
- package/dist/fixtures/update-verification/prisma/schema.prisma +10 -0
- package/dist/fixtures/update-verification/sql/functions.sql +16 -0
- package/dist/fixtures/update-verification/src/CsharpFixture.cs +6 -0
- package/dist/fixtures/update-verification/src/JavaFixture.java +9 -0
- package/dist/fixtures/update-verification/src/javascript.js +7 -0
- package/dist/fixtures/update-verification/src/python_fixture.py +6 -0
- package/dist/fixtures/update-verification/src/typescript.ts +7 -0
- package/dist/fixtures/update-verification/terraform/main.tf +7 -0
- package/dist/fixtures/update-verification/terraform/modules/verified-child/main.tf +3 -0
- package/dist/src/agent-integration.js +26 -3
- package/dist/src/backup-retention.js +811 -0
- package/dist/src/cli-model.js +49 -8
- package/dist/src/docs-sections.js +1 -0
- package/dist/src/doctor.js +142 -12
- package/dist/src/engine/embeddings.js +8 -1
- package/dist/src/engine/index.js +1272 -645
- package/dist/src/engine/parse-pool-resources.js +96 -0
- package/dist/src/engine/parse-pool.js +352 -0
- package/dist/src/engine/parse-protocol.js +1 -0
- package/dist/src/engine/parse-worker.js +51 -0
- package/dist/src/engine/reflink-copy.js +23 -0
- package/dist/src/engine/seal-command.js +116 -0
- package/dist/src/engine/seal.js +270 -0
- package/dist/src/engine/sealed-open.js +116 -0
- package/dist/src/engine/sealed-query.js +49 -0
- package/dist/src/init.js +251 -183
- package/dist/src/manager-update.js +455 -0
- package/dist/src/release-preflight.js +106 -95
- package/dist/src/repair-lease.js +77 -8
- package/dist/src/response-budget.js +5 -1
- package/dist/src/server.js +18 -5
- package/dist/src/shared-index/compatibility.js +4 -2
- package/dist/src/shared-index/selection.js +1 -1
- package/dist/src/tools/knodin-tools.js +85 -13
- package/dist/src/update-ceremony.js +20 -24
- package/dist/src/update-coordination.js +158 -0
- package/dist/src/update-executor.js +355 -0
- package/dist/src/update-verifier.js +358 -0
- package/dist/src/worktree-seed.js +172 -0
- package/docs/BACKUP-RETENTION.md +54 -0
- package/docs/CLI.md +6 -0
- package/docs/DOCTOR-AND-UPDATES.md +44 -12
- package/docs/SHARED-INDEX-CONTRACT.md +9 -0
- package/docs/SIGNED-UPDATES.md +29 -22
- package/docs/releases/0.10.0.md +127 -0
- package/docs/releases/0.9.0.md +43 -0
- package/package.json +8 -1
- package/roadmap/competitive-roadmap.md +8 -0
package/dist/src/engine/index.js
CHANGED
|
@@ -25,16 +25,23 @@ import { runReadOnlyLspQuery } from "../lsp-readonly.js";
|
|
|
25
25
|
import { acquireRepairLease } from "../repair-lease.js";
|
|
26
26
|
import { analyzeResourceReachability, resourceFingerprint, } from "../resource-reachability.js";
|
|
27
27
|
import { contentFingerprint, writeStructuralSnapshot, } from "../structural-snapshot.js";
|
|
28
|
+
import { acquireLifecycleCoordination } from "../update-coordination.js";
|
|
28
29
|
import { KNODIN_VERSION } from "../version.js";
|
|
29
30
|
import * as ann from "./ann-hnsw.js";
|
|
30
31
|
import { allocateCandidate, assertCandidate, discardCandidateFiles, promoteCandidateFile, recoverInterruptedPromotion, } from "./candidate-database.js";
|
|
31
32
|
import { computeSimilarity, generateEmbedding, generateEmbeddings, } from "./embeddings.js";
|
|
32
33
|
import { walkRepoFiles } from "./file-walker.js";
|
|
33
34
|
import { clearGitHistorySignalCache, collectGitHistorySignals, } from "./git-history.js";
|
|
35
|
+
// Runtime import, but not a cycle: parse-pool imports only TYPES from here,
|
|
36
|
+
// which erase at compile time. parse-worker's runtime import of this module
|
|
37
|
+
// resolves inside the worker thread, never in this one.
|
|
38
|
+
import { createParsePool, parseWorkersEnabled } from "./parse-pool.js";
|
|
34
39
|
import { beginPerfPhase, measurePerfPhase, measurePerfPhaseSync } from "./perf.js";
|
|
35
40
|
import { isIndexablePath, makeWatchIgnorePredicate } from "./prune.js";
|
|
41
|
+
import { reflinkCopyFile } from "./reflink-copy.js";
|
|
36
42
|
import { readSarifLog, SARIF_DEFAULT_LIMITS, } from "./sarif-import.js";
|
|
37
43
|
import { readScipIndex, SCIP_DEFAULT_LIMITS, } from "./scip-import.js";
|
|
44
|
+
import { listSealedFiles, readSealedSource } from "./seal.js";
|
|
38
45
|
import { isIndexableSourcePath } from "./source-policy.js";
|
|
39
46
|
import { Database } from "./sqlite.js";
|
|
40
47
|
import { lookupMirror, mayWriteToRepository, resolveDbPath, resolveStateDir, } from "./state-paths.js";
|
|
@@ -500,10 +507,13 @@ function persistSymbolIdentities(db, repoPath, filePaths) {
|
|
|
500
507
|
}
|
|
501
508
|
sourceLines.set(row.filePath, lines);
|
|
502
509
|
}
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
510
|
+
// Only the symbol's FIRST line is used. This used to slice the whole
|
|
511
|
+
// body, join it into one string and split that back apart to take
|
|
512
|
+
// element zero — building the entire declaration text of every symbol
|
|
513
|
+
// to read one line of it, which cost 2.9ms per symbol on a real
|
|
514
|
+
// corpus. `lines` comes from `split(/\r?\n/)`, so no element contains
|
|
515
|
+
// a newline and indexing is exactly equivalent.
|
|
516
|
+
const declaration = (lines[Math.max(0, row.startLine - 1)] ?? "")
|
|
507
517
|
// `(?<!\s)` pins the match to the start of a whitespace run; without it
|
|
508
518
|
// every position inside the run rescans it, which is quadratic.
|
|
509
519
|
//
|
|
@@ -783,8 +793,8 @@ function freshnessMechanismFor(repoPath, policy) {
|
|
|
783
793
|
}
|
|
784
794
|
// Bumped whenever any repo is (re)indexed, so cached community-detection results
|
|
785
795
|
// below can be invalidated cheaply instead of recomputed on every call.
|
|
786
|
-
export const KNODIN_SCHEMA_VERSION =
|
|
787
|
-
const LAST_REBUILD_SCHEMA_VERSION =
|
|
796
|
+
export const KNODIN_SCHEMA_VERSION = 24;
|
|
797
|
+
const LAST_REBUILD_SCHEMA_VERSION = 24;
|
|
788
798
|
let indexGeneration = 0;
|
|
789
799
|
const resourceReachabilityCache = new Map();
|
|
790
800
|
const RESOURCE_REACHABILITY_CACHE_LIMIT = 32;
|
|
@@ -969,7 +979,10 @@ function writePersistedStatusAudit(repoPath, databasePath, databaseFingerprint,
|
|
|
969
979
|
};
|
|
970
980
|
try {
|
|
971
981
|
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
972
|
-
fs.writeFileSync(temporary, `${JSON.stringify(snapshot)}\n`, {
|
|
982
|
+
fs.writeFileSync(temporary, `${JSON.stringify(snapshot)}\n`, {
|
|
983
|
+
mode: 0o600,
|
|
984
|
+
flag: "wx",
|
|
985
|
+
});
|
|
973
986
|
fs.renameSync(temporary, target);
|
|
974
987
|
}
|
|
975
988
|
finally {
|
|
@@ -1122,6 +1135,19 @@ export function getSearchCacheStats() {
|
|
|
1122
1135
|
communityLookupCacheEntries: searchCommunityLookupCache.size,
|
|
1123
1136
|
};
|
|
1124
1137
|
}
|
|
1138
|
+
/**
|
|
1139
|
+
* How many repositories still have a live engine owner in this process.
|
|
1140
|
+
*
|
|
1141
|
+
* The process-global caches are released only when this reaches zero (see
|
|
1142
|
+
* `close()`), so a test asserting that those caches emptied is really
|
|
1143
|
+
* asserting that it held the last engine. The suite shares one process per
|
|
1144
|
+
* shard (`isolate: false`), where that is a property of the whole shard rather
|
|
1145
|
+
* than of any single spec — hence exposing it, so such an assertion can state
|
|
1146
|
+
* its precondition instead of depending on test ordering.
|
|
1147
|
+
*/
|
|
1148
|
+
export function getOpenRepositoryEngineCount() {
|
|
1149
|
+
return repositoryEngineOwners.size;
|
|
1150
|
+
}
|
|
1125
1151
|
/** Reset C46 work counters without changing cached production state. */
|
|
1126
1152
|
export function resetSearchCacheStats() {
|
|
1127
1153
|
for (const key of Object.keys(searchCacheStats)) {
|
|
@@ -1282,12 +1308,8 @@ function computeFlows(db, resolvedRepoPath) {
|
|
|
1282
1308
|
const isExported = (relFile, name) => {
|
|
1283
1309
|
let src = sourceCache.get(relFile);
|
|
1284
1310
|
if (src === undefined) {
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
}
|
|
1288
|
-
catch {
|
|
1289
|
-
src = null;
|
|
1290
|
-
}
|
|
1311
|
+
const read = readWholeSource(resolvedRepoPath, relFile);
|
|
1312
|
+
src = read.availability.state === "available" ? read.text : null;
|
|
1291
1313
|
sourceCache.set(relFile, src);
|
|
1292
1314
|
}
|
|
1293
1315
|
if (!src)
|
|
@@ -3177,15 +3199,18 @@ function resolveImportPath(sourceFile, importSource, repoPath) {
|
|
|
3177
3199
|
// Extensionless specifier: try appending each known source extension.
|
|
3178
3200
|
for (const e of RESOLVE_EXTS)
|
|
3179
3201
|
candidates.push(absolutePath + e);
|
|
3202
|
+
// Probed through the resolver, not the filesystem: a sealed repository has
|
|
3203
|
+
// no working tree, and `fs.existsSync` there would fail every candidate and
|
|
3204
|
+
// silently report the module unresolvable rather than erroring.
|
|
3180
3205
|
for (const c of candidates) {
|
|
3181
|
-
if (
|
|
3206
|
+
if (sourceFileExists(repoPath, path.relative(repoPath, c))) {
|
|
3182
3207
|
return path.relative(repoPath, c);
|
|
3183
3208
|
}
|
|
3184
3209
|
}
|
|
3185
3210
|
// Directory import -> its index file.
|
|
3186
3211
|
for (const e of RESOLVE_EXTS) {
|
|
3187
3212
|
const p = path.join(absolutePath, `index${e}`);
|
|
3188
|
-
if (
|
|
3213
|
+
if (sourceFileExists(repoPath, path.relative(repoPath, p))) {
|
|
3189
3214
|
return path.relative(repoPath, p);
|
|
3190
3215
|
}
|
|
3191
3216
|
}
|
|
@@ -3209,11 +3234,12 @@ function getPackageMap(repoPath) {
|
|
|
3209
3234
|
return cached.value;
|
|
3210
3235
|
const map = new Map();
|
|
3211
3236
|
let cacheable = true;
|
|
3212
|
-
for (const rel of
|
|
3213
|
-
accept: (file) => path.posix.basename(file) === "package.json",
|
|
3214
|
-
})) {
|
|
3237
|
+
for (const rel of listSourceFiles(repoPath, (file) => path.posix.basename(file) === "package.json")) {
|
|
3215
3238
|
try {
|
|
3216
|
-
const
|
|
3239
|
+
const manifest = readWholeSource(repoPath, rel);
|
|
3240
|
+
if (manifest.availability.state !== "available")
|
|
3241
|
+
continue;
|
|
3242
|
+
const pkg = JSON.parse(manifest.text);
|
|
3217
3243
|
if (pkg && typeof pkg.name === "string") {
|
|
3218
3244
|
const dir = path.posix.dirname(rel);
|
|
3219
3245
|
map.set(pkg.name, {
|
|
@@ -3229,7 +3255,10 @@ function getPackageMap(repoPath) {
|
|
|
3229
3255
|
}
|
|
3230
3256
|
}
|
|
3231
3257
|
if (cacheable)
|
|
3232
|
-
packageMapCache.set(resolvedRepoPath, {
|
|
3258
|
+
packageMapCache.set(resolvedRepoPath, {
|
|
3259
|
+
generation: indexGeneration,
|
|
3260
|
+
value: map,
|
|
3261
|
+
});
|
|
3233
3262
|
return map;
|
|
3234
3263
|
}
|
|
3235
3264
|
/** Best source entry file for a package: prefer source-y fields, else src/index, else index. */
|
|
@@ -3243,7 +3272,7 @@ function resolvePackageEntry(repoPath, dir, pkg) {
|
|
|
3243
3272
|
}
|
|
3244
3273
|
for (const rel of ["src/index.ts", "src/index.tsx", "src/index.js", "index.ts", "index.js"]) {
|
|
3245
3274
|
const p = path.join(dir, rel);
|
|
3246
|
-
if (
|
|
3275
|
+
if (sourceFileExists(repoPath, p))
|
|
3247
3276
|
return p;
|
|
3248
3277
|
}
|
|
3249
3278
|
return null;
|
|
@@ -3389,10 +3418,10 @@ function findClassFilePath(className, repoPath) {
|
|
|
3389
3418
|
* LWC claim, so the LWC path uses this stricter resolver instead.
|
|
3390
3419
|
*/
|
|
3391
3420
|
function findUniqueApexClassFilePath(className, repoPath) {
|
|
3392
|
-
|
|
3393
|
-
|
|
3394
|
-
|
|
3395
|
-
});
|
|
3421
|
+
// Enumerated through the resolver so a sealed repository answers from its
|
|
3422
|
+
// coverage. Still bounded to two matches, because "unique" is the whole
|
|
3423
|
+
// contract: a second match must resolve to null rather than a first hit.
|
|
3424
|
+
const matches = listSourceFiles(repoPath, (file) => path.posix.basename(file) === `${className}.cls`).slice(0, 2);
|
|
3396
3425
|
return matches.length === 1 ? matches[0] : null;
|
|
3397
3426
|
}
|
|
3398
3427
|
/**
|
|
@@ -3433,7 +3462,7 @@ function experienceSiteFile(sitePath, category, value, repoPath) {
|
|
|
3433
3462
|
return null;
|
|
3434
3463
|
const extension = category === "assets" ? "" : ".json";
|
|
3435
3464
|
const target = `${sitePath}/${category}/${name}${extension}`;
|
|
3436
|
-
return
|
|
3465
|
+
return sourceFileExists(repoPath, target) ? target : null;
|
|
3437
3466
|
}
|
|
3438
3467
|
function findExperienceComponentFile(value, repoPath) {
|
|
3439
3468
|
if (typeof value !== "string")
|
|
@@ -3444,7 +3473,7 @@ function findExperienceComponentFile(value, repoPath) {
|
|
|
3444
3473
|
return (findSalesforceLwcComponentFile(component, repoPath) ??
|
|
3445
3474
|
(() => {
|
|
3446
3475
|
const candidate = `force-app/main/default/aura/${component}/${component}.cmp`;
|
|
3447
|
-
return
|
|
3476
|
+
return sourceFileExists(repoPath, candidate) ? candidate : null;
|
|
3448
3477
|
})());
|
|
3449
3478
|
}
|
|
3450
3479
|
/**
|
|
@@ -3482,7 +3511,7 @@ function auraBundleFile(relativePath) {
|
|
|
3482
3511
|
function findLwcComponentFile(fromFile, componentName, repoPath) {
|
|
3483
3512
|
const lwcRoot = path.dirname(path.dirname(fromFile));
|
|
3484
3513
|
const candidate = path.join(lwcRoot, componentName, `${componentName}.js`);
|
|
3485
|
-
return
|
|
3514
|
+
return sourceFileExists(repoPath, candidate) ? candidate : null;
|
|
3486
3515
|
}
|
|
3487
3516
|
function findApexMethodFilePath(className, methodName, repoPath) {
|
|
3488
3517
|
const classFile = findUniqueApexClassFilePath(className, repoPath);
|
|
@@ -3544,13 +3573,10 @@ function findUniqueInvocableApexMethod(className, methodName, repoPath) {
|
|
|
3544
3573
|
const file = findUniqueApexClassFilePath(className, repoPath);
|
|
3545
3574
|
if (!file)
|
|
3546
3575
|
return null;
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
source = fs.readFileSync(path.join(repoPath, file), "utf8");
|
|
3550
|
-
}
|
|
3551
|
-
catch {
|
|
3576
|
+
const read = readWholeSource(repoPath, file);
|
|
3577
|
+
if (read.availability.state !== "available")
|
|
3552
3578
|
return null;
|
|
3553
|
-
|
|
3579
|
+
const source = read.text;
|
|
3554
3580
|
const matches = [];
|
|
3555
3581
|
for (const match of source.matchAll(/@InvocableMethod\b[\s\S]{0,300}?\b(?:global|public)\s+static\s(?:\s|[^({;]*?(?<!\s))\s+([a-z]\w*)\s*\(/gi)) {
|
|
3556
3582
|
if (!isExecutablePosition(source, match.index ?? 0) || match[1] !== methodName)
|
|
@@ -3590,7 +3616,7 @@ function findSalesforceLwcComponentFile(componentName, repoPath) {
|
|
|
3590
3616
|
if (!/^[A-Za-z]\w*$/.test(componentName))
|
|
3591
3617
|
return null;
|
|
3592
3618
|
const candidate = `force-app/main/default/lwc/${componentName}/${componentName}.js`;
|
|
3593
|
-
return
|
|
3619
|
+
return sourceFileExists(repoPath, candidate) ? candidate : null;
|
|
3594
3620
|
}
|
|
3595
3621
|
function apexMethodIsDeclared(content, methodName) {
|
|
3596
3622
|
const escaped = methodName.replace(/[.*+?^${}()|[\]\\]/g, String.raw `\$&`);
|
|
@@ -3746,10 +3772,81 @@ function extractApexPlatformDependencies(content, repoPath) {
|
|
|
3746
3772
|
return dependencies;
|
|
3747
3773
|
}
|
|
3748
3774
|
/** Index C20's static, deployable Salesforce DX metadata without interpreting formulas or runtime values. */
|
|
3775
|
+
/** Symbol kinds emitted for Salesforce DX metadata, one per metadata type. */
|
|
3776
|
+
export const SALESFORCE_METADATA_KINDS = [
|
|
3777
|
+
"salesforce_object",
|
|
3778
|
+
"salesforce_field",
|
|
3779
|
+
"salesforce_record_type",
|
|
3780
|
+
"salesforce_permission_set",
|
|
3781
|
+
"salesforce_flow",
|
|
3782
|
+
"salesforce_workflow",
|
|
3783
|
+
"salesforce_approval",
|
|
3784
|
+
];
|
|
3785
|
+
const SALESFORCE_KIND_BY_TYPE = {
|
|
3786
|
+
object: "salesforce_object",
|
|
3787
|
+
field: "salesforce_field",
|
|
3788
|
+
recordType: "salesforce_record_type",
|
|
3789
|
+
permissionSet: "salesforce_permission_set",
|
|
3790
|
+
flow: "salesforce_flow",
|
|
3791
|
+
workflow: "salesforce_workflow",
|
|
3792
|
+
approval: "salesforce_approval",
|
|
3793
|
+
};
|
|
3794
|
+
/**
|
|
3795
|
+
* Give a metadata symbol a kind and a summary that describe it.
|
|
3796
|
+
*
|
|
3797
|
+
* Every Salesforce symbol previously shared one kind and one identical
|
|
3798
|
+
* boilerplate summary, which made 36,763 of them indistinguishable to both FTS
|
|
3799
|
+
* (which indexes name, kind and summary) and to embeddings. A field's type,
|
|
3800
|
+
* label and lookup target are sitting in the file and were simply discarded.
|
|
3801
|
+
*/
|
|
3802
|
+
function describeSalesforceMetadata(metadata, content) {
|
|
3803
|
+
const kind = SALESFORCE_KIND_BY_TYPE[metadata.type] ?? "salesforce_metadata";
|
|
3804
|
+
const first = (tag) => staticXmlTagValues(content, tag)[0]?.value?.trim() || undefined;
|
|
3805
|
+
if (metadata.type === "field") {
|
|
3806
|
+
const fieldType = first("type");
|
|
3807
|
+
const referenceTo = first("referenceTo");
|
|
3808
|
+
const label = first("label");
|
|
3809
|
+
const parts = [
|
|
3810
|
+
`Salesforce field ${metadata.object}.${metadata.field}`,
|
|
3811
|
+
fieldType ? (referenceTo ? `${fieldType}(${referenceTo})` : fieldType) : undefined,
|
|
3812
|
+
label ? `"${label}"` : undefined,
|
|
3813
|
+
].filter(Boolean);
|
|
3814
|
+
return { kind, summary: parts.join(" — ") };
|
|
3815
|
+
}
|
|
3816
|
+
if (metadata.type === "object") {
|
|
3817
|
+
const label = first("label");
|
|
3818
|
+
// A `__e` suffix is the Platform Event marker, and callers ask about
|
|
3819
|
+
// those specifically ("which Platform Event triggers lack a retry guard").
|
|
3820
|
+
const platformEvent = metadata.object.endsWith("__e");
|
|
3821
|
+
const parts = [
|
|
3822
|
+
`Salesforce ${platformEvent ? "Platform Event" : "object"} ${metadata.object}`,
|
|
3823
|
+
label ? `"${label}"` : undefined,
|
|
3824
|
+
].filter(Boolean);
|
|
3825
|
+
return { kind, summary: parts.join(" — ") };
|
|
3826
|
+
}
|
|
3827
|
+
const label = first("label") ?? first("masterLabel");
|
|
3828
|
+
const name = metadata.type === "flow"
|
|
3829
|
+
? metadata.name
|
|
3830
|
+
: metadata.type === "permissionSet"
|
|
3831
|
+
? metadata.name
|
|
3832
|
+
: metadata.type === "recordType"
|
|
3833
|
+
? `${metadata.object}.${metadata.recordType}`
|
|
3834
|
+
: metadata.type === "approval"
|
|
3835
|
+
? `${metadata.object}.${metadata.name}`
|
|
3836
|
+
: metadata.object;
|
|
3837
|
+
const parts = [`Salesforce ${metadata.type} ${name}`, label ? `"${label}"` : undefined].filter(Boolean);
|
|
3838
|
+
return { kind, summary: parts.join(" — ") };
|
|
3839
|
+
}
|
|
3749
3840
|
async function indexSalesforceMetadataFile(content, relativePath, repoPath, db, metadata) {
|
|
3750
3841
|
const dependencies = [];
|
|
3751
3842
|
const add = (to, kind, sourceEvidence, methodReference) => {
|
|
3752
|
-
dependencies.push({
|
|
3843
|
+
dependencies.push({
|
|
3844
|
+
to,
|
|
3845
|
+
kind,
|
|
3846
|
+
confidence: 1,
|
|
3847
|
+
sourceEvidence,
|
|
3848
|
+
methodReference,
|
|
3849
|
+
});
|
|
3753
3850
|
};
|
|
3754
3851
|
const addField = (object, value, kind, evidence) => {
|
|
3755
3852
|
const qualified = value.includes(".") ? value : `${object}.${value}`;
|
|
@@ -3760,9 +3857,17 @@ async function indexSalesforceMetadataFile(content, relativePath, repoPath, db,
|
|
|
3760
3857
|
case "object":
|
|
3761
3858
|
add(metadata.object, "salesforce_metadata_object", "Salesforce DX CustomObject file path");
|
|
3762
3859
|
break;
|
|
3763
|
-
case "field":
|
|
3860
|
+
case "field": {
|
|
3764
3861
|
add(`${metadata.object}.${metadata.field}`, "salesforce_metadata_field", "Salesforce DX CustomField file path");
|
|
3862
|
+
// A Lookup/MasterDetail field names the object it points at. Emitting
|
|
3863
|
+
// it as an edge is what turns "which objects are near the 40-lookup
|
|
3864
|
+
// limit" into a graph query instead of a grep over raw XML.
|
|
3865
|
+
for (const entry of staticXmlTagValues(content, "referenceTo")) {
|
|
3866
|
+
if (staticSalesforceIdentifier(entry.value))
|
|
3867
|
+
add(entry.value, "salesforce_lookup_target", entry.evidence);
|
|
3868
|
+
}
|
|
3765
3869
|
break;
|
|
3870
|
+
}
|
|
3766
3871
|
case "recordType":
|
|
3767
3872
|
add(`${metadata.object}.${metadata.recordType}`, "salesforce_record_type", "Salesforce DX RecordType file path");
|
|
3768
3873
|
break;
|
|
@@ -3851,16 +3956,17 @@ async function indexSalesforceMetadataFile(content, relativePath, repoPath, db,
|
|
|
3851
3956
|
db.run("DELETE FROM dependencies WHERE fromFile = ?", [relativePath]);
|
|
3852
3957
|
db.run("DELETE FROM mcp_tools WHERE filePath = ?", [relativePath]);
|
|
3853
3958
|
const flowSymbol = metadata.type === "flow" ? metadata.name : path.basename(relativePath, ".xml");
|
|
3959
|
+
const descriptor = describeSalesforceMetadata(metadata, content);
|
|
3854
3960
|
db.run(`INSERT INTO symbols (name, kind, filePath, startLine, endLine, startCol, endCol, summary)
|
|
3855
3961
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
3856
3962
|
flowSymbol,
|
|
3857
|
-
|
|
3963
|
+
descriptor.kind,
|
|
3858
3964
|
relativePath,
|
|
3859
3965
|
1,
|
|
3860
3966
|
content.split("\n").length,
|
|
3861
3967
|
0,
|
|
3862
3968
|
0,
|
|
3863
|
-
|
|
3969
|
+
descriptor.summary,
|
|
3864
3970
|
]);
|
|
3865
3971
|
const insertRef = db.prepare(`INSERT INTO "references" (callerSymbol, callerFile, calleeSymbol, calleeFile, line, column, kind, confidence)
|
|
3866
3972
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
@@ -3899,7 +4005,7 @@ async function refreshFlowsForApexFiles(db, repoPath, apexFiles) {
|
|
|
3899
4005
|
if (classNames.size === 0)
|
|
3900
4006
|
return [];
|
|
3901
4007
|
const flowFiles = db
|
|
3902
|
-
.query("SELECT DISTINCT filePath FROM symbols WHERE kind = '
|
|
4008
|
+
.query("SELECT DISTINCT filePath FROM symbols WHERE kind = 'salesforce_flow' AND filePath LIKE 'force-app/main/default/flows/%.flow-meta.xml' ORDER BY filePath")
|
|
3903
4009
|
.all();
|
|
3904
4010
|
const refreshed = [];
|
|
3905
4011
|
for (const { filePath } of flowFiles) {
|
|
@@ -5579,10 +5685,6 @@ async function indexPrismaFile(content, tree, relativePath, _repoPath, db) {
|
|
|
5579
5685
|
const hasRelation = colDecl.text.includes("@relation");
|
|
5580
5686
|
if (hasRelation) {
|
|
5581
5687
|
dependencies.push({
|
|
5582
|
-
from: modelName,
|
|
5583
|
-
to: targetModel,
|
|
5584
|
-
kind: "orm-relation",
|
|
5585
|
-
}, {
|
|
5586
5688
|
from: relativePath,
|
|
5587
5689
|
to: targetModel,
|
|
5588
5690
|
kind: "orm-relation",
|
|
@@ -5605,7 +5707,6 @@ async function indexPrismaFile(content, tree, relativePath, _repoPath, db) {
|
|
|
5605
5707
|
const modelBlockRegex = /model\s+(\w+)\s*\{([^}]+)\}/g;
|
|
5606
5708
|
let mBlock = modelBlockRegex.exec(content);
|
|
5607
5709
|
while (mBlock !== null) {
|
|
5608
|
-
const srcModel = mBlock[1];
|
|
5609
5710
|
const blockContent = mBlock[2];
|
|
5610
5711
|
// `(?<!\w)` pins the leading field name to a word start; without it the scan
|
|
5611
5712
|
// restarts inside every identifier and re-walks it, which is quadratic.
|
|
@@ -5613,11 +5714,6 @@ async function indexPrismaFile(content, tree, relativePath, _repoPath, db) {
|
|
|
5613
5714
|
let rel = relationRegex.exec(blockContent);
|
|
5614
5715
|
while (rel !== null) {
|
|
5615
5716
|
const targetModel = rel[1];
|
|
5616
|
-
dependencies.push({
|
|
5617
|
-
from: srcModel,
|
|
5618
|
-
to: targetModel,
|
|
5619
|
-
kind: "orm-relation",
|
|
5620
|
-
});
|
|
5621
5717
|
dependencies.push({
|
|
5622
5718
|
from: relativePath,
|
|
5623
5719
|
to: targetModel,
|
|
@@ -5891,262 +5987,444 @@ export async function indexDbtManifestFile(content, _absolutePath, relativePath,
|
|
|
5891
5987
|
}
|
|
5892
5988
|
}
|
|
5893
5989
|
/**
|
|
5894
|
-
*
|
|
5990
|
+
* Builds a `FileIndexResult` from an already-parsed tree -- no DB access,
|
|
5991
|
+
* no `db` parameter. Factored out of `indexFile` so this same logic is
|
|
5992
|
+
* usable both from the current synchronous caller and, later, from a worker
|
|
5993
|
+
* thread that owns its own read+parse step (see the parallel-parsing plan).
|
|
5994
|
+
*/
|
|
5995
|
+
function buildFileIndexResult(tree, content, isPython, isApex, absolutePath, relativePath, repoPath, mcpRegistrations) {
|
|
5996
|
+
const { definitions, references, fileDependencies, tableDependencies, ormDependencies, importBindings, apiContracts, } = measurePerfPhaseSync("parse_extract", () => extractSymbolsAndReferences(tree.rootNode, isPython, isApex, absolutePath));
|
|
5997
|
+
const apexPlatformDependencies = isApex ? extractApexPlatformDependencies(content, repoPath) : [];
|
|
5998
|
+
// Resolve each callee to the file its definition lives in, so references
|
|
5999
|
+
// are file-qualified instead of name-only. Precedence: an imported name
|
|
6000
|
+
// resolves to its module's file; otherwise a name defined in THIS file
|
|
6001
|
+
// resolves to this file; otherwise unresolved (null) — external, dynamic,
|
|
6002
|
+
// or a member call — which consumers treat as best-effort.
|
|
6003
|
+
const localDefNames = new Set(definitions.map((d) => d.name));
|
|
6004
|
+
const resolveCalleeFile = (calleeName, explicitModule) => {
|
|
6005
|
+
// null means a member call on an ordinary value (`obj.run()`): do not
|
|
6006
|
+
// accidentally reuse an unrelated named import called `run`. undefined
|
|
6007
|
+
// means a bare call and is eligible for normal import-binding resolution.
|
|
6008
|
+
const mod = explicitModule === null ? undefined : (explicitModule ?? importBindings.get(calleeName));
|
|
6009
|
+
if (mod) {
|
|
6010
|
+
// Resolve the module (relative or workspace package alias) to a file,
|
|
6011
|
+
// then follow re-export barrels to the file that actually defines the
|
|
6012
|
+
// symbol (so `@pkg` -> index barrel -> definition file).
|
|
6013
|
+
const moduleFile = resolveModuleToFile(relativePath, mod, repoPath);
|
|
6014
|
+
if (moduleFile)
|
|
6015
|
+
return followReExports(repoPath, moduleFile, calleeName, 0);
|
|
6016
|
+
return null;
|
|
6017
|
+
}
|
|
6018
|
+
if (localDefNames.has(calleeName))
|
|
6019
|
+
return relativePath;
|
|
6020
|
+
return null;
|
|
6021
|
+
};
|
|
6022
|
+
const resolvedReferences = references.map((ref) => ({
|
|
6023
|
+
callerSymbol: ref.callerSymbol,
|
|
6024
|
+
calleeSymbol: ref.calleeSymbol,
|
|
6025
|
+
calleeFile: resolveCalleeFile(ref.calleeSymbol, ref.calleeModule),
|
|
6026
|
+
line: ref.line,
|
|
6027
|
+
column: ref.column,
|
|
6028
|
+
kind: ref.kind ?? "call",
|
|
6029
|
+
}));
|
|
6030
|
+
const dependencies = [];
|
|
6031
|
+
for (const depSource of fileDependencies) {
|
|
6032
|
+
const resolved = resolveImportPath(relativePath, depSource, repoPath);
|
|
6033
|
+
if (resolved)
|
|
6034
|
+
dependencies.push({ toFile: resolved, kind: "import" });
|
|
6035
|
+
}
|
|
6036
|
+
if (tableDependencies) {
|
|
6037
|
+
for (const tableName of tableDependencies)
|
|
6038
|
+
dependencies.push({ toFile: tableName, kind: "table" });
|
|
6039
|
+
}
|
|
6040
|
+
if (ormDependencies) {
|
|
6041
|
+
for (const dep of ormDependencies)
|
|
6042
|
+
dependencies.push({ toFile: dep.to, kind: dep.kind });
|
|
6043
|
+
}
|
|
6044
|
+
const dedupedApexPlatformDependencies = [];
|
|
6045
|
+
const seenApexPlatformDependencies = new Set();
|
|
6046
|
+
for (const dependency of apexPlatformDependencies) {
|
|
6047
|
+
const key = `${dependency.to}\0${dependency.kind}`;
|
|
6048
|
+
if (seenApexPlatformDependencies.has(key))
|
|
6049
|
+
continue;
|
|
6050
|
+
seenApexPlatformDependencies.add(key);
|
|
6051
|
+
dedupedApexPlatformDependencies.push(dependency);
|
|
6052
|
+
}
|
|
6053
|
+
return {
|
|
6054
|
+
relativePath,
|
|
6055
|
+
mcpRegistrations,
|
|
6056
|
+
definitions,
|
|
6057
|
+
references: resolvedReferences,
|
|
6058
|
+
apiContracts,
|
|
6059
|
+
dependencies,
|
|
6060
|
+
apexPlatformDependencies: dedupedApexPlatformDependencies,
|
|
6061
|
+
};
|
|
6062
|
+
}
|
|
6063
|
+
/**
|
|
6064
|
+
* Writes a single `FileIndexResult` inside its own transaction -- identical
|
|
6065
|
+
* shape to `indexFile`'s previous inline write, just parameterized on
|
|
6066
|
+
* already-extracted data instead of closure-captured locals.
|
|
6067
|
+
*/
|
|
6068
|
+
function writeFileIndexResult(db, result) {
|
|
6069
|
+
const finishWrite = beginPerfPhase("sqlite_write");
|
|
6070
|
+
db.run("BEGIN TRANSACTION;");
|
|
6071
|
+
try {
|
|
6072
|
+
writeFileIndexResultUnwrapped(db, result);
|
|
6073
|
+
db.run("COMMIT;");
|
|
6074
|
+
}
|
|
6075
|
+
catch (error) {
|
|
6076
|
+
db.run("ROLLBACK;");
|
|
6077
|
+
throw error;
|
|
6078
|
+
}
|
|
6079
|
+
finally {
|
|
6080
|
+
finishWrite();
|
|
6081
|
+
}
|
|
6082
|
+
}
|
|
6083
|
+
/** The actual delete+insert statements, run inside a transaction/savepoint the caller owns. */
|
|
6084
|
+
function writeFileIndexResultUnwrapped(db, result) {
|
|
6085
|
+
const { relativePath, mcpRegistrations, definitions, references, apiContracts, dependencies } = result;
|
|
6086
|
+
db.run('DELETE FROM "references" WHERE callerFile = ?', [relativePath]);
|
|
6087
|
+
deleteSymbolsForFile(db, relativePath);
|
|
6088
|
+
db.run("DELETE FROM dependencies WHERE fromFile = ?", [relativePath]);
|
|
6089
|
+
db.run("DELETE FROM mcp_tools WHERE filePath = ?", [relativePath]);
|
|
6090
|
+
db.run("DELETE FROM api_contracts WHERE filePath = ?", [relativePath]);
|
|
6091
|
+
const insertMcp = db.prepare(`
|
|
6092
|
+
INSERT INTO mcp_tools(name, description, schemaSymbol, handlerSymbol, filePath, line, confidence, associationKey)
|
|
6093
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
6094
|
+
`);
|
|
6095
|
+
for (const registration of mcpRegistrations) {
|
|
6096
|
+
insertMcp.run(registration.name, registration.description ?? null, registration.schemaSymbol ?? null, registration.handlerSymbol ?? null, relativePath, registration.line, registration.confidence, registration.associationKey ?? null);
|
|
6097
|
+
}
|
|
6098
|
+
insertMcp.finalize();
|
|
6099
|
+
const insertSym = db.prepare(`
|
|
6100
|
+
INSERT INTO symbols (name, kind, filePath, startLine, endLine, startCol, endCol, summary)
|
|
6101
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
6102
|
+
`);
|
|
6103
|
+
for (const def of definitions) {
|
|
6104
|
+
insertSym.run(def.name, def.kind, relativePath, def.startLine, def.endLine, def.startCol, def.endCol, def.summary);
|
|
6105
|
+
}
|
|
6106
|
+
insertSym.finalize();
|
|
6107
|
+
const insertRef = db.prepare(`
|
|
6108
|
+
INSERT INTO "references" (callerSymbol, callerFile, calleeSymbol, calleeFile, line, column, kind)
|
|
6109
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
6110
|
+
`);
|
|
6111
|
+
for (const ref of references) {
|
|
6112
|
+
insertRef.run(ref.callerSymbol, relativePath, ref.calleeSymbol, ref.calleeFile, ref.line, ref.column, ref.kind);
|
|
6113
|
+
}
|
|
6114
|
+
insertRef.finalize();
|
|
6115
|
+
const insertApiContract = db.prepare(`
|
|
6116
|
+
INSERT INTO api_contracts (kind, framework, method, path, origin, handler, middleware, responseFields, responseKnown, declaredResponseFields, accessedFields, unresolved, filePath, line, confidence, evidence)
|
|
6117
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
6118
|
+
`);
|
|
6119
|
+
for (const fact of apiContracts) {
|
|
6120
|
+
insertApiContract.run(fact.kind, fact.framework, fact.method, fact.path, fact.origin, fact.handler, JSON.stringify(fact.middleware), JSON.stringify(fact.responseFields), fact.responseKnown ? 1 : 0, JSON.stringify(fact.declaredResponseFields), JSON.stringify(fact.accessedFields), fact.unresolved ? 1 : 0, relativePath, fact.line, fact.confidence, fact.evidence);
|
|
6121
|
+
}
|
|
6122
|
+
insertApiContract.finalize();
|
|
6123
|
+
const insertDep = db.prepare(`
|
|
6124
|
+
INSERT INTO dependencies (fromFile, toFile, kind)
|
|
6125
|
+
VALUES (?, ?, ?)
|
|
6126
|
+
`);
|
|
6127
|
+
const insertApexPlatformDependency = db.prepare("INSERT INTO dependencies (fromFile, toFile, kind, confidence, sourceEvidence) VALUES (?, ?, ?, ?, ?)");
|
|
6128
|
+
for (const dep of dependencies) {
|
|
6129
|
+
insertDep.run(relativePath, dep.toFile, dep.kind);
|
|
6130
|
+
}
|
|
6131
|
+
insertDep.finalize();
|
|
6132
|
+
for (const dependency of result.apexPlatformDependencies) {
|
|
6133
|
+
insertApexPlatformDependency.run(relativePath, dependency.to, dependency.kind, 1, dependency.sourceEvidence);
|
|
6134
|
+
}
|
|
6135
|
+
insertApexPlatformDependency.finalize();
|
|
6136
|
+
}
|
|
6137
|
+
/**
|
|
6138
|
+
* Write a batch of `FileIndexResult`s in a single transaction, isolating
|
|
6139
|
+
* each file behind its own SAVEPOINT so one file's write failure rolls back
|
|
6140
|
+
* only that file and does not lose the rest of the batch -- the same
|
|
6141
|
+
* per-file error-isolation contract `indexFile`'s single-file transaction
|
|
6142
|
+
* always had, just amortized over one BEGIN/COMMIT per batch instead of per
|
|
6143
|
+
* file. Returns the files whose write failed, mirroring `indexFile`'s own
|
|
6144
|
+
* catch (caller tallies into `unparsedByExtension` the same way).
|
|
6145
|
+
*/
|
|
6146
|
+
export function writeFileIndexResultBatch(db, results) {
|
|
6147
|
+
if (results.length === 0)
|
|
6148
|
+
return [];
|
|
6149
|
+
const failures = [];
|
|
6150
|
+
const finishWrite = beginPerfPhase("sqlite_write");
|
|
6151
|
+
db.run("BEGIN TRANSACTION;");
|
|
6152
|
+
try {
|
|
6153
|
+
results.forEach((result, i) => {
|
|
6154
|
+
const savepoint = `f${i}`;
|
|
6155
|
+
db.run(`SAVEPOINT ${savepoint};`);
|
|
6156
|
+
try {
|
|
6157
|
+
writeFileIndexResultUnwrapped(db, result);
|
|
6158
|
+
db.run(`RELEASE ${savepoint};`);
|
|
6159
|
+
}
|
|
6160
|
+
catch (error) {
|
|
6161
|
+
db.run(`ROLLBACK TO ${savepoint};`);
|
|
6162
|
+
db.run(`RELEASE ${savepoint};`);
|
|
6163
|
+
failures.push({ relativePath: result.relativePath, error });
|
|
6164
|
+
}
|
|
6165
|
+
});
|
|
6166
|
+
db.run("COMMIT;");
|
|
6167
|
+
}
|
|
6168
|
+
catch (error) {
|
|
6169
|
+
db.run("ROLLBACK;");
|
|
6170
|
+
throw error;
|
|
6171
|
+
}
|
|
6172
|
+
finally {
|
|
6173
|
+
finishWrite();
|
|
6174
|
+
}
|
|
6175
|
+
return failures;
|
|
6176
|
+
}
|
|
6177
|
+
const INDEX_WRITE_BATCH_SIZE = (() => {
|
|
6178
|
+
const raw = Number(process.env.KNODIN_INDEX_WRITE_BATCH_SIZE);
|
|
6179
|
+
return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : 200;
|
|
6180
|
+
})();
|
|
6181
|
+
/**
|
|
6182
|
+
* Read, parse, and extract one file through the generic tree-sitter path.
|
|
5895
6183
|
*
|
|
5896
|
-
*
|
|
5897
|
-
*
|
|
5898
|
-
*
|
|
5899
|
-
*
|
|
5900
|
-
* contributing nothing looks exactly like a language with nothing to say.
|
|
6184
|
+
* Deliberately free of any `Database` parameter so the whole read → parse →
|
|
6185
|
+
* extract sequence can run off the main thread. `indexFile` is the only
|
|
6186
|
+
* in-tree caller today; the parse pool becomes the second, and both must go
|
|
6187
|
+
* through here so routing can never drift between them.
|
|
5901
6188
|
*/
|
|
5902
|
-
async function
|
|
6189
|
+
export async function extractGenericFileForIndex(absolutePath, relativePath, repoPath) {
|
|
6190
|
+
const lang = await measurePerfPhase("parser_wasm_init", () => getLanguageForFile(absolutePath));
|
|
6191
|
+
if (!lang)
|
|
6192
|
+
return { kind: "unparsed" };
|
|
6193
|
+
const content = measurePerfPhaseSync("read", () => fs.readFileSync(absolutePath, "utf-8"));
|
|
6194
|
+
// Skip minified / generated bundles (e.g. a `*.bundle.js`): they're not
|
|
6195
|
+
// readable source and would inject thousands of junk "symbols" that pollute
|
|
6196
|
+
// the graph, communities, and hubs. Heuristic: a very high average line
|
|
6197
|
+
// length (minified code packs everything onto a few enormous lines).
|
|
6198
|
+
if (isMinifiedSource(content))
|
|
6199
|
+
return { kind: "unparsed" };
|
|
6200
|
+
const isSql = absolutePath.endsWith(".sql") || absolutePath.endsWith(".pkb") || absolutePath.endsWith(".pks");
|
|
6201
|
+
// SQL: neutralize dollar-quoted bodies that crash the grammar's scanner.
|
|
6202
|
+
// The tree and every downstream indexer must see the SAME string, so pass
|
|
6203
|
+
// `parseContent` (not the raw file) forward.
|
|
6204
|
+
const parseContent = isSql ? sanitizeDollarQuotedSql(content) : content;
|
|
6205
|
+
const parser = new Parser();
|
|
6206
|
+
parser.setLanguage(lang);
|
|
6207
|
+
// A malformed/pathological input can make the wasm parser throw rather than
|
|
6208
|
+
// return an ERROR tree. Skip that one file instead of aborting the file's
|
|
6209
|
+
// index (and without dumping the wasm source into the log).
|
|
6210
|
+
let tree;
|
|
6211
|
+
try {
|
|
6212
|
+
tree = measurePerfPhaseSync("parse_extract", () => parser.parse(parseContent));
|
|
6213
|
+
}
|
|
6214
|
+
catch (e) {
|
|
6215
|
+
console.warn(`knodin: skipped ${relativePath} (parser error: ${e.message})`);
|
|
6216
|
+
return { kind: "skipped", reason: e.message };
|
|
6217
|
+
}
|
|
6218
|
+
if (!tree)
|
|
6219
|
+
return { kind: "skipped", reason: "parser returned no tree" };
|
|
6220
|
+
const mcpRegistrations = /\.[cm]?[jt]sx?$/.test(absolutePath) && /@modelcontextprotocol\/sdk/.test(content)
|
|
6221
|
+
? extractMcpToolRegistrations(content, relativePath, tree.rootNode)
|
|
6222
|
+
: [];
|
|
6223
|
+
const isPython = absolutePath.endsWith(".py");
|
|
6224
|
+
const isApex = absolutePath.endsWith(".cls") || absolutePath.endsWith(".trigger");
|
|
6225
|
+
const isVisualforce = absolutePath.endsWith(".page") || absolutePath.endsWith(".component");
|
|
6226
|
+
const isPrisma = absolutePath.endsWith(".prisma");
|
|
6227
|
+
const isWorkday = absolutePath.endsWith(".clp") ||
|
|
6228
|
+
absolutePath.endsWith(".ws") ||
|
|
6229
|
+
(absolutePath.endsWith(".xml") && isWorkdayStudioFile(content, absolutePath));
|
|
6230
|
+
const special = (route) => ({
|
|
6231
|
+
kind: "special",
|
|
6232
|
+
route,
|
|
6233
|
+
content,
|
|
6234
|
+
parseContent,
|
|
6235
|
+
tree: tree,
|
|
6236
|
+
});
|
|
6237
|
+
if (isWorkday)
|
|
6238
|
+
return special("workday");
|
|
6239
|
+
if (isVisualforce)
|
|
6240
|
+
return special("visualforce");
|
|
6241
|
+
if (isSql)
|
|
6242
|
+
return special("sql");
|
|
6243
|
+
if (isPrisma)
|
|
6244
|
+
return special("prisma");
|
|
6245
|
+
return {
|
|
6246
|
+
kind: "result",
|
|
6247
|
+
result: buildFileIndexResult(tree, content, isPython, isApex, absolutePath, relativePath, repoPath, mcpRegistrations),
|
|
6248
|
+
};
|
|
6249
|
+
}
|
|
6250
|
+
/**
|
|
6251
|
+
* The single source of truth for indexer routing.
|
|
6252
|
+
*
|
|
6253
|
+
* `indexFile` dispatches on this, and the parse pool decides worker
|
|
6254
|
+
* eligibility from it. Both must consult the SAME function: a second copy of
|
|
6255
|
+
* this ordering would drift the moment a file type is added, and the failure
|
|
6256
|
+
* mode is silent — a markdown file routed to a worker comes back "unparsed"
|
|
6257
|
+
* rather than reaching `indexMarkdownFile`, which reads as "this file has no
|
|
6258
|
+
* symbols" instead of as an error.
|
|
6259
|
+
*
|
|
6260
|
+
* Order is significant and matches the historical dispatch exactly.
|
|
6261
|
+
*/
|
|
6262
|
+
export function classifyIndexRoute(absolutePath, relativePath, options) {
|
|
6263
|
+
const baseName = path.basename(absolutePath).toLowerCase();
|
|
6264
|
+
if (!options?.skipDbt && baseName === "manifest.json")
|
|
6265
|
+
return { kind: "dbt-candidate" };
|
|
6266
|
+
if (absolutePath.endsWith(".lsif") || baseName === "dump.lsif")
|
|
6267
|
+
return { kind: "lsif" };
|
|
6268
|
+
if (absolutePath.endsWith(".tf") ||
|
|
6269
|
+
absolutePath.endsWith(".tfvars") ||
|
|
6270
|
+
absolutePath.endsWith(".hcl"))
|
|
6271
|
+
return { kind: "terraform" };
|
|
6272
|
+
if (absolutePath.endsWith(".dockerfile") || baseName === "dockerfile")
|
|
6273
|
+
return { kind: "dockerfile" };
|
|
6274
|
+
const salesforceExperience = salesforceExperienceFile(relativePath);
|
|
6275
|
+
if (salesforceExperience)
|
|
6276
|
+
return { kind: "salesforce-experience", file: salesforceExperience };
|
|
6277
|
+
const lwcBundle = lwcBundleFile(relativePath);
|
|
6278
|
+
if (lwcBundle)
|
|
6279
|
+
return { kind: "lwc-bundle", file: lwcBundle };
|
|
6280
|
+
const auraBundle = auraBundleFile(relativePath);
|
|
6281
|
+
if (auraBundle)
|
|
6282
|
+
return { kind: "aura-bundle", file: auraBundle };
|
|
6283
|
+
const salesforceMetadata = salesforceMetadataFile(relativePath);
|
|
6284
|
+
if (salesforceMetadata)
|
|
6285
|
+
return { kind: "salesforce-metadata", file: salesforceMetadata };
|
|
6286
|
+
if (relativePath.endsWith(".md") || relativePath.endsWith(".mdx"))
|
|
6287
|
+
return { kind: "markdown" };
|
|
6288
|
+
if (relativePath.endsWith(".yml") || relativePath.endsWith(".yaml"))
|
|
6289
|
+
return { kind: "yaml" };
|
|
6290
|
+
if (relativePath.endsWith(".csproj") ||
|
|
6291
|
+
relativePath.endsWith(".sln") ||
|
|
6292
|
+
relativePath.endsWith(".props") ||
|
|
6293
|
+
relativePath.endsWith(".targets"))
|
|
6294
|
+
return { kind: "dotnet-project" };
|
|
6295
|
+
if (relativePath.endsWith(".cshtml") || relativePath.endsWith(".razor"))
|
|
6296
|
+
return { kind: "razor" };
|
|
6297
|
+
return { kind: "generic" };
|
|
6298
|
+
}
|
|
6299
|
+
/**
|
|
6300
|
+
* Whether a file may be parsed off the main thread.
|
|
6301
|
+
*
|
|
6302
|
+
* Only the generic tree-sitter path qualifies: every other route calls an
|
|
6303
|
+
* indexer that writes to the database, and the database has exactly one
|
|
6304
|
+
* writer. `dbt-candidate` is excluded on purpose — it cannot be resolved
|
|
6305
|
+
* without reading the file, and `manifest.json` is rare enough that sending
|
|
6306
|
+
* it down the sequential path costs nothing.
|
|
6307
|
+
*/
|
|
6308
|
+
export function isWorkerEligibleFile(absolutePath, relativePath) {
|
|
6309
|
+
if (classifyIndexRoute(absolutePath, relativePath).kind !== "generic")
|
|
6310
|
+
return false;
|
|
6311
|
+
// `generic` is necessary but NOT sufficient. `extractGenericFileForIndex`
|
|
6312
|
+
// can still return a `special` route — visualforce, workday, sql, prisma —
|
|
6313
|
+
// each of which hands off to a main-thread indexer that opens its own
|
|
6314
|
+
// BEGIN/COMMIT against the single-writer database. Sending one to a worker
|
|
6315
|
+
// leaves it to the pool's fallback, which re-indexes it inline while pooled
|
|
6316
|
+
// results are still accumulating for a batched write, and the two
|
|
6317
|
+
// interleave nondeterministically: observed as a Visualforce controller
|
|
6318
|
+
// edge that resolved on some runs and not others, at roughly 1 in 3.
|
|
6319
|
+
//
|
|
6320
|
+
// Decided from the path alone, so it can be answered before reading the
|
|
6321
|
+
// file. `.xml` is excluded conservatively because Workday detection needs
|
|
6322
|
+
// the content; an XML file indexing inline costs nothing.
|
|
6323
|
+
return !SPECIAL_ROUTE_EXTENSIONS.some((extension) => absolutePath.endsWith(extension));
|
|
6324
|
+
}
|
|
6325
|
+
/**
|
|
6326
|
+
* Extensions that may reach a `special` route in `extractGenericFileForIndex`.
|
|
6327
|
+
*
|
|
6328
|
+
* Kept adjacent to that function's own extension tests on purpose: if a new
|
|
6329
|
+
* special route is added there without a matching entry here, the file becomes
|
|
6330
|
+
* worker-eligible again and the failure is silent rather than loud.
|
|
6331
|
+
*/
|
|
6332
|
+
const SPECIAL_ROUTE_EXTENSIONS = [
|
|
6333
|
+
".page",
|
|
6334
|
+
".component",
|
|
6335
|
+
".clp",
|
|
6336
|
+
".ws",
|
|
6337
|
+
".xml",
|
|
6338
|
+
".sql",
|
|
6339
|
+
".pkb",
|
|
6340
|
+
".pks",
|
|
6341
|
+
".prisma",
|
|
6342
|
+
];
|
|
6343
|
+
async function indexFile(absolutePath, relativePath, repoPath, db, unparsed, collectResult) {
|
|
5903
6344
|
try {
|
|
5904
6345
|
invalidateScipFacts(db);
|
|
5905
|
-
const
|
|
5906
|
-
|
|
6346
|
+
const read = () => measurePerfPhaseSync("read", () => fs.readFileSync(absolutePath, "utf-8"));
|
|
6347
|
+
let route = classifyIndexRoute(absolutePath, relativePath);
|
|
6348
|
+
if (route.kind === "dbt-candidate") {
|
|
5907
6349
|
const content = fs.readFileSync(absolutePath, "utf-8");
|
|
5908
6350
|
if (isDbtManifest(content)) {
|
|
5909
6351
|
await indexDbtManifestFile(content, absolutePath, relativePath, repoPath, db);
|
|
5910
6352
|
return;
|
|
5911
6353
|
}
|
|
6354
|
+
// Not a dbt manifest after all. Re-classify without the dbt check so
|
|
6355
|
+
// the file still reaches whatever route it would otherwise have taken.
|
|
6356
|
+
route = classifyIndexRoute(absolutePath, relativePath, { skipDbt: true });
|
|
5912
6357
|
}
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
|
|
5916
|
-
|
|
5917
|
-
|
|
5918
|
-
|
|
5919
|
-
|
|
5920
|
-
|
|
5921
|
-
|
|
5922
|
-
|
|
5923
|
-
|
|
5924
|
-
|
|
5925
|
-
|
|
5926
|
-
|
|
5927
|
-
|
|
5928
|
-
|
|
5929
|
-
|
|
5930
|
-
|
|
5931
|
-
|
|
5932
|
-
|
|
5933
|
-
|
|
5934
|
-
|
|
5935
|
-
|
|
5936
|
-
|
|
5937
|
-
|
|
5938
|
-
|
|
5939
|
-
|
|
5940
|
-
|
|
5941
|
-
|
|
5942
|
-
|
|
5943
|
-
|
|
5944
|
-
|
|
5945
|
-
|
|
5946
|
-
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
const salesforceMetadata = salesforceMetadataFile(relativePath);
|
|
5950
|
-
if (salesforceMetadata) {
|
|
5951
|
-
const content = measurePerfPhaseSync("read", () => fs.readFileSync(absolutePath, "utf-8"));
|
|
5952
|
-
await indexSalesforceMetadataFile(content, relativePath, repoPath, db, salesforceMetadata);
|
|
5953
|
-
return;
|
|
5954
|
-
}
|
|
5955
|
-
if (relativePath.endsWith(".md") || relativePath.endsWith(".mdx")) {
|
|
5956
|
-
const content = measurePerfPhaseSync("read", () => fs.readFileSync(absolutePath, "utf-8"));
|
|
5957
|
-
await indexMarkdownFile(content, relativePath, repoPath, db);
|
|
5958
|
-
return;
|
|
5959
|
-
}
|
|
5960
|
-
if (relativePath.endsWith(".yml") || relativePath.endsWith(".yaml")) {
|
|
5961
|
-
const content = measurePerfPhaseSync("read", () => fs.readFileSync(absolutePath, "utf-8"));
|
|
5962
|
-
await indexYamlFile(content, relativePath, repoPath, db);
|
|
5963
|
-
return;
|
|
5964
|
-
}
|
|
5965
|
-
if (relativePath.endsWith(".csproj") ||
|
|
5966
|
-
relativePath.endsWith(".sln") ||
|
|
5967
|
-
relativePath.endsWith(".props") ||
|
|
5968
|
-
relativePath.endsWith(".targets")) {
|
|
5969
|
-
const content = measurePerfPhaseSync("read", () => fs.readFileSync(absolutePath, "utf-8"));
|
|
5970
|
-
await indexDotnetProjectFile(content, relativePath, repoPath, db);
|
|
5971
|
-
return;
|
|
5972
|
-
}
|
|
5973
|
-
if (relativePath.endsWith(".cshtml") || relativePath.endsWith(".razor")) {
|
|
5974
|
-
const content = measurePerfPhaseSync("read", () => fs.readFileSync(absolutePath, "utf-8"));
|
|
5975
|
-
await indexRazorFile(content, relativePath, repoPath, db);
|
|
5976
|
-
return;
|
|
5977
|
-
}
|
|
5978
|
-
const lang = await measurePerfPhase("parser_wasm_init", () => getLanguageForFile(absolutePath));
|
|
5979
|
-
if (!lang) {
|
|
5980
|
-
if (unparsed)
|
|
5981
|
-
tallyOne(unparsed, extensionBucket(relativePath));
|
|
5982
|
-
return;
|
|
6358
|
+
switch (route.kind) {
|
|
6359
|
+
case "lsif":
|
|
6360
|
+
await indexLsifDump(absolutePath, relativePath, repoPath, db);
|
|
6361
|
+
return;
|
|
6362
|
+
case "terraform":
|
|
6363
|
+
await indexTerraformFile(fs.readFileSync(absolutePath, "utf-8"), relativePath, repoPath, db);
|
|
6364
|
+
return;
|
|
6365
|
+
case "dockerfile":
|
|
6366
|
+
await indexDockerFile(fs.readFileSync(absolutePath, "utf-8"), relativePath, repoPath, db);
|
|
6367
|
+
return;
|
|
6368
|
+
case "salesforce-experience":
|
|
6369
|
+
await indexSalesforceExperienceFile(read(), relativePath, repoPath, db, route.file);
|
|
6370
|
+
return;
|
|
6371
|
+
case "lwc-bundle":
|
|
6372
|
+
await indexLwcBundleFile(read(), relativePath, repoPath, db, route.file);
|
|
6373
|
+
return;
|
|
6374
|
+
case "aura-bundle":
|
|
6375
|
+
await indexAuraBundleFile(read(), relativePath, repoPath, db, route.file);
|
|
6376
|
+
return;
|
|
6377
|
+
case "salesforce-metadata":
|
|
6378
|
+
await indexSalesforceMetadataFile(read(), relativePath, repoPath, db, route.file);
|
|
6379
|
+
return;
|
|
6380
|
+
case "markdown":
|
|
6381
|
+
await indexMarkdownFile(read(), relativePath, repoPath, db);
|
|
6382
|
+
return;
|
|
6383
|
+
case "yaml":
|
|
6384
|
+
await indexYamlFile(read(), relativePath, repoPath, db);
|
|
6385
|
+
return;
|
|
6386
|
+
case "dotnet-project":
|
|
6387
|
+
await indexDotnetProjectFile(read(), relativePath, repoPath, db);
|
|
6388
|
+
return;
|
|
6389
|
+
case "razor":
|
|
6390
|
+
await indexRazorFile(read(), relativePath, repoPath, db);
|
|
6391
|
+
return;
|
|
6392
|
+
default:
|
|
6393
|
+
break;
|
|
5983
6394
|
}
|
|
5984
|
-
const
|
|
5985
|
-
|
|
5986
|
-
// readable source and would inject thousands of junk "symbols" that pollute
|
|
5987
|
-
// the graph, communities, and hubs. Heuristic: a very high average line
|
|
5988
|
-
// length (minified code packs everything onto a few enormous lines).
|
|
5989
|
-
if (isMinifiedSource(content)) {
|
|
6395
|
+
const extraction = await extractGenericFileForIndex(absolutePath, relativePath, repoPath);
|
|
6396
|
+
if (extraction.kind === "unparsed") {
|
|
5990
6397
|
if (unparsed)
|
|
5991
6398
|
tallyOne(unparsed, extensionBucket(relativePath));
|
|
5992
6399
|
return;
|
|
5993
6400
|
}
|
|
5994
|
-
|
|
5995
|
-
|
|
5996
|
-
|
|
5997
|
-
|
|
5998
|
-
// The tree and every downstream indexer must see the SAME string, so pass
|
|
5999
|
-
// `parseContent` (not the raw file) forward.
|
|
6000
|
-
const parseContent = isSql ? sanitizeDollarQuotedSql(content) : content;
|
|
6001
|
-
const parser = new Parser();
|
|
6002
|
-
parser.setLanguage(lang);
|
|
6003
|
-
// A malformed/pathological input can make the wasm parser throw rather than
|
|
6004
|
-
// return an ERROR tree. Skip that one file instead of aborting the file's
|
|
6005
|
-
// index (and without dumping the wasm source into the log).
|
|
6006
|
-
let tree;
|
|
6007
|
-
try {
|
|
6008
|
-
tree = measurePerfPhaseSync("parse_extract", () => parser.parse(parseContent));
|
|
6009
|
-
}
|
|
6010
|
-
catch (e) {
|
|
6011
|
-
console.warn(`knodin: skipped ${relativePath} (parser error: ${e.message})`);
|
|
6012
|
-
return;
|
|
6013
|
-
}
|
|
6014
|
-
if (!tree)
|
|
6015
|
-
return;
|
|
6016
|
-
const mcpRegistrations = /\.[cm]?[jt]sx?$/.test(absolutePath) && /@modelcontextprotocol\/sdk/.test(content)
|
|
6017
|
-
? extractMcpToolRegistrations(content, relativePath, tree.rootNode)
|
|
6018
|
-
: [];
|
|
6019
|
-
const isPython = absolutePath.endsWith(".py");
|
|
6020
|
-
const isApex = absolutePath.endsWith(".cls") || absolutePath.endsWith(".trigger");
|
|
6021
|
-
const isVisualforce = absolutePath.endsWith(".page") || absolutePath.endsWith(".component");
|
|
6022
|
-
const isPrisma = absolutePath.endsWith(".prisma");
|
|
6023
|
-
const isWorkday = absolutePath.endsWith(".clp") ||
|
|
6024
|
-
absolutePath.endsWith(".ws") ||
|
|
6025
|
-
(absolutePath.endsWith(".xml") && isWorkdayStudioFile(content, absolutePath));
|
|
6026
|
-
if (isWorkday) {
|
|
6027
|
-
await indexWorkdayFile(content, tree, relativePath, repoPath, db);
|
|
6028
|
-
return;
|
|
6029
|
-
}
|
|
6030
|
-
if (isVisualforce) {
|
|
6031
|
-
await indexVisualforceFile(content, tree, relativePath, repoPath, db);
|
|
6032
|
-
return;
|
|
6033
|
-
}
|
|
6034
|
-
if (isSql) {
|
|
6035
|
-
await indexSqlFile(parseContent, tree, relativePath, repoPath, db);
|
|
6401
|
+
// A parse failure abandons this one file without tallying it as a
|
|
6402
|
+
// coverage gap: the file IS supported, it just did not parse, and
|
|
6403
|
+
// `extractGenericFileForIndex` has already warned about it.
|
|
6404
|
+
if (extraction.kind === "skipped")
|
|
6036
6405
|
return;
|
|
6037
|
-
|
|
6038
|
-
|
|
6039
|
-
|
|
6040
|
-
|
|
6041
|
-
|
|
6042
|
-
|
|
6043
|
-
|
|
6044
|
-
|
|
6045
|
-
|
|
6046
|
-
|
|
6047
|
-
|
|
6048
|
-
|
|
6049
|
-
|
|
6050
|
-
|
|
6051
|
-
|
|
6052
|
-
const resolveCalleeFile = (calleeName, explicitModule) => {
|
|
6053
|
-
// null means a member call on an ordinary value (`obj.run()`): do not
|
|
6054
|
-
// accidentally reuse an unrelated named import called `run`. undefined
|
|
6055
|
-
// means a bare call and is eligible for normal import-binding resolution.
|
|
6056
|
-
const mod = explicitModule === null ? undefined : (explicitModule ?? importBindings.get(calleeName));
|
|
6057
|
-
if (mod) {
|
|
6058
|
-
// Resolve the module (relative or workspace package alias) to a file,
|
|
6059
|
-
// then follow re-export barrels to the file that actually defines the
|
|
6060
|
-
// symbol (so `@pkg` -> index barrel -> definition file).
|
|
6061
|
-
const moduleFile = resolveModuleToFile(relativePath, mod, repoPath);
|
|
6062
|
-
if (moduleFile)
|
|
6063
|
-
return followReExports(repoPath, moduleFile, calleeName, 0);
|
|
6064
|
-
return null;
|
|
6065
|
-
}
|
|
6066
|
-
if (localDefNames.has(calleeName))
|
|
6067
|
-
return relativePath;
|
|
6068
|
-
return null;
|
|
6069
|
-
};
|
|
6070
|
-
// Execute all database operations inside a single transaction synchronously
|
|
6071
|
-
const finishWrite = beginPerfPhase("sqlite_write");
|
|
6072
|
-
db.run("BEGIN TRANSACTION;");
|
|
6073
|
-
try {
|
|
6074
|
-
db.run('DELETE FROM "references" WHERE callerFile = ?', [relativePath]);
|
|
6075
|
-
deleteSymbolsForFile(db, relativePath);
|
|
6076
|
-
db.run("DELETE FROM dependencies WHERE fromFile = ?", [relativePath]);
|
|
6077
|
-
db.run("DELETE FROM mcp_tools WHERE filePath = ?", [relativePath]);
|
|
6078
|
-
db.run("DELETE FROM api_contracts WHERE filePath = ?", [relativePath]);
|
|
6079
|
-
const insertMcp = db.prepare(`
|
|
6080
|
-
INSERT INTO mcp_tools(name, description, schemaSymbol, handlerSymbol, filePath, line, confidence, associationKey)
|
|
6081
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
6082
|
-
`);
|
|
6083
|
-
for (const registration of mcpRegistrations) {
|
|
6084
|
-
insertMcp.run(registration.name, registration.description ?? null, registration.schemaSymbol ?? null, registration.handlerSymbol ?? null, relativePath, registration.line, registration.confidence, registration.associationKey ?? null);
|
|
6085
|
-
}
|
|
6086
|
-
insertMcp.finalize();
|
|
6087
|
-
const insertSym = db.prepare(`
|
|
6088
|
-
INSERT INTO symbols (name, kind, filePath, startLine, endLine, startCol, endCol, summary)
|
|
6089
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
6090
|
-
`);
|
|
6091
|
-
for (const def of definitions) {
|
|
6092
|
-
insertSym.run(def.name, def.kind, relativePath, def.startLine, def.endLine, def.startCol, def.endCol, def.summary);
|
|
6093
|
-
}
|
|
6094
|
-
insertSym.finalize();
|
|
6095
|
-
const insertRef = db.prepare(`
|
|
6096
|
-
INSERT INTO "references" (callerSymbol, callerFile, calleeSymbol, calleeFile, line, column, kind)
|
|
6097
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
6098
|
-
`);
|
|
6099
|
-
for (const ref of references) {
|
|
6100
|
-
insertRef.run(ref.callerSymbol, relativePath, ref.calleeSymbol, resolveCalleeFile(ref.calleeSymbol, ref.calleeModule), ref.line, ref.column, ref.kind ?? "call");
|
|
6101
|
-
}
|
|
6102
|
-
insertRef.finalize();
|
|
6103
|
-
const insertApiContract = db.prepare(`
|
|
6104
|
-
INSERT INTO api_contracts (kind, framework, method, path, origin, handler, middleware, responseFields, responseKnown, declaredResponseFields, accessedFields, unresolved, filePath, line, confidence, evidence)
|
|
6105
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
6106
|
-
`);
|
|
6107
|
-
for (const fact of apiContracts) {
|
|
6108
|
-
insertApiContract.run(fact.kind, fact.framework, fact.method, fact.path, fact.origin, fact.handler, JSON.stringify(fact.middleware), JSON.stringify(fact.responseFields), fact.responseKnown ? 1 : 0, JSON.stringify(fact.declaredResponseFields), JSON.stringify(fact.accessedFields), fact.unresolved ? 1 : 0, relativePath, fact.line, fact.confidence, fact.evidence);
|
|
6109
|
-
}
|
|
6110
|
-
insertApiContract.finalize();
|
|
6111
|
-
const insertDep = db.prepare(`
|
|
6112
|
-
INSERT INTO dependencies (fromFile, toFile, kind)
|
|
6113
|
-
VALUES (?, ?, ?)
|
|
6114
|
-
`);
|
|
6115
|
-
for (const depSource of fileDependencies) {
|
|
6116
|
-
const resolved = resolveImportPath(relativePath, depSource, repoPath);
|
|
6117
|
-
if (resolved) {
|
|
6118
|
-
insertDep.run(relativePath, resolved, "import");
|
|
6119
|
-
}
|
|
6120
|
-
}
|
|
6121
|
-
if (tableDependencies) {
|
|
6122
|
-
for (const tableName of tableDependencies) {
|
|
6123
|
-
insertDep.run(relativePath, tableName, "table");
|
|
6124
|
-
}
|
|
6125
|
-
}
|
|
6126
|
-
if (ormDependencies) {
|
|
6127
|
-
for (const dep of ormDependencies) {
|
|
6128
|
-
insertDep.run(relativePath, dep.to, dep.kind);
|
|
6129
|
-
}
|
|
6130
|
-
}
|
|
6131
|
-
insertDep.finalize();
|
|
6132
|
-
const insertApexPlatformDependency = db.prepare("INSERT INTO dependencies (fromFile, toFile, kind, confidence, sourceEvidence) VALUES (?, ?, ?, ?, ?)");
|
|
6133
|
-
const seenApexPlatformDependencies = new Set();
|
|
6134
|
-
for (const dependency of apexPlatformDependencies) {
|
|
6135
|
-
const key = `${dependency.to}\0${dependency.kind}`;
|
|
6136
|
-
if (seenApexPlatformDependencies.has(key))
|
|
6137
|
-
continue;
|
|
6138
|
-
seenApexPlatformDependencies.add(key);
|
|
6139
|
-
insertApexPlatformDependency.run(relativePath, dependency.to, dependency.kind, 1, dependency.sourceEvidence);
|
|
6406
|
+
if (extraction.kind === "special") {
|
|
6407
|
+
const { content, parseContent, tree } = extraction;
|
|
6408
|
+
switch (extraction.route) {
|
|
6409
|
+
case "workday":
|
|
6410
|
+
await indexWorkdayFile(content, tree, relativePath, repoPath, db);
|
|
6411
|
+
return;
|
|
6412
|
+
case "visualforce":
|
|
6413
|
+
await indexVisualforceFile(content, tree, relativePath, repoPath, db);
|
|
6414
|
+
return;
|
|
6415
|
+
case "sql":
|
|
6416
|
+
await indexSqlFile(parseContent, tree, relativePath, repoPath, db);
|
|
6417
|
+
return;
|
|
6418
|
+
case "prisma":
|
|
6419
|
+
await indexPrismaFile(content, tree, relativePath, repoPath, db);
|
|
6420
|
+
return;
|
|
6140
6421
|
}
|
|
6141
|
-
insertApexPlatformDependency.finalize();
|
|
6142
|
-
db.run("COMMIT;");
|
|
6143
6422
|
}
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
throw error;
|
|
6423
|
+
if (collectResult) {
|
|
6424
|
+
collectResult(extraction.result);
|
|
6147
6425
|
}
|
|
6148
|
-
|
|
6149
|
-
|
|
6426
|
+
else {
|
|
6427
|
+
writeFileIndexResult(db, extraction.result);
|
|
6150
6428
|
}
|
|
6151
6429
|
}
|
|
6152
6430
|
catch (error) {
|
|
@@ -6252,7 +6530,10 @@ function reconcileTypeScriptDi(db, repoPath) {
|
|
|
6252
6530
|
.replace(/^type\s+/, "")
|
|
6253
6531
|
.split(/\s+as\s+/);
|
|
6254
6532
|
if (parts[0])
|
|
6255
|
-
imports.set(parts[1] ?? parts[0], {
|
|
6533
|
+
imports.set(parts[1] ?? parts[0], {
|
|
6534
|
+
original: parts[0],
|
|
6535
|
+
module: match[2],
|
|
6536
|
+
});
|
|
6256
6537
|
}
|
|
6257
6538
|
}
|
|
6258
6539
|
for (const match of masked.matchAll(/\bimport\s+([A-Za-z_$][\w$]*)\s+from\s*["']([^"']+)["']/g))
|
|
@@ -6307,11 +6588,15 @@ function reconcileTypeScriptDi(db, repoPath) {
|
|
|
6307
6588
|
return { reason: `Token import ${token} does not resolve uniquely.` };
|
|
6308
6589
|
const definitionFile = followReExports(repoPath, moduleFile, imported.original, 0);
|
|
6309
6590
|
if (!exactSymbol(definitionFile, imported.original))
|
|
6310
|
-
return {
|
|
6591
|
+
return {
|
|
6592
|
+
reason: `Token ${token} does not have one indexed declaration.`,
|
|
6593
|
+
};
|
|
6311
6594
|
return { value: `symbol:${definitionFile}#${imported.original}` };
|
|
6312
6595
|
}
|
|
6313
6596
|
if (!exactSymbol(file, token))
|
|
6314
|
-
return {
|
|
6597
|
+
return {
|
|
6598
|
+
reason: `Token ${token} does not have one indexed declaration.`,
|
|
6599
|
+
};
|
|
6315
6600
|
return { value: `symbol:${file}#${token}` };
|
|
6316
6601
|
};
|
|
6317
6602
|
const implementation = (name) => {
|
|
@@ -6471,7 +6756,10 @@ function reconcileTypeScriptDi(db, repoPath) {
|
|
|
6471
6756
|
? bindingCandidates.filter((binding) => binding.framework === resolution.framework && binding.token === resolution.token)
|
|
6472
6757
|
: (bindingsByToken.get(`${resolution.framework}\0${resolution.registry}\0${resolution.token}`) ?? []);
|
|
6473
6758
|
if (bindings.length === 1)
|
|
6474
|
-
candidates.push({
|
|
6759
|
+
candidates.push({
|
|
6760
|
+
...resolution,
|
|
6761
|
+
implementation: bindings[0].implementation,
|
|
6762
|
+
});
|
|
6475
6763
|
else
|
|
6476
6764
|
candidates.push({
|
|
6477
6765
|
...resolution,
|
|
@@ -6844,7 +7132,9 @@ function mirrorSnapshotFor(repoPath) {
|
|
|
6844
7132
|
const record = lookupMirror(repoPath);
|
|
6845
7133
|
if (!record)
|
|
6846
7134
|
return {};
|
|
6847
|
-
return {
|
|
7135
|
+
return {
|
|
7136
|
+
mirror: { url: record.url, sha: record.sha, fetchedAt: record.fetchedAt },
|
|
7137
|
+
};
|
|
6848
7138
|
}
|
|
6849
7139
|
/**
|
|
6850
7140
|
* Compares indexed symbols against embedded ones. A repository with no symbols
|
|
@@ -7196,6 +7486,37 @@ function parseGitStatusPaths(output) {
|
|
|
7196
7486
|
* Only files actually re-indexed here are re-embedded (indexEmbeddings is
|
|
7197
7487
|
* incremental), so a clean reconcile does no work.
|
|
7198
7488
|
*/
|
|
7489
|
+
/**
|
|
7490
|
+
* Full mtime/size comparison against `index_state`, used both when there is
|
|
7491
|
+
* no git HEAD at all and as a correctness fallback when a git-diff baseline
|
|
7492
|
+
* turns out to be unresolvable (see `reconcileIndex`). Not a substitute for
|
|
7493
|
+
* the git fast path in general: it only detects drift relative to *this*
|
|
7494
|
+
* checkout's own recorded state, so a database whose `index_state` rows came
|
|
7495
|
+
* from a different physical checkout (fresh mtimes on every file) degrades
|
|
7496
|
+
* to "everything changed" here rather than a cheap targeted diff.
|
|
7497
|
+
*/
|
|
7498
|
+
function detectDriftByStat(repoPath, db, canonicalFiles) {
|
|
7499
|
+
const changed = new Set();
|
|
7500
|
+
const state = new Map();
|
|
7501
|
+
for (const row of db
|
|
7502
|
+
.query("SELECT filePath, mtimeMs, size FROM index_state")
|
|
7503
|
+
.all()) {
|
|
7504
|
+
state.set(row.filePath, row);
|
|
7505
|
+
}
|
|
7506
|
+
for (const file of canonicalFiles) {
|
|
7507
|
+
const prev = state.get(file);
|
|
7508
|
+
try {
|
|
7509
|
+
const st = fs.statSync(path.join(repoPath, file));
|
|
7510
|
+
if (!prev || Math.abs(prev.mtimeMs - st.mtimeMs) >= 1.0 || prev.size !== st.size) {
|
|
7511
|
+
changed.add(file);
|
|
7512
|
+
}
|
|
7513
|
+
}
|
|
7514
|
+
catch (_) {
|
|
7515
|
+
changed.add(file);
|
|
7516
|
+
}
|
|
7517
|
+
}
|
|
7518
|
+
return changed;
|
|
7519
|
+
}
|
|
7199
7520
|
async function reconcileIndex(repoPath, db, progress) {
|
|
7200
7521
|
try {
|
|
7201
7522
|
progress?.("collecting-files", 0, "Checking existing local graph for changes");
|
|
@@ -7215,6 +7536,7 @@ async function reconcileIndex(repoPath, db, progress) {
|
|
|
7215
7536
|
changed.add(file);
|
|
7216
7537
|
}
|
|
7217
7538
|
catch (_) { }
|
|
7539
|
+
let baselineUnresolved = false;
|
|
7218
7540
|
if (lastHead && lastHead !== head) {
|
|
7219
7541
|
try {
|
|
7220
7542
|
const diff = child_process.execFileSync("git", ["diff", "--name-only", "-z", lastHead, head, "--"], {
|
|
@@ -7225,7 +7547,26 @@ async function reconcileIndex(repoPath, db, progress) {
|
|
|
7225
7547
|
for (const file of splitNulPaths(diff))
|
|
7226
7548
|
changed.add(file);
|
|
7227
7549
|
}
|
|
7228
|
-
catch (_) {
|
|
7550
|
+
catch (_) {
|
|
7551
|
+
// `lastHead` is not resolvable in this checkout's object database
|
|
7552
|
+
// (a shallow clone missing that commit, or a database seeded from
|
|
7553
|
+
// an unrelated checkout). Silently treating this as "nothing
|
|
7554
|
+
// changed" would let the graph advance `lastIndexedHead` to the
|
|
7555
|
+
// new HEAD below while having missed every real change in
|
|
7556
|
+
// between — fall back to a full comparison instead.
|
|
7557
|
+
baselineUnresolved = true;
|
|
7558
|
+
}
|
|
7559
|
+
}
|
|
7560
|
+
if (baselineUnresolved) {
|
|
7561
|
+
console.warn(`knodin: cached baseline commit ${lastHead} is not resolvable in this checkout (shallow clone or foreign database) — falling back to a full file-drift comparison`);
|
|
7562
|
+
canonicalFiles = collectRepoFiles(repoPath);
|
|
7563
|
+
if (canonicalFiles.length > 50000) {
|
|
7564
|
+
console.warn(`knodin: skipping full-drift fallback reconcile (${canonicalFiles.length} files exceeds 50k bound)`);
|
|
7565
|
+
}
|
|
7566
|
+
else {
|
|
7567
|
+
for (const file of detectDriftByStat(repoPath, db, canonicalFiles))
|
|
7568
|
+
changed.add(file);
|
|
7569
|
+
}
|
|
7229
7570
|
}
|
|
7230
7571
|
}
|
|
7231
7572
|
else {
|
|
@@ -7235,24 +7576,8 @@ async function reconcileIndex(repoPath, db, progress) {
|
|
|
7235
7576
|
console.warn(`knodin: skipping freshness reconcile (${canonicalFiles.length} files exceeds 50k bound)`);
|
|
7236
7577
|
return 0;
|
|
7237
7578
|
}
|
|
7238
|
-
const
|
|
7239
|
-
|
|
7240
|
-
.query("SELECT filePath, mtimeMs, size FROM index_state")
|
|
7241
|
-
.all()) {
|
|
7242
|
-
state.set(row.filePath, row);
|
|
7243
|
-
}
|
|
7244
|
-
for (const file of canonicalFiles) {
|
|
7245
|
-
const prev = state.get(file);
|
|
7246
|
-
try {
|
|
7247
|
-
const st = fs.statSync(path.join(repoPath, file));
|
|
7248
|
-
if (!prev || Math.abs(prev.mtimeMs - st.mtimeMs) >= 1.0 || prev.size !== st.size) {
|
|
7249
|
-
changed.add(file);
|
|
7250
|
-
}
|
|
7251
|
-
}
|
|
7252
|
-
catch (_) {
|
|
7253
|
-
changed.add(file);
|
|
7254
|
-
}
|
|
7255
|
-
}
|
|
7579
|
+
for (const file of detectDriftByStat(repoPath, db, canonicalFiles))
|
|
7580
|
+
changed.add(file);
|
|
7256
7581
|
}
|
|
7257
7582
|
// A row in index_state whose file is gone or no longer satisfies the
|
|
7258
7583
|
// canonical source policy must be purged. Keep those legacy rows in the
|
|
@@ -7332,6 +7657,7 @@ async function reconcileIndex(repoPath, db, progress) {
|
|
|
7332
7657
|
// "dirty, and indexed that way" from "dirty, and moved since".
|
|
7333
7658
|
if (head)
|
|
7334
7659
|
setMeta(db, "lastIndexedHead", head);
|
|
7660
|
+
setMeta(db, "knodinVersion", KNODIN_VERSION);
|
|
7335
7661
|
setMeta(db, "lastSuccessfulReconciliation", new Date().toISOString());
|
|
7336
7662
|
recordFreshnessBaseline(repoPath, db);
|
|
7337
7663
|
return reindexed;
|
|
@@ -7711,20 +8037,99 @@ async function indexRepo(repoPath, db, progress, skipEmbeddings = false, publish
|
|
|
7711
8037
|
db.run("DELETE FROM dependencies;");
|
|
7712
8038
|
db.run("DELETE FROM mcp_tools;");
|
|
7713
8039
|
db.run("DELETE FROM index_state;");
|
|
7714
|
-
// Process files sequentially to ensure thread-safe SQLite transactions
|
|
8040
|
+
// Process files sequentially to ensure thread-safe SQLite transactions.
|
|
8041
|
+
// Extraction (buildFileIndexResult, pure) still happens one file at a
|
|
8042
|
+
// time here; only the write is batched, via `pendingWrites`, so the
|
|
8043
|
+
// generic tree-sitter path amortizes one BEGIN/COMMIT over many files
|
|
8044
|
+
// instead of paying it per file (per-file failure isolation is preserved
|
|
8045
|
+
// through per-file SAVEPOINTs inside `writeFileIndexResultBatch`).
|
|
7715
8046
|
let lastYieldAt = Date.now();
|
|
7716
|
-
|
|
7717
|
-
|
|
7718
|
-
|
|
8047
|
+
const pendingWrites = [];
|
|
8048
|
+
const flushPendingWrites = () => {
|
|
8049
|
+
if (pendingWrites.length === 0)
|
|
8050
|
+
return;
|
|
8051
|
+
const failures = writeFileIndexResultBatch(db, pendingWrites);
|
|
8052
|
+
for (const failure of failures) {
|
|
8053
|
+
console.error(`Error indexing file ${failure.relativePath}:`, failure.error);
|
|
8054
|
+
tallyOne(unparsedByExtension, extensionBucket(failure.relativePath));
|
|
8055
|
+
}
|
|
8056
|
+
pendingWrites.length = 0;
|
|
8057
|
+
};
|
|
8058
|
+
// Progress is reported against a single counter so the pool and the
|
|
8059
|
+
// sequential path can interleave without the count going backwards.
|
|
8060
|
+
let processed = 0;
|
|
8061
|
+
const afterFile = async (file) => {
|
|
8062
|
+
if (pendingWrites.length >= INDEX_WRITE_BATCH_SIZE)
|
|
8063
|
+
flushPendingWrites();
|
|
7719
8064
|
recordIndexState(db, repoPath, file);
|
|
7720
|
-
|
|
8065
|
+
processed++;
|
|
8066
|
+
progress?.("indexing-files", processed, `Indexing ${files.length.toLocaleString()} files`, {
|
|
7721
8067
|
phaseTotal: files.length,
|
|
7722
8068
|
});
|
|
7723
8069
|
if (Date.now() - lastYieldAt >= INDEX_EVENT_LOOP_YIELD_MS) {
|
|
7724
8070
|
await yieldToIndexEventLoop();
|
|
7725
8071
|
lastYieldAt = Date.now();
|
|
7726
8072
|
}
|
|
8073
|
+
};
|
|
8074
|
+
const indexOneInline = async (file) => {
|
|
8075
|
+
await indexFile(path.join(repoPath, file), file, repoPath, db, unparsedByExtension, (result) => {
|
|
8076
|
+
pendingWrites.push(result);
|
|
8077
|
+
});
|
|
8078
|
+
};
|
|
8079
|
+
// Opt-in for now. `createParsePool` returns null whenever the machine says
|
|
8080
|
+
// not to spawn, so an unsuitable host silently keeps today's behavior
|
|
8081
|
+
// rather than discovering the problem partway through an index.
|
|
8082
|
+
const pool = parseWorkersEnabled() ? createParsePool() : null;
|
|
8083
|
+
try {
|
|
8084
|
+
if (pool) {
|
|
8085
|
+
// Only the generic tree-sitter route may leave the main thread; every
|
|
8086
|
+
// other route calls an indexer that writes to the database, and the
|
|
8087
|
+
// database has exactly one writer. `isWorkerEligibleFile` is derived
|
|
8088
|
+
// from the same classifier `indexFile` dispatches on, so the two
|
|
8089
|
+
// cannot disagree about where a file belongs.
|
|
8090
|
+
const eligible = [];
|
|
8091
|
+
const inline = [];
|
|
8092
|
+
for (const file of files) {
|
|
8093
|
+
if (isWorkerEligibleFile(path.join(repoPath, file), file))
|
|
8094
|
+
eligible.push({ absolutePath: path.join(repoPath, file), relativePath: file, repoPath });
|
|
8095
|
+
else
|
|
8096
|
+
inline.push(file);
|
|
8097
|
+
}
|
|
8098
|
+
await pool.run(eligible, async (task, outcome) => {
|
|
8099
|
+
switch (outcome.kind) {
|
|
8100
|
+
case "result":
|
|
8101
|
+
pendingWrites.push(outcome.result);
|
|
8102
|
+
break;
|
|
8103
|
+
case "unparsed":
|
|
8104
|
+
tallyOne(unparsedByExtension, extensionBucket(task.relativePath));
|
|
8105
|
+
break;
|
|
8106
|
+
case "skipped":
|
|
8107
|
+
break;
|
|
8108
|
+
default:
|
|
8109
|
+
// The pool could not finish this file (worker crash, or it
|
|
8110
|
+
// turned out to need a main-thread indexer). Never dropped:
|
|
8111
|
+
// run it through the sequential path exactly as before.
|
|
8112
|
+
await indexOneInline(task.relativePath);
|
|
8113
|
+
break;
|
|
8114
|
+
}
|
|
8115
|
+
await afterFile(task.relativePath);
|
|
8116
|
+
});
|
|
8117
|
+
for (const file of inline) {
|
|
8118
|
+
await indexOneInline(file);
|
|
8119
|
+
await afterFile(file);
|
|
8120
|
+
}
|
|
8121
|
+
}
|
|
8122
|
+
else {
|
|
8123
|
+
for (const file of files) {
|
|
8124
|
+
await indexOneInline(file);
|
|
8125
|
+
await afterFile(file);
|
|
8126
|
+
}
|
|
8127
|
+
}
|
|
8128
|
+
}
|
|
8129
|
+
finally {
|
|
8130
|
+
await pool?.close();
|
|
7727
8131
|
}
|
|
8132
|
+
flushPendingWrites();
|
|
7728
8133
|
progress?.("finalizing", 0, "Persisting symbol identities and semantic index");
|
|
7729
8134
|
persistSymbolIdentities(db, repoPath);
|
|
7730
8135
|
reconcileTypeScriptDi(db, repoPath);
|
|
@@ -7736,6 +8141,7 @@ async function indexRepo(repoPath, db, progress, skipEmbeddings = false, publish
|
|
|
7736
8141
|
await indexEmbeddings(db, repoPath, progress);
|
|
7737
8142
|
// Remember the HEAD we indexed at so a later cold start can diff forward.
|
|
7738
8143
|
setMeta(db, "lastIndexedHead", gitHead(repoPath) ?? "");
|
|
8144
|
+
setMeta(db, "knodinVersion", KNODIN_VERSION);
|
|
7739
8145
|
setMeta(db, "lastSuccessfulReconciliation", new Date().toISOString());
|
|
7740
8146
|
setMeta(db, "mcpBackfillVersion", "17");
|
|
7741
8147
|
setMeta(db, COVERAGE_UNPARSED_META_KEY, JSON.stringify(Object.fromEntries(unparsedByExtension)));
|
|
@@ -7932,7 +8338,7 @@ async function suspendFileWatcher(repoPath) {
|
|
|
7932
8338
|
return true;
|
|
7933
8339
|
}
|
|
7934
8340
|
/** Open and initialize SQLite connection and full repo index. */
|
|
7935
|
-
async function getOrInitDb(repoPath, options = {}) {
|
|
8341
|
+
export async function getOrInitDb(repoPath, options = {}) {
|
|
7936
8342
|
const normalizedPath = path.resolve(repoPath);
|
|
7937
8343
|
const selectedDatabasePath = explicitDatabasePath(normalizedPath, options.databasePath);
|
|
7938
8344
|
if (!selectedDatabasePath)
|
|
@@ -8025,6 +8431,8 @@ async function getOrInitDb(repoPath, options = {}) {
|
|
|
8025
8431
|
// newly indexable JSON files. v22 admits markdown as a link-only
|
|
8026
8432
|
// participant, so previously skipped documentation must be visited once
|
|
8027
8433
|
// to populate its doc_link / doc_link_broken edges.
|
|
8434
|
+
// v23 changes the embedding runtime from FP32 to q8. Source topology is
|
|
8435
|
+
// unchanged, so migrate in place and regenerate only semantic vectors.
|
|
8028
8436
|
// Highest version whose upgrade needs the stored data REBUILT. Versions
|
|
8029
8437
|
// above it migrate in place, so an upgrade costs a DELETE rather than a
|
|
8030
8438
|
// full re-index + re-embed (~40 min of CPU on an 18k-symbol corpus).
|
|
@@ -8034,6 +8442,7 @@ async function getOrInitDb(repoPath, options = {}) {
|
|
|
8034
8442
|
// v13 -> v14 purges embeddings orphaned while the FK cascade was inert.
|
|
8035
8443
|
// Runs after the CREATE TABLEs below, since the table must exist.
|
|
8036
8444
|
const needsOrphanPurge = storedVersion < KNODIN_SCHEMA_VERSION;
|
|
8445
|
+
const needsEmbeddingRepresentationRefresh = storedVersion >= 22 && storedVersion < 23;
|
|
8037
8446
|
if (storedVersion < LAST_REBUILD_SCHEMA_VERSION) {
|
|
8038
8447
|
db.run("DROP TRIGGER IF EXISTS after_symbol_insert;");
|
|
8039
8448
|
db.run("DROP TRIGGER IF EXISTS after_symbol_delete;");
|
|
@@ -8215,9 +8624,21 @@ async function getOrInitDb(repoPath, options = {}) {
|
|
|
8215
8624
|
DELETE FROM symbols_fts WHERE symbolId = old.id;
|
|
8216
8625
|
END;
|
|
8217
8626
|
`);
|
|
8627
|
+
// Scoped to the columns the FTS table actually mirrors. An unscoped
|
|
8628
|
+
// AFTER UPDATE fires for every column, so writing `identity` -- which
|
|
8629
|
+
// symbols_fts does not index -- rewrote an FTS row each time.
|
|
8630
|
+
// `persistSymbolIdentities` writes identity for every symbol on a full
|
|
8631
|
+
// index, and that alone measured 39.1s of a 40.0s pass (19,638 rows at
|
|
8632
|
+
// ~2ms each) on a 1,926-file corpus.
|
|
8633
|
+
//
|
|
8634
|
+
// Dropped unconditionally rather than guarded by `IF NOT EXISTS`,
|
|
8635
|
+
// because an existing database already carries the unscoped trigger and
|
|
8636
|
+
// would otherwise keep it forever. Recreating a trigger on open is
|
|
8637
|
+
// cheap and needs no schema-version bump or forced reindex.
|
|
8638
|
+
db.run("DROP TRIGGER IF EXISTS after_symbol_update;");
|
|
8218
8639
|
db.run(`
|
|
8219
|
-
CREATE TRIGGER
|
|
8220
|
-
AFTER UPDATE ON symbols
|
|
8640
|
+
CREATE TRIGGER after_symbol_update
|
|
8641
|
+
AFTER UPDATE OF name, kind, summary ON symbols
|
|
8221
8642
|
BEGIN
|
|
8222
8643
|
UPDATE symbols_fts
|
|
8223
8644
|
SET name = new.name,
|
|
@@ -8252,6 +8673,8 @@ async function getOrInitDb(repoPath, options = {}) {
|
|
|
8252
8673
|
if (needsOrphanPurge) {
|
|
8253
8674
|
purgeOrphanEmbeddings(db);
|
|
8254
8675
|
}
|
|
8676
|
+
if (needsEmbeddingRepresentationRefresh)
|
|
8677
|
+
db.run("DELETE FROM symbol_embeddings;");
|
|
8255
8678
|
finishDbOpenMigration();
|
|
8256
8679
|
// Check if we need to do a full index (e.g. if the database is brand new/empty)
|
|
8257
8680
|
let needsIndex = true;
|
|
@@ -8447,12 +8870,92 @@ function matchEndpoints(clientPath, serverPath) {
|
|
|
8447
8870
|
serverSegment.toLocaleLowerCase("en-US");
|
|
8448
8871
|
});
|
|
8449
8872
|
}
|
|
8450
|
-
/**
|
|
8451
|
-
|
|
8452
|
-
|
|
8453
|
-
|
|
8454
|
-
|
|
8455
|
-
|
|
8873
|
+
/**
|
|
8874
|
+
* Repositories being served from a sealed artifact rather than a working tree.
|
|
8875
|
+
*
|
|
8876
|
+
* Keyed by the repository path a caller passes in, so every existing reader
|
|
8877
|
+
* keeps its signature. This is the ONE place that decides where source bytes
|
|
8878
|
+
* come from: widening the chokepoint instead of threading a resolver through
|
|
8879
|
+
* ~20 call sites is what keeps a sealed answer from depending on whether a
|
|
8880
|
+
* particular reader remembered to ask.
|
|
8881
|
+
*/
|
|
8882
|
+
const sealedRepositories = new Map();
|
|
8883
|
+
export function registerSealedRepository(repoPath, db) {
|
|
8884
|
+
sealedRepositories.set(path.resolve(repoPath), db);
|
|
8885
|
+
}
|
|
8886
|
+
export function unregisterSealedRepository(repoPath) {
|
|
8887
|
+
sealedRepositories.delete(path.resolve(repoPath));
|
|
8888
|
+
}
|
|
8889
|
+
export function isSealedRepository(repoPath) {
|
|
8890
|
+
return sealedRepositories.has(path.resolve(repoPath));
|
|
8891
|
+
}
|
|
8892
|
+
/**
|
|
8893
|
+
* Whole-file source, from the sealed artifact when there is one.
|
|
8894
|
+
*
|
|
8895
|
+
* Returns an availability rather than a bare string, and NEVER `""` for
|
|
8896
|
+
* missing content: an empty string is indistinguishable from an empty file,
|
|
8897
|
+
* and downstream that reads as "this file has nothing in it" rather than "this
|
|
8898
|
+
* was not covered" — the silent false negative a consumer needs to detect in
|
|
8899
|
+
* order to fall back to live source.
|
|
8900
|
+
*/
|
|
8901
|
+
function readWholeSource(repoPath, relativePath) {
|
|
8902
|
+
const sealed = sealedRepositories.get(path.resolve(repoPath));
|
|
8903
|
+
if (sealed) {
|
|
8904
|
+
const text = readSealedSource(sealed, relativePath);
|
|
8905
|
+
return text === null
|
|
8906
|
+
? {
|
|
8907
|
+
text: "",
|
|
8908
|
+
availability: {
|
|
8909
|
+
state: "absent",
|
|
8910
|
+
reason: `not covered by the sealed artifact: ${relativePath}`,
|
|
8911
|
+
},
|
|
8912
|
+
}
|
|
8913
|
+
: { text, availability: { state: "available" } };
|
|
8914
|
+
}
|
|
8915
|
+
try {
|
|
8916
|
+
return {
|
|
8917
|
+
text: fs.readFileSync(path.resolve(repoPath, relativePath), "utf-8"),
|
|
8918
|
+
availability: { state: "available" },
|
|
8919
|
+
};
|
|
8920
|
+
}
|
|
8921
|
+
catch (error) {
|
|
8922
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
8923
|
+
return error?.code === "ENOENT"
|
|
8924
|
+
? { text: "", availability: { state: "absent", reason } }
|
|
8925
|
+
: { text: "", availability: { state: "read-failed", reason, retryable: true } };
|
|
8926
|
+
}
|
|
8927
|
+
}
|
|
8928
|
+
/**
|
|
8929
|
+
* Whether a file is present as source, without reading it.
|
|
8930
|
+
*
|
|
8931
|
+
* A sealed artifact answers from its embedded coverage, not the filesystem —
|
|
8932
|
+
* a sealed repository has no working tree, so `fs.existsSync` there would
|
|
8933
|
+
* report every file missing and quietly invert the meaning of any check built
|
|
8934
|
+
* on it (a test file "not present", a module path "unresolvable").
|
|
8935
|
+
*/
|
|
8936
|
+
function sourceFileExists(repoPath, relativePath) {
|
|
8937
|
+
const sealed = sealedRepositories.get(path.resolve(repoPath));
|
|
8938
|
+
if (sealed)
|
|
8939
|
+
return readSealedSource(sealed, relativePath) !== null;
|
|
8940
|
+
return fs.existsSync(path.resolve(repoPath, relativePath));
|
|
8941
|
+
}
|
|
8942
|
+
/**
|
|
8943
|
+
* Enumerate source files, from the sealed artifact's coverage when there is
|
|
8944
|
+
* one. A sealed repository has no directory tree, so anything that discovers
|
|
8945
|
+
* files by traversal must ask coverage instead of the filesystem.
|
|
8946
|
+
*/
|
|
8947
|
+
function listSourceFiles(repoPath, accept) {
|
|
8948
|
+
const sealed = sealedRepositories.get(path.resolve(repoPath));
|
|
8949
|
+
if (sealed)
|
|
8950
|
+
return listSealedFiles(sealed).filter(accept);
|
|
8951
|
+
return walkRepoFiles(repoPath, { accept });
|
|
8952
|
+
}
|
|
8953
|
+
/** Get verbatim source with line numbers, and say so when it is unavailable. */
|
|
8954
|
+
function readSourceRange(repoPath, relativePath, startLine, endLine, maxLines) {
|
|
8955
|
+
const whole = readWholeSource(repoPath, relativePath);
|
|
8956
|
+
if (whole.availability.state !== "available")
|
|
8957
|
+
return { text: "", availability: whole.availability };
|
|
8958
|
+
const content = whole.text;
|
|
8456
8959
|
const lines = content.split("\n");
|
|
8457
8960
|
const totalLines = endLine - startLine + 1;
|
|
8458
8961
|
let slice = lines.slice(startLine - 1, endLine);
|
|
@@ -8465,7 +8968,30 @@ function getSourceRange(repoPath, relativePath, startLine, endLine, maxLines) {
|
|
|
8465
8968
|
if (truncated) {
|
|
8466
8969
|
result += `\n... (truncated: showing lines ${startLine}-${startLine + slice.length - 1} of ${totalLines} total — use \`query large_functions\` to confirm size, or read the file directly for the remainder)`;
|
|
8467
8970
|
}
|
|
8468
|
-
return result;
|
|
8971
|
+
return { text: result, availability: { state: "available" } };
|
|
8972
|
+
}
|
|
8973
|
+
/**
|
|
8974
|
+
* One trimmed source line, used as edge evidence, or undefined if unreadable.
|
|
8975
|
+
*
|
|
8976
|
+
* The map builders previously read this inline and unguarded, so a single
|
|
8977
|
+
* missing file threw and aborted the ENTIRE map rather than costing one edge
|
|
8978
|
+
* its evidence string. Evidence is decoration on an edge; the edge itself is
|
|
8979
|
+
* derived from the graph and stands without it.
|
|
8980
|
+
*/
|
|
8981
|
+
function evidenceLine(root, relativePath, line) {
|
|
8982
|
+
const whole = readWholeSource(root, relativePath);
|
|
8983
|
+
if (whole.availability.state !== "available")
|
|
8984
|
+
return undefined;
|
|
8985
|
+
return whole.text.split(/\r?\n/)[line - 1]?.trim().slice(0, 240);
|
|
8986
|
+
}
|
|
8987
|
+
/**
|
|
8988
|
+
* Text-only view of `readSourceRange`, for callers that genuinely do not
|
|
8989
|
+
* surface availability. Kept deliberately thin and separate so that dropping
|
|
8990
|
+
* the availability is a visible choice at the call site rather than the
|
|
8991
|
+
* default behaviour of the only available helper.
|
|
8992
|
+
*/
|
|
8993
|
+
function getSourceRange(repoPath, relativePath, startLine, endLine, maxLines) {
|
|
8994
|
+
return readSourceRange(repoPath, relativePath, startLine, endLine, maxLines).text;
|
|
8469
8995
|
}
|
|
8470
8996
|
/** Recursively traverse backwards from callee to caller to build dependent trees. */
|
|
8471
8997
|
function _findCallersRecursive(db, targetSymbol, depth, visited = new Set()) {
|
|
@@ -10673,7 +11199,9 @@ function withStaleness(engine, claimRepository, openDb, openPolicy) {
|
|
|
10673
11199
|
},
|
|
10674
11200
|
};
|
|
10675
11201
|
}
|
|
10676
|
-
const DEFAULT_ENGINE_OPEN_POLICY = Object.freeze({
|
|
11202
|
+
const DEFAULT_ENGINE_OPEN_POLICY = Object.freeze({
|
|
11203
|
+
watcher: "enabled",
|
|
11204
|
+
});
|
|
10677
11205
|
/** Default SQLite + Tree-Sitter engine. */
|
|
10678
11206
|
export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
10679
11207
|
const owner = Symbol("knodin-engine");
|
|
@@ -10682,11 +11210,18 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
10682
11210
|
const candidateAudits = new Map();
|
|
10683
11211
|
const candidates = new Map();
|
|
10684
11212
|
let closed = false;
|
|
11213
|
+
const sealed = openPolicy.mode === "sealed";
|
|
10685
11214
|
const openDb = (repoPath, options = {}) => getOrInitDb(repoPath, {
|
|
10686
11215
|
...options,
|
|
10687
|
-
watcher: options.watcher ?? openPolicy.watcher,
|
|
11216
|
+
watcher: sealed ? "disabled" : (options.watcher ?? openPolicy.watcher),
|
|
10688
11217
|
watcherFileLimit: options.watcherFileLimit ?? openPolicy.watcherFileLimit,
|
|
10689
11218
|
databasePath: options.databasePath ?? openPolicy.databasePath,
|
|
11219
|
+
// A sealed artifact has no working tree. Reconciling against the empty
|
|
11220
|
+
// mount would find none of the indexed files on disk and prune every
|
|
11221
|
+
// symbol, so the artifact would answer "not found" for content it
|
|
11222
|
+
// actually carries — a negative that looks entirely correct.
|
|
11223
|
+
skipContentReconciliation: sealed || options.skipContentReconciliation,
|
|
11224
|
+
skipFreshnessGuard: sealed || options.skipFreshnessGuard,
|
|
10690
11225
|
});
|
|
10691
11226
|
const claimRepository = (repoPath) => {
|
|
10692
11227
|
if (closed)
|
|
@@ -10891,8 +11426,19 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
10891
11426
|
const header = fs.readFileSync(source).subarray(0, 16).toString("binary");
|
|
10892
11427
|
if (header !== "SQLite format 3\u0000")
|
|
10893
11428
|
throw new Error("candidate source is not a SQLite database");
|
|
10894
|
-
|
|
11429
|
+
const { reflinked } = reflinkCopyFile(source, candidate.databasePath);
|
|
11430
|
+
candidate.reflinked = reflinked;
|
|
10895
11431
|
fs.chmodSync(candidate.databasePath, 0o600);
|
|
11432
|
+
if (options.sourceLabel) {
|
|
11433
|
+
const stamp = new Database(candidate.databasePath);
|
|
11434
|
+
try {
|
|
11435
|
+
setMeta(stamp, "seededFromWorktree", options.sourceLabel);
|
|
11436
|
+
setMeta(stamp, "reflinkBacked", String(reflinked));
|
|
11437
|
+
}
|
|
11438
|
+
finally {
|
|
11439
|
+
stamp.close();
|
|
11440
|
+
}
|
|
11441
|
+
}
|
|
10896
11442
|
}
|
|
10897
11443
|
else {
|
|
10898
11444
|
await getOrInitDb(resolved, {
|
|
@@ -10979,11 +11525,27 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
10979
11525
|
}
|
|
10980
11526
|
}
|
|
10981
11527
|
}
|
|
11528
|
+
// A candidate populated from a foreign source (a downloaded shared-index
|
|
11529
|
+
// artifact, or another worktree's database) carries index_state mtimes
|
|
11530
|
+
// stamped wherever that source was built -- not this `resolved` checkout.
|
|
11531
|
+
// Every file outside `reconciled` is, by construction, content-identical
|
|
11532
|
+
// to what the caller already verified via its own diff, so refresh only
|
|
11533
|
+
// the mtime/size bookkeeping against this checkout's actual disk instead
|
|
11534
|
+
// of re-parsing/re-embedding: a plain stat, not a re-index.
|
|
11535
|
+
const reconciledSet = new Set(reconciled);
|
|
11536
|
+
const knownPaths = db
|
|
11537
|
+
.query("SELECT filePath FROM index_state")
|
|
11538
|
+
.all()
|
|
11539
|
+
.map((row) => row.filePath)
|
|
11540
|
+
.filter((file) => !reconciledSet.has(file));
|
|
11541
|
+
for (const file of knownPaths)
|
|
11542
|
+
recordIndexState(db, resolved, file);
|
|
10982
11543
|
await refreshFlowsForApexFiles(db, resolved, reconciled);
|
|
10983
11544
|
persistSymbolIdentities(db, resolved, reconciled);
|
|
10984
11545
|
reconcileTypeScriptDi(db, resolved);
|
|
10985
11546
|
await indexEmbeddings(db, resolved);
|
|
10986
11547
|
setMeta(db, "lastIndexedHead", gitHead(resolved) ?? "");
|
|
11548
|
+
setMeta(db, "knodinVersion", KNODIN_VERSION);
|
|
10987
11549
|
setMeta(db, "lastSuccessfulReconciliation", new Date().toISOString());
|
|
10988
11550
|
recordFreshnessBaseline(resolved, db);
|
|
10989
11551
|
return { reconciled };
|
|
@@ -11049,14 +11611,20 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
11049
11611
|
candidateAudits.set(candidate.id, databaseSha256);
|
|
11050
11612
|
else
|
|
11051
11613
|
candidateAudits.delete(candidate.id);
|
|
11052
|
-
return {
|
|
11614
|
+
return {
|
|
11615
|
+
healthy: issues.length === 0,
|
|
11616
|
+
issues: [...new Set(issues)],
|
|
11617
|
+
databaseSha256,
|
|
11618
|
+
};
|
|
11053
11619
|
},
|
|
11054
11620
|
async promoteCandidate(candidate) {
|
|
11055
11621
|
assertCandidate(candidate.repo, candidate);
|
|
11056
11622
|
const auditedDigest = candidateAudits.get(candidate.id);
|
|
11057
11623
|
if (!auditedDigest)
|
|
11058
11624
|
throw new Error("candidate must pass deep audit before promotion");
|
|
11059
|
-
const confirmation = await engine.auditCandidate(candidate, {
|
|
11625
|
+
const confirmation = await engine.auditCandidate(candidate, {
|
|
11626
|
+
mode: "deep",
|
|
11627
|
+
});
|
|
11060
11628
|
if (!confirmation.healthy || confirmation.databaseSha256 !== auditedDigest)
|
|
11061
11629
|
throw new Error("candidate or repository changed after deep audit");
|
|
11062
11630
|
closeCandidateDatabase(candidate);
|
|
@@ -11154,7 +11722,12 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
11154
11722
|
}
|
|
11155
11723
|
const summary = primaryDef.summary ||
|
|
11156
11724
|
`Symbol "${symbol}" is a ${primaryDef.kind} defined in ${primaryDef.filePath}.`;
|
|
11157
|
-
|
|
11725
|
+
// Availability is carried, not discarded: an empty `source` here would
|
|
11726
|
+
// otherwise be indistinguishable from the not-found and ambiguous
|
|
11727
|
+
// branches above, which also return "".
|
|
11728
|
+
const sourceRead = readSourceRange(targetRepoPath, primaryDef.filePath, primaryDef.startLine, primaryDef.endLine, 500);
|
|
11729
|
+
const source = sourceRead.text;
|
|
11730
|
+
const sourceAvailability = sourceRead.availability.state === "available" ? undefined : sourceRead.availability;
|
|
11158
11731
|
// DIRECT callers/callees (depth 1) — the immediate, navigable neighbours.
|
|
11159
11732
|
// A deep transitive closure is deliberately NOT returned: on a large repo
|
|
11160
11733
|
// it explodes to thousands of (often name-conflated) nodes and megabytes
|
|
@@ -11185,10 +11758,11 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
11185
11758
|
confidence: c.confidence,
|
|
11186
11759
|
...(c.kind?.startsWith("di_")
|
|
11187
11760
|
? {
|
|
11188
|
-
|
|
11189
|
-
|
|
11190
|
-
|
|
11191
|
-
|
|
11761
|
+
// Was an unguarded read: a single missing file threw and aborted
|
|
11762
|
+
// the whole `explain`, rather than costing one caller its
|
|
11763
|
+
// evidence string. `evidenceLine` also serves a sealed
|
|
11764
|
+
// repository, which has no working tree to read from.
|
|
11765
|
+
sourceEvidence: evidenceLine(c.repoPath || targetRepoPath, c.evidenceFile ?? c.filePath, c.evidenceLine ?? c.lineNumber) ?? "",
|
|
11192
11766
|
}
|
|
11193
11767
|
: {}),
|
|
11194
11768
|
...(definition
|
|
@@ -11209,15 +11783,7 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
11209
11783
|
.all(primaryDef.name, primaryDef.filePath)
|
|
11210
11784
|
.map((row) => {
|
|
11211
11785
|
const separator = row.calleeSymbol.indexOf(":");
|
|
11212
|
-
|
|
11213
|
-
try {
|
|
11214
|
-
evidence =
|
|
11215
|
-
fs
|
|
11216
|
-
.readFileSync(path.join(targetRepoPath, row.callerFile), "utf8")
|
|
11217
|
-
.split(/\r?\n/)[row.line - 1]?.trim()
|
|
11218
|
-
.slice(0, 240) ?? "";
|
|
11219
|
-
}
|
|
11220
|
-
catch { }
|
|
11786
|
+
const evidence = evidenceLine(targetRepoPath, row.callerFile, row.line) ?? "";
|
|
11221
11787
|
return {
|
|
11222
11788
|
kind: separator < 0 ? "di_unresolved" : row.calleeSymbol.slice(0, separator),
|
|
11223
11789
|
reason: separator < 0 ? row.calleeSymbol : row.calleeSymbol.slice(separator + 1),
|
|
@@ -11240,12 +11806,16 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
11240
11806
|
symbol,
|
|
11241
11807
|
summary,
|
|
11242
11808
|
source,
|
|
11809
|
+
...(sourceAvailability ? { sourceAvailability } : {}),
|
|
11243
11810
|
callers: allCallers.slice(0, cap),
|
|
11244
11811
|
callerCount: allCallers.length,
|
|
11245
11812
|
callees: allCallees.slice(0, cap),
|
|
11246
11813
|
calleeCount: allCallees.length,
|
|
11247
11814
|
...(omissions.length > 0
|
|
11248
|
-
? {
|
|
11815
|
+
? {
|
|
11816
|
+
omissions: omissions.slice(0, cap),
|
|
11817
|
+
omissionCount: omissions.length,
|
|
11818
|
+
}
|
|
11249
11819
|
: {}),
|
|
11250
11820
|
blastRadius: allBlastFiles.slice(0, cap),
|
|
11251
11821
|
blastRadiusCount: allBlastFiles.length,
|
|
@@ -11283,7 +11853,10 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
11283
11853
|
let hasTest = false;
|
|
11284
11854
|
const testPaths = getTestFilePaths(filePath);
|
|
11285
11855
|
for (const tp of testPaths) {
|
|
11286
|
-
|
|
11856
|
+
// Sealed repositories have no working tree, so an `fs.existsSync`
|
|
11857
|
+
// here would report every test file missing and invert the
|
|
11858
|
+
// meaning of this flag rather than failing visibly.
|
|
11859
|
+
if (sourceFileExists(repoPath, tp)) {
|
|
11287
11860
|
hasTest = true;
|
|
11288
11861
|
break;
|
|
11289
11862
|
}
|
|
@@ -11292,13 +11865,16 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
11292
11865
|
let tree = null;
|
|
11293
11866
|
try {
|
|
11294
11867
|
const absPath = path.resolve(repoPath, filePath);
|
|
11295
|
-
|
|
11296
|
-
|
|
11868
|
+
// Content comes from the resolver so a sealed repository can be
|
|
11869
|
+
// reviewed; the language is still chosen from the path, which
|
|
11870
|
+
// needs no working tree.
|
|
11871
|
+
const source = readWholeSource(repoPath, filePath);
|
|
11872
|
+
if (source.availability.state === "available") {
|
|
11297
11873
|
const lang = await getLanguageForFile(absPath);
|
|
11298
11874
|
if (lang) {
|
|
11299
11875
|
const parser = new Parser();
|
|
11300
11876
|
parser.setLanguage(lang);
|
|
11301
|
-
tree = parser.parse(
|
|
11877
|
+
tree = parser.parse(source.text);
|
|
11302
11878
|
}
|
|
11303
11879
|
}
|
|
11304
11880
|
}
|
|
@@ -11675,10 +12251,7 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
11675
12251
|
if (!calleeFile || calleeFile === r.callerFile)
|
|
11676
12252
|
continue;
|
|
11677
12253
|
addExtractedEdge(r.callerFile, calleeFile, r.kind.startsWith("di_") ? r.kind : "calls", r.confidence, r.kind.startsWith("di_")
|
|
11678
|
-
?
|
|
11679
|
-
.readFileSync(path.join(resolvedRepoPath, r.callerFile), "utf8")
|
|
11680
|
-
.split(/\r?\n/)[r.line - 1]?.trim()
|
|
11681
|
-
.slice(0, 240)
|
|
12254
|
+
? evidenceLine(resolvedRepoPath, r.callerFile, r.line)
|
|
11682
12255
|
: undefined);
|
|
11683
12256
|
}
|
|
11684
12257
|
return {
|
|
@@ -11792,10 +12365,7 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
11792
12365
|
}
|
|
11793
12366
|
if (calleePrefixed && calleePrefixed !== callerPrefixed) {
|
|
11794
12367
|
addExtractedEdge(callerPrefixed, calleePrefixed, r.kind.startsWith("di_") ? r.kind : "calls", r.confidence, r.kind.startsWith("di_")
|
|
11795
|
-
?
|
|
11796
|
-
.readFileSync(path.join(repo.path, r.callerFile), "utf8")
|
|
11797
|
-
.split(/\r?\n/)[r.line - 1]?.trim()
|
|
11798
|
-
.slice(0, 240)
|
|
12368
|
+
? evidenceLine(repo.path, r.callerFile, r.line)
|
|
11799
12369
|
: undefined);
|
|
11800
12370
|
}
|
|
11801
12371
|
// 2. Cross-repo Routing: If it does not exist locally but matches endpoints in other repos
|
|
@@ -12209,8 +12779,12 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
12209
12779
|
const count = (sql) => db.query(sql).get()?.count ?? 0;
|
|
12210
12780
|
const orphaned = {
|
|
12211
12781
|
embeddings: count(`SELECT COUNT(*) count FROM symbol_embeddings WHERE ${ORPHANED_EMBEDDING_PREDICATE}`),
|
|
12212
|
-
references: count(
|
|
12213
|
-
|
|
12782
|
+
references: count(`SELECT COUNT(*) count FROM "references" r
|
|
12783
|
+
WHERE NOT EXISTS (SELECT 1 FROM index_state i WHERE i.filePath=r.callerFile)
|
|
12784
|
+
AND NOT (r.callerFile LIKE '.knodin/dbt/%' AND EXISTS (SELECT 1 FROM symbols s WHERE s.filePath=r.callerFile))`),
|
|
12785
|
+
dependencies: count(`SELECT COUNT(*) count FROM dependencies d
|
|
12786
|
+
WHERE NOT EXISTS (SELECT 1 FROM index_state i WHERE i.filePath=d.fromFile)
|
|
12787
|
+
AND NOT (d.fromFile LIKE '.knodin/dbt/%' AND EXISTS (SELECT 1 FROM symbols s WHERE s.filePath=d.fromFile))`),
|
|
12214
12788
|
};
|
|
12215
12789
|
const records = staleFiles.map((file) => `indexed file no longer exists: ${file}`);
|
|
12216
12790
|
for (const file of ineligibleFiles)
|
|
@@ -12253,9 +12827,18 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
12253
12827
|
missing: { files: missingFiles, records },
|
|
12254
12828
|
lastSuccessfulReconciliation: getMeta(db, "lastSuccessfulReconciliation") ?? null,
|
|
12255
12829
|
lastIndexedHead: getMeta(db, "lastIndexedHead") ?? "",
|
|
12830
|
+
seededFromWorktree: getMeta(db, "seededFromWorktree") || undefined,
|
|
12831
|
+
reflinkBacked: (() => {
|
|
12832
|
+
const value = getMeta(db, "reflinkBacked");
|
|
12833
|
+
return value === "" || value === null ? undefined : value === "true";
|
|
12834
|
+
})(),
|
|
12256
12835
|
freshness,
|
|
12257
12836
|
indexGeneration,
|
|
12258
|
-
verification: {
|
|
12837
|
+
verification: {
|
|
12838
|
+
mode: "deep-audit",
|
|
12839
|
+
auditVerifiedAt: verifiedAt,
|
|
12840
|
+
verifiedAt,
|
|
12841
|
+
},
|
|
12259
12842
|
freshnessMechanism: freshnessMechanismFor(resolved, openPolicy),
|
|
12260
12843
|
repairSteps: needsRepair
|
|
12261
12844
|
? [
|
|
@@ -12636,6 +13219,7 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
12636
13219
|
if (after.freshness.state === "stale-head" || after.freshness.state === "unknown") {
|
|
12637
13220
|
setMeta(db, "lastIndexedHead", gitHead(resolved) ?? "");
|
|
12638
13221
|
}
|
|
13222
|
+
setMeta(db, "knodinVersion", KNODIN_VERSION);
|
|
12639
13223
|
setMeta(db, "lastSuccessfulReconciliation", new Date().toISOString());
|
|
12640
13224
|
recordFreshnessBaseline(resolved, db);
|
|
12641
13225
|
statusCache.delete(databaseCacheKey(resolved, openPolicy.databasePath));
|
|
@@ -12736,292 +13320,308 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
12736
13320
|
},
|
|
12737
13321
|
async index(repoPath, files, clean = false, options) {
|
|
12738
13322
|
const resolved = path.resolve(repoPath);
|
|
12739
|
-
|
|
12740
|
-
|
|
12741
|
-
|
|
12742
|
-
|
|
12743
|
-
|
|
12744
|
-
|
|
12745
|
-
|
|
12746
|
-
|
|
12747
|
-
|
|
13323
|
+
const indexCoordination = acquireLifecycleCoordination({
|
|
13324
|
+
command: "index",
|
|
13325
|
+
repository: resolved,
|
|
13326
|
+
verificationToken: process.env.KNODIN_UPDATE_VERIFICATION_TOKEN,
|
|
13327
|
+
});
|
|
13328
|
+
try {
|
|
13329
|
+
if (clean &&
|
|
13330
|
+
(!files || files.length === 0) &&
|
|
13331
|
+
!openPolicy.databasePath &&
|
|
13332
|
+
!process.env.VITEST &&
|
|
13333
|
+
process.env.NODE_ENV !== "test") {
|
|
13334
|
+
const progress = createIndexProgressReporter(options?.onProgress);
|
|
13335
|
+
progress("starting", 0, "Creating isolated clean-index candidate");
|
|
13336
|
+
const candidate = await engine.createCandidate(resolved);
|
|
13337
|
+
let promoted = false;
|
|
13338
|
+
try {
|
|
13339
|
+
const reconciliation = await engine.reconcileCandidate(candidate, resolved, undefined, {
|
|
13340
|
+
onProgress: progress,
|
|
13341
|
+
skipEmbeddings: options?.skipEmbeddings,
|
|
13342
|
+
});
|
|
13343
|
+
progress("verifying", 0, "Deep-auditing clean-index candidate", {
|
|
13344
|
+
phaseTotal: 1,
|
|
13345
|
+
});
|
|
13346
|
+
const audit = await engine.auditCandidate(candidate, {
|
|
13347
|
+
mode: "deep",
|
|
13348
|
+
});
|
|
13349
|
+
if (!audit.healthy)
|
|
13350
|
+
throw new Error(`clean-index candidate audit failed: ${audit.issues.join(", ")}`);
|
|
13351
|
+
await engine.promoteCandidate(candidate);
|
|
13352
|
+
promoted = true;
|
|
13353
|
+
const activeDb = await openDb(resolved, {
|
|
13354
|
+
skipContentReconciliation: true,
|
|
13355
|
+
skipFreshnessGuard: true,
|
|
13356
|
+
watcher: openPolicy.watcher,
|
|
13357
|
+
});
|
|
13358
|
+
if (!process.env.VITEST && process.env.NODE_ENV !== "test")
|
|
13359
|
+
publishStructuralSnapshot(resolved, activeDb);
|
|
13360
|
+
const health = await engine.status(resolved, {
|
|
13361
|
+
audit: "deep",
|
|
13362
|
+
ignoreActiveOperation: true,
|
|
13363
|
+
});
|
|
13364
|
+
const issueCount = health.missing.files.length + health.missing.records.length;
|
|
13365
|
+
progress("verifying", 1, health.status === "healthy"
|
|
13366
|
+
? "Clean-index candidate promoted and verified"
|
|
13367
|
+
: `Promoted graph has ${issueCount.toLocaleString()} issue(s)`, { phaseTotal: 1 });
|
|
13368
|
+
progress("completed", reconciliation.reconciled.length, "Clean index completed", {
|
|
13369
|
+
phaseTotal: reconciliation.reconciled.length,
|
|
13370
|
+
});
|
|
13371
|
+
return {
|
|
13372
|
+
indexed: reconciliation.reconciled,
|
|
13373
|
+
unchanged: [],
|
|
13374
|
+
verification: {
|
|
13375
|
+
status: health.status,
|
|
13376
|
+
issueCount,
|
|
13377
|
+
missing: health.missing,
|
|
13378
|
+
verifiedAt: health.verification.verifiedAt,
|
|
13379
|
+
},
|
|
13380
|
+
};
|
|
13381
|
+
}
|
|
13382
|
+
catch (error) {
|
|
13383
|
+
if (!promoted && candidates.has(candidate.id))
|
|
13384
|
+
await engine.discardCandidate(candidate);
|
|
13385
|
+
if (openPolicy.watcher === "enabled" && fs.existsSync(resolveDbPath(resolved)))
|
|
13386
|
+
await openDb(resolved, {
|
|
13387
|
+
skipContentReconciliation: true,
|
|
13388
|
+
skipFreshnessGuard: true,
|
|
13389
|
+
});
|
|
13390
|
+
throw error;
|
|
13391
|
+
}
|
|
13392
|
+
}
|
|
13393
|
+
const watcherStateBefore = watcherStates.get(resolved)?.watcher;
|
|
13394
|
+
if (!openPolicy.databasePath)
|
|
13395
|
+
await suspendFileWatcher(resolved);
|
|
12748
13396
|
try {
|
|
12749
|
-
|
|
13397
|
+
invalidateResolutionCaches(repoPath);
|
|
13398
|
+
const indexed = [];
|
|
13399
|
+
const unchanged = [];
|
|
13400
|
+
let scipReport;
|
|
13401
|
+
let sarifReport;
|
|
13402
|
+
const progress = createIndexProgressReporter(options?.onProgress);
|
|
13403
|
+
progress("starting", 0, "Opening local graph database");
|
|
13404
|
+
const db = await openDb(repoPath, {
|
|
12750
13405
|
onProgress: progress,
|
|
12751
|
-
|
|
12752
|
-
|
|
12753
|
-
|
|
12754
|
-
|
|
12755
|
-
if (!audit.healthy)
|
|
12756
|
-
throw new Error(`clean-index candidate audit failed: ${audit.issues.join(", ")}`);
|
|
12757
|
-
await engine.promoteCandidate(candidate);
|
|
12758
|
-
promoted = true;
|
|
12759
|
-
const activeDb = await openDb(resolved, {
|
|
13406
|
+
watcher: "disabled",
|
|
13407
|
+
// The explicit lifecycle operation owns all content work. Otherwise
|
|
13408
|
+
// getOrInitDb's freshness guard can perform a full, invisible
|
|
13409
|
+
// reconciliation before a targeted index even starts.
|
|
12760
13410
|
skipContentReconciliation: true,
|
|
12761
13411
|
skipFreshnessGuard: true,
|
|
12762
|
-
watcher: openPolicy.watcher,
|
|
12763
|
-
});
|
|
12764
|
-
if (!process.env.VITEST && process.env.NODE_ENV !== "test")
|
|
12765
|
-
publishStructuralSnapshot(resolved, activeDb);
|
|
12766
|
-
const health = await engine.status(resolved, {
|
|
12767
|
-
audit: "deep",
|
|
12768
|
-
ignoreActiveOperation: true,
|
|
12769
|
-
});
|
|
12770
|
-
const issueCount = health.missing.files.length + health.missing.records.length;
|
|
12771
|
-
progress("verifying", 1, health.status === "healthy"
|
|
12772
|
-
? "Clean-index candidate promoted and verified"
|
|
12773
|
-
: `Promoted graph has ${issueCount.toLocaleString()} issue(s)`, { phaseTotal: 1 });
|
|
12774
|
-
progress("completed", reconciliation.reconciled.length, "Clean index completed", {
|
|
12775
|
-
phaseTotal: reconciliation.reconciled.length,
|
|
12776
13412
|
});
|
|
12777
|
-
|
|
12778
|
-
|
|
12779
|
-
|
|
12780
|
-
|
|
12781
|
-
status: health.status,
|
|
12782
|
-
issueCount,
|
|
12783
|
-
missing: health.missing,
|
|
12784
|
-
verifiedAt: health.verification.verifiedAt,
|
|
12785
|
-
},
|
|
12786
|
-
};
|
|
12787
|
-
}
|
|
12788
|
-
catch (error) {
|
|
12789
|
-
if (!promoted && candidates.has(candidate.id))
|
|
12790
|
-
await engine.discardCandidate(candidate);
|
|
12791
|
-
if (openPolicy.watcher === "enabled" && fs.existsSync(resolveDbPath(resolved)))
|
|
12792
|
-
await openDb(resolved, {
|
|
12793
|
-
skipContentReconciliation: true,
|
|
12794
|
-
skipFreshnessGuard: true,
|
|
13413
|
+
let completionMessage = "Local graph is current";
|
|
13414
|
+
if (files && files.length > 0) {
|
|
13415
|
+
progress("indexing-files", 0, `Checking ${files.length.toLocaleString()} requested files`, {
|
|
13416
|
+
phaseTotal: files.length,
|
|
12795
13417
|
});
|
|
12796
|
-
|
|
12797
|
-
|
|
12798
|
-
|
|
12799
|
-
|
|
12800
|
-
|
|
12801
|
-
|
|
12802
|
-
|
|
12803
|
-
|
|
12804
|
-
|
|
12805
|
-
|
|
12806
|
-
|
|
12807
|
-
|
|
12808
|
-
|
|
12809
|
-
|
|
12810
|
-
|
|
12811
|
-
|
|
12812
|
-
|
|
12813
|
-
|
|
12814
|
-
|
|
12815
|
-
|
|
12816
|
-
|
|
12817
|
-
|
|
12818
|
-
|
|
12819
|
-
|
|
12820
|
-
|
|
12821
|
-
|
|
12822
|
-
|
|
12823
|
-
|
|
12824
|
-
|
|
12825
|
-
|
|
12826
|
-
const absPath = path.resolve(repoPath, relativePath);
|
|
12827
|
-
const needsUpdate = clean || fileDriftedFromIndexState(db, repoPath, relativePath);
|
|
12828
|
-
if (!needsUpdate) {
|
|
12829
|
-
unchanged.push(relativePath);
|
|
13418
|
+
for (const [fileIndex, file] of files.entries()) {
|
|
13419
|
+
const relativePath = path.isAbsolute(file) ? path.relative(repoPath, file) : file;
|
|
13420
|
+
const absPath = path.resolve(repoPath, relativePath);
|
|
13421
|
+
const needsUpdate = clean || fileDriftedFromIndexState(db, repoPath, relativePath);
|
|
13422
|
+
if (!needsUpdate) {
|
|
13423
|
+
unchanged.push(relativePath);
|
|
13424
|
+
}
|
|
13425
|
+
else if (fs.existsSync(absPath)) {
|
|
13426
|
+
await indexFile(absPath, relativePath, repoPath, db);
|
|
13427
|
+
recordIndexState(db, repoPath, relativePath);
|
|
13428
|
+
}
|
|
13429
|
+
else {
|
|
13430
|
+
// File was deleted
|
|
13431
|
+
db.run("BEGIN TRANSACTION;");
|
|
13432
|
+
try {
|
|
13433
|
+
deleteSymbolsForFile(db, relativePath);
|
|
13434
|
+
db.run('DELETE FROM "references" WHERE callerFile = ?', [relativePath]);
|
|
13435
|
+
db.run("DELETE FROM dependencies WHERE fromFile = ?", [relativePath]);
|
|
13436
|
+
db.run("DELETE FROM mcp_tools WHERE filePath = ?", [relativePath]);
|
|
13437
|
+
db.run("COMMIT;");
|
|
13438
|
+
}
|
|
13439
|
+
catch (e) {
|
|
13440
|
+
db.run("ROLLBACK;");
|
|
13441
|
+
console.error(`Error deleting file from index: ${relativePath}`, e);
|
|
13442
|
+
}
|
|
13443
|
+
removeIndexState(db, relativePath);
|
|
13444
|
+
}
|
|
13445
|
+
if (needsUpdate)
|
|
13446
|
+
indexed.push(relativePath);
|
|
13447
|
+
progress("indexing-files", fileIndex + 1, `Checking ${files.length.toLocaleString()} requested files`, { phaseTotal: files.length });
|
|
12830
13448
|
}
|
|
12831
|
-
|
|
12832
|
-
|
|
12833
|
-
|
|
13449
|
+
const refreshedFlows = await refreshFlowsForApexFiles(db, repoPath, indexed);
|
|
13450
|
+
for (const flow of refreshedFlows)
|
|
13451
|
+
if (!indexed.includes(flow))
|
|
13452
|
+
indexed.push(flow);
|
|
13453
|
+
if (indexed.length > 0) {
|
|
13454
|
+
progress("finalizing", 0, "Refreshing identities and semantic embeddings");
|
|
13455
|
+
persistSymbolIdentities(db, repoPath, indexed);
|
|
13456
|
+
reconcileTypeScriptDi(db, repoPath);
|
|
13457
|
+
await indexEmbeddings(db, repoPath, progress);
|
|
13458
|
+
indexGeneration++;
|
|
12834
13459
|
}
|
|
12835
|
-
|
|
12836
|
-
|
|
12837
|
-
|
|
12838
|
-
|
|
12839
|
-
|
|
12840
|
-
|
|
12841
|
-
|
|
12842
|
-
|
|
12843
|
-
|
|
12844
|
-
|
|
12845
|
-
|
|
12846
|
-
|
|
12847
|
-
|
|
13460
|
+
const alreadyCurrent = unchanged.length === 1
|
|
13461
|
+
? "1 requested file already current"
|
|
13462
|
+
: `${unchanged.length.toLocaleString()} requested files already current`;
|
|
13463
|
+
completionMessage =
|
|
13464
|
+
indexed.length === 0
|
|
13465
|
+
? alreadyCurrent
|
|
13466
|
+
: `Indexed ${indexed.length.toLocaleString()} requested file(s); ${alreadyCurrent}`;
|
|
13467
|
+
}
|
|
13468
|
+
else {
|
|
13469
|
+
// Full re-index or incremental sync
|
|
13470
|
+
let hasSymbols = false;
|
|
13471
|
+
try {
|
|
13472
|
+
const countResult = db
|
|
13473
|
+
.query("SELECT COUNT(*) as count FROM symbols")
|
|
13474
|
+
.all();
|
|
13475
|
+
if (countResult && countResult.length > 0 && countResult[0].count > 0) {
|
|
13476
|
+
hasSymbols = true;
|
|
12848
13477
|
}
|
|
12849
|
-
removeIndexState(db, relativePath);
|
|
12850
13478
|
}
|
|
12851
|
-
|
|
12852
|
-
|
|
12853
|
-
|
|
12854
|
-
|
|
12855
|
-
|
|
12856
|
-
|
|
12857
|
-
|
|
12858
|
-
|
|
12859
|
-
|
|
12860
|
-
|
|
12861
|
-
|
|
12862
|
-
|
|
13479
|
+
catch (_) { }
|
|
13480
|
+
if (!clean && hasSymbols) {
|
|
13481
|
+
progress("collecting-files", 0, "Checking existing local graph for changes");
|
|
13482
|
+
await reconcileIndex(repoPath, db, progress);
|
|
13483
|
+
}
|
|
13484
|
+
else {
|
|
13485
|
+
await indexRepo(repoPath, db, progress, options?.skipEmbeddings === true);
|
|
13486
|
+
}
|
|
13487
|
+
// Reuse the same candidate set as indexing/reconciliation. This keeps
|
|
13488
|
+
// LWC HTML/CSS bundle members visible in the response without admitting
|
|
13489
|
+
// arbitrary web assets to the source index.
|
|
13490
|
+
// Appended in a loop, not `push(...files)`: spreading an array as
|
|
13491
|
+
// arguments overflows the call stack once a repository is large
|
|
13492
|
+
// enough. Observed on a 40k-file checkout, which is well within the
|
|
13493
|
+
// range knodin is meant to index.
|
|
13494
|
+
for (const file of collectRepoFiles(repoPath))
|
|
13495
|
+
indexed.push(file);
|
|
13496
|
+
completionMessage = `${indexed.length.toLocaleString()} repository file(s) are current`;
|
|
13497
|
+
}
|
|
13498
|
+
if (options?.scip) {
|
|
13499
|
+
const facts = readScipIndex(repoPath, options.scip);
|
|
13500
|
+
const affectedFiles = persistScipFacts(db, repoPath, facts);
|
|
13501
|
+
progress("finalizing", 0, "Refreshing SCIP identities and semantic embeddings");
|
|
12863
13502
|
await indexEmbeddings(db, repoPath, progress);
|
|
12864
13503
|
indexGeneration++;
|
|
13504
|
+
const bounds = { ...SCIP_DEFAULT_LIMITS, ...options.scip };
|
|
13505
|
+
scipReport = {
|
|
13506
|
+
provenance: "scip",
|
|
13507
|
+
inputPath: facts.inputPath,
|
|
13508
|
+
bytes: facts.bytes,
|
|
13509
|
+
files: facts.files,
|
|
13510
|
+
facts: facts.facts,
|
|
13511
|
+
languages: facts.languages,
|
|
13512
|
+
indexers: facts.indexers,
|
|
13513
|
+
elapsedMs: facts.elapsedMs,
|
|
13514
|
+
bounds: {
|
|
13515
|
+
maxBytes: bounds.maxBytes,
|
|
13516
|
+
maxFiles: bounds.maxFiles,
|
|
13517
|
+
maxFacts: bounds.maxFacts,
|
|
13518
|
+
timeoutMs: bounds.timeoutMs,
|
|
13519
|
+
},
|
|
13520
|
+
};
|
|
13521
|
+
for (const file of affectedFiles)
|
|
13522
|
+
if (!indexed.includes(file))
|
|
13523
|
+
indexed.push(file);
|
|
12865
13524
|
}
|
|
12866
|
-
|
|
12867
|
-
|
|
12868
|
-
|
|
12869
|
-
|
|
12870
|
-
|
|
12871
|
-
|
|
12872
|
-
|
|
12873
|
-
|
|
12874
|
-
|
|
12875
|
-
|
|
12876
|
-
|
|
12877
|
-
|
|
12878
|
-
|
|
12879
|
-
.
|
|
12880
|
-
.
|
|
12881
|
-
|
|
12882
|
-
|
|
12883
|
-
|
|
12884
|
-
|
|
12885
|
-
|
|
12886
|
-
|
|
12887
|
-
|
|
12888
|
-
await reconcileIndex(repoPath, db, progress);
|
|
13525
|
+
if (options?.sarif) {
|
|
13526
|
+
const facts = readSarifLog(repoPath, options.sarif);
|
|
13527
|
+
persistSarifFindings(db, facts);
|
|
13528
|
+
// Findings are not graph facts, so no re-embedding and no generation
|
|
13529
|
+
// bump: the graph is unchanged and existing answers stay valid.
|
|
13530
|
+
const bounds = { ...SARIF_DEFAULT_LIMITS, ...options.sarif };
|
|
13531
|
+
sarifReport = {
|
|
13532
|
+
provenance: "sarif",
|
|
13533
|
+
inputPath: facts.inputPath,
|
|
13534
|
+
bytes: facts.bytes,
|
|
13535
|
+
tools: facts.tools,
|
|
13536
|
+
rules: facts.rules,
|
|
13537
|
+
files: facts.files,
|
|
13538
|
+
findings: facts.findings.length,
|
|
13539
|
+
skippedUnresolved: facts.skippedUnresolved,
|
|
13540
|
+
elapsedMs: facts.elapsedMs,
|
|
13541
|
+
bounds: {
|
|
13542
|
+
maxBytes: bounds.maxBytes,
|
|
13543
|
+
maxFindings: bounds.maxFindings,
|
|
13544
|
+
timeoutMs: bounds.timeoutMs,
|
|
13545
|
+
},
|
|
13546
|
+
};
|
|
12889
13547
|
}
|
|
12890
|
-
|
|
12891
|
-
|
|
12892
|
-
}
|
|
12893
|
-
|
|
12894
|
-
// LWC HTML/CSS bundle members visible in the response without admitting
|
|
12895
|
-
// arbitrary web assets to the source index.
|
|
12896
|
-
// Appended in a loop, not `push(...files)`: spreading an array as
|
|
12897
|
-
// arguments overflows the call stack once a repository is large
|
|
12898
|
-
// enough. Observed on a 40k-file checkout, which is well within the
|
|
12899
|
-
// range knodin is meant to index.
|
|
12900
|
-
for (const file of collectRepoFiles(repoPath))
|
|
12901
|
-
indexed.push(file);
|
|
12902
|
-
completionMessage = `${indexed.length.toLocaleString()} repository file(s) are current`;
|
|
12903
|
-
}
|
|
12904
|
-
if (options?.scip) {
|
|
12905
|
-
const facts = readScipIndex(repoPath, options.scip);
|
|
12906
|
-
const affectedFiles = persistScipFacts(db, repoPath, facts);
|
|
12907
|
-
progress("finalizing", 0, "Refreshing SCIP identities and semantic embeddings");
|
|
12908
|
-
await indexEmbeddings(db, repoPath, progress);
|
|
12909
|
-
indexGeneration++;
|
|
12910
|
-
const bounds = { ...SCIP_DEFAULT_LIMITS, ...options.scip };
|
|
12911
|
-
scipReport = {
|
|
12912
|
-
provenance: "scip",
|
|
12913
|
-
inputPath: facts.inputPath,
|
|
12914
|
-
bytes: facts.bytes,
|
|
12915
|
-
files: facts.files,
|
|
12916
|
-
facts: facts.facts,
|
|
12917
|
-
languages: facts.languages,
|
|
12918
|
-
indexers: facts.indexers,
|
|
12919
|
-
elapsedMs: facts.elapsedMs,
|
|
12920
|
-
bounds: {
|
|
12921
|
-
maxBytes: bounds.maxBytes,
|
|
12922
|
-
maxFiles: bounds.maxFiles,
|
|
12923
|
-
maxFacts: bounds.maxFacts,
|
|
12924
|
-
timeoutMs: bounds.timeoutMs,
|
|
12925
|
-
},
|
|
12926
|
-
};
|
|
12927
|
-
for (const file of affectedFiles)
|
|
12928
|
-
if (!indexed.includes(file))
|
|
12929
|
-
indexed.push(file);
|
|
12930
|
-
}
|
|
12931
|
-
if (options?.sarif) {
|
|
12932
|
-
const facts = readSarifLog(repoPath, options.sarif);
|
|
12933
|
-
persistSarifFindings(db, facts);
|
|
12934
|
-
// Findings are not graph facts, so no re-embedding and no generation
|
|
12935
|
-
// bump: the graph is unchanged and existing answers stay valid.
|
|
12936
|
-
const bounds = { ...SARIF_DEFAULT_LIMITS, ...options.sarif };
|
|
12937
|
-
sarifReport = {
|
|
12938
|
-
provenance: "sarif",
|
|
12939
|
-
inputPath: facts.inputPath,
|
|
12940
|
-
bytes: facts.bytes,
|
|
12941
|
-
tools: facts.tools,
|
|
12942
|
-
rules: facts.rules,
|
|
12943
|
-
files: facts.files,
|
|
12944
|
-
findings: facts.findings.length,
|
|
12945
|
-
skippedUnresolved: facts.skippedUnresolved,
|
|
12946
|
-
elapsedMs: facts.elapsedMs,
|
|
12947
|
-
bounds: {
|
|
12948
|
-
maxBytes: bounds.maxBytes,
|
|
12949
|
-
maxFindings: bounds.maxFindings,
|
|
12950
|
-
timeoutMs: bounds.timeoutMs,
|
|
12951
|
-
},
|
|
12952
|
-
};
|
|
12953
|
-
}
|
|
12954
|
-
progress("verifying", 0, "Verifying local graph health", {
|
|
12955
|
-
phaseTotal: 1,
|
|
12956
|
-
});
|
|
12957
|
-
let health = await engine.status(repoPath, {
|
|
12958
|
-
audit: "deep",
|
|
12959
|
-
ignoreActiveOperation: true,
|
|
12960
|
-
});
|
|
12961
|
-
const contentVerified = health.coverage.sourceFiles === health.coverage.indexedFiles &&
|
|
12962
|
-
health.missing.files.length === 0 &&
|
|
12963
|
-
health.missing.records.length === 0 &&
|
|
12964
|
-
Object.values(health.orphaned).every((count) => count === 0);
|
|
12965
|
-
if (contentVerified &&
|
|
12966
|
-
(health.freshness.state === "stale-head" || health.freshness.state === "unknown")) {
|
|
12967
|
-
setMeta(db, "lastIndexedHead", gitHead(repoPath) ?? "");
|
|
12968
|
-
setMeta(db, "lastSuccessfulReconciliation", new Date().toISOString());
|
|
12969
|
-
recordFreshnessBaseline(repoPath, db);
|
|
12970
|
-
statusCache.delete(databaseCacheKey(repoPath, openPolicy.databasePath));
|
|
12971
|
-
health = await engine.status(repoPath, {
|
|
13548
|
+
progress("verifying", 0, "Verifying local graph health", {
|
|
13549
|
+
phaseTotal: 1,
|
|
13550
|
+
});
|
|
13551
|
+
let health = await engine.status(repoPath, {
|
|
12972
13552
|
audit: "deep",
|
|
12973
13553
|
ignoreActiveOperation: true,
|
|
12974
13554
|
});
|
|
12975
|
-
|
|
12976
|
-
|
|
12977
|
-
|
|
12978
|
-
|
|
12979
|
-
|
|
12980
|
-
|
|
12981
|
-
// Explicit indexing counts as a successful reconciliation only after
|
|
12982
|
-
// the audit proves that no requested or unrequested drift remains.
|
|
12983
|
-
if (files && files.length > 0) {
|
|
13555
|
+
const contentVerified = health.coverage.sourceFiles === health.coverage.indexedFiles &&
|
|
13556
|
+
health.missing.files.length === 0 &&
|
|
13557
|
+
health.missing.records.length === 0 &&
|
|
13558
|
+
Object.values(health.orphaned).every((count) => count === 0);
|
|
13559
|
+
if (contentVerified &&
|
|
13560
|
+
(health.freshness.state === "stale-head" || health.freshness.state === "unknown")) {
|
|
12984
13561
|
setMeta(db, "lastIndexedHead", gitHead(repoPath) ?? "");
|
|
13562
|
+
setMeta(db, "knodinVersion", KNODIN_VERSION);
|
|
12985
13563
|
setMeta(db, "lastSuccessfulReconciliation", new Date().toISOString());
|
|
12986
13564
|
recordFreshnessBaseline(repoPath, db);
|
|
13565
|
+
statusCache.delete(databaseCacheKey(repoPath, openPolicy.databasePath));
|
|
13566
|
+
health = await engine.status(repoPath, {
|
|
13567
|
+
audit: "deep",
|
|
13568
|
+
ignoreActiveOperation: true,
|
|
13569
|
+
});
|
|
13570
|
+
}
|
|
13571
|
+
// The postcondition belongs to this index result. Do not turn the next
|
|
13572
|
+
// explicit `status` call into a cache hit against an internal audit.
|
|
13573
|
+
statusCache.delete(databaseCacheKey(repoPath, openPolicy.databasePath));
|
|
13574
|
+
const issueCount = health.missing.files.length + health.missing.records.length;
|
|
13575
|
+
if (health.status === "healthy") {
|
|
13576
|
+
// Explicit indexing counts as a successful reconciliation only after
|
|
13577
|
+
// the audit proves that no requested or unrequested drift remains.
|
|
13578
|
+
if (files && files.length > 0) {
|
|
13579
|
+
setMeta(db, "lastIndexedHead", gitHead(repoPath) ?? "");
|
|
13580
|
+
setMeta(db, "knodinVersion", KNODIN_VERSION);
|
|
13581
|
+
setMeta(db, "lastSuccessfulReconciliation", new Date().toISOString());
|
|
13582
|
+
recordFreshnessBaseline(repoPath, db);
|
|
13583
|
+
}
|
|
13584
|
+
noteFreshness(repoPath, "fresh");
|
|
13585
|
+
// Test engines use an in-memory graph and must not create untracked
|
|
13586
|
+
// repository artifacts that pollute Git-diff/review fixtures.
|
|
13587
|
+
if (!process.env.VITEST && process.env.NODE_ENV !== "test")
|
|
13588
|
+
publishStructuralSnapshot(repoPath, db);
|
|
13589
|
+
}
|
|
13590
|
+
else {
|
|
13591
|
+
freshnessProbes.delete(path.resolve(repoPath));
|
|
12987
13592
|
}
|
|
12988
|
-
|
|
12989
|
-
|
|
12990
|
-
|
|
12991
|
-
|
|
12992
|
-
|
|
13593
|
+
progress("verifying", 1, health.status === "healthy"
|
|
13594
|
+
? "Local graph health verified"
|
|
13595
|
+
: `Local graph still has ${issueCount.toLocaleString()} issue(s)`, { phaseTotal: 1 });
|
|
13596
|
+
const completed = files?.length ?? indexed.length;
|
|
13597
|
+
progress("completed", completed, health.status === "healthy"
|
|
13598
|
+
? completionMessage
|
|
13599
|
+
: `${completionMessage}; verification found ${issueCount.toLocaleString()} issue(s)`, { phaseTotal: completed });
|
|
13600
|
+
return {
|
|
13601
|
+
indexed,
|
|
13602
|
+
unchanged,
|
|
13603
|
+
scip: scipReport,
|
|
13604
|
+
sarif: sarifReport,
|
|
13605
|
+
verification: {
|
|
13606
|
+
status: health.status,
|
|
13607
|
+
issueCount,
|
|
13608
|
+
missing: health.missing,
|
|
13609
|
+
verifiedAt: health.verification.verifiedAt,
|
|
13610
|
+
},
|
|
13611
|
+
};
|
|
13612
|
+
}
|
|
13613
|
+
finally {
|
|
13614
|
+
const activeDb = dbInstances.get(resolved);
|
|
13615
|
+
if (!openPolicy.databasePath &&
|
|
13616
|
+
activeDb &&
|
|
13617
|
+
openPolicy.watcher === "enabled" &&
|
|
13618
|
+
watcherStateBefore !== "scale-limited" &&
|
|
13619
|
+
watcherStateBefore !== "resource-limited")
|
|
13620
|
+
startFileWatcher(resolved, activeDb, openPolicy.watcherFileLimit);
|
|
12993
13621
|
}
|
|
12994
|
-
else {
|
|
12995
|
-
freshnessProbes.delete(path.resolve(repoPath));
|
|
12996
|
-
}
|
|
12997
|
-
progress("verifying", 1, health.status === "healthy"
|
|
12998
|
-
? "Local graph health verified"
|
|
12999
|
-
: `Local graph still has ${issueCount.toLocaleString()} issue(s)`, { phaseTotal: 1 });
|
|
13000
|
-
const completed = files?.length ?? indexed.length;
|
|
13001
|
-
progress("completed", completed, health.status === "healthy"
|
|
13002
|
-
? completionMessage
|
|
13003
|
-
: `${completionMessage}; verification found ${issueCount.toLocaleString()} issue(s)`, { phaseTotal: completed });
|
|
13004
|
-
return {
|
|
13005
|
-
indexed,
|
|
13006
|
-
unchanged,
|
|
13007
|
-
scip: scipReport,
|
|
13008
|
-
sarif: sarifReport,
|
|
13009
|
-
verification: {
|
|
13010
|
-
status: health.status,
|
|
13011
|
-
issueCount,
|
|
13012
|
-
missing: health.missing,
|
|
13013
|
-
verifiedAt: health.verification.verifiedAt,
|
|
13014
|
-
},
|
|
13015
|
-
};
|
|
13016
13622
|
}
|
|
13017
13623
|
finally {
|
|
13018
|
-
|
|
13019
|
-
if (!openPolicy.databasePath &&
|
|
13020
|
-
activeDb &&
|
|
13021
|
-
openPolicy.watcher === "enabled" &&
|
|
13022
|
-
watcherStateBefore !== "scale-limited" &&
|
|
13023
|
-
watcherStateBefore !== "resource-limited")
|
|
13024
|
-
startFileWatcher(resolved, activeDb, openPolicy.watcherFileLimit);
|
|
13624
|
+
indexCoordination.release();
|
|
13025
13625
|
}
|
|
13026
13626
|
},
|
|
13027
13627
|
async search(query, repoPath, limit, options = {}) {
|
|
@@ -14051,7 +14651,10 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
14051
14651
|
if (!impactOptions.includeDataFlow)
|
|
14052
14652
|
return undefined;
|
|
14053
14653
|
try {
|
|
14054
|
-
const
|
|
14654
|
+
const whole = readWholeSource(resolvedRepoPath, file);
|
|
14655
|
+
if (whole.availability.state !== "available")
|
|
14656
|
+
return undefined;
|
|
14657
|
+
const text = whole.text.split("\n")[line - 1];
|
|
14055
14658
|
if (!text)
|
|
14056
14659
|
return undefined;
|
|
14057
14660
|
const escaped = callee.replace(/[.*+?^${}()|[\]\\]/g, String.raw `\$&`);
|
|
@@ -14384,11 +14987,8 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
14384
14987
|
const readSource = (relFile) => {
|
|
14385
14988
|
if (sourceCache.has(relFile))
|
|
14386
14989
|
return sourceCache.get(relFile) ?? null;
|
|
14387
|
-
|
|
14388
|
-
|
|
14389
|
-
src = fs.readFileSync(path.join(repoPath, relFile), "utf-8");
|
|
14390
|
-
}
|
|
14391
|
-
catch { }
|
|
14990
|
+
const read = readWholeSource(repoPath, relFile);
|
|
14991
|
+
const src = read.availability.state === "available" ? read.text : null;
|
|
14392
14992
|
sourceCache.set(relFile, src);
|
|
14393
14993
|
return src;
|
|
14394
14994
|
};
|
|
@@ -14452,6 +15052,13 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
14452
15052
|
return false;
|
|
14453
15053
|
if (isTestFilePath(r.filePath))
|
|
14454
15054
|
return false;
|
|
15055
|
+
// Unreadable file: there is no basis to judge this symbol either
|
|
15056
|
+
// way. Everything below infers "unused" from the ABSENCE of
|
|
15057
|
+
// evidence, so a failed read silently produced a confident "dead"
|
|
15058
|
+
// verdict — and acting on that means deleting live code. Unknown
|
|
15059
|
+
// must not collapse into dead.
|
|
15060
|
+
if (readSource(r.filePath) === null)
|
|
15061
|
+
return false;
|
|
14455
15062
|
if (isExportedSymbol(r.filePath, r.name)) {
|
|
14456
15063
|
const importers = importersByFile.get(r.filePath) ?? [];
|
|
14457
15064
|
// An exported symbol in an otherwise unimported file may be a
|
|
@@ -14522,7 +15129,7 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
14522
15129
|
});
|
|
14523
15130
|
if (!destination.selected)
|
|
14524
15131
|
return empty("The destination endpoint was not resolved uniquely.");
|
|
14525
|
-
if (selectedTarget.kind !== "
|
|
15132
|
+
if (selectedTarget.kind !== "salesforce_flow" ||
|
|
14526
15133
|
!selectedTarget.filePath.endsWith(".flow-meta.xml") ||
|
|
14527
15134
|
!destination.selected.filePath.endsWith(".cls"))
|
|
14528
15135
|
return empty("The endpoints are not a supported Salesforce Flow-to-Apex boundary.");
|
|
@@ -14530,7 +15137,10 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
14530
15137
|
const actionName = `${className}.${destination.selected.name}`;
|
|
14531
15138
|
let flowEvidence = "";
|
|
14532
15139
|
try {
|
|
14533
|
-
const
|
|
15140
|
+
const flowRead = readWholeSource(primary.path, selectedTarget.filePath);
|
|
15141
|
+
if (flowRead.availability.state !== "available")
|
|
15142
|
+
throw new Error(flowRead.availability.reason);
|
|
15143
|
+
const flowSource = flowRead.text;
|
|
14534
15144
|
const matchingActions = [
|
|
14535
15145
|
...flowSource.matchAll(/<actionCalls>([\s\S]*?)<\/actionCalls>/g),
|
|
14536
15146
|
]
|
|
@@ -14864,7 +15474,10 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
14864
15474
|
return undefined;
|
|
14865
15475
|
try {
|
|
14866
15476
|
const absolutePath = path.join(resolvedRepoPath, ref.callerFile);
|
|
14867
|
-
const
|
|
15477
|
+
const read = readWholeSource(resolvedRepoPath, ref.callerFile);
|
|
15478
|
+
if (read.availability.state !== "available")
|
|
15479
|
+
return undefined;
|
|
15480
|
+
const content = read.text;
|
|
14868
15481
|
const language = await getLanguageForFile(absolutePath);
|
|
14869
15482
|
if (!language)
|
|
14870
15483
|
return undefined;
|
|
@@ -15056,21 +15669,27 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
15056
15669
|
}
|
|
15057
15670
|
else {
|
|
15058
15671
|
inputs = indexed.flatMap(({ filePath }) => {
|
|
15059
|
-
const
|
|
15060
|
-
|
|
15061
|
-
const content = fs.readFileSync(absolute, "utf8");
|
|
15062
|
-
if (fileDriftedFromIndexState(db, resolvedRepoPath, filePath))
|
|
15063
|
-
diskDrifted = true;
|
|
15064
|
-
return [{ path: filePath, content }];
|
|
15065
|
-
}
|
|
15066
|
-
catch {
|
|
15672
|
+
const read = readWholeSource(resolvedRepoPath, filePath);
|
|
15673
|
+
if (read.availability.state !== "available") {
|
|
15067
15674
|
diskDrifted = true;
|
|
15068
15675
|
return [];
|
|
15069
15676
|
}
|
|
15677
|
+
// Drift is a working-tree concept: a sealed artifact's bytes
|
|
15678
|
+
// came from the attested commit and cannot have moved since,
|
|
15679
|
+
// and the mtime/size comparison would report every file
|
|
15680
|
+
// drifted because there is no file to stat.
|
|
15681
|
+
if (!isSealedRepository(resolvedRepoPath) &&
|
|
15682
|
+
fileDriftedFromIndexState(db, resolvedRepoPath, filePath))
|
|
15683
|
+
diskDrifted = true;
|
|
15684
|
+
return [{ path: filePath, content: read.text }];
|
|
15070
15685
|
});
|
|
15071
15686
|
diskFingerprint = resourceFingerprint(inputs);
|
|
15072
15687
|
if (!diskDrifted)
|
|
15073
|
-
setBoundedCache(resourceCorpusCache, resolvedRepoPath, {
|
|
15688
|
+
setBoundedCache(resourceCorpusCache, resolvedRepoPath, {
|
|
15689
|
+
generation: indexGeneration,
|
|
15690
|
+
inputs,
|
|
15691
|
+
fingerprint: diskFingerprint,
|
|
15692
|
+
}, RESOURCE_CORPUS_CACHE_LIMIT);
|
|
15074
15693
|
}
|
|
15075
15694
|
// Disk state is part of the cache identity. A generation-only key can
|
|
15076
15695
|
// return a previously fresh result during the freshness-probe TTL after
|
|
@@ -15080,7 +15699,10 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
15080
15699
|
const cached = resourceReachabilityCache.get(cacheKey);
|
|
15081
15700
|
if (cached?.generation === indexGeneration) {
|
|
15082
15701
|
const result = structuredClone(cached.result);
|
|
15083
|
-
return finish([], {
|
|
15702
|
+
return finish([], {
|
|
15703
|
+
count: result.paths.length,
|
|
15704
|
+
resourceReachability: result,
|
|
15705
|
+
});
|
|
15084
15706
|
}
|
|
15085
15707
|
const result = analyzeResourceReachability(inputs, {
|
|
15086
15708
|
maxItems: cap,
|
|
@@ -15090,7 +15712,10 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
15090
15712
|
expectedFingerprint: diskDrifted ? `indexed:${diskFingerprint}` : diskFingerprint,
|
|
15091
15713
|
});
|
|
15092
15714
|
setBoundedCache(resourceReachabilityCache, cacheKey, { generation: indexGeneration, result: structuredClone(result) }, RESOURCE_REACHABILITY_CACHE_LIMIT);
|
|
15093
|
-
return finish([], {
|
|
15715
|
+
return finish([], {
|
|
15716
|
+
count: result.paths.length,
|
|
15717
|
+
resourceReachability: result,
|
|
15718
|
+
});
|
|
15094
15719
|
}
|
|
15095
15720
|
case "flow_analysis": {
|
|
15096
15721
|
// C8 deliberately does not persist a PDG. This is a bounded, local
|
|
@@ -15105,8 +15730,10 @@ export function createEngine(openPolicy = DEFAULT_ENGINE_OPEN_POLICY) {
|
|
|
15105
15730
|
: undefined;
|
|
15106
15731
|
if (!language)
|
|
15107
15732
|
return finish([]);
|
|
15108
|
-
const
|
|
15109
|
-
|
|
15733
|
+
const flowRead = readWholeSource(resolvedRepoPath, selectedTarget.filePath);
|
|
15734
|
+
if (flowRead.availability.state !== "available")
|
|
15735
|
+
return finish([]);
|
|
15736
|
+
const sourceLines = flowRead.text.split(/\r?\n/);
|
|
15110
15737
|
const functionLines = sourceLines.slice(selectedTarget.startLine - 1, selectedTarget.endLine);
|
|
15111
15738
|
const facts = [];
|
|
15112
15739
|
const variableFilter = options.flowVariable;
|