@aryam/fixmap-core 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -1
- package/dist/annotation-store.d.ts +5 -0
- package/dist/annotation-store.js +117 -0
- package/dist/annotation-store.js.map +1 -0
- package/dist/annotations.d.ts +57 -0
- package/dist/annotations.js +221 -0
- package/dist/annotations.js.map +1 -0
- package/dist/architecture-history.d.ts +33 -0
- package/dist/architecture-history.js +234 -0
- package/dist/architecture-history.js.map +1 -0
- package/dist/architecture.d.ts +124 -0
- package/dist/architecture.js +348 -0
- package/dist/architecture.js.map +1 -0
- package/dist/artifacts.d.ts +9 -0
- package/dist/artifacts.js +82 -0
- package/dist/artifacts.js.map +1 -0
- package/dist/ask.d.ts +48 -0
- package/dist/ask.js +182 -0
- package/dist/ask.js.map +1 -0
- package/dist/browser.d.ts +66 -3
- package/dist/browser.js +34 -2
- package/dist/browser.js.map +1 -1
- package/dist/capabilities.d.ts +39 -0
- package/dist/capabilities.js +172 -0
- package/dist/capabilities.js.map +1 -0
- package/dist/capability-history.d.ts +49 -0
- package/dist/capability-history.js +176 -0
- package/dist/capability-history.js.map +1 -0
- package/dist/change-conflicts.d.ts +34 -0
- package/dist/change-conflicts.js +140 -0
- package/dist/change-conflicts.js.map +1 -0
- package/dist/change-scope.d.ts +81 -0
- package/dist/change-scope.js +316 -0
- package/dist/change-scope.js.map +1 -0
- package/dist/characterization.d.ts +61 -0
- package/dist/characterization.js +169 -0
- package/dist/characterization.js.map +1 -0
- package/dist/ci-matrix.d.ts +53 -0
- package/dist/ci-matrix.js +170 -0
- package/dist/ci-matrix.js.map +1 -0
- package/dist/compare.js +2 -2
- package/dist/compare.js.map +1 -1
- package/dist/composer-projects.d.ts +32 -0
- package/dist/composer-projects.js +246 -0
- package/dist/composer-projects.js.map +1 -0
- package/dist/context.d.ts +1 -1
- package/dist/context.js +26 -4
- package/dist/context.js.map +1 -1
- package/dist/contracts.d.ts +75 -0
- package/dist/contracts.js +526 -0
- package/dist/contracts.js.map +1 -0
- package/dist/custom-language-context.d.ts +43 -0
- package/dist/custom-language-context.js +89 -0
- package/dist/custom-language-context.js.map +1 -0
- package/dist/decisions.d.ts +64 -0
- package/dist/decisions.js +338 -0
- package/dist/decisions.js.map +1 -0
- package/dist/dossier.d.ts +63 -0
- package/dist/dossier.js +176 -0
- package/dist/dossier.js.map +1 -0
- package/dist/dotnet-projects.d.ts +26 -0
- package/dist/dotnet-projects.js +171 -0
- package/dist/dotnet-projects.js.map +1 -0
- package/dist/editor-loader.d.ts +9 -0
- package/dist/editor-loader.js +18 -0
- package/dist/editor-loader.js.map +1 -0
- package/dist/editor-protocol.d.ts +35 -0
- package/dist/editor-protocol.js +200 -0
- package/dist/editor-protocol.js.map +1 -0
- package/dist/editor-session.d.ts +15 -0
- package/dist/editor-session.js +27 -0
- package/dist/editor-session.js.map +1 -0
- package/dist/editor-transport.d.ts +10 -0
- package/dist/editor-transport.js +75 -0
- package/dist/editor-transport.js.map +1 -0
- package/dist/enterprise-policy.d.ts +82 -0
- package/dist/enterprise-policy.js +170 -0
- package/dist/enterprise-policy.js.map +1 -0
- package/dist/evidence-bundle.d.ts +13 -0
- package/dist/evidence-bundle.js +105 -0
- package/dist/evidence-bundle.js.map +1 -0
- package/dist/evidence.d.ts +125 -0
- package/dist/evidence.js +252 -0
- package/dist/evidence.js.map +1 -0
- package/dist/explain.js +7 -0
- package/dist/explain.js.map +1 -1
- package/dist/go-projects.d.ts +22 -0
- package/dist/go-projects.js +146 -0
- package/dist/go-projects.js.map +1 -0
- package/dist/graph.js +9 -0
- package/dist/graph.js.map +1 -1
- package/dist/grounding.d.ts +4 -1
- package/dist/grounding.js +70 -13
- package/dist/grounding.js.map +1 -1
- package/dist/identity-graph.d.ts +106 -0
- package/dist/identity-graph.js +353 -0
- package/dist/identity-graph.js.map +1 -0
- package/dist/impact.d.ts +2 -1
- package/dist/impact.js +12 -4
- package/dist/impact.js.map +1 -1
- package/dist/import-graph.d.ts +7 -1
- package/dist/import-graph.js +616 -34
- package/dist/import-graph.js.map +1 -1
- package/dist/incident.d.ts +74 -0
- package/dist/incident.js +212 -0
- package/dist/incident.js.map +1 -0
- package/dist/index.d.ts +91 -4
- package/dist/index.js +47 -3
- package/dist/index.js.map +1 -1
- package/dist/language-adapters.d.ts +33 -0
- package/dist/language-adapters.js +390 -0
- package/dist/language-adapters.js.map +1 -0
- package/dist/language-registry.d.ts +26 -0
- package/dist/language-registry.js +57 -0
- package/dist/language-registry.js.map +1 -0
- package/dist/languages.d.ts +20 -5
- package/dist/languages.js +197 -19
- package/dist/languages.js.map +1 -1
- package/dist/markdown.d.ts +1 -0
- package/dist/markdown.js +7 -1
- package/dist/markdown.js.map +1 -1
- package/dist/migration.d.ts +56 -0
- package/dist/migration.js +154 -0
- package/dist/migration.js.map +1 -0
- package/dist/outcomes.d.ts +57 -0
- package/dist/outcomes.js +146 -0
- package/dist/outcomes.js.map +1 -0
- package/dist/ownership.d.ts +29 -0
- package/dist/ownership.js +189 -0
- package/dist/ownership.js.map +1 -0
- package/dist/plan-alternatives.d.ts +74 -0
- package/dist/plan-alternatives.js +205 -0
- package/dist/plan-alternatives.js.map +1 -0
- package/dist/plan.d.ts +7 -0
- package/dist/plan.js +41 -8
- package/dist/plan.js.map +1 -1
- package/dist/rank.d.ts +64 -1
- package/dist/rank.js +278 -26
- package/dist/rank.js.map +1 -1
- package/dist/repo-scan.d.ts +8 -1
- package/dist/repo-scan.js +341 -97
- package/dist/repo-scan.js.map +1 -1
- package/dist/report.d.ts +16 -0
- package/dist/report.js +298 -33
- package/dist/report.js.map +1 -1
- package/dist/retrieval.d.ts +37 -0
- package/dist/retrieval.js +150 -21
- package/dist/retrieval.js.map +1 -1
- package/dist/reverse-docs.d.ts +46 -0
- package/dist/reverse-docs.js +179 -0
- package/dist/reverse-docs.js.map +1 -0
- package/dist/ruby-projects.d.ts +20 -0
- package/dist/ruby-projects.js +114 -0
- package/dist/ruby-projects.js.map +1 -0
- package/dist/runtime-evidence.d.ts +90 -0
- package/dist/runtime-evidence.js +186 -0
- package/dist/runtime-evidence.js.map +1 -0
- package/dist/rust-projects.d.ts +17 -0
- package/dist/rust-projects.js +125 -0
- package/dist/rust-projects.js.map +1 -0
- package/dist/sandbox.d.ts +59 -0
- package/dist/sandbox.js +202 -0
- package/dist/sandbox.js.map +1 -0
- package/dist/semantic-cache.d.ts +8 -0
- package/dist/semantic-cache.js +149 -0
- package/dist/semantic-cache.js.map +1 -0
- package/dist/semantic.d.ts +78 -0
- package/dist/semantic.js +237 -0
- package/dist/semantic.js.map +1 -0
- package/dist/sensitive-data.d.ts +8 -0
- package/dist/sensitive-data.js +157 -0
- package/dist/sensitive-data.js.map +1 -0
- package/dist/signals.js +1 -1
- package/dist/signals.js.map +1 -1
- package/dist/supply-chain.d.ts +38 -0
- package/dist/supply-chain.js +206 -0
- package/dist/supply-chain.js.map +1 -0
- package/dist/test-reliability.d.ts +56 -0
- package/dist/test-reliability.js +191 -0
- package/dist/test-reliability.js.map +1 -0
- package/dist/text.js +8 -1
- package/dist/text.js.map +1 -1
- package/dist/transformers-embedding.d.ts +16 -0
- package/dist/transformers-embedding.js +127 -0
- package/dist/transformers-embedding.js.map +1 -0
- package/dist/types.d.ts +79 -2
- package/dist/validate.js +172 -0
- package/dist/validate.js.map +1 -1
- package/dist/verify.js +136 -2
- package/dist/verify.js.map +1 -1
- package/dist/workspace.d.ts +86 -0
- package/dist/workspace.js +557 -0
- package/dist/workspace.js.map +1 -0
- package/package.json +1 -1
package/dist/repo-scan.js
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
2
|
import { createHash, randomUUID } from "node:crypto";
|
|
3
|
-
import {
|
|
3
|
+
import { createReadStream } from "node:fs";
|
|
4
|
+
import { mkdir, open, readdir, readFile, realpath, rename, stat, unlink, writeFile } from "node:fs/promises";
|
|
4
5
|
import { homedir } from "node:os";
|
|
5
6
|
import { dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
6
7
|
import { promisify } from "node:util";
|
|
7
8
|
import { ALWAYS_IGNORED_DIRS, GENERATED_DIRS, SOURCE_FILE_EXTENSIONS, isGeneratedPath } from "./paths.js";
|
|
9
|
+
import { isLanguageTestPath } from "./language-adapters.js";
|
|
10
|
+
import { markdownCode } from "./markdown.js";
|
|
8
11
|
import { DIAGNOSTIC_SPEC_LIMIT, truncateForDiagnostic } from "./text.js";
|
|
9
12
|
// Vendored source is intentionally scannable and receives a ranking penalty. Git mode has
|
|
10
13
|
// always kept tracked vendor/ files; walk mode must not silently use a smaller corpus.
|
|
@@ -17,9 +20,10 @@ const CONVENTIONAL_DOCUMENT_NAMES = new Set([
|
|
|
17
20
|
"authors", "changelog", "code_of_conduct", "contributing", "license", "notice", "readme", "security"
|
|
18
21
|
]);
|
|
19
22
|
const CONVENTIONAL_CONFIG_NAMES = new Set([
|
|
20
|
-
".dockerignore", ".editorconfig", ".gitattributes", ".gitignore", ".npmignore",
|
|
21
|
-
"codeowners", "dockerfile", "gemfile", "jenkinsfile", "makefile", "procfile", "rakefile", "vagrantfile",
|
|
22
|
-
"pom.xml", "build.gradle", "settings.gradle"
|
|
23
|
+
".dockerignore", ".editorconfig", ".gitattributes", ".gitignore", ".npmignore", ".rspec",
|
|
24
|
+
"cargo.toml", "codeowners", "dockerfile", "gemfile", "go.mod", "go.work", "jenkinsfile", "makefile", "procfile", "rakefile", "vagrantfile",
|
|
25
|
+
"pom.xml", "build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts", "gradlew", "gradlew.bat",
|
|
26
|
+
"mvnw", "mvnw.cmd", "pest.php", "phpunit.xml", "phpunit.xml.dist", "pyproject.toml", "pytest.ini", "setup.cfg", "tox.ini"
|
|
23
27
|
]);
|
|
24
28
|
/**
|
|
25
29
|
* A single-file component is mostly template and style. Sampling the whole file let markup
|
|
@@ -42,16 +46,24 @@ const TEST_PATTERNS = [
|
|
|
42
46
|
const MAX_TEXT_SAMPLE_BYTES = 64_000;
|
|
43
47
|
const MAX_DIFF_TEXT_CHARS = 200_000;
|
|
44
48
|
const MAX_SCANNED_FILES = 25_000;
|
|
49
|
+
// Metadata reads, bounded content samples, and realpath resolution are independent per file.
|
|
50
|
+
// Running a small batch together removes the dominant cold-scan latency on large Windows
|
|
51
|
+
// checkouts without opening an unbounded number of file handles.
|
|
52
|
+
const TRACKED_SCAN_IO_CONCURRENCY = 32;
|
|
45
53
|
const GIT_MAX_BUFFER = 10 * 1024 * 1024;
|
|
46
54
|
const GIT_HISTORY_MAX_BUFFER = 24 * 1024 * 1024;
|
|
47
55
|
const MAX_HISTORY_COMMITS = 1_000;
|
|
48
56
|
const MAX_HISTORY_FILES_PER_COMMIT = 30;
|
|
49
57
|
const exec = promisify(execFile);
|
|
50
|
-
const SCAN_CACHE_VERSION =
|
|
58
|
+
const SCAN_CACHE_VERSION = 9;
|
|
51
59
|
const SCAN_CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
52
60
|
const SCAN_CACHE_MAX_FUTURE_SKEW_MS = 5 * 60 * 1000;
|
|
53
61
|
const SCAN_CACHE_FILE = /^[a-f0-9]{24}-[a-f0-9]{24}\.json$/;
|
|
54
62
|
const SCAN_CACHE_TEMP_FILE = /^[a-f0-9]{24}-[a-f0-9]{24}\.json\.\d+-[0-9a-f-]+\.tmp$/i;
|
|
63
|
+
// Reject records generated with HEAD-relative dirty detection: their staged blob
|
|
64
|
+
// fingerprints can describe different bytes from the cached worktree sample.
|
|
65
|
+
const INCREMENTAL_INDEX_VERSION = 3;
|
|
66
|
+
const INCREMENTAL_INDEX_TEMP_FILE = /^[a-f0-9]{24}-index-v2\.json\.\d+-[0-9a-f-]+\.tmp$/i;
|
|
55
67
|
export async function scanRepo(input) {
|
|
56
68
|
const repoRoot = resolve(input.repoRoot);
|
|
57
69
|
if (!(await isDirectory(repoRoot))) {
|
|
@@ -76,9 +88,13 @@ export async function scanRepo(input) {
|
|
|
76
88
|
? cacheRoot
|
|
77
89
|
: undefined;
|
|
78
90
|
const cacheDecision = input.useCache === true
|
|
79
|
-
? await buildScanCacheLocation(repoRoot, cacheRoot, internalPaths, input.includeHistory === true)
|
|
91
|
+
? await buildScanCacheLocation(repoRoot, cacheRoot, internalPaths, input.includeHistory === true, input.languageRegistry?.cacheIdentity)
|
|
80
92
|
: undefined;
|
|
81
93
|
const cacheLocation = cacheDecision?.location;
|
|
94
|
+
const incrementalIndexLocation = input.useCache === true &&
|
|
95
|
+
!sameFilesystemPath(cacheRoot, repoRoot) && containedPath(repoRoot, cacheRoot) === undefined
|
|
96
|
+
? buildIncrementalIndexLocation(repoRoot, cacheRoot, input.languageRegistry?.cacheIdentity)
|
|
97
|
+
: undefined;
|
|
82
98
|
if (input.useCache === false) {
|
|
83
99
|
diagnostics.push({
|
|
84
100
|
code: "cache-bypass",
|
|
@@ -112,8 +128,9 @@ export async function scanRepo(input) {
|
|
|
112
128
|
});
|
|
113
129
|
}
|
|
114
130
|
else {
|
|
115
|
-
|
|
116
|
-
|
|
131
|
+
const listed = await listFiles(repoRoot, diagnostics, internalCacheRoot, internalPaths, incrementalIndexLocation, input.languageRegistry);
|
|
132
|
+
files = listed.files;
|
|
133
|
+
trackedFiles = listed.trackedFiles;
|
|
117
134
|
packageScripts = await readPackageScripts(repoRoot, files, diagnostics);
|
|
118
135
|
packageManager = detectPackageManager(files, diagnostics);
|
|
119
136
|
history = input.includeHistory === true
|
|
@@ -153,6 +170,10 @@ export async function scanRepo(input) {
|
|
|
153
170
|
...(history ? { history } : {})
|
|
154
171
|
};
|
|
155
172
|
}
|
|
173
|
+
function buildIncrementalIndexLocation(root, cacheRoot, registryIdentity) {
|
|
174
|
+
const repoKey = hashText(resolve(root) + (registryIdentity ? `\0${registryIdentity}` : ""));
|
|
175
|
+
return { path: join(cacheRoot, `${repoKey}-index-v2.json`), repoKey };
|
|
176
|
+
}
|
|
156
177
|
function configuredScanCacheRoot() {
|
|
157
178
|
return resolve(process.env.FIXMAP_CACHE_DIR ?? join(process.env.LOCALAPPDATA ?? process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache"), "fixmap", "scans"));
|
|
158
179
|
}
|
|
@@ -189,7 +210,7 @@ function gitPathspec(internalPaths) {
|
|
|
189
210
|
.sort((a, b) => a.localeCompare(b))
|
|
190
211
|
.map((path) => `:(exclude,literal)${path}`)];
|
|
191
212
|
}
|
|
192
|
-
async function buildScanCacheLocation(root, cacheRoot, internalPaths, includeHistory) {
|
|
213
|
+
async function buildScanCacheLocation(root, cacheRoot, internalPaths, includeHistory, registryIdentity) {
|
|
193
214
|
if (sameFilesystemPath(cacheRoot, root) || containedPath(root, cacheRoot) !== undefined) {
|
|
194
215
|
return {
|
|
195
216
|
skipReason: "Repository scan caching was skipped because FIXMAP_CACHE_DIR is inside the scanned repository. " +
|
|
@@ -197,21 +218,23 @@ async function buildScanCacheLocation(root, cacheRoot, internalPaths, includeHis
|
|
|
197
218
|
};
|
|
198
219
|
}
|
|
199
220
|
try {
|
|
200
|
-
const
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
221
|
+
const { stdout: status } = await exec("git", ["status", "--porcelain=v2", "--branch", "--no-ahead-behind", "-z", "--untracked-files=all", ...gitPathspec(internalPaths)], {
|
|
222
|
+
cwd: root,
|
|
223
|
+
maxBuffer: GIT_MAX_BUFFER
|
|
224
|
+
});
|
|
225
|
+
const entries = status.split("\0");
|
|
226
|
+
const head = entries.find((entry) => /^# branch\.oid [a-f0-9]{40,64}$/i.test(entry))?.slice(13);
|
|
227
|
+
// An unborn checkout reports `(initial)`, not an exact commit identity.
|
|
228
|
+
if (!head)
|
|
229
|
+
throw new Error("No committed Git identity in status");
|
|
207
230
|
// Untracked files are scanner inputs but are absent from `git diff`. Do not cache that
|
|
208
231
|
// state rather than keying it on names alone and serving stale contents after an edit.
|
|
209
|
-
if (
|
|
232
|
+
if (entries.some((entry) => entry.startsWith("? "))) {
|
|
210
233
|
return {
|
|
211
234
|
skipReason: "Repository scan caching was skipped because untracked files are scanner inputs and can change without a stable git diff."
|
|
212
235
|
};
|
|
213
236
|
}
|
|
214
|
-
const dirtyDiff =
|
|
237
|
+
const dirtyDiff = entries.some((entry) => /^[12u] /.test(entry))
|
|
215
238
|
? (await exec("git", ["diff", "--binary", "--no-ext-diff", "HEAD", ...gitPathspec(internalPaths)], {
|
|
216
239
|
cwd: root,
|
|
217
240
|
maxBuffer: GIT_MAX_BUFFER
|
|
@@ -224,6 +247,7 @@ async function buildScanCacheLocation(root, cacheRoot, internalPaths, includeHis
|
|
|
224
247
|
status,
|
|
225
248
|
dirtyDiff,
|
|
226
249
|
includeHistory ? "history" : "no-history",
|
|
250
|
+
...(registryIdentity ? [registryIdentity] : []),
|
|
227
251
|
...[...internalPaths].sort((a, b) => a.localeCompare(b))
|
|
228
252
|
].join("\0"));
|
|
229
253
|
return { location: {
|
|
@@ -265,6 +289,28 @@ async function readScanCache(location) {
|
|
|
265
289
|
return undefined;
|
|
266
290
|
}
|
|
267
291
|
}
|
|
292
|
+
async function readIncrementalScanIndex(location) {
|
|
293
|
+
try {
|
|
294
|
+
const candidate = JSON.parse(await readFile(location.path, "utf8"));
|
|
295
|
+
if (candidate.version !== INCREMENTAL_INDEX_VERSION || candidate.repoKey !== location.repoKey ||
|
|
296
|
+
typeof candidate.updatedAt !== "string" || !Number.isFinite(Date.parse(candidate.updatedAt)) ||
|
|
297
|
+
!Array.isArray(candidate.files))
|
|
298
|
+
return undefined;
|
|
299
|
+
const entries = new Map();
|
|
300
|
+
for (const entry of candidate.files) {
|
|
301
|
+
if (!isRecord(entry) || !isCachedRelativePath(entry.path) ||
|
|
302
|
+
typeof entry.fingerprint !== "string" || !/^(?:git|worktree):[a-f0-9]{40,64}$/i.test(entry.fingerprint) ||
|
|
303
|
+
!isCachedRepoFile(entry.file) || entry.file.path !== entry.path || entries.has(entry.path)) {
|
|
304
|
+
return undefined;
|
|
305
|
+
}
|
|
306
|
+
entries.set(entry.path, entry);
|
|
307
|
+
}
|
|
308
|
+
return entries;
|
|
309
|
+
}
|
|
310
|
+
catch {
|
|
311
|
+
return undefined;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
268
314
|
function isCachedHistory(candidate) {
|
|
269
315
|
if (!isRecord(candidate) || !Array.isArray(candidate.commits) ||
|
|
270
316
|
typeof candidate.inspectedCommits !== "number" || !Number.isSafeInteger(candidate.inspectedCommits) || candidate.inspectedCommits < 0 ||
|
|
@@ -273,8 +319,9 @@ function isCachedHistory(candidate) {
|
|
|
273
319
|
return false;
|
|
274
320
|
}
|
|
275
321
|
return candidate.commits.every((commit) => {
|
|
276
|
-
if (!isRecord(commit) || typeof commit.hash !== "string" || !/^[a-f0-9]{40}$/i.test(commit.hash) ||
|
|
322
|
+
if (!isRecord(commit) || typeof commit.hash !== "string" || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/i.test(commit.hash) ||
|
|
277
323
|
typeof commit.committedAt !== "number" || !Number.isSafeInteger(commit.committedAt) || commit.committedAt < 0 ||
|
|
324
|
+
(commit.author !== undefined && (typeof commit.author !== "string" || !commit.author.trim() || commit.author.length > 200 || /[\0-\x1f\x7f]/.test(commit.author))) ||
|
|
278
325
|
!Array.isArray(commit.files))
|
|
279
326
|
return false;
|
|
280
327
|
return commit.files.every(isCachedRelativePath);
|
|
@@ -286,6 +333,8 @@ function isCachedRepoFile(candidate) {
|
|
|
286
333
|
const validSkipReason = candidate.textSampleSkipReason === "too-large" ||
|
|
287
334
|
candidate.textSampleSkipReason === "not-text" || candidate.textSampleSkipReason === "unreadable";
|
|
288
335
|
return isCachedRelativePath(candidate.path) &&
|
|
336
|
+
typeof candidate.contentFingerprint === "string" &&
|
|
337
|
+
/^(?:git|worktree):[a-f0-9]{40,64}$/i.test(candidate.contentFingerprint) &&
|
|
289
338
|
typeof candidate.extension === "string" &&
|
|
290
339
|
typeof candidate.sizeBytes === "number" && Number.isFinite(candidate.sizeBytes) && candidate.sizeBytes >= 0 &&
|
|
291
340
|
typeof candidate.isTest === "boolean" &&
|
|
@@ -347,6 +396,26 @@ async function writeScanCache(location, cached) {
|
|
|
347
396
|
catch { /* The rename may already have consumed it. */ }
|
|
348
397
|
}
|
|
349
398
|
}
|
|
399
|
+
async function writeIncrementalScanIndex(location, files) {
|
|
400
|
+
const temporaryPath = `${location.path}.${process.pid}-${randomUUID()}.tmp`;
|
|
401
|
+
const index = {
|
|
402
|
+
version: INCREMENTAL_INDEX_VERSION,
|
|
403
|
+
repoKey: location.repoKey,
|
|
404
|
+
updatedAt: new Date().toISOString(),
|
|
405
|
+
files
|
|
406
|
+
};
|
|
407
|
+
try {
|
|
408
|
+
await mkdir(dirname(location.path), { recursive: true });
|
|
409
|
+
await writeFile(temporaryPath, `${JSON.stringify(index)}\n`, { encoding: "utf8", flag: "wx" });
|
|
410
|
+
await rename(temporaryPath, location.path);
|
|
411
|
+
}
|
|
412
|
+
catch {
|
|
413
|
+
try {
|
|
414
|
+
await unlink(temporaryPath);
|
|
415
|
+
}
|
|
416
|
+
catch { /* The rename may already have consumed it. */ }
|
|
417
|
+
}
|
|
418
|
+
}
|
|
350
419
|
async function pruneExpiredScanCache(cacheRoot) {
|
|
351
420
|
try {
|
|
352
421
|
const entries = await readdir(cacheRoot, { withFileTypes: true });
|
|
@@ -355,7 +424,8 @@ async function pruneExpiredScanCache(cacheRoot) {
|
|
|
355
424
|
// A killed process can leave its uniquely named atomic-write file behind. It is not
|
|
356
425
|
// readable as a cache entry, but without pruning it the cache directory grows forever.
|
|
357
426
|
// The same seven-day horizon preserves any plausible active writer.
|
|
358
|
-
.filter((entry) => entry.isFile() && (SCAN_CACHE_FILE.test(entry.name) || SCAN_CACHE_TEMP_FILE.test(entry.name)
|
|
427
|
+
.filter((entry) => entry.isFile() && (SCAN_CACHE_FILE.test(entry.name) || SCAN_CACHE_TEMP_FILE.test(entry.name) ||
|
|
428
|
+
INCREMENTAL_INDEX_TEMP_FILE.test(entry.name)))
|
|
359
429
|
.map(async (entry) => {
|
|
360
430
|
const path = join(cacheRoot, entry.name);
|
|
361
431
|
try {
|
|
@@ -402,18 +472,53 @@ async function listTrackedPaths(root, internalPaths) {
|
|
|
402
472
|
function resolveDiffSpec(input) {
|
|
403
473
|
return input.diffSpec ?? (input.baseRef ? `${input.baseRef}...${input.headRef ?? "HEAD"}` : undefined);
|
|
404
474
|
}
|
|
405
|
-
async function listFiles(root, diagnostics, internalCacheRoot, internalPaths) {
|
|
475
|
+
async function listFiles(root, diagnostics, internalCacheRoot, internalPaths, incrementalIndexLocation, languageRegistry) {
|
|
406
476
|
const gitPaths = await listGitPaths(root);
|
|
407
477
|
const visiblePaths = gitPaths?.paths.filter((path) => !hasInternalPath(internalPaths, normalizePath(path)) && !isInternalCachePath(root, path, internalCacheRoot));
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
478
|
+
let files;
|
|
479
|
+
if (gitPaths) {
|
|
480
|
+
const previous = incrementalIndexLocation
|
|
481
|
+
? await readIncrementalScanIndex(incrementalIndexLocation)
|
|
482
|
+
: undefined;
|
|
483
|
+
const built = await buildFilesFromPaths(root, visiblePaths ?? [], diagnostics, gitPaths.gitLinks, gitPaths.fingerprints, previous, languageRegistry);
|
|
484
|
+
files = built.files;
|
|
485
|
+
if (incrementalIndexLocation) {
|
|
486
|
+
await writeIncrementalScanIndex(incrementalIndexLocation, built.indexedFiles);
|
|
487
|
+
}
|
|
488
|
+
if (previous && built.reused > 0) {
|
|
489
|
+
diagnostics.push({
|
|
490
|
+
code: "incremental-index-hit",
|
|
491
|
+
severity: "info",
|
|
492
|
+
message: `Reused ${built.reused.toLocaleString()} unchanged file record${built.reused === 1 ? "" : "s"} from the persistent index ` +
|
|
493
|
+
`and refreshed ${built.refreshed.toLocaleString()} changed or new path${built.refreshed === 1 ? "" : "s"}.`
|
|
494
|
+
});
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
else {
|
|
498
|
+
const state = { count: 0, limitReported: false, linkedPaths: [] };
|
|
499
|
+
files = (await walkFiles(root, root, diagnostics, state, internalCacheRoot, internalPaths, languageRegistry))
|
|
411
500
|
.sort((a, b) => a.path.localeCompare(b.path));
|
|
501
|
+
if (state.linkedPaths.length > 0) {
|
|
502
|
+
const paths = [...new Set(state.linkedPaths)].sort();
|
|
503
|
+
diagnostics.push({
|
|
504
|
+
code: "linked-paths-skipped",
|
|
505
|
+
severity: "info",
|
|
506
|
+
message: `Skipped ${paths.length} symbolic link or junction ${paths.length === 1 ? "path" : "paths"} during the filesystem scan ` +
|
|
507
|
+
`to avoid leaving the repository or following a loop: ${paths.slice(0, 5).map(markdownCode).join(", ")}${paths.length > 5 ? ", ..." : ""}.`,
|
|
508
|
+
paths: paths.slice(0, 8)
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
}
|
|
412
512
|
// These are properties of the scanned files, not of git. Keeping them here makes an
|
|
413
513
|
// extracted archive and a checkout report the same content limitations.
|
|
414
514
|
reportUnreadContent(diagnostics, files);
|
|
415
515
|
reportGeneratedDominance(diagnostics, files);
|
|
416
|
-
|
|
516
|
+
// The staged index already enumerates every tracked path, including deleted
|
|
517
|
+
// worktree files and gitlinks. Retain the fallback if Git discovery failed.
|
|
518
|
+
const trackedFiles = gitPaths
|
|
519
|
+
? gitPaths.trackedPaths.filter((path) => !hasInternalPath(internalPaths, path))
|
|
520
|
+
: await listTrackedPaths(root, internalPaths);
|
|
521
|
+
return { files, trackedFiles };
|
|
417
522
|
}
|
|
418
523
|
function isInternalCachePath(root, path, internalCacheRoot) {
|
|
419
524
|
if (!internalCacheRoot)
|
|
@@ -431,25 +536,46 @@ function isInternalCachePath(root, path, internalCacheRoot) {
|
|
|
431
536
|
}
|
|
432
537
|
async function listGitPaths(root) {
|
|
433
538
|
try {
|
|
434
|
-
const [{ stdout }, { stdout: staged }] = await Promise.all([
|
|
539
|
+
const [{ stdout }, { stdout: staged }, { stdout: dirty }] = await Promise.all([
|
|
435
540
|
exec("git", ["ls-files", "--cached", "--others", "--exclude-standard", "-z"], {
|
|
436
541
|
cwd: root,
|
|
437
542
|
maxBuffer: GIT_MAX_BUFFER
|
|
438
543
|
}),
|
|
439
|
-
exec("git", ["ls-files", "--stage", "-z"], { cwd: root, maxBuffer: GIT_MAX_BUFFER })
|
|
544
|
+
exec("git", ["ls-files", "--stage", "-z"], { cwd: root, maxBuffer: GIT_MAX_BUFFER }),
|
|
545
|
+
// Fingerprints come from the index, so only index/worktree equality permits
|
|
546
|
+
// reuse. Comparing with HEAD misses unstaged edits that undo a staged edit.
|
|
547
|
+
// This also works before the first commit, without a failing HEAD probe.
|
|
548
|
+
exec("git", ["diff", "--name-only", "-z", "--"], { cwd: root, maxBuffer: GIT_MAX_BUFFER })
|
|
440
549
|
]);
|
|
441
|
-
const gitLinks = new Set(
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
550
|
+
const gitLinks = new Set();
|
|
551
|
+
const fingerprints = new Map();
|
|
552
|
+
const trackedPaths = [];
|
|
553
|
+
for (const entry of staged.split("\0")) {
|
|
554
|
+
const match = /^(\d+)\s+([0-9a-f]+)\s+\d+\t(.+)$/is.exec(entry);
|
|
555
|
+
if (!match?.[1] || !match[2] || !match[3])
|
|
556
|
+
continue;
|
|
557
|
+
const path = normalizePath(match[3]);
|
|
558
|
+
trackedPaths.push(path);
|
|
559
|
+
if (match[1] === "160000") {
|
|
560
|
+
gitLinks.add(path);
|
|
561
|
+
}
|
|
562
|
+
else if (!/^0+$/.test(match[2])) {
|
|
563
|
+
fingerprints.set(path, `git:${match[2].toLowerCase()}`);
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
for (const path of dirty.split("\0").filter(Boolean).map(normalizePath)) {
|
|
567
|
+
fingerprints.delete(path);
|
|
568
|
+
}
|
|
569
|
+
return { paths: [...new Set(stdout.split("\0").filter(Boolean))], trackedPaths, gitLinks, fingerprints };
|
|
446
570
|
}
|
|
447
571
|
catch {
|
|
448
572
|
return undefined;
|
|
449
573
|
}
|
|
450
574
|
}
|
|
451
|
-
async function buildFilesFromPaths(root, paths, diagnostics, knownGitLinks = new Set()) {
|
|
575
|
+
async function buildFilesFromPaths(root, paths, diagnostics, knownGitLinks = new Set(), fingerprints = new Map(), previous, languageRegistry) {
|
|
452
576
|
const results = [];
|
|
577
|
+
const indexedByPath = new Map();
|
|
578
|
+
const reusedPaths = new Set();
|
|
453
579
|
const absent = [];
|
|
454
580
|
const gitLinks = [];
|
|
455
581
|
// Git can hand back two tracked paths that are one file on disk: a symlink beside its
|
|
@@ -461,61 +587,117 @@ async function buildFilesFromPaths(root, paths, diagnostics, knownGitLinks = new
|
|
|
461
587
|
// git listed first would have made that an alphabetical accident.
|
|
462
588
|
const seenRealPaths = new Map();
|
|
463
589
|
const linked = [];
|
|
590
|
+
const escapedLinks = [];
|
|
464
591
|
// macOS temporary directories commonly enter through /var while realpath returns
|
|
465
592
|
// /private/var. Resolve the repository root once so that prefix alias does not make
|
|
466
593
|
// every ordinary file look like a symlink.
|
|
467
594
|
const realRoot = await resolveRealPath(root);
|
|
468
|
-
|
|
469
|
-
if (results.length >= MAX_SCANNED_FILES) {
|
|
470
|
-
// Git handed us the whole tracked list, so the remainder is known exactly.
|
|
471
|
-
// Saying which directories went unread lets a reader judge whether the
|
|
472
|
-
// truncation touched the code their task is about.
|
|
473
|
-
reportScanLimit(diagnostics, paths.slice(index).map(normalizePath));
|
|
474
|
-
break;
|
|
475
|
-
}
|
|
595
|
+
const preparePath = async (rawPath, index) => {
|
|
476
596
|
const relativePath = normalizePath(rawPath);
|
|
477
597
|
if (isInAlwaysIgnoredDir(relativePath)) {
|
|
478
|
-
|
|
598
|
+
return { index, relativePath, status: "ignored" };
|
|
479
599
|
}
|
|
480
600
|
if (knownGitLinks.has(relativePath)) {
|
|
481
|
-
|
|
482
|
-
continue;
|
|
483
|
-
}
|
|
484
|
-
const scanned = await toRepoFile(join(root, rawPath), relativePath);
|
|
485
|
-
if (scanned.status === "absent") {
|
|
486
|
-
absent.push(relativePath);
|
|
487
|
-
continue;
|
|
601
|
+
return { index, relativePath, status: "git-link" };
|
|
488
602
|
}
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
if (
|
|
504
|
-
|
|
505
|
-
|
|
603
|
+
const absolutePath = join(root, rawPath);
|
|
604
|
+
const fingerprint = fingerprints.get(relativePath) ?? await hashWorktreeFile(absolutePath);
|
|
605
|
+
const prior = fingerprint ? previous?.get(relativePath) : undefined;
|
|
606
|
+
const reused = prior && prior.fingerprint === fingerprint ? prior.file : undefined;
|
|
607
|
+
const scanned = await toRepoFile(absolutePath, relativePath, fingerprint, reused, languageRegistry);
|
|
608
|
+
return { index, relativePath, status: "scanned", fingerprint, reused: reused !== undefined, scanned };
|
|
609
|
+
};
|
|
610
|
+
let limitReached = false;
|
|
611
|
+
for (let start = 0; start < paths.length && !limitReached; start += TRACKED_SCAN_IO_CONCURRENCY) {
|
|
612
|
+
const batch = paths.slice(start, start + TRACKED_SCAN_IO_CONCURRENCY);
|
|
613
|
+
const prepared = await Promise.all(batch.map((rawPath, offset) => preparePath(rawPath, start + offset)));
|
|
614
|
+
// Reduce in the exact order Git supplied. Alias selection, the scan budget, and every
|
|
615
|
+
// diagnostic therefore remain stable even though the filesystem work above overlaps.
|
|
616
|
+
for (const candidate of prepared) {
|
|
617
|
+
if (results.length >= MAX_SCANNED_FILES) {
|
|
618
|
+
// Git handed us the whole tracked list, so the remainder is known exactly.
|
|
619
|
+
// Saying which directories went unread lets a reader judge whether the
|
|
620
|
+
// truncation touched the code their task is about.
|
|
621
|
+
reportScanLimit(diagnostics, paths.slice(candidate.index).map(normalizePath));
|
|
622
|
+
limitReached = true;
|
|
623
|
+
break;
|
|
624
|
+
}
|
|
625
|
+
if (candidate.status === "ignored")
|
|
626
|
+
continue;
|
|
627
|
+
if (candidate.status === "git-link") {
|
|
628
|
+
gitLinks.push(candidate.relativePath);
|
|
629
|
+
continue;
|
|
506
630
|
}
|
|
507
|
-
|
|
508
|
-
|
|
631
|
+
if (candidate.status !== "scanned")
|
|
632
|
+
continue;
|
|
633
|
+
const { relativePath, fingerprint, reused, scanned } = candidate;
|
|
634
|
+
if (scanned.status === "absent") {
|
|
635
|
+
absent.push(relativePath);
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
if (scanned.status === "not-a-file") {
|
|
639
|
+
gitLinks.push(relativePath);
|
|
640
|
+
continue;
|
|
641
|
+
}
|
|
642
|
+
if (scanned.status !== "ok") {
|
|
643
|
+
continue;
|
|
644
|
+
}
|
|
645
|
+
if (!containedPath(realRoot, scanned.realPath)) {
|
|
646
|
+
escapedLinks.push(relativePath);
|
|
647
|
+
continue;
|
|
648
|
+
}
|
|
649
|
+
const seenIndex = seenRealPaths.get(scanned.realPath);
|
|
650
|
+
if (seenIndex !== undefined) {
|
|
651
|
+
const seenFile = results[seenIndex];
|
|
652
|
+
// Looking only at the leaf with lstat misses Windows junctions, where the linked
|
|
653
|
+
// object is an ancestor directory. Comparing literal and resolved paths covers both.
|
|
654
|
+
const seenIsAlias = !sameFilesystemPath(resolve(realRoot, seenFile.path), scanned.realPath);
|
|
655
|
+
const currentIsAlias = !sameFilesystemPath(resolve(realRoot, relativePath), scanned.realPath);
|
|
656
|
+
if (seenIsAlias && !currentIsAlias) {
|
|
657
|
+
linked.push({ path: seenFile.path, target: relativePath });
|
|
658
|
+
indexedByPath.delete(seenFile.path);
|
|
659
|
+
reusedPaths.delete(seenFile.path);
|
|
660
|
+
results[seenIndex] = scanned.file;
|
|
661
|
+
if (fingerprint) {
|
|
662
|
+
indexedByPath.set(relativePath, { path: relativePath, fingerprint, file: scanned.file });
|
|
663
|
+
if (reused)
|
|
664
|
+
reusedPaths.add(relativePath);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
else {
|
|
668
|
+
linked.push({ path: relativePath, target: seenFile.path });
|
|
669
|
+
}
|
|
670
|
+
continue;
|
|
671
|
+
}
|
|
672
|
+
seenRealPaths.set(scanned.realPath, results.length);
|
|
673
|
+
results.push(scanned.file);
|
|
674
|
+
if (fingerprint) {
|
|
675
|
+
indexedByPath.set(relativePath, { path: relativePath, fingerprint, file: scanned.file });
|
|
676
|
+
if (reused)
|
|
677
|
+
reusedPaths.add(relativePath);
|
|
509
678
|
}
|
|
510
|
-
continue;
|
|
511
679
|
}
|
|
512
|
-
seenRealPaths.set(scanned.realPath, results.length);
|
|
513
|
-
results.push(scanned.file);
|
|
514
680
|
}
|
|
515
681
|
reportAbsentTrackedPaths(diagnostics, absent);
|
|
516
682
|
reportLinkedDuplicates(diagnostics, linked);
|
|
683
|
+
reportEscapedLinks(diagnostics, escapedLinks);
|
|
517
684
|
reportSkippedSubmodules(diagnostics, gitLinks);
|
|
518
|
-
|
|
685
|
+
const files = results.sort((a, b) => a.path.localeCompare(b.path));
|
|
686
|
+
const indexedFiles = files.flatMap((file) => {
|
|
687
|
+
const indexed = indexedByPath.get(file.path);
|
|
688
|
+
return indexed ? [indexed] : [];
|
|
689
|
+
});
|
|
690
|
+
const reused = files.filter((file) => reusedPaths.has(file.path)).length;
|
|
691
|
+
return { files, indexedFiles, reused, refreshed: Math.max(0, indexedFiles.length - reused) };
|
|
692
|
+
}
|
|
693
|
+
async function hashWorktreeFile(path) {
|
|
694
|
+
return await new Promise((resolveHash) => {
|
|
695
|
+
const hash = createHash("sha256");
|
|
696
|
+
const input = createReadStream(path);
|
|
697
|
+
input.on("data", (chunk) => hash.update(chunk));
|
|
698
|
+
input.on("error", () => resolveHash(undefined));
|
|
699
|
+
input.on("end", () => resolveHash(`worktree:${hash.digest("hex")}`));
|
|
700
|
+
});
|
|
519
701
|
}
|
|
520
702
|
/**
|
|
521
703
|
* A tracked path with nothing on disk was dropped silently, so the plan read as a complete
|
|
@@ -579,9 +761,8 @@ function reportUnreadContent(diagnostics, files) {
|
|
|
579
761
|
diagnostics.push({
|
|
580
762
|
code: "content-too-large",
|
|
581
763
|
severity: "warning",
|
|
582
|
-
message: `${unread.length.toLocaleString()} source file${unread.length === 1 ? "" : "s"}
|
|
583
|
-
|
|
584
|
-
`${unread.length > 3 ? ", …" : ""}. Files over ${(MAX_TEXT_SAMPLE_BYTES / 1000).toLocaleString()} kB are not sampled.`,
|
|
764
|
+
message: `${unread.length.toLocaleString()} source file${unread.length === 1 ? "" : "s"} exceeded the complete text window — largest: ${sample}` +
|
|
765
|
+
`${unread.length > 3 ? ", …" : ""}. Files over ${(MAX_TEXT_SAMPLE_BYTES / 1000).toLocaleString()} kB use bounded head and distributed retrieval samples; context and grounding remain incomplete.`,
|
|
585
766
|
paths: unread.slice(0, 8).map((file) => file.path)
|
|
586
767
|
});
|
|
587
768
|
}
|
|
@@ -636,7 +817,20 @@ function reportLinkedDuplicates(diagnostics, linked) {
|
|
|
636
817
|
`${linked.length > 3 ? ", …" : ""}.`
|
|
637
818
|
});
|
|
638
819
|
}
|
|
639
|
-
|
|
820
|
+
function reportEscapedLinks(diagnostics, paths) {
|
|
821
|
+
if (paths.length === 0)
|
|
822
|
+
return;
|
|
823
|
+
const unique = [...new Set(paths)].sort();
|
|
824
|
+
diagnostics.push({
|
|
825
|
+
code: "linked-paths-skipped",
|
|
826
|
+
severity: "info",
|
|
827
|
+
message: `Skipped ${unique.length.toLocaleString()} tracked or untracked linked ${unique.length === 1 ? "path" : "paths"} ` +
|
|
828
|
+
`because ${unique.length === 1 ? "it resolves" : "they resolve"} outside the repository: ` +
|
|
829
|
+
`${unique.slice(0, 5).map(markdownCode).join(", ")}${unique.length > 5 ? ", ..." : ""}.`,
|
|
830
|
+
paths: unique.slice(0, 8)
|
|
831
|
+
});
|
|
832
|
+
}
|
|
833
|
+
async function walkFiles(root, current, diagnostics, state, internalCacheRoot, internalPaths, languageRegistry) {
|
|
640
834
|
let entries;
|
|
641
835
|
try {
|
|
642
836
|
entries = await readdir(current, { withFileTypes: true });
|
|
@@ -653,6 +847,10 @@ async function walkFiles(root, current, diagnostics, state, internalCacheRoot, i
|
|
|
653
847
|
}
|
|
654
848
|
break;
|
|
655
849
|
}
|
|
850
|
+
if (entry.isSymbolicLink()) {
|
|
851
|
+
state.linkedPaths.push(normalizePath(relative(root, join(current, entry.name))));
|
|
852
|
+
continue;
|
|
853
|
+
}
|
|
656
854
|
if (entry.isDirectory()) {
|
|
657
855
|
if (WALK_IGNORED_DIRS.has(entry.name)) {
|
|
658
856
|
continue;
|
|
@@ -660,7 +858,7 @@ async function walkFiles(root, current, diagnostics, state, internalCacheRoot, i
|
|
|
660
858
|
const directory = join(current, entry.name);
|
|
661
859
|
if (internalCacheRoot && sameFilesystemPath(directory, internalCacheRoot))
|
|
662
860
|
continue;
|
|
663
|
-
results.push(...await walkFiles(root, directory, diagnostics, state, internalCacheRoot, internalPaths));
|
|
861
|
+
results.push(...await walkFiles(root, directory, diagnostics, state, internalCacheRoot, internalPaths, languageRegistry));
|
|
664
862
|
continue;
|
|
665
863
|
}
|
|
666
864
|
if (!entry.isFile()) {
|
|
@@ -670,7 +868,8 @@ async function walkFiles(root, current, diagnostics, state, internalCacheRoot, i
|
|
|
670
868
|
const relativePath = normalizePath(relative(root, absolutePath));
|
|
671
869
|
if (hasInternalPath(internalPaths, relativePath) || isInternalCachePath(root, relativePath, internalCacheRoot))
|
|
672
870
|
continue;
|
|
673
|
-
const
|
|
871
|
+
const fingerprint = await hashWorktreeFile(absolutePath);
|
|
872
|
+
const scanned = await toRepoFile(absolutePath, relativePath, fingerprint, undefined, languageRegistry);
|
|
674
873
|
if (scanned.status === "ok") {
|
|
675
874
|
results.push(scanned.file);
|
|
676
875
|
state.count += 1;
|
|
@@ -678,7 +877,7 @@ async function walkFiles(root, current, diagnostics, state, internalCacheRoot, i
|
|
|
678
877
|
}
|
|
679
878
|
return results;
|
|
680
879
|
}
|
|
681
|
-
async function toRepoFile(absolutePath, relativePath) {
|
|
880
|
+
async function toRepoFile(absolutePath, relativePath, contentFingerprint, reusable, languageRegistry) {
|
|
682
881
|
let fileStat;
|
|
683
882
|
try {
|
|
684
883
|
fileStat = await stat(absolutePath);
|
|
@@ -689,26 +888,38 @@ async function toRepoFile(absolutePath, relativePath) {
|
|
|
689
888
|
if (!fileStat.isFile()) {
|
|
690
889
|
return { status: "not-a-file" };
|
|
691
890
|
}
|
|
891
|
+
if (reusable?.path === relativePath && contentFingerprint) {
|
|
892
|
+
return {
|
|
893
|
+
status: "ok",
|
|
894
|
+
realPath: await resolveRealPath(absolutePath),
|
|
895
|
+
file: { ...reusable, contentFingerprint, sizeBytes: fileStat.size }
|
|
896
|
+
};
|
|
897
|
+
}
|
|
692
898
|
const extension = extname(relativePath).toLowerCase();
|
|
693
899
|
const conventionalKind = classifyConventionalTextFile(relativePath);
|
|
694
|
-
const
|
|
900
|
+
const customSource = languageRegistry?.customForExtension(extension) !== undefined;
|
|
901
|
+
const isSource = customSource || SOURCE_EXTENSIONS.has(extension) || conventionalKind !== undefined;
|
|
695
902
|
const sample = isSource
|
|
696
903
|
? await readTextSample(absolutePath, fileStat.size)
|
|
697
904
|
: { text: "", complete: true };
|
|
698
905
|
if (SFC_EXTENSIONS.has(extension) && sample.text) {
|
|
699
906
|
sample.text = extractScriptBlocks(sample.text);
|
|
907
|
+
if (sample.searchText)
|
|
908
|
+
sample.searchText = extractScriptBlocks(sample.searchText);
|
|
700
909
|
}
|
|
701
910
|
return {
|
|
702
911
|
status: "ok",
|
|
703
912
|
realPath: await resolveRealPath(absolutePath),
|
|
704
913
|
file: {
|
|
705
914
|
path: relativePath,
|
|
915
|
+
...(contentFingerprint ? { contentFingerprint } : {}),
|
|
706
916
|
extension,
|
|
707
917
|
sizeBytes: fileStat.size,
|
|
708
|
-
isTest: TEST_PATTERNS.some((pattern) => pattern.test(relativePath)),
|
|
918
|
+
isTest: isLanguageTestPath(relativePath, extension) || TEST_PATTERNS.some((pattern) => pattern.test(relativePath)),
|
|
709
919
|
isSource,
|
|
710
|
-
kind: classifyFile(relativePath, extension),
|
|
920
|
+
kind: customSource ? "code" : classifyFile(relativePath, extension),
|
|
711
921
|
textSample: sample.text,
|
|
922
|
+
...(sample.searchText ? { searchTextSample: sample.searchText } : {}),
|
|
712
923
|
textSampleComplete: sample.complete,
|
|
713
924
|
...(sample.skipReason ? { textSampleSkipReason: sample.skipReason } : {})
|
|
714
925
|
}
|
|
@@ -851,8 +1062,7 @@ async function readDiff(repoRoot, diffSpec, diagnostics, internalPaths) {
|
|
|
851
1062
|
.map((path) => path.trim())
|
|
852
1063
|
.filter(Boolean)
|
|
853
1064
|
.map(normalizePath);
|
|
854
|
-
const
|
|
855
|
-
const changedFiles = [...new Set([...tracked, ...untracked])].sort((a, b) => a.localeCompare(b));
|
|
1065
|
+
const changedFiles = [...new Set(tracked)].sort((a, b) => a.localeCompare(b));
|
|
856
1066
|
diagnostics.push({
|
|
857
1067
|
code: "diff-resolved",
|
|
858
1068
|
severity: "info",
|
|
@@ -1001,16 +1211,19 @@ async function readRepositoryHistory(root, repositoryPaths, diagnostics) {
|
|
|
1001
1211
|
try {
|
|
1002
1212
|
const [{ stdout: shallowText }, { stdout: countText }, { stdout: logText }] = await Promise.all([
|
|
1003
1213
|
exec("git", ["rev-parse", "--is-shallow-repository"], { cwd: root, maxBuffer: GIT_MAX_BUFFER }),
|
|
1004
|
-
exec("git", ["rev-list", "--count", "--no-merges", "HEAD"], { cwd: root, maxBuffer: GIT_MAX_BUFFER }),
|
|
1214
|
+
exec("git", ["rev-list", "--count", "--no-merges", "HEAD", "--", "."], { cwd: root, maxBuffer: GIT_MAX_BUFFER }),
|
|
1005
1215
|
exec("git", [
|
|
1006
1216
|
"-c", "core.quotepath=false",
|
|
1007
1217
|
"log",
|
|
1218
|
+
"--relative",
|
|
1008
1219
|
"--no-merges",
|
|
1009
1220
|
"-n", String(MAX_HISTORY_COMMITS),
|
|
1010
|
-
"--format=%x1e%H%x1f%ct",
|
|
1221
|
+
"--format=%x1e%H%x1f%ct%x1f%aN",
|
|
1011
1222
|
"--name-only",
|
|
1012
1223
|
"-z",
|
|
1013
|
-
"HEAD"
|
|
1224
|
+
"HEAD",
|
|
1225
|
+
"--",
|
|
1226
|
+
"."
|
|
1014
1227
|
], { cwd: root, maxBuffer: GIT_HISTORY_MAX_BUFFER })
|
|
1015
1228
|
]);
|
|
1016
1229
|
const parsed = parseHistoryLog(logText, repositoryPaths);
|
|
@@ -1057,7 +1270,7 @@ async function readRepositoryHistory(root, repositoryPaths, diagnostics) {
|
|
|
1057
1270
|
return undefined;
|
|
1058
1271
|
}
|
|
1059
1272
|
}
|
|
1060
|
-
function parseHistoryLog(logText, repositoryPaths) {
|
|
1273
|
+
export function parseHistoryLog(logText, repositoryPaths) {
|
|
1061
1274
|
const commits = [];
|
|
1062
1275
|
let inspectedCommits = 0;
|
|
1063
1276
|
let skippedLargeCommits = 0;
|
|
@@ -1070,8 +1283,11 @@ function parseHistoryLog(logText, repositoryPaths) {
|
|
|
1070
1283
|
if (separator === -1)
|
|
1071
1284
|
continue;
|
|
1072
1285
|
const hash = header.slice(0, separator).trim();
|
|
1073
|
-
const
|
|
1074
|
-
|
|
1286
|
+
const secondSeparator = header.indexOf("\x1f", separator + 1);
|
|
1287
|
+
const committedAt = Number.parseInt(header.slice(separator + 1, secondSeparator === -1 ? undefined : secondSeparator).trim(), 10);
|
|
1288
|
+
const rawAuthor = secondSeparator === -1 ? "" : header.slice(secondSeparator + 1).trim();
|
|
1289
|
+
const author = rawAuthor && !/[\0-\x1f\x7f]/.test(rawAuthor) ? rawAuthor.slice(0, 200) : undefined;
|
|
1290
|
+
if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/i.test(hash) || !Number.isSafeInteger(committedAt) || committedAt < 0)
|
|
1075
1291
|
continue;
|
|
1076
1292
|
inspectedCommits += 1;
|
|
1077
1293
|
const allFiles = [...new Set(fields
|
|
@@ -1085,7 +1301,7 @@ function parseHistoryLog(logText, repositoryPaths) {
|
|
|
1085
1301
|
const currentFiles = allFiles.filter((path) => repositoryPaths.has(path));
|
|
1086
1302
|
if (currentFiles.length === 0)
|
|
1087
1303
|
continue;
|
|
1088
|
-
commits.push({ hash, committedAt, files: currentFiles });
|
|
1304
|
+
commits.push({ hash, committedAt, ...(author ? { author } : {}), files: currentFiles });
|
|
1089
1305
|
}
|
|
1090
1306
|
return { commits, inspectedCommits, skippedLargeCommits };
|
|
1091
1307
|
}
|
|
@@ -1180,16 +1396,15 @@ function classifyConventionalTextFile(path) {
|
|
|
1180
1396
|
return "documentation";
|
|
1181
1397
|
if (CONVENTIONAL_CONFIG_NAMES.has(name))
|
|
1182
1398
|
return "config";
|
|
1183
|
-
if (/\.(?:csproj|fsproj|vbproj)$/.test(name))
|
|
1399
|
+
if (/\.(?:csproj|fsproj|vbproj|sln)$/.test(name))
|
|
1184
1400
|
return "config";
|
|
1185
1401
|
return undefined;
|
|
1186
1402
|
}
|
|
1187
1403
|
async function readTextSample(path, sizeBytes) {
|
|
1188
|
-
if (sizeBytes > MAX_TEXT_SAMPLE_BYTES) {
|
|
1189
|
-
return { text: "", complete: false, skipReason: "too-large" };
|
|
1190
|
-
}
|
|
1191
1404
|
try {
|
|
1192
|
-
const bytes =
|
|
1405
|
+
const bytes = sizeBytes <= MAX_TEXT_SAMPLE_BYTES
|
|
1406
|
+
? await readFile(path)
|
|
1407
|
+
: await readFileWindow(path, 0, MAX_TEXT_SAMPLE_BYTES);
|
|
1193
1408
|
// A NUL byte does not occur in real source. Treating such a file as text produced a
|
|
1194
1409
|
// garbled sample that matched nothing, while the path still scored — so a binary blob
|
|
1195
1410
|
// named like source ranked on its name alone with no way to tell from the report.
|
|
@@ -1198,12 +1413,41 @@ async function readTextSample(path, sizeBytes) {
|
|
|
1198
1413
|
if (bytes.includes(0)) {
|
|
1199
1414
|
return { text: "", complete: false, skipReason: "not-text" };
|
|
1200
1415
|
}
|
|
1201
|
-
|
|
1416
|
+
const text = bytes.toString("utf8");
|
|
1417
|
+
if (sizeBytes <= MAX_TEXT_SAMPLE_BYTES)
|
|
1418
|
+
return { text, complete: true };
|
|
1419
|
+
const searchBytes = await readDistributedWindows(path, sizeBytes);
|
|
1420
|
+
if (searchBytes.includes(0))
|
|
1421
|
+
return { text: "", complete: false, skipReason: "not-text" };
|
|
1422
|
+
return { text, searchText: searchBytes.toString("utf8"), complete: false, skipReason: "too-large" };
|
|
1202
1423
|
}
|
|
1203
1424
|
catch {
|
|
1204
1425
|
return { text: "", complete: false, skipReason: "unreadable" };
|
|
1205
1426
|
}
|
|
1206
1427
|
}
|
|
1428
|
+
async function readFileWindow(path, position, length) {
|
|
1429
|
+
const handle = await open(path, "r");
|
|
1430
|
+
try {
|
|
1431
|
+
const buffer = Buffer.allocUnsafe(length);
|
|
1432
|
+
const { bytesRead } = await handle.read(buffer, 0, length, position);
|
|
1433
|
+
return buffer.subarray(0, bytesRead);
|
|
1434
|
+
}
|
|
1435
|
+
finally {
|
|
1436
|
+
await handle.close();
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
async function readDistributedWindows(path, sizeBytes) {
|
|
1440
|
+
const windowBytes = Math.floor(MAX_TEXT_SAMPLE_BYTES / 4);
|
|
1441
|
+
const maximumStart = Math.max(0, sizeBytes - windowBytes);
|
|
1442
|
+
const starts = [...new Set([
|
|
1443
|
+
0,
|
|
1444
|
+
Math.floor(maximumStart / 3),
|
|
1445
|
+
Math.floor((maximumStart * 2) / 3),
|
|
1446
|
+
maximumStart
|
|
1447
|
+
])];
|
|
1448
|
+
const windows = await Promise.all(starts.map((position) => readFileWindow(path, position, windowBytes)));
|
|
1449
|
+
return Buffer.concat(windows.flatMap((window, index) => index === 0 ? [window] : [Buffer.from("\n"), window]));
|
|
1450
|
+
}
|
|
1207
1451
|
async function listUntrackedPaths(repoRoot, internalPaths = new Set()) {
|
|
1208
1452
|
try {
|
|
1209
1453
|
const { stdout } = await exec("git", ["ls-files", "--others", "--exclude-standard", "-z"], { cwd: repoRoot, maxBuffer: GIT_MAX_BUFFER });
|