@jcode.labs/ragmir 6.1.6 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -49
- package/dist/cli.js +194 -24
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +24 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +127 -24
- package/dist/config.js.map +1 -1
- package/dist/context-resources.d.ts.map +1 -1
- package/dist/context-resources.js +19 -8
- package/dist/context-resources.js.map +1 -1
- package/dist/defaults.d.ts +5 -0
- package/dist/defaults.d.ts.map +1 -1
- package/dist/defaults.js +16 -0
- package/dist/defaults.js.map +1 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +37 -5
- package/dist/doctor.js.map +1 -1
- package/dist/evaluate.d.ts.map +1 -1
- package/dist/evaluate.js +11 -6
- package/dist/evaluate.js.map +1 -1
- package/dist/files.d.ts +27 -1
- package/dist/files.d.ts.map +1 -1
- package/dist/files.js +344 -100
- package/dist/files.js.map +1 -1
- package/dist/freshness.d.ts +16 -0
- package/dist/freshness.d.ts.map +1 -0
- package/dist/freshness.js +97 -0
- package/dist/freshness.js.map +1 -0
- package/dist/generation-retention.d.ts +12 -0
- package/dist/generation-retention.d.ts.map +1 -1
- package/dist/generation-retention.js +13 -2
- package/dist/generation-retention.js.map +1 -1
- package/dist/git-hooks.d.ts +25 -0
- package/dist/git-hooks.d.ts.map +1 -0
- package/dist/git-hooks.js +173 -0
- package/dist/git-hooks.js.map +1 -0
- package/dist/index-diagnostics.d.ts +9 -1
- package/dist/index-diagnostics.d.ts.map +1 -1
- package/dist/index-diagnostics.js +35 -0
- package/dist/index-diagnostics.js.map +1 -1
- package/dist/index-policy.d.ts.map +1 -1
- package/dist/index-policy.js +4 -0
- package/dist/index-policy.js.map +1 -1
- package/dist/ingest.d.ts +6 -0
- package/dist/ingest.d.ts.map +1 -1
- package/dist/ingest.js +44 -21
- package/dist/ingest.js.map +1 -1
- package/dist/init.d.ts.map +1 -1
- package/dist/init.js +19 -4
- package/dist/init.js.map +1 -1
- package/dist/knowledge-bases.d.ts +18 -1
- package/dist/knowledge-bases.d.ts.map +1 -1
- package/dist/knowledge-bases.js +73 -4
- package/dist/knowledge-bases.js.map +1 -1
- package/dist/mcp-output.d.ts +14 -3
- package/dist/mcp-output.d.ts.map +1 -1
- package/dist/mcp-output.js +37 -0
- package/dist/mcp-output.js.map +1 -1
- package/dist/mcp-registrations.d.ts +9 -0
- package/dist/mcp-registrations.d.ts.map +1 -0
- package/dist/mcp-registrations.js +247 -0
- package/dist/mcp-registrations.js.map +1 -0
- package/dist/mcp-scope.d.ts +32 -0
- package/dist/mcp-scope.d.ts.map +1 -0
- package/dist/mcp-scope.js +114 -0
- package/dist/mcp-scope.js.map +1 -0
- package/dist/mcp-session.d.ts +37 -0
- package/dist/mcp-session.d.ts.map +1 -0
- package/dist/mcp-session.js +53 -0
- package/dist/mcp-session.js.map +1 -0
- package/dist/mcp-summaries.d.ts +5 -0
- package/dist/mcp-summaries.d.ts.map +1 -1
- package/dist/mcp-summaries.js +30 -0
- package/dist/mcp-summaries.js.map +1 -1
- package/dist/mcp.d.ts +6 -0
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +410 -103
- package/dist/mcp.js.map +1 -1
- package/dist/ocr.d.ts +51 -0
- package/dist/ocr.d.ts.map +1 -1
- package/dist/ocr.js +210 -4
- package/dist/ocr.js.map +1 -1
- package/dist/package-manager.d.ts.map +1 -1
- package/dist/package-manager.js +2 -1
- package/dist/package-manager.js.map +1 -1
- package/dist/preview.d.ts.map +1 -1
- package/dist/preview.js +3 -7
- package/dist/preview.js.map +1 -1
- package/dist/quality-report.d.ts.map +1 -1
- package/dist/quality-report.js +1 -1
- package/dist/quality-report.js.map +1 -1
- package/dist/query.d.ts.map +1 -1
- package/dist/query.js +235 -34
- package/dist/query.js.map +1 -1
- package/dist/ranking.d.ts +56 -6
- package/dist/ranking.d.ts.map +1 -1
- package/dist/ranking.js +187 -16
- package/dist/ranking.js.map +1 -1
- package/dist/retrieval-diversity.d.ts +5 -0
- package/dist/retrieval-diversity.d.ts.map +1 -1
- package/dist/retrieval-diversity.js +4 -2
- package/dist/retrieval-diversity.js.map +1 -1
- package/dist/search-output.d.ts.map +1 -1
- package/dist/search-output.js +2 -0
- package/dist/search-output.js.map +1 -1
- package/dist/security.js +1 -1
- package/dist/security.js.map +1 -1
- package/dist/setup-prompt.d.ts +1 -1
- package/dist/setup-prompt.d.ts.map +1 -1
- package/dist/setup-prompt.js +3 -3
- package/dist/setup.js +1 -1
- package/dist/setup.js.map +1 -1
- package/dist/skill.d.ts +2 -0
- package/dist/skill.d.ts.map +1 -1
- package/dist/skill.js +122 -14
- package/dist/skill.js.map +1 -1
- package/dist/source-paths.d.ts +36 -0
- package/dist/source-paths.d.ts.map +1 -0
- package/dist/source-paths.js +157 -0
- package/dist/source-paths.js.map +1 -0
- package/dist/source-weights.d.ts +28 -0
- package/dist/source-weights.d.ts.map +1 -0
- package/dist/source-weights.js +241 -0
- package/dist/source-weights.js.map +1 -0
- package/dist/storage-maintenance.d.ts +3 -1
- package/dist/storage-maintenance.d.ts.map +1 -1
- package/dist/storage-maintenance.js +21 -2
- package/dist/storage-maintenance.js.map +1 -1
- package/dist/storage-usage.d.ts +21 -0
- package/dist/storage-usage.d.ts.map +1 -0
- package/dist/storage-usage.js +118 -0
- package/dist/storage-usage.js.map +1 -0
- package/dist/types.d.ts +127 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +3 -1
package/dist/files.js
CHANGED
|
@@ -3,10 +3,12 @@ import { createReadStream, existsSync } from "node:fs";
|
|
|
3
3
|
import { readFile, stat } from "node:fs/promises";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import fg from "fast-glob";
|
|
6
|
+
import picomatch from "picomatch";
|
|
6
7
|
import { DEFAULT_CONFIG, LEGACY_KB_DIR, LEGACY_PRIVATE_DIR, RAGMIR_DIR } from "./defaults.js";
|
|
7
8
|
import { activeIngestionMetrics } from "./ingestion-metrics.js";
|
|
8
9
|
import { operationSignal, throwIfAborted } from "./operation.js";
|
|
9
10
|
import { readSourceFingerprintCache, reusableSourceFingerprint, sourceFingerprintRecord, writeSourceFingerprintCache, } from "./source-fingerprint-cache.js";
|
|
11
|
+
import { canonicalRelativePath, canonicalSourcePath, compareCodePoints, isCaseInsensitiveFileSystem, resolveExistingPath, sourcePathKey, unicodeGlobVariants, } from "./source-paths.js";
|
|
10
12
|
const GENERATED_SOURCE_READMES = new Set([
|
|
11
13
|
`${DEFAULT_CONFIG.rawDir}/README.md`,
|
|
12
14
|
`${LEGACY_PRIVATE_DIR}/README.md`,
|
|
@@ -87,6 +89,72 @@ export const DEFAULT_FAST_GLOB_IGNORES = [
|
|
|
87
89
|
`**/${LEGACY_KB_DIR}/**`,
|
|
88
90
|
`**/${RAGMIR_DIR}/**`,
|
|
89
91
|
];
|
|
92
|
+
/**
|
|
93
|
+
* Tool state, dependency caches, lockfiles, and generated bundles that drown out project knowledge.
|
|
94
|
+
* They are skipped unless `defaultExcludes` is false or a source entry names the directory itself.
|
|
95
|
+
*/
|
|
96
|
+
export const DEFAULT_EXCLUDED_DIRECTORIES = [
|
|
97
|
+
".agents",
|
|
98
|
+
".angular",
|
|
99
|
+
".cache",
|
|
100
|
+
".claude",
|
|
101
|
+
".cline",
|
|
102
|
+
".codex",
|
|
103
|
+
".cursor",
|
|
104
|
+
".dart_tool",
|
|
105
|
+
".expo",
|
|
106
|
+
".gitnexus",
|
|
107
|
+
".gradle",
|
|
108
|
+
".idea",
|
|
109
|
+
".kimi",
|
|
110
|
+
".mypy_cache",
|
|
111
|
+
".next",
|
|
112
|
+
".nuxt",
|
|
113
|
+
".nyc_output",
|
|
114
|
+
".opencode",
|
|
115
|
+
".parcel-cache",
|
|
116
|
+
".pnpm-store",
|
|
117
|
+
".pytest_cache",
|
|
118
|
+
".ruff_cache",
|
|
119
|
+
".svelte-kit",
|
|
120
|
+
".terraform",
|
|
121
|
+
".tox",
|
|
122
|
+
".turbo",
|
|
123
|
+
".venv",
|
|
124
|
+
".windsurf",
|
|
125
|
+
".yarn",
|
|
126
|
+
"__pycache__",
|
|
127
|
+
"bower_components",
|
|
128
|
+
"coverage",
|
|
129
|
+
"DerivedData",
|
|
130
|
+
"Pods",
|
|
131
|
+
];
|
|
132
|
+
export const DEFAULT_EXCLUDED_FILES = [
|
|
133
|
+
"*.map",
|
|
134
|
+
"*.min.css",
|
|
135
|
+
"*.min.js",
|
|
136
|
+
"*.min.mjs",
|
|
137
|
+
"bun.lock",
|
|
138
|
+
"bun.lockb",
|
|
139
|
+
"Cargo.lock",
|
|
140
|
+
"composer.lock",
|
|
141
|
+
"deno.lock",
|
|
142
|
+
"flake.lock",
|
|
143
|
+
"Gemfile.lock",
|
|
144
|
+
"go.sum",
|
|
145
|
+
"gradle.lockfile",
|
|
146
|
+
"mix.lock",
|
|
147
|
+
"npm-shrinkwrap.json",
|
|
148
|
+
"package-lock.json",
|
|
149
|
+
"packages.lock.json",
|
|
150
|
+
"Pipfile.lock",
|
|
151
|
+
"pnpm-lock.yaml",
|
|
152
|
+
"Podfile.lock",
|
|
153
|
+
"poetry.lock",
|
|
154
|
+
"pubspec.lock",
|
|
155
|
+
"uv.lock",
|
|
156
|
+
"yarn.lock",
|
|
157
|
+
];
|
|
90
158
|
const GLOB_PATTERN_CHARS = /[*?[{]/u;
|
|
91
159
|
export const DEFAULT_SUPPORTED_EXTENSIONS = new Set([
|
|
92
160
|
".atom",
|
|
@@ -173,44 +241,191 @@ export async function listSourceFiles(config, options = {}) {
|
|
|
173
241
|
export async function inventorySourceFiles(config, options = {}) {
|
|
174
242
|
const signal = operationSignal(options);
|
|
175
243
|
throwIfAborted(signal);
|
|
176
|
-
const
|
|
244
|
+
const discovery = await discoverSourceCandidates(config, signal);
|
|
245
|
+
const candidates = new Map(discovery.candidates.map((candidate) => [candidate.absolutePath, candidate]));
|
|
177
246
|
throwIfAborted(signal);
|
|
178
|
-
const
|
|
179
|
-
?
|
|
180
|
-
: await
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
247
|
+
const cachedFingerprints = config.sourceFingerprintMode === "strict"
|
|
248
|
+
? new Map()
|
|
249
|
+
: await readSourceFingerprintCache(config);
|
|
250
|
+
const usableCache = cachedFingerprints ?? new Map();
|
|
251
|
+
const nextFingerprints = new Map();
|
|
252
|
+
const files = new Map();
|
|
253
|
+
const verifiedAt = new Date().toISOString();
|
|
254
|
+
let contentBytesRead = 0;
|
|
255
|
+
let hashedFiles = 0;
|
|
256
|
+
let reusedFingerprints = 0;
|
|
257
|
+
await mapLimit([...candidates.values()], config.ingestConcurrency, signal, async (candidate) => {
|
|
258
|
+
const identity = sourceFileIdentity(candidate.absolutePath, candidate.info);
|
|
259
|
+
const reusable = reusableSourceFingerprint(usableCache.get(candidate.absolutePath), identity, config.sourceFingerprintMode);
|
|
260
|
+
let checksum;
|
|
261
|
+
if (reusable) {
|
|
262
|
+
checksum = reusable.checksum;
|
|
263
|
+
reusedFingerprints += 1;
|
|
264
|
+
nextFingerprints.set(candidate.absolutePath, reusable);
|
|
265
|
+
}
|
|
266
|
+
else {
|
|
267
|
+
const hashed = await checksumFile(candidate.absolutePath, signal);
|
|
268
|
+
const currentIdentity = sourceFileIdentity(candidate.absolutePath, sourceEntryStats(await stat(candidate.absolutePath)));
|
|
269
|
+
if (!sameSourceFileIdentity(identity, currentIdentity)) {
|
|
270
|
+
throw new Error(`Source file changed while it was being hashed: ${candidate.relativePath}. Retry ingestion.`);
|
|
271
|
+
}
|
|
272
|
+
checksum = hashed.checksum;
|
|
273
|
+
hashedFiles += 1;
|
|
274
|
+
contentBytesRead += hashed.bytesRead;
|
|
275
|
+
nextFingerprints.set(candidate.absolutePath, sourceFingerprintRecord(identity, checksum, verifiedAt));
|
|
276
|
+
}
|
|
277
|
+
files.set(candidate.absolutePath, {
|
|
278
|
+
absolutePath: candidate.absolutePath,
|
|
279
|
+
relativePath: candidate.relativePath,
|
|
280
|
+
source: candidate.source,
|
|
281
|
+
extension: candidate.extension,
|
|
282
|
+
bytes: candidate.info.size,
|
|
283
|
+
mtimeMs: candidate.info.mtimeMs,
|
|
284
|
+
checksum,
|
|
285
|
+
});
|
|
286
|
+
});
|
|
287
|
+
if (options.writeFingerprintCache !== false) {
|
|
288
|
+
await writeSourceFingerprintCache([...nextFingerprints.values()].sort((left, right) => compareCodePoints(left.absolutePath, right.absolutePath)), config);
|
|
289
|
+
}
|
|
188
290
|
throwIfAborted(signal);
|
|
291
|
+
return {
|
|
292
|
+
discoveredFiles: discovery.discoveredFiles,
|
|
293
|
+
supportedFiles: [...files.values()].sort((a, b) => compareCodePoints(a.relativePath, b.relativePath)),
|
|
294
|
+
skippedFiles: discovery.skippedFiles,
|
|
295
|
+
contentBytesRead,
|
|
296
|
+
hashedFiles,
|
|
297
|
+
reusedFingerprints,
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Discover indexable source files with their size and modification time, without reading content.
|
|
302
|
+
* Freshness checks compare this O(files) view with the index manifest.
|
|
303
|
+
*/
|
|
304
|
+
export async function discoverSourceFiles(config, options = {}) {
|
|
305
|
+
const signal = operationSignal(options);
|
|
306
|
+
const discovery = await discoverSourceCandidates(config, signal);
|
|
307
|
+
return {
|
|
308
|
+
files: discovery.candidates
|
|
309
|
+
.map((candidate) => ({
|
|
310
|
+
absolutePath: candidate.absolutePath,
|
|
311
|
+
relativePath: candidate.relativePath,
|
|
312
|
+
bytes: candidate.info.size,
|
|
313
|
+
mtimeMs: candidate.info.mtimeMs,
|
|
314
|
+
identity: sourceFileIdentity(candidate.absolutePath, candidate.info),
|
|
315
|
+
}))
|
|
316
|
+
.sort((left, right) => compareCodePoints(left.relativePath, right.relativePath)),
|
|
317
|
+
skippedFiles: discovery.skippedFiles,
|
|
318
|
+
discoveredFiles: discovery.discoveredFiles,
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
/** Explain source rules that silently select nothing or leave excluded content reachable. */
|
|
322
|
+
export async function sourceRuleDiagnostics(config, supportedFiles, options = {}) {
|
|
323
|
+
const signal = operationSignal(options);
|
|
324
|
+
const inputs = await sourceInputs(config, signal);
|
|
325
|
+
const caseInsensitive = isCaseInsensitiveFileSystem(config.projectRoot);
|
|
326
|
+
const supportedPaths = supportedFiles.map((file) => file.relativePath);
|
|
327
|
+
const unmatchedPatterns = inputs.patterns.filter((pattern) => {
|
|
328
|
+
const matches = picomatch(pattern, { dot: true, nocase: caseInsensitive });
|
|
329
|
+
return !supportedPaths.some((relativePath) => matches(relativePath));
|
|
330
|
+
});
|
|
331
|
+
const indexedByChecksum = new Map();
|
|
332
|
+
for (const file of supportedFiles) {
|
|
333
|
+
if (!indexedByChecksum.has(file.checksum)) {
|
|
334
|
+
indexedByChecksum.set(file.checksum, file.relativePath);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
const unmatchedExclusions = [];
|
|
338
|
+
const excludedCopies = [];
|
|
339
|
+
for (const pattern of inputs.ignorePatterns) {
|
|
340
|
+
throwIfAborted(signal);
|
|
341
|
+
if (GLOB_PATTERN_CHARS.test(pattern)) {
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
const resolved = resolveExistingPath(path.resolve(config.projectRoot, pattern), caseInsensitive);
|
|
345
|
+
if (resolved === null) {
|
|
346
|
+
unmatchedExclusions.push(pattern);
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
const info = await stat(resolved);
|
|
350
|
+
if (!info.isFile() || info.size > config.maxFileBytes) {
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
const { checksum } = await checksumFile(resolved, signal);
|
|
354
|
+
const indexedCopy = indexedByChecksum.get(checksum);
|
|
355
|
+
if (indexedCopy !== undefined) {
|
|
356
|
+
excludedCopies.push({ excluded: pattern, indexedCopy });
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
const portabilityWarnings = config.sources.flatMap((entry) => {
|
|
360
|
+
const value = entry.trim().replace(/^!/u, "").trim();
|
|
361
|
+
if (!value || value.startsWith("#")) {
|
|
362
|
+
return [];
|
|
363
|
+
}
|
|
364
|
+
if (value.includes("\\")) {
|
|
365
|
+
return [
|
|
366
|
+
`Source entry "${entry}" uses backslashes; write forward slashes so the config works on macOS, Linux, and Windows.`,
|
|
367
|
+
];
|
|
368
|
+
}
|
|
369
|
+
if (path.isAbsolute(value)) {
|
|
370
|
+
return [
|
|
371
|
+
`Source entry "${entry}" is absolute; a project-relative path keeps the config portable across machines.`,
|
|
372
|
+
];
|
|
373
|
+
}
|
|
374
|
+
return [];
|
|
375
|
+
});
|
|
376
|
+
return {
|
|
377
|
+
unmatchedSources: inputs.unresolvedRoots,
|
|
378
|
+
unmatchedPatterns,
|
|
379
|
+
unmatchedExclusions,
|
|
380
|
+
excludedCopies,
|
|
381
|
+
portabilityWarnings,
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
async function discoverSourceCandidates(config, signal) {
|
|
385
|
+
const inputs = await sourceInputs(config, signal);
|
|
386
|
+
throwIfAborted(signal);
|
|
387
|
+
const caseInsensitive = isCaseInsensitiveFileSystem(config.projectRoot);
|
|
388
|
+
const isExcluded = sourceExclusionMatcher(inputs.ignorePatterns, caseInsensitive);
|
|
189
389
|
const candidates = new Map();
|
|
190
390
|
const skippedFiles = new Map();
|
|
191
391
|
const discoveredPaths = new Set();
|
|
192
392
|
const allowedExtensions = supportedExtensions(config);
|
|
193
393
|
const recordSourceCandidate = (absolutePath, info, source) => {
|
|
194
394
|
throwIfAborted(signal);
|
|
195
|
-
|
|
395
|
+
const relativePath = canonicalRelativePath(config.projectRoot, absolutePath);
|
|
396
|
+
if (isExcluded(relativePath) || GENERATED_SOURCE_READMES.has(relativePath)) {
|
|
196
397
|
return;
|
|
197
398
|
}
|
|
198
|
-
const
|
|
199
|
-
|
|
399
|
+
const key = sourcePathKey(relativePath, caseInsensitive);
|
|
400
|
+
const existing = candidates.get(key);
|
|
401
|
+
if (existing) {
|
|
402
|
+
if (existing.absolutePath !== absolutePath) {
|
|
403
|
+
// Linux and Windows can hold two names that differ only by Unicode form; both would cite
|
|
404
|
+
// the same canonical path, so the second one is reported instead of silently shadowed.
|
|
405
|
+
skippedFiles.set(absolutePath, {
|
|
406
|
+
relativePath,
|
|
407
|
+
source: canonicalSourcePath(source),
|
|
408
|
+
extension: path.extname(absolutePath).toLowerCase() || NO_EXTENSION,
|
|
409
|
+
bytes: info.size,
|
|
410
|
+
reason: "path-collision",
|
|
411
|
+
recommendation: skippedRecommendation("path-collision", ""),
|
|
412
|
+
});
|
|
413
|
+
}
|
|
200
414
|
return;
|
|
201
415
|
}
|
|
202
|
-
discoveredPaths.add(
|
|
416
|
+
discoveredPaths.add(key);
|
|
203
417
|
const extension = path.extname(absolutePath).toLowerCase();
|
|
204
|
-
const
|
|
418
|
+
const canonicalSource = canonicalSourcePath(source);
|
|
419
|
+
const skipped = skippedSourceFile(absolutePath, relativePath, canonicalSource, extension, info.size);
|
|
205
420
|
if (skipped) {
|
|
206
|
-
skippedFiles.set(
|
|
421
|
+
skippedFiles.set(key, skipped);
|
|
207
422
|
return;
|
|
208
423
|
}
|
|
209
424
|
if (!isSupportedSourceFile(absolutePath, extension, allowedExtensions)) {
|
|
210
425
|
const normalizedExtension = extension || NO_EXTENSION;
|
|
211
|
-
skippedFiles.set(
|
|
426
|
+
skippedFiles.set(key, {
|
|
212
427
|
relativePath,
|
|
213
|
-
source,
|
|
428
|
+
source: canonicalSource,
|
|
214
429
|
extension: normalizedExtension,
|
|
215
430
|
bytes: info.size,
|
|
216
431
|
reason: "unsupported-extension",
|
|
@@ -220,9 +435,9 @@ export async function inventorySourceFiles(config, options = {}) {
|
|
|
220
435
|
}
|
|
221
436
|
if (info.size > config.maxFileBytes) {
|
|
222
437
|
const normalizedExtension = extension || NO_EXTENSION;
|
|
223
|
-
skippedFiles.set(
|
|
438
|
+
skippedFiles.set(key, {
|
|
224
439
|
relativePath,
|
|
225
|
-
source,
|
|
440
|
+
source: canonicalSource,
|
|
226
441
|
extension: normalizedExtension,
|
|
227
442
|
bytes: info.size,
|
|
228
443
|
reason: "oversized",
|
|
@@ -230,54 +445,61 @@ export async function inventorySourceFiles(config, options = {}) {
|
|
|
230
445
|
});
|
|
231
446
|
return;
|
|
232
447
|
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
448
|
+
candidates.set(key, {
|
|
449
|
+
absolutePath,
|
|
450
|
+
info,
|
|
451
|
+
source: canonicalSource,
|
|
452
|
+
relativePath,
|
|
453
|
+
extension,
|
|
454
|
+
});
|
|
236
455
|
};
|
|
237
456
|
for (const root of inputs.roots) {
|
|
238
457
|
throwIfAborted(signal);
|
|
239
|
-
|
|
240
|
-
continue;
|
|
241
|
-
}
|
|
242
|
-
const rootInfo = await stat(root);
|
|
458
|
+
const rootInfo = await stat(root.path);
|
|
243
459
|
throwIfAborted(signal);
|
|
244
460
|
const entries = rootInfo.isDirectory()
|
|
245
461
|
? (await fg("**/*", {
|
|
246
|
-
cwd: root,
|
|
462
|
+
cwd: root.path,
|
|
247
463
|
absolute: true,
|
|
248
464
|
onlyFiles: true,
|
|
249
465
|
dot: true,
|
|
250
466
|
followSymbolicLinks: false,
|
|
467
|
+
caseSensitiveMatch: !caseInsensitive,
|
|
251
468
|
ignore: [
|
|
252
469
|
...DEFAULT_FAST_GLOB_IGNORES,
|
|
253
|
-
...
|
|
470
|
+
...defaultExcludeGlobs(config, [root.relativePath]),
|
|
471
|
+
...unicodeGlobVariants(ignorePatternsForRoot(root.relativePath, inputs.ignorePatterns)),
|
|
254
472
|
],
|
|
255
473
|
objectMode: true,
|
|
256
474
|
stats: true,
|
|
257
475
|
unique: true,
|
|
258
476
|
}))
|
|
259
|
-
: [{ path: root, stats: sourceEntryStats(rootInfo) }];
|
|
477
|
+
: [{ path: root.path, stats: sourceEntryStats(rootInfo) }];
|
|
260
478
|
throwIfAborted(signal);
|
|
261
479
|
for (const entry of entries) {
|
|
262
480
|
throwIfAborted(signal);
|
|
263
|
-
const absolutePath = path.
|
|
481
|
+
const absolutePath = path.resolve(root.path, entry.path);
|
|
264
482
|
const info = entry.stats ?? sourceEntryStats(await stat(absolutePath));
|
|
265
|
-
const relativePath = path.relative(config.projectRoot, absolutePath);
|
|
266
483
|
const source = rootInfo.isDirectory()
|
|
267
|
-
? path.relative(root, absolutePath) || path.basename(absolutePath)
|
|
268
|
-
: relativePath || path.basename(absolutePath);
|
|
484
|
+
? path.relative(root.path, absolutePath) || path.basename(absolutePath)
|
|
485
|
+
: root.relativePath || path.basename(absolutePath);
|
|
269
486
|
recordSourceCandidate(absolutePath, info, source);
|
|
270
487
|
}
|
|
271
488
|
}
|
|
272
|
-
|
|
489
|
+
for (const patterns of patternGroupsByDefaultExcludes(config, inputs.patterns)) {
|
|
273
490
|
throwIfAborted(signal);
|
|
274
|
-
const entries = (await fg(
|
|
491
|
+
const entries = (await fg(unicodeGlobVariants(patterns), {
|
|
275
492
|
cwd: config.projectRoot,
|
|
276
493
|
absolute: true,
|
|
277
494
|
onlyFiles: true,
|
|
278
495
|
dot: true,
|
|
279
496
|
followSymbolicLinks: false,
|
|
280
|
-
|
|
497
|
+
caseSensitiveMatch: !caseInsensitive,
|
|
498
|
+
ignore: [
|
|
499
|
+
...DEFAULT_FAST_GLOB_IGNORES,
|
|
500
|
+
...defaultExcludeGlobs(config, patterns),
|
|
501
|
+
...unicodeGlobVariants(inputs.ignorePatterns),
|
|
502
|
+
],
|
|
281
503
|
objectMode: true,
|
|
282
504
|
stats: true,
|
|
283
505
|
unique: true,
|
|
@@ -285,67 +507,71 @@ export async function inventorySourceFiles(config, options = {}) {
|
|
|
285
507
|
throwIfAborted(signal);
|
|
286
508
|
for (const entry of entries) {
|
|
287
509
|
throwIfAborted(signal);
|
|
288
|
-
const absolutePath = path.
|
|
289
|
-
? entry.path
|
|
290
|
-
: path.resolve(config.projectRoot, entry.path);
|
|
510
|
+
const absolutePath = path.resolve(config.projectRoot, entry.path);
|
|
291
511
|
const info = entry.stats ?? sourceEntryStats(await stat(absolutePath));
|
|
292
|
-
const relativePath =
|
|
512
|
+
const relativePath = canonicalRelativePath(config.projectRoot, absolutePath);
|
|
293
513
|
recordSourceCandidate(absolutePath, info, relativePath || path.basename(absolutePath));
|
|
294
514
|
}
|
|
295
515
|
}
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
:
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
516
|
+
return {
|
|
517
|
+
candidates: [...candidates.values()],
|
|
518
|
+
skippedFiles: [...skippedFiles.values()].sort((a, b) => compareCodePoints(a.relativePath, b.relativePath)),
|
|
519
|
+
discoveredFiles: discoveredPaths.size,
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
/**
|
|
523
|
+
* Match exclusion rules against canonical project-relative paths, independent of the on-disk
|
|
524
|
+
* Unicode form, and with the volume's case rules. A rule that names a directory excludes its files,
|
|
525
|
+
* as fast-glob ignore patterns do.
|
|
526
|
+
*/
|
|
527
|
+
function sourceExclusionMatcher(patterns, caseInsensitive) {
|
|
528
|
+
if (patterns.length === 0) {
|
|
529
|
+
return () => false;
|
|
530
|
+
}
|
|
531
|
+
const matches = picomatch([...patterns], { dot: true, nocase: caseInsensitive });
|
|
532
|
+
return (relativePath) => {
|
|
533
|
+
if (matches(relativePath)) {
|
|
534
|
+
return true;
|
|
314
535
|
}
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
if (!sameSourceFileIdentity(identity, currentIdentity)) {
|
|
319
|
-
throw new Error(`Source file changed while it was being hashed: ${candidate.relativePath}. Retry ingestion.`);
|
|
536
|
+
for (let index = relativePath.lastIndexOf("/"); index > 0; index = relativePath.lastIndexOf("/", index - 1)) {
|
|
537
|
+
if (matches(relativePath.slice(0, index))) {
|
|
538
|
+
return true;
|
|
320
539
|
}
|
|
321
|
-
checksum = hashed.checksum;
|
|
322
|
-
hashedFiles += 1;
|
|
323
|
-
contentBytesRead += hashed.bytesRead;
|
|
324
|
-
nextFingerprints.set(candidate.absolutePath, sourceFingerprintRecord(identity, checksum, verifiedAt));
|
|
325
540
|
}
|
|
326
|
-
|
|
327
|
-
absolutePath: candidate.absolutePath,
|
|
328
|
-
relativePath: candidate.relativePath,
|
|
329
|
-
source: candidate.source,
|
|
330
|
-
extension: candidate.extension,
|
|
331
|
-
bytes: candidate.info.size,
|
|
332
|
-
mtimeMs: candidate.info.mtimeMs,
|
|
333
|
-
checksum,
|
|
334
|
-
});
|
|
335
|
-
});
|
|
336
|
-
if (options.writeFingerprintCache !== false) {
|
|
337
|
-
await writeSourceFingerprintCache([...nextFingerprints.values()].sort((left, right) => left.absolutePath.localeCompare(right.absolutePath)), config);
|
|
338
|
-
}
|
|
339
|
-
throwIfAborted(signal);
|
|
340
|
-
return {
|
|
341
|
-
discoveredFiles: discoveredPaths.size,
|
|
342
|
-
supportedFiles: [...files.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)),
|
|
343
|
-
skippedFiles: [...skippedFiles.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)),
|
|
344
|
-
contentBytesRead,
|
|
345
|
-
hashedFiles,
|
|
346
|
-
reusedFingerprints,
|
|
541
|
+
return false;
|
|
347
542
|
};
|
|
348
543
|
}
|
|
544
|
+
/** Built-in exclusions, minus the ones a source entry targets explicitly. */
|
|
545
|
+
function defaultExcludeGlobs(config, sourceEntries) {
|
|
546
|
+
if (!config.defaultExcludes) {
|
|
547
|
+
return [];
|
|
548
|
+
}
|
|
549
|
+
const named = new Set(sourceEntries.flatMap((entry) => literalSegments(entry)));
|
|
550
|
+
return [
|
|
551
|
+
...DEFAULT_EXCLUDED_DIRECTORIES.filter((directory) => !named.has(directory)).map((directory) => `**/${directory}/**`),
|
|
552
|
+
...DEFAULT_EXCLUDED_FILES.filter((file) => !named.has(file)).map((file) => `**/${file}`),
|
|
553
|
+
];
|
|
554
|
+
}
|
|
555
|
+
/** Split include patterns so that one naming `.claude/` does not re-enable it for every pattern. */
|
|
556
|
+
function patternGroupsByDefaultExcludes(config, patterns) {
|
|
557
|
+
const groups = new Map();
|
|
558
|
+
for (const pattern of patterns) {
|
|
559
|
+
const key = defaultExcludeGlobs(config, [pattern]).join("\0");
|
|
560
|
+
const group = groups.get(key);
|
|
561
|
+
if (group) {
|
|
562
|
+
group.push(pattern);
|
|
563
|
+
}
|
|
564
|
+
else {
|
|
565
|
+
groups.set(key, [pattern]);
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
return [...groups.values()];
|
|
569
|
+
}
|
|
570
|
+
function literalSegments(entry) {
|
|
571
|
+
return entry
|
|
572
|
+
.split("/")
|
|
573
|
+
.filter((segment) => segment.length > 0 && !GLOB_PATTERN_CHARS.test(segment));
|
|
574
|
+
}
|
|
349
575
|
async function checksumFile(filePath, signal) {
|
|
350
576
|
const hash = createHash("sha256");
|
|
351
577
|
let bytesRead = 0;
|
|
@@ -420,10 +646,9 @@ function sameSourceFileIdentity(left, right) {
|
|
|
420
646
|
left.ino === right.ino &&
|
|
421
647
|
left.mode === right.mode);
|
|
422
648
|
}
|
|
423
|
-
function ignorePatternsForRoot(
|
|
424
|
-
const relativeRoot = path.relative(projectRoot, root).replaceAll(path.sep, "/");
|
|
649
|
+
function ignorePatternsForRoot(relativeRoot, patterns) {
|
|
425
650
|
if (!relativeRoot) {
|
|
426
|
-
return patterns;
|
|
651
|
+
return [...patterns];
|
|
427
652
|
}
|
|
428
653
|
const prefix = `${relativeRoot}/`;
|
|
429
654
|
return patterns.flatMap((pattern) => {
|
|
@@ -447,7 +672,8 @@ export function summarizeUnsupportedExtensions(skippedFiles) {
|
|
|
447
672
|
}
|
|
448
673
|
async function sourceInputs(config, signal) {
|
|
449
674
|
throwIfAborted(signal);
|
|
450
|
-
const
|
|
675
|
+
const caseInsensitive = isCaseInsensitiveFileSystem(config.projectRoot);
|
|
676
|
+
const requestedRoots = [config.rawDir];
|
|
451
677
|
const patterns = [];
|
|
452
678
|
const ignorePatterns = [];
|
|
453
679
|
const classifyEntry = (entry) => {
|
|
@@ -463,7 +689,7 @@ async function sourceInputs(config, signal) {
|
|
|
463
689
|
patterns.push(sourcePattern(config.projectRoot, trimmed));
|
|
464
690
|
return;
|
|
465
691
|
}
|
|
466
|
-
|
|
692
|
+
requestedRoots.push(path.isAbsolute(trimmed) ? trimmed : path.resolve(config.projectRoot, trimmed));
|
|
467
693
|
};
|
|
468
694
|
// Inline `sources` from config.json are the primary mechanism; the legacy
|
|
469
695
|
// sources.txt file is still read when present so existing projects keep working.
|
|
@@ -485,14 +711,29 @@ async function sourceInputs(config, signal) {
|
|
|
485
711
|
classifyEntry(line);
|
|
486
712
|
}
|
|
487
713
|
}
|
|
714
|
+
const roots = [];
|
|
715
|
+
const unresolvedRoots = [];
|
|
716
|
+
for (const requested of requestedRoots) {
|
|
717
|
+
// Linux and Windows compare names byte for byte, so a configured NFC folder name must still
|
|
718
|
+
// reach an NFD folder copied from macOS or an archive.
|
|
719
|
+
const resolved = resolveExistingPath(requested, caseInsensitive);
|
|
720
|
+
if (resolved === null) {
|
|
721
|
+
if (requested !== config.rawDir) {
|
|
722
|
+
unresolvedRoots.push(canonicalRelativePath(config.projectRoot, requested));
|
|
723
|
+
}
|
|
724
|
+
continue;
|
|
725
|
+
}
|
|
726
|
+
roots.push({
|
|
727
|
+
path: resolved,
|
|
728
|
+
relativePath: canonicalRelativePath(config.projectRoot, resolved),
|
|
729
|
+
});
|
|
730
|
+
}
|
|
488
731
|
throwIfAborted(signal);
|
|
489
|
-
return { roots, patterns, ignorePatterns };
|
|
732
|
+
return { roots, patterns, ignorePatterns, unresolvedRoots };
|
|
490
733
|
}
|
|
491
734
|
function sourcePattern(projectRoot, input) {
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
}
|
|
495
|
-
return input.replaceAll(path.sep, "/");
|
|
735
|
+
const pattern = path.isAbsolute(input) ? path.relative(projectRoot, input) : input;
|
|
736
|
+
return canonicalSourcePath(pattern).replace(/^(?:\.\/)+/u, "");
|
|
496
737
|
}
|
|
497
738
|
export function isSensitiveFilePath(absolutePath) {
|
|
498
739
|
const baseName = path.basename(absolutePath).toLowerCase();
|
|
@@ -521,6 +762,9 @@ function skippedSourceFile(absolutePath, relativePath, source, extension, bytes)
|
|
|
521
762
|
};
|
|
522
763
|
}
|
|
523
764
|
function skippedRecommendation(reason, extension) {
|
|
765
|
+
if (reason === "path-collision") {
|
|
766
|
+
return "Rename one of the files: their names differ only by Unicode normalization, so they would share one citation path.";
|
|
767
|
+
}
|
|
524
768
|
if (reason === "sensitive-name") {
|
|
525
769
|
return "Review manually; secret-like files are skipped to avoid indexing credentials or private keys.";
|
|
526
770
|
}
|