@jcode.labs/ragmir 6.1.5 → 6.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/README.md +43 -49
  2. package/dist/cli.js +194 -24
  3. package/dist/cli.js.map +1 -1
  4. package/dist/config.d.ts +24 -2
  5. package/dist/config.d.ts.map +1 -1
  6. package/dist/config.js +127 -24
  7. package/dist/config.js.map +1 -1
  8. package/dist/context-resources.d.ts.map +1 -1
  9. package/dist/context-resources.js +19 -8
  10. package/dist/context-resources.js.map +1 -1
  11. package/dist/defaults.d.ts +5 -0
  12. package/dist/defaults.d.ts.map +1 -1
  13. package/dist/defaults.js +16 -0
  14. package/dist/defaults.js.map +1 -1
  15. package/dist/doctor.d.ts.map +1 -1
  16. package/dist/doctor.js +37 -5
  17. package/dist/doctor.js.map +1 -1
  18. package/dist/evaluate.d.ts.map +1 -1
  19. package/dist/evaluate.js +11 -6
  20. package/dist/evaluate.js.map +1 -1
  21. package/dist/files.d.ts +27 -1
  22. package/dist/files.d.ts.map +1 -1
  23. package/dist/files.js +344 -100
  24. package/dist/files.js.map +1 -1
  25. package/dist/freshness.d.ts +16 -0
  26. package/dist/freshness.d.ts.map +1 -0
  27. package/dist/freshness.js +97 -0
  28. package/dist/freshness.js.map +1 -0
  29. package/dist/generation-retention.d.ts +12 -0
  30. package/dist/generation-retention.d.ts.map +1 -1
  31. package/dist/generation-retention.js +13 -2
  32. package/dist/generation-retention.js.map +1 -1
  33. package/dist/git-hooks.d.ts +25 -0
  34. package/dist/git-hooks.d.ts.map +1 -0
  35. package/dist/git-hooks.js +173 -0
  36. package/dist/git-hooks.js.map +1 -0
  37. package/dist/index-diagnostics.d.ts +9 -1
  38. package/dist/index-diagnostics.d.ts.map +1 -1
  39. package/dist/index-diagnostics.js +35 -0
  40. package/dist/index-diagnostics.js.map +1 -1
  41. package/dist/index-policy.d.ts.map +1 -1
  42. package/dist/index-policy.js +4 -0
  43. package/dist/index-policy.js.map +1 -1
  44. package/dist/ingest.d.ts +6 -0
  45. package/dist/ingest.d.ts.map +1 -1
  46. package/dist/ingest.js +44 -21
  47. package/dist/ingest.js.map +1 -1
  48. package/dist/init.d.ts.map +1 -1
  49. package/dist/init.js +19 -4
  50. package/dist/init.js.map +1 -1
  51. package/dist/knowledge-bases.d.ts +18 -1
  52. package/dist/knowledge-bases.d.ts.map +1 -1
  53. package/dist/knowledge-bases.js +73 -4
  54. package/dist/knowledge-bases.js.map +1 -1
  55. package/dist/mcp-output.d.ts +14 -3
  56. package/dist/mcp-output.d.ts.map +1 -1
  57. package/dist/mcp-output.js +37 -0
  58. package/dist/mcp-output.js.map +1 -1
  59. package/dist/mcp-registrations.d.ts +9 -0
  60. package/dist/mcp-registrations.d.ts.map +1 -0
  61. package/dist/mcp-registrations.js +247 -0
  62. package/dist/mcp-registrations.js.map +1 -0
  63. package/dist/mcp-scope.d.ts +32 -0
  64. package/dist/mcp-scope.d.ts.map +1 -0
  65. package/dist/mcp-scope.js +114 -0
  66. package/dist/mcp-scope.js.map +1 -0
  67. package/dist/mcp-session.d.ts +37 -0
  68. package/dist/mcp-session.d.ts.map +1 -0
  69. package/dist/mcp-session.js +53 -0
  70. package/dist/mcp-session.js.map +1 -0
  71. package/dist/mcp-summaries.d.ts +5 -0
  72. package/dist/mcp-summaries.d.ts.map +1 -1
  73. package/dist/mcp-summaries.js +30 -0
  74. package/dist/mcp-summaries.js.map +1 -1
  75. package/dist/mcp.d.ts +6 -0
  76. package/dist/mcp.d.ts.map +1 -1
  77. package/dist/mcp.js +410 -103
  78. package/dist/mcp.js.map +1 -1
  79. package/dist/ocr.d.ts +51 -0
  80. package/dist/ocr.d.ts.map +1 -1
  81. package/dist/ocr.js +210 -4
  82. package/dist/ocr.js.map +1 -1
  83. package/dist/package-manager.d.ts.map +1 -1
  84. package/dist/package-manager.js +2 -1
  85. package/dist/package-manager.js.map +1 -1
  86. package/dist/preview.d.ts.map +1 -1
  87. package/dist/preview.js +3 -7
  88. package/dist/preview.js.map +1 -1
  89. package/dist/quality-report.d.ts.map +1 -1
  90. package/dist/quality-report.js +1 -1
  91. package/dist/quality-report.js.map +1 -1
  92. package/dist/query.d.ts.map +1 -1
  93. package/dist/query.js +254 -40
  94. package/dist/query.js.map +1 -1
  95. package/dist/ranking.d.ts +56 -6
  96. package/dist/ranking.d.ts.map +1 -1
  97. package/dist/ranking.js +187 -16
  98. package/dist/ranking.js.map +1 -1
  99. package/dist/retrieval-diversity.d.ts +5 -0
  100. package/dist/retrieval-diversity.d.ts.map +1 -1
  101. package/dist/retrieval-diversity.js +4 -2
  102. package/dist/retrieval-diversity.js.map +1 -1
  103. package/dist/search-output.d.ts.map +1 -1
  104. package/dist/search-output.js +2 -0
  105. package/dist/search-output.js.map +1 -1
  106. package/dist/security.js +1 -1
  107. package/dist/security.js.map +1 -1
  108. package/dist/setup-prompt.d.ts +1 -1
  109. package/dist/setup-prompt.d.ts.map +1 -1
  110. package/dist/setup-prompt.js +3 -3
  111. package/dist/setup.js +1 -1
  112. package/dist/setup.js.map +1 -1
  113. package/dist/skill.d.ts +2 -0
  114. package/dist/skill.d.ts.map +1 -1
  115. package/dist/skill.js +122 -14
  116. package/dist/skill.js.map +1 -1
  117. package/dist/source-paths.d.ts +36 -0
  118. package/dist/source-paths.d.ts.map +1 -0
  119. package/dist/source-paths.js +157 -0
  120. package/dist/source-paths.js.map +1 -0
  121. package/dist/source-weights.d.ts +28 -0
  122. package/dist/source-weights.d.ts.map +1 -0
  123. package/dist/source-weights.js +241 -0
  124. package/dist/source-weights.js.map +1 -0
  125. package/dist/storage-maintenance.d.ts +3 -1
  126. package/dist/storage-maintenance.d.ts.map +1 -1
  127. package/dist/storage-maintenance.js +21 -2
  128. package/dist/storage-maintenance.js.map +1 -1
  129. package/dist/storage-usage.d.ts +21 -0
  130. package/dist/storage-usage.d.ts.map +1 -0
  131. package/dist/storage-usage.js +118 -0
  132. package/dist/storage-usage.js.map +1 -0
  133. package/dist/types.d.ts +127 -3
  134. package/dist/types.d.ts.map +1 -1
  135. package/dist/version.d.ts +1 -1
  136. package/dist/version.js +1 -1
  137. package/package.json +3 -1
package/dist/files.js CHANGED
@@ -3,10 +3,12 @@ import { createReadStream, existsSync } from "node:fs";
3
3
  import { readFile, stat } from "node:fs/promises";
4
4
  import path from "node:path";
5
5
  import fg from "fast-glob";
6
+ import picomatch from "picomatch";
6
7
  import { DEFAULT_CONFIG, LEGACY_KB_DIR, LEGACY_PRIVATE_DIR, RAGMIR_DIR } from "./defaults.js";
7
8
  import { activeIngestionMetrics } from "./ingestion-metrics.js";
8
9
  import { operationSignal, throwIfAborted } from "./operation.js";
9
10
  import { readSourceFingerprintCache, reusableSourceFingerprint, sourceFingerprintRecord, writeSourceFingerprintCache, } from "./source-fingerprint-cache.js";
11
+ import { canonicalRelativePath, canonicalSourcePath, compareCodePoints, isCaseInsensitiveFileSystem, resolveExistingPath, sourcePathKey, unicodeGlobVariants, } from "./source-paths.js";
10
12
  const GENERATED_SOURCE_READMES = new Set([
11
13
  `${DEFAULT_CONFIG.rawDir}/README.md`,
12
14
  `${LEGACY_PRIVATE_DIR}/README.md`,
@@ -87,6 +89,72 @@ export const DEFAULT_FAST_GLOB_IGNORES = [
87
89
  `**/${LEGACY_KB_DIR}/**`,
88
90
  `**/${RAGMIR_DIR}/**`,
89
91
  ];
92
+ /**
93
+ * Tool state, dependency caches, lockfiles, and generated bundles that drown out project knowledge.
94
+ * They are skipped unless `defaultExcludes` is false or a source entry names the directory itself.
95
+ */
96
+ export const DEFAULT_EXCLUDED_DIRECTORIES = [
97
+ ".agents",
98
+ ".angular",
99
+ ".cache",
100
+ ".claude",
101
+ ".cline",
102
+ ".codex",
103
+ ".cursor",
104
+ ".dart_tool",
105
+ ".expo",
106
+ ".gitnexus",
107
+ ".gradle",
108
+ ".idea",
109
+ ".kimi",
110
+ ".mypy_cache",
111
+ ".next",
112
+ ".nuxt",
113
+ ".nyc_output",
114
+ ".opencode",
115
+ ".parcel-cache",
116
+ ".pnpm-store",
117
+ ".pytest_cache",
118
+ ".ruff_cache",
119
+ ".svelte-kit",
120
+ ".terraform",
121
+ ".tox",
122
+ ".turbo",
123
+ ".venv",
124
+ ".windsurf",
125
+ ".yarn",
126
+ "__pycache__",
127
+ "bower_components",
128
+ "coverage",
129
+ "DerivedData",
130
+ "Pods",
131
+ ];
132
+ export const DEFAULT_EXCLUDED_FILES = [
133
+ "*.map",
134
+ "*.min.css",
135
+ "*.min.js",
136
+ "*.min.mjs",
137
+ "bun.lock",
138
+ "bun.lockb",
139
+ "Cargo.lock",
140
+ "composer.lock",
141
+ "deno.lock",
142
+ "flake.lock",
143
+ "Gemfile.lock",
144
+ "go.sum",
145
+ "gradle.lockfile",
146
+ "mix.lock",
147
+ "npm-shrinkwrap.json",
148
+ "package-lock.json",
149
+ "packages.lock.json",
150
+ "Pipfile.lock",
151
+ "pnpm-lock.yaml",
152
+ "Podfile.lock",
153
+ "poetry.lock",
154
+ "pubspec.lock",
155
+ "uv.lock",
156
+ "yarn.lock",
157
+ ];
90
158
  const GLOB_PATTERN_CHARS = /[*?[{]/u;
91
159
  export const DEFAULT_SUPPORTED_EXTENSIONS = new Set([
92
160
  ".atom",
@@ -173,44 +241,191 @@ export async function listSourceFiles(config, options = {}) {
173
241
  export async function inventorySourceFiles(config, options = {}) {
174
242
  const signal = operationSignal(options);
175
243
  throwIfAborted(signal);
176
- const inputs = await sourceInputs(config, signal);
244
+ const discovery = await discoverSourceCandidates(config, signal);
245
+ const candidates = new Map(discovery.candidates.map((candidate) => [candidate.absolutePath, candidate]));
177
246
  throwIfAborted(signal);
178
- const excludedPaths = new Set(inputs.ignorePatterns.length === 0
179
- ? []
180
- : await fg(inputs.ignorePatterns, {
181
- cwd: config.projectRoot,
182
- absolute: true,
183
- onlyFiles: true,
184
- dot: true,
185
- followSymbolicLinks: false,
186
- unique: true,
187
- }));
247
+ const cachedFingerprints = config.sourceFingerprintMode === "strict"
248
+ ? new Map()
249
+ : await readSourceFingerprintCache(config);
250
+ const usableCache = cachedFingerprints ?? new Map();
251
+ const nextFingerprints = new Map();
252
+ const files = new Map();
253
+ const verifiedAt = new Date().toISOString();
254
+ let contentBytesRead = 0;
255
+ let hashedFiles = 0;
256
+ let reusedFingerprints = 0;
257
+ await mapLimit([...candidates.values()], config.ingestConcurrency, signal, async (candidate) => {
258
+ const identity = sourceFileIdentity(candidate.absolutePath, candidate.info);
259
+ const reusable = reusableSourceFingerprint(usableCache.get(candidate.absolutePath), identity, config.sourceFingerprintMode);
260
+ let checksum;
261
+ if (reusable) {
262
+ checksum = reusable.checksum;
263
+ reusedFingerprints += 1;
264
+ nextFingerprints.set(candidate.absolutePath, reusable);
265
+ }
266
+ else {
267
+ const hashed = await checksumFile(candidate.absolutePath, signal);
268
+ const currentIdentity = sourceFileIdentity(candidate.absolutePath, sourceEntryStats(await stat(candidate.absolutePath)));
269
+ if (!sameSourceFileIdentity(identity, currentIdentity)) {
270
+ throw new Error(`Source file changed while it was being hashed: ${candidate.relativePath}. Retry ingestion.`);
271
+ }
272
+ checksum = hashed.checksum;
273
+ hashedFiles += 1;
274
+ contentBytesRead += hashed.bytesRead;
275
+ nextFingerprints.set(candidate.absolutePath, sourceFingerprintRecord(identity, checksum, verifiedAt));
276
+ }
277
+ files.set(candidate.absolutePath, {
278
+ absolutePath: candidate.absolutePath,
279
+ relativePath: candidate.relativePath,
280
+ source: candidate.source,
281
+ extension: candidate.extension,
282
+ bytes: candidate.info.size,
283
+ mtimeMs: candidate.info.mtimeMs,
284
+ checksum,
285
+ });
286
+ });
287
+ if (options.writeFingerprintCache !== false) {
288
+ await writeSourceFingerprintCache([...nextFingerprints.values()].sort((left, right) => compareCodePoints(left.absolutePath, right.absolutePath)), config);
289
+ }
188
290
  throwIfAborted(signal);
291
+ return {
292
+ discoveredFiles: discovery.discoveredFiles,
293
+ supportedFiles: [...files.values()].sort((a, b) => compareCodePoints(a.relativePath, b.relativePath)),
294
+ skippedFiles: discovery.skippedFiles,
295
+ contentBytesRead,
296
+ hashedFiles,
297
+ reusedFingerprints,
298
+ };
299
+ }
300
+ /**
301
+ * Discover indexable source files with their size and modification time, without reading content.
302
+ * Freshness checks compare this O(files) view with the index manifest.
303
+ */
304
+ export async function discoverSourceFiles(config, options = {}) {
305
+ const signal = operationSignal(options);
306
+ const discovery = await discoverSourceCandidates(config, signal);
307
+ return {
308
+ files: discovery.candidates
309
+ .map((candidate) => ({
310
+ absolutePath: candidate.absolutePath,
311
+ relativePath: candidate.relativePath,
312
+ bytes: candidate.info.size,
313
+ mtimeMs: candidate.info.mtimeMs,
314
+ identity: sourceFileIdentity(candidate.absolutePath, candidate.info),
315
+ }))
316
+ .sort((left, right) => compareCodePoints(left.relativePath, right.relativePath)),
317
+ skippedFiles: discovery.skippedFiles,
318
+ discoveredFiles: discovery.discoveredFiles,
319
+ };
320
+ }
321
+ /** Explain source rules that silently select nothing or leave excluded content reachable. */
322
+ export async function sourceRuleDiagnostics(config, supportedFiles, options = {}) {
323
+ const signal = operationSignal(options);
324
+ const inputs = await sourceInputs(config, signal);
325
+ const caseInsensitive = isCaseInsensitiveFileSystem(config.projectRoot);
326
+ const supportedPaths = supportedFiles.map((file) => file.relativePath);
327
+ const unmatchedPatterns = inputs.patterns.filter((pattern) => {
328
+ const matches = picomatch(pattern, { dot: true, nocase: caseInsensitive });
329
+ return !supportedPaths.some((relativePath) => matches(relativePath));
330
+ });
331
+ const indexedByChecksum = new Map();
332
+ for (const file of supportedFiles) {
333
+ if (!indexedByChecksum.has(file.checksum)) {
334
+ indexedByChecksum.set(file.checksum, file.relativePath);
335
+ }
336
+ }
337
+ const unmatchedExclusions = [];
338
+ const excludedCopies = [];
339
+ for (const pattern of inputs.ignorePatterns) {
340
+ throwIfAborted(signal);
341
+ if (GLOB_PATTERN_CHARS.test(pattern)) {
342
+ continue;
343
+ }
344
+ const resolved = resolveExistingPath(path.resolve(config.projectRoot, pattern), caseInsensitive);
345
+ if (resolved === null) {
346
+ unmatchedExclusions.push(pattern);
347
+ continue;
348
+ }
349
+ const info = await stat(resolved);
350
+ if (!info.isFile() || info.size > config.maxFileBytes) {
351
+ continue;
352
+ }
353
+ const { checksum } = await checksumFile(resolved, signal);
354
+ const indexedCopy = indexedByChecksum.get(checksum);
355
+ if (indexedCopy !== undefined) {
356
+ excludedCopies.push({ excluded: pattern, indexedCopy });
357
+ }
358
+ }
359
+ const portabilityWarnings = config.sources.flatMap((entry) => {
360
+ const value = entry.trim().replace(/^!/u, "").trim();
361
+ if (!value || value.startsWith("#")) {
362
+ return [];
363
+ }
364
+ if (value.includes("\\")) {
365
+ return [
366
+ `Source entry "${entry}" uses backslashes; write forward slashes so the config works on macOS, Linux, and Windows.`,
367
+ ];
368
+ }
369
+ if (path.isAbsolute(value)) {
370
+ return [
371
+ `Source entry "${entry}" is absolute; a project-relative path keeps the config portable across machines.`,
372
+ ];
373
+ }
374
+ return [];
375
+ });
376
+ return {
377
+ unmatchedSources: inputs.unresolvedRoots,
378
+ unmatchedPatterns,
379
+ unmatchedExclusions,
380
+ excludedCopies,
381
+ portabilityWarnings,
382
+ };
383
+ }
384
+ async function discoverSourceCandidates(config, signal) {
385
+ const inputs = await sourceInputs(config, signal);
386
+ throwIfAborted(signal);
387
+ const caseInsensitive = isCaseInsensitiveFileSystem(config.projectRoot);
388
+ const isExcluded = sourceExclusionMatcher(inputs.ignorePatterns, caseInsensitive);
189
389
  const candidates = new Map();
190
390
  const skippedFiles = new Map();
191
391
  const discoveredPaths = new Set();
192
392
  const allowedExtensions = supportedExtensions(config);
193
393
  const recordSourceCandidate = (absolutePath, info, source) => {
194
394
  throwIfAborted(signal);
195
- if (excludedPaths.has(absolutePath)) {
395
+ const relativePath = canonicalRelativePath(config.projectRoot, absolutePath);
396
+ if (isExcluded(relativePath) || GENERATED_SOURCE_READMES.has(relativePath)) {
196
397
  return;
197
398
  }
198
- const relativePath = path.relative(config.projectRoot, absolutePath);
199
- if (GENERATED_SOURCE_READMES.has(relativePath)) {
399
+ const key = sourcePathKey(relativePath, caseInsensitive);
400
+ const existing = candidates.get(key);
401
+ if (existing) {
402
+ if (existing.absolutePath !== absolutePath) {
403
+ // Linux and Windows can hold two names that differ only by Unicode form; both would cite
404
+ // the same canonical path, so the second one is reported instead of silently shadowed.
405
+ skippedFiles.set(absolutePath, {
406
+ relativePath,
407
+ source: canonicalSourcePath(source),
408
+ extension: path.extname(absolutePath).toLowerCase() || NO_EXTENSION,
409
+ bytes: info.size,
410
+ reason: "path-collision",
411
+ recommendation: skippedRecommendation("path-collision", ""),
412
+ });
413
+ }
200
414
  return;
201
415
  }
202
- discoveredPaths.add(absolutePath);
416
+ discoveredPaths.add(key);
203
417
  const extension = path.extname(absolutePath).toLowerCase();
204
- const skipped = skippedSourceFile(absolutePath, relativePath, source, extension, info.size);
418
+ const canonicalSource = canonicalSourcePath(source);
419
+ const skipped = skippedSourceFile(absolutePath, relativePath, canonicalSource, extension, info.size);
205
420
  if (skipped) {
206
- skippedFiles.set(absolutePath, skipped);
421
+ skippedFiles.set(key, skipped);
207
422
  return;
208
423
  }
209
424
  if (!isSupportedSourceFile(absolutePath, extension, allowedExtensions)) {
210
425
  const normalizedExtension = extension || NO_EXTENSION;
211
- skippedFiles.set(absolutePath, {
426
+ skippedFiles.set(key, {
212
427
  relativePath,
213
- source,
428
+ source: canonicalSource,
214
429
  extension: normalizedExtension,
215
430
  bytes: info.size,
216
431
  reason: "unsupported-extension",
@@ -220,9 +435,9 @@ export async function inventorySourceFiles(config, options = {}) {
220
435
  }
221
436
  if (info.size > config.maxFileBytes) {
222
437
  const normalizedExtension = extension || NO_EXTENSION;
223
- skippedFiles.set(absolutePath, {
438
+ skippedFiles.set(key, {
224
439
  relativePath,
225
- source,
440
+ source: canonicalSource,
226
441
  extension: normalizedExtension,
227
442
  bytes: info.size,
228
443
  reason: "oversized",
@@ -230,54 +445,61 @@ export async function inventorySourceFiles(config, options = {}) {
230
445
  });
231
446
  return;
232
447
  }
233
- if (!candidates.has(absolutePath)) {
234
- candidates.set(absolutePath, { absolutePath, info, source, relativePath, extension });
235
- }
448
+ candidates.set(key, {
449
+ absolutePath,
450
+ info,
451
+ source: canonicalSource,
452
+ relativePath,
453
+ extension,
454
+ });
236
455
  };
237
456
  for (const root of inputs.roots) {
238
457
  throwIfAborted(signal);
239
- if (!existsSync(root)) {
240
- continue;
241
- }
242
- const rootInfo = await stat(root);
458
+ const rootInfo = await stat(root.path);
243
459
  throwIfAborted(signal);
244
460
  const entries = rootInfo.isDirectory()
245
461
  ? (await fg("**/*", {
246
- cwd: root,
462
+ cwd: root.path,
247
463
  absolute: true,
248
464
  onlyFiles: true,
249
465
  dot: true,
250
466
  followSymbolicLinks: false,
467
+ caseSensitiveMatch: !caseInsensitive,
251
468
  ignore: [
252
469
  ...DEFAULT_FAST_GLOB_IGNORES,
253
- ...ignorePatternsForRoot(root, config.projectRoot, inputs.ignorePatterns),
470
+ ...defaultExcludeGlobs(config, [root.relativePath]),
471
+ ...unicodeGlobVariants(ignorePatternsForRoot(root.relativePath, inputs.ignorePatterns)),
254
472
  ],
255
473
  objectMode: true,
256
474
  stats: true,
257
475
  unique: true,
258
476
  }))
259
- : [{ path: root, stats: sourceEntryStats(rootInfo) }];
477
+ : [{ path: root.path, stats: sourceEntryStats(rootInfo) }];
260
478
  throwIfAborted(signal);
261
479
  for (const entry of entries) {
262
480
  throwIfAborted(signal);
263
- const absolutePath = path.isAbsolute(entry.path) ? entry.path : path.resolve(root, entry.path);
481
+ const absolutePath = path.resolve(root.path, entry.path);
264
482
  const info = entry.stats ?? sourceEntryStats(await stat(absolutePath));
265
- const relativePath = path.relative(config.projectRoot, absolutePath);
266
483
  const source = rootInfo.isDirectory()
267
- ? path.relative(root, absolutePath) || path.basename(absolutePath)
268
- : relativePath || path.basename(absolutePath);
484
+ ? path.relative(root.path, absolutePath) || path.basename(absolutePath)
485
+ : root.relativePath || path.basename(absolutePath);
269
486
  recordSourceCandidate(absolutePath, info, source);
270
487
  }
271
488
  }
272
- if (inputs.patterns.length > 0) {
489
+ for (const patterns of patternGroupsByDefaultExcludes(config, inputs.patterns)) {
273
490
  throwIfAborted(signal);
274
- const entries = (await fg(inputs.patterns, {
491
+ const entries = (await fg(unicodeGlobVariants(patterns), {
275
492
  cwd: config.projectRoot,
276
493
  absolute: true,
277
494
  onlyFiles: true,
278
495
  dot: true,
279
496
  followSymbolicLinks: false,
280
- ignore: [...DEFAULT_FAST_GLOB_IGNORES, ...inputs.ignorePatterns],
497
+ caseSensitiveMatch: !caseInsensitive,
498
+ ignore: [
499
+ ...DEFAULT_FAST_GLOB_IGNORES,
500
+ ...defaultExcludeGlobs(config, patterns),
501
+ ...unicodeGlobVariants(inputs.ignorePatterns),
502
+ ],
281
503
  objectMode: true,
282
504
  stats: true,
283
505
  unique: true,
@@ -285,67 +507,71 @@ export async function inventorySourceFiles(config, options = {}) {
285
507
  throwIfAborted(signal);
286
508
  for (const entry of entries) {
287
509
  throwIfAborted(signal);
288
- const absolutePath = path.isAbsolute(entry.path)
289
- ? entry.path
290
- : path.resolve(config.projectRoot, entry.path);
510
+ const absolutePath = path.resolve(config.projectRoot, entry.path);
291
511
  const info = entry.stats ?? sourceEntryStats(await stat(absolutePath));
292
- const relativePath = path.relative(config.projectRoot, absolutePath);
512
+ const relativePath = canonicalRelativePath(config.projectRoot, absolutePath);
293
513
  recordSourceCandidate(absolutePath, info, relativePath || path.basename(absolutePath));
294
514
  }
295
515
  }
296
- const cachedFingerprints = config.sourceFingerprintMode === "strict"
297
- ? new Map()
298
- : await readSourceFingerprintCache(config);
299
- const usableCache = cachedFingerprints ?? new Map();
300
- const nextFingerprints = new Map();
301
- const files = new Map();
302
- const verifiedAt = new Date().toISOString();
303
- let contentBytesRead = 0;
304
- let hashedFiles = 0;
305
- let reusedFingerprints = 0;
306
- await mapLimit([...candidates.values()], config.ingestConcurrency, signal, async (candidate) => {
307
- const identity = sourceFileIdentity(candidate.absolutePath, candidate.info);
308
- const reusable = reusableSourceFingerprint(usableCache.get(candidate.absolutePath), identity, config.sourceFingerprintMode);
309
- let checksum;
310
- if (reusable) {
311
- checksum = reusable.checksum;
312
- reusedFingerprints += 1;
313
- nextFingerprints.set(candidate.absolutePath, reusable);
516
+ return {
517
+ candidates: [...candidates.values()],
518
+ skippedFiles: [...skippedFiles.values()].sort((a, b) => compareCodePoints(a.relativePath, b.relativePath)),
519
+ discoveredFiles: discoveredPaths.size,
520
+ };
521
+ }
522
+ /**
523
+ * Match exclusion rules against canonical project-relative paths, independent of the on-disk
524
+ * Unicode form, and with the volume's case rules. A rule that names a directory excludes its files,
525
+ * as fast-glob ignore patterns do.
526
+ */
527
+ function sourceExclusionMatcher(patterns, caseInsensitive) {
528
+ if (patterns.length === 0) {
529
+ return () => false;
530
+ }
531
+ const matches = picomatch([...patterns], { dot: true, nocase: caseInsensitive });
532
+ return (relativePath) => {
533
+ if (matches(relativePath)) {
534
+ return true;
314
535
  }
315
- else {
316
- const hashed = await checksumFile(candidate.absolutePath, signal);
317
- const currentIdentity = sourceFileIdentity(candidate.absolutePath, sourceEntryStats(await stat(candidate.absolutePath)));
318
- if (!sameSourceFileIdentity(identity, currentIdentity)) {
319
- throw new Error(`Source file changed while it was being hashed: ${candidate.relativePath}. Retry ingestion.`);
536
+ for (let index = relativePath.lastIndexOf("/"); index > 0; index = relativePath.lastIndexOf("/", index - 1)) {
537
+ if (matches(relativePath.slice(0, index))) {
538
+ return true;
320
539
  }
321
- checksum = hashed.checksum;
322
- hashedFiles += 1;
323
- contentBytesRead += hashed.bytesRead;
324
- nextFingerprints.set(candidate.absolutePath, sourceFingerprintRecord(identity, checksum, verifiedAt));
325
540
  }
326
- files.set(candidate.absolutePath, {
327
- absolutePath: candidate.absolutePath,
328
- relativePath: candidate.relativePath,
329
- source: candidate.source,
330
- extension: candidate.extension,
331
- bytes: candidate.info.size,
332
- mtimeMs: candidate.info.mtimeMs,
333
- checksum,
334
- });
335
- });
336
- if (options.writeFingerprintCache !== false) {
337
- await writeSourceFingerprintCache([...nextFingerprints.values()].sort((left, right) => left.absolutePath.localeCompare(right.absolutePath)), config);
338
- }
339
- throwIfAborted(signal);
340
- return {
341
- discoveredFiles: discoveredPaths.size,
342
- supportedFiles: [...files.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)),
343
- skippedFiles: [...skippedFiles.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)),
344
- contentBytesRead,
345
- hashedFiles,
346
- reusedFingerprints,
541
+ return false;
347
542
  };
348
543
  }
544
+ /** Built-in exclusions, minus the ones a source entry targets explicitly. */
545
+ function defaultExcludeGlobs(config, sourceEntries) {
546
+ if (!config.defaultExcludes) {
547
+ return [];
548
+ }
549
+ const named = new Set(sourceEntries.flatMap((entry) => literalSegments(entry)));
550
+ return [
551
+ ...DEFAULT_EXCLUDED_DIRECTORIES.filter((directory) => !named.has(directory)).map((directory) => `**/${directory}/**`),
552
+ ...DEFAULT_EXCLUDED_FILES.filter((file) => !named.has(file)).map((file) => `**/${file}`),
553
+ ];
554
+ }
555
+ /** Split include patterns so that one naming `.claude/` does not re-enable it for every pattern. */
556
+ function patternGroupsByDefaultExcludes(config, patterns) {
557
+ const groups = new Map();
558
+ for (const pattern of patterns) {
559
+ const key = defaultExcludeGlobs(config, [pattern]).join("\0");
560
+ const group = groups.get(key);
561
+ if (group) {
562
+ group.push(pattern);
563
+ }
564
+ else {
565
+ groups.set(key, [pattern]);
566
+ }
567
+ }
568
+ return [...groups.values()];
569
+ }
570
+ function literalSegments(entry) {
571
+ return entry
572
+ .split("/")
573
+ .filter((segment) => segment.length > 0 && !GLOB_PATTERN_CHARS.test(segment));
574
+ }
349
575
  async function checksumFile(filePath, signal) {
350
576
  const hash = createHash("sha256");
351
577
  let bytesRead = 0;
@@ -420,10 +646,9 @@ function sameSourceFileIdentity(left, right) {
420
646
  left.ino === right.ino &&
421
647
  left.mode === right.mode);
422
648
  }
423
- function ignorePatternsForRoot(root, projectRoot, patterns) {
424
- const relativeRoot = path.relative(projectRoot, root).replaceAll(path.sep, "/");
649
+ function ignorePatternsForRoot(relativeRoot, patterns) {
425
650
  if (!relativeRoot) {
426
- return patterns;
651
+ return [...patterns];
427
652
  }
428
653
  const prefix = `${relativeRoot}/`;
429
654
  return patterns.flatMap((pattern) => {
@@ -447,7 +672,8 @@ export function summarizeUnsupportedExtensions(skippedFiles) {
447
672
  }
448
673
  async function sourceInputs(config, signal) {
449
674
  throwIfAborted(signal);
450
- const roots = [config.rawDir];
675
+ const caseInsensitive = isCaseInsensitiveFileSystem(config.projectRoot);
676
+ const requestedRoots = [config.rawDir];
451
677
  const patterns = [];
452
678
  const ignorePatterns = [];
453
679
  const classifyEntry = (entry) => {
@@ -463,7 +689,7 @@ async function sourceInputs(config, signal) {
463
689
  patterns.push(sourcePattern(config.projectRoot, trimmed));
464
690
  return;
465
691
  }
466
- roots.push(path.isAbsolute(trimmed) ? trimmed : path.resolve(config.projectRoot, trimmed));
692
+ requestedRoots.push(path.isAbsolute(trimmed) ? trimmed : path.resolve(config.projectRoot, trimmed));
467
693
  };
468
694
  // Inline `sources` from config.json are the primary mechanism; the legacy
469
695
  // sources.txt file is still read when present so existing projects keep working.
@@ -485,14 +711,29 @@ async function sourceInputs(config, signal) {
485
711
  classifyEntry(line);
486
712
  }
487
713
  }
714
+ const roots = [];
715
+ const unresolvedRoots = [];
716
+ for (const requested of requestedRoots) {
717
+ // Linux and Windows compare names byte for byte, so a configured NFC folder name must still
718
+ // reach an NFD folder copied from macOS or an archive.
719
+ const resolved = resolveExistingPath(requested, caseInsensitive);
720
+ if (resolved === null) {
721
+ if (requested !== config.rawDir) {
722
+ unresolvedRoots.push(canonicalRelativePath(config.projectRoot, requested));
723
+ }
724
+ continue;
725
+ }
726
+ roots.push({
727
+ path: resolved,
728
+ relativePath: canonicalRelativePath(config.projectRoot, resolved),
729
+ });
730
+ }
488
731
  throwIfAborted(signal);
489
- return { roots, patterns, ignorePatterns };
732
+ return { roots, patterns, ignorePatterns, unresolvedRoots };
490
733
  }
491
734
  function sourcePattern(projectRoot, input) {
492
- if (path.isAbsolute(input)) {
493
- return path.relative(projectRoot, input).replaceAll(path.sep, "/");
494
- }
495
- return input.replaceAll(path.sep, "/");
735
+ const pattern = path.isAbsolute(input) ? path.relative(projectRoot, input) : input;
736
+ return canonicalSourcePath(pattern).replace(/^(?:\.\/)+/u, "");
496
737
  }
497
738
  export function isSensitiveFilePath(absolutePath) {
498
739
  const baseName = path.basename(absolutePath).toLowerCase();
@@ -521,6 +762,9 @@ function skippedSourceFile(absolutePath, relativePath, source, extension, bytes)
521
762
  };
522
763
  }
523
764
  function skippedRecommendation(reason, extension) {
765
+ if (reason === "path-collision") {
766
+ return "Rename one of the files: their names differ only by Unicode normalization, so they would share one citation path.";
767
+ }
524
768
  if (reason === "sensitive-name") {
525
769
  return "Review manually; secret-like files are skipped to avoid indexing credentials or private keys.";
526
770
  }