@ninjaxtools/slopdex 0.3.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  // src/code-index.ts
2
- import { lstat, readFile } from "node:fs/promises";
2
+ import { lstat as lstat2, readFile, readdir } from "node:fs/promises";
3
3
  import path6 from "node:path";
4
4
 
5
5
  // src/errors.ts
@@ -15,6 +15,12 @@ var GitDivergenceError = class extends CodeIndexError {
15
15
  this.name = "GitDivergenceError";
16
16
  }
17
17
  };
18
+ var GitUnavailableError = class extends CodeIndexError {
19
+ constructor(message, options) {
20
+ super(message, options);
21
+ this.name = "GitUnavailableError";
22
+ }
23
+ };
18
24
  var IncompatibleIndexError = class extends CodeIndexError {
19
25
  constructor(message) {
20
26
  super(message);
@@ -24,6 +30,7 @@ var IncompatibleIndexError = class extends CodeIndexError {
24
30
 
25
31
  // src/git/repository.ts
26
32
  import { spawn } from "node:child_process";
33
+ import { lstat } from "node:fs/promises";
27
34
  import path from "node:path";
28
35
  var GitRepository = class {
29
36
  constructor(rootDir) {
@@ -37,7 +44,13 @@ var GitRepository = class {
37
44
  const stderr = [];
38
45
  child.stdout.on("data", (value) => stdout.push(value));
39
46
  child.stderr.on("data", (value) => stderr.push(value));
40
- child.on("error", reject);
47
+ child.on("error", (error) => {
48
+ if (error.code === "ENOENT") {
49
+ reject(new GitUnavailableError("Git executable is not available.", { cause: error }));
50
+ return;
51
+ }
52
+ reject(error);
53
+ });
41
54
  child.on("close", (code) => {
42
55
  const exitCode = code ?? -1;
43
56
  if (exitCode === 0 || allowExitOne && exitCode === 1) {
@@ -49,29 +62,51 @@ var GitRepository = class {
49
62
  });
50
63
  }
51
64
  async assertRepository() {
52
- const result = await this.#run(["rev-parse", "--is-inside-work-tree"]);
65
+ let result;
66
+ try {
67
+ result = await this.#run(["rev-parse", "--is-inside-work-tree"]);
68
+ } catch (error) {
69
+ if (error instanceof GitUnavailableError) throw error;
70
+ try {
71
+ await lstat(path.join(this.rootDir, ".git"));
72
+ } catch (markerError) {
73
+ if (markerError.code === "ENOENT") {
74
+ throw new GitUnavailableError(`${this.rootDir} is not a Git worktree.`, { cause: error });
75
+ }
76
+ throw markerError;
77
+ }
78
+ throw error;
79
+ }
53
80
  if (result.stdout.toString("utf8").trim() !== "true") {
54
- throw new CodeIndexError(`${this.rootDir} is not a Git worktree.`);
81
+ throw new GitUnavailableError(`${this.rootDir} is not a Git worktree.`);
55
82
  }
56
83
  const topLevel = (await this.#run(["rev-parse", "--show-toplevel"])).stdout.toString("utf8").trim();
57
84
  if (path.resolve(topLevel) !== path.resolve(this.rootDir)) {
58
85
  throw new CodeIndexError(`rootDir must be the Git worktree root: ${topLevel}`);
59
86
  }
60
87
  }
61
- async assertCleanWorkingTree(excludePaths = []) {
88
+ async workTreeChanges(base, excludePaths = []) {
62
89
  const pathspecs = [".", ...excludePaths.map((filePath) => `:(top,literal,exclude)${filePath}`)];
63
90
  const result = await this.#run([
64
- "status",
65
- "--porcelain=v1",
91
+ "diff",
92
+ "--name-status",
66
93
  "-z",
67
- "--untracked-files=all",
68
- "--ignore-submodules=none",
94
+ "--find-renames",
95
+ base,
69
96
  "--",
70
97
  ...pathspecs
71
98
  ]);
72
- if (result.stdout.length > 0) {
73
- throw new CodeIndexError("Cannot update from Git: the working tree has uncommitted changes. Commit or stash them first.");
74
- }
99
+ const changes = parseChanges(splitNull(result.stdout));
100
+ const untracked = await this.#run([
101
+ "ls-files",
102
+ "--others",
103
+ "--exclude-standard",
104
+ "-z",
105
+ "--",
106
+ ...pathspecs
107
+ ]);
108
+ for (const filePath of splitNull(untracked.stdout)) changes.push({ status: "A", path: filePath });
109
+ return changes;
75
110
  }
76
111
  async resolveCommit(ref) {
77
112
  const result = await this.#run(["rev-parse", "--verify", "--end-of-options", `${ref}^{commit}`]);
@@ -103,26 +138,28 @@ var GitRepository = class {
103
138
  }
104
139
  async diff(base, target) {
105
140
  const result = await this.#run(["diff", "--name-status", "-z", "--find-renames", `${base}..${target}`, "--"]);
106
- const values = splitNull(result.stdout);
107
- const changes = [];
108
- for (let index = 0; index < values.length; ) {
109
- const rawStatus = values[index++];
110
- if (!rawStatus) continue;
111
- const status = rawStatus[0];
112
- if (status === "R" || status === "C") {
113
- const oldPath = values[index++];
114
- const filePath = values[index++];
115
- if (oldPath && filePath) changes.push({ status, oldPath, path: filePath });
116
- } else if (status === "A" || status === "M" || status === "D" || status === "T") {
117
- const filePath = values[index++];
118
- if (filePath) changes.push({ status, path: filePath });
119
- } else {
120
- throw new CodeIndexError(`Unsupported Git diff status: ${rawStatus}`);
121
- }
122
- }
123
- return changes;
141
+ return parseChanges(splitNull(result.stdout));
124
142
  }
125
143
  };
144
+ function parseChanges(values) {
145
+ const changes = [];
146
+ for (let index = 0; index < values.length; ) {
147
+ const rawStatus = values[index++];
148
+ if (!rawStatus) continue;
149
+ const status = rawStatus[0];
150
+ if (status === "R" || status === "C") {
151
+ const oldPath = values[index++];
152
+ const filePath = values[index++];
153
+ if (oldPath && filePath) changes.push({ status, oldPath, path: filePath });
154
+ } else if (status === "A" || status === "M" || status === "D" || status === "T") {
155
+ const filePath = values[index++];
156
+ if (filePath) changes.push({ status, path: filePath });
157
+ } else {
158
+ throw new CodeIndexError(`Unsupported Git diff status: ${rawStatus}`);
159
+ }
160
+ }
161
+ return changes;
162
+ }
126
163
  function splitNull(buffer) {
127
164
  const value = buffer.toString("utf8");
128
165
  const parts = value.split("\0");
@@ -446,6 +483,7 @@ function parseCallables(relativePath, content, onWarning = console.warn) {
446
483
  startColumn: node.startPosition.column + 1,
447
484
  endLine: node.endPosition.row + 1,
448
485
  endColumn: node.endPosition.column + 1,
486
+ lineCount: node.endPosition.row - node.startPosition.row + 1,
449
487
  source,
450
488
  sourceHash: sha256(source),
451
489
  embeddingInput
@@ -475,18 +513,24 @@ var SourcePolicy = class {
475
513
  includes(relativePath) {
476
514
  const normalized = relativePath.replaceAll("\\", "/");
477
515
  if (!languageForPath(normalized)) return false;
478
- if (normalized.split("/").some((segment) => DEFAULT_EXCLUDED_SEGMENTS.has(segment))) return false;
516
+ if (hasExcludedSegment(normalized)) return false;
479
517
  if (this.#exclude.some((pattern) => path4.matchesGlob(normalized, pattern))) return false;
480
518
  return this.#include.length === 0 || this.#include.some((pattern) => path4.matchesGlob(normalized, pattern));
481
519
  }
520
+ traversesDirectory(relativePath) {
521
+ return !hasExcludedSegment(relativePath.replaceAll("\\", "/"));
522
+ }
482
523
  };
524
+ function hasExcludedSegment(relativePath) {
525
+ return relativePath.split("/").some((segment) => DEFAULT_EXCLUDED_SEGMENTS.has(segment));
526
+ }
483
527
 
484
528
  // src/storage/database.ts
485
529
  import { mkdirSync } from "node:fs";
486
530
  import path5 from "node:path";
487
531
  import { DatabaseSync } from "node:sqlite";
488
532
  import * as sqliteVec from "sqlite-vec";
489
- var SCHEMA_VERSION = "1";
533
+ var SCHEMA_VERSION = "2";
490
534
  var IndexDatabase = class {
491
535
  #db;
492
536
  #rootDir;
@@ -498,15 +542,30 @@ var IndexDatabase = class {
498
542
  this.#rootDir = rootDir;
499
543
  this.#profile = profile;
500
544
  this.#db = new DatabaseSync(indexPath, { allowExtension: true, readOnly });
501
- sqliteVec.load(this.#db);
502
- this.#db.enableLoadExtension(false);
503
- if (readOnly) {
504
- this.#db.exec("PRAGMA foreign_keys=ON");
505
- } else {
506
- this.#db.exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; PRAGMA foreign_keys=ON;");
507
- this.#migrate();
545
+ try {
546
+ sqliteVec.load(this.#db);
547
+ this.#db.enableLoadExtension(false);
548
+ const regexes = /* @__PURE__ */ new Map();
549
+ this.#db.function("slopdex_regexp", { deterministic: true }, (pattern, value) => {
550
+ if (typeof pattern !== "string" || typeof value !== "string") return 0;
551
+ let regex = regexes.get(pattern);
552
+ if (!regex) {
553
+ regex = new RegExp(pattern);
554
+ regexes.set(pattern, regex);
555
+ }
556
+ return regex.test(value) ? 1 : 0;
557
+ });
558
+ if (readOnly) {
559
+ this.#db.exec("PRAGMA foreign_keys=ON");
560
+ } else {
561
+ this.#db.exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; PRAGMA foreign_keys=ON;");
562
+ this.#migrate();
563
+ }
564
+ this.#validateMetadata(readOnly);
565
+ } catch (error) {
566
+ this.#db.close();
567
+ throw error;
508
568
  }
509
- this.#validateMetadata(readOnly);
510
569
  }
511
570
  close() {
512
571
  this.#db.close();
@@ -545,6 +604,7 @@ var IndexDatabase = class {
545
604
  start_column INTEGER NOT NULL,
546
605
  end_line INTEGER NOT NULL,
547
606
  end_column INTEGER NOT NULL,
607
+ line_count INTEGER NOT NULL,
548
608
  source TEXT NOT NULL,
549
609
  source_hash TEXT NOT NULL,
550
610
  embedding_input TEXT NOT NULL,
@@ -553,13 +613,31 @@ var IndexDatabase = class {
553
613
  source_mode TEXT NOT NULL CHECK(source_mode IN ('git', 'working-tree')),
554
614
  embedding_id INTEGER NOT NULL REFERENCES embeddings(id)
555
615
  );
616
+ CREATE TABLE IF NOT EXISTS callable_provenance (
617
+ identity_key TEXT NOT NULL,
618
+ source_hash TEXT NOT NULL,
619
+ first_seen_commit TEXT NOT NULL,
620
+ PRIMARY KEY(identity_key, source_hash)
621
+ );
556
622
  CREATE INDEX IF NOT EXISTS functions_path ON functions(path);
557
623
  CREATE INDEX IF NOT EXISTS functions_embedding ON functions(embedding_id);
624
+ INSERT OR IGNORE INTO callable_provenance(identity_key, source_hash, first_seen_commit)
625
+ SELECT identity_key, source_hash, first_seen_commit FROM functions WHERE first_seen_commit IS NOT NULL;
558
626
  `);
559
627
  const fileColumns = this.#db.prepare("PRAGMA table_info(files)").all();
560
628
  if (!fileColumns.some((column) => column.name === "previous_path")) {
561
629
  this.#db.exec("ALTER TABLE files ADD COLUMN previous_path TEXT");
562
630
  }
631
+ const functionColumns = this.#db.prepare("PRAGMA table_info(functions)").all();
632
+ const needsLineCount = !functionColumns.some((column) => column.name === "line_count");
633
+ const schemaVersion = this.#metadata("schema_version");
634
+ if (needsLineCount || schemaVersion === "1") {
635
+ this.#transaction(() => {
636
+ if (needsLineCount) this.#db.exec("ALTER TABLE functions ADD COLUMN line_count INTEGER NOT NULL DEFAULT 1");
637
+ this.#db.exec("UPDATE functions SET line_count = end_line - start_line + 1");
638
+ if (schemaVersion === "1") this.#setMetadata("schema_version", SCHEMA_VERSION);
639
+ });
640
+ }
563
641
  }
564
642
  #metadata(key) {
565
643
  const row = this.#db.prepare("SELECT value FROM metadata WHERE key = ?").get(key);
@@ -571,6 +649,9 @@ var IndexDatabase = class {
571
649
  ON CONFLICT(key) DO UPDATE SET value = excluded.value
572
650
  `).run(key, value);
573
651
  }
652
+ #deleteMetadata(key) {
653
+ this.#db.prepare("DELETE FROM metadata WHERE key = ?").run(key);
654
+ }
574
655
  #validateMetadata(readOnly) {
575
656
  const schemaVersion = this.#metadata("schema_version");
576
657
  if (readOnly && !schemaVersion) throw new IncompatibleIndexError("Target file is not a slopdex index.");
@@ -619,6 +700,13 @@ var IndexDatabase = class {
619
700
  FROM files WHERE source_mode = 'working-tree' ORDER BY path
620
701
  `).all();
621
702
  }
703
+ getFileStates() {
704
+ return this.#db.prepare(`
705
+ SELECT path, content_hash AS contentHash, blob_oid AS blobOid,
706
+ source_mode AS sourceMode, previous_path AS previousPath
707
+ FROM files ORDER BY path
708
+ `).all();
709
+ }
622
710
  getEmbeddingKeys(keys) {
623
711
  const found = /* @__PURE__ */ new Set();
624
712
  const statement = this.#db.prepare("SELECT 1 FROM embeddings WHERE embedding_key = ?");
@@ -632,36 +720,44 @@ var IndexDatabase = class {
632
720
  if (options.expectedCheckpoint !== void 0 && this.getCheckpoint() !== options.expectedCheckpoint) {
633
721
  throw new CodeIndexError("Git checkpoint changed while the update was being prepared.");
634
722
  }
723
+ if (options.expectedGeneration !== void 0 && this.getGeneration() !== options.expectedGeneration) {
724
+ throw new CodeIndexError("Index changed while the update was being prepared; retry the update.");
725
+ }
635
726
  const stats = {
636
- filesUpdated: options.files.length,
727
+ filesUpdated: 0,
637
728
  filesDeleted: 0,
638
729
  functionsAdded: 0,
639
730
  functionsUpdated: 0,
640
731
  functionsDeleted: 0,
641
732
  embeddingsCreated: 0,
642
- checkpoint: options.checkpoint ?? this.getCheckpoint()
733
+ checkpoint: options.checkpoint !== void 0 ? options.checkpoint : this.getCheckpoint()
643
734
  };
644
735
  const deleteFile = this.#db.prepare("DELETE FROM files WHERE path = ?");
645
736
  const countFunctions = this.#db.prepare("SELECT COUNT(*) AS count FROM functions WHERE path = ?");
646
- const uniqueDeletes = new Set(options.deletePaths);
647
- for (const file of options.files) {
648
- uniqueDeletes.delete(file.path);
649
- if (file.previousPath) uniqueDeletes.delete(file.previousPath);
650
- if (file.replacePath) uniqueDeletes.delete(file.replacePath);
651
- }
652
- for (const filePath of uniqueDeletes) {
653
- const count = Number(countFunctions.get(filePath).count);
654
- const result = deleteFile.run(filePath);
655
- if (result.changes > 0) {
656
- stats.filesDeleted += 1;
657
- stats.functionsDeleted += count;
737
+ const applyStage = (files, deletePaths) => {
738
+ stats.filesUpdated += files.length;
739
+ const uniqueDeletes = new Set(deletePaths);
740
+ for (const file of files) {
741
+ uniqueDeletes.delete(file.path);
742
+ if (file.previousPath) uniqueDeletes.delete(file.previousPath);
743
+ if (file.replacePath) uniqueDeletes.delete(file.replacePath);
658
744
  }
659
- }
660
- for (const file of options.files) this.#replaceFile(file, stats);
661
- this.#db.exec("DELETE FROM embeddings WHERE id NOT IN (SELECT DISTINCT embedding_id FROM functions)");
745
+ for (const filePath of uniqueDeletes) {
746
+ const count = Number(countFunctions.get(filePath).count);
747
+ const result = deleteFile.run(filePath);
748
+ if (result.changes > 0) {
749
+ stats.filesDeleted += 1;
750
+ stats.functionsDeleted += count;
751
+ }
752
+ }
753
+ for (const file of files) this.#replaceFile(file, stats);
754
+ };
755
+ applyStage(options.files, options.deletePaths);
756
+ if (options.workingTree) applyStage(options.workingTree.files, options.workingTree.deletePaths);
662
757
  const generation = this.getGeneration() + 1;
663
758
  this.#setMetadata("generation", String(generation));
664
- if (options.checkpoint) this.#setMetadata("git_checkpoint", options.checkpoint);
759
+ if (options.checkpoint === null) this.#deleteMetadata("git_checkpoint");
760
+ else if (options.checkpoint !== void 0) this.#setMetadata("git_checkpoint", options.checkpoint);
665
761
  return stats;
666
762
  });
667
763
  }
@@ -671,6 +767,12 @@ var IndexDatabase = class {
671
767
  const existingFile = this.#db.prepare("SELECT previous_path FROM files WHERE path = ?").get(sourcePath);
672
768
  const oldMatches = reconcileFunctions(file.callables, oldRows);
673
769
  if (sourcePath !== file.path) {
770
+ const displacedRows = this.#rowsForPath(file.path);
771
+ const displaced = this.#db.prepare("DELETE FROM files WHERE path = ?").run(file.path);
772
+ if (displaced.changes > 0) {
773
+ stats.filesDeleted += 1;
774
+ stats.functionsDeleted += displacedRows.length;
775
+ }
674
776
  this.#db.prepare("DELETE FROM files WHERE path = ?").run(sourcePath);
675
777
  } else {
676
778
  this.#db.prepare("DELETE FROM files WHERE path = ?").run(file.path);
@@ -702,13 +804,16 @@ var IndexDatabase = class {
702
804
  if (!embedding) throw new CodeIndexError(`Missing embedding for ${callable.qualifiedName}.`);
703
805
  const old = oldMatches.get(callable);
704
806
  if (old) usedIds.add(old.id);
705
- const firstSeenCommit = old?.first_seen_commit ?? file.indexedCommit;
807
+ const remembered = this.#db.prepare(`
808
+ SELECT first_seen_commit FROM callable_provenance WHERE identity_key = ? AND source_hash = ?
809
+ `).get(callable.identityKey, callable.sourceHash);
810
+ const firstSeenCommit = old?.first_seen_commit ?? remembered?.first_seen_commit ?? file.indexedCommit;
706
811
  this.#db.prepare(`
707
812
  INSERT INTO functions(
708
813
  id, path, language, kind, name, qualified_name, signature, identity_key,
709
- start_line, start_column, end_line, end_column, source, source_hash,
814
+ start_line, start_column, end_line, end_column, line_count, source, source_hash,
710
815
  embedding_input, first_seen_commit, last_seen_commit, source_mode, embedding_id
711
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
816
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
712
817
  `).run(
713
818
  old?.id ?? null,
714
819
  file.path,
@@ -722,6 +827,7 @@ var IndexDatabase = class {
722
827
  callable.startColumn,
723
828
  callable.endLine,
724
829
  callable.endColumn,
830
+ callable.lineCount,
725
831
  callable.source,
726
832
  callable.sourceHash,
727
833
  callable.embeddingInput,
@@ -732,6 +838,11 @@ var IndexDatabase = class {
732
838
  );
733
839
  if (old) stats.functionsUpdated += 1;
734
840
  else stats.functionsAdded += 1;
841
+ if (firstSeenCommit) {
842
+ this.#db.prepare(`
843
+ INSERT OR IGNORE INTO callable_provenance(identity_key, source_hash, first_seen_commit) VALUES (?, ?, ?)
844
+ `).run(callable.identityKey, callable.sourceHash, firstSeenCommit);
845
+ }
735
846
  }
736
847
  stats.functionsDeleted += oldRows.length - usedIds.size;
737
848
  }
@@ -754,19 +865,28 @@ var IndexDatabase = class {
754
865
  return results;
755
866
  }
756
867
  searchVector(vector, options) {
868
+ const excludePaths = options.excludePaths ?? [];
869
+ const pathFilter = excludePaths.length > 0 ? `AND f.path NOT IN (${excludePaths.map(() => "?").join(", ")})` : "";
757
870
  const rows = this.#db.prepare(`
758
871
  SELECT f.*, 1.0 - vec_distance_cosine(e.vector, ?) AS similarity
759
872
  FROM functions f
760
873
  JOIN embeddings e ON e.id = f.embedding_id
761
874
  WHERE (? IS NULL OR f.id != ?)
875
+ ${pathFilter}
876
+ AND f.line_count >= ?
877
+ AND (? IS NULL OR slopdex_regexp(?, f.qualified_name))
762
878
  AND (1.0 - vec_distance_cosine(e.vector, ?)) >= ?
763
- AND (? IS NULL OR (1.0 - vec_distance_cosine(e.vector, ?)) <= ?)
879
+ AND (? IS NULL OR (1.0 - vec_distance_cosine(e.vector, ?)) < ?)
764
880
  ORDER BY similarity DESC, f.id ASC
765
881
  LIMIT ?
766
882
  `).all(
767
883
  vectorBuffer(vector),
768
884
  options.excludeId ?? null,
769
885
  options.excludeId ?? null,
886
+ ...excludePaths,
887
+ options.minLines ?? 1,
888
+ options.nameRegex ?? null,
889
+ options.nameRegex ?? null,
770
890
  vectorBuffer(vector),
771
891
  options.minSimilarity,
772
892
  options.maxSimilarity ?? null,
@@ -815,6 +935,7 @@ function toIndexedFunction(row) {
815
935
  startColumn: row.start_column,
816
936
  endLine: row.end_line,
817
937
  endColumn: row.end_column,
938
+ lineCount: row.line_count,
818
939
  source: row.source,
819
940
  sourceHash: row.source_hash,
820
941
  embeddingInput: row.embedding_input,
@@ -909,7 +1030,7 @@ var CodeIndex = class {
909
1030
  throwIfAborted(options.signal);
910
1031
  if (!this.#policy.includes(relativePath)) throw new CodeIndexError(`Unsupported or excluded source file: ${relativePath}`);
911
1032
  const absolutePath = path6.join(this.rootDir, relativePath);
912
- const info = await lstat(absolutePath);
1033
+ const info = await lstat2(absolutePath);
913
1034
  if (info.isSymbolicLink()) throw new CodeIndexError(`Symbolic links are not supported: ${relativePath}`);
914
1035
  if (!info.isFile()) throw new CodeIndexError(`Not a regular file: ${relativePath}`);
915
1036
  if (info.size > this.#maxFileSize) throw new CodeIndexError(`File exceeds maxFileSize: ${relativePath}`);
@@ -930,50 +1051,210 @@ var CodeIndex = class {
930
1051
  deletePaths: [...deletePaths, ...renameMap.values()]
931
1052
  });
932
1053
  }
1054
+ async updateFromWorkingTree(options = {}) {
1055
+ throwIfAborted(options.signal);
1056
+ const generation = this.#database.getGeneration();
1057
+ const indexedFiles = this.#database.getFileStates();
1058
+ const sourcePaths = await this.#workingTreeSourcePaths(options.signal);
1059
+ const prepared = [];
1060
+ const skippedPaths = /* @__PURE__ */ new Set();
1061
+ for (const relativePath of sourcePaths) {
1062
+ throwIfAborted(options.signal);
1063
+ const absolutePath = path6.join(this.rootDir, relativePath);
1064
+ const info = await lstat2(absolutePath);
1065
+ if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize) {
1066
+ skippedPaths.add(relativePath);
1067
+ continue;
1068
+ }
1069
+ const content = await readFile(absolutePath);
1070
+ prepared.push(this.#prepareFile(relativePath, content, {
1071
+ blobOid: null,
1072
+ sourceMode: "working-tree",
1073
+ indexedCommit: null
1074
+ }));
1075
+ }
1076
+ await this.#attachEmbeddings(prepared, options.signal);
1077
+ const pathsAgain = await this.#workingTreeSourcePaths(options.signal);
1078
+ if (JSON.stringify(pathsAgain) !== JSON.stringify(sourcePaths)) {
1079
+ throw new CodeIndexError("Working-tree files changed while indexing; retry the update.");
1080
+ }
1081
+ for (const file of prepared) {
1082
+ try {
1083
+ const info = await lstat2(path6.join(this.rootDir, file.path));
1084
+ const content = await readFile(path6.join(this.rootDir, file.path));
1085
+ if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize || sha256(content.toString("utf8")) !== file.contentHash) {
1086
+ throw new CodeIndexError(`Working-tree file changed while indexing: ${file.path}`);
1087
+ }
1088
+ } catch (error) {
1089
+ if (error instanceof CodeIndexError) throw error;
1090
+ throw new CodeIndexError(`Working-tree file changed while indexing: ${file.path}`, { cause: error });
1091
+ }
1092
+ }
1093
+ for (const relativePath of skippedPaths) {
1094
+ const info = await lstat2(path6.join(this.rootDir, relativePath));
1095
+ if (info.isFile() && !info.isSymbolicLink() && info.size <= this.#maxFileSize) {
1096
+ throw new CodeIndexError(`Working-tree file changed while indexing: ${relativePath}`);
1097
+ }
1098
+ }
1099
+ return this.#database.applyUpdate({
1100
+ files: prepared,
1101
+ deletePaths: indexedFiles.map((file) => file.path),
1102
+ checkpoint: null,
1103
+ expectedGeneration: generation
1104
+ });
1105
+ }
933
1106
  async updateFromGit(options = {}) {
934
1107
  throwIfAborted(options.signal);
935
1108
  await this.#git.assertRepository();
936
1109
  const relativeIndexPath = path6.relative(this.rootDir, this.indexPath).replaceAll(path6.sep, "/");
937
1110
  const indexArtifacts = relativeIndexPath && !relativeIndexPath.startsWith("../") ? [relativeIndexPath, `${relativeIndexPath}-shm`, `${relativeIndexPath}-wal`, `${relativeIndexPath}-journal`] : [];
938
- await this.#git.assertCleanWorkingTree(indexArtifacts);
939
1111
  const target = await this.#git.resolveCommit(options.target ?? "HEAD");
1112
+ const head = await this.#git.resolveCommit("HEAD");
1113
+ const overlayWorkingTree = target === head && options.includeWorkingTree !== false;
940
1114
  const checkpoint = this.#database.getCheckpoint();
941
- let rebuild = checkpoint === null;
1115
+ const generation = this.#database.getGeneration();
1116
+ let reconcileAll = checkpoint === null;
942
1117
  if (checkpoint && !await this.#git.isAncestor(checkpoint, target)) {
943
1118
  if (!options.rebuildOnDivergence) throw new GitDivergenceError(checkpoint, target);
944
- rebuild = true;
1119
+ reconcileAll = true;
945
1120
  }
946
1121
  const tree = await this.#git.listTree(target);
947
- const workingFiles = this.#database.getWorkingTreeFiles();
948
- const changes = rebuild || !checkpoint ? [] : await this.#git.diff(checkpoint, target);
1122
+ const indexedFiles = this.#database.getFileStates();
1123
+ const indexedByPath = new Map(indexedFiles.map((file) => [file.path, file]));
1124
+ const workingFiles = indexedFiles.filter((file) => file.sourceMode === "working-tree");
1125
+ const workingChanges = overlayWorkingTree ? await this.#git.workTreeChanges(target, indexArtifacts) : [];
1126
+ const visibleWorkingPaths = new Set(
1127
+ workingChanges.filter((change) => change.status !== "D").map((change) => change.path)
1128
+ );
1129
+ const changes = reconcileAll || !checkpoint ? [] : await this.#git.diff(checkpoint, target);
949
1130
  const upserts = /* @__PURE__ */ new Map();
950
1131
  const deletes = /* @__PURE__ */ new Set();
951
- if (rebuild) {
952
- for (const existing of this.#database.allFilePaths()) deletes.add(existing);
953
- for (const entry of tree.values()) {
954
- if (this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize) upserts.set(entry.path, { entry });
1132
+ const eligibleEntries = [...tree.values()].filter((entry) => this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize);
1133
+ const eligibleTargetPaths = new Set(eligibleEntries.map((entry) => entry.path));
1134
+ if (reconcileAll) {
1135
+ const renameCandidates = indexedFiles.filter((file) => file.sourceMode === "git" && file.blobOid && !eligibleTargetPaths.has(file.path));
1136
+ const claimedRenameSources = /* @__PURE__ */ new Set();
1137
+ for (const entry of eligibleEntries) {
1138
+ const indexed = indexedByPath.get(entry.path);
1139
+ if (indexed?.sourceMode === "git" && indexed.blobOid === entry.oid) continue;
1140
+ const matches = renameCandidates.filter((candidate) => candidate.blobOid === entry.oid && !claimedRenameSources.has(candidate.path));
1141
+ const previousPath = matches.length === 1 ? matches[0].path : void 0;
1142
+ if (previousPath) claimedRenameSources.add(previousPath);
1143
+ upserts.set(entry.path, { entry, ...previousPath ? { previousPath } : {} });
1144
+ }
1145
+ for (const indexed of indexedFiles) {
1146
+ if (!eligibleTargetPaths.has(indexed.path)) deletes.add(indexed.path);
955
1147
  }
956
1148
  } else {
957
1149
  this.#classifyGitChanges(changes, tree, upserts, deletes);
958
- for (const { path: dirtyPath } of workingFiles) {
959
- const entry = tree.get(dirtyPath);
960
- if (entry && this.#policy.includes(dirtyPath) && entry.size <= this.#maxFileSize) upserts.set(dirtyPath, { entry });
961
- else deletes.add(dirtyPath);
962
- }
963
- const eligibleTargetPaths = new Set(
964
- [...tree.values()].filter((entry) => this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize).map((entry) => entry.path)
965
- );
966
- const indexedPaths = new Set(this.#database.allFilePaths());
1150
+ const indexedPaths = new Set(indexedByPath.keys());
967
1151
  for (const targetPath of eligibleTargetPaths) {
968
- if (!indexedPaths.has(targetPath) && !upserts.has(targetPath)) {
969
- upserts.set(targetPath, { entry: tree.get(targetPath) });
1152
+ const indexed = indexedByPath.get(targetPath);
1153
+ const entry = tree.get(targetPath);
1154
+ if ((!indexed || indexed.sourceMode === "git" && indexed.blobOid !== entry.oid) && !upserts.has(targetPath)) {
1155
+ upserts.set(targetPath, { entry });
970
1156
  }
971
1157
  }
972
1158
  for (const indexedPath of indexedPaths) {
973
1159
  if (!eligibleTargetPaths.has(indexedPath)) deletes.add(indexedPath);
974
1160
  }
975
1161
  }
976
- if (!rebuild && checkpoint === target && upserts.size === 0 && deletes.size === 0) return emptyStats(checkpoint);
1162
+ for (const dirtyFile of workingFiles) {
1163
+ const entry = tree.get(dirtyFile.path);
1164
+ const previousEntry = dirtyFile.previousPath ? tree.get(dirtyFile.previousPath) : void 0;
1165
+ if (previousEntry && this.#policy.includes(previousEntry.path) && previousEntry.size <= this.#maxFileSize && (entry !== void 0 || !visibleWorkingPaths.has(dirtyFile.path))) {
1166
+ const currentPlan = entry ? upserts.get(dirtyFile.path) : void 0;
1167
+ upserts.delete(dirtyFile.path);
1168
+ upserts.delete(previousEntry.path);
1169
+ upserts.set(previousEntry.path, { entry: previousEntry, previousPath: dirtyFile.path });
1170
+ if (entry && this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize) {
1171
+ upserts.set(dirtyFile.path, currentPlan ?? { entry });
1172
+ }
1173
+ continue;
1174
+ }
1175
+ if (entry && this.#policy.includes(dirtyFile.path) && entry.size <= this.#maxFileSize) {
1176
+ const planned = upserts.get(dirtyFile.path);
1177
+ if (planned?.previousPath && !indexedByPath.has(planned.previousPath)) {
1178
+ upserts.set(dirtyFile.path, { entry, previousPath: dirtyFile.path });
1179
+ } else if (!planned) {
1180
+ upserts.set(dirtyFile.path, { entry });
1181
+ }
1182
+ continue;
1183
+ }
1184
+ if (previousEntry && this.#policy.includes(previousEntry.path) && previousEntry.size <= this.#maxFileSize) {
1185
+ upserts.set(previousEntry.path, { entry: previousEntry, previousPath: dirtyFile.path });
1186
+ } else {
1187
+ deletes.add(dirtyFile.path);
1188
+ }
1189
+ }
1190
+ for (const [filePath, { entry, previousPath }] of upserts) {
1191
+ const indexed = indexedByPath.get(filePath);
1192
+ if (!previousPath && indexed?.sourceMode === "git" && indexed.blobOid === entry.oid) upserts.delete(filePath);
1193
+ }
1194
+ const workingPrepared = [];
1195
+ const workingDeletes = /* @__PURE__ */ new Set();
1196
+ const skippedWorkingPaths = /* @__PURE__ */ new Set();
1197
+ for (const change of workingChanges) {
1198
+ throwIfAborted(options.signal);
1199
+ if (change.status === "D") {
1200
+ upserts.delete(change.path);
1201
+ workingDeletes.add(change.path);
1202
+ continue;
1203
+ }
1204
+ let previousPath;
1205
+ let replacePath;
1206
+ if (change.status === "R") {
1207
+ const plannedBase = upserts.get(change.oldPath);
1208
+ const existingAtTarget = indexedByPath.get(change.oldPath);
1209
+ const existingAtFinal = indexedByPath.get(change.path);
1210
+ const continuingRename = existingAtFinal?.sourceMode === "working-tree" && existingAtFinal.previousPath === change.oldPath;
1211
+ replacePath = continuingRename ? change.path : existingAtTarget ? change.oldPath : existingAtFinal ? change.path : plannedBase?.previousPath && indexedByPath.has(plannedBase.previousPath) ? plannedBase.previousPath : void 0;
1212
+ previousPath = existingAtFinal?.previousPath ?? change.oldPath;
1213
+ upserts.delete(change.oldPath);
1214
+ upserts.delete(change.path);
1215
+ if (replacePath) deletes.delete(replacePath);
1216
+ workingDeletes.add(change.oldPath);
1217
+ } else {
1218
+ const planned = upserts.get(change.path);
1219
+ const existing = indexedByPath.get(change.path);
1220
+ replacePath = existing ? change.path : planned?.previousPath && indexedByPath.has(planned.previousPath) ? planned.previousPath : void 0;
1221
+ previousPath = existing?.previousPath ?? planned?.previousPath;
1222
+ upserts.delete(change.path);
1223
+ if (replacePath) deletes.delete(replacePath);
1224
+ }
1225
+ const relativePath = change.path;
1226
+ if (!this.#policy.includes(relativePath)) {
1227
+ if (replacePath) workingDeletes.add(replacePath);
1228
+ workingDeletes.add(relativePath);
1229
+ continue;
1230
+ }
1231
+ const absolutePath = path6.join(this.rootDir, relativePath);
1232
+ const info = await lstat2(absolutePath);
1233
+ if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize) {
1234
+ if (replacePath) workingDeletes.add(replacePath);
1235
+ workingDeletes.add(relativePath);
1236
+ skippedWorkingPaths.add(relativePath);
1237
+ continue;
1238
+ }
1239
+ const content = await readFile(absolutePath);
1240
+ if (!replacePath && change.status === "A") {
1241
+ const contentHash = sha256(content.toString("utf8"));
1242
+ const renameCandidates = workingFiles.filter((file) => file.path !== relativePath && !visibleWorkingPaths.has(file.path) && deletes.has(file.path) && file.contentHash === contentHash);
1243
+ const renamed = renameCandidates.length === 1 ? renameCandidates[0] : void 0;
1244
+ if (renamed) {
1245
+ replacePath = renamed.path;
1246
+ previousPath = renamed.previousPath ?? renamed.path;
1247
+ deletes.delete(renamed.path);
1248
+ }
1249
+ }
1250
+ workingPrepared.push(this.#prepareFile(relativePath, content, {
1251
+ blobOid: null,
1252
+ sourceMode: "working-tree",
1253
+ indexedCommit: null,
1254
+ ...previousPath ? { previousPath } : {},
1255
+ ...replacePath ? { replacePath } : {}
1256
+ }));
1257
+ }
977
1258
  const prepared = [];
978
1259
  for (const { entry, previousPath } of upserts.values()) {
979
1260
  throwIfAborted(options.signal);
@@ -989,14 +1270,46 @@ var CodeIndex = class {
989
1270
  ...previousPath ? { replacePath: previousPath } : {}
990
1271
  }));
991
1272
  }
992
- await this.#attachEmbeddings(prepared, options.signal);
1273
+ await this.#attachEmbeddings([...prepared, ...workingPrepared], options.signal);
1274
+ const noChanges = !reconcileAll && checkpoint === target && prepared.length === 0 && deletes.size === 0 && workingPrepared.length === 0 && workingDeletes.size === 0;
993
1275
  const resolvedAgain = await this.#git.resolveCommit(options.target ?? "HEAD");
994
1276
  if (resolvedAgain !== target) throw new CodeIndexError("Target Git ref changed while indexing; retry the update.");
1277
+ if (overlayWorkingTree) {
1278
+ const headAgain = await this.#git.resolveCommit("HEAD");
1279
+ const workingChangesAgain = await this.#git.workTreeChanges(target, indexArtifacts);
1280
+ if (headAgain !== head || JSON.stringify(workingChangesAgain) !== JSON.stringify(workingChanges)) {
1281
+ throw new CodeIndexError("Git HEAD or working-tree changes changed while indexing; retry the update.");
1282
+ }
1283
+ for (const file of workingPrepared) {
1284
+ const info = await lstat2(path6.join(this.rootDir, file.path));
1285
+ const content = await readFile(path6.join(this.rootDir, file.path));
1286
+ if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize || sha256(content.toString("utf8")) !== file.contentHash) {
1287
+ throw new CodeIndexError(`Working-tree file changed while indexing: ${file.path}`);
1288
+ }
1289
+ }
1290
+ for (const relativePath of skippedWorkingPaths) {
1291
+ const info = await lstat2(path6.join(this.rootDir, relativePath));
1292
+ if (info.isFile() && !info.isSymbolicLink() && info.size <= this.#maxFileSize) {
1293
+ throw new CodeIndexError(`Working-tree file changed while indexing: ${relativePath}`);
1294
+ }
1295
+ }
1296
+ }
1297
+ if (noChanges) {
1298
+ if (this.#database.getCheckpoint() !== checkpoint || this.#database.getGeneration() !== generation) {
1299
+ throw new CodeIndexError("Index changed while the update was being prepared; retry the update.");
1300
+ }
1301
+ return emptyStats(checkpoint);
1302
+ }
995
1303
  return this.#database.applyUpdate({
996
1304
  files: prepared,
997
1305
  deletePaths: [...deletes],
1306
+ workingTree: {
1307
+ files: workingPrepared,
1308
+ deletePaths: [...workingDeletes]
1309
+ },
998
1310
  checkpoint: target,
999
- expectedCheckpoint: checkpoint
1311
+ expectedCheckpoint: checkpoint,
1312
+ expectedGeneration: generation
1000
1313
  });
1001
1314
  }
1002
1315
  #classifyGitChanges(changes, tree, upserts, deletes) {
@@ -1018,6 +1331,28 @@ var CodeIndex = class {
1018
1331
  else deletes.add(change.path);
1019
1332
  }
1020
1333
  }
1334
+ async #workingTreeSourcePaths(signal) {
1335
+ const relativeIndexPath = path6.relative(this.rootDir, this.indexPath).replaceAll(path6.sep, "/");
1336
+ const indexArtifacts = new Set(relativeIndexPath && !relativeIndexPath.startsWith("../") ? [relativeIndexPath, `${relativeIndexPath}-shm`, `${relativeIndexPath}-wal`, `${relativeIndexPath}-journal`] : []);
1337
+ const directories = [""];
1338
+ const sourcePaths = [];
1339
+ while (directories.length > 0) {
1340
+ throwIfAborted(signal);
1341
+ const relativeDirectory = directories.shift();
1342
+ const entries = await readdir(path6.join(this.rootDir, relativeDirectory), { withFileTypes: true });
1343
+ entries.sort((left, right) => left.name.localeCompare(right.name));
1344
+ for (const entry of entries) {
1345
+ const relativePath = relativeDirectory ? `${relativeDirectory.replaceAll(path6.sep, "/")}/${entry.name}` : entry.name;
1346
+ if (entry.isDirectory()) {
1347
+ if (this.#policy.traversesDirectory(relativePath)) directories.push(relativePath);
1348
+ } else if (entry.isFile() && !indexArtifacts.has(relativePath) && this.#policy.includes(relativePath)) {
1349
+ sourcePaths.push(relativePath);
1350
+ }
1351
+ }
1352
+ }
1353
+ sourcePaths.sort();
1354
+ return sourcePaths;
1355
+ }
1021
1356
  #prepareFile(relativePath, buffer, provenance) {
1022
1357
  const content = buffer.toString("utf8");
1023
1358
  const callables = parseCallables(relativePath, content, this.#onWarning);
@@ -1076,7 +1411,10 @@ var CodeIndex = class {
1076
1411
  limit: options.limit,
1077
1412
  minSimilarity: options.minSimilarity,
1078
1413
  ...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
1079
- excludeId: functionId
1414
+ excludeId: functionId,
1415
+ ...options.excludePaths !== void 0 ? { excludePaths: options.excludePaths } : {},
1416
+ ...options.minLines !== void 0 ? { minLines: options.minLines } : {},
1417
+ ...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {}
1080
1418
  });
1081
1419
  }
1082
1420
  vectorForFunction(functionId) {
@@ -1086,15 +1424,15 @@ var CodeIndex = class {
1086
1424
  return this.#database.searchVector(normalizeEmbeddingVector(vector, this.provider.profile.dimensions), options);
1087
1425
  }
1088
1426
  async sourceFunctions(filter) {
1089
- const functions = filter.type === "all" ? this.allFunctions() : await this.#functionsAddedSince(filter.commit);
1427
+ const functions = filter.type === "all" ? this.allFunctions() : filter.type === "uncommitted" ? this.allFunctions().filter((callable) => callable.sourceMode === "working-tree") : await this.#functionsChangedSince(filter.commit);
1090
1428
  if (!filter.path) return functions;
1091
1429
  const sourcePath = normalizeRelativePath(this.rootDir, filter.path);
1092
1430
  return functions.filter((callable) => callable.path === sourcePath || callable.path.startsWith(`${sourcePath}/`));
1093
1431
  }
1094
- async #functionsAddedSince(baseRef) {
1432
+ async #functionsChangedSince(baseRef) {
1095
1433
  await this.#git.assertRepository();
1096
1434
  const checkpoint = this.#database.getCheckpoint();
1097
- if (!checkpoint) throw new CodeIndexError("added-since requires an index with a Git checkpoint.");
1435
+ if (!checkpoint) throw new CodeIndexError("changed-since requires an index with a Git checkpoint.");
1098
1436
  const base = await this.#git.resolveCommit(baseRef);
1099
1437
  if (!await this.#git.isAncestor(base, checkpoint)) {
1100
1438
  throw new GitDivergenceError(base, checkpoint);
@@ -1104,18 +1442,17 @@ var CodeIndex = class {
1104
1442
  const comparisonPaths = /* @__PURE__ */ new Map();
1105
1443
  for (const change of changes) {
1106
1444
  if (change.status === "D") continue;
1107
- if (change.status === "R") comparisonPaths.set(change.path, change.oldPath);
1108
- else if (change.status === "A" || change.status === "C") comparisonPaths.set(change.path, null);
1109
- else comparisonPaths.set(change.path, change.path);
1445
+ if (change.status === "R") comparisonPaths.set(change.path, { basePath: change.oldPath, pathChanged: true });
1446
+ else if (change.status === "A" || change.status === "C") comparisonPaths.set(change.path, { basePath: null, pathChanged: false });
1447
+ else comparisonPaths.set(change.path, { basePath: change.path, pathChanged: false });
1110
1448
  }
1111
1449
  for (const dirtyFile of this.#database.getWorkingTreeFiles()) {
1112
- if (!comparisonPaths.has(dirtyFile.path)) {
1113
- comparisonPaths.set(dirtyFile.path, dirtyFile.previousPath ?? dirtyFile.path);
1114
- }
1450
+ const basePath = dirtyFile.previousPath ?? comparisonPaths.get(dirtyFile.path)?.basePath ?? dirtyFile.path;
1451
+ comparisonPaths.set(dirtyFile.path, { basePath, pathChanged: basePath !== dirtyFile.path });
1115
1452
  }
1116
1453
  const current = this.#database.functionsForPaths([...comparisonPaths.keys()]);
1117
1454
  const baseFunctionsByCurrentPath = /* @__PURE__ */ new Map();
1118
- for (const [currentPath, basePath] of comparisonPaths) {
1455
+ for (const [currentPath, { basePath }] of comparisonPaths) {
1119
1456
  if (!basePath) {
1120
1457
  baseFunctionsByCurrentPath.set(currentPath, []);
1121
1458
  continue;
@@ -1129,19 +1466,25 @@ var CodeIndex = class {
1129
1466
  baseFunctionsByCurrentPath.set(currentPath, parseCallables(basePath, content, this.#onWarning));
1130
1467
  }
1131
1468
  const currentByPath = groupBy(current, (callable) => callable.path);
1132
- const addedIds = /* @__PURE__ */ new Set();
1469
+ const changedIds = /* @__PURE__ */ new Set();
1133
1470
  for (const [currentPath, currentFunctions] of currentByPath) {
1134
- for (const id of addedFunctionIds(currentFunctions, baseFunctionsByCurrentPath.get(currentPath) ?? [])) {
1135
- addedIds.add(id);
1471
+ const comparison = comparisonPaths.get(currentPath);
1472
+ for (const id of changedFunctionIds(
1473
+ currentFunctions,
1474
+ baseFunctionsByCurrentPath.get(currentPath) ?? [],
1475
+ comparison.pathChanged
1476
+ )) {
1477
+ changedIds.add(id);
1136
1478
  }
1137
1479
  }
1138
- return current.filter((callable) => addedIds.has(callable.id));
1480
+ return current.filter((callable) => changedIds.has(callable.id));
1139
1481
  }
1140
1482
  };
1141
- function addedFunctionIds(current, base) {
1483
+ function changedFunctionIds(current, base, pathChanged) {
1484
+ if (pathChanged) return current.map((callable) => callable.id);
1142
1485
  const currentGroups = groupBy(current, (callable) => `${callable.qualifiedName}\0${callable.kind}`);
1143
1486
  const baseGroups = groupBy(base, (callable) => `${callable.qualifiedName}\0${callable.kind}`);
1144
- const added = [];
1487
+ const changed = [];
1145
1488
  for (const [key, currentGroup] of currentGroups) {
1146
1489
  const baseGroup = baseGroups.get(key) ?? [];
1147
1490
  const unmatchedBase = new Set(baseGroup.map((_, index) => index));
@@ -1155,13 +1498,9 @@ function addedFunctionIds(current, base) {
1155
1498
  unmatchedCurrent.delete(currentIndex);
1156
1499
  }
1157
1500
  }
1158
- for (const currentIndex of unmatchedCurrent) {
1159
- const baseIndex = unmatchedBase.values().next().value;
1160
- if (baseIndex === void 0) added.push(currentGroup[currentIndex].id);
1161
- else unmatchedBase.delete(baseIndex);
1162
- }
1501
+ for (const currentIndex of unmatchedCurrent) changed.push(currentGroup[currentIndex].id);
1163
1502
  }
1164
- return added;
1503
+ return changed;
1165
1504
  }
1166
1505
  function normalizeProfile(profile) {
1167
1506
  return {
@@ -1203,7 +1542,280 @@ function groupBy(values, keyFor) {
1203
1542
  return groups;
1204
1543
  }
1205
1544
 
1545
+ // src/analysis/cohesion.ts
1546
+ import path7 from "node:path";
1547
+ async function analyzeCohesion(options) {
1548
+ const neighbors = options.neighbors ?? 20;
1549
+ const limit = options.limit ?? 50;
1550
+ const minSimilarity = options.minSimilarity ?? 0.8;
1551
+ const minLines = options.minLines ?? 2;
1552
+ const sourceFilter = options.sourceFilter ?? { type: "all" };
1553
+ assertPositiveInteger(neighbors, "neighbors");
1554
+ assertPositiveInteger(limit, "limit");
1555
+ assertPositiveInteger(minLines, "minLines");
1556
+ if (!Number.isFinite(minSimilarity) || minSimilarity < -1 || minSimilarity >= 1) {
1557
+ throw new CodeIndexError("minSimilarity must be at least -1 and less than 1.");
1558
+ }
1559
+ if (options.maxSimilarity !== void 0 && (!Number.isFinite(options.maxSimilarity) || options.maxSimilarity <= minSimilarity)) {
1560
+ throw new CodeIndexError("maxSimilarity must be greater than minSimilarity.");
1561
+ }
1562
+ const nameRegex = compileNameRegex(options.nameRegex);
1563
+ const candidateFunctions = options.source.allFunctions().filter((callable) => callable.lineCount >= minLines && (!nameRegex || nameRegex.test(callable.qualifiedName)));
1564
+ const candidateIds = new Set(candidateFunctions.map((callable) => callable.id));
1565
+ const sourceFunctions = (await options.source.sourceFunctions(sourceFilter)).filter((callable) => candidateIds.has(callable.id));
1566
+ const neighborCache = /* @__PURE__ */ new Map();
1567
+ const neighborsFor = (callable) => {
1568
+ throwIfAborted(options.signal);
1569
+ const matches = options.source.similarToFunction(callable.id, {
1570
+ limit: neighbors,
1571
+ minSimilarity,
1572
+ ...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
1573
+ minLines,
1574
+ ...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {}
1575
+ }).filter((match) => candidateIds.has(match.function.id));
1576
+ neighborCache.set(callable.id, new Set(matches.map((match) => match.function.id)));
1577
+ return matches;
1578
+ };
1579
+ const edges = /* @__PURE__ */ new Map();
1580
+ for (let index = 0; index < sourceFunctions.length; index += 1) {
1581
+ const source = sourceFunctions[index];
1582
+ for (const match of neighborsFor(source)) {
1583
+ const [left, right] = orderedFunctions(source, match.function);
1584
+ const key = pairKey(left.id, right.id);
1585
+ const existing = edges.get(key);
1586
+ if (!existing || match.similarity > existing.similarity) {
1587
+ edges.set(key, { left, right, similarity: match.similarity });
1588
+ }
1589
+ }
1590
+ options.onProgress?.({ completed: index + 1, total: sourceFunctions.length });
1591
+ }
1592
+ const scoredPairs = [...edges.values()].map((edge) => {
1593
+ const leftNeighbors = neighborCache.get(edge.left.id);
1594
+ const rightNeighbors = neighborCache.get(edge.right.id);
1595
+ const reciprocal = leftNeighbors && rightNeighbors ? leftNeighbors.has(edge.right.id) && rightNeighbors.has(edge.left.id) : null;
1596
+ return scorePair(edge, reciprocal, minSimilarity);
1597
+ }).sort(comparePairs);
1598
+ scoredPairs.forEach((pair, index) => {
1599
+ pair.rank = index + 1;
1600
+ });
1601
+ const repositoryScope = sourceFilter.type === "all" && sourceFilter.path === void 0;
1602
+ const summary = summarize(
1603
+ repositoryScope ? "repository" : "selected-sources",
1604
+ sourceFunctions.length,
1605
+ candidateFunctions.length,
1606
+ scoredPairs
1607
+ );
1608
+ const reportedPairs = scoredPairs.slice(0, limit);
1609
+ const files = summarizeFiles(sourceFunctions, scoredPairs).filter((file) => file.internalAffinity + file.sameFolderAffinity + file.externalAffinity > 0).slice(0, limit);
1610
+ const groups = buildGroups(reportedPairs);
1611
+ const status = options.source.status();
1612
+ return {
1613
+ schemaVersion: 1,
1614
+ repository: {
1615
+ generation: status.generation,
1616
+ gitCheckpoint: status.gitCheckpoint,
1617
+ embeddingProfile: status.embeddingProfile
1618
+ },
1619
+ parameters: {
1620
+ neighbors,
1621
+ limit,
1622
+ minSimilarity,
1623
+ ...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
1624
+ minLines,
1625
+ ...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {},
1626
+ sourceFilter
1627
+ },
1628
+ summary,
1629
+ pairs: reportedPairs,
1630
+ files,
1631
+ groups
1632
+ };
1633
+ }
1634
+ function cohesionLocation(leftPath, rightPath) {
1635
+ const normalizedLeft = leftPath.replaceAll("\\", "/");
1636
+ const normalizedRight = rightPath.replaceAll("\\", "/");
1637
+ const leftDirectory = directoryParts(normalizedLeft);
1638
+ const rightDirectory = directoryParts(normalizedRight);
1639
+ let commonLength = 0;
1640
+ while (commonLength < leftDirectory.length && commonLength < rightDirectory.length && leftDirectory[commonLength] === rightDirectory[commonLength]) {
1641
+ commonLength += 1;
1642
+ }
1643
+ const commonParts = leftDirectory.slice(0, commonLength);
1644
+ const commonAncestor = commonParts.length > 0 ? commonParts.join("/") : null;
1645
+ const sameFile = normalizedLeft === normalizedRight;
1646
+ const folderHops = sameFile ? 0 : leftDirectory.length - commonLength + rightDirectory.length - commonLength;
1647
+ return {
1648
+ category: sameFile ? "same-file" : folderHops === 0 ? "same-folder" : "different-folder",
1649
+ physicalDistance: sameFile ? 0 : 1 + folderHops,
1650
+ folderHops,
1651
+ commonAncestor,
1652
+ sourceTestPair: isTestPath(normalizedLeft) !== isTestPath(normalizedRight)
1653
+ };
1654
+ }
1655
+ function scorePair(edge, reciprocal, minSimilarity) {
1656
+ const location = cohesionLocation(edge.left.path, edge.right.path);
1657
+ const semanticWeight = clamp((edge.similarity - minSimilarity) / (1 - minSimilarity));
1658
+ const separationWeight = 1 - Math.exp(-location.physicalDistance / 2);
1659
+ return {
1660
+ rank: 0,
1661
+ left: edge.left,
1662
+ right: edge.right,
1663
+ similarity: edge.similarity,
1664
+ reciprocal,
1665
+ semanticWeight,
1666
+ separationWeight,
1667
+ cohesionGap: semanticWeight * separationWeight,
1668
+ location
1669
+ };
1670
+ }
1671
+ function summarize(scope, functionsAnalyzed, candidateFunctions, pairs) {
1672
+ const totalWeight = pairs.reduce((sum, pair) => sum + pair.semanticWeight, 0);
1673
+ const sameFileWeight = pairs.filter((pair) => pair.location.category === "same-file").reduce((sum, pair) => sum + pair.semanticWeight, 0);
1674
+ const sameFolderWeight = pairs.filter((pair) => pair.location.category === "same-folder").reduce((sum, pair) => sum + pair.semanticWeight, 0);
1675
+ const remoteWeight = pairs.filter((pair) => pair.location.category === "different-folder").reduce((sum, pair) => sum + pair.semanticWeight, 0);
1676
+ const weightedDistance = pairs.reduce(
1677
+ (sum, pair) => sum + pair.semanticWeight * pair.location.physicalDistance,
1678
+ 0
1679
+ );
1680
+ return {
1681
+ scope,
1682
+ functionsAnalyzed,
1683
+ candidateFunctions,
1684
+ semanticEdges: pairs.length,
1685
+ sameFileRatio: ratio(sameFileWeight, totalWeight),
1686
+ sameFolderRatio: ratio(sameFolderWeight, totalWeight),
1687
+ remoteRatio: ratio(remoteWeight, totalWeight),
1688
+ weightedMeanDistance: ratio(weightedDistance, totalWeight)
1689
+ };
1690
+ }
1691
+ function summarizeFiles(sourceFunctions, pairs) {
1692
+ const sourceIds = new Set(sourceFunctions.map((callable) => callable.id));
1693
+ const counts = /* @__PURE__ */ new Map();
1694
+ for (const callable of sourceFunctions) counts.set(callable.path, (counts.get(callable.path) ?? 0) + 1);
1695
+ const reports = /* @__PURE__ */ new Map();
1696
+ for (const [filePath, functionCount] of counts) {
1697
+ reports.set(filePath, {
1698
+ path: filePath,
1699
+ functionCount,
1700
+ internalAffinity: 0,
1701
+ sameFolderAffinity: 0,
1702
+ externalAffinity: 0,
1703
+ externalAffinityRatio: 0
1704
+ });
1705
+ }
1706
+ for (const pair of pairs) {
1707
+ if (sourceIds.has(pair.left.id)) addFileAffinity(reports.get(pair.left.path), pair.right, pair);
1708
+ if (sourceIds.has(pair.right.id)) addFileAffinity(reports.get(pair.right.path), pair.left, pair);
1709
+ }
1710
+ for (const report of reports.values()) {
1711
+ const total = report.internalAffinity + report.sameFolderAffinity + report.externalAffinity;
1712
+ report.externalAffinityRatio = ratio(report.externalAffinity, total);
1713
+ }
1714
+ return [...reports.values()].sort((left, right) => right.externalAffinityRatio - left.externalAffinityRatio || right.externalAffinity - left.externalAffinity || left.path.localeCompare(right.path));
1715
+ }
1716
+ function addFileAffinity(report, match, pair) {
1717
+ if (pair.location.category === "same-file") report.internalAffinity += pair.semanticWeight;
1718
+ else if (pair.location.category === "same-folder") report.sameFolderAffinity += pair.semanticWeight;
1719
+ else {
1720
+ report.externalAffinity += pair.semanticWeight;
1721
+ updateStrongestExternal(report, match, pair);
1722
+ }
1723
+ }
1724
+ function updateStrongestExternal(report, callable, pair) {
1725
+ if (!report.strongestExternalMatch || pair.cohesionGap > report.strongestExternalMatch.cohesionGap) {
1726
+ report.strongestExternalMatch = {
1727
+ function: callable,
1728
+ similarity: pair.similarity,
1729
+ cohesionGap: pair.cohesionGap
1730
+ };
1731
+ }
1732
+ }
1733
+ function buildGroups(pairs) {
1734
+ const functions = /* @__PURE__ */ new Map();
1735
+ const neighbors = /* @__PURE__ */ new Map();
1736
+ for (const pair of pairs) {
1737
+ functions.set(pair.left.id, pair.left);
1738
+ functions.set(pair.right.id, pair.right);
1739
+ addNeighbor(neighbors, pair.left.id, pair.right.id);
1740
+ addNeighbor(neighbors, pair.right.id, pair.left.id);
1741
+ }
1742
+ const seen = /* @__PURE__ */ new Set();
1743
+ const groups = [];
1744
+ for (const start of neighbors.keys()) {
1745
+ if (seen.has(start)) continue;
1746
+ const pending = [start];
1747
+ const memberIds = /* @__PURE__ */ new Set();
1748
+ while (pending.length > 0) {
1749
+ const id = pending.pop();
1750
+ if (seen.has(id)) continue;
1751
+ seen.add(id);
1752
+ memberIds.add(id);
1753
+ for (const neighbor of neighbors.get(id) ?? []) pending.push(neighbor);
1754
+ }
1755
+ const groupPairs = pairs.filter((pair) => memberIds.has(pair.left.id) && memberIds.has(pair.right.id));
1756
+ const members = [...memberIds].map((id) => functions.get(id)).sort(compareFunctions);
1757
+ groups.push({
1758
+ rank: 0,
1759
+ memberCount: members.length,
1760
+ fileCount: new Set(members.map((member) => member.path)).size,
1761
+ minimumEdgeSimilarity: Math.min(...groupPairs.map((pair) => pair.similarity)),
1762
+ maximumEdgeSimilarity: Math.max(...groupPairs.map((pair) => pair.similarity)),
1763
+ maximumPhysicalDistance: Math.max(...groupPairs.map((pair) => pair.location.physicalDistance)),
1764
+ maximumCohesionGap: Math.max(...groupPairs.map((pair) => pair.cohesionGap)),
1765
+ members
1766
+ });
1767
+ }
1768
+ groups.sort((left, right) => right.maximumCohesionGap - left.maximumCohesionGap || right.memberCount - left.memberCount || compareFunctions(left.members[0], right.members[0]));
1769
+ groups.forEach((group, index) => {
1770
+ group.rank = index + 1;
1771
+ });
1772
+ return groups;
1773
+ }
1774
+ function compileNameRegex(pattern) {
1775
+ if (pattern === void 0) return void 0;
1776
+ try {
1777
+ return new RegExp(pattern);
1778
+ } catch (error) {
1779
+ throw new CodeIndexError(`Invalid name regex: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
1780
+ }
1781
+ }
1782
+ function directoryParts(filePath) {
1783
+ const directory = path7.posix.dirname(filePath.replaceAll("\\", "/"));
1784
+ return directory === "." ? [] : directory.split("/").filter(Boolean);
1785
+ }
1786
+ function isTestPath(filePath) {
1787
+ const normalized = filePath.replaceAll("\\", "/").toLowerCase();
1788
+ const segments = normalized.split("/");
1789
+ const fileName = segments.at(-1) ?? "";
1790
+ return segments.some((segment) => segment === "test" || segment === "tests" || segment === "__tests__") || /\.(?:test|spec)\.[cm]?[jt]sx?$/.test(fileName);
1791
+ }
1792
+ function orderedFunctions(left, right) {
1793
+ return compareFunctions(left, right) <= 0 ? [left, right] : [right, left];
1794
+ }
1795
+ function compareFunctions(left, right) {
1796
+ return left.path.localeCompare(right.path) || left.startLine - right.startLine || left.startColumn - right.startColumn || left.qualifiedName.localeCompare(right.qualifiedName) || left.id - right.id;
1797
+ }
1798
+ function comparePairs(left, right) {
1799
+ return right.cohesionGap - left.cohesionGap || right.similarity - left.similarity || Number(right.reciprocal === true) - Number(left.reciprocal === true) || compareFunctions(left.left, right.left) || compareFunctions(left.right, right.right);
1800
+ }
1801
+ function pairKey(leftId, rightId) {
1802
+ return leftId < rightId ? `${leftId}:${rightId}` : `${rightId}:${leftId}`;
1803
+ }
1804
+ function addNeighbor(neighbors, source, target) {
1805
+ const values = neighbors.get(source) ?? /* @__PURE__ */ new Set();
1806
+ values.add(target);
1807
+ neighbors.set(source, values);
1808
+ }
1809
+ function ratio(numerator, denominator) {
1810
+ return denominator === 0 ? 0 : numerator / denominator;
1811
+ }
1812
+ function clamp(value) {
1813
+ return Math.max(0, Math.min(1, value));
1814
+ }
1815
+
1206
1816
  // src/search/cross-search.ts
1817
+ import { realpath, stat } from "node:fs/promises";
1818
+ import path8 from "node:path";
1207
1819
  async function* crossSearch(options) {
1208
1820
  const target = options.target ?? options.source;
1209
1821
  const sourceProfile = JSON.stringify(options.source.status().embeddingProfile);
@@ -1213,21 +1825,55 @@ async function* crossSearch(options) {
1213
1825
  }
1214
1826
  const limit = options.limitPerFunction ?? 5;
1215
1827
  assertPositiveInteger(limit, "limitPerFunction");
1216
- const sourceFunctions = await options.source.sourceFunctions(options.sourceFilter ?? { type: "all" });
1217
- const sameIndex = target.indexPath === options.source.indexPath;
1828
+ const minLines = options.minLines ?? 2;
1829
+ assertPositiveInteger(minLines, "minLines");
1830
+ const nameRegex = compileNameRegex2(options.nameRegex);
1831
+ const sourceFunctions = (await options.source.sourceFunctions(options.sourceFilter ?? { type: "all" })).filter((callable) => callable.lineCount >= minLines && (!nameRegex || nameRegex.test(callable.qualifiedName)));
1832
+ const sameIndex = target.indexPath === options.source.indexPath || await fileIdentity(target.indexPath) === await fileIdentity(options.source.indexPath);
1833
+ const sourceRoot = options.crossFileOnly ? await canonicalRoot(options.source.rootDir) : void 0;
1834
+ const targetRoot = options.crossFileOnly ? sameIndex ? sourceRoot : await canonicalRoot(target.rootDir) : void 0;
1835
+ const canonicalFiles = /* @__PURE__ */ new Map();
1836
+ const canonicalFile = (root, filePath) => {
1837
+ const absolutePath = path8.resolve(root, filePath);
1838
+ let result = canonicalFiles.get(absolutePath);
1839
+ if (!result) {
1840
+ result = fileIdentity(absolutePath);
1841
+ canonicalFiles.set(absolutePath, result);
1842
+ }
1843
+ return result;
1844
+ };
1845
+ const targetPathsByCanonicalFile = /* @__PURE__ */ new Map();
1846
+ if (targetRoot) {
1847
+ const targetPaths = [...new Set(target.allFunctions().map((callable) => callable.path))];
1848
+ await Promise.all(targetPaths.map(async (targetPath) => {
1849
+ const canonicalPath = await canonicalFile(targetRoot, targetPath);
1850
+ const paths = targetPathsByCanonicalFile.get(canonicalPath) ?? [];
1851
+ paths.push(targetPath);
1852
+ targetPathsByCanonicalFile.set(canonicalPath, paths);
1853
+ }));
1854
+ }
1218
1855
  const seenPairs = /* @__PURE__ */ new Set();
1219
1856
  for (let index = 0; index < sourceFunctions.length; index += 1) {
1220
1857
  throwIfAborted(options.signal);
1221
1858
  const source = sourceFunctions[index];
1222
1859
  const vector = options.source.vectorForFunction(source.id);
1860
+ const canonicalSourceFile = sourceRoot ? await canonicalFile(sourceRoot, source.path) : void 0;
1861
+ const excludedTargetPaths = canonicalSourceFile ? targetPathsByCanonicalFile.get(canonicalSourceFile) : void 0;
1862
+ const excludePaths = excludedTargetPaths ? { excludePaths: excludedTargetPaths } : {};
1223
1863
  const candidates = sameIndex ? options.source.similarToFunction(source.id, {
1224
1864
  limit,
1225
1865
  minSimilarity: options.minSimilarity ?? -1,
1226
- ...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {}
1866
+ ...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
1867
+ minLines,
1868
+ ...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {},
1869
+ ...excludePaths
1227
1870
  }) : target.searchByVector(vector, {
1228
1871
  limit,
1229
1872
  minSimilarity: options.minSimilarity ?? -1,
1230
- ...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {}
1873
+ ...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
1874
+ minLines,
1875
+ ...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {},
1876
+ ...excludePaths
1231
1877
  });
1232
1878
  const matches = sameIndex && !options.includeSymmetricDuplicates ? candidates.filter((match) => {
1233
1879
  const pair = source.id < match.function.id ? `${source.id}:${match.function.id}` : `${match.function.id}:${source.id}`;
@@ -1239,6 +1885,29 @@ async function* crossSearch(options) {
1239
1885
  options.onProgress?.({ completed: index + 1, total: sourceFunctions.length });
1240
1886
  }
1241
1887
  }
1888
+ function compileNameRegex2(pattern) {
1889
+ if (pattern === void 0) return void 0;
1890
+ try {
1891
+ return new RegExp(pattern);
1892
+ } catch (error) {
1893
+ throw new CodeIndexError(`Invalid name regex: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
1894
+ }
1895
+ }
1896
+ async function canonicalRoot(root) {
1897
+ try {
1898
+ return await realpath(root);
1899
+ } catch {
1900
+ return path8.resolve(root);
1901
+ }
1902
+ }
1903
+ async function fileIdentity(filePath) {
1904
+ try {
1905
+ const metadata = await stat(filePath);
1906
+ return `inode:${metadata.dev}:${metadata.ino}`;
1907
+ } catch {
1908
+ return `path:${path8.resolve(filePath)}`;
1909
+ }
1910
+ }
1242
1911
 
1243
1912
  // src/embeddings/openai.ts
1244
1913
  import { Tiktoken } from "js-tiktoken/lite";
@@ -1305,8 +1974,8 @@ var OpenAIEmbeddingProvider = class {
1305
1974
  constructor(options = {}) {
1306
1975
  this.#apiKey = options.apiKey ?? process.env.OPENAI_API_KEY ?? "";
1307
1976
  if (!this.#apiKey) throw new Error("OPENAI_API_KEY is required.");
1308
- const model = options.model ?? "text-embedding-3-small";
1309
- const dimensions = options.dimensions ?? 1536;
1977
+ const model = options.model ?? "text-embedding-3-large";
1978
+ const dimensions = options.dimensions ?? 3072;
1310
1979
  if (!Number.isInteger(dimensions) || dimensions < 1) throw new Error("dimensions must be a positive integer.");
1311
1980
  this.profile = { provider: "openai", model, dimensions, strategyVersion: "callable-v1" };
1312
1981
  this.#url = `${(options.baseUrl ?? "https://api.openai.com/v1").replace(/\/$/, "")}/embeddings`;
@@ -1390,24 +2059,35 @@ function updateFiles(index, options) {
1390
2059
  function updateFromGit(index, options) {
1391
2060
  return index.updateFromGit(options);
1392
2061
  }
2062
+ function updateFromWorkingTree(index, options) {
2063
+ return index.updateFromWorkingTree(options);
2064
+ }
1393
2065
  function similaritySearch(index, options) {
1394
2066
  return index.similaritySearch(options);
1395
2067
  }
1396
2068
  function crossSearchFunctions(options) {
1397
2069
  return crossSearch(options);
1398
2070
  }
2071
+ function analyzeCodeCohesion(options) {
2072
+ return analyzeCohesion(options);
2073
+ }
1399
2074
  export {
1400
2075
  CodeIndex,
1401
2076
  CodeIndexError,
1402
2077
  GitDivergenceError,
2078
+ GitUnavailableError,
1403
2079
  IncompatibleIndexError,
1404
2080
  JinaEmbeddingProvider,
1405
2081
  OpenAIEmbeddingProvider,
2082
+ analyzeCodeCohesion,
2083
+ analyzeCohesion,
2084
+ cohesionLocation,
1406
2085
  crossSearch,
1407
2086
  crossSearchFunctions,
1408
2087
  openCodeIndex,
1409
2088
  similaritySearch,
1410
2089
  updateFiles,
1411
- updateFromGit
2090
+ updateFromGit,
2091
+ updateFromWorkingTree
1412
2092
  };
1413
2093
  //# sourceMappingURL=index.js.map