@ninjaxtools/slopdex 0.3.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/slopdex/SKILL.md +173 -24
- package/README.md +93 -94
- package/dist/cli.js +1254 -198
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +129 -2
- package/dist/index.js +796 -116
- package/dist/index.js.map +1 -1
- package/package.json +5 -1
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/code-index.ts
|
|
2
|
-
import { lstat, readFile } from "node:fs/promises";
|
|
2
|
+
import { lstat as lstat2, readFile, readdir } from "node:fs/promises";
|
|
3
3
|
import path6 from "node:path";
|
|
4
4
|
|
|
5
5
|
// src/errors.ts
|
|
@@ -15,6 +15,12 @@ var GitDivergenceError = class extends CodeIndexError {
|
|
|
15
15
|
this.name = "GitDivergenceError";
|
|
16
16
|
}
|
|
17
17
|
};
|
|
18
|
+
var GitUnavailableError = class extends CodeIndexError {
|
|
19
|
+
constructor(message, options) {
|
|
20
|
+
super(message, options);
|
|
21
|
+
this.name = "GitUnavailableError";
|
|
22
|
+
}
|
|
23
|
+
};
|
|
18
24
|
var IncompatibleIndexError = class extends CodeIndexError {
|
|
19
25
|
constructor(message) {
|
|
20
26
|
super(message);
|
|
@@ -24,6 +30,7 @@ var IncompatibleIndexError = class extends CodeIndexError {
|
|
|
24
30
|
|
|
25
31
|
// src/git/repository.ts
|
|
26
32
|
import { spawn } from "node:child_process";
|
|
33
|
+
import { lstat } from "node:fs/promises";
|
|
27
34
|
import path from "node:path";
|
|
28
35
|
var GitRepository = class {
|
|
29
36
|
constructor(rootDir) {
|
|
@@ -37,7 +44,13 @@ var GitRepository = class {
|
|
|
37
44
|
const stderr = [];
|
|
38
45
|
child.stdout.on("data", (value) => stdout.push(value));
|
|
39
46
|
child.stderr.on("data", (value) => stderr.push(value));
|
|
40
|
-
child.on("error",
|
|
47
|
+
child.on("error", (error) => {
|
|
48
|
+
if (error.code === "ENOENT") {
|
|
49
|
+
reject(new GitUnavailableError("Git executable is not available.", { cause: error }));
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
reject(error);
|
|
53
|
+
});
|
|
41
54
|
child.on("close", (code) => {
|
|
42
55
|
const exitCode = code ?? -1;
|
|
43
56
|
if (exitCode === 0 || allowExitOne && exitCode === 1) {
|
|
@@ -49,29 +62,51 @@ var GitRepository = class {
|
|
|
49
62
|
});
|
|
50
63
|
}
|
|
51
64
|
async assertRepository() {
|
|
52
|
-
|
|
65
|
+
let result;
|
|
66
|
+
try {
|
|
67
|
+
result = await this.#run(["rev-parse", "--is-inside-work-tree"]);
|
|
68
|
+
} catch (error) {
|
|
69
|
+
if (error instanceof GitUnavailableError) throw error;
|
|
70
|
+
try {
|
|
71
|
+
await lstat(path.join(this.rootDir, ".git"));
|
|
72
|
+
} catch (markerError) {
|
|
73
|
+
if (markerError.code === "ENOENT") {
|
|
74
|
+
throw new GitUnavailableError(`${this.rootDir} is not a Git worktree.`, { cause: error });
|
|
75
|
+
}
|
|
76
|
+
throw markerError;
|
|
77
|
+
}
|
|
78
|
+
throw error;
|
|
79
|
+
}
|
|
53
80
|
if (result.stdout.toString("utf8").trim() !== "true") {
|
|
54
|
-
throw new
|
|
81
|
+
throw new GitUnavailableError(`${this.rootDir} is not a Git worktree.`);
|
|
55
82
|
}
|
|
56
83
|
const topLevel = (await this.#run(["rev-parse", "--show-toplevel"])).stdout.toString("utf8").trim();
|
|
57
84
|
if (path.resolve(topLevel) !== path.resolve(this.rootDir)) {
|
|
58
85
|
throw new CodeIndexError(`rootDir must be the Git worktree root: ${topLevel}`);
|
|
59
86
|
}
|
|
60
87
|
}
|
|
61
|
-
async
|
|
88
|
+
async workTreeChanges(base, excludePaths = []) {
|
|
62
89
|
const pathspecs = [".", ...excludePaths.map((filePath) => `:(top,literal,exclude)${filePath}`)];
|
|
63
90
|
const result = await this.#run([
|
|
64
|
-
"
|
|
65
|
-
"--
|
|
91
|
+
"diff",
|
|
92
|
+
"--name-status",
|
|
66
93
|
"-z",
|
|
67
|
-
"--
|
|
68
|
-
|
|
94
|
+
"--find-renames",
|
|
95
|
+
base,
|
|
69
96
|
"--",
|
|
70
97
|
...pathspecs
|
|
71
98
|
]);
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
99
|
+
const changes = parseChanges(splitNull(result.stdout));
|
|
100
|
+
const untracked = await this.#run([
|
|
101
|
+
"ls-files",
|
|
102
|
+
"--others",
|
|
103
|
+
"--exclude-standard",
|
|
104
|
+
"-z",
|
|
105
|
+
"--",
|
|
106
|
+
...pathspecs
|
|
107
|
+
]);
|
|
108
|
+
for (const filePath of splitNull(untracked.stdout)) changes.push({ status: "A", path: filePath });
|
|
109
|
+
return changes;
|
|
75
110
|
}
|
|
76
111
|
async resolveCommit(ref) {
|
|
77
112
|
const result = await this.#run(["rev-parse", "--verify", "--end-of-options", `${ref}^{commit}`]);
|
|
@@ -103,26 +138,28 @@ var GitRepository = class {
|
|
|
103
138
|
}
|
|
104
139
|
async diff(base, target) {
|
|
105
140
|
const result = await this.#run(["diff", "--name-status", "-z", "--find-renames", `${base}..${target}`, "--"]);
|
|
106
|
-
|
|
107
|
-
const changes = [];
|
|
108
|
-
for (let index = 0; index < values.length; ) {
|
|
109
|
-
const rawStatus = values[index++];
|
|
110
|
-
if (!rawStatus) continue;
|
|
111
|
-
const status = rawStatus[0];
|
|
112
|
-
if (status === "R" || status === "C") {
|
|
113
|
-
const oldPath = values[index++];
|
|
114
|
-
const filePath = values[index++];
|
|
115
|
-
if (oldPath && filePath) changes.push({ status, oldPath, path: filePath });
|
|
116
|
-
} else if (status === "A" || status === "M" || status === "D" || status === "T") {
|
|
117
|
-
const filePath = values[index++];
|
|
118
|
-
if (filePath) changes.push({ status, path: filePath });
|
|
119
|
-
} else {
|
|
120
|
-
throw new CodeIndexError(`Unsupported Git diff status: ${rawStatus}`);
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
return changes;
|
|
141
|
+
return parseChanges(splitNull(result.stdout));
|
|
124
142
|
}
|
|
125
143
|
};
|
|
144
|
+
function parseChanges(values) {
|
|
145
|
+
const changes = [];
|
|
146
|
+
for (let index = 0; index < values.length; ) {
|
|
147
|
+
const rawStatus = values[index++];
|
|
148
|
+
if (!rawStatus) continue;
|
|
149
|
+
const status = rawStatus[0];
|
|
150
|
+
if (status === "R" || status === "C") {
|
|
151
|
+
const oldPath = values[index++];
|
|
152
|
+
const filePath = values[index++];
|
|
153
|
+
if (oldPath && filePath) changes.push({ status, oldPath, path: filePath });
|
|
154
|
+
} else if (status === "A" || status === "M" || status === "D" || status === "T") {
|
|
155
|
+
const filePath = values[index++];
|
|
156
|
+
if (filePath) changes.push({ status, path: filePath });
|
|
157
|
+
} else {
|
|
158
|
+
throw new CodeIndexError(`Unsupported Git diff status: ${rawStatus}`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return changes;
|
|
162
|
+
}
|
|
126
163
|
function splitNull(buffer) {
|
|
127
164
|
const value = buffer.toString("utf8");
|
|
128
165
|
const parts = value.split("\0");
|
|
@@ -446,6 +483,7 @@ function parseCallables(relativePath, content, onWarning = console.warn) {
|
|
|
446
483
|
startColumn: node.startPosition.column + 1,
|
|
447
484
|
endLine: node.endPosition.row + 1,
|
|
448
485
|
endColumn: node.endPosition.column + 1,
|
|
486
|
+
lineCount: node.endPosition.row - node.startPosition.row + 1,
|
|
449
487
|
source,
|
|
450
488
|
sourceHash: sha256(source),
|
|
451
489
|
embeddingInput
|
|
@@ -475,18 +513,24 @@ var SourcePolicy = class {
|
|
|
475
513
|
includes(relativePath) {
|
|
476
514
|
const normalized = relativePath.replaceAll("\\", "/");
|
|
477
515
|
if (!languageForPath(normalized)) return false;
|
|
478
|
-
if (normalized
|
|
516
|
+
if (hasExcludedSegment(normalized)) return false;
|
|
479
517
|
if (this.#exclude.some((pattern) => path4.matchesGlob(normalized, pattern))) return false;
|
|
480
518
|
return this.#include.length === 0 || this.#include.some((pattern) => path4.matchesGlob(normalized, pattern));
|
|
481
519
|
}
|
|
520
|
+
traversesDirectory(relativePath) {
|
|
521
|
+
return !hasExcludedSegment(relativePath.replaceAll("\\", "/"));
|
|
522
|
+
}
|
|
482
523
|
};
|
|
524
|
+
function hasExcludedSegment(relativePath) {
|
|
525
|
+
return relativePath.split("/").some((segment) => DEFAULT_EXCLUDED_SEGMENTS.has(segment));
|
|
526
|
+
}
|
|
483
527
|
|
|
484
528
|
// src/storage/database.ts
|
|
485
529
|
import { mkdirSync } from "node:fs";
|
|
486
530
|
import path5 from "node:path";
|
|
487
531
|
import { DatabaseSync } from "node:sqlite";
|
|
488
532
|
import * as sqliteVec from "sqlite-vec";
|
|
489
|
-
var SCHEMA_VERSION = "
|
|
533
|
+
var SCHEMA_VERSION = "2";
|
|
490
534
|
var IndexDatabase = class {
|
|
491
535
|
#db;
|
|
492
536
|
#rootDir;
|
|
@@ -498,15 +542,30 @@ var IndexDatabase = class {
|
|
|
498
542
|
this.#rootDir = rootDir;
|
|
499
543
|
this.#profile = profile;
|
|
500
544
|
this.#db = new DatabaseSync(indexPath, { allowExtension: true, readOnly });
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
545
|
+
try {
|
|
546
|
+
sqliteVec.load(this.#db);
|
|
547
|
+
this.#db.enableLoadExtension(false);
|
|
548
|
+
const regexes = /* @__PURE__ */ new Map();
|
|
549
|
+
this.#db.function("slopdex_regexp", { deterministic: true }, (pattern, value) => {
|
|
550
|
+
if (typeof pattern !== "string" || typeof value !== "string") return 0;
|
|
551
|
+
let regex = regexes.get(pattern);
|
|
552
|
+
if (!regex) {
|
|
553
|
+
regex = new RegExp(pattern);
|
|
554
|
+
regexes.set(pattern, regex);
|
|
555
|
+
}
|
|
556
|
+
return regex.test(value) ? 1 : 0;
|
|
557
|
+
});
|
|
558
|
+
if (readOnly) {
|
|
559
|
+
this.#db.exec("PRAGMA foreign_keys=ON");
|
|
560
|
+
} else {
|
|
561
|
+
this.#db.exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; PRAGMA foreign_keys=ON;");
|
|
562
|
+
this.#migrate();
|
|
563
|
+
}
|
|
564
|
+
this.#validateMetadata(readOnly);
|
|
565
|
+
} catch (error) {
|
|
566
|
+
this.#db.close();
|
|
567
|
+
throw error;
|
|
508
568
|
}
|
|
509
|
-
this.#validateMetadata(readOnly);
|
|
510
569
|
}
|
|
511
570
|
close() {
|
|
512
571
|
this.#db.close();
|
|
@@ -545,6 +604,7 @@ var IndexDatabase = class {
|
|
|
545
604
|
start_column INTEGER NOT NULL,
|
|
546
605
|
end_line INTEGER NOT NULL,
|
|
547
606
|
end_column INTEGER NOT NULL,
|
|
607
|
+
line_count INTEGER NOT NULL,
|
|
548
608
|
source TEXT NOT NULL,
|
|
549
609
|
source_hash TEXT NOT NULL,
|
|
550
610
|
embedding_input TEXT NOT NULL,
|
|
@@ -553,13 +613,31 @@ var IndexDatabase = class {
|
|
|
553
613
|
source_mode TEXT NOT NULL CHECK(source_mode IN ('git', 'working-tree')),
|
|
554
614
|
embedding_id INTEGER NOT NULL REFERENCES embeddings(id)
|
|
555
615
|
);
|
|
616
|
+
CREATE TABLE IF NOT EXISTS callable_provenance (
|
|
617
|
+
identity_key TEXT NOT NULL,
|
|
618
|
+
source_hash TEXT NOT NULL,
|
|
619
|
+
first_seen_commit TEXT NOT NULL,
|
|
620
|
+
PRIMARY KEY(identity_key, source_hash)
|
|
621
|
+
);
|
|
556
622
|
CREATE INDEX IF NOT EXISTS functions_path ON functions(path);
|
|
557
623
|
CREATE INDEX IF NOT EXISTS functions_embedding ON functions(embedding_id);
|
|
624
|
+
INSERT OR IGNORE INTO callable_provenance(identity_key, source_hash, first_seen_commit)
|
|
625
|
+
SELECT identity_key, source_hash, first_seen_commit FROM functions WHERE first_seen_commit IS NOT NULL;
|
|
558
626
|
`);
|
|
559
627
|
const fileColumns = this.#db.prepare("PRAGMA table_info(files)").all();
|
|
560
628
|
if (!fileColumns.some((column) => column.name === "previous_path")) {
|
|
561
629
|
this.#db.exec("ALTER TABLE files ADD COLUMN previous_path TEXT");
|
|
562
630
|
}
|
|
631
|
+
const functionColumns = this.#db.prepare("PRAGMA table_info(functions)").all();
|
|
632
|
+
const needsLineCount = !functionColumns.some((column) => column.name === "line_count");
|
|
633
|
+
const schemaVersion = this.#metadata("schema_version");
|
|
634
|
+
if (needsLineCount || schemaVersion === "1") {
|
|
635
|
+
this.#transaction(() => {
|
|
636
|
+
if (needsLineCount) this.#db.exec("ALTER TABLE functions ADD COLUMN line_count INTEGER NOT NULL DEFAULT 1");
|
|
637
|
+
this.#db.exec("UPDATE functions SET line_count = end_line - start_line + 1");
|
|
638
|
+
if (schemaVersion === "1") this.#setMetadata("schema_version", SCHEMA_VERSION);
|
|
639
|
+
});
|
|
640
|
+
}
|
|
563
641
|
}
|
|
564
642
|
#metadata(key) {
|
|
565
643
|
const row = this.#db.prepare("SELECT value FROM metadata WHERE key = ?").get(key);
|
|
@@ -571,6 +649,9 @@ var IndexDatabase = class {
|
|
|
571
649
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
572
650
|
`).run(key, value);
|
|
573
651
|
}
|
|
652
|
+
#deleteMetadata(key) {
|
|
653
|
+
this.#db.prepare("DELETE FROM metadata WHERE key = ?").run(key);
|
|
654
|
+
}
|
|
574
655
|
#validateMetadata(readOnly) {
|
|
575
656
|
const schemaVersion = this.#metadata("schema_version");
|
|
576
657
|
if (readOnly && !schemaVersion) throw new IncompatibleIndexError("Target file is not a slopdex index.");
|
|
@@ -619,6 +700,13 @@ var IndexDatabase = class {
|
|
|
619
700
|
FROM files WHERE source_mode = 'working-tree' ORDER BY path
|
|
620
701
|
`).all();
|
|
621
702
|
}
|
|
703
|
+
getFileStates() {
|
|
704
|
+
return this.#db.prepare(`
|
|
705
|
+
SELECT path, content_hash AS contentHash, blob_oid AS blobOid,
|
|
706
|
+
source_mode AS sourceMode, previous_path AS previousPath
|
|
707
|
+
FROM files ORDER BY path
|
|
708
|
+
`).all();
|
|
709
|
+
}
|
|
622
710
|
getEmbeddingKeys(keys) {
|
|
623
711
|
const found = /* @__PURE__ */ new Set();
|
|
624
712
|
const statement = this.#db.prepare("SELECT 1 FROM embeddings WHERE embedding_key = ?");
|
|
@@ -632,36 +720,44 @@ var IndexDatabase = class {
|
|
|
632
720
|
if (options.expectedCheckpoint !== void 0 && this.getCheckpoint() !== options.expectedCheckpoint) {
|
|
633
721
|
throw new CodeIndexError("Git checkpoint changed while the update was being prepared.");
|
|
634
722
|
}
|
|
723
|
+
if (options.expectedGeneration !== void 0 && this.getGeneration() !== options.expectedGeneration) {
|
|
724
|
+
throw new CodeIndexError("Index changed while the update was being prepared; retry the update.");
|
|
725
|
+
}
|
|
635
726
|
const stats = {
|
|
636
|
-
filesUpdated:
|
|
727
|
+
filesUpdated: 0,
|
|
637
728
|
filesDeleted: 0,
|
|
638
729
|
functionsAdded: 0,
|
|
639
730
|
functionsUpdated: 0,
|
|
640
731
|
functionsDeleted: 0,
|
|
641
732
|
embeddingsCreated: 0,
|
|
642
|
-
checkpoint: options.checkpoint
|
|
733
|
+
checkpoint: options.checkpoint !== void 0 ? options.checkpoint : this.getCheckpoint()
|
|
643
734
|
};
|
|
644
735
|
const deleteFile = this.#db.prepare("DELETE FROM files WHERE path = ?");
|
|
645
736
|
const countFunctions = this.#db.prepare("SELECT COUNT(*) AS count FROM functions WHERE path = ?");
|
|
646
|
-
const
|
|
647
|
-
|
|
648
|
-
uniqueDeletes
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
const count = Number(countFunctions.get(filePath).count);
|
|
654
|
-
const result = deleteFile.run(filePath);
|
|
655
|
-
if (result.changes > 0) {
|
|
656
|
-
stats.filesDeleted += 1;
|
|
657
|
-
stats.functionsDeleted += count;
|
|
737
|
+
const applyStage = (files, deletePaths) => {
|
|
738
|
+
stats.filesUpdated += files.length;
|
|
739
|
+
const uniqueDeletes = new Set(deletePaths);
|
|
740
|
+
for (const file of files) {
|
|
741
|
+
uniqueDeletes.delete(file.path);
|
|
742
|
+
if (file.previousPath) uniqueDeletes.delete(file.previousPath);
|
|
743
|
+
if (file.replacePath) uniqueDeletes.delete(file.replacePath);
|
|
658
744
|
}
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
745
|
+
for (const filePath of uniqueDeletes) {
|
|
746
|
+
const count = Number(countFunctions.get(filePath).count);
|
|
747
|
+
const result = deleteFile.run(filePath);
|
|
748
|
+
if (result.changes > 0) {
|
|
749
|
+
stats.filesDeleted += 1;
|
|
750
|
+
stats.functionsDeleted += count;
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
for (const file of files) this.#replaceFile(file, stats);
|
|
754
|
+
};
|
|
755
|
+
applyStage(options.files, options.deletePaths);
|
|
756
|
+
if (options.workingTree) applyStage(options.workingTree.files, options.workingTree.deletePaths);
|
|
662
757
|
const generation = this.getGeneration() + 1;
|
|
663
758
|
this.#setMetadata("generation", String(generation));
|
|
664
|
-
if (options.checkpoint) this.#
|
|
759
|
+
if (options.checkpoint === null) this.#deleteMetadata("git_checkpoint");
|
|
760
|
+
else if (options.checkpoint !== void 0) this.#setMetadata("git_checkpoint", options.checkpoint);
|
|
665
761
|
return stats;
|
|
666
762
|
});
|
|
667
763
|
}
|
|
@@ -671,6 +767,12 @@ var IndexDatabase = class {
|
|
|
671
767
|
const existingFile = this.#db.prepare("SELECT previous_path FROM files WHERE path = ?").get(sourcePath);
|
|
672
768
|
const oldMatches = reconcileFunctions(file.callables, oldRows);
|
|
673
769
|
if (sourcePath !== file.path) {
|
|
770
|
+
const displacedRows = this.#rowsForPath(file.path);
|
|
771
|
+
const displaced = this.#db.prepare("DELETE FROM files WHERE path = ?").run(file.path);
|
|
772
|
+
if (displaced.changes > 0) {
|
|
773
|
+
stats.filesDeleted += 1;
|
|
774
|
+
stats.functionsDeleted += displacedRows.length;
|
|
775
|
+
}
|
|
674
776
|
this.#db.prepare("DELETE FROM files WHERE path = ?").run(sourcePath);
|
|
675
777
|
} else {
|
|
676
778
|
this.#db.prepare("DELETE FROM files WHERE path = ?").run(file.path);
|
|
@@ -702,13 +804,16 @@ var IndexDatabase = class {
|
|
|
702
804
|
if (!embedding) throw new CodeIndexError(`Missing embedding for ${callable.qualifiedName}.`);
|
|
703
805
|
const old = oldMatches.get(callable);
|
|
704
806
|
if (old) usedIds.add(old.id);
|
|
705
|
-
const
|
|
807
|
+
const remembered = this.#db.prepare(`
|
|
808
|
+
SELECT first_seen_commit FROM callable_provenance WHERE identity_key = ? AND source_hash = ?
|
|
809
|
+
`).get(callable.identityKey, callable.sourceHash);
|
|
810
|
+
const firstSeenCommit = old?.first_seen_commit ?? remembered?.first_seen_commit ?? file.indexedCommit;
|
|
706
811
|
this.#db.prepare(`
|
|
707
812
|
INSERT INTO functions(
|
|
708
813
|
id, path, language, kind, name, qualified_name, signature, identity_key,
|
|
709
|
-
start_line, start_column, end_line, end_column, source, source_hash,
|
|
814
|
+
start_line, start_column, end_line, end_column, line_count, source, source_hash,
|
|
710
815
|
embedding_input, first_seen_commit, last_seen_commit, source_mode, embedding_id
|
|
711
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
816
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
712
817
|
`).run(
|
|
713
818
|
old?.id ?? null,
|
|
714
819
|
file.path,
|
|
@@ -722,6 +827,7 @@ var IndexDatabase = class {
|
|
|
722
827
|
callable.startColumn,
|
|
723
828
|
callable.endLine,
|
|
724
829
|
callable.endColumn,
|
|
830
|
+
callable.lineCount,
|
|
725
831
|
callable.source,
|
|
726
832
|
callable.sourceHash,
|
|
727
833
|
callable.embeddingInput,
|
|
@@ -732,6 +838,11 @@ var IndexDatabase = class {
|
|
|
732
838
|
);
|
|
733
839
|
if (old) stats.functionsUpdated += 1;
|
|
734
840
|
else stats.functionsAdded += 1;
|
|
841
|
+
if (firstSeenCommit) {
|
|
842
|
+
this.#db.prepare(`
|
|
843
|
+
INSERT OR IGNORE INTO callable_provenance(identity_key, source_hash, first_seen_commit) VALUES (?, ?, ?)
|
|
844
|
+
`).run(callable.identityKey, callable.sourceHash, firstSeenCommit);
|
|
845
|
+
}
|
|
735
846
|
}
|
|
736
847
|
stats.functionsDeleted += oldRows.length - usedIds.size;
|
|
737
848
|
}
|
|
@@ -754,19 +865,28 @@ var IndexDatabase = class {
|
|
|
754
865
|
return results;
|
|
755
866
|
}
|
|
756
867
|
searchVector(vector, options) {
|
|
868
|
+
const excludePaths = options.excludePaths ?? [];
|
|
869
|
+
const pathFilter = excludePaths.length > 0 ? `AND f.path NOT IN (${excludePaths.map(() => "?").join(", ")})` : "";
|
|
757
870
|
const rows = this.#db.prepare(`
|
|
758
871
|
SELECT f.*, 1.0 - vec_distance_cosine(e.vector, ?) AS similarity
|
|
759
872
|
FROM functions f
|
|
760
873
|
JOIN embeddings e ON e.id = f.embedding_id
|
|
761
874
|
WHERE (? IS NULL OR f.id != ?)
|
|
875
|
+
${pathFilter}
|
|
876
|
+
AND f.line_count >= ?
|
|
877
|
+
AND (? IS NULL OR slopdex_regexp(?, f.qualified_name))
|
|
762
878
|
AND (1.0 - vec_distance_cosine(e.vector, ?)) >= ?
|
|
763
|
-
AND (? IS NULL OR (1.0 - vec_distance_cosine(e.vector, ?))
|
|
879
|
+
AND (? IS NULL OR (1.0 - vec_distance_cosine(e.vector, ?)) < ?)
|
|
764
880
|
ORDER BY similarity DESC, f.id ASC
|
|
765
881
|
LIMIT ?
|
|
766
882
|
`).all(
|
|
767
883
|
vectorBuffer(vector),
|
|
768
884
|
options.excludeId ?? null,
|
|
769
885
|
options.excludeId ?? null,
|
|
886
|
+
...excludePaths,
|
|
887
|
+
options.minLines ?? 1,
|
|
888
|
+
options.nameRegex ?? null,
|
|
889
|
+
options.nameRegex ?? null,
|
|
770
890
|
vectorBuffer(vector),
|
|
771
891
|
options.minSimilarity,
|
|
772
892
|
options.maxSimilarity ?? null,
|
|
@@ -815,6 +935,7 @@ function toIndexedFunction(row) {
|
|
|
815
935
|
startColumn: row.start_column,
|
|
816
936
|
endLine: row.end_line,
|
|
817
937
|
endColumn: row.end_column,
|
|
938
|
+
lineCount: row.line_count,
|
|
818
939
|
source: row.source,
|
|
819
940
|
sourceHash: row.source_hash,
|
|
820
941
|
embeddingInput: row.embedding_input,
|
|
@@ -909,7 +1030,7 @@ var CodeIndex = class {
|
|
|
909
1030
|
throwIfAborted(options.signal);
|
|
910
1031
|
if (!this.#policy.includes(relativePath)) throw new CodeIndexError(`Unsupported or excluded source file: ${relativePath}`);
|
|
911
1032
|
const absolutePath = path6.join(this.rootDir, relativePath);
|
|
912
|
-
const info = await
|
|
1033
|
+
const info = await lstat2(absolutePath);
|
|
913
1034
|
if (info.isSymbolicLink()) throw new CodeIndexError(`Symbolic links are not supported: ${relativePath}`);
|
|
914
1035
|
if (!info.isFile()) throw new CodeIndexError(`Not a regular file: ${relativePath}`);
|
|
915
1036
|
if (info.size > this.#maxFileSize) throw new CodeIndexError(`File exceeds maxFileSize: ${relativePath}`);
|
|
@@ -930,50 +1051,210 @@ var CodeIndex = class {
|
|
|
930
1051
|
deletePaths: [...deletePaths, ...renameMap.values()]
|
|
931
1052
|
});
|
|
932
1053
|
}
|
|
1054
|
+
async updateFromWorkingTree(options = {}) {
|
|
1055
|
+
throwIfAborted(options.signal);
|
|
1056
|
+
const generation = this.#database.getGeneration();
|
|
1057
|
+
const indexedFiles = this.#database.getFileStates();
|
|
1058
|
+
const sourcePaths = await this.#workingTreeSourcePaths(options.signal);
|
|
1059
|
+
const prepared = [];
|
|
1060
|
+
const skippedPaths = /* @__PURE__ */ new Set();
|
|
1061
|
+
for (const relativePath of sourcePaths) {
|
|
1062
|
+
throwIfAborted(options.signal);
|
|
1063
|
+
const absolutePath = path6.join(this.rootDir, relativePath);
|
|
1064
|
+
const info = await lstat2(absolutePath);
|
|
1065
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize) {
|
|
1066
|
+
skippedPaths.add(relativePath);
|
|
1067
|
+
continue;
|
|
1068
|
+
}
|
|
1069
|
+
const content = await readFile(absolutePath);
|
|
1070
|
+
prepared.push(this.#prepareFile(relativePath, content, {
|
|
1071
|
+
blobOid: null,
|
|
1072
|
+
sourceMode: "working-tree",
|
|
1073
|
+
indexedCommit: null
|
|
1074
|
+
}));
|
|
1075
|
+
}
|
|
1076
|
+
await this.#attachEmbeddings(prepared, options.signal);
|
|
1077
|
+
const pathsAgain = await this.#workingTreeSourcePaths(options.signal);
|
|
1078
|
+
if (JSON.stringify(pathsAgain) !== JSON.stringify(sourcePaths)) {
|
|
1079
|
+
throw new CodeIndexError("Working-tree files changed while indexing; retry the update.");
|
|
1080
|
+
}
|
|
1081
|
+
for (const file of prepared) {
|
|
1082
|
+
try {
|
|
1083
|
+
const info = await lstat2(path6.join(this.rootDir, file.path));
|
|
1084
|
+
const content = await readFile(path6.join(this.rootDir, file.path));
|
|
1085
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize || sha256(content.toString("utf8")) !== file.contentHash) {
|
|
1086
|
+
throw new CodeIndexError(`Working-tree file changed while indexing: ${file.path}`);
|
|
1087
|
+
}
|
|
1088
|
+
} catch (error) {
|
|
1089
|
+
if (error instanceof CodeIndexError) throw error;
|
|
1090
|
+
throw new CodeIndexError(`Working-tree file changed while indexing: ${file.path}`, { cause: error });
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
for (const relativePath of skippedPaths) {
|
|
1094
|
+
const info = await lstat2(path6.join(this.rootDir, relativePath));
|
|
1095
|
+
if (info.isFile() && !info.isSymbolicLink() && info.size <= this.#maxFileSize) {
|
|
1096
|
+
throw new CodeIndexError(`Working-tree file changed while indexing: ${relativePath}`);
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
return this.#database.applyUpdate({
|
|
1100
|
+
files: prepared,
|
|
1101
|
+
deletePaths: indexedFiles.map((file) => file.path),
|
|
1102
|
+
checkpoint: null,
|
|
1103
|
+
expectedGeneration: generation
|
|
1104
|
+
});
|
|
1105
|
+
}
|
|
933
1106
|
async updateFromGit(options = {}) {
|
|
934
1107
|
throwIfAborted(options.signal);
|
|
935
1108
|
await this.#git.assertRepository();
|
|
936
1109
|
const relativeIndexPath = path6.relative(this.rootDir, this.indexPath).replaceAll(path6.sep, "/");
|
|
937
1110
|
const indexArtifacts = relativeIndexPath && !relativeIndexPath.startsWith("../") ? [relativeIndexPath, `${relativeIndexPath}-shm`, `${relativeIndexPath}-wal`, `${relativeIndexPath}-journal`] : [];
|
|
938
|
-
await this.#git.assertCleanWorkingTree(indexArtifacts);
|
|
939
1111
|
const target = await this.#git.resolveCommit(options.target ?? "HEAD");
|
|
1112
|
+
const head = await this.#git.resolveCommit("HEAD");
|
|
1113
|
+
const overlayWorkingTree = target === head && options.includeWorkingTree !== false;
|
|
940
1114
|
const checkpoint = this.#database.getCheckpoint();
|
|
941
|
-
|
|
1115
|
+
const generation = this.#database.getGeneration();
|
|
1116
|
+
let reconcileAll = checkpoint === null;
|
|
942
1117
|
if (checkpoint && !await this.#git.isAncestor(checkpoint, target)) {
|
|
943
1118
|
if (!options.rebuildOnDivergence) throw new GitDivergenceError(checkpoint, target);
|
|
944
|
-
|
|
1119
|
+
reconcileAll = true;
|
|
945
1120
|
}
|
|
946
1121
|
const tree = await this.#git.listTree(target);
|
|
947
|
-
const
|
|
948
|
-
const
|
|
1122
|
+
const indexedFiles = this.#database.getFileStates();
|
|
1123
|
+
const indexedByPath = new Map(indexedFiles.map((file) => [file.path, file]));
|
|
1124
|
+
const workingFiles = indexedFiles.filter((file) => file.sourceMode === "working-tree");
|
|
1125
|
+
const workingChanges = overlayWorkingTree ? await this.#git.workTreeChanges(target, indexArtifacts) : [];
|
|
1126
|
+
const visibleWorkingPaths = new Set(
|
|
1127
|
+
workingChanges.filter((change) => change.status !== "D").map((change) => change.path)
|
|
1128
|
+
);
|
|
1129
|
+
const changes = reconcileAll || !checkpoint ? [] : await this.#git.diff(checkpoint, target);
|
|
949
1130
|
const upserts = /* @__PURE__ */ new Map();
|
|
950
1131
|
const deletes = /* @__PURE__ */ new Set();
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
1132
|
+
const eligibleEntries = [...tree.values()].filter((entry) => this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize);
|
|
1133
|
+
const eligibleTargetPaths = new Set(eligibleEntries.map((entry) => entry.path));
|
|
1134
|
+
if (reconcileAll) {
|
|
1135
|
+
const renameCandidates = indexedFiles.filter((file) => file.sourceMode === "git" && file.blobOid && !eligibleTargetPaths.has(file.path));
|
|
1136
|
+
const claimedRenameSources = /* @__PURE__ */ new Set();
|
|
1137
|
+
for (const entry of eligibleEntries) {
|
|
1138
|
+
const indexed = indexedByPath.get(entry.path);
|
|
1139
|
+
if (indexed?.sourceMode === "git" && indexed.blobOid === entry.oid) continue;
|
|
1140
|
+
const matches = renameCandidates.filter((candidate) => candidate.blobOid === entry.oid && !claimedRenameSources.has(candidate.path));
|
|
1141
|
+
const previousPath = matches.length === 1 ? matches[0].path : void 0;
|
|
1142
|
+
if (previousPath) claimedRenameSources.add(previousPath);
|
|
1143
|
+
upserts.set(entry.path, { entry, ...previousPath ? { previousPath } : {} });
|
|
1144
|
+
}
|
|
1145
|
+
for (const indexed of indexedFiles) {
|
|
1146
|
+
if (!eligibleTargetPaths.has(indexed.path)) deletes.add(indexed.path);
|
|
955
1147
|
}
|
|
956
1148
|
} else {
|
|
957
1149
|
this.#classifyGitChanges(changes, tree, upserts, deletes);
|
|
958
|
-
|
|
959
|
-
const entry = tree.get(dirtyPath);
|
|
960
|
-
if (entry && this.#policy.includes(dirtyPath) && entry.size <= this.#maxFileSize) upserts.set(dirtyPath, { entry });
|
|
961
|
-
else deletes.add(dirtyPath);
|
|
962
|
-
}
|
|
963
|
-
const eligibleTargetPaths = new Set(
|
|
964
|
-
[...tree.values()].filter((entry) => this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize).map((entry) => entry.path)
|
|
965
|
-
);
|
|
966
|
-
const indexedPaths = new Set(this.#database.allFilePaths());
|
|
1150
|
+
const indexedPaths = new Set(indexedByPath.keys());
|
|
967
1151
|
for (const targetPath of eligibleTargetPaths) {
|
|
968
|
-
|
|
969
|
-
|
|
1152
|
+
const indexed = indexedByPath.get(targetPath);
|
|
1153
|
+
const entry = tree.get(targetPath);
|
|
1154
|
+
if ((!indexed || indexed.sourceMode === "git" && indexed.blobOid !== entry.oid) && !upserts.has(targetPath)) {
|
|
1155
|
+
upserts.set(targetPath, { entry });
|
|
970
1156
|
}
|
|
971
1157
|
}
|
|
972
1158
|
for (const indexedPath of indexedPaths) {
|
|
973
1159
|
if (!eligibleTargetPaths.has(indexedPath)) deletes.add(indexedPath);
|
|
974
1160
|
}
|
|
975
1161
|
}
|
|
976
|
-
|
|
1162
|
+
for (const dirtyFile of workingFiles) {
|
|
1163
|
+
const entry = tree.get(dirtyFile.path);
|
|
1164
|
+
const previousEntry = dirtyFile.previousPath ? tree.get(dirtyFile.previousPath) : void 0;
|
|
1165
|
+
if (previousEntry && this.#policy.includes(previousEntry.path) && previousEntry.size <= this.#maxFileSize && (entry !== void 0 || !visibleWorkingPaths.has(dirtyFile.path))) {
|
|
1166
|
+
const currentPlan = entry ? upserts.get(dirtyFile.path) : void 0;
|
|
1167
|
+
upserts.delete(dirtyFile.path);
|
|
1168
|
+
upserts.delete(previousEntry.path);
|
|
1169
|
+
upserts.set(previousEntry.path, { entry: previousEntry, previousPath: dirtyFile.path });
|
|
1170
|
+
if (entry && this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize) {
|
|
1171
|
+
upserts.set(dirtyFile.path, currentPlan ?? { entry });
|
|
1172
|
+
}
|
|
1173
|
+
continue;
|
|
1174
|
+
}
|
|
1175
|
+
if (entry && this.#policy.includes(dirtyFile.path) && entry.size <= this.#maxFileSize) {
|
|
1176
|
+
const planned = upserts.get(dirtyFile.path);
|
|
1177
|
+
if (planned?.previousPath && !indexedByPath.has(planned.previousPath)) {
|
|
1178
|
+
upserts.set(dirtyFile.path, { entry, previousPath: dirtyFile.path });
|
|
1179
|
+
} else if (!planned) {
|
|
1180
|
+
upserts.set(dirtyFile.path, { entry });
|
|
1181
|
+
}
|
|
1182
|
+
continue;
|
|
1183
|
+
}
|
|
1184
|
+
if (previousEntry && this.#policy.includes(previousEntry.path) && previousEntry.size <= this.#maxFileSize) {
|
|
1185
|
+
upserts.set(previousEntry.path, { entry: previousEntry, previousPath: dirtyFile.path });
|
|
1186
|
+
} else {
|
|
1187
|
+
deletes.add(dirtyFile.path);
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
for (const [filePath, { entry, previousPath }] of upserts) {
|
|
1191
|
+
const indexed = indexedByPath.get(filePath);
|
|
1192
|
+
if (!previousPath && indexed?.sourceMode === "git" && indexed.blobOid === entry.oid) upserts.delete(filePath);
|
|
1193
|
+
}
|
|
1194
|
+
const workingPrepared = [];
|
|
1195
|
+
const workingDeletes = /* @__PURE__ */ new Set();
|
|
1196
|
+
const skippedWorkingPaths = /* @__PURE__ */ new Set();
|
|
1197
|
+
for (const change of workingChanges) {
|
|
1198
|
+
throwIfAborted(options.signal);
|
|
1199
|
+
if (change.status === "D") {
|
|
1200
|
+
upserts.delete(change.path);
|
|
1201
|
+
workingDeletes.add(change.path);
|
|
1202
|
+
continue;
|
|
1203
|
+
}
|
|
1204
|
+
let previousPath;
|
|
1205
|
+
let replacePath;
|
|
1206
|
+
if (change.status === "R") {
|
|
1207
|
+
const plannedBase = upserts.get(change.oldPath);
|
|
1208
|
+
const existingAtTarget = indexedByPath.get(change.oldPath);
|
|
1209
|
+
const existingAtFinal = indexedByPath.get(change.path);
|
|
1210
|
+
const continuingRename = existingAtFinal?.sourceMode === "working-tree" && existingAtFinal.previousPath === change.oldPath;
|
|
1211
|
+
replacePath = continuingRename ? change.path : existingAtTarget ? change.oldPath : existingAtFinal ? change.path : plannedBase?.previousPath && indexedByPath.has(plannedBase.previousPath) ? plannedBase.previousPath : void 0;
|
|
1212
|
+
previousPath = existingAtFinal?.previousPath ?? change.oldPath;
|
|
1213
|
+
upserts.delete(change.oldPath);
|
|
1214
|
+
upserts.delete(change.path);
|
|
1215
|
+
if (replacePath) deletes.delete(replacePath);
|
|
1216
|
+
workingDeletes.add(change.oldPath);
|
|
1217
|
+
} else {
|
|
1218
|
+
const planned = upserts.get(change.path);
|
|
1219
|
+
const existing = indexedByPath.get(change.path);
|
|
1220
|
+
replacePath = existing ? change.path : planned?.previousPath && indexedByPath.has(planned.previousPath) ? planned.previousPath : void 0;
|
|
1221
|
+
previousPath = existing?.previousPath ?? planned?.previousPath;
|
|
1222
|
+
upserts.delete(change.path);
|
|
1223
|
+
if (replacePath) deletes.delete(replacePath);
|
|
1224
|
+
}
|
|
1225
|
+
const relativePath = change.path;
|
|
1226
|
+
if (!this.#policy.includes(relativePath)) {
|
|
1227
|
+
if (replacePath) workingDeletes.add(replacePath);
|
|
1228
|
+
workingDeletes.add(relativePath);
|
|
1229
|
+
continue;
|
|
1230
|
+
}
|
|
1231
|
+
const absolutePath = path6.join(this.rootDir, relativePath);
|
|
1232
|
+
const info = await lstat2(absolutePath);
|
|
1233
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize) {
|
|
1234
|
+
if (replacePath) workingDeletes.add(replacePath);
|
|
1235
|
+
workingDeletes.add(relativePath);
|
|
1236
|
+
skippedWorkingPaths.add(relativePath);
|
|
1237
|
+
continue;
|
|
1238
|
+
}
|
|
1239
|
+
const content = await readFile(absolutePath);
|
|
1240
|
+
if (!replacePath && change.status === "A") {
|
|
1241
|
+
const contentHash = sha256(content.toString("utf8"));
|
|
1242
|
+
const renameCandidates = workingFiles.filter((file) => file.path !== relativePath && !visibleWorkingPaths.has(file.path) && deletes.has(file.path) && file.contentHash === contentHash);
|
|
1243
|
+
const renamed = renameCandidates.length === 1 ? renameCandidates[0] : void 0;
|
|
1244
|
+
if (renamed) {
|
|
1245
|
+
replacePath = renamed.path;
|
|
1246
|
+
previousPath = renamed.previousPath ?? renamed.path;
|
|
1247
|
+
deletes.delete(renamed.path);
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
workingPrepared.push(this.#prepareFile(relativePath, content, {
|
|
1251
|
+
blobOid: null,
|
|
1252
|
+
sourceMode: "working-tree",
|
|
1253
|
+
indexedCommit: null,
|
|
1254
|
+
...previousPath ? { previousPath } : {},
|
|
1255
|
+
...replacePath ? { replacePath } : {}
|
|
1256
|
+
}));
|
|
1257
|
+
}
|
|
977
1258
|
const prepared = [];
|
|
978
1259
|
for (const { entry, previousPath } of upserts.values()) {
|
|
979
1260
|
throwIfAborted(options.signal);
|
|
@@ -989,14 +1270,46 @@ var CodeIndex = class {
|
|
|
989
1270
|
...previousPath ? { replacePath: previousPath } : {}
|
|
990
1271
|
}));
|
|
991
1272
|
}
|
|
992
|
-
await this.#attachEmbeddings(prepared, options.signal);
|
|
1273
|
+
await this.#attachEmbeddings([...prepared, ...workingPrepared], options.signal);
|
|
1274
|
+
const noChanges = !reconcileAll && checkpoint === target && prepared.length === 0 && deletes.size === 0 && workingPrepared.length === 0 && workingDeletes.size === 0;
|
|
993
1275
|
const resolvedAgain = await this.#git.resolveCommit(options.target ?? "HEAD");
|
|
994
1276
|
if (resolvedAgain !== target) throw new CodeIndexError("Target Git ref changed while indexing; retry the update.");
|
|
1277
|
+
if (overlayWorkingTree) {
|
|
1278
|
+
const headAgain = await this.#git.resolveCommit("HEAD");
|
|
1279
|
+
const workingChangesAgain = await this.#git.workTreeChanges(target, indexArtifacts);
|
|
1280
|
+
if (headAgain !== head || JSON.stringify(workingChangesAgain) !== JSON.stringify(workingChanges)) {
|
|
1281
|
+
throw new CodeIndexError("Git HEAD or working-tree changes changed while indexing; retry the update.");
|
|
1282
|
+
}
|
|
1283
|
+
for (const file of workingPrepared) {
|
|
1284
|
+
const info = await lstat2(path6.join(this.rootDir, file.path));
|
|
1285
|
+
const content = await readFile(path6.join(this.rootDir, file.path));
|
|
1286
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize || sha256(content.toString("utf8")) !== file.contentHash) {
|
|
1287
|
+
throw new CodeIndexError(`Working-tree file changed while indexing: ${file.path}`);
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
for (const relativePath of skippedWorkingPaths) {
|
|
1291
|
+
const info = await lstat2(path6.join(this.rootDir, relativePath));
|
|
1292
|
+
if (info.isFile() && !info.isSymbolicLink() && info.size <= this.#maxFileSize) {
|
|
1293
|
+
throw new CodeIndexError(`Working-tree file changed while indexing: ${relativePath}`);
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
if (noChanges) {
|
|
1298
|
+
if (this.#database.getCheckpoint() !== checkpoint || this.#database.getGeneration() !== generation) {
|
|
1299
|
+
throw new CodeIndexError("Index changed while the update was being prepared; retry the update.");
|
|
1300
|
+
}
|
|
1301
|
+
return emptyStats(checkpoint);
|
|
1302
|
+
}
|
|
995
1303
|
return this.#database.applyUpdate({
|
|
996
1304
|
files: prepared,
|
|
997
1305
|
deletePaths: [...deletes],
|
|
1306
|
+
workingTree: {
|
|
1307
|
+
files: workingPrepared,
|
|
1308
|
+
deletePaths: [...workingDeletes]
|
|
1309
|
+
},
|
|
998
1310
|
checkpoint: target,
|
|
999
|
-
expectedCheckpoint: checkpoint
|
|
1311
|
+
expectedCheckpoint: checkpoint,
|
|
1312
|
+
expectedGeneration: generation
|
|
1000
1313
|
});
|
|
1001
1314
|
}
|
|
1002
1315
|
#classifyGitChanges(changes, tree, upserts, deletes) {
|
|
@@ -1018,6 +1331,28 @@ var CodeIndex = class {
|
|
|
1018
1331
|
else deletes.add(change.path);
|
|
1019
1332
|
}
|
|
1020
1333
|
}
|
|
1334
|
+
async #workingTreeSourcePaths(signal) {
|
|
1335
|
+
const relativeIndexPath = path6.relative(this.rootDir, this.indexPath).replaceAll(path6.sep, "/");
|
|
1336
|
+
const indexArtifacts = new Set(relativeIndexPath && !relativeIndexPath.startsWith("../") ? [relativeIndexPath, `${relativeIndexPath}-shm`, `${relativeIndexPath}-wal`, `${relativeIndexPath}-journal`] : []);
|
|
1337
|
+
const directories = [""];
|
|
1338
|
+
const sourcePaths = [];
|
|
1339
|
+
while (directories.length > 0) {
|
|
1340
|
+
throwIfAborted(signal);
|
|
1341
|
+
const relativeDirectory = directories.shift();
|
|
1342
|
+
const entries = await readdir(path6.join(this.rootDir, relativeDirectory), { withFileTypes: true });
|
|
1343
|
+
entries.sort((left, right) => left.name.localeCompare(right.name));
|
|
1344
|
+
for (const entry of entries) {
|
|
1345
|
+
const relativePath = relativeDirectory ? `${relativeDirectory.replaceAll(path6.sep, "/")}/${entry.name}` : entry.name;
|
|
1346
|
+
if (entry.isDirectory()) {
|
|
1347
|
+
if (this.#policy.traversesDirectory(relativePath)) directories.push(relativePath);
|
|
1348
|
+
} else if (entry.isFile() && !indexArtifacts.has(relativePath) && this.#policy.includes(relativePath)) {
|
|
1349
|
+
sourcePaths.push(relativePath);
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
sourcePaths.sort();
|
|
1354
|
+
return sourcePaths;
|
|
1355
|
+
}
|
|
1021
1356
|
#prepareFile(relativePath, buffer, provenance) {
|
|
1022
1357
|
const content = buffer.toString("utf8");
|
|
1023
1358
|
const callables = parseCallables(relativePath, content, this.#onWarning);
|
|
@@ -1076,7 +1411,10 @@ var CodeIndex = class {
|
|
|
1076
1411
|
limit: options.limit,
|
|
1077
1412
|
minSimilarity: options.minSimilarity,
|
|
1078
1413
|
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
|
|
1079
|
-
excludeId: functionId
|
|
1414
|
+
excludeId: functionId,
|
|
1415
|
+
...options.excludePaths !== void 0 ? { excludePaths: options.excludePaths } : {},
|
|
1416
|
+
...options.minLines !== void 0 ? { minLines: options.minLines } : {},
|
|
1417
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {}
|
|
1080
1418
|
});
|
|
1081
1419
|
}
|
|
1082
1420
|
vectorForFunction(functionId) {
|
|
@@ -1086,15 +1424,15 @@ var CodeIndex = class {
|
|
|
1086
1424
|
return this.#database.searchVector(normalizeEmbeddingVector(vector, this.provider.profile.dimensions), options);
|
|
1087
1425
|
}
|
|
1088
1426
|
async sourceFunctions(filter) {
|
|
1089
|
-
const functions = filter.type === "all" ? this.allFunctions() : await this.#
|
|
1427
|
+
const functions = filter.type === "all" ? this.allFunctions() : filter.type === "uncommitted" ? this.allFunctions().filter((callable) => callable.sourceMode === "working-tree") : await this.#functionsChangedSince(filter.commit);
|
|
1090
1428
|
if (!filter.path) return functions;
|
|
1091
1429
|
const sourcePath = normalizeRelativePath(this.rootDir, filter.path);
|
|
1092
1430
|
return functions.filter((callable) => callable.path === sourcePath || callable.path.startsWith(`${sourcePath}/`));
|
|
1093
1431
|
}
|
|
1094
|
-
async #
|
|
1432
|
+
async #functionsChangedSince(baseRef) {
|
|
1095
1433
|
await this.#git.assertRepository();
|
|
1096
1434
|
const checkpoint = this.#database.getCheckpoint();
|
|
1097
|
-
if (!checkpoint) throw new CodeIndexError("
|
|
1435
|
+
if (!checkpoint) throw new CodeIndexError("changed-since requires an index with a Git checkpoint.");
|
|
1098
1436
|
const base = await this.#git.resolveCommit(baseRef);
|
|
1099
1437
|
if (!await this.#git.isAncestor(base, checkpoint)) {
|
|
1100
1438
|
throw new GitDivergenceError(base, checkpoint);
|
|
@@ -1104,18 +1442,17 @@ var CodeIndex = class {
|
|
|
1104
1442
|
const comparisonPaths = /* @__PURE__ */ new Map();
|
|
1105
1443
|
for (const change of changes) {
|
|
1106
1444
|
if (change.status === "D") continue;
|
|
1107
|
-
if (change.status === "R") comparisonPaths.set(change.path, change.oldPath);
|
|
1108
|
-
else if (change.status === "A" || change.status === "C") comparisonPaths.set(change.path, null);
|
|
1109
|
-
else comparisonPaths.set(change.path, change.path);
|
|
1445
|
+
if (change.status === "R") comparisonPaths.set(change.path, { basePath: change.oldPath, pathChanged: true });
|
|
1446
|
+
else if (change.status === "A" || change.status === "C") comparisonPaths.set(change.path, { basePath: null, pathChanged: false });
|
|
1447
|
+
else comparisonPaths.set(change.path, { basePath: change.path, pathChanged: false });
|
|
1110
1448
|
}
|
|
1111
1449
|
for (const dirtyFile of this.#database.getWorkingTreeFiles()) {
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
}
|
|
1450
|
+
const basePath = dirtyFile.previousPath ?? comparisonPaths.get(dirtyFile.path)?.basePath ?? dirtyFile.path;
|
|
1451
|
+
comparisonPaths.set(dirtyFile.path, { basePath, pathChanged: basePath !== dirtyFile.path });
|
|
1115
1452
|
}
|
|
1116
1453
|
const current = this.#database.functionsForPaths([...comparisonPaths.keys()]);
|
|
1117
1454
|
const baseFunctionsByCurrentPath = /* @__PURE__ */ new Map();
|
|
1118
|
-
for (const [currentPath, basePath] of comparisonPaths) {
|
|
1455
|
+
for (const [currentPath, { basePath }] of comparisonPaths) {
|
|
1119
1456
|
if (!basePath) {
|
|
1120
1457
|
baseFunctionsByCurrentPath.set(currentPath, []);
|
|
1121
1458
|
continue;
|
|
@@ -1129,19 +1466,25 @@ var CodeIndex = class {
|
|
|
1129
1466
|
baseFunctionsByCurrentPath.set(currentPath, parseCallables(basePath, content, this.#onWarning));
|
|
1130
1467
|
}
|
|
1131
1468
|
const currentByPath = groupBy(current, (callable) => callable.path);
|
|
1132
|
-
const
|
|
1469
|
+
const changedIds = /* @__PURE__ */ new Set();
|
|
1133
1470
|
for (const [currentPath, currentFunctions] of currentByPath) {
|
|
1134
|
-
|
|
1135
|
-
|
|
1471
|
+
const comparison = comparisonPaths.get(currentPath);
|
|
1472
|
+
for (const id of changedFunctionIds(
|
|
1473
|
+
currentFunctions,
|
|
1474
|
+
baseFunctionsByCurrentPath.get(currentPath) ?? [],
|
|
1475
|
+
comparison.pathChanged
|
|
1476
|
+
)) {
|
|
1477
|
+
changedIds.add(id);
|
|
1136
1478
|
}
|
|
1137
1479
|
}
|
|
1138
|
-
return current.filter((callable) =>
|
|
1480
|
+
return current.filter((callable) => changedIds.has(callable.id));
|
|
1139
1481
|
}
|
|
1140
1482
|
};
|
|
1141
|
-
function
|
|
1483
|
+
function changedFunctionIds(current, base, pathChanged) {
|
|
1484
|
+
if (pathChanged) return current.map((callable) => callable.id);
|
|
1142
1485
|
const currentGroups = groupBy(current, (callable) => `${callable.qualifiedName}\0${callable.kind}`);
|
|
1143
1486
|
const baseGroups = groupBy(base, (callable) => `${callable.qualifiedName}\0${callable.kind}`);
|
|
1144
|
-
const
|
|
1487
|
+
const changed = [];
|
|
1145
1488
|
for (const [key, currentGroup] of currentGroups) {
|
|
1146
1489
|
const baseGroup = baseGroups.get(key) ?? [];
|
|
1147
1490
|
const unmatchedBase = new Set(baseGroup.map((_, index) => index));
|
|
@@ -1155,13 +1498,9 @@ function addedFunctionIds(current, base) {
|
|
|
1155
1498
|
unmatchedCurrent.delete(currentIndex);
|
|
1156
1499
|
}
|
|
1157
1500
|
}
|
|
1158
|
-
for (const currentIndex of unmatchedCurrent)
|
|
1159
|
-
const baseIndex = unmatchedBase.values().next().value;
|
|
1160
|
-
if (baseIndex === void 0) added.push(currentGroup[currentIndex].id);
|
|
1161
|
-
else unmatchedBase.delete(baseIndex);
|
|
1162
|
-
}
|
|
1501
|
+
for (const currentIndex of unmatchedCurrent) changed.push(currentGroup[currentIndex].id);
|
|
1163
1502
|
}
|
|
1164
|
-
return
|
|
1503
|
+
return changed;
|
|
1165
1504
|
}
|
|
1166
1505
|
function normalizeProfile(profile) {
|
|
1167
1506
|
return {
|
|
@@ -1203,7 +1542,280 @@ function groupBy(values, keyFor) {
|
|
|
1203
1542
|
return groups;
|
|
1204
1543
|
}
|
|
1205
1544
|
|
|
1545
|
+
// src/analysis/cohesion.ts
|
|
1546
|
+
import path7 from "node:path";
|
|
1547
|
+
async function analyzeCohesion(options) {
|
|
1548
|
+
const neighbors = options.neighbors ?? 20;
|
|
1549
|
+
const limit = options.limit ?? 50;
|
|
1550
|
+
const minSimilarity = options.minSimilarity ?? 0.8;
|
|
1551
|
+
const minLines = options.minLines ?? 2;
|
|
1552
|
+
const sourceFilter = options.sourceFilter ?? { type: "all" };
|
|
1553
|
+
assertPositiveInteger(neighbors, "neighbors");
|
|
1554
|
+
assertPositiveInteger(limit, "limit");
|
|
1555
|
+
assertPositiveInteger(minLines, "minLines");
|
|
1556
|
+
if (!Number.isFinite(minSimilarity) || minSimilarity < -1 || minSimilarity >= 1) {
|
|
1557
|
+
throw new CodeIndexError("minSimilarity must be at least -1 and less than 1.");
|
|
1558
|
+
}
|
|
1559
|
+
if (options.maxSimilarity !== void 0 && (!Number.isFinite(options.maxSimilarity) || options.maxSimilarity <= minSimilarity)) {
|
|
1560
|
+
throw new CodeIndexError("maxSimilarity must be greater than minSimilarity.");
|
|
1561
|
+
}
|
|
1562
|
+
const nameRegex = compileNameRegex(options.nameRegex);
|
|
1563
|
+
const candidateFunctions = options.source.allFunctions().filter((callable) => callable.lineCount >= minLines && (!nameRegex || nameRegex.test(callable.qualifiedName)));
|
|
1564
|
+
const candidateIds = new Set(candidateFunctions.map((callable) => callable.id));
|
|
1565
|
+
const sourceFunctions = (await options.source.sourceFunctions(sourceFilter)).filter((callable) => candidateIds.has(callable.id));
|
|
1566
|
+
const neighborCache = /* @__PURE__ */ new Map();
|
|
1567
|
+
const neighborsFor = (callable) => {
|
|
1568
|
+
throwIfAborted(options.signal);
|
|
1569
|
+
const matches = options.source.similarToFunction(callable.id, {
|
|
1570
|
+
limit: neighbors,
|
|
1571
|
+
minSimilarity,
|
|
1572
|
+
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
|
|
1573
|
+
minLines,
|
|
1574
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {}
|
|
1575
|
+
}).filter((match) => candidateIds.has(match.function.id));
|
|
1576
|
+
neighborCache.set(callable.id, new Set(matches.map((match) => match.function.id)));
|
|
1577
|
+
return matches;
|
|
1578
|
+
};
|
|
1579
|
+
const edges = /* @__PURE__ */ new Map();
|
|
1580
|
+
for (let index = 0; index < sourceFunctions.length; index += 1) {
|
|
1581
|
+
const source = sourceFunctions[index];
|
|
1582
|
+
for (const match of neighborsFor(source)) {
|
|
1583
|
+
const [left, right] = orderedFunctions(source, match.function);
|
|
1584
|
+
const key = pairKey(left.id, right.id);
|
|
1585
|
+
const existing = edges.get(key);
|
|
1586
|
+
if (!existing || match.similarity > existing.similarity) {
|
|
1587
|
+
edges.set(key, { left, right, similarity: match.similarity });
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
options.onProgress?.({ completed: index + 1, total: sourceFunctions.length });
|
|
1591
|
+
}
|
|
1592
|
+
const scoredPairs = [...edges.values()].map((edge) => {
|
|
1593
|
+
const leftNeighbors = neighborCache.get(edge.left.id);
|
|
1594
|
+
const rightNeighbors = neighborCache.get(edge.right.id);
|
|
1595
|
+
const reciprocal = leftNeighbors && rightNeighbors ? leftNeighbors.has(edge.right.id) && rightNeighbors.has(edge.left.id) : null;
|
|
1596
|
+
return scorePair(edge, reciprocal, minSimilarity);
|
|
1597
|
+
}).sort(comparePairs);
|
|
1598
|
+
scoredPairs.forEach((pair, index) => {
|
|
1599
|
+
pair.rank = index + 1;
|
|
1600
|
+
});
|
|
1601
|
+
const repositoryScope = sourceFilter.type === "all" && sourceFilter.path === void 0;
|
|
1602
|
+
const summary = summarize(
|
|
1603
|
+
repositoryScope ? "repository" : "selected-sources",
|
|
1604
|
+
sourceFunctions.length,
|
|
1605
|
+
candidateFunctions.length,
|
|
1606
|
+
scoredPairs
|
|
1607
|
+
);
|
|
1608
|
+
const reportedPairs = scoredPairs.slice(0, limit);
|
|
1609
|
+
const files = summarizeFiles(sourceFunctions, scoredPairs).filter((file) => file.internalAffinity + file.sameFolderAffinity + file.externalAffinity > 0).slice(0, limit);
|
|
1610
|
+
const groups = buildGroups(reportedPairs);
|
|
1611
|
+
const status = options.source.status();
|
|
1612
|
+
return {
|
|
1613
|
+
schemaVersion: 1,
|
|
1614
|
+
repository: {
|
|
1615
|
+
generation: status.generation,
|
|
1616
|
+
gitCheckpoint: status.gitCheckpoint,
|
|
1617
|
+
embeddingProfile: status.embeddingProfile
|
|
1618
|
+
},
|
|
1619
|
+
parameters: {
|
|
1620
|
+
neighbors,
|
|
1621
|
+
limit,
|
|
1622
|
+
minSimilarity,
|
|
1623
|
+
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
|
|
1624
|
+
minLines,
|
|
1625
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {},
|
|
1626
|
+
sourceFilter
|
|
1627
|
+
},
|
|
1628
|
+
summary,
|
|
1629
|
+
pairs: reportedPairs,
|
|
1630
|
+
files,
|
|
1631
|
+
groups
|
|
1632
|
+
};
|
|
1633
|
+
}
|
|
1634
|
+
function cohesionLocation(leftPath, rightPath) {
|
|
1635
|
+
const normalizedLeft = leftPath.replaceAll("\\", "/");
|
|
1636
|
+
const normalizedRight = rightPath.replaceAll("\\", "/");
|
|
1637
|
+
const leftDirectory = directoryParts(normalizedLeft);
|
|
1638
|
+
const rightDirectory = directoryParts(normalizedRight);
|
|
1639
|
+
let commonLength = 0;
|
|
1640
|
+
while (commonLength < leftDirectory.length && commonLength < rightDirectory.length && leftDirectory[commonLength] === rightDirectory[commonLength]) {
|
|
1641
|
+
commonLength += 1;
|
|
1642
|
+
}
|
|
1643
|
+
const commonParts = leftDirectory.slice(0, commonLength);
|
|
1644
|
+
const commonAncestor = commonParts.length > 0 ? commonParts.join("/") : null;
|
|
1645
|
+
const sameFile = normalizedLeft === normalizedRight;
|
|
1646
|
+
const folderHops = sameFile ? 0 : leftDirectory.length - commonLength + rightDirectory.length - commonLength;
|
|
1647
|
+
return {
|
|
1648
|
+
category: sameFile ? "same-file" : folderHops === 0 ? "same-folder" : "different-folder",
|
|
1649
|
+
physicalDistance: sameFile ? 0 : 1 + folderHops,
|
|
1650
|
+
folderHops,
|
|
1651
|
+
commonAncestor,
|
|
1652
|
+
sourceTestPair: isTestPath(normalizedLeft) !== isTestPath(normalizedRight)
|
|
1653
|
+
};
|
|
1654
|
+
}
|
|
1655
|
+
function scorePair(edge, reciprocal, minSimilarity) {
|
|
1656
|
+
const location = cohesionLocation(edge.left.path, edge.right.path);
|
|
1657
|
+
const semanticWeight = clamp((edge.similarity - minSimilarity) / (1 - minSimilarity));
|
|
1658
|
+
const separationWeight = 1 - Math.exp(-location.physicalDistance / 2);
|
|
1659
|
+
return {
|
|
1660
|
+
rank: 0,
|
|
1661
|
+
left: edge.left,
|
|
1662
|
+
right: edge.right,
|
|
1663
|
+
similarity: edge.similarity,
|
|
1664
|
+
reciprocal,
|
|
1665
|
+
semanticWeight,
|
|
1666
|
+
separationWeight,
|
|
1667
|
+
cohesionGap: semanticWeight * separationWeight,
|
|
1668
|
+
location
|
|
1669
|
+
};
|
|
1670
|
+
}
|
|
1671
|
+
function summarize(scope, functionsAnalyzed, candidateFunctions, pairs) {
|
|
1672
|
+
const totalWeight = pairs.reduce((sum, pair) => sum + pair.semanticWeight, 0);
|
|
1673
|
+
const sameFileWeight = pairs.filter((pair) => pair.location.category === "same-file").reduce((sum, pair) => sum + pair.semanticWeight, 0);
|
|
1674
|
+
const sameFolderWeight = pairs.filter((pair) => pair.location.category === "same-folder").reduce((sum, pair) => sum + pair.semanticWeight, 0);
|
|
1675
|
+
const remoteWeight = pairs.filter((pair) => pair.location.category === "different-folder").reduce((sum, pair) => sum + pair.semanticWeight, 0);
|
|
1676
|
+
const weightedDistance = pairs.reduce(
|
|
1677
|
+
(sum, pair) => sum + pair.semanticWeight * pair.location.physicalDistance,
|
|
1678
|
+
0
|
|
1679
|
+
);
|
|
1680
|
+
return {
|
|
1681
|
+
scope,
|
|
1682
|
+
functionsAnalyzed,
|
|
1683
|
+
candidateFunctions,
|
|
1684
|
+
semanticEdges: pairs.length,
|
|
1685
|
+
sameFileRatio: ratio(sameFileWeight, totalWeight),
|
|
1686
|
+
sameFolderRatio: ratio(sameFolderWeight, totalWeight),
|
|
1687
|
+
remoteRatio: ratio(remoteWeight, totalWeight),
|
|
1688
|
+
weightedMeanDistance: ratio(weightedDistance, totalWeight)
|
|
1689
|
+
};
|
|
1690
|
+
}
|
|
1691
|
+
function summarizeFiles(sourceFunctions, pairs) {
|
|
1692
|
+
const sourceIds = new Set(sourceFunctions.map((callable) => callable.id));
|
|
1693
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1694
|
+
for (const callable of sourceFunctions) counts.set(callable.path, (counts.get(callable.path) ?? 0) + 1);
|
|
1695
|
+
const reports = /* @__PURE__ */ new Map();
|
|
1696
|
+
for (const [filePath, functionCount] of counts) {
|
|
1697
|
+
reports.set(filePath, {
|
|
1698
|
+
path: filePath,
|
|
1699
|
+
functionCount,
|
|
1700
|
+
internalAffinity: 0,
|
|
1701
|
+
sameFolderAffinity: 0,
|
|
1702
|
+
externalAffinity: 0,
|
|
1703
|
+
externalAffinityRatio: 0
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1706
|
+
for (const pair of pairs) {
|
|
1707
|
+
if (sourceIds.has(pair.left.id)) addFileAffinity(reports.get(pair.left.path), pair.right, pair);
|
|
1708
|
+
if (sourceIds.has(pair.right.id)) addFileAffinity(reports.get(pair.right.path), pair.left, pair);
|
|
1709
|
+
}
|
|
1710
|
+
for (const report of reports.values()) {
|
|
1711
|
+
const total = report.internalAffinity + report.sameFolderAffinity + report.externalAffinity;
|
|
1712
|
+
report.externalAffinityRatio = ratio(report.externalAffinity, total);
|
|
1713
|
+
}
|
|
1714
|
+
return [...reports.values()].sort((left, right) => right.externalAffinityRatio - left.externalAffinityRatio || right.externalAffinity - left.externalAffinity || left.path.localeCompare(right.path));
|
|
1715
|
+
}
|
|
1716
|
+
function addFileAffinity(report, match, pair) {
|
|
1717
|
+
if (pair.location.category === "same-file") report.internalAffinity += pair.semanticWeight;
|
|
1718
|
+
else if (pair.location.category === "same-folder") report.sameFolderAffinity += pair.semanticWeight;
|
|
1719
|
+
else {
|
|
1720
|
+
report.externalAffinity += pair.semanticWeight;
|
|
1721
|
+
updateStrongestExternal(report, match, pair);
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
function updateStrongestExternal(report, callable, pair) {
|
|
1725
|
+
if (!report.strongestExternalMatch || pair.cohesionGap > report.strongestExternalMatch.cohesionGap) {
|
|
1726
|
+
report.strongestExternalMatch = {
|
|
1727
|
+
function: callable,
|
|
1728
|
+
similarity: pair.similarity,
|
|
1729
|
+
cohesionGap: pair.cohesionGap
|
|
1730
|
+
};
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
function buildGroups(pairs) {
|
|
1734
|
+
const functions = /* @__PURE__ */ new Map();
|
|
1735
|
+
const neighbors = /* @__PURE__ */ new Map();
|
|
1736
|
+
for (const pair of pairs) {
|
|
1737
|
+
functions.set(pair.left.id, pair.left);
|
|
1738
|
+
functions.set(pair.right.id, pair.right);
|
|
1739
|
+
addNeighbor(neighbors, pair.left.id, pair.right.id);
|
|
1740
|
+
addNeighbor(neighbors, pair.right.id, pair.left.id);
|
|
1741
|
+
}
|
|
1742
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1743
|
+
const groups = [];
|
|
1744
|
+
for (const start of neighbors.keys()) {
|
|
1745
|
+
if (seen.has(start)) continue;
|
|
1746
|
+
const pending = [start];
|
|
1747
|
+
const memberIds = /* @__PURE__ */ new Set();
|
|
1748
|
+
while (pending.length > 0) {
|
|
1749
|
+
const id = pending.pop();
|
|
1750
|
+
if (seen.has(id)) continue;
|
|
1751
|
+
seen.add(id);
|
|
1752
|
+
memberIds.add(id);
|
|
1753
|
+
for (const neighbor of neighbors.get(id) ?? []) pending.push(neighbor);
|
|
1754
|
+
}
|
|
1755
|
+
const groupPairs = pairs.filter((pair) => memberIds.has(pair.left.id) && memberIds.has(pair.right.id));
|
|
1756
|
+
const members = [...memberIds].map((id) => functions.get(id)).sort(compareFunctions);
|
|
1757
|
+
groups.push({
|
|
1758
|
+
rank: 0,
|
|
1759
|
+
memberCount: members.length,
|
|
1760
|
+
fileCount: new Set(members.map((member) => member.path)).size,
|
|
1761
|
+
minimumEdgeSimilarity: Math.min(...groupPairs.map((pair) => pair.similarity)),
|
|
1762
|
+
maximumEdgeSimilarity: Math.max(...groupPairs.map((pair) => pair.similarity)),
|
|
1763
|
+
maximumPhysicalDistance: Math.max(...groupPairs.map((pair) => pair.location.physicalDistance)),
|
|
1764
|
+
maximumCohesionGap: Math.max(...groupPairs.map((pair) => pair.cohesionGap)),
|
|
1765
|
+
members
|
|
1766
|
+
});
|
|
1767
|
+
}
|
|
1768
|
+
groups.sort((left, right) => right.maximumCohesionGap - left.maximumCohesionGap || right.memberCount - left.memberCount || compareFunctions(left.members[0], right.members[0]));
|
|
1769
|
+
groups.forEach((group, index) => {
|
|
1770
|
+
group.rank = index + 1;
|
|
1771
|
+
});
|
|
1772
|
+
return groups;
|
|
1773
|
+
}
|
|
1774
|
+
function compileNameRegex(pattern) {
|
|
1775
|
+
if (pattern === void 0) return void 0;
|
|
1776
|
+
try {
|
|
1777
|
+
return new RegExp(pattern);
|
|
1778
|
+
} catch (error) {
|
|
1779
|
+
throw new CodeIndexError(`Invalid name regex: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
1780
|
+
}
|
|
1781
|
+
}
|
|
1782
|
+
function directoryParts(filePath) {
|
|
1783
|
+
const directory = path7.posix.dirname(filePath.replaceAll("\\", "/"));
|
|
1784
|
+
return directory === "." ? [] : directory.split("/").filter(Boolean);
|
|
1785
|
+
}
|
|
1786
|
+
function isTestPath(filePath) {
|
|
1787
|
+
const normalized = filePath.replaceAll("\\", "/").toLowerCase();
|
|
1788
|
+
const segments = normalized.split("/");
|
|
1789
|
+
const fileName = segments.at(-1) ?? "";
|
|
1790
|
+
return segments.some((segment) => segment === "test" || segment === "tests" || segment === "__tests__") || /\.(?:test|spec)\.[cm]?[jt]sx?$/.test(fileName);
|
|
1791
|
+
}
|
|
1792
|
+
function orderedFunctions(left, right) {
|
|
1793
|
+
return compareFunctions(left, right) <= 0 ? [left, right] : [right, left];
|
|
1794
|
+
}
|
|
1795
|
+
function compareFunctions(left, right) {
|
|
1796
|
+
return left.path.localeCompare(right.path) || left.startLine - right.startLine || left.startColumn - right.startColumn || left.qualifiedName.localeCompare(right.qualifiedName) || left.id - right.id;
|
|
1797
|
+
}
|
|
1798
|
+
function comparePairs(left, right) {
|
|
1799
|
+
return right.cohesionGap - left.cohesionGap || right.similarity - left.similarity || Number(right.reciprocal === true) - Number(left.reciprocal === true) || compareFunctions(left.left, right.left) || compareFunctions(left.right, right.right);
|
|
1800
|
+
}
|
|
1801
|
+
function pairKey(leftId, rightId) {
|
|
1802
|
+
return leftId < rightId ? `${leftId}:${rightId}` : `${rightId}:${leftId}`;
|
|
1803
|
+
}
|
|
1804
|
+
function addNeighbor(neighbors, source, target) {
|
|
1805
|
+
const values = neighbors.get(source) ?? /* @__PURE__ */ new Set();
|
|
1806
|
+
values.add(target);
|
|
1807
|
+
neighbors.set(source, values);
|
|
1808
|
+
}
|
|
1809
|
+
function ratio(numerator, denominator) {
|
|
1810
|
+
return denominator === 0 ? 0 : numerator / denominator;
|
|
1811
|
+
}
|
|
1812
|
+
function clamp(value) {
|
|
1813
|
+
return Math.max(0, Math.min(1, value));
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1206
1816
|
// src/search/cross-search.ts
|
|
1817
|
+
import { realpath, stat } from "node:fs/promises";
|
|
1818
|
+
import path8 from "node:path";
|
|
1207
1819
|
async function* crossSearch(options) {
|
|
1208
1820
|
const target = options.target ?? options.source;
|
|
1209
1821
|
const sourceProfile = JSON.stringify(options.source.status().embeddingProfile);
|
|
@@ -1213,21 +1825,55 @@ async function* crossSearch(options) {
|
|
|
1213
1825
|
}
|
|
1214
1826
|
const limit = options.limitPerFunction ?? 5;
|
|
1215
1827
|
assertPositiveInteger(limit, "limitPerFunction");
|
|
1216
|
-
const
|
|
1217
|
-
|
|
1828
|
+
const minLines = options.minLines ?? 2;
|
|
1829
|
+
assertPositiveInteger(minLines, "minLines");
|
|
1830
|
+
const nameRegex = compileNameRegex2(options.nameRegex);
|
|
1831
|
+
const sourceFunctions = (await options.source.sourceFunctions(options.sourceFilter ?? { type: "all" })).filter((callable) => callable.lineCount >= minLines && (!nameRegex || nameRegex.test(callable.qualifiedName)));
|
|
1832
|
+
const sameIndex = target.indexPath === options.source.indexPath || await fileIdentity(target.indexPath) === await fileIdentity(options.source.indexPath);
|
|
1833
|
+
const sourceRoot = options.crossFileOnly ? await canonicalRoot(options.source.rootDir) : void 0;
|
|
1834
|
+
const targetRoot = options.crossFileOnly ? sameIndex ? sourceRoot : await canonicalRoot(target.rootDir) : void 0;
|
|
1835
|
+
const canonicalFiles = /* @__PURE__ */ new Map();
|
|
1836
|
+
const canonicalFile = (root, filePath) => {
|
|
1837
|
+
const absolutePath = path8.resolve(root, filePath);
|
|
1838
|
+
let result = canonicalFiles.get(absolutePath);
|
|
1839
|
+
if (!result) {
|
|
1840
|
+
result = fileIdentity(absolutePath);
|
|
1841
|
+
canonicalFiles.set(absolutePath, result);
|
|
1842
|
+
}
|
|
1843
|
+
return result;
|
|
1844
|
+
};
|
|
1845
|
+
const targetPathsByCanonicalFile = /* @__PURE__ */ new Map();
|
|
1846
|
+
if (targetRoot) {
|
|
1847
|
+
const targetPaths = [...new Set(target.allFunctions().map((callable) => callable.path))];
|
|
1848
|
+
await Promise.all(targetPaths.map(async (targetPath) => {
|
|
1849
|
+
const canonicalPath = await canonicalFile(targetRoot, targetPath);
|
|
1850
|
+
const paths = targetPathsByCanonicalFile.get(canonicalPath) ?? [];
|
|
1851
|
+
paths.push(targetPath);
|
|
1852
|
+
targetPathsByCanonicalFile.set(canonicalPath, paths);
|
|
1853
|
+
}));
|
|
1854
|
+
}
|
|
1218
1855
|
const seenPairs = /* @__PURE__ */ new Set();
|
|
1219
1856
|
for (let index = 0; index < sourceFunctions.length; index += 1) {
|
|
1220
1857
|
throwIfAborted(options.signal);
|
|
1221
1858
|
const source = sourceFunctions[index];
|
|
1222
1859
|
const vector = options.source.vectorForFunction(source.id);
|
|
1860
|
+
const canonicalSourceFile = sourceRoot ? await canonicalFile(sourceRoot, source.path) : void 0;
|
|
1861
|
+
const excludedTargetPaths = canonicalSourceFile ? targetPathsByCanonicalFile.get(canonicalSourceFile) : void 0;
|
|
1862
|
+
const excludePaths = excludedTargetPaths ? { excludePaths: excludedTargetPaths } : {};
|
|
1223
1863
|
const candidates = sameIndex ? options.source.similarToFunction(source.id, {
|
|
1224
1864
|
limit,
|
|
1225
1865
|
minSimilarity: options.minSimilarity ?? -1,
|
|
1226
|
-
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {}
|
|
1866
|
+
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
|
|
1867
|
+
minLines,
|
|
1868
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {},
|
|
1869
|
+
...excludePaths
|
|
1227
1870
|
}) : target.searchByVector(vector, {
|
|
1228
1871
|
limit,
|
|
1229
1872
|
minSimilarity: options.minSimilarity ?? -1,
|
|
1230
|
-
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {}
|
|
1873
|
+
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
|
|
1874
|
+
minLines,
|
|
1875
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {},
|
|
1876
|
+
...excludePaths
|
|
1231
1877
|
});
|
|
1232
1878
|
const matches = sameIndex && !options.includeSymmetricDuplicates ? candidates.filter((match) => {
|
|
1233
1879
|
const pair = source.id < match.function.id ? `${source.id}:${match.function.id}` : `${match.function.id}:${source.id}`;
|
|
@@ -1239,6 +1885,29 @@ async function* crossSearch(options) {
|
|
|
1239
1885
|
options.onProgress?.({ completed: index + 1, total: sourceFunctions.length });
|
|
1240
1886
|
}
|
|
1241
1887
|
}
|
|
1888
|
+
function compileNameRegex2(pattern) {
|
|
1889
|
+
if (pattern === void 0) return void 0;
|
|
1890
|
+
try {
|
|
1891
|
+
return new RegExp(pattern);
|
|
1892
|
+
} catch (error) {
|
|
1893
|
+
throw new CodeIndexError(`Invalid name regex: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
1894
|
+
}
|
|
1895
|
+
}
|
|
1896
|
+
async function canonicalRoot(root) {
|
|
1897
|
+
try {
|
|
1898
|
+
return await realpath(root);
|
|
1899
|
+
} catch {
|
|
1900
|
+
return path8.resolve(root);
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
async function fileIdentity(filePath) {
|
|
1904
|
+
try {
|
|
1905
|
+
const metadata = await stat(filePath);
|
|
1906
|
+
return `inode:${metadata.dev}:${metadata.ino}`;
|
|
1907
|
+
} catch {
|
|
1908
|
+
return `path:${path8.resolve(filePath)}`;
|
|
1909
|
+
}
|
|
1910
|
+
}
|
|
1242
1911
|
|
|
1243
1912
|
// src/embeddings/openai.ts
|
|
1244
1913
|
import { Tiktoken } from "js-tiktoken/lite";
|
|
@@ -1305,8 +1974,8 @@ var OpenAIEmbeddingProvider = class {
|
|
|
1305
1974
|
constructor(options = {}) {
|
|
1306
1975
|
this.#apiKey = options.apiKey ?? process.env.OPENAI_API_KEY ?? "";
|
|
1307
1976
|
if (!this.#apiKey) throw new Error("OPENAI_API_KEY is required.");
|
|
1308
|
-
const model = options.model ?? "text-embedding-3-
|
|
1309
|
-
const dimensions = options.dimensions ??
|
|
1977
|
+
const model = options.model ?? "text-embedding-3-large";
|
|
1978
|
+
const dimensions = options.dimensions ?? 3072;
|
|
1310
1979
|
if (!Number.isInteger(dimensions) || dimensions < 1) throw new Error("dimensions must be a positive integer.");
|
|
1311
1980
|
this.profile = { provider: "openai", model, dimensions, strategyVersion: "callable-v1" };
|
|
1312
1981
|
this.#url = `${(options.baseUrl ?? "https://api.openai.com/v1").replace(/\/$/, "")}/embeddings`;
|
|
@@ -1390,24 +2059,35 @@ function updateFiles(index, options) {
|
|
|
1390
2059
|
function updateFromGit(index, options) {
|
|
1391
2060
|
return index.updateFromGit(options);
|
|
1392
2061
|
}
|
|
2062
|
+
function updateFromWorkingTree(index, options) {
|
|
2063
|
+
return index.updateFromWorkingTree(options);
|
|
2064
|
+
}
|
|
1393
2065
|
function similaritySearch(index, options) {
|
|
1394
2066
|
return index.similaritySearch(options);
|
|
1395
2067
|
}
|
|
1396
2068
|
function crossSearchFunctions(options) {
|
|
1397
2069
|
return crossSearch(options);
|
|
1398
2070
|
}
|
|
2071
|
+
function analyzeCodeCohesion(options) {
|
|
2072
|
+
return analyzeCohesion(options);
|
|
2073
|
+
}
|
|
1399
2074
|
export {
|
|
1400
2075
|
CodeIndex,
|
|
1401
2076
|
CodeIndexError,
|
|
1402
2077
|
GitDivergenceError,
|
|
2078
|
+
GitUnavailableError,
|
|
1403
2079
|
IncompatibleIndexError,
|
|
1404
2080
|
JinaEmbeddingProvider,
|
|
1405
2081
|
OpenAIEmbeddingProvider,
|
|
2082
|
+
analyzeCodeCohesion,
|
|
2083
|
+
analyzeCohesion,
|
|
2084
|
+
cohesionLocation,
|
|
1406
2085
|
crossSearch,
|
|
1407
2086
|
crossSearchFunctions,
|
|
1408
2087
|
openCodeIndex,
|
|
1409
2088
|
similaritySearch,
|
|
1410
2089
|
updateFiles,
|
|
1411
|
-
updateFromGit
|
|
2090
|
+
updateFromGit,
|
|
2091
|
+
updateFromWorkingTree
|
|
1412
2092
|
};
|
|
1413
2093
|
//# sourceMappingURL=index.js.map
|