@ninjaxtools/slopdex 0.3.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/slopdex/SKILL.md +173 -24
- package/README.md +93 -94
- package/dist/cli.js +1254 -198
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +129 -2
- package/dist/index.js +796 -116
- package/dist/index.js.map +1 -1
- package/package.json +5 -1
package/dist/cli.js
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
// src/cli.ts
|
|
4
4
|
import { existsSync, readFileSync, rmSync } from "node:fs";
|
|
5
|
-
import
|
|
5
|
+
import path9 from "node:path";
|
|
6
6
|
import { parseArgs } from "node:util";
|
|
7
7
|
|
|
8
8
|
// src/code-index.ts
|
|
9
|
-
import { lstat, readFile } from "node:fs/promises";
|
|
9
|
+
import { lstat as lstat2, readFile, readdir } from "node:fs/promises";
|
|
10
10
|
import path6 from "node:path";
|
|
11
11
|
|
|
12
12
|
// src/errors.ts
|
|
@@ -22,6 +22,12 @@ var GitDivergenceError = class extends CodeIndexError {
|
|
|
22
22
|
this.name = "GitDivergenceError";
|
|
23
23
|
}
|
|
24
24
|
};
|
|
25
|
+
var GitUnavailableError = class extends CodeIndexError {
|
|
26
|
+
constructor(message, options) {
|
|
27
|
+
super(message, options);
|
|
28
|
+
this.name = "GitUnavailableError";
|
|
29
|
+
}
|
|
30
|
+
};
|
|
25
31
|
var IncompatibleIndexError = class extends CodeIndexError {
|
|
26
32
|
constructor(message) {
|
|
27
33
|
super(message);
|
|
@@ -31,6 +37,7 @@ var IncompatibleIndexError = class extends CodeIndexError {
|
|
|
31
37
|
|
|
32
38
|
// src/git/repository.ts
|
|
33
39
|
import { spawn } from "node:child_process";
|
|
40
|
+
import { lstat } from "node:fs/promises";
|
|
34
41
|
import path from "node:path";
|
|
35
42
|
var GitRepository = class {
|
|
36
43
|
constructor(rootDir) {
|
|
@@ -44,7 +51,13 @@ var GitRepository = class {
|
|
|
44
51
|
const stderr = [];
|
|
45
52
|
child.stdout.on("data", (value) => stdout.push(value));
|
|
46
53
|
child.stderr.on("data", (value) => stderr.push(value));
|
|
47
|
-
child.on("error",
|
|
54
|
+
child.on("error", (error) => {
|
|
55
|
+
if (error.code === "ENOENT") {
|
|
56
|
+
reject(new GitUnavailableError("Git executable is not available.", { cause: error }));
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
reject(error);
|
|
60
|
+
});
|
|
48
61
|
child.on("close", (code) => {
|
|
49
62
|
const exitCode = code ?? -1;
|
|
50
63
|
if (exitCode === 0 || allowExitOne && exitCode === 1) {
|
|
@@ -56,29 +69,51 @@ var GitRepository = class {
|
|
|
56
69
|
});
|
|
57
70
|
}
|
|
58
71
|
async assertRepository() {
|
|
59
|
-
|
|
72
|
+
let result;
|
|
73
|
+
try {
|
|
74
|
+
result = await this.#run(["rev-parse", "--is-inside-work-tree"]);
|
|
75
|
+
} catch (error) {
|
|
76
|
+
if (error instanceof GitUnavailableError) throw error;
|
|
77
|
+
try {
|
|
78
|
+
await lstat(path.join(this.rootDir, ".git"));
|
|
79
|
+
} catch (markerError) {
|
|
80
|
+
if (markerError.code === "ENOENT") {
|
|
81
|
+
throw new GitUnavailableError(`${this.rootDir} is not a Git worktree.`, { cause: error });
|
|
82
|
+
}
|
|
83
|
+
throw markerError;
|
|
84
|
+
}
|
|
85
|
+
throw error;
|
|
86
|
+
}
|
|
60
87
|
if (result.stdout.toString("utf8").trim() !== "true") {
|
|
61
|
-
throw new
|
|
88
|
+
throw new GitUnavailableError(`${this.rootDir} is not a Git worktree.`);
|
|
62
89
|
}
|
|
63
90
|
const topLevel = (await this.#run(["rev-parse", "--show-toplevel"])).stdout.toString("utf8").trim();
|
|
64
91
|
if (path.resolve(topLevel) !== path.resolve(this.rootDir)) {
|
|
65
92
|
throw new CodeIndexError(`rootDir must be the Git worktree root: ${topLevel}`);
|
|
66
93
|
}
|
|
67
94
|
}
|
|
68
|
-
async
|
|
95
|
+
async workTreeChanges(base, excludePaths = []) {
|
|
69
96
|
const pathspecs = [".", ...excludePaths.map((filePath) => `:(top,literal,exclude)${filePath}`)];
|
|
70
97
|
const result = await this.#run([
|
|
71
|
-
"
|
|
72
|
-
"--
|
|
98
|
+
"diff",
|
|
99
|
+
"--name-status",
|
|
73
100
|
"-z",
|
|
74
|
-
"--
|
|
75
|
-
|
|
101
|
+
"--find-renames",
|
|
102
|
+
base,
|
|
76
103
|
"--",
|
|
77
104
|
...pathspecs
|
|
78
105
|
]);
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
106
|
+
const changes = parseChanges(splitNull(result.stdout));
|
|
107
|
+
const untracked = await this.#run([
|
|
108
|
+
"ls-files",
|
|
109
|
+
"--others",
|
|
110
|
+
"--exclude-standard",
|
|
111
|
+
"-z",
|
|
112
|
+
"--",
|
|
113
|
+
...pathspecs
|
|
114
|
+
]);
|
|
115
|
+
for (const filePath of splitNull(untracked.stdout)) changes.push({ status: "A", path: filePath });
|
|
116
|
+
return changes;
|
|
82
117
|
}
|
|
83
118
|
async resolveCommit(ref) {
|
|
84
119
|
const result = await this.#run(["rev-parse", "--verify", "--end-of-options", `${ref}^{commit}`]);
|
|
@@ -110,26 +145,28 @@ var GitRepository = class {
|
|
|
110
145
|
}
|
|
111
146
|
async diff(base, target) {
|
|
112
147
|
const result = await this.#run(["diff", "--name-status", "-z", "--find-renames", `${base}..${target}`, "--"]);
|
|
113
|
-
|
|
114
|
-
const changes = [];
|
|
115
|
-
for (let index = 0; index < values.length; ) {
|
|
116
|
-
const rawStatus = values[index++];
|
|
117
|
-
if (!rawStatus) continue;
|
|
118
|
-
const status = rawStatus[0];
|
|
119
|
-
if (status === "R" || status === "C") {
|
|
120
|
-
const oldPath = values[index++];
|
|
121
|
-
const filePath = values[index++];
|
|
122
|
-
if (oldPath && filePath) changes.push({ status, oldPath, path: filePath });
|
|
123
|
-
} else if (status === "A" || status === "M" || status === "D" || status === "T") {
|
|
124
|
-
const filePath = values[index++];
|
|
125
|
-
if (filePath) changes.push({ status, path: filePath });
|
|
126
|
-
} else {
|
|
127
|
-
throw new CodeIndexError(`Unsupported Git diff status: ${rawStatus}`);
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
return changes;
|
|
148
|
+
return parseChanges(splitNull(result.stdout));
|
|
131
149
|
}
|
|
132
150
|
};
|
|
151
|
+
function parseChanges(values) {
|
|
152
|
+
const changes = [];
|
|
153
|
+
for (let index = 0; index < values.length; ) {
|
|
154
|
+
const rawStatus = values[index++];
|
|
155
|
+
if (!rawStatus) continue;
|
|
156
|
+
const status = rawStatus[0];
|
|
157
|
+
if (status === "R" || status === "C") {
|
|
158
|
+
const oldPath = values[index++];
|
|
159
|
+
const filePath = values[index++];
|
|
160
|
+
if (oldPath && filePath) changes.push({ status, oldPath, path: filePath });
|
|
161
|
+
} else if (status === "A" || status === "M" || status === "D" || status === "T") {
|
|
162
|
+
const filePath = values[index++];
|
|
163
|
+
if (filePath) changes.push({ status, path: filePath });
|
|
164
|
+
} else {
|
|
165
|
+
throw new CodeIndexError(`Unsupported Git diff status: ${rawStatus}`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return changes;
|
|
169
|
+
}
|
|
133
170
|
function splitNull(buffer) {
|
|
134
171
|
const value = buffer.toString("utf8");
|
|
135
172
|
const parts = value.split("\0");
|
|
@@ -453,6 +490,7 @@ function parseCallables(relativePath, content, onWarning = console.warn) {
|
|
|
453
490
|
startColumn: node.startPosition.column + 1,
|
|
454
491
|
endLine: node.endPosition.row + 1,
|
|
455
492
|
endColumn: node.endPosition.column + 1,
|
|
493
|
+
lineCount: node.endPosition.row - node.startPosition.row + 1,
|
|
456
494
|
source,
|
|
457
495
|
sourceHash: sha256(source),
|
|
458
496
|
embeddingInput
|
|
@@ -482,18 +520,24 @@ var SourcePolicy = class {
|
|
|
482
520
|
includes(relativePath) {
|
|
483
521
|
const normalized = relativePath.replaceAll("\\", "/");
|
|
484
522
|
if (!languageForPath(normalized)) return false;
|
|
485
|
-
if (normalized
|
|
523
|
+
if (hasExcludedSegment(normalized)) return false;
|
|
486
524
|
if (this.#exclude.some((pattern) => path4.matchesGlob(normalized, pattern))) return false;
|
|
487
525
|
return this.#include.length === 0 || this.#include.some((pattern) => path4.matchesGlob(normalized, pattern));
|
|
488
526
|
}
|
|
527
|
+
traversesDirectory(relativePath) {
|
|
528
|
+
return !hasExcludedSegment(relativePath.replaceAll("\\", "/"));
|
|
529
|
+
}
|
|
489
530
|
};
|
|
531
|
+
function hasExcludedSegment(relativePath) {
|
|
532
|
+
return relativePath.split("/").some((segment) => DEFAULT_EXCLUDED_SEGMENTS.has(segment));
|
|
533
|
+
}
|
|
490
534
|
|
|
491
535
|
// src/storage/database.ts
|
|
492
536
|
import { mkdirSync } from "node:fs";
|
|
493
537
|
import path5 from "node:path";
|
|
494
538
|
import { DatabaseSync } from "node:sqlite";
|
|
495
539
|
import * as sqliteVec from "sqlite-vec";
|
|
496
|
-
var SCHEMA_VERSION = "
|
|
540
|
+
var SCHEMA_VERSION = "2";
|
|
497
541
|
var IndexDatabase = class {
|
|
498
542
|
#db;
|
|
499
543
|
#rootDir;
|
|
@@ -505,15 +549,30 @@ var IndexDatabase = class {
|
|
|
505
549
|
this.#rootDir = rootDir;
|
|
506
550
|
this.#profile = profile;
|
|
507
551
|
this.#db = new DatabaseSync(indexPath, { allowExtension: true, readOnly });
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
552
|
+
try {
|
|
553
|
+
sqliteVec.load(this.#db);
|
|
554
|
+
this.#db.enableLoadExtension(false);
|
|
555
|
+
const regexes = /* @__PURE__ */ new Map();
|
|
556
|
+
this.#db.function("slopdex_regexp", { deterministic: true }, (pattern, value) => {
|
|
557
|
+
if (typeof pattern !== "string" || typeof value !== "string") return 0;
|
|
558
|
+
let regex = regexes.get(pattern);
|
|
559
|
+
if (!regex) {
|
|
560
|
+
regex = new RegExp(pattern);
|
|
561
|
+
regexes.set(pattern, regex);
|
|
562
|
+
}
|
|
563
|
+
return regex.test(value) ? 1 : 0;
|
|
564
|
+
});
|
|
565
|
+
if (readOnly) {
|
|
566
|
+
this.#db.exec("PRAGMA foreign_keys=ON");
|
|
567
|
+
} else {
|
|
568
|
+
this.#db.exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; PRAGMA foreign_keys=ON;");
|
|
569
|
+
this.#migrate();
|
|
570
|
+
}
|
|
571
|
+
this.#validateMetadata(readOnly);
|
|
572
|
+
} catch (error) {
|
|
573
|
+
this.#db.close();
|
|
574
|
+
throw error;
|
|
515
575
|
}
|
|
516
|
-
this.#validateMetadata(readOnly);
|
|
517
576
|
}
|
|
518
577
|
close() {
|
|
519
578
|
this.#db.close();
|
|
@@ -552,6 +611,7 @@ var IndexDatabase = class {
|
|
|
552
611
|
start_column INTEGER NOT NULL,
|
|
553
612
|
end_line INTEGER NOT NULL,
|
|
554
613
|
end_column INTEGER NOT NULL,
|
|
614
|
+
line_count INTEGER NOT NULL,
|
|
555
615
|
source TEXT NOT NULL,
|
|
556
616
|
source_hash TEXT NOT NULL,
|
|
557
617
|
embedding_input TEXT NOT NULL,
|
|
@@ -560,13 +620,31 @@ var IndexDatabase = class {
|
|
|
560
620
|
source_mode TEXT NOT NULL CHECK(source_mode IN ('git', 'working-tree')),
|
|
561
621
|
embedding_id INTEGER NOT NULL REFERENCES embeddings(id)
|
|
562
622
|
);
|
|
623
|
+
CREATE TABLE IF NOT EXISTS callable_provenance (
|
|
624
|
+
identity_key TEXT NOT NULL,
|
|
625
|
+
source_hash TEXT NOT NULL,
|
|
626
|
+
first_seen_commit TEXT NOT NULL,
|
|
627
|
+
PRIMARY KEY(identity_key, source_hash)
|
|
628
|
+
);
|
|
563
629
|
CREATE INDEX IF NOT EXISTS functions_path ON functions(path);
|
|
564
630
|
CREATE INDEX IF NOT EXISTS functions_embedding ON functions(embedding_id);
|
|
631
|
+
INSERT OR IGNORE INTO callable_provenance(identity_key, source_hash, first_seen_commit)
|
|
632
|
+
SELECT identity_key, source_hash, first_seen_commit FROM functions WHERE first_seen_commit IS NOT NULL;
|
|
565
633
|
`);
|
|
566
634
|
const fileColumns = this.#db.prepare("PRAGMA table_info(files)").all();
|
|
567
635
|
if (!fileColumns.some((column) => column.name === "previous_path")) {
|
|
568
636
|
this.#db.exec("ALTER TABLE files ADD COLUMN previous_path TEXT");
|
|
569
637
|
}
|
|
638
|
+
const functionColumns = this.#db.prepare("PRAGMA table_info(functions)").all();
|
|
639
|
+
const needsLineCount = !functionColumns.some((column) => column.name === "line_count");
|
|
640
|
+
const schemaVersion = this.#metadata("schema_version");
|
|
641
|
+
if (needsLineCount || schemaVersion === "1") {
|
|
642
|
+
this.#transaction(() => {
|
|
643
|
+
if (needsLineCount) this.#db.exec("ALTER TABLE functions ADD COLUMN line_count INTEGER NOT NULL DEFAULT 1");
|
|
644
|
+
this.#db.exec("UPDATE functions SET line_count = end_line - start_line + 1");
|
|
645
|
+
if (schemaVersion === "1") this.#setMetadata("schema_version", SCHEMA_VERSION);
|
|
646
|
+
});
|
|
647
|
+
}
|
|
570
648
|
}
|
|
571
649
|
#metadata(key) {
|
|
572
650
|
const row = this.#db.prepare("SELECT value FROM metadata WHERE key = ?").get(key);
|
|
@@ -578,6 +656,9 @@ var IndexDatabase = class {
|
|
|
578
656
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
579
657
|
`).run(key, value);
|
|
580
658
|
}
|
|
659
|
+
#deleteMetadata(key) {
|
|
660
|
+
this.#db.prepare("DELETE FROM metadata WHERE key = ?").run(key);
|
|
661
|
+
}
|
|
581
662
|
#validateMetadata(readOnly) {
|
|
582
663
|
const schemaVersion = this.#metadata("schema_version");
|
|
583
664
|
if (readOnly && !schemaVersion) throw new IncompatibleIndexError("Target file is not a slopdex index.");
|
|
@@ -626,6 +707,13 @@ var IndexDatabase = class {
|
|
|
626
707
|
FROM files WHERE source_mode = 'working-tree' ORDER BY path
|
|
627
708
|
`).all();
|
|
628
709
|
}
|
|
710
|
+
getFileStates() {
|
|
711
|
+
return this.#db.prepare(`
|
|
712
|
+
SELECT path, content_hash AS contentHash, blob_oid AS blobOid,
|
|
713
|
+
source_mode AS sourceMode, previous_path AS previousPath
|
|
714
|
+
FROM files ORDER BY path
|
|
715
|
+
`).all();
|
|
716
|
+
}
|
|
629
717
|
getEmbeddingKeys(keys) {
|
|
630
718
|
const found = /* @__PURE__ */ new Set();
|
|
631
719
|
const statement = this.#db.prepare("SELECT 1 FROM embeddings WHERE embedding_key = ?");
|
|
@@ -639,36 +727,44 @@ var IndexDatabase = class {
|
|
|
639
727
|
if (options.expectedCheckpoint !== void 0 && this.getCheckpoint() !== options.expectedCheckpoint) {
|
|
640
728
|
throw new CodeIndexError("Git checkpoint changed while the update was being prepared.");
|
|
641
729
|
}
|
|
730
|
+
if (options.expectedGeneration !== void 0 && this.getGeneration() !== options.expectedGeneration) {
|
|
731
|
+
throw new CodeIndexError("Index changed while the update was being prepared; retry the update.");
|
|
732
|
+
}
|
|
642
733
|
const stats = {
|
|
643
|
-
filesUpdated:
|
|
734
|
+
filesUpdated: 0,
|
|
644
735
|
filesDeleted: 0,
|
|
645
736
|
functionsAdded: 0,
|
|
646
737
|
functionsUpdated: 0,
|
|
647
738
|
functionsDeleted: 0,
|
|
648
739
|
embeddingsCreated: 0,
|
|
649
|
-
checkpoint: options.checkpoint
|
|
740
|
+
checkpoint: options.checkpoint !== void 0 ? options.checkpoint : this.getCheckpoint()
|
|
650
741
|
};
|
|
651
742
|
const deleteFile = this.#db.prepare("DELETE FROM files WHERE path = ?");
|
|
652
743
|
const countFunctions = this.#db.prepare("SELECT COUNT(*) AS count FROM functions WHERE path = ?");
|
|
653
|
-
const
|
|
654
|
-
|
|
655
|
-
uniqueDeletes
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
const count = Number(countFunctions.get(filePath).count);
|
|
661
|
-
const result = deleteFile.run(filePath);
|
|
662
|
-
if (result.changes > 0) {
|
|
663
|
-
stats.filesDeleted += 1;
|
|
664
|
-
stats.functionsDeleted += count;
|
|
744
|
+
const applyStage = (files, deletePaths) => {
|
|
745
|
+
stats.filesUpdated += files.length;
|
|
746
|
+
const uniqueDeletes = new Set(deletePaths);
|
|
747
|
+
for (const file of files) {
|
|
748
|
+
uniqueDeletes.delete(file.path);
|
|
749
|
+
if (file.previousPath) uniqueDeletes.delete(file.previousPath);
|
|
750
|
+
if (file.replacePath) uniqueDeletes.delete(file.replacePath);
|
|
665
751
|
}
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
752
|
+
for (const filePath of uniqueDeletes) {
|
|
753
|
+
const count = Number(countFunctions.get(filePath).count);
|
|
754
|
+
const result = deleteFile.run(filePath);
|
|
755
|
+
if (result.changes > 0) {
|
|
756
|
+
stats.filesDeleted += 1;
|
|
757
|
+
stats.functionsDeleted += count;
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
for (const file of files) this.#replaceFile(file, stats);
|
|
761
|
+
};
|
|
762
|
+
applyStage(options.files, options.deletePaths);
|
|
763
|
+
if (options.workingTree) applyStage(options.workingTree.files, options.workingTree.deletePaths);
|
|
669
764
|
const generation = this.getGeneration() + 1;
|
|
670
765
|
this.#setMetadata("generation", String(generation));
|
|
671
|
-
if (options.checkpoint) this.#
|
|
766
|
+
if (options.checkpoint === null) this.#deleteMetadata("git_checkpoint");
|
|
767
|
+
else if (options.checkpoint !== void 0) this.#setMetadata("git_checkpoint", options.checkpoint);
|
|
672
768
|
return stats;
|
|
673
769
|
});
|
|
674
770
|
}
|
|
@@ -678,6 +774,12 @@ var IndexDatabase = class {
|
|
|
678
774
|
const existingFile = this.#db.prepare("SELECT previous_path FROM files WHERE path = ?").get(sourcePath);
|
|
679
775
|
const oldMatches = reconcileFunctions(file.callables, oldRows);
|
|
680
776
|
if (sourcePath !== file.path) {
|
|
777
|
+
const displacedRows = this.#rowsForPath(file.path);
|
|
778
|
+
const displaced = this.#db.prepare("DELETE FROM files WHERE path = ?").run(file.path);
|
|
779
|
+
if (displaced.changes > 0) {
|
|
780
|
+
stats.filesDeleted += 1;
|
|
781
|
+
stats.functionsDeleted += displacedRows.length;
|
|
782
|
+
}
|
|
681
783
|
this.#db.prepare("DELETE FROM files WHERE path = ?").run(sourcePath);
|
|
682
784
|
} else {
|
|
683
785
|
this.#db.prepare("DELETE FROM files WHERE path = ?").run(file.path);
|
|
@@ -709,13 +811,16 @@ var IndexDatabase = class {
|
|
|
709
811
|
if (!embedding) throw new CodeIndexError(`Missing embedding for ${callable.qualifiedName}.`);
|
|
710
812
|
const old = oldMatches.get(callable);
|
|
711
813
|
if (old) usedIds.add(old.id);
|
|
712
|
-
const
|
|
814
|
+
const remembered = this.#db.prepare(`
|
|
815
|
+
SELECT first_seen_commit FROM callable_provenance WHERE identity_key = ? AND source_hash = ?
|
|
816
|
+
`).get(callable.identityKey, callable.sourceHash);
|
|
817
|
+
const firstSeenCommit = old?.first_seen_commit ?? remembered?.first_seen_commit ?? file.indexedCommit;
|
|
713
818
|
this.#db.prepare(`
|
|
714
819
|
INSERT INTO functions(
|
|
715
820
|
id, path, language, kind, name, qualified_name, signature, identity_key,
|
|
716
|
-
start_line, start_column, end_line, end_column, source, source_hash,
|
|
821
|
+
start_line, start_column, end_line, end_column, line_count, source, source_hash,
|
|
717
822
|
embedding_input, first_seen_commit, last_seen_commit, source_mode, embedding_id
|
|
718
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
823
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
719
824
|
`).run(
|
|
720
825
|
old?.id ?? null,
|
|
721
826
|
file.path,
|
|
@@ -729,6 +834,7 @@ var IndexDatabase = class {
|
|
|
729
834
|
callable.startColumn,
|
|
730
835
|
callable.endLine,
|
|
731
836
|
callable.endColumn,
|
|
837
|
+
callable.lineCount,
|
|
732
838
|
callable.source,
|
|
733
839
|
callable.sourceHash,
|
|
734
840
|
callable.embeddingInput,
|
|
@@ -739,6 +845,11 @@ var IndexDatabase = class {
|
|
|
739
845
|
);
|
|
740
846
|
if (old) stats.functionsUpdated += 1;
|
|
741
847
|
else stats.functionsAdded += 1;
|
|
848
|
+
if (firstSeenCommit) {
|
|
849
|
+
this.#db.prepare(`
|
|
850
|
+
INSERT OR IGNORE INTO callable_provenance(identity_key, source_hash, first_seen_commit) VALUES (?, ?, ?)
|
|
851
|
+
`).run(callable.identityKey, callable.sourceHash, firstSeenCommit);
|
|
852
|
+
}
|
|
742
853
|
}
|
|
743
854
|
stats.functionsDeleted += oldRows.length - usedIds.size;
|
|
744
855
|
}
|
|
@@ -761,19 +872,28 @@ var IndexDatabase = class {
|
|
|
761
872
|
return results;
|
|
762
873
|
}
|
|
763
874
|
searchVector(vector, options) {
|
|
875
|
+
const excludePaths = options.excludePaths ?? [];
|
|
876
|
+
const pathFilter = excludePaths.length > 0 ? `AND f.path NOT IN (${excludePaths.map(() => "?").join(", ")})` : "";
|
|
764
877
|
const rows = this.#db.prepare(`
|
|
765
878
|
SELECT f.*, 1.0 - vec_distance_cosine(e.vector, ?) AS similarity
|
|
766
879
|
FROM functions f
|
|
767
880
|
JOIN embeddings e ON e.id = f.embedding_id
|
|
768
881
|
WHERE (? IS NULL OR f.id != ?)
|
|
882
|
+
${pathFilter}
|
|
883
|
+
AND f.line_count >= ?
|
|
884
|
+
AND (? IS NULL OR slopdex_regexp(?, f.qualified_name))
|
|
769
885
|
AND (1.0 - vec_distance_cosine(e.vector, ?)) >= ?
|
|
770
|
-
AND (? IS NULL OR (1.0 - vec_distance_cosine(e.vector, ?))
|
|
886
|
+
AND (? IS NULL OR (1.0 - vec_distance_cosine(e.vector, ?)) < ?)
|
|
771
887
|
ORDER BY similarity DESC, f.id ASC
|
|
772
888
|
LIMIT ?
|
|
773
889
|
`).all(
|
|
774
890
|
vectorBuffer(vector),
|
|
775
891
|
options.excludeId ?? null,
|
|
776
892
|
options.excludeId ?? null,
|
|
893
|
+
...excludePaths,
|
|
894
|
+
options.minLines ?? 1,
|
|
895
|
+
options.nameRegex ?? null,
|
|
896
|
+
options.nameRegex ?? null,
|
|
777
897
|
vectorBuffer(vector),
|
|
778
898
|
options.minSimilarity,
|
|
779
899
|
options.maxSimilarity ?? null,
|
|
@@ -822,6 +942,7 @@ function toIndexedFunction(row) {
|
|
|
822
942
|
startColumn: row.start_column,
|
|
823
943
|
endLine: row.end_line,
|
|
824
944
|
endColumn: row.end_column,
|
|
945
|
+
lineCount: row.line_count,
|
|
825
946
|
source: row.source,
|
|
826
947
|
sourceHash: row.source_hash,
|
|
827
948
|
embeddingInput: row.embedding_input,
|
|
@@ -916,7 +1037,7 @@ var CodeIndex = class {
|
|
|
916
1037
|
throwIfAborted(options.signal);
|
|
917
1038
|
if (!this.#policy.includes(relativePath)) throw new CodeIndexError(`Unsupported or excluded source file: ${relativePath}`);
|
|
918
1039
|
const absolutePath = path6.join(this.rootDir, relativePath);
|
|
919
|
-
const info = await
|
|
1040
|
+
const info = await lstat2(absolutePath);
|
|
920
1041
|
if (info.isSymbolicLink()) throw new CodeIndexError(`Symbolic links are not supported: ${relativePath}`);
|
|
921
1042
|
if (!info.isFile()) throw new CodeIndexError(`Not a regular file: ${relativePath}`);
|
|
922
1043
|
if (info.size > this.#maxFileSize) throw new CodeIndexError(`File exceeds maxFileSize: ${relativePath}`);
|
|
@@ -937,50 +1058,210 @@ var CodeIndex = class {
|
|
|
937
1058
|
deletePaths: [...deletePaths, ...renameMap.values()]
|
|
938
1059
|
});
|
|
939
1060
|
}
|
|
1061
|
+
async updateFromWorkingTree(options = {}) {
|
|
1062
|
+
throwIfAborted(options.signal);
|
|
1063
|
+
const generation = this.#database.getGeneration();
|
|
1064
|
+
const indexedFiles = this.#database.getFileStates();
|
|
1065
|
+
const sourcePaths = await this.#workingTreeSourcePaths(options.signal);
|
|
1066
|
+
const prepared = [];
|
|
1067
|
+
const skippedPaths = /* @__PURE__ */ new Set();
|
|
1068
|
+
for (const relativePath of sourcePaths) {
|
|
1069
|
+
throwIfAborted(options.signal);
|
|
1070
|
+
const absolutePath = path6.join(this.rootDir, relativePath);
|
|
1071
|
+
const info = await lstat2(absolutePath);
|
|
1072
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize) {
|
|
1073
|
+
skippedPaths.add(relativePath);
|
|
1074
|
+
continue;
|
|
1075
|
+
}
|
|
1076
|
+
const content = await readFile(absolutePath);
|
|
1077
|
+
prepared.push(this.#prepareFile(relativePath, content, {
|
|
1078
|
+
blobOid: null,
|
|
1079
|
+
sourceMode: "working-tree",
|
|
1080
|
+
indexedCommit: null
|
|
1081
|
+
}));
|
|
1082
|
+
}
|
|
1083
|
+
await this.#attachEmbeddings(prepared, options.signal);
|
|
1084
|
+
const pathsAgain = await this.#workingTreeSourcePaths(options.signal);
|
|
1085
|
+
if (JSON.stringify(pathsAgain) !== JSON.stringify(sourcePaths)) {
|
|
1086
|
+
throw new CodeIndexError("Working-tree files changed while indexing; retry the update.");
|
|
1087
|
+
}
|
|
1088
|
+
for (const file of prepared) {
|
|
1089
|
+
try {
|
|
1090
|
+
const info = await lstat2(path6.join(this.rootDir, file.path));
|
|
1091
|
+
const content = await readFile(path6.join(this.rootDir, file.path));
|
|
1092
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize || sha256(content.toString("utf8")) !== file.contentHash) {
|
|
1093
|
+
throw new CodeIndexError(`Working-tree file changed while indexing: ${file.path}`);
|
|
1094
|
+
}
|
|
1095
|
+
} catch (error) {
|
|
1096
|
+
if (error instanceof CodeIndexError) throw error;
|
|
1097
|
+
throw new CodeIndexError(`Working-tree file changed while indexing: ${file.path}`, { cause: error });
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
for (const relativePath of skippedPaths) {
|
|
1101
|
+
const info = await lstat2(path6.join(this.rootDir, relativePath));
|
|
1102
|
+
if (info.isFile() && !info.isSymbolicLink() && info.size <= this.#maxFileSize) {
|
|
1103
|
+
throw new CodeIndexError(`Working-tree file changed while indexing: ${relativePath}`);
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
return this.#database.applyUpdate({
|
|
1107
|
+
files: prepared,
|
|
1108
|
+
deletePaths: indexedFiles.map((file) => file.path),
|
|
1109
|
+
checkpoint: null,
|
|
1110
|
+
expectedGeneration: generation
|
|
1111
|
+
});
|
|
1112
|
+
}
|
|
940
1113
|
async updateFromGit(options = {}) {
|
|
941
1114
|
throwIfAborted(options.signal);
|
|
942
1115
|
await this.#git.assertRepository();
|
|
943
1116
|
const relativeIndexPath = path6.relative(this.rootDir, this.indexPath).replaceAll(path6.sep, "/");
|
|
944
1117
|
const indexArtifacts = relativeIndexPath && !relativeIndexPath.startsWith("../") ? [relativeIndexPath, `${relativeIndexPath}-shm`, `${relativeIndexPath}-wal`, `${relativeIndexPath}-journal`] : [];
|
|
945
|
-
await this.#git.assertCleanWorkingTree(indexArtifacts);
|
|
946
1118
|
const target = await this.#git.resolveCommit(options.target ?? "HEAD");
|
|
1119
|
+
const head = await this.#git.resolveCommit("HEAD");
|
|
1120
|
+
const overlayWorkingTree = target === head && options.includeWorkingTree !== false;
|
|
947
1121
|
const checkpoint = this.#database.getCheckpoint();
|
|
948
|
-
|
|
1122
|
+
const generation = this.#database.getGeneration();
|
|
1123
|
+
let reconcileAll = checkpoint === null;
|
|
949
1124
|
if (checkpoint && !await this.#git.isAncestor(checkpoint, target)) {
|
|
950
1125
|
if (!options.rebuildOnDivergence) throw new GitDivergenceError(checkpoint, target);
|
|
951
|
-
|
|
1126
|
+
reconcileAll = true;
|
|
952
1127
|
}
|
|
953
1128
|
const tree = await this.#git.listTree(target);
|
|
954
|
-
const
|
|
955
|
-
const
|
|
1129
|
+
const indexedFiles = this.#database.getFileStates();
|
|
1130
|
+
const indexedByPath = new Map(indexedFiles.map((file) => [file.path, file]));
|
|
1131
|
+
const workingFiles = indexedFiles.filter((file) => file.sourceMode === "working-tree");
|
|
1132
|
+
const workingChanges = overlayWorkingTree ? await this.#git.workTreeChanges(target, indexArtifacts) : [];
|
|
1133
|
+
const visibleWorkingPaths = new Set(
|
|
1134
|
+
workingChanges.filter((change) => change.status !== "D").map((change) => change.path)
|
|
1135
|
+
);
|
|
1136
|
+
const changes = reconcileAll || !checkpoint ? [] : await this.#git.diff(checkpoint, target);
|
|
956
1137
|
const upserts = /* @__PURE__ */ new Map();
|
|
957
1138
|
const deletes = /* @__PURE__ */ new Set();
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
1139
|
+
const eligibleEntries = [...tree.values()].filter((entry) => this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize);
|
|
1140
|
+
const eligibleTargetPaths = new Set(eligibleEntries.map((entry) => entry.path));
|
|
1141
|
+
if (reconcileAll) {
|
|
1142
|
+
const renameCandidates = indexedFiles.filter((file) => file.sourceMode === "git" && file.blobOid && !eligibleTargetPaths.has(file.path));
|
|
1143
|
+
const claimedRenameSources = /* @__PURE__ */ new Set();
|
|
1144
|
+
for (const entry of eligibleEntries) {
|
|
1145
|
+
const indexed = indexedByPath.get(entry.path);
|
|
1146
|
+
if (indexed?.sourceMode === "git" && indexed.blobOid === entry.oid) continue;
|
|
1147
|
+
const matches = renameCandidates.filter((candidate) => candidate.blobOid === entry.oid && !claimedRenameSources.has(candidate.path));
|
|
1148
|
+
const previousPath = matches.length === 1 ? matches[0].path : void 0;
|
|
1149
|
+
if (previousPath) claimedRenameSources.add(previousPath);
|
|
1150
|
+
upserts.set(entry.path, { entry, ...previousPath ? { previousPath } : {} });
|
|
1151
|
+
}
|
|
1152
|
+
for (const indexed of indexedFiles) {
|
|
1153
|
+
if (!eligibleTargetPaths.has(indexed.path)) deletes.add(indexed.path);
|
|
962
1154
|
}
|
|
963
1155
|
} else {
|
|
964
1156
|
this.#classifyGitChanges(changes, tree, upserts, deletes);
|
|
965
|
-
|
|
966
|
-
const entry = tree.get(dirtyPath);
|
|
967
|
-
if (entry && this.#policy.includes(dirtyPath) && entry.size <= this.#maxFileSize) upserts.set(dirtyPath, { entry });
|
|
968
|
-
else deletes.add(dirtyPath);
|
|
969
|
-
}
|
|
970
|
-
const eligibleTargetPaths = new Set(
|
|
971
|
-
[...tree.values()].filter((entry) => this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize).map((entry) => entry.path)
|
|
972
|
-
);
|
|
973
|
-
const indexedPaths = new Set(this.#database.allFilePaths());
|
|
1157
|
+
const indexedPaths = new Set(indexedByPath.keys());
|
|
974
1158
|
for (const targetPath of eligibleTargetPaths) {
|
|
975
|
-
|
|
976
|
-
|
|
1159
|
+
const indexed = indexedByPath.get(targetPath);
|
|
1160
|
+
const entry = tree.get(targetPath);
|
|
1161
|
+
if ((!indexed || indexed.sourceMode === "git" && indexed.blobOid !== entry.oid) && !upserts.has(targetPath)) {
|
|
1162
|
+
upserts.set(targetPath, { entry });
|
|
977
1163
|
}
|
|
978
1164
|
}
|
|
979
1165
|
for (const indexedPath of indexedPaths) {
|
|
980
1166
|
if (!eligibleTargetPaths.has(indexedPath)) deletes.add(indexedPath);
|
|
981
1167
|
}
|
|
982
1168
|
}
|
|
983
|
-
|
|
1169
|
+
for (const dirtyFile of workingFiles) {
|
|
1170
|
+
const entry = tree.get(dirtyFile.path);
|
|
1171
|
+
const previousEntry = dirtyFile.previousPath ? tree.get(dirtyFile.previousPath) : void 0;
|
|
1172
|
+
if (previousEntry && this.#policy.includes(previousEntry.path) && previousEntry.size <= this.#maxFileSize && (entry !== void 0 || !visibleWorkingPaths.has(dirtyFile.path))) {
|
|
1173
|
+
const currentPlan = entry ? upserts.get(dirtyFile.path) : void 0;
|
|
1174
|
+
upserts.delete(dirtyFile.path);
|
|
1175
|
+
upserts.delete(previousEntry.path);
|
|
1176
|
+
upserts.set(previousEntry.path, { entry: previousEntry, previousPath: dirtyFile.path });
|
|
1177
|
+
if (entry && this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize) {
|
|
1178
|
+
upserts.set(dirtyFile.path, currentPlan ?? { entry });
|
|
1179
|
+
}
|
|
1180
|
+
continue;
|
|
1181
|
+
}
|
|
1182
|
+
if (entry && this.#policy.includes(dirtyFile.path) && entry.size <= this.#maxFileSize) {
|
|
1183
|
+
const planned = upserts.get(dirtyFile.path);
|
|
1184
|
+
if (planned?.previousPath && !indexedByPath.has(planned.previousPath)) {
|
|
1185
|
+
upserts.set(dirtyFile.path, { entry, previousPath: dirtyFile.path });
|
|
1186
|
+
} else if (!planned) {
|
|
1187
|
+
upserts.set(dirtyFile.path, { entry });
|
|
1188
|
+
}
|
|
1189
|
+
continue;
|
|
1190
|
+
}
|
|
1191
|
+
if (previousEntry && this.#policy.includes(previousEntry.path) && previousEntry.size <= this.#maxFileSize) {
|
|
1192
|
+
upserts.set(previousEntry.path, { entry: previousEntry, previousPath: dirtyFile.path });
|
|
1193
|
+
} else {
|
|
1194
|
+
deletes.add(dirtyFile.path);
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
for (const [filePath, { entry, previousPath }] of upserts) {
|
|
1198
|
+
const indexed = indexedByPath.get(filePath);
|
|
1199
|
+
if (!previousPath && indexed?.sourceMode === "git" && indexed.blobOid === entry.oid) upserts.delete(filePath);
|
|
1200
|
+
}
|
|
1201
|
+
const workingPrepared = [];
|
|
1202
|
+
const workingDeletes = /* @__PURE__ */ new Set();
|
|
1203
|
+
const skippedWorkingPaths = /* @__PURE__ */ new Set();
|
|
1204
|
+
for (const change of workingChanges) {
|
|
1205
|
+
throwIfAborted(options.signal);
|
|
1206
|
+
if (change.status === "D") {
|
|
1207
|
+
upserts.delete(change.path);
|
|
1208
|
+
workingDeletes.add(change.path);
|
|
1209
|
+
continue;
|
|
1210
|
+
}
|
|
1211
|
+
let previousPath;
|
|
1212
|
+
let replacePath;
|
|
1213
|
+
if (change.status === "R") {
|
|
1214
|
+
const plannedBase = upserts.get(change.oldPath);
|
|
1215
|
+
const existingAtTarget = indexedByPath.get(change.oldPath);
|
|
1216
|
+
const existingAtFinal = indexedByPath.get(change.path);
|
|
1217
|
+
const continuingRename = existingAtFinal?.sourceMode === "working-tree" && existingAtFinal.previousPath === change.oldPath;
|
|
1218
|
+
replacePath = continuingRename ? change.path : existingAtTarget ? change.oldPath : existingAtFinal ? change.path : plannedBase?.previousPath && indexedByPath.has(plannedBase.previousPath) ? plannedBase.previousPath : void 0;
|
|
1219
|
+
previousPath = existingAtFinal?.previousPath ?? change.oldPath;
|
|
1220
|
+
upserts.delete(change.oldPath);
|
|
1221
|
+
upserts.delete(change.path);
|
|
1222
|
+
if (replacePath) deletes.delete(replacePath);
|
|
1223
|
+
workingDeletes.add(change.oldPath);
|
|
1224
|
+
} else {
|
|
1225
|
+
const planned = upserts.get(change.path);
|
|
1226
|
+
const existing = indexedByPath.get(change.path);
|
|
1227
|
+
replacePath = existing ? change.path : planned?.previousPath && indexedByPath.has(planned.previousPath) ? planned.previousPath : void 0;
|
|
1228
|
+
previousPath = existing?.previousPath ?? planned?.previousPath;
|
|
1229
|
+
upserts.delete(change.path);
|
|
1230
|
+
if (replacePath) deletes.delete(replacePath);
|
|
1231
|
+
}
|
|
1232
|
+
const relativePath = change.path;
|
|
1233
|
+
if (!this.#policy.includes(relativePath)) {
|
|
1234
|
+
if (replacePath) workingDeletes.add(replacePath);
|
|
1235
|
+
workingDeletes.add(relativePath);
|
|
1236
|
+
continue;
|
|
1237
|
+
}
|
|
1238
|
+
const absolutePath = path6.join(this.rootDir, relativePath);
|
|
1239
|
+
const info = await lstat2(absolutePath);
|
|
1240
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize) {
|
|
1241
|
+
if (replacePath) workingDeletes.add(replacePath);
|
|
1242
|
+
workingDeletes.add(relativePath);
|
|
1243
|
+
skippedWorkingPaths.add(relativePath);
|
|
1244
|
+
continue;
|
|
1245
|
+
}
|
|
1246
|
+
const content = await readFile(absolutePath);
|
|
1247
|
+
if (!replacePath && change.status === "A") {
|
|
1248
|
+
const contentHash = sha256(content.toString("utf8"));
|
|
1249
|
+
const renameCandidates = workingFiles.filter((file) => file.path !== relativePath && !visibleWorkingPaths.has(file.path) && deletes.has(file.path) && file.contentHash === contentHash);
|
|
1250
|
+
const renamed = renameCandidates.length === 1 ? renameCandidates[0] : void 0;
|
|
1251
|
+
if (renamed) {
|
|
1252
|
+
replacePath = renamed.path;
|
|
1253
|
+
previousPath = renamed.previousPath ?? renamed.path;
|
|
1254
|
+
deletes.delete(renamed.path);
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
workingPrepared.push(this.#prepareFile(relativePath, content, {
|
|
1258
|
+
blobOid: null,
|
|
1259
|
+
sourceMode: "working-tree",
|
|
1260
|
+
indexedCommit: null,
|
|
1261
|
+
...previousPath ? { previousPath } : {},
|
|
1262
|
+
...replacePath ? { replacePath } : {}
|
|
1263
|
+
}));
|
|
1264
|
+
}
|
|
984
1265
|
const prepared = [];
|
|
985
1266
|
for (const { entry, previousPath } of upserts.values()) {
|
|
986
1267
|
throwIfAborted(options.signal);
|
|
@@ -996,14 +1277,46 @@ var CodeIndex = class {
|
|
|
996
1277
|
...previousPath ? { replacePath: previousPath } : {}
|
|
997
1278
|
}));
|
|
998
1279
|
}
|
|
999
|
-
await this.#attachEmbeddings(prepared, options.signal);
|
|
1280
|
+
await this.#attachEmbeddings([...prepared, ...workingPrepared], options.signal);
|
|
1281
|
+
const noChanges = !reconcileAll && checkpoint === target && prepared.length === 0 && deletes.size === 0 && workingPrepared.length === 0 && workingDeletes.size === 0;
|
|
1000
1282
|
const resolvedAgain = await this.#git.resolveCommit(options.target ?? "HEAD");
|
|
1001
1283
|
if (resolvedAgain !== target) throw new CodeIndexError("Target Git ref changed while indexing; retry the update.");
|
|
1284
|
+
if (overlayWorkingTree) {
|
|
1285
|
+
const headAgain = await this.#git.resolveCommit("HEAD");
|
|
1286
|
+
const workingChangesAgain = await this.#git.workTreeChanges(target, indexArtifacts);
|
|
1287
|
+
if (headAgain !== head || JSON.stringify(workingChangesAgain) !== JSON.stringify(workingChanges)) {
|
|
1288
|
+
throw new CodeIndexError("Git HEAD or working-tree changes changed while indexing; retry the update.");
|
|
1289
|
+
}
|
|
1290
|
+
for (const file of workingPrepared) {
|
|
1291
|
+
const info = await lstat2(path6.join(this.rootDir, file.path));
|
|
1292
|
+
const content = await readFile(path6.join(this.rootDir, file.path));
|
|
1293
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size > this.#maxFileSize || sha256(content.toString("utf8")) !== file.contentHash) {
|
|
1294
|
+
throw new CodeIndexError(`Working-tree file changed while indexing: ${file.path}`);
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
for (const relativePath of skippedWorkingPaths) {
|
|
1298
|
+
const info = await lstat2(path6.join(this.rootDir, relativePath));
|
|
1299
|
+
if (info.isFile() && !info.isSymbolicLink() && info.size <= this.#maxFileSize) {
|
|
1300
|
+
throw new CodeIndexError(`Working-tree file changed while indexing: ${relativePath}`);
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
if (noChanges) {
|
|
1305
|
+
if (this.#database.getCheckpoint() !== checkpoint || this.#database.getGeneration() !== generation) {
|
|
1306
|
+
throw new CodeIndexError("Index changed while the update was being prepared; retry the update.");
|
|
1307
|
+
}
|
|
1308
|
+
return emptyStats(checkpoint);
|
|
1309
|
+
}
|
|
1002
1310
|
return this.#database.applyUpdate({
|
|
1003
1311
|
files: prepared,
|
|
1004
1312
|
deletePaths: [...deletes],
|
|
1313
|
+
workingTree: {
|
|
1314
|
+
files: workingPrepared,
|
|
1315
|
+
deletePaths: [...workingDeletes]
|
|
1316
|
+
},
|
|
1005
1317
|
checkpoint: target,
|
|
1006
|
-
expectedCheckpoint: checkpoint
|
|
1318
|
+
expectedCheckpoint: checkpoint,
|
|
1319
|
+
expectedGeneration: generation
|
|
1007
1320
|
});
|
|
1008
1321
|
}
|
|
1009
1322
|
#classifyGitChanges(changes, tree, upserts, deletes) {
|
|
@@ -1025,6 +1338,28 @@ var CodeIndex = class {
|
|
|
1025
1338
|
else deletes.add(change.path);
|
|
1026
1339
|
}
|
|
1027
1340
|
}
|
|
1341
|
+
async #workingTreeSourcePaths(signal) {
|
|
1342
|
+
const relativeIndexPath = path6.relative(this.rootDir, this.indexPath).replaceAll(path6.sep, "/");
|
|
1343
|
+
const indexArtifacts = new Set(relativeIndexPath && !relativeIndexPath.startsWith("../") ? [relativeIndexPath, `${relativeIndexPath}-shm`, `${relativeIndexPath}-wal`, `${relativeIndexPath}-journal`] : []);
|
|
1344
|
+
const directories = [""];
|
|
1345
|
+
const sourcePaths = [];
|
|
1346
|
+
while (directories.length > 0) {
|
|
1347
|
+
throwIfAborted(signal);
|
|
1348
|
+
const relativeDirectory = directories.shift();
|
|
1349
|
+
const entries = await readdir(path6.join(this.rootDir, relativeDirectory), { withFileTypes: true });
|
|
1350
|
+
entries.sort((left, right) => left.name.localeCompare(right.name));
|
|
1351
|
+
for (const entry of entries) {
|
|
1352
|
+
const relativePath = relativeDirectory ? `${relativeDirectory.replaceAll(path6.sep, "/")}/${entry.name}` : entry.name;
|
|
1353
|
+
if (entry.isDirectory()) {
|
|
1354
|
+
if (this.#policy.traversesDirectory(relativePath)) directories.push(relativePath);
|
|
1355
|
+
} else if (entry.isFile() && !indexArtifacts.has(relativePath) && this.#policy.includes(relativePath)) {
|
|
1356
|
+
sourcePaths.push(relativePath);
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
sourcePaths.sort();
|
|
1361
|
+
return sourcePaths;
|
|
1362
|
+
}
|
|
1028
1363
|
#prepareFile(relativePath, buffer, provenance) {
|
|
1029
1364
|
const content = buffer.toString("utf8");
|
|
1030
1365
|
const callables = parseCallables(relativePath, content, this.#onWarning);
|
|
@@ -1083,7 +1418,10 @@ var CodeIndex = class {
|
|
|
1083
1418
|
limit: options.limit,
|
|
1084
1419
|
minSimilarity: options.minSimilarity,
|
|
1085
1420
|
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
|
|
1086
|
-
excludeId: functionId
|
|
1421
|
+
excludeId: functionId,
|
|
1422
|
+
...options.excludePaths !== void 0 ? { excludePaths: options.excludePaths } : {},
|
|
1423
|
+
...options.minLines !== void 0 ? { minLines: options.minLines } : {},
|
|
1424
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {}
|
|
1087
1425
|
});
|
|
1088
1426
|
}
|
|
1089
1427
|
vectorForFunction(functionId) {
|
|
@@ -1093,15 +1431,15 @@ var CodeIndex = class {
|
|
|
1093
1431
|
return this.#database.searchVector(normalizeEmbeddingVector(vector, this.provider.profile.dimensions), options);
|
|
1094
1432
|
}
|
|
1095
1433
|
async sourceFunctions(filter) {
|
|
1096
|
-
const functions = filter.type === "all" ? this.allFunctions() : await this.#
|
|
1434
|
+
const functions = filter.type === "all" ? this.allFunctions() : filter.type === "uncommitted" ? this.allFunctions().filter((callable) => callable.sourceMode === "working-tree") : await this.#functionsChangedSince(filter.commit);
|
|
1097
1435
|
if (!filter.path) return functions;
|
|
1098
1436
|
const sourcePath = normalizeRelativePath(this.rootDir, filter.path);
|
|
1099
1437
|
return functions.filter((callable) => callable.path === sourcePath || callable.path.startsWith(`${sourcePath}/`));
|
|
1100
1438
|
}
|
|
1101
|
-
async #
|
|
1439
|
+
async #functionsChangedSince(baseRef) {
|
|
1102
1440
|
await this.#git.assertRepository();
|
|
1103
1441
|
const checkpoint = this.#database.getCheckpoint();
|
|
1104
|
-
if (!checkpoint) throw new CodeIndexError("
|
|
1442
|
+
if (!checkpoint) throw new CodeIndexError("changed-since requires an index with a Git checkpoint.");
|
|
1105
1443
|
const base = await this.#git.resolveCommit(baseRef);
|
|
1106
1444
|
if (!await this.#git.isAncestor(base, checkpoint)) {
|
|
1107
1445
|
throw new GitDivergenceError(base, checkpoint);
|
|
@@ -1111,18 +1449,17 @@ var CodeIndex = class {
|
|
|
1111
1449
|
const comparisonPaths = /* @__PURE__ */ new Map();
|
|
1112
1450
|
for (const change of changes) {
|
|
1113
1451
|
if (change.status === "D") continue;
|
|
1114
|
-
if (change.status === "R") comparisonPaths.set(change.path, change.oldPath);
|
|
1115
|
-
else if (change.status === "A" || change.status === "C") comparisonPaths.set(change.path, null);
|
|
1116
|
-
else comparisonPaths.set(change.path, change.path);
|
|
1452
|
+
if (change.status === "R") comparisonPaths.set(change.path, { basePath: change.oldPath, pathChanged: true });
|
|
1453
|
+
else if (change.status === "A" || change.status === "C") comparisonPaths.set(change.path, { basePath: null, pathChanged: false });
|
|
1454
|
+
else comparisonPaths.set(change.path, { basePath: change.path, pathChanged: false });
|
|
1117
1455
|
}
|
|
1118
1456
|
for (const dirtyFile of this.#database.getWorkingTreeFiles()) {
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
}
|
|
1457
|
+
const basePath = dirtyFile.previousPath ?? comparisonPaths.get(dirtyFile.path)?.basePath ?? dirtyFile.path;
|
|
1458
|
+
comparisonPaths.set(dirtyFile.path, { basePath, pathChanged: basePath !== dirtyFile.path });
|
|
1122
1459
|
}
|
|
1123
1460
|
const current = this.#database.functionsForPaths([...comparisonPaths.keys()]);
|
|
1124
1461
|
const baseFunctionsByCurrentPath = /* @__PURE__ */ new Map();
|
|
1125
|
-
for (const [currentPath, basePath] of comparisonPaths) {
|
|
1462
|
+
for (const [currentPath, { basePath }] of comparisonPaths) {
|
|
1126
1463
|
if (!basePath) {
|
|
1127
1464
|
baseFunctionsByCurrentPath.set(currentPath, []);
|
|
1128
1465
|
continue;
|
|
@@ -1136,19 +1473,25 @@ var CodeIndex = class {
|
|
|
1136
1473
|
baseFunctionsByCurrentPath.set(currentPath, parseCallables(basePath, content, this.#onWarning));
|
|
1137
1474
|
}
|
|
1138
1475
|
const currentByPath = groupBy(current, (callable) => callable.path);
|
|
1139
|
-
const
|
|
1476
|
+
const changedIds = /* @__PURE__ */ new Set();
|
|
1140
1477
|
for (const [currentPath, currentFunctions] of currentByPath) {
|
|
1141
|
-
|
|
1142
|
-
|
|
1478
|
+
const comparison = comparisonPaths.get(currentPath);
|
|
1479
|
+
for (const id of changedFunctionIds(
|
|
1480
|
+
currentFunctions,
|
|
1481
|
+
baseFunctionsByCurrentPath.get(currentPath) ?? [],
|
|
1482
|
+
comparison.pathChanged
|
|
1483
|
+
)) {
|
|
1484
|
+
changedIds.add(id);
|
|
1143
1485
|
}
|
|
1144
1486
|
}
|
|
1145
|
-
return current.filter((callable) =>
|
|
1487
|
+
return current.filter((callable) => changedIds.has(callable.id));
|
|
1146
1488
|
}
|
|
1147
1489
|
};
|
|
1148
|
-
function
|
|
1490
|
+
function changedFunctionIds(current, base, pathChanged) {
|
|
1491
|
+
if (pathChanged) return current.map((callable) => callable.id);
|
|
1149
1492
|
const currentGroups = groupBy(current, (callable) => `${callable.qualifiedName}\0${callable.kind}`);
|
|
1150
1493
|
const baseGroups = groupBy(base, (callable) => `${callable.qualifiedName}\0${callable.kind}`);
|
|
1151
|
-
const
|
|
1494
|
+
const changed = [];
|
|
1152
1495
|
for (const [key, currentGroup] of currentGroups) {
|
|
1153
1496
|
const baseGroup = baseGroups.get(key) ?? [];
|
|
1154
1497
|
const unmatchedBase = new Set(baseGroup.map((_, index) => index));
|
|
@@ -1162,13 +1505,9 @@ function addedFunctionIds(current, base) {
|
|
|
1162
1505
|
unmatchedCurrent.delete(currentIndex);
|
|
1163
1506
|
}
|
|
1164
1507
|
}
|
|
1165
|
-
for (const currentIndex of unmatchedCurrent)
|
|
1166
|
-
const baseIndex = unmatchedBase.values().next().value;
|
|
1167
|
-
if (baseIndex === void 0) added.push(currentGroup[currentIndex].id);
|
|
1168
|
-
else unmatchedBase.delete(baseIndex);
|
|
1169
|
-
}
|
|
1508
|
+
for (const currentIndex of unmatchedCurrent) changed.push(currentGroup[currentIndex].id);
|
|
1170
1509
|
}
|
|
1171
|
-
return
|
|
1510
|
+
return changed;
|
|
1172
1511
|
}
|
|
1173
1512
|
function normalizeProfile(profile) {
|
|
1174
1513
|
return {
|
|
@@ -1210,6 +1549,277 @@ function groupBy(values, keyFor) {
|
|
|
1210
1549
|
return groups;
|
|
1211
1550
|
}
|
|
1212
1551
|
|
|
1552
|
+
// src/analysis/cohesion.ts
|
|
1553
|
+
import path7 from "node:path";
|
|
1554
|
+
async function analyzeCohesion(options) {
|
|
1555
|
+
const neighbors = options.neighbors ?? 20;
|
|
1556
|
+
const limit = options.limit ?? 50;
|
|
1557
|
+
const minSimilarity = options.minSimilarity ?? 0.8;
|
|
1558
|
+
const minLines = options.minLines ?? 2;
|
|
1559
|
+
const sourceFilter = options.sourceFilter ?? { type: "all" };
|
|
1560
|
+
assertPositiveInteger(neighbors, "neighbors");
|
|
1561
|
+
assertPositiveInteger(limit, "limit");
|
|
1562
|
+
assertPositiveInteger(minLines, "minLines");
|
|
1563
|
+
if (!Number.isFinite(minSimilarity) || minSimilarity < -1 || minSimilarity >= 1) {
|
|
1564
|
+
throw new CodeIndexError("minSimilarity must be at least -1 and less than 1.");
|
|
1565
|
+
}
|
|
1566
|
+
if (options.maxSimilarity !== void 0 && (!Number.isFinite(options.maxSimilarity) || options.maxSimilarity <= minSimilarity)) {
|
|
1567
|
+
throw new CodeIndexError("maxSimilarity must be greater than minSimilarity.");
|
|
1568
|
+
}
|
|
1569
|
+
const nameRegex = compileNameRegex(options.nameRegex);
|
|
1570
|
+
const candidateFunctions = options.source.allFunctions().filter((callable) => callable.lineCount >= minLines && (!nameRegex || nameRegex.test(callable.qualifiedName)));
|
|
1571
|
+
const candidateIds = new Set(candidateFunctions.map((callable) => callable.id));
|
|
1572
|
+
const sourceFunctions = (await options.source.sourceFunctions(sourceFilter)).filter((callable) => candidateIds.has(callable.id));
|
|
1573
|
+
const neighborCache = /* @__PURE__ */ new Map();
|
|
1574
|
+
const neighborsFor = (callable) => {
|
|
1575
|
+
throwIfAborted(options.signal);
|
|
1576
|
+
const matches = options.source.similarToFunction(callable.id, {
|
|
1577
|
+
limit: neighbors,
|
|
1578
|
+
minSimilarity,
|
|
1579
|
+
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
|
|
1580
|
+
minLines,
|
|
1581
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {}
|
|
1582
|
+
}).filter((match) => candidateIds.has(match.function.id));
|
|
1583
|
+
neighborCache.set(callable.id, new Set(matches.map((match) => match.function.id)));
|
|
1584
|
+
return matches;
|
|
1585
|
+
};
|
|
1586
|
+
const edges = /* @__PURE__ */ new Map();
|
|
1587
|
+
for (let index = 0; index < sourceFunctions.length; index += 1) {
|
|
1588
|
+
const source = sourceFunctions[index];
|
|
1589
|
+
for (const match of neighborsFor(source)) {
|
|
1590
|
+
const [left, right] = orderedFunctions(source, match.function);
|
|
1591
|
+
const key = pairKey(left.id, right.id);
|
|
1592
|
+
const existing = edges.get(key);
|
|
1593
|
+
if (!existing || match.similarity > existing.similarity) {
|
|
1594
|
+
edges.set(key, { left, right, similarity: match.similarity });
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
options.onProgress?.({ completed: index + 1, total: sourceFunctions.length });
|
|
1598
|
+
}
|
|
1599
|
+
const scoredPairs = [...edges.values()].map((edge) => {
|
|
1600
|
+
const leftNeighbors = neighborCache.get(edge.left.id);
|
|
1601
|
+
const rightNeighbors = neighborCache.get(edge.right.id);
|
|
1602
|
+
const reciprocal = leftNeighbors && rightNeighbors ? leftNeighbors.has(edge.right.id) && rightNeighbors.has(edge.left.id) : null;
|
|
1603
|
+
return scorePair(edge, reciprocal, minSimilarity);
|
|
1604
|
+
}).sort(comparePairs);
|
|
1605
|
+
scoredPairs.forEach((pair, index) => {
|
|
1606
|
+
pair.rank = index + 1;
|
|
1607
|
+
});
|
|
1608
|
+
const repositoryScope = sourceFilter.type === "all" && sourceFilter.path === void 0;
|
|
1609
|
+
const summary = summarize(
|
|
1610
|
+
repositoryScope ? "repository" : "selected-sources",
|
|
1611
|
+
sourceFunctions.length,
|
|
1612
|
+
candidateFunctions.length,
|
|
1613
|
+
scoredPairs
|
|
1614
|
+
);
|
|
1615
|
+
const reportedPairs = scoredPairs.slice(0, limit);
|
|
1616
|
+
const files = summarizeFiles(sourceFunctions, scoredPairs).filter((file) => file.internalAffinity + file.sameFolderAffinity + file.externalAffinity > 0).slice(0, limit);
|
|
1617
|
+
const groups = buildGroups(reportedPairs);
|
|
1618
|
+
const status = options.source.status();
|
|
1619
|
+
return {
|
|
1620
|
+
schemaVersion: 1,
|
|
1621
|
+
repository: {
|
|
1622
|
+
generation: status.generation,
|
|
1623
|
+
gitCheckpoint: status.gitCheckpoint,
|
|
1624
|
+
embeddingProfile: status.embeddingProfile
|
|
1625
|
+
},
|
|
1626
|
+
parameters: {
|
|
1627
|
+
neighbors,
|
|
1628
|
+
limit,
|
|
1629
|
+
minSimilarity,
|
|
1630
|
+
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
|
|
1631
|
+
minLines,
|
|
1632
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {},
|
|
1633
|
+
sourceFilter
|
|
1634
|
+
},
|
|
1635
|
+
summary,
|
|
1636
|
+
pairs: reportedPairs,
|
|
1637
|
+
files,
|
|
1638
|
+
groups
|
|
1639
|
+
};
|
|
1640
|
+
}
|
|
1641
|
+
function cohesionLocation(leftPath, rightPath) {
|
|
1642
|
+
const normalizedLeft = leftPath.replaceAll("\\", "/");
|
|
1643
|
+
const normalizedRight = rightPath.replaceAll("\\", "/");
|
|
1644
|
+
const leftDirectory = directoryParts(normalizedLeft);
|
|
1645
|
+
const rightDirectory = directoryParts(normalizedRight);
|
|
1646
|
+
let commonLength = 0;
|
|
1647
|
+
while (commonLength < leftDirectory.length && commonLength < rightDirectory.length && leftDirectory[commonLength] === rightDirectory[commonLength]) {
|
|
1648
|
+
commonLength += 1;
|
|
1649
|
+
}
|
|
1650
|
+
const commonParts = leftDirectory.slice(0, commonLength);
|
|
1651
|
+
const commonAncestor = commonParts.length > 0 ? commonParts.join("/") : null;
|
|
1652
|
+
const sameFile = normalizedLeft === normalizedRight;
|
|
1653
|
+
const folderHops = sameFile ? 0 : leftDirectory.length - commonLength + rightDirectory.length - commonLength;
|
|
1654
|
+
return {
|
|
1655
|
+
category: sameFile ? "same-file" : folderHops === 0 ? "same-folder" : "different-folder",
|
|
1656
|
+
physicalDistance: sameFile ? 0 : 1 + folderHops,
|
|
1657
|
+
folderHops,
|
|
1658
|
+
commonAncestor,
|
|
1659
|
+
sourceTestPair: isTestPath(normalizedLeft) !== isTestPath(normalizedRight)
|
|
1660
|
+
};
|
|
1661
|
+
}
|
|
1662
|
+
function scorePair(edge, reciprocal, minSimilarity) {
|
|
1663
|
+
const location = cohesionLocation(edge.left.path, edge.right.path);
|
|
1664
|
+
const semanticWeight = clamp((edge.similarity - minSimilarity) / (1 - minSimilarity));
|
|
1665
|
+
const separationWeight = 1 - Math.exp(-location.physicalDistance / 2);
|
|
1666
|
+
return {
|
|
1667
|
+
rank: 0,
|
|
1668
|
+
left: edge.left,
|
|
1669
|
+
right: edge.right,
|
|
1670
|
+
similarity: edge.similarity,
|
|
1671
|
+
reciprocal,
|
|
1672
|
+
semanticWeight,
|
|
1673
|
+
separationWeight,
|
|
1674
|
+
cohesionGap: semanticWeight * separationWeight,
|
|
1675
|
+
location
|
|
1676
|
+
};
|
|
1677
|
+
}
|
|
1678
|
+
function summarize(scope, functionsAnalyzed, candidateFunctions, pairs) {
|
|
1679
|
+
const totalWeight = pairs.reduce((sum, pair) => sum + pair.semanticWeight, 0);
|
|
1680
|
+
const sameFileWeight = pairs.filter((pair) => pair.location.category === "same-file").reduce((sum, pair) => sum + pair.semanticWeight, 0);
|
|
1681
|
+
const sameFolderWeight = pairs.filter((pair) => pair.location.category === "same-folder").reduce((sum, pair) => sum + pair.semanticWeight, 0);
|
|
1682
|
+
const remoteWeight = pairs.filter((pair) => pair.location.category === "different-folder").reduce((sum, pair) => sum + pair.semanticWeight, 0);
|
|
1683
|
+
const weightedDistance = pairs.reduce(
|
|
1684
|
+
(sum, pair) => sum + pair.semanticWeight * pair.location.physicalDistance,
|
|
1685
|
+
0
|
|
1686
|
+
);
|
|
1687
|
+
return {
|
|
1688
|
+
scope,
|
|
1689
|
+
functionsAnalyzed,
|
|
1690
|
+
candidateFunctions,
|
|
1691
|
+
semanticEdges: pairs.length,
|
|
1692
|
+
sameFileRatio: ratio(sameFileWeight, totalWeight),
|
|
1693
|
+
sameFolderRatio: ratio(sameFolderWeight, totalWeight),
|
|
1694
|
+
remoteRatio: ratio(remoteWeight, totalWeight),
|
|
1695
|
+
weightedMeanDistance: ratio(weightedDistance, totalWeight)
|
|
1696
|
+
};
|
|
1697
|
+
}
|
|
1698
|
+
function summarizeFiles(sourceFunctions, pairs) {
|
|
1699
|
+
const sourceIds = new Set(sourceFunctions.map((callable) => callable.id));
|
|
1700
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1701
|
+
for (const callable of sourceFunctions) counts.set(callable.path, (counts.get(callable.path) ?? 0) + 1);
|
|
1702
|
+
const reports = /* @__PURE__ */ new Map();
|
|
1703
|
+
for (const [filePath, functionCount] of counts) {
|
|
1704
|
+
reports.set(filePath, {
|
|
1705
|
+
path: filePath,
|
|
1706
|
+
functionCount,
|
|
1707
|
+
internalAffinity: 0,
|
|
1708
|
+
sameFolderAffinity: 0,
|
|
1709
|
+
externalAffinity: 0,
|
|
1710
|
+
externalAffinityRatio: 0
|
|
1711
|
+
});
|
|
1712
|
+
}
|
|
1713
|
+
for (const pair of pairs) {
|
|
1714
|
+
if (sourceIds.has(pair.left.id)) addFileAffinity(reports.get(pair.left.path), pair.right, pair);
|
|
1715
|
+
if (sourceIds.has(pair.right.id)) addFileAffinity(reports.get(pair.right.path), pair.left, pair);
|
|
1716
|
+
}
|
|
1717
|
+
for (const report of reports.values()) {
|
|
1718
|
+
const total = report.internalAffinity + report.sameFolderAffinity + report.externalAffinity;
|
|
1719
|
+
report.externalAffinityRatio = ratio(report.externalAffinity, total);
|
|
1720
|
+
}
|
|
1721
|
+
return [...reports.values()].sort((left, right) => right.externalAffinityRatio - left.externalAffinityRatio || right.externalAffinity - left.externalAffinity || left.path.localeCompare(right.path));
|
|
1722
|
+
}
|
|
1723
|
+
function addFileAffinity(report, match, pair) {
|
|
1724
|
+
if (pair.location.category === "same-file") report.internalAffinity += pair.semanticWeight;
|
|
1725
|
+
else if (pair.location.category === "same-folder") report.sameFolderAffinity += pair.semanticWeight;
|
|
1726
|
+
else {
|
|
1727
|
+
report.externalAffinity += pair.semanticWeight;
|
|
1728
|
+
updateStrongestExternal(report, match, pair);
|
|
1729
|
+
}
|
|
1730
|
+
}
|
|
1731
|
+
function updateStrongestExternal(report, callable, pair) {
|
|
1732
|
+
if (!report.strongestExternalMatch || pair.cohesionGap > report.strongestExternalMatch.cohesionGap) {
|
|
1733
|
+
report.strongestExternalMatch = {
|
|
1734
|
+
function: callable,
|
|
1735
|
+
similarity: pair.similarity,
|
|
1736
|
+
cohesionGap: pair.cohesionGap
|
|
1737
|
+
};
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
function buildGroups(pairs) {
|
|
1741
|
+
const functions = /* @__PURE__ */ new Map();
|
|
1742
|
+
const neighbors = /* @__PURE__ */ new Map();
|
|
1743
|
+
for (const pair of pairs) {
|
|
1744
|
+
functions.set(pair.left.id, pair.left);
|
|
1745
|
+
functions.set(pair.right.id, pair.right);
|
|
1746
|
+
addNeighbor(neighbors, pair.left.id, pair.right.id);
|
|
1747
|
+
addNeighbor(neighbors, pair.right.id, pair.left.id);
|
|
1748
|
+
}
|
|
1749
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1750
|
+
const groups = [];
|
|
1751
|
+
for (const start of neighbors.keys()) {
|
|
1752
|
+
if (seen.has(start)) continue;
|
|
1753
|
+
const pending = [start];
|
|
1754
|
+
const memberIds = /* @__PURE__ */ new Set();
|
|
1755
|
+
while (pending.length > 0) {
|
|
1756
|
+
const id = pending.pop();
|
|
1757
|
+
if (seen.has(id)) continue;
|
|
1758
|
+
seen.add(id);
|
|
1759
|
+
memberIds.add(id);
|
|
1760
|
+
for (const neighbor of neighbors.get(id) ?? []) pending.push(neighbor);
|
|
1761
|
+
}
|
|
1762
|
+
const groupPairs = pairs.filter((pair) => memberIds.has(pair.left.id) && memberIds.has(pair.right.id));
|
|
1763
|
+
const members = [...memberIds].map((id) => functions.get(id)).sort(compareFunctions);
|
|
1764
|
+
groups.push({
|
|
1765
|
+
rank: 0,
|
|
1766
|
+
memberCount: members.length,
|
|
1767
|
+
fileCount: new Set(members.map((member) => member.path)).size,
|
|
1768
|
+
minimumEdgeSimilarity: Math.min(...groupPairs.map((pair) => pair.similarity)),
|
|
1769
|
+
maximumEdgeSimilarity: Math.max(...groupPairs.map((pair) => pair.similarity)),
|
|
1770
|
+
maximumPhysicalDistance: Math.max(...groupPairs.map((pair) => pair.location.physicalDistance)),
|
|
1771
|
+
maximumCohesionGap: Math.max(...groupPairs.map((pair) => pair.cohesionGap)),
|
|
1772
|
+
members
|
|
1773
|
+
});
|
|
1774
|
+
}
|
|
1775
|
+
groups.sort((left, right) => right.maximumCohesionGap - left.maximumCohesionGap || right.memberCount - left.memberCount || compareFunctions(left.members[0], right.members[0]));
|
|
1776
|
+
groups.forEach((group, index) => {
|
|
1777
|
+
group.rank = index + 1;
|
|
1778
|
+
});
|
|
1779
|
+
return groups;
|
|
1780
|
+
}
|
|
1781
|
+
function compileNameRegex(pattern) {
|
|
1782
|
+
if (pattern === void 0) return void 0;
|
|
1783
|
+
try {
|
|
1784
|
+
return new RegExp(pattern);
|
|
1785
|
+
} catch (error) {
|
|
1786
|
+
throw new CodeIndexError(`Invalid name regex: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
function directoryParts(filePath) {
|
|
1790
|
+
const directory = path7.posix.dirname(filePath.replaceAll("\\", "/"));
|
|
1791
|
+
return directory === "." ? [] : directory.split("/").filter(Boolean);
|
|
1792
|
+
}
|
|
1793
|
+
function isTestPath(filePath) {
|
|
1794
|
+
const normalized = filePath.replaceAll("\\", "/").toLowerCase();
|
|
1795
|
+
const segments = normalized.split("/");
|
|
1796
|
+
const fileName = segments.at(-1) ?? "";
|
|
1797
|
+
return segments.some((segment) => segment === "test" || segment === "tests" || segment === "__tests__") || /\.(?:test|spec)\.[cm]?[jt]sx?$/.test(fileName);
|
|
1798
|
+
}
|
|
1799
|
+
function orderedFunctions(left, right) {
|
|
1800
|
+
return compareFunctions(left, right) <= 0 ? [left, right] : [right, left];
|
|
1801
|
+
}
|
|
1802
|
+
function compareFunctions(left, right) {
|
|
1803
|
+
return left.path.localeCompare(right.path) || left.startLine - right.startLine || left.startColumn - right.startColumn || left.qualifiedName.localeCompare(right.qualifiedName) || left.id - right.id;
|
|
1804
|
+
}
|
|
1805
|
+
function comparePairs(left, right) {
|
|
1806
|
+
return right.cohesionGap - left.cohesionGap || right.similarity - left.similarity || Number(right.reciprocal === true) - Number(left.reciprocal === true) || compareFunctions(left.left, right.left) || compareFunctions(left.right, right.right);
|
|
1807
|
+
}
|
|
1808
|
+
function pairKey(leftId, rightId) {
|
|
1809
|
+
return leftId < rightId ? `${leftId}:${rightId}` : `${rightId}:${leftId}`;
|
|
1810
|
+
}
|
|
1811
|
+
function addNeighbor(neighbors, source, target) {
|
|
1812
|
+
const values = neighbors.get(source) ?? /* @__PURE__ */ new Set();
|
|
1813
|
+
values.add(target);
|
|
1814
|
+
neighbors.set(source, values);
|
|
1815
|
+
}
|
|
1816
|
+
function ratio(numerator, denominator) {
|
|
1817
|
+
return denominator === 0 ? 0 : numerator / denominator;
|
|
1818
|
+
}
|
|
1819
|
+
function clamp(value) {
|
|
1820
|
+
return Math.max(0, Math.min(1, value));
|
|
1821
|
+
}
|
|
1822
|
+
|
|
1213
1823
|
// src/embeddings/http.ts
|
|
1214
1824
|
async function requestEmbeddings(options) {
|
|
1215
1825
|
let lastError;
|
|
@@ -1318,8 +1928,8 @@ var OpenAIEmbeddingProvider = class {
|
|
|
1318
1928
|
constructor(options = {}) {
|
|
1319
1929
|
this.#apiKey = options.apiKey ?? process.env.OPENAI_API_KEY ?? "";
|
|
1320
1930
|
if (!this.#apiKey) throw new Error("OPENAI_API_KEY is required.");
|
|
1321
|
-
const model = options.model ?? "text-embedding-3-
|
|
1322
|
-
const dimensions = options.dimensions ??
|
|
1931
|
+
const model = options.model ?? "text-embedding-3-large";
|
|
1932
|
+
const dimensions = options.dimensions ?? 3072;
|
|
1323
1933
|
if (!Number.isInteger(dimensions) || dimensions < 1) throw new Error("dimensions must be a positive integer.");
|
|
1324
1934
|
this.profile = { provider: "openai", model, dimensions, strategyVersion: "callable-v1" };
|
|
1325
1935
|
this.#url = `${(options.baseUrl ?? "https://api.openai.com/v1").replace(/\/$/, "")}/embeddings`;
|
|
@@ -1357,8 +1967,87 @@ function formatSimilaritySummary(matches, source) {
|
|
|
1357
1967
|
if (!source) return lines.length > 0 ? lines.join("\n") : "No matches.";
|
|
1358
1968
|
return [functionName(source), ...lines.length > 0 ? lines : [" No matches."]].join("\n");
|
|
1359
1969
|
}
|
|
1970
|
+
function formatSimilarityClusters(results, sameIndex) {
|
|
1971
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
1972
|
+
const neighbors = /* @__PURE__ */ new Map();
|
|
1973
|
+
const edges = [];
|
|
1974
|
+
const connect = (left, right) => {
|
|
1975
|
+
const leftNeighbors = neighbors.get(left) ?? /* @__PURE__ */ new Set();
|
|
1976
|
+
const rightNeighbors = neighbors.get(right) ?? /* @__PURE__ */ new Set();
|
|
1977
|
+
leftNeighbors.add(right);
|
|
1978
|
+
rightNeighbors.add(left);
|
|
1979
|
+
neighbors.set(left, leftNeighbors);
|
|
1980
|
+
neighbors.set(right, rightNeighbors);
|
|
1981
|
+
};
|
|
1982
|
+
for (const result of results) {
|
|
1983
|
+
const sourceKey = `${sameIndex ? "index" : "source"}:${result.source.id}`;
|
|
1984
|
+
nodes.set(sourceKey, { function: result.source, role: "source" });
|
|
1985
|
+
for (const match of result.matches) {
|
|
1986
|
+
const matchKey = `${sameIndex ? "index" : "target"}:${match.function.id}`;
|
|
1987
|
+
nodes.set(matchKey, { function: match.function, role: sameIndex ? "source" : "target" });
|
|
1988
|
+
connect(sourceKey, matchKey);
|
|
1989
|
+
edges.push({ left: sourceKey, right: matchKey, similarity: match.similarity });
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1993
|
+
const clusters = [];
|
|
1994
|
+
for (const start of neighbors.keys()) {
|
|
1995
|
+
if (seen.has(start)) continue;
|
|
1996
|
+
const pending = [start];
|
|
1997
|
+
const keys = /* @__PURE__ */ new Set();
|
|
1998
|
+
while (pending.length > 0) {
|
|
1999
|
+
const key = pending.pop();
|
|
2000
|
+
if (seen.has(key)) continue;
|
|
2001
|
+
seen.add(key);
|
|
2002
|
+
keys.add(key);
|
|
2003
|
+
for (const neighbor of neighbors.get(key) ?? []) pending.push(neighbor);
|
|
2004
|
+
}
|
|
2005
|
+
const similarities = edges.filter((edge) => keys.has(edge.left) && keys.has(edge.right)).map((edge) => edge.similarity);
|
|
2006
|
+
clusters.push({
|
|
2007
|
+
members: [...keys].map((key) => nodes.get(key)).sort((left, right) => clusterFunctionName(left, sameIndex).localeCompare(clusterFunctionName(right, sameIndex))),
|
|
2008
|
+
min: Math.min(...similarities),
|
|
2009
|
+
max: Math.max(...similarities)
|
|
2010
|
+
});
|
|
2011
|
+
}
|
|
2012
|
+
clusters.sort((left, right) => right.members.length - left.members.length || clusterFunctionName(left.members[0], sameIndex).localeCompare(clusterFunctionName(right.members[0], sameIndex)));
|
|
2013
|
+
if (clusters.length === 0) return "No clusters.";
|
|
2014
|
+
return clusters.map((cluster, index) => {
|
|
2015
|
+
const similarity = cluster.min === cluster.max ? cluster.min.toFixed(4) : `${cluster.min.toFixed(4)}-${cluster.max.toFixed(4)}`;
|
|
2016
|
+
return [
|
|
2017
|
+
`Cluster ${index + 1} (${cluster.members.length} functions, similarity ${similarity})`,
|
|
2018
|
+
...cluster.members.map((member) => ` ${clusterFunctionName(member, sameIndex)}`)
|
|
2019
|
+
].join("\n");
|
|
2020
|
+
}).join("\n\n");
|
|
2021
|
+
}
|
|
2022
|
+
function formatCohesionSummary(report) {
|
|
2023
|
+
const { summary } = report;
|
|
2024
|
+
const edgeLabel = summary.semanticEdges === 1 ? "edge" : "edges";
|
|
2025
|
+
const lines = [
|
|
2026
|
+
`Cohesion: ${summary.functionsAnalyzed} functions analyzed, ${summary.semanticEdges} semantic ${edgeLabel}`,
|
|
2027
|
+
` same file ${(summary.sameFileRatio * 100).toFixed(1)}% same folder ${(summary.sameFolderRatio * 100).toFixed(1)}% remote ${(summary.remoteRatio * 100).toFixed(1)}% mean distance ${summary.weightedMeanDistance.toFixed(2)}`
|
|
2028
|
+
];
|
|
2029
|
+
if (report.pairs.length === 0) return [...lines, "No cohesion gaps."].join("\n");
|
|
2030
|
+
lines.push("");
|
|
2031
|
+
for (const pair of report.pairs) {
|
|
2032
|
+
lines.push(
|
|
2033
|
+
`${pair.rank}. gap ${pair.cohesionGap.toFixed(4)} similarity ${pair.similarity.toFixed(4)} distance ${pair.location.physicalDistance}${pair.reciprocal ? " reciprocal" : ""}`,
|
|
2034
|
+
` ${functionLocation(pair.left)}`,
|
|
2035
|
+
` ${functionLocation(pair.right)}`
|
|
2036
|
+
);
|
|
2037
|
+
}
|
|
2038
|
+
return lines.join("\n");
|
|
2039
|
+
}
|
|
2040
|
+
function clusterFunctionName(member, sameIndex) {
|
|
2041
|
+
const role = sameIndex ? "" : `[${member.role}] `;
|
|
2042
|
+
return `${role}${member.function.path}:${member.function.startLine}:${member.function.startColumn} :: ${member.function.qualifiedName}`;
|
|
2043
|
+
}
|
|
2044
|
+
function functionLocation(value) {
|
|
2045
|
+
return `${value.path}:${value.startLine}:${value.startColumn} :: ${value.qualifiedName}`;
|
|
2046
|
+
}
|
|
1360
2047
|
|
|
1361
2048
|
// src/search/cross-search.ts
|
|
2049
|
+
import { realpath, stat } from "node:fs/promises";
|
|
2050
|
+
import path8 from "node:path";
|
|
1362
2051
|
async function* crossSearch(options) {
|
|
1363
2052
|
const target = options.target ?? options.source;
|
|
1364
2053
|
const sourceProfile = JSON.stringify(options.source.status().embeddingProfile);
|
|
@@ -1368,21 +2057,55 @@ async function* crossSearch(options) {
|
|
|
1368
2057
|
}
|
|
1369
2058
|
const limit = options.limitPerFunction ?? 5;
|
|
1370
2059
|
assertPositiveInteger(limit, "limitPerFunction");
|
|
1371
|
-
const
|
|
1372
|
-
|
|
2060
|
+
const minLines = options.minLines ?? 2;
|
|
2061
|
+
assertPositiveInteger(minLines, "minLines");
|
|
2062
|
+
const nameRegex = compileNameRegex2(options.nameRegex);
|
|
2063
|
+
const sourceFunctions = (await options.source.sourceFunctions(options.sourceFilter ?? { type: "all" })).filter((callable) => callable.lineCount >= minLines && (!nameRegex || nameRegex.test(callable.qualifiedName)));
|
|
2064
|
+
const sameIndex = target.indexPath === options.source.indexPath || await fileIdentity(target.indexPath) === await fileIdentity(options.source.indexPath);
|
|
2065
|
+
const sourceRoot = options.crossFileOnly ? await canonicalRoot(options.source.rootDir) : void 0;
|
|
2066
|
+
const targetRoot = options.crossFileOnly ? sameIndex ? sourceRoot : await canonicalRoot(target.rootDir) : void 0;
|
|
2067
|
+
const canonicalFiles = /* @__PURE__ */ new Map();
|
|
2068
|
+
const canonicalFile = (root, filePath) => {
|
|
2069
|
+
const absolutePath = path8.resolve(root, filePath);
|
|
2070
|
+
let result = canonicalFiles.get(absolutePath);
|
|
2071
|
+
if (!result) {
|
|
2072
|
+
result = fileIdentity(absolutePath);
|
|
2073
|
+
canonicalFiles.set(absolutePath, result);
|
|
2074
|
+
}
|
|
2075
|
+
return result;
|
|
2076
|
+
};
|
|
2077
|
+
const targetPathsByCanonicalFile = /* @__PURE__ */ new Map();
|
|
2078
|
+
if (targetRoot) {
|
|
2079
|
+
const targetPaths = [...new Set(target.allFunctions().map((callable) => callable.path))];
|
|
2080
|
+
await Promise.all(targetPaths.map(async (targetPath) => {
|
|
2081
|
+
const canonicalPath = await canonicalFile(targetRoot, targetPath);
|
|
2082
|
+
const paths = targetPathsByCanonicalFile.get(canonicalPath) ?? [];
|
|
2083
|
+
paths.push(targetPath);
|
|
2084
|
+
targetPathsByCanonicalFile.set(canonicalPath, paths);
|
|
2085
|
+
}));
|
|
2086
|
+
}
|
|
1373
2087
|
const seenPairs = /* @__PURE__ */ new Set();
|
|
1374
2088
|
for (let index = 0; index < sourceFunctions.length; index += 1) {
|
|
1375
2089
|
throwIfAborted(options.signal);
|
|
1376
2090
|
const source = sourceFunctions[index];
|
|
1377
2091
|
const vector = options.source.vectorForFunction(source.id);
|
|
2092
|
+
const canonicalSourceFile = sourceRoot ? await canonicalFile(sourceRoot, source.path) : void 0;
|
|
2093
|
+
const excludedTargetPaths = canonicalSourceFile ? targetPathsByCanonicalFile.get(canonicalSourceFile) : void 0;
|
|
2094
|
+
const excludePaths = excludedTargetPaths ? { excludePaths: excludedTargetPaths } : {};
|
|
1378
2095
|
const candidates = sameIndex ? options.source.similarToFunction(source.id, {
|
|
1379
2096
|
limit,
|
|
1380
2097
|
minSimilarity: options.minSimilarity ?? -1,
|
|
1381
|
-
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {}
|
|
2098
|
+
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
|
|
2099
|
+
minLines,
|
|
2100
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {},
|
|
2101
|
+
...excludePaths
|
|
1382
2102
|
}) : target.searchByVector(vector, {
|
|
1383
2103
|
limit,
|
|
1384
2104
|
minSimilarity: options.minSimilarity ?? -1,
|
|
1385
|
-
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {}
|
|
2105
|
+
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
|
|
2106
|
+
minLines,
|
|
2107
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {},
|
|
2108
|
+
...excludePaths
|
|
1386
2109
|
});
|
|
1387
2110
|
const matches = sameIndex && !options.includeSymmetricDuplicates ? candidates.filter((match) => {
|
|
1388
2111
|
const pair = source.id < match.function.id ? `${source.id}:${match.function.id}` : `${match.function.id}:${source.id}`;
|
|
@@ -1394,33 +2117,72 @@ async function* crossSearch(options) {
|
|
|
1394
2117
|
options.onProgress?.({ completed: index + 1, total: sourceFunctions.length });
|
|
1395
2118
|
}
|
|
1396
2119
|
}
|
|
2120
|
+
function compileNameRegex2(pattern) {
|
|
2121
|
+
if (pattern === void 0) return void 0;
|
|
2122
|
+
try {
|
|
2123
|
+
return new RegExp(pattern);
|
|
2124
|
+
} catch (error) {
|
|
2125
|
+
throw new CodeIndexError(`Invalid name regex: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
2126
|
+
}
|
|
2127
|
+
}
|
|
2128
|
+
async function canonicalRoot(root) {
|
|
2129
|
+
try {
|
|
2130
|
+
return await realpath(root);
|
|
2131
|
+
} catch {
|
|
2132
|
+
return path8.resolve(root);
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
async function fileIdentity(filePath) {
|
|
2136
|
+
try {
|
|
2137
|
+
const metadata = await stat(filePath);
|
|
2138
|
+
return `inode:${metadata.dev}:${metadata.ino}`;
|
|
2139
|
+
} catch {
|
|
2140
|
+
return `path:${path8.resolve(filePath)}`;
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
1397
2143
|
|
|
1398
2144
|
// src/cli.ts
|
|
1399
|
-
var parsed =
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
2145
|
+
var parsed = (() => {
|
|
2146
|
+
try {
|
|
2147
|
+
return parseArgs({
|
|
2148
|
+
args: process.argv.slice(2),
|
|
2149
|
+
allowPositionals: true,
|
|
2150
|
+
strict: true,
|
|
2151
|
+
options: {
|
|
2152
|
+
root: { type: "string", default: process.cwd() },
|
|
2153
|
+
config: { type: "string" },
|
|
2154
|
+
index: { type: "string" },
|
|
2155
|
+
provider: { type: "string" },
|
|
2156
|
+
model: { type: "string" },
|
|
2157
|
+
dimensions: { type: "string" },
|
|
2158
|
+
target: { type: "string" },
|
|
2159
|
+
"target-root": { type: "string" },
|
|
2160
|
+
"target-index": { type: "string" },
|
|
2161
|
+
"target-config": { type: "string" },
|
|
2162
|
+
"changed-since": { type: "string" },
|
|
2163
|
+
uncommitted: { type: "boolean", default: false },
|
|
2164
|
+
"source-path": { type: "string" },
|
|
2165
|
+
"min-lines": { type: "string" },
|
|
2166
|
+
regex: { type: "string" },
|
|
2167
|
+
limit: { type: "string" },
|
|
2168
|
+
neighbors: { type: "string" },
|
|
2169
|
+
threshold: { type: "string" },
|
|
2170
|
+
format: { type: "string" },
|
|
2171
|
+
"include-source": { type: "boolean", default: false },
|
|
2172
|
+
"include-symmetric-duplicates": { type: "boolean", default: false },
|
|
2173
|
+
"cross-file-only": { type: "boolean", default: false },
|
|
2174
|
+
"rebuild-on-divergence": { type: "boolean", default: false },
|
|
2175
|
+
"force-reindex": { type: "boolean", default: false },
|
|
2176
|
+
"no-reindex": { type: "boolean", default: false },
|
|
2177
|
+
help: { type: "boolean", short: "h", default: false }
|
|
2178
|
+
}
|
|
2179
|
+
});
|
|
2180
|
+
} catch (error) {
|
|
2181
|
+
process.stderr.write(`slopdex: ${error instanceof Error ? error.message : String(error)}
|
|
2182
|
+
`);
|
|
2183
|
+
process.exit(2);
|
|
1422
2184
|
}
|
|
1423
|
-
});
|
|
2185
|
+
})();
|
|
1424
2186
|
var [command, ...positionals] = parsed.positionals;
|
|
1425
2187
|
if (parsed.values.help || !command) {
|
|
1426
2188
|
printHelp();
|
|
@@ -1433,7 +2195,8 @@ void main().catch((error) => {
|
|
|
1433
2195
|
process.exitCode = error instanceof CodeIndexError ? 2 : 1;
|
|
1434
2196
|
});
|
|
1435
2197
|
async function main() {
|
|
1436
|
-
|
|
2198
|
+
validateInvocation();
|
|
2199
|
+
const rootDir = path9.resolve(parsed.values.root);
|
|
1437
2200
|
const config = loadConfig(rootDir, parsed.values.config);
|
|
1438
2201
|
const provider = createProvider(config);
|
|
1439
2202
|
const indexOptions = {
|
|
@@ -1447,10 +2210,14 @@ async function main() {
|
|
|
1447
2210
|
...config.maxFileSize ? { maxFileSize: config.maxFileSize } : {},
|
|
1448
2211
|
...config.embeddingBatchSize ? { embeddingBatchSize: config.embeddingBatchSize } : {}
|
|
1449
2212
|
};
|
|
1450
|
-
const
|
|
2213
|
+
const updateTarget = command === "update-git" ? parsed.values.target ?? "HEAD" : "HEAD";
|
|
2214
|
+
const updateStats = await ensureIndexUpdated(
|
|
1451
2215
|
indexOptions,
|
|
1452
2216
|
"index",
|
|
1453
|
-
|
|
2217
|
+
updateTarget,
|
|
2218
|
+
parsed.values["rebuild-on-divergence"],
|
|
2219
|
+
parsed.values["force-reindex"],
|
|
2220
|
+
parsed.values["no-reindex"]
|
|
1454
2221
|
);
|
|
1455
2222
|
const index = new CodeIndex(indexOptions);
|
|
1456
2223
|
try {
|
|
@@ -1467,11 +2234,7 @@ async function main() {
|
|
|
1467
2234
|
printJson(await index.updateFiles({ delete: positionals }));
|
|
1468
2235
|
break;
|
|
1469
2236
|
case "update-git": {
|
|
1470
|
-
|
|
1471
|
-
...parsed.values.target ? { target: parsed.values.target } : {},
|
|
1472
|
-
rebuildOnDivergence: parsed.values["rebuild-on-divergence"]
|
|
1473
|
-
});
|
|
1474
|
-
printJson(stats);
|
|
2237
|
+
printJson(updateStats);
|
|
1475
2238
|
break;
|
|
1476
2239
|
}
|
|
1477
2240
|
case "search": {
|
|
@@ -1484,7 +2247,9 @@ async function main() {
|
|
|
1484
2247
|
minSimilarity: threshold.min,
|
|
1485
2248
|
...threshold.max !== void 0 ? { maxSimilarity: threshold.max } : {}
|
|
1486
2249
|
});
|
|
1487
|
-
|
|
2250
|
+
const format = outputFormat("json");
|
|
2251
|
+
if (format === "clusters") throw new CodeIndexError("clusters format is only available for cross-search.");
|
|
2252
|
+
if (format === "summary") process.stdout.write(`${formatSimilaritySummary(results)}
|
|
1488
2253
|
`);
|
|
1489
2254
|
else printJson(results.map(presentMatch));
|
|
1490
2255
|
break;
|
|
@@ -1492,6 +2257,25 @@ async function main() {
|
|
|
1492
2257
|
case "cross-search":
|
|
1493
2258
|
await runCrossSearch(index, indexOptions, provider);
|
|
1494
2259
|
break;
|
|
2260
|
+
case "cohesion": {
|
|
2261
|
+
const threshold = cohesionThreshold();
|
|
2262
|
+
const report = await analyzeCohesion({
|
|
2263
|
+
source: index,
|
|
2264
|
+
sourceFilter: crossSearchSourceFilter(),
|
|
2265
|
+
neighbors: positiveIntegerOption(parsed.values.neighbors, 20, "neighbors"),
|
|
2266
|
+
limit: positiveIntegerOption(parsed.values.limit, 50, "limit"),
|
|
2267
|
+
minSimilarity: threshold.min,
|
|
2268
|
+
...threshold.max !== void 0 ? { maxSimilarity: threshold.max } : {},
|
|
2269
|
+
minLines: minimumLines(),
|
|
2270
|
+
...parsed.values.regex !== void 0 ? { nameRegex: parsed.values.regex } : {}
|
|
2271
|
+
});
|
|
2272
|
+
const format = outputFormat("json");
|
|
2273
|
+
if (format === "clusters") throw new CodeIndexError("clusters format is only available for cross-search.");
|
|
2274
|
+
if (format === "summary") process.stdout.write(`${formatCohesionSummary(report)}
|
|
2275
|
+
`);
|
|
2276
|
+
else printJson(presentCohesionReport(report, parsed.values["include-source"]));
|
|
2277
|
+
break;
|
|
2278
|
+
}
|
|
1495
2279
|
default:
|
|
1496
2280
|
throw new CodeIndexError(`Unknown command: ${command}`);
|
|
1497
2281
|
}
|
|
@@ -1505,33 +2289,54 @@ async function runCrossSearch(source, sourceOptions, provider) {
|
|
|
1505
2289
|
if (targetRoot && !targetPath || !targetRoot && targetPath) {
|
|
1506
2290
|
throw new CodeIndexError("Cross-index search requires both --target-root and --target-index.");
|
|
1507
2291
|
}
|
|
1508
|
-
const
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
2292
|
+
const targetRootDir = targetRoot ? path9.resolve(targetRoot) : void 0;
|
|
2293
|
+
const resolvedTargetPath = targetPath ? path9.resolve(targetPath) : void 0;
|
|
2294
|
+
const usesSourceAsTarget = targetRootDir === source.rootDir && resolvedTargetPath === source.indexPath;
|
|
2295
|
+
const targetConfig = targetRootDir && !usesSourceAsTarget ? loadConfig(targetRootDir, parsed.values["target-config"]) : void 0;
|
|
2296
|
+
const targetOptions = targetRootDir && resolvedTargetPath && !usesSourceAsTarget ? {
|
|
2297
|
+
rootDir: targetRootDir,
|
|
2298
|
+
indexPath: resolvedTargetPath,
|
|
2299
|
+
provider,
|
|
2300
|
+
...sourceOptions.onWarning ? { onWarning: sourceOptions.onWarning } : {},
|
|
2301
|
+
...targetConfig?.include ? { include: targetConfig.include } : {},
|
|
2302
|
+
...targetConfig?.exclude ? { exclude: targetConfig.exclude } : {},
|
|
2303
|
+
...targetConfig?.maxFileSize ? { maxFileSize: targetConfig.maxFileSize } : {},
|
|
2304
|
+
...targetConfig?.embeddingBatchSize ? { embeddingBatchSize: targetConfig.embeddingBatchSize } : {}
|
|
1513
2305
|
} : void 0;
|
|
1514
|
-
if (targetOptions)
|
|
2306
|
+
if (targetOptions) {
|
|
2307
|
+
await ensureIndexUpdated(
|
|
2308
|
+
targetOptions,
|
|
2309
|
+
"target index",
|
|
2310
|
+
"HEAD",
|
|
2311
|
+
parsed.values["rebuild-on-divergence"],
|
|
2312
|
+
parsed.values["force-reindex"],
|
|
2313
|
+
parsed.values["no-reindex"]
|
|
2314
|
+
);
|
|
2315
|
+
}
|
|
1515
2316
|
const target = targetOptions ? new CodeIndex({ ...targetOptions, readOnly: true }) : void 0;
|
|
1516
|
-
const format = outputFormat();
|
|
2317
|
+
const format = outputFormat("clusters");
|
|
1517
2318
|
const threshold = similarityThreshold();
|
|
2319
|
+
const searchOptions = {
|
|
2320
|
+
source,
|
|
2321
|
+
...target ? { target } : {},
|
|
2322
|
+
sourceFilter: crossSearchSourceFilter(),
|
|
2323
|
+
limitPerFunction: numberOption(parsed.values.limit, 5, "limit"),
|
|
2324
|
+
minSimilarity: threshold.min,
|
|
2325
|
+
...threshold.max !== void 0 ? { maxSimilarity: threshold.max } : {},
|
|
2326
|
+
includeSymmetricDuplicates: parsed.values["include-symmetric-duplicates"],
|
|
2327
|
+
crossFileOnly: parsed.values["cross-file-only"],
|
|
2328
|
+
minLines: minimumLines(),
|
|
2329
|
+
...parsed.values.regex !== void 0 ? { nameRegex: parsed.values.regex } : {}
|
|
2330
|
+
};
|
|
1518
2331
|
try {
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
type: "all",
|
|
1528
|
-
...parsed.values["source-path"] ? { path: parsed.values["source-path"] } : {}
|
|
1529
|
-
},
|
|
1530
|
-
limitPerFunction: numberOption(parsed.values.limit, 5, "limit"),
|
|
1531
|
-
minSimilarity: threshold.min,
|
|
1532
|
-
...threshold.max !== void 0 ? { maxSimilarity: threshold.max } : {},
|
|
1533
|
-
includeSymmetricDuplicates: parsed.values["include-symmetric-duplicates"]
|
|
1534
|
-
})) {
|
|
2332
|
+
if (format === "clusters") {
|
|
2333
|
+
const results = [];
|
|
2334
|
+
for await (const result of crossSearch(searchOptions)) results.push(result);
|
|
2335
|
+
process.stdout.write(`${formatSimilarityClusters(results, !target)}
|
|
2336
|
+
`);
|
|
2337
|
+
return;
|
|
2338
|
+
}
|
|
2339
|
+
for await (const result of crossSearch(searchOptions)) {
|
|
1535
2340
|
if (format === "summary") {
|
|
1536
2341
|
process.stdout.write(`${formatSimilaritySummary(result.matches, result.source)}
|
|
1537
2342
|
`);
|
|
@@ -1547,26 +2352,86 @@ async function runCrossSearch(source, sourceOptions, provider) {
|
|
|
1547
2352
|
target?.close();
|
|
1548
2353
|
}
|
|
1549
2354
|
}
|
|
1550
|
-
async function
|
|
1551
|
-
const
|
|
2355
|
+
async function ensureIndexUpdated(options, label, target, rebuildOnDivergence, forceRebuild, noReindex) {
|
|
2356
|
+
const initialized = await initializeMissingIndex(options, label, target, noReindex);
|
|
2357
|
+
if (initialized) return initialized;
|
|
2358
|
+
try {
|
|
2359
|
+
const index = new CodeIndex(options);
|
|
2360
|
+
try {
|
|
2361
|
+
return await refreshIndex(index, label, target, rebuildOnDivergence, noReindex);
|
|
2362
|
+
} finally {
|
|
2363
|
+
index.close();
|
|
2364
|
+
}
|
|
2365
|
+
} catch (error) {
|
|
2366
|
+
if (!forceRebuild || !(error instanceof IncompatibleIndexError)) throw error;
|
|
2367
|
+
process.stderr.write(
|
|
2368
|
+
`slopdex: warning: ${label} is incompatible (${error.message}); rebuilding automatically because --force-reindex was specified.
|
|
2369
|
+
`
|
|
2370
|
+
);
|
|
2371
|
+
const indexPath = resolveIndexPath(options);
|
|
2372
|
+
removeIndexArtifacts(indexPath);
|
|
2373
|
+
return initializeIndex(options, indexPath, label, target, noReindex);
|
|
2374
|
+
}
|
|
2375
|
+
}
|
|
2376
|
+
async function initializeMissingIndex(options, label, target, noReindex) {
|
|
2377
|
+
const indexPath = resolveIndexPath(options);
|
|
1552
2378
|
if (existsSync(indexPath)) return null;
|
|
1553
|
-
process.stderr.write(
|
|
1554
|
-
`
|
|
2379
|
+
process.stderr.write(
|
|
2380
|
+
`slopdex: ${label} not found at ${indexPath}; initializing automatically from ${target}${noReindex ? "" : " and the working tree"}.
|
|
2381
|
+
`
|
|
2382
|
+
);
|
|
2383
|
+
return initializeIndex(options, indexPath, label, target, noReindex);
|
|
2384
|
+
}
|
|
2385
|
+
async function initializeIndex(options, indexPath, label, target, noReindex) {
|
|
1555
2386
|
const index = new CodeIndex({ ...options, indexPath });
|
|
1556
2387
|
let initialized = false;
|
|
1557
2388
|
try {
|
|
1558
|
-
const stats = await index
|
|
2389
|
+
const stats = await refreshIndex(index, label, target, false, noReindex);
|
|
1559
2390
|
initialized = true;
|
|
1560
2391
|
return stats;
|
|
1561
2392
|
} finally {
|
|
1562
2393
|
index.close();
|
|
1563
2394
|
if (!initialized) {
|
|
1564
|
-
|
|
2395
|
+
removeIndexArtifacts(indexPath);
|
|
1565
2396
|
}
|
|
1566
2397
|
}
|
|
1567
2398
|
}
|
|
2399
|
+
async function refreshIndex(index, label, target, rebuildOnDivergence, noReindex) {
|
|
2400
|
+
try {
|
|
2401
|
+
return await index.updateFromGit({ target, rebuildOnDivergence, includeWorkingTree: !noReindex });
|
|
2402
|
+
} catch (error) {
|
|
2403
|
+
if (!(error instanceof GitUnavailableError)) throw error;
|
|
2404
|
+
const status = index.status();
|
|
2405
|
+
if (noReindex && status.fileCount > 0) {
|
|
2406
|
+
process.stderr.write(
|
|
2407
|
+
`slopdex: warning: no Git repository is available for ${label}; full working-tree re-index skipped because --no-reindex was specified.
|
|
2408
|
+
`
|
|
2409
|
+
);
|
|
2410
|
+
return {
|
|
2411
|
+
filesUpdated: 0,
|
|
2412
|
+
filesDeleted: 0,
|
|
2413
|
+
functionsAdded: 0,
|
|
2414
|
+
functionsUpdated: 0,
|
|
2415
|
+
functionsDeleted: 0,
|
|
2416
|
+
embeddingsCreated: 0,
|
|
2417
|
+
checkpoint: status.gitCheckpoint
|
|
2418
|
+
};
|
|
2419
|
+
}
|
|
2420
|
+
process.stderr.write(
|
|
2421
|
+
`slopdex: warning: no Git repository is available for ${label}; re-indexing all source files from the working tree.
|
|
2422
|
+
`
|
|
2423
|
+
);
|
|
2424
|
+
return await index.updateFromWorkingTree();
|
|
2425
|
+
}
|
|
2426
|
+
}
|
|
2427
|
+
function resolveIndexPath(options) {
|
|
2428
|
+
return path9.resolve(options.indexPath ?? path9.join(path9.resolve(options.rootDir), ".slopdex", "index.sqlite"));
|
|
2429
|
+
}
|
|
2430
|
+
function removeIndexArtifacts(indexPath) {
|
|
2431
|
+
for (const suffix of ["", "-shm", "-wal", "-journal"]) rmSync(`${indexPath}${suffix}`, { force: true });
|
|
2432
|
+
}
|
|
1568
2433
|
function loadConfig(rootDir, configuredPath) {
|
|
1569
|
-
const configPath =
|
|
2434
|
+
const configPath = path9.resolve(configuredPath ?? path9.join(rootDir, ".slopdex", "config.json"));
|
|
1570
2435
|
if (!existsSync(configPath)) return commandLineConfig({});
|
|
1571
2436
|
const value = JSON.parse(readFileSync(configPath, "utf8"));
|
|
1572
2437
|
return commandLineConfig(value);
|
|
@@ -1603,27 +2468,108 @@ function numberOption(value, defaultValue, name) {
|
|
|
1603
2468
|
if (!Number.isFinite(parsedValue)) throw new CodeIndexError(`${name} must be a number.`);
|
|
1604
2469
|
return parsedValue;
|
|
1605
2470
|
}
|
|
1606
|
-
function
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
2471
|
+
function validateInvocation() {
|
|
2472
|
+
switch (command) {
|
|
2473
|
+
case "status":
|
|
2474
|
+
case "update-git":
|
|
2475
|
+
return;
|
|
2476
|
+
case "update-files":
|
|
2477
|
+
if (positionals.length === 0) throw new CodeIndexError("update-files requires at least one path.");
|
|
2478
|
+
return;
|
|
2479
|
+
case "delete-files":
|
|
2480
|
+
if (positionals.length === 0) throw new CodeIndexError("delete-files requires at least one path.");
|
|
2481
|
+
return;
|
|
2482
|
+
case "search": {
|
|
2483
|
+
if (!positionals.join(" ").trim()) throw new CodeIndexError("search requires a query.");
|
|
2484
|
+
validateLimit(10);
|
|
2485
|
+
similarityThreshold();
|
|
2486
|
+
if (outputFormat("json") === "clusters") throw new CodeIndexError("clusters format is only available for cross-search.");
|
|
2487
|
+
return;
|
|
2488
|
+
}
|
|
2489
|
+
case "cross-search":
|
|
2490
|
+
if (Boolean(parsed.values["target-root"]) !== Boolean(parsed.values["target-index"])) {
|
|
2491
|
+
throw new CodeIndexError("Cross-index search requires both --target-root and --target-index.");
|
|
2492
|
+
}
|
|
2493
|
+
if (parsed.values["target-config"] && !parsed.values["target-root"]) {
|
|
2494
|
+
throw new CodeIndexError("--target-config requires --target-root and --target-index.");
|
|
2495
|
+
}
|
|
2496
|
+
validateLimit(5);
|
|
2497
|
+
similarityThreshold();
|
|
2498
|
+
outputFormat("clusters");
|
|
2499
|
+
minimumLines();
|
|
2500
|
+
functionNameRegex();
|
|
2501
|
+
crossSearchSourceFilter();
|
|
2502
|
+
return;
|
|
2503
|
+
case "cohesion":
|
|
2504
|
+
positiveIntegerOption(parsed.values.limit, 50, "limit");
|
|
2505
|
+
positiveIntegerOption(parsed.values.neighbors, 20, "neighbors");
|
|
2506
|
+
cohesionThreshold();
|
|
2507
|
+
if (outputFormat("json") === "clusters") throw new CodeIndexError("clusters format is only available for cross-search.");
|
|
2508
|
+
minimumLines();
|
|
2509
|
+
functionNameRegex();
|
|
2510
|
+
crossSearchSourceFilter();
|
|
2511
|
+
return;
|
|
2512
|
+
default:
|
|
2513
|
+
throw new CodeIndexError(`Unknown command: ${command}`);
|
|
1611
2514
|
}
|
|
1612
|
-
|
|
2515
|
+
}
|
|
2516
|
+
function validateLimit(defaultValue) {
|
|
2517
|
+
positiveIntegerOption(parsed.values.limit, defaultValue, "limit");
|
|
2518
|
+
}
|
|
2519
|
+
function positiveIntegerOption(value, defaultValue, name) {
|
|
2520
|
+
const parsedValue = numberOption(value, defaultValue, name);
|
|
2521
|
+
if (!Number.isInteger(parsedValue) || parsedValue < 1) throw new CodeIndexError(`${name} must be a positive integer.`);
|
|
2522
|
+
return parsedValue;
|
|
2523
|
+
}
|
|
2524
|
+
function similarityThreshold(defaultMin = -1) {
|
|
2525
|
+
const threshold = parsed.values.threshold;
|
|
2526
|
+
if (threshold === void 0) return { min: defaultMin };
|
|
1613
2527
|
const number = "[+-]?(?:\\d+(?:\\.\\d*)?|\\.\\d+)(?:e[+-]?\\d+)?";
|
|
1614
2528
|
const range = new RegExp(`^\\s*(${number})\\s*-\\s*(${number})\\s*$`, "i").exec(threshold);
|
|
1615
2529
|
if (!range) return { min: numberOption(threshold, -1, "threshold") };
|
|
1616
2530
|
const min = Number(range[1]);
|
|
1617
2531
|
const max = Number(range[2]);
|
|
1618
2532
|
if (!Number.isFinite(min) || !Number.isFinite(max)) throw new CodeIndexError("threshold range bounds must be numbers.");
|
|
1619
|
-
if (min
|
|
2533
|
+
if (min >= max) throw new CodeIndexError("threshold range minimum must be less than its maximum.");
|
|
1620
2534
|
return { min, max };
|
|
1621
2535
|
}
|
|
1622
|
-
function
|
|
1623
|
-
const
|
|
1624
|
-
if (
|
|
2536
|
+
function cohesionThreshold() {
|
|
2537
|
+
const threshold = similarityThreshold(0.8);
|
|
2538
|
+
if (threshold.min < -1 || threshold.min >= 1) {
|
|
2539
|
+
throw new CodeIndexError("cohesion threshold must be at least -1 and less than 1.");
|
|
2540
|
+
}
|
|
2541
|
+
return threshold;
|
|
2542
|
+
}
|
|
2543
|
+
function outputFormat(defaultValue) {
|
|
2544
|
+
const value = parsed.values.format ?? defaultValue;
|
|
2545
|
+
if (value !== "json" && value !== "summary" && value !== "clusters") {
|
|
2546
|
+
throw new CodeIndexError("format must be json, summary, or clusters.");
|
|
2547
|
+
}
|
|
1625
2548
|
return value;
|
|
1626
2549
|
}
|
|
2550
|
+
function minimumLines() {
|
|
2551
|
+
const value = numberOption(parsed.values["min-lines"], 2, "min-lines");
|
|
2552
|
+
if (!Number.isInteger(value) || value < 1) throw new CodeIndexError("min-lines must be a positive integer.");
|
|
2553
|
+
return value;
|
|
2554
|
+
}
|
|
2555
|
+
function functionNameRegex() {
|
|
2556
|
+
const pattern = parsed.values.regex;
|
|
2557
|
+
if (pattern === void 0) return void 0;
|
|
2558
|
+
try {
|
|
2559
|
+
return new RegExp(pattern);
|
|
2560
|
+
} catch (error) {
|
|
2561
|
+
throw new CodeIndexError(`Invalid --regex value: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
2562
|
+
}
|
|
2563
|
+
}
|
|
2564
|
+
function crossSearchSourceFilter() {
|
|
2565
|
+
const changedSince = parsed.values["changed-since"];
|
|
2566
|
+
const uncommitted = parsed.values.uncommitted;
|
|
2567
|
+
if (changedSince && uncommitted) throw new CodeIndexError("Use either --changed-since or --uncommitted, not both.");
|
|
2568
|
+
const pathOption = parsed.values["source-path"] ? { path: parsed.values["source-path"] } : {};
|
|
2569
|
+
if (changedSince) return { type: "changed-since", commit: changedSince, ...pathOption };
|
|
2570
|
+
if (uncommitted) return { type: "uncommitted", ...pathOption };
|
|
2571
|
+
return { type: "all", ...pathOption };
|
|
2572
|
+
}
|
|
1627
2573
|
function presentFunction(value) {
|
|
1628
2574
|
const { embeddingInput: _embeddingInput, embeddingId: _embeddingId, ...result } = value;
|
|
1629
2575
|
return result;
|
|
@@ -1631,6 +2577,41 @@ function presentFunction(value) {
|
|
|
1631
2577
|
function presentMatch(value) {
|
|
1632
2578
|
return { similarity: value.similarity, function: presentFunction(value.function) };
|
|
1633
2579
|
}
|
|
2580
|
+
function presentCohesionReport(report, includeSource) {
|
|
2581
|
+
const presentCohesionFunction = (value) => ({
|
|
2582
|
+
path: value.path,
|
|
2583
|
+
qualifiedName: value.qualifiedName,
|
|
2584
|
+
kind: value.kind,
|
|
2585
|
+
signature: value.signature,
|
|
2586
|
+
startLine: value.startLine,
|
|
2587
|
+
startColumn: value.startColumn,
|
|
2588
|
+
endLine: value.endLine,
|
|
2589
|
+
endColumn: value.endColumn,
|
|
2590
|
+
lineCount: value.lineCount,
|
|
2591
|
+
...includeSource ? { source: value.source } : {}
|
|
2592
|
+
});
|
|
2593
|
+
return {
|
|
2594
|
+
...report,
|
|
2595
|
+
pairs: report.pairs.map((pair) => ({
|
|
2596
|
+
...pair,
|
|
2597
|
+
left: presentCohesionFunction(pair.left),
|
|
2598
|
+
right: presentCohesionFunction(pair.right)
|
|
2599
|
+
})),
|
|
2600
|
+
files: report.files.map((file) => ({
|
|
2601
|
+
...file,
|
|
2602
|
+
...file.strongestExternalMatch ? {
|
|
2603
|
+
strongestExternalMatch: {
|
|
2604
|
+
...file.strongestExternalMatch,
|
|
2605
|
+
function: presentCohesionFunction(file.strongestExternalMatch.function)
|
|
2606
|
+
}
|
|
2607
|
+
} : {}
|
|
2608
|
+
})),
|
|
2609
|
+
groups: report.groups.map((group) => ({
|
|
2610
|
+
...group,
|
|
2611
|
+
members: group.members.map(presentCohesionFunction)
|
|
2612
|
+
}))
|
|
2613
|
+
};
|
|
2614
|
+
}
|
|
1634
2615
|
function printJson(value) {
|
|
1635
2616
|
process.stdout.write(`${JSON.stringify(value, null, 2)}
|
|
1636
2617
|
`);
|
|
@@ -1642,9 +2623,79 @@ Commands:
|
|
|
1642
2623
|
status Show index metadata
|
|
1643
2624
|
update-files <path...> Index specific working-tree files
|
|
1644
2625
|
delete-files <path...> Remove specific files from the index
|
|
1645
|
-
update-git Index
|
|
2626
|
+
update-git Index a Git snapshot plus working-tree changes
|
|
1646
2627
|
search <query> Search functions by semantic similarity
|
|
1647
2628
|
cross-search Find nearest functions for each source function
|
|
2629
|
+
cohesion Rank related functions separated across the repository
|
|
2630
|
+
|
|
2631
|
+
Analysis Examples:
|
|
2632
|
+
Duplicate Analysis:
|
|
2633
|
+
slopdex cross-search --cross-file-only --min-lines 4 --threshold 0.9 --limit 5
|
|
2634
|
+
|
|
2635
|
+
Cluster 1 (3 functions, similarity 0.9124-0.9568)
|
|
2636
|
+
src/auth/session.ts:18:1 :: validateSession
|
|
2637
|
+
src/http/middleware.ts:42:1 :: authenticate
|
|
2638
|
+
src/users/user-service.ts:27:3 :: UserService.authenticate
|
|
2639
|
+
|
|
2640
|
+
This found three substantial authentication functions in separate files with very
|
|
2641
|
+
high similarity. Review them for repeated validation or session logic that could be shared.
|
|
2642
|
+
Middleware and service locations may be intentional architectural layers, so this is
|
|
2643
|
+
evidence to inspect rather than proof they should merge. Connected components may use
|
|
2644
|
+
transitive links, so every function need not directly match every other function.
|
|
2645
|
+
|
|
2646
|
+
Cohesion Analysis:
|
|
2647
|
+
slopdex cohesion --threshold 0.8 --neighbors 20 --limit 50 --format summary
|
|
2648
|
+
|
|
2649
|
+
Cohesion: 184 functions analyzed, 37 semantic edges
|
|
2650
|
+
same file 35.1% same folder 29.7% remote 35.2% mean distance 1.84
|
|
2651
|
+
|
|
2652
|
+
1. gap 0.6053 similarity 0.9400 distance 4 reciprocal
|
|
2653
|
+
src/auth/session.ts:18:1 :: validateSession
|
|
2654
|
+
packages/http/middleware.ts:42:1 :: authenticate
|
|
2655
|
+
|
|
2656
|
+
There is no universal pass/fail cutoff, but this warrants review: 35.2% of weighted
|
|
2657
|
+
semantic affinity crosses folders, mean distance 1.84 exceeds the same-folder distance
|
|
2658
|
+
of one, and the top pair is 0.94 similar but four units apart. A more cohesive result
|
|
2659
|
+
would concentrate affinity in the same file or folder and have few high-gap remote
|
|
2660
|
+
pairs. Compare modules or history rather than treating one percentage as a fixed
|
|
2661
|
+
threshold, and allow for intentionally separate architectural responsibilities.
|
|
2662
|
+
|
|
2663
|
+
Reading Analysis Output:
|
|
2664
|
+
Compare values only with the same embedding profile and similar threshold, neighbor,
|
|
2665
|
+
source-filter, and minimum-line settings.
|
|
2666
|
+
|
|
2667
|
+
Duplicate cluster line:
|
|
2668
|
+
Cluster N Display order by function count, then name; not severity
|
|
2669
|
+
functions Unique connected callables; higher may mean a larger duplicate family
|
|
2670
|
+
similarity range Raw cosine edge range; higher means stronger model-assessed resemblance
|
|
2671
|
+
A high minimum means all observed links are strong; a wide range may
|
|
2672
|
+
indicate that a weaker transitive edge joined tighter matches
|
|
2673
|
+
callable location path:line:column :: qualifiedFunctionName; inspect architectural roles
|
|
2674
|
+
|
|
2675
|
+
Cohesion headline:
|
|
2676
|
+
functions analyzed Filtered source coverage; higher or lower does not mean better or worse
|
|
2677
|
+
semantic edges Unique qualifying neighbor pairs before --limit; not a quality score
|
|
2678
|
+
More neighbors or a lower threshold generally increases this count
|
|
2679
|
+
|
|
2680
|
+
Cohesion distribution:
|
|
2681
|
+
same file Higher generally means related implementation is co-located
|
|
2682
|
+
same folder Higher means related modules remain locally grouped
|
|
2683
|
+
remote Higher means more affinity crosses folders and weaker physical cohesion
|
|
2684
|
+
mean distance Lower is generally more cohesive (same file 0, same folder 1)
|
|
2685
|
+
|
|
2686
|
+
Ranked cohesion finding:
|
|
2687
|
+
rank Lower number means higher review priority
|
|
2688
|
+
gap 0-to-1 combined signal; higher means strongly related and farther apart
|
|
2689
|
+
similarity Higher means greater resemblance; values are model-specific
|
|
2690
|
+
distance Higher means more file and directory-tree separation
|
|
2691
|
+
reciprocal Mutual top-neighbor relationship; strengthens confidence when present
|
|
2692
|
+
|
|
2693
|
+
JSON diagnostics:
|
|
2694
|
+
semanticWeight Higher means similarity is farther above --threshold
|
|
2695
|
+
separationWeight Higher means greater path distance, saturating near one
|
|
2696
|
+
sourceTestPair True flags a possibly intentional source/test relationship
|
|
2697
|
+
externalAffinityRatio
|
|
2698
|
+
Higher means more of a file's related affinity lies outside its folder
|
|
1648
2699
|
|
|
1649
2700
|
Options:
|
|
1650
2701
|
--root <path> Repository root (default: current directory)
|
|
@@ -1655,17 +2706,25 @@ Options:
|
|
|
1655
2706
|
--dimensions <number> Embedding dimensions
|
|
1656
2707
|
--target <ref> Target ref for update-git (default: HEAD)
|
|
1657
2708
|
--rebuild-on-divergence Rebuild after a rebase or branch change
|
|
2709
|
+
--force-reindex Rebuild an incompatible existing index
|
|
2710
|
+
--no-reindex Skip worktree overlays or reuse a non-Git index
|
|
1658
2711
|
--limit <number> Search result limit
|
|
2712
|
+
--neighbors <number> Semantic neighbors per function for cohesion (default: 20)
|
|
1659
2713
|
--threshold <number|range> Show similarities at/above a value or within a range
|
|
1660
|
-
--
|
|
1661
|
-
--
|
|
2714
|
+
--format <json|summary|clusters> Output format (search: json; cross-search: clusters)
|
|
2715
|
+
--include-source Include function source in cohesion JSON
|
|
1662
2716
|
--include-symmetric-duplicates Show both directions of same-index matches
|
|
1663
|
-
--
|
|
2717
|
+
--cross-file-only Exclude matches from the source file
|
|
2718
|
+
--min-lines <number> Minimum callable length for cross-search (default: 2)
|
|
2719
|
+
--regex <regex> Restrict cross-search to matching qualified names
|
|
2720
|
+
--changed-since <commit> Search added, modified, or moved functions
|
|
2721
|
+
--uncommitted Search functions from uncommitted files
|
|
1664
2722
|
--source-path <path> Restrict cross-search sources to a file or directory
|
|
1665
2723
|
--target-root <path> Root of a second indexed codebase
|
|
1666
2724
|
--target-index <path> SQLite path of a second index
|
|
2725
|
+
--target-config <path> Config file for a second indexed codebase
|
|
1667
2726
|
|
|
1668
|
-
Examples:
|
|
2727
|
+
Other Examples:
|
|
1669
2728
|
Show metadata for the current index:
|
|
1670
2729
|
slopdex status
|
|
1671
2730
|
|
|
@@ -1675,15 +2734,12 @@ Examples:
|
|
|
1675
2734
|
Remove deleted files from the index:
|
|
1676
2735
|
slopdex delete-files src/removed.ts
|
|
1677
2736
|
|
|
1678
|
-
Index
|
|
2737
|
+
Index HEAD and overlay uncommitted working-tree changes:
|
|
1679
2738
|
slopdex update-git --target HEAD
|
|
1680
2739
|
|
|
1681
2740
|
Find functions matching a semantic query:
|
|
1682
2741
|
slopdex search "validate an authenticated session" --limit 10
|
|
1683
2742
|
|
|
1684
|
-
Review similar functions as duplicate-code candidates:
|
|
1685
|
-
slopdex cross-search --format summary --threshold 0.8 --limit 5
|
|
1686
|
-
|
|
1687
2743
|
Review functions under a path against the whole codebase:
|
|
1688
2744
|
slopdex cross-search --source-path src/services --format summary
|
|
1689
2745
|
`);
|