opencode-codebase-index 0.5.2 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +208 -105
- package/commands/definition.md +24 -0
- package/dist/cli.cjs +1442 -200
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +1415 -179
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +217 -80
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +218 -87
- package/dist/index.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +8 -1
package/dist/cli.js
CHANGED
|
@@ -7,13 +7,7 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
|
7
7
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
8
8
|
var __getProtoOf = Object.getPrototypeOf;
|
|
9
9
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
10
|
-
var
|
|
11
|
-
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
12
|
-
}) : x)(function(x) {
|
|
13
|
-
if (typeof require !== "undefined") return require.apply(this, arguments);
|
|
14
|
-
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
15
|
-
});
|
|
16
|
-
var __commonJS = (cb, mod) => function __require2() {
|
|
10
|
+
var __commonJS = (cb, mod) => function __require() {
|
|
17
11
|
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
18
12
|
};
|
|
19
13
|
var __copyProps = (to, from, except, desc) => {
|
|
@@ -35,7 +29,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
35
29
|
|
|
36
30
|
// node_modules/eventemitter3/index.js
|
|
37
31
|
var require_eventemitter3 = __commonJS({
|
|
38
|
-
"node_modules/eventemitter3/index.js"(exports,
|
|
32
|
+
"node_modules/eventemitter3/index.js"(exports, module2) {
|
|
39
33
|
"use strict";
|
|
40
34
|
var has = Object.prototype.hasOwnProperty;
|
|
41
35
|
var prefix = "~";
|
|
@@ -189,15 +183,15 @@ var require_eventemitter3 = __commonJS({
|
|
|
189
183
|
EventEmitter2.prototype.addListener = EventEmitter2.prototype.on;
|
|
190
184
|
EventEmitter2.prefixed = prefix;
|
|
191
185
|
EventEmitter2.EventEmitter = EventEmitter2;
|
|
192
|
-
if ("undefined" !== typeof
|
|
193
|
-
|
|
186
|
+
if ("undefined" !== typeof module2) {
|
|
187
|
+
module2.exports = EventEmitter2;
|
|
194
188
|
}
|
|
195
189
|
}
|
|
196
190
|
});
|
|
197
191
|
|
|
198
192
|
// node_modules/ignore/index.js
|
|
199
193
|
var require_ignore = __commonJS({
|
|
200
|
-
"node_modules/ignore/index.js"(exports,
|
|
194
|
+
"node_modules/ignore/index.js"(exports, module2) {
|
|
201
195
|
"use strict";
|
|
202
196
|
function makeArray(subject) {
|
|
203
197
|
return Array.isArray(subject) ? subject : [subject];
|
|
@@ -493,7 +487,7 @@ var require_ignore = __commonJS({
|
|
|
493
487
|
// path matching.
|
|
494
488
|
// - check `string` either `MODE_IGNORE` or `MODE_CHECK_IGNORE`
|
|
495
489
|
// @returns {TestResult} true if a file is ignored
|
|
496
|
-
test(
|
|
490
|
+
test(path10, checkUnignored, mode) {
|
|
497
491
|
let ignored = false;
|
|
498
492
|
let unignored = false;
|
|
499
493
|
let matchedRule;
|
|
@@ -502,7 +496,7 @@ var require_ignore = __commonJS({
|
|
|
502
496
|
if (unignored === negative && ignored !== unignored || negative && !ignored && !unignored && !checkUnignored) {
|
|
503
497
|
return;
|
|
504
498
|
}
|
|
505
|
-
const matched = rule[mode].test(
|
|
499
|
+
const matched = rule[mode].test(path10);
|
|
506
500
|
if (!matched) {
|
|
507
501
|
return;
|
|
508
502
|
}
|
|
@@ -523,17 +517,17 @@ var require_ignore = __commonJS({
|
|
|
523
517
|
var throwError = (message, Ctor) => {
|
|
524
518
|
throw new Ctor(message);
|
|
525
519
|
};
|
|
526
|
-
var checkPath = (
|
|
527
|
-
if (!isString(
|
|
520
|
+
var checkPath = (path10, originalPath, doThrow) => {
|
|
521
|
+
if (!isString(path10)) {
|
|
528
522
|
return doThrow(
|
|
529
523
|
`path must be a string, but got \`${originalPath}\``,
|
|
530
524
|
TypeError
|
|
531
525
|
);
|
|
532
526
|
}
|
|
533
|
-
if (!
|
|
527
|
+
if (!path10) {
|
|
534
528
|
return doThrow(`path must not be empty`, TypeError);
|
|
535
529
|
}
|
|
536
|
-
if (checkPath.isNotRelative(
|
|
530
|
+
if (checkPath.isNotRelative(path10)) {
|
|
537
531
|
const r = "`path.relative()`d";
|
|
538
532
|
return doThrow(
|
|
539
533
|
`path should be a ${r} string, but got "${originalPath}"`,
|
|
@@ -542,7 +536,7 @@ var require_ignore = __commonJS({
|
|
|
542
536
|
}
|
|
543
537
|
return true;
|
|
544
538
|
};
|
|
545
|
-
var isNotRelative = (
|
|
539
|
+
var isNotRelative = (path10) => REGEX_TEST_INVALID_PATH.test(path10);
|
|
546
540
|
checkPath.isNotRelative = isNotRelative;
|
|
547
541
|
checkPath.convert = (p) => p;
|
|
548
542
|
var Ignore2 = class {
|
|
@@ -572,19 +566,19 @@ var require_ignore = __commonJS({
|
|
|
572
566
|
}
|
|
573
567
|
// @returns {TestResult}
|
|
574
568
|
_test(originalPath, cache, checkUnignored, slices) {
|
|
575
|
-
const
|
|
569
|
+
const path10 = originalPath && checkPath.convert(originalPath);
|
|
576
570
|
checkPath(
|
|
577
|
-
|
|
571
|
+
path10,
|
|
578
572
|
originalPath,
|
|
579
573
|
this._strictPathCheck ? throwError : RETURN_FALSE
|
|
580
574
|
);
|
|
581
|
-
return this._t(
|
|
575
|
+
return this._t(path10, cache, checkUnignored, slices);
|
|
582
576
|
}
|
|
583
|
-
checkIgnore(
|
|
584
|
-
if (!REGEX_TEST_TRAILING_SLASH.test(
|
|
585
|
-
return this.test(
|
|
577
|
+
checkIgnore(path10) {
|
|
578
|
+
if (!REGEX_TEST_TRAILING_SLASH.test(path10)) {
|
|
579
|
+
return this.test(path10);
|
|
586
580
|
}
|
|
587
|
-
const slices =
|
|
581
|
+
const slices = path10.split(SLASH).filter(Boolean);
|
|
588
582
|
slices.pop();
|
|
589
583
|
if (slices.length) {
|
|
590
584
|
const parent = this._t(
|
|
@@ -597,18 +591,18 @@ var require_ignore = __commonJS({
|
|
|
597
591
|
return parent;
|
|
598
592
|
}
|
|
599
593
|
}
|
|
600
|
-
return this._rules.test(
|
|
594
|
+
return this._rules.test(path10, false, MODE_CHECK_IGNORE);
|
|
601
595
|
}
|
|
602
|
-
_t(
|
|
603
|
-
if (
|
|
604
|
-
return cache[
|
|
596
|
+
_t(path10, cache, checkUnignored, slices) {
|
|
597
|
+
if (path10 in cache) {
|
|
598
|
+
return cache[path10];
|
|
605
599
|
}
|
|
606
600
|
if (!slices) {
|
|
607
|
-
slices =
|
|
601
|
+
slices = path10.split(SLASH).filter(Boolean);
|
|
608
602
|
}
|
|
609
603
|
slices.pop();
|
|
610
604
|
if (!slices.length) {
|
|
611
|
-
return cache[
|
|
605
|
+
return cache[path10] = this._rules.test(path10, checkUnignored, MODE_IGNORE);
|
|
612
606
|
}
|
|
613
607
|
const parent = this._t(
|
|
614
608
|
slices.join(SLASH) + SLASH,
|
|
@@ -616,29 +610,29 @@ var require_ignore = __commonJS({
|
|
|
616
610
|
checkUnignored,
|
|
617
611
|
slices
|
|
618
612
|
);
|
|
619
|
-
return cache[
|
|
613
|
+
return cache[path10] = parent.ignored ? parent : this._rules.test(path10, checkUnignored, MODE_IGNORE);
|
|
620
614
|
}
|
|
621
|
-
ignores(
|
|
622
|
-
return this._test(
|
|
615
|
+
ignores(path10) {
|
|
616
|
+
return this._test(path10, this._ignoreCache, false).ignored;
|
|
623
617
|
}
|
|
624
618
|
createFilter() {
|
|
625
|
-
return (
|
|
619
|
+
return (path10) => !this.ignores(path10);
|
|
626
620
|
}
|
|
627
621
|
filter(paths) {
|
|
628
622
|
return makeArray(paths).filter(this.createFilter());
|
|
629
623
|
}
|
|
630
624
|
// @returns {TestResult}
|
|
631
|
-
test(
|
|
632
|
-
return this._test(
|
|
625
|
+
test(path10) {
|
|
626
|
+
return this._test(path10, this._testCache, true);
|
|
633
627
|
}
|
|
634
628
|
};
|
|
635
629
|
var factory = (options) => new Ignore2(options);
|
|
636
|
-
var isPathValid = (
|
|
630
|
+
var isPathValid = (path10) => checkPath(path10 && checkPath.convert(path10), path10, RETURN_FALSE);
|
|
637
631
|
var setupWindows = () => {
|
|
638
632
|
const makePosix = (str) => /^\\\\\?\\/.test(str) || /["<>|\u0000-\u001F]+/u.test(str) ? str : str.replace(/\\/g, "/");
|
|
639
633
|
checkPath.convert = makePosix;
|
|
640
634
|
const REGEX_TEST_WINDOWS_PATH_ABSOLUTE = /^[a-z]:\//i;
|
|
641
|
-
checkPath.isNotRelative = (
|
|
635
|
+
checkPath.isNotRelative = (path10) => REGEX_TEST_WINDOWS_PATH_ABSOLUTE.test(path10) || isNotRelative(path10);
|
|
642
636
|
};
|
|
643
637
|
if (
|
|
644
638
|
// Detect `process` so that it can run in browsers.
|
|
@@ -646,18 +640,18 @@ var require_ignore = __commonJS({
|
|
|
646
640
|
) {
|
|
647
641
|
setupWindows();
|
|
648
642
|
}
|
|
649
|
-
|
|
643
|
+
module2.exports = factory;
|
|
650
644
|
factory.default = factory;
|
|
651
|
-
|
|
652
|
-
define(
|
|
645
|
+
module2.exports.isPathValid = isPathValid;
|
|
646
|
+
define(module2.exports, /* @__PURE__ */ Symbol.for("setupWindows"), setupWindows);
|
|
653
647
|
}
|
|
654
648
|
});
|
|
655
649
|
|
|
656
650
|
// src/cli.ts
|
|
657
651
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
658
|
-
import { existsSync as
|
|
659
|
-
import * as
|
|
660
|
-
import * as
|
|
652
|
+
import { existsSync as existsSync6, readFileSync as readFileSync8 } from "fs";
|
|
653
|
+
import * as path9 from "path";
|
|
654
|
+
import * as os4 from "os";
|
|
661
655
|
|
|
662
656
|
// src/config/constants.ts
|
|
663
657
|
var DEFAULT_INCLUDE = [
|
|
@@ -665,7 +659,7 @@ var DEFAULT_INCLUDE = [
|
|
|
665
659
|
"**/*.{py,pyi}",
|
|
666
660
|
"**/*.{go,rs,java,kt,scala}",
|
|
667
661
|
"**/*.{c,cpp,cc,h,hpp}",
|
|
668
|
-
"**/*.{rb,php,swift}",
|
|
662
|
+
"**/*.{rb,php,inc,swift}",
|
|
669
663
|
"**/*.{vue,svelte,astro}",
|
|
670
664
|
"**/*.{sql,graphql,proto}",
|
|
671
665
|
"**/*.{yaml,yml,toml}",
|
|
@@ -760,6 +754,27 @@ var DEFAULT_PROVIDER_MODELS = {
|
|
|
760
754
|
"ollama": "nomic-embed-text"
|
|
761
755
|
};
|
|
762
756
|
|
|
757
|
+
// src/config/env-substitution.ts
|
|
758
|
+
var ENV_REFERENCE_PATTERN = /^\{env:([A-Z_][A-Z0-9_]*)\}$/;
|
|
759
|
+
var ENV_REFERENCE_LIKE_PATTERN = /\{env:[^}]+\}/;
|
|
760
|
+
function substituteEnvString(value, keyPath) {
|
|
761
|
+
const match = value.match(ENV_REFERENCE_PATTERN);
|
|
762
|
+
if (!match) {
|
|
763
|
+
if (ENV_REFERENCE_LIKE_PATTERN.test(value)) {
|
|
764
|
+
throw new Error(
|
|
765
|
+
`Invalid environment variable reference at '${keyPath}'. Expected the entire string to match '{env:VAR_NAME}' with VAR_NAME matching [A-Z_][A-Z0-9_]*.`
|
|
766
|
+
);
|
|
767
|
+
}
|
|
768
|
+
return value;
|
|
769
|
+
}
|
|
770
|
+
const variableName = match[1];
|
|
771
|
+
const envValue = process.env[variableName];
|
|
772
|
+
if (envValue === void 0) {
|
|
773
|
+
throw new Error(`Missing environment variable '${variableName}' referenced by config at '${keyPath}'.`);
|
|
774
|
+
}
|
|
775
|
+
return envValue;
|
|
776
|
+
}
|
|
777
|
+
|
|
763
778
|
// src/config/schema.ts
|
|
764
779
|
function getDefaultIndexingConfig() {
|
|
765
780
|
return {
|
|
@@ -817,11 +832,27 @@ function isValidScope(value) {
|
|
|
817
832
|
function isStringArray(value) {
|
|
818
833
|
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
819
834
|
}
|
|
835
|
+
function getResolvedString(value, keyPath) {
|
|
836
|
+
if (typeof value !== "string") {
|
|
837
|
+
return void 0;
|
|
838
|
+
}
|
|
839
|
+
return substituteEnvString(value, keyPath);
|
|
840
|
+
}
|
|
841
|
+
function getResolvedStringArray(value, keyPath) {
|
|
842
|
+
if (!isStringArray(value)) {
|
|
843
|
+
return void 0;
|
|
844
|
+
}
|
|
845
|
+
return value.map((item, index) => substituteEnvString(item, `${keyPath}[${index}]`));
|
|
846
|
+
}
|
|
820
847
|
function isValidLogLevel(value) {
|
|
821
848
|
return typeof value === "string" && VALID_LOG_LEVELS.includes(value);
|
|
822
849
|
}
|
|
823
850
|
function parseConfig(raw) {
|
|
824
851
|
const input = raw && typeof raw === "object" ? raw : {};
|
|
852
|
+
const embeddingProviderValue = getResolvedString(input.embeddingProvider, "$root.embeddingProvider");
|
|
853
|
+
const scopeValue = getResolvedString(input.scope, "$root.scope");
|
|
854
|
+
const includeValue = getResolvedStringArray(input.include, "$root.include");
|
|
855
|
+
const excludeValue = getResolvedStringArray(input.exclude, "$root.exclude");
|
|
825
856
|
const defaultIndexing = getDefaultIndexingConfig();
|
|
826
857
|
const defaultSearch = getDefaultSearchConfig();
|
|
827
858
|
const defaultDebug = getDefaultDebugConfig();
|
|
@@ -864,19 +895,23 @@ function parseConfig(raw) {
|
|
|
864
895
|
let embeddingProvider;
|
|
865
896
|
let embeddingModel = void 0;
|
|
866
897
|
let customProvider = void 0;
|
|
867
|
-
if (
|
|
898
|
+
if (embeddingProviderValue === "custom") {
|
|
868
899
|
embeddingProvider = "custom";
|
|
869
900
|
const rawCustom = input.customProvider && typeof input.customProvider === "object" ? input.customProvider : null;
|
|
870
|
-
|
|
901
|
+
const baseUrlValue = getResolvedString(rawCustom?.baseUrl, "$root.customProvider.baseUrl");
|
|
902
|
+
const modelValue = getResolvedString(rawCustom?.model, "$root.customProvider.model");
|
|
903
|
+
const apiKeyValue = getResolvedString(rawCustom?.apiKey, "$root.customProvider.apiKey");
|
|
904
|
+
if (rawCustom && typeof baseUrlValue === "string" && baseUrlValue.trim().length > 0 && typeof modelValue === "string" && modelValue.trim().length > 0 && typeof rawCustom.dimensions === "number" && Number.isInteger(rawCustom.dimensions) && rawCustom.dimensions > 0) {
|
|
871
905
|
customProvider = {
|
|
872
|
-
baseUrl:
|
|
873
|
-
model:
|
|
906
|
+
baseUrl: baseUrlValue.trim().replace(/\/+$/, ""),
|
|
907
|
+
model: modelValue,
|
|
874
908
|
dimensions: rawCustom.dimensions,
|
|
875
|
-
apiKey:
|
|
909
|
+
apiKey: apiKeyValue,
|
|
876
910
|
maxTokens: typeof rawCustom.maxTokens === "number" ? rawCustom.maxTokens : void 0,
|
|
877
911
|
timeoutMs: typeof rawCustom.timeoutMs === "number" ? Math.max(1e3, rawCustom.timeoutMs) : void 0,
|
|
878
912
|
concurrency: typeof rawCustom.concurrency === "number" ? Math.max(1, Math.floor(rawCustom.concurrency)) : void 0,
|
|
879
|
-
requestIntervalMs: typeof rawCustom.requestIntervalMs === "number" ? Math.max(0, Math.floor(rawCustom.requestIntervalMs)) : void 0
|
|
913
|
+
requestIntervalMs: typeof rawCustom.requestIntervalMs === "number" ? Math.max(0, Math.floor(rawCustom.requestIntervalMs)) : void 0,
|
|
914
|
+
maxBatchSize: typeof rawCustom.maxBatchSize === "number" ? Math.max(1, Math.floor(rawCustom.maxBatchSize)) : typeof rawCustom.max_batch_size === "number" ? Math.max(1, Math.floor(rawCustom.max_batch_size)) : void 0
|
|
880
915
|
};
|
|
881
916
|
if (!/\/v\d+\/?$/.test(customProvider.baseUrl)) {
|
|
882
917
|
console.warn(
|
|
@@ -888,10 +923,16 @@ function parseConfig(raw) {
|
|
|
888
923
|
"embeddingProvider is 'custom' but customProvider config is missing or invalid. Required fields: baseUrl (string), model (string), dimensions (positive integer)."
|
|
889
924
|
);
|
|
890
925
|
}
|
|
891
|
-
} else if (isValidProvider(
|
|
892
|
-
embeddingProvider =
|
|
893
|
-
|
|
894
|
-
|
|
926
|
+
} else if (isValidProvider(embeddingProviderValue)) {
|
|
927
|
+
embeddingProvider = embeddingProviderValue;
|
|
928
|
+
const rawEmbeddingModel = input.embeddingModel;
|
|
929
|
+
if (typeof rawEmbeddingModel === "string") {
|
|
930
|
+
const embeddingModelValue = substituteEnvString(rawEmbeddingModel, "$root.embeddingModel");
|
|
931
|
+
if (embeddingModelValue) {
|
|
932
|
+
embeddingModel = isValidModel(embeddingModelValue, embeddingProvider) ? embeddingModelValue : DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
933
|
+
}
|
|
934
|
+
} else if (rawEmbeddingModel) {
|
|
935
|
+
embeddingModel = DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
895
936
|
}
|
|
896
937
|
} else {
|
|
897
938
|
embeddingProvider = "auto";
|
|
@@ -900,9 +941,9 @@ function parseConfig(raw) {
|
|
|
900
941
|
embeddingProvider,
|
|
901
942
|
embeddingModel,
|
|
902
943
|
customProvider,
|
|
903
|
-
scope: isValidScope(
|
|
904
|
-
include:
|
|
905
|
-
exclude:
|
|
944
|
+
scope: isValidScope(scopeValue) ? scopeValue : "project",
|
|
945
|
+
include: includeValue ?? DEFAULT_INCLUDE,
|
|
946
|
+
exclude: excludeValue ?? DEFAULT_EXCLUDE,
|
|
906
947
|
indexing,
|
|
907
948
|
search,
|
|
908
949
|
debug
|
|
@@ -915,13 +956,197 @@ function getDefaultModelForProvider(provider) {
|
|
|
915
956
|
}
|
|
916
957
|
var availableProviders = Object.keys(EMBEDDING_MODELS);
|
|
917
958
|
|
|
918
|
-
// src/
|
|
919
|
-
import
|
|
920
|
-
|
|
959
|
+
// src/eval/cli.ts
|
|
960
|
+
import * as path8 from "path";
|
|
961
|
+
|
|
962
|
+
// src/eval/compare.ts
|
|
963
|
+
function metricDelta(current, baseline) {
|
|
964
|
+
const absolute = current - baseline;
|
|
965
|
+
const relativePct = baseline === 0 ? current === 0 ? 0 : 100 : absolute / baseline * 100;
|
|
966
|
+
return {
|
|
967
|
+
current,
|
|
968
|
+
baseline,
|
|
969
|
+
absolute,
|
|
970
|
+
relativePct
|
|
971
|
+
};
|
|
972
|
+
}
|
|
973
|
+
function compareSummaries(current, baseline, againstPath) {
|
|
974
|
+
return {
|
|
975
|
+
againstPath,
|
|
976
|
+
deltas: {
|
|
977
|
+
hitAt1: metricDelta(current.metrics.hitAt1, baseline.metrics.hitAt1),
|
|
978
|
+
hitAt3: metricDelta(current.metrics.hitAt3, baseline.metrics.hitAt3),
|
|
979
|
+
hitAt5: metricDelta(current.metrics.hitAt5, baseline.metrics.hitAt5),
|
|
980
|
+
hitAt10: metricDelta(current.metrics.hitAt10, baseline.metrics.hitAt10),
|
|
981
|
+
mrrAt10: metricDelta(current.metrics.mrrAt10, baseline.metrics.mrrAt10),
|
|
982
|
+
ndcgAt10: metricDelta(current.metrics.ndcgAt10, baseline.metrics.ndcgAt10),
|
|
983
|
+
latencyP50Ms: metricDelta(current.metrics.latencyMs.p50, baseline.metrics.latencyMs.p50),
|
|
984
|
+
latencyP95Ms: metricDelta(current.metrics.latencyMs.p95, baseline.metrics.latencyMs.p95),
|
|
985
|
+
latencyP99Ms: metricDelta(current.metrics.latencyMs.p99, baseline.metrics.latencyMs.p99),
|
|
986
|
+
embeddingCallCount: metricDelta(
|
|
987
|
+
current.metrics.embedding.callCount,
|
|
988
|
+
baseline.metrics.embedding.callCount
|
|
989
|
+
),
|
|
990
|
+
estimatedCostUsd: metricDelta(
|
|
991
|
+
current.metrics.embedding.estimatedCostUsd,
|
|
992
|
+
baseline.metrics.embedding.estimatedCostUsd
|
|
993
|
+
)
|
|
994
|
+
}
|
|
995
|
+
};
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
// src/eval/reports.ts
|
|
999
|
+
import { mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
1000
|
+
import * as path from "path";
|
|
1001
|
+
function formatPct(value) {
|
|
1002
|
+
return `${(value * 100).toFixed(2)}%`;
|
|
1003
|
+
}
|
|
1004
|
+
function formatMs(value) {
|
|
1005
|
+
return `${value.toFixed(3)}ms`;
|
|
1006
|
+
}
|
|
1007
|
+
function formatUsd(value) {
|
|
1008
|
+
return `$${value.toFixed(6)}`;
|
|
1009
|
+
}
|
|
1010
|
+
function signed(value, digits = 4) {
|
|
1011
|
+
const formatted = value.toFixed(digits);
|
|
1012
|
+
return value > 0 ? `+${formatted}` : formatted;
|
|
1013
|
+
}
|
|
1014
|
+
function loadSummary(summaryPath) {
|
|
1015
|
+
const raw = readFileSync(summaryPath, "utf-8");
|
|
1016
|
+
return JSON.parse(raw);
|
|
1017
|
+
}
|
|
1018
|
+
function createRunDirectory(outputRoot, timestampOverride) {
|
|
1019
|
+
const timestamp = (timestampOverride ?? (/* @__PURE__ */ new Date()).toISOString()).replace(/[:.]/g, "-");
|
|
1020
|
+
const dir = path.join(outputRoot, timestamp);
|
|
1021
|
+
mkdirSync(dir, { recursive: true });
|
|
1022
|
+
return dir;
|
|
1023
|
+
}
|
|
1024
|
+
function writeJson(filePath, value) {
|
|
1025
|
+
writeFileSync(filePath, JSON.stringify(value, null, 2), "utf-8");
|
|
1026
|
+
}
|
|
1027
|
+
function writeText(filePath, value) {
|
|
1028
|
+
writeFileSync(filePath, value, "utf-8");
|
|
1029
|
+
}
|
|
1030
|
+
function createSummaryMarkdown(summary, comparison, gate, sweep) {
|
|
1031
|
+
const lines = [];
|
|
1032
|
+
lines.push("# Evaluation Summary");
|
|
1033
|
+
lines.push("");
|
|
1034
|
+
lines.push(`- Generated: ${summary.generatedAt}`);
|
|
1035
|
+
lines.push(`- Dataset: ${summary.datasetName} (v${summary.datasetVersion})`);
|
|
1036
|
+
lines.push(`- Query count: ${summary.queryCount}`);
|
|
1037
|
+
lines.push(
|
|
1038
|
+
`- Search config: fusion=${summary.searchConfig.fusionStrategy}, hybridWeight=${summary.searchConfig.hybridWeight}, rrfK=${summary.searchConfig.rrfK}, rerankTopN=${summary.searchConfig.rerankTopN}`
|
|
1039
|
+
);
|
|
1040
|
+
lines.push("");
|
|
1041
|
+
lines.push("## Metrics");
|
|
1042
|
+
lines.push("");
|
|
1043
|
+
lines.push("| Metric | Value |");
|
|
1044
|
+
lines.push("|---|---:|");
|
|
1045
|
+
lines.push(`| Hit@1 | ${formatPct(summary.metrics.hitAt1)} |`);
|
|
1046
|
+
lines.push(`| Hit@3 | ${formatPct(summary.metrics.hitAt3)} |`);
|
|
1047
|
+
lines.push(`| Hit@5 | ${formatPct(summary.metrics.hitAt5)} |`);
|
|
1048
|
+
lines.push(`| Hit@10 | ${formatPct(summary.metrics.hitAt10)} |`);
|
|
1049
|
+
lines.push(`| MRR@10 | ${summary.metrics.mrrAt10.toFixed(4)} |`);
|
|
1050
|
+
lines.push(`| nDCG@10 | ${summary.metrics.ndcgAt10.toFixed(4)} |`);
|
|
1051
|
+
lines.push(`| Latency p50 | ${formatMs(summary.metrics.latencyMs.p50)} |`);
|
|
1052
|
+
lines.push(`| Latency p95 | ${formatMs(summary.metrics.latencyMs.p95)} |`);
|
|
1053
|
+
lines.push(`| Latency p99 | ${formatMs(summary.metrics.latencyMs.p99)} |`);
|
|
1054
|
+
lines.push(`| Embedding calls | ${summary.metrics.embedding.callCount} |`);
|
|
1055
|
+
lines.push(`| Embedding tokens | ${summary.metrics.tokenEstimate.embeddingTokensUsed} |`);
|
|
1056
|
+
lines.push(`| Estimated embedding cost | ${formatUsd(summary.metrics.embedding.estimatedCostUsd)} |`);
|
|
1057
|
+
lines.push("");
|
|
1058
|
+
lines.push("## Failure Buckets");
|
|
1059
|
+
lines.push("");
|
|
1060
|
+
lines.push("| Bucket | Count |");
|
|
1061
|
+
lines.push("|---|---:|");
|
|
1062
|
+
lines.push(
|
|
1063
|
+
`| wrong-file | ${summary.metrics.failureBuckets["wrong-file"]} |`
|
|
1064
|
+
);
|
|
1065
|
+
lines.push(
|
|
1066
|
+
`| wrong-symbol | ${summary.metrics.failureBuckets["wrong-symbol"]} |`
|
|
1067
|
+
);
|
|
1068
|
+
lines.push(
|
|
1069
|
+
`| docs/tests outranking source | ${summary.metrics.failureBuckets["docs-tests-outranking-source"]} |`
|
|
1070
|
+
);
|
|
1071
|
+
lines.push(
|
|
1072
|
+
`| no relevant hit in top-k | ${summary.metrics.failureBuckets["no-relevant-hit-top-k"]} |`
|
|
1073
|
+
);
|
|
1074
|
+
lines.push("");
|
|
1075
|
+
if (comparison) {
|
|
1076
|
+
lines.push("## Comparison vs Baseline");
|
|
1077
|
+
lines.push("");
|
|
1078
|
+
lines.push(`- Against: ${comparison.againstPath}`);
|
|
1079
|
+
lines.push("");
|
|
1080
|
+
lines.push("| Metric | Baseline | Current | Delta |");
|
|
1081
|
+
lines.push("|---|---:|---:|---:|");
|
|
1082
|
+
lines.push(
|
|
1083
|
+
`| Hit@5 | ${formatPct(comparison.deltas.hitAt5.baseline)} | ${formatPct(comparison.deltas.hitAt5.current)} | ${signed(comparison.deltas.hitAt5.absolute)} |`
|
|
1084
|
+
);
|
|
1085
|
+
lines.push(
|
|
1086
|
+
`| MRR@10 | ${comparison.deltas.mrrAt10.baseline.toFixed(4)} | ${comparison.deltas.mrrAt10.current.toFixed(4)} | ${signed(comparison.deltas.mrrAt10.absolute)} |`
|
|
1087
|
+
);
|
|
1088
|
+
lines.push(
|
|
1089
|
+
`| nDCG@10 | ${comparison.deltas.ndcgAt10.baseline.toFixed(4)} | ${comparison.deltas.ndcgAt10.current.toFixed(4)} | ${signed(comparison.deltas.ndcgAt10.absolute)} |`
|
|
1090
|
+
);
|
|
1091
|
+
lines.push(
|
|
1092
|
+
`| p95 latency (ms) | ${comparison.deltas.latencyP95Ms.baseline.toFixed(3)} | ${comparison.deltas.latencyP95Ms.current.toFixed(3)} | ${signed(comparison.deltas.latencyP95Ms.absolute, 3)} |`
|
|
1093
|
+
);
|
|
1094
|
+
lines.push("");
|
|
1095
|
+
}
|
|
1096
|
+
if (gate) {
|
|
1097
|
+
lines.push("## CI Gate");
|
|
1098
|
+
lines.push("");
|
|
1099
|
+
lines.push(`- Result: ${gate.passed ? "PASS \u2705" : "FAIL \u274C"}`);
|
|
1100
|
+
if (gate.violations.length > 0) {
|
|
1101
|
+
lines.push("- Violations:");
|
|
1102
|
+
for (const violation of gate.violations) {
|
|
1103
|
+
lines.push(` - ${violation.metric}: ${violation.message}`);
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
lines.push("");
|
|
1107
|
+
}
|
|
1108
|
+
if (sweep) {
|
|
1109
|
+
lines.push("## Parameter Sweep");
|
|
1110
|
+
lines.push("");
|
|
1111
|
+
lines.push(`- Run count: ${sweep.runCount}`);
|
|
1112
|
+
if (sweep.bestByHitAt5) {
|
|
1113
|
+
lines.push(
|
|
1114
|
+
`- Best Hit@5: ${formatPct(sweep.bestByHitAt5.summary.metrics.hitAt5)} with fusion=${sweep.bestByHitAt5.searchConfig.fusionStrategy}, hybridWeight=${sweep.bestByHitAt5.searchConfig.hybridWeight}, rrfK=${sweep.bestByHitAt5.searchConfig.rrfK}, rerankTopN=${sweep.bestByHitAt5.searchConfig.rerankTopN}`
|
|
1115
|
+
);
|
|
1116
|
+
}
|
|
1117
|
+
if (sweep.bestByMrrAt10) {
|
|
1118
|
+
lines.push(
|
|
1119
|
+
`- Best MRR@10: ${sweep.bestByMrrAt10.summary.metrics.mrrAt10.toFixed(4)} with fusion=${sweep.bestByMrrAt10.searchConfig.fusionStrategy}, hybridWeight=${sweep.bestByMrrAt10.searchConfig.hybridWeight}, rrfK=${sweep.bestByMrrAt10.searchConfig.rrfK}, rerankTopN=${sweep.bestByMrrAt10.searchConfig.rerankTopN}`
|
|
1120
|
+
);
|
|
1121
|
+
}
|
|
1122
|
+
if (sweep.bestByP95Latency) {
|
|
1123
|
+
lines.push(
|
|
1124
|
+
`- Best p95 latency: ${formatMs(sweep.bestByP95Latency.summary.metrics.latencyMs.p95)} with fusion=${sweep.bestByP95Latency.searchConfig.fusionStrategy}, hybridWeight=${sweep.bestByP95Latency.searchConfig.hybridWeight}, rrfK=${sweep.bestByP95Latency.searchConfig.rrfK}, rerankTopN=${sweep.bestByP95Latency.searchConfig.rerankTopN}`
|
|
1125
|
+
);
|
|
1126
|
+
}
|
|
1127
|
+
lines.push("");
|
|
1128
|
+
}
|
|
1129
|
+
return `${lines.join("\n")}
|
|
1130
|
+
`;
|
|
1131
|
+
}
|
|
1132
|
+
function buildPerQueryArtifact(perQuery) {
|
|
1133
|
+
return {
|
|
1134
|
+
queryCount: perQuery.length,
|
|
1135
|
+
queries: [...perQuery].sort((a, b) => a.id.localeCompare(b.id))
|
|
1136
|
+
};
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
// src/eval/runner.ts
|
|
1140
|
+
import { existsSync as existsSync5 } from "fs";
|
|
1141
|
+
import { readFileSync as readFileSync7 } from "fs";
|
|
1142
|
+
import { rmSync } from "fs";
|
|
1143
|
+
import * as os3 from "os";
|
|
1144
|
+
import * as path7 from "path";
|
|
1145
|
+
import { performance as performance3 } from "perf_hooks";
|
|
921
1146
|
|
|
922
1147
|
// src/indexer/index.ts
|
|
923
|
-
import { existsSync as existsSync4, readFileSync as
|
|
924
|
-
import * as
|
|
1148
|
+
import { existsSync as existsSync4, readFileSync as readFileSync5, writeFileSync as writeFileSync2, renameSync, unlinkSync, promises as fsPromises2 } from "fs";
|
|
1149
|
+
import * as path6 from "path";
|
|
925
1150
|
import { performance as performance2 } from "perf_hooks";
|
|
926
1151
|
|
|
927
1152
|
// node_modules/eventemitter3/index.mjs
|
|
@@ -946,7 +1171,7 @@ function pTimeout(promise, options) {
|
|
|
946
1171
|
} = options;
|
|
947
1172
|
let timer;
|
|
948
1173
|
let abortHandler;
|
|
949
|
-
const wrappedPromise = new Promise((
|
|
1174
|
+
const wrappedPromise = new Promise((resolve5, reject) => {
|
|
950
1175
|
if (typeof milliseconds !== "number" || Math.sign(milliseconds) !== 1) {
|
|
951
1176
|
throw new TypeError(`Expected \`milliseconds\` to be a positive number, got \`${milliseconds}\``);
|
|
952
1177
|
}
|
|
@@ -960,7 +1185,7 @@ function pTimeout(promise, options) {
|
|
|
960
1185
|
};
|
|
961
1186
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
962
1187
|
}
|
|
963
|
-
promise.then(
|
|
1188
|
+
promise.then(resolve5, reject);
|
|
964
1189
|
if (milliseconds === Number.POSITIVE_INFINITY) {
|
|
965
1190
|
return;
|
|
966
1191
|
}
|
|
@@ -968,7 +1193,7 @@ function pTimeout(promise, options) {
|
|
|
968
1193
|
timer = customTimers.setTimeout.call(void 0, () => {
|
|
969
1194
|
if (fallback) {
|
|
970
1195
|
try {
|
|
971
|
-
|
|
1196
|
+
resolve5(fallback());
|
|
972
1197
|
} catch (error) {
|
|
973
1198
|
reject(error);
|
|
974
1199
|
}
|
|
@@ -978,7 +1203,7 @@ function pTimeout(promise, options) {
|
|
|
978
1203
|
promise.cancel();
|
|
979
1204
|
}
|
|
980
1205
|
if (message === false) {
|
|
981
|
-
|
|
1206
|
+
resolve5();
|
|
982
1207
|
} else if (message instanceof Error) {
|
|
983
1208
|
reject(message);
|
|
984
1209
|
} else {
|
|
@@ -1368,7 +1593,7 @@ var PQueue = class extends import_index.default {
|
|
|
1368
1593
|
// Assign unique ID if not provided
|
|
1369
1594
|
id: options.id ?? (this.#idAssigner++).toString()
|
|
1370
1595
|
};
|
|
1371
|
-
return new Promise((
|
|
1596
|
+
return new Promise((resolve5, reject) => {
|
|
1372
1597
|
const taskSymbol = /* @__PURE__ */ Symbol(`task-${options.id}`);
|
|
1373
1598
|
this.#queue.enqueue(async () => {
|
|
1374
1599
|
this.#pending++;
|
|
@@ -1406,7 +1631,7 @@ var PQueue = class extends import_index.default {
|
|
|
1406
1631
|
})]);
|
|
1407
1632
|
}
|
|
1408
1633
|
const result = await operation;
|
|
1409
|
-
|
|
1634
|
+
resolve5(result);
|
|
1410
1635
|
this.emit("completed", result);
|
|
1411
1636
|
} catch (error) {
|
|
1412
1637
|
reject(error);
|
|
@@ -1563,13 +1788,13 @@ var PQueue = class extends import_index.default {
|
|
|
1563
1788
|
});
|
|
1564
1789
|
}
|
|
1565
1790
|
async #onEvent(event, filter) {
|
|
1566
|
-
return new Promise((
|
|
1791
|
+
return new Promise((resolve5) => {
|
|
1567
1792
|
const listener = () => {
|
|
1568
1793
|
if (filter && !filter()) {
|
|
1569
1794
|
return;
|
|
1570
1795
|
}
|
|
1571
1796
|
this.off(event, listener);
|
|
1572
|
-
|
|
1797
|
+
resolve5();
|
|
1573
1798
|
};
|
|
1574
1799
|
this.on(event, listener);
|
|
1575
1800
|
});
|
|
@@ -1854,7 +2079,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
1854
2079
|
const finalDelay = Math.min(delayTime, remainingTime);
|
|
1855
2080
|
options.signal?.throwIfAborted();
|
|
1856
2081
|
if (finalDelay > 0) {
|
|
1857
|
-
await new Promise((
|
|
2082
|
+
await new Promise((resolve5, reject) => {
|
|
1858
2083
|
const onAbort = () => {
|
|
1859
2084
|
clearTimeout(timeoutToken);
|
|
1860
2085
|
options.signal?.removeEventListener("abort", onAbort);
|
|
@@ -1862,7 +2087,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
1862
2087
|
};
|
|
1863
2088
|
const timeoutToken = setTimeout(() => {
|
|
1864
2089
|
options.signal?.removeEventListener("abort", onAbort);
|
|
1865
|
-
|
|
2090
|
+
resolve5();
|
|
1866
2091
|
}, finalDelay);
|
|
1867
2092
|
if (options.unref) {
|
|
1868
2093
|
timeoutToken.unref?.();
|
|
@@ -1923,17 +2148,17 @@ async function pRetry(input, options = {}) {
|
|
|
1923
2148
|
}
|
|
1924
2149
|
|
|
1925
2150
|
// src/embeddings/detector.ts
|
|
1926
|
-
import { existsSync, readFileSync } from "fs";
|
|
1927
|
-
import * as
|
|
2151
|
+
import { existsSync, readFileSync as readFileSync2 } from "fs";
|
|
2152
|
+
import * as path2 from "path";
|
|
1928
2153
|
import * as os from "os";
|
|
1929
2154
|
function getOpenCodeAuthPath() {
|
|
1930
|
-
return
|
|
2155
|
+
return path2.join(os.homedir(), ".local", "share", "opencode", "auth.json");
|
|
1931
2156
|
}
|
|
1932
2157
|
function loadOpenCodeAuth() {
|
|
1933
2158
|
const authPath = getOpenCodeAuthPath();
|
|
1934
2159
|
try {
|
|
1935
2160
|
if (existsSync(authPath)) {
|
|
1936
|
-
return JSON.parse(
|
|
2161
|
+
return JSON.parse(readFileSync2(authPath, "utf-8"));
|
|
1937
2162
|
}
|
|
1938
2163
|
} catch {
|
|
1939
2164
|
}
|
|
@@ -2090,7 +2315,8 @@ function createCustomProviderInfo(config) {
|
|
|
2090
2315
|
dimensions: config.dimensions,
|
|
2091
2316
|
maxTokens: config.maxTokens ?? 8192,
|
|
2092
2317
|
costPer1MTokens: 0,
|
|
2093
|
-
timeoutMs: config.timeoutMs ?? 3e4
|
|
2318
|
+
timeoutMs: config.timeoutMs ?? 3e4,
|
|
2319
|
+
maxBatchSize: config.maxBatchSize
|
|
2094
2320
|
}
|
|
2095
2321
|
};
|
|
2096
2322
|
}
|
|
@@ -2353,21 +2579,24 @@ var CustomEmbeddingProvider = class {
|
|
|
2353
2579
|
this.credentials = credentials;
|
|
2354
2580
|
this.modelInfo = modelInfo;
|
|
2355
2581
|
}
|
|
2356
|
-
|
|
2357
|
-
const
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
return
|
|
2366
|
-
embedding: result.embeddings[0],
|
|
2367
|
-
tokensUsed: result.totalTokensUsed
|
|
2368
|
-
};
|
|
2582
|
+
splitIntoRequestBatches(texts) {
|
|
2583
|
+
const maxBatchSize = this.modelInfo.maxBatchSize;
|
|
2584
|
+
if (!maxBatchSize || texts.length <= maxBatchSize) {
|
|
2585
|
+
return [texts];
|
|
2586
|
+
}
|
|
2587
|
+
const batches = [];
|
|
2588
|
+
for (let i = 0; i < texts.length; i += maxBatchSize) {
|
|
2589
|
+
batches.push(texts.slice(i, i + maxBatchSize));
|
|
2590
|
+
}
|
|
2591
|
+
return batches;
|
|
2369
2592
|
}
|
|
2370
|
-
async
|
|
2593
|
+
async embedRequest(texts) {
|
|
2594
|
+
if (texts.length === 0) {
|
|
2595
|
+
return {
|
|
2596
|
+
embeddings: [],
|
|
2597
|
+
totalTokensUsed: 0
|
|
2598
|
+
};
|
|
2599
|
+
}
|
|
2371
2600
|
const headers = {
|
|
2372
2601
|
"Content-Type": "application/json"
|
|
2373
2602
|
};
|
|
@@ -2428,6 +2657,34 @@ var CustomEmbeddingProvider = class {
|
|
|
2428
2657
|
}
|
|
2429
2658
|
throw new Error("Custom embedding API returned unexpected response format. Expected OpenAI-compatible format with data[].embedding.");
|
|
2430
2659
|
}
|
|
2660
|
+
async embedQuery(query) {
|
|
2661
|
+
const result = await this.embedBatch([query]);
|
|
2662
|
+
return {
|
|
2663
|
+
embedding: result.embeddings[0],
|
|
2664
|
+
tokensUsed: result.totalTokensUsed
|
|
2665
|
+
};
|
|
2666
|
+
}
|
|
2667
|
+
async embedDocument(document) {
|
|
2668
|
+
const result = await this.embedBatch([document]);
|
|
2669
|
+
return {
|
|
2670
|
+
embedding: result.embeddings[0],
|
|
2671
|
+
tokensUsed: result.totalTokensUsed
|
|
2672
|
+
};
|
|
2673
|
+
}
|
|
2674
|
+
async embedBatch(texts) {
|
|
2675
|
+
const requestBatches = this.splitIntoRequestBatches(texts);
|
|
2676
|
+
const embeddings = [];
|
|
2677
|
+
let totalTokensUsed = 0;
|
|
2678
|
+
for (const batch of requestBatches) {
|
|
2679
|
+
const result = await this.embedRequest(batch);
|
|
2680
|
+
embeddings.push(...result.embeddings);
|
|
2681
|
+
totalTokensUsed += result.totalTokensUsed;
|
|
2682
|
+
}
|
|
2683
|
+
return {
|
|
2684
|
+
embeddings,
|
|
2685
|
+
totalTokensUsed
|
|
2686
|
+
};
|
|
2687
|
+
}
|
|
2431
2688
|
getModelInfo() {
|
|
2432
2689
|
return this.modelInfo;
|
|
2433
2690
|
}
|
|
@@ -2435,8 +2692,8 @@ var CustomEmbeddingProvider = class {
|
|
|
2435
2692
|
|
|
2436
2693
|
// src/utils/files.ts
|
|
2437
2694
|
var import_ignore = __toESM(require_ignore(), 1);
|
|
2438
|
-
import { existsSync as existsSync2, readFileSync as
|
|
2439
|
-
import * as
|
|
2695
|
+
import { existsSync as existsSync2, readFileSync as readFileSync3, promises as fsPromises } from "fs";
|
|
2696
|
+
import * as path3 from "path";
|
|
2440
2697
|
function createIgnoreFilter(projectRoot) {
|
|
2441
2698
|
const ig = (0, import_ignore.default)();
|
|
2442
2699
|
const defaultIgnores = [
|
|
@@ -2453,9 +2710,9 @@ function createIgnoreFilter(projectRoot) {
|
|
|
2453
2710
|
".opencode"
|
|
2454
2711
|
];
|
|
2455
2712
|
ig.add(defaultIgnores);
|
|
2456
|
-
const gitignorePath =
|
|
2713
|
+
const gitignorePath = path3.join(projectRoot, ".gitignore");
|
|
2457
2714
|
if (existsSync2(gitignorePath)) {
|
|
2458
|
-
const gitignoreContent =
|
|
2715
|
+
const gitignoreContent = readFileSync3(gitignorePath, "utf-8");
|
|
2459
2716
|
ig.add(gitignoreContent);
|
|
2460
2717
|
}
|
|
2461
2718
|
return ig;
|
|
@@ -2471,8 +2728,8 @@ function matchGlob(filePath, pattern) {
|
|
|
2471
2728
|
async function* walkDirectory(dir, projectRoot, includePatterns, excludePatterns, ignoreFilter, maxFileSize, skipped) {
|
|
2472
2729
|
const entries = await fsPromises.readdir(dir, { withFileTypes: true });
|
|
2473
2730
|
for (const entry of entries) {
|
|
2474
|
-
const fullPath =
|
|
2475
|
-
const relativePath =
|
|
2731
|
+
const fullPath = path3.join(dir, entry.name);
|
|
2732
|
+
const relativePath = path3.relative(projectRoot, fullPath);
|
|
2476
2733
|
if (ignoreFilter.ignores(relativePath)) {
|
|
2477
2734
|
if (entry.isFile()) {
|
|
2478
2735
|
skipped.push({ path: relativePath, reason: "gitignore" });
|
|
@@ -2533,6 +2790,9 @@ async function collectFiles(projectRoot, includePatterns, excludePatterns, maxFi
|
|
|
2533
2790
|
}
|
|
2534
2791
|
|
|
2535
2792
|
// src/utils/cost.ts
|
|
2793
|
+
function estimateTokens(text) {
|
|
2794
|
+
return Math.ceil(text.length / 4);
|
|
2795
|
+
}
|
|
2536
2796
|
function estimateChunksFromFiles(files) {
|
|
2537
2797
|
let totalChunks = 0;
|
|
2538
2798
|
for (const file of files) {
|
|
@@ -2887,8 +3147,9 @@ function initializeLogger(config) {
|
|
|
2887
3147
|
}
|
|
2888
3148
|
|
|
2889
3149
|
// src/native/index.ts
|
|
2890
|
-
import * as
|
|
3150
|
+
import * as path4 from "path";
|
|
2891
3151
|
import * as os2 from "os";
|
|
3152
|
+
import * as module from "module";
|
|
2892
3153
|
import { fileURLToPath } from "url";
|
|
2893
3154
|
function getNativeBinding() {
|
|
2894
3155
|
const platform2 = os2.platform();
|
|
@@ -2908,17 +3169,22 @@ function getNativeBinding() {
|
|
|
2908
3169
|
throw new Error(`Unsupported platform: ${platform2}-${arch2}`);
|
|
2909
3170
|
}
|
|
2910
3171
|
let currentDir;
|
|
3172
|
+
let requireTarget;
|
|
2911
3173
|
if (typeof import.meta !== "undefined" && import.meta.url) {
|
|
2912
|
-
currentDir =
|
|
3174
|
+
currentDir = path4.dirname(fileURLToPath(import.meta.url));
|
|
3175
|
+
requireTarget = import.meta.url;
|
|
2913
3176
|
} else if (typeof __dirname !== "undefined") {
|
|
2914
3177
|
currentDir = __dirname;
|
|
3178
|
+
requireTarget = __filename;
|
|
2915
3179
|
} else {
|
|
2916
3180
|
currentDir = process.cwd();
|
|
3181
|
+
requireTarget = path4.join(currentDir, "index.js");
|
|
2917
3182
|
}
|
|
2918
3183
|
const isDevMode = currentDir.includes("/src/native");
|
|
2919
|
-
const packageRoot = isDevMode ?
|
|
2920
|
-
const nativePath =
|
|
2921
|
-
|
|
3184
|
+
const packageRoot = isDevMode ? path4.resolve(currentDir, "../..") : path4.resolve(currentDir, "..");
|
|
3185
|
+
const nativePath = path4.join(packageRoot, "native", bindingName);
|
|
3186
|
+
const require2 = module.createRequire(requireTarget);
|
|
3187
|
+
return require2(nativePath);
|
|
2922
3188
|
}
|
|
2923
3189
|
var native = getNativeBinding();
|
|
2924
3190
|
function parseFiles(files) {
|
|
@@ -3036,7 +3302,7 @@ var VectorStore = class {
|
|
|
3036
3302
|
var CHARS_PER_TOKEN = 4;
|
|
3037
3303
|
var MAX_BATCH_TOKENS = 7500;
|
|
3038
3304
|
var MAX_SINGLE_CHUNK_TOKENS = 2e3;
|
|
3039
|
-
function
|
|
3305
|
+
function estimateTokens2(text) {
|
|
3040
3306
|
return Math.ceil(text.length / CHARS_PER_TOKEN);
|
|
3041
3307
|
}
|
|
3042
3308
|
function createEmbeddingText(chunk, filePath) {
|
|
@@ -3101,7 +3367,7 @@ function createDynamicBatches(chunks) {
|
|
|
3101
3367
|
let currentBatch = [];
|
|
3102
3368
|
let currentTokens = 0;
|
|
3103
3369
|
for (const chunk of chunks) {
|
|
3104
|
-
const chunkTokens =
|
|
3370
|
+
const chunkTokens = estimateTokens2(chunk.text);
|
|
3105
3371
|
if (currentBatch.length > 0 && currentTokens + chunkTokens > MAX_BATCH_TOKENS) {
|
|
3106
3372
|
batches.push(currentBatch);
|
|
3107
3373
|
currentBatch = [];
|
|
@@ -3398,11 +3664,40 @@ var Database = class {
|
|
|
3398
3664
|
};
|
|
3399
3665
|
|
|
3400
3666
|
// src/git/index.ts
|
|
3401
|
-
import { existsSync as existsSync3, readFileSync as
|
|
3402
|
-
import * as
|
|
3403
|
-
|
|
3667
|
+
import { existsSync as existsSync3, readFileSync as readFileSync4, readdirSync, statSync } from "fs";
|
|
3668
|
+
import * as path5 from "path";
|
|
3669
|
+
function readPackedRefs(gitDir) {
|
|
3670
|
+
const packedRefsPath = path5.join(gitDir, "packed-refs");
|
|
3671
|
+
if (!existsSync3(packedRefsPath)) {
|
|
3672
|
+
return [];
|
|
3673
|
+
}
|
|
3674
|
+
try {
|
|
3675
|
+
return readFileSync4(packedRefsPath, "utf-8").split("\n").map((line) => line.trim()).filter((line) => line.length > 0 && !line.startsWith("#") && !line.startsWith("^"));
|
|
3676
|
+
} catch {
|
|
3677
|
+
return [];
|
|
3678
|
+
}
|
|
3679
|
+
}
|
|
3680
|
+
function resolveCommonGitDir(gitDir) {
|
|
3681
|
+
const commonDirPath = path5.join(gitDir, "commondir");
|
|
3682
|
+
if (!existsSync3(commonDirPath)) {
|
|
3683
|
+
return gitDir;
|
|
3684
|
+
}
|
|
3685
|
+
try {
|
|
3686
|
+
const raw = readFileSync4(commonDirPath, "utf-8").trim();
|
|
3687
|
+
if (!raw) {
|
|
3688
|
+
return gitDir;
|
|
3689
|
+
}
|
|
3690
|
+
const resolved = path5.isAbsolute(raw) ? raw : path5.resolve(gitDir, raw);
|
|
3691
|
+
if (existsSync3(resolved)) {
|
|
3692
|
+
return resolved;
|
|
3693
|
+
}
|
|
3694
|
+
} catch {
|
|
3695
|
+
return gitDir;
|
|
3696
|
+
}
|
|
3697
|
+
return gitDir;
|
|
3698
|
+
}
|
|
3404
3699
|
function resolveGitDir(repoRoot) {
|
|
3405
|
-
const gitPath =
|
|
3700
|
+
const gitPath = path5.join(repoRoot, ".git");
|
|
3406
3701
|
if (!existsSync3(gitPath)) {
|
|
3407
3702
|
return null;
|
|
3408
3703
|
}
|
|
@@ -3412,11 +3707,11 @@ function resolveGitDir(repoRoot) {
|
|
|
3412
3707
|
return gitPath;
|
|
3413
3708
|
}
|
|
3414
3709
|
if (stat.isFile()) {
|
|
3415
|
-
const content =
|
|
3710
|
+
const content = readFileSync4(gitPath, "utf-8").trim();
|
|
3416
3711
|
const match = content.match(/^gitdir:\s*(.+)$/);
|
|
3417
3712
|
if (match) {
|
|
3418
3713
|
const gitdir = match[1];
|
|
3419
|
-
const resolvedPath =
|
|
3714
|
+
const resolvedPath = path5.isAbsolute(gitdir) ? gitdir : path5.resolve(repoRoot, gitdir);
|
|
3420
3715
|
if (existsSync3(resolvedPath)) {
|
|
3421
3716
|
return resolvedPath;
|
|
3422
3717
|
}
|
|
@@ -3434,12 +3729,12 @@ function getCurrentBranch(repoRoot) {
|
|
|
3434
3729
|
if (!gitDir) {
|
|
3435
3730
|
return null;
|
|
3436
3731
|
}
|
|
3437
|
-
const headPath =
|
|
3732
|
+
const headPath = path5.join(gitDir, "HEAD");
|
|
3438
3733
|
if (!existsSync3(headPath)) {
|
|
3439
3734
|
return null;
|
|
3440
3735
|
}
|
|
3441
3736
|
try {
|
|
3442
|
-
const headContent =
|
|
3737
|
+
const headContent = readFileSync4(headPath, "utf-8").trim();
|
|
3443
3738
|
const match = headContent.match(/^ref: refs\/heads\/(.+)$/);
|
|
3444
3739
|
if (match) {
|
|
3445
3740
|
return match[1];
|
|
@@ -3454,37 +3749,20 @@ function getCurrentBranch(repoRoot) {
|
|
|
3454
3749
|
}
|
|
3455
3750
|
function getBaseBranch(repoRoot) {
|
|
3456
3751
|
const gitDir = resolveGitDir(repoRoot);
|
|
3752
|
+
const refStoreDir = gitDir ? resolveCommonGitDir(gitDir) : null;
|
|
3457
3753
|
const candidates = ["main", "master", "develop", "trunk"];
|
|
3458
|
-
if (
|
|
3754
|
+
if (refStoreDir) {
|
|
3459
3755
|
for (const candidate of candidates) {
|
|
3460
|
-
const refPath =
|
|
3756
|
+
const refPath = path5.join(refStoreDir, "refs", "heads", candidate);
|
|
3461
3757
|
if (existsSync3(refPath)) {
|
|
3462
3758
|
return candidate;
|
|
3463
3759
|
}
|
|
3464
|
-
const
|
|
3465
|
-
if (
|
|
3466
|
-
|
|
3467
|
-
const content = readFileSync3(packedRefsPath, "utf-8");
|
|
3468
|
-
if (content.includes(`refs/heads/${candidate}`)) {
|
|
3469
|
-
return candidate;
|
|
3470
|
-
}
|
|
3471
|
-
} catch {
|
|
3472
|
-
}
|
|
3760
|
+
const packedRefs = readPackedRefs(refStoreDir);
|
|
3761
|
+
if (packedRefs.some((line) => line.endsWith(` refs/heads/${candidate}`))) {
|
|
3762
|
+
return candidate;
|
|
3473
3763
|
}
|
|
3474
3764
|
}
|
|
3475
3765
|
}
|
|
3476
|
-
try {
|
|
3477
|
-
const result = execSync("git remote show origin", {
|
|
3478
|
-
cwd: repoRoot,
|
|
3479
|
-
encoding: "utf-8",
|
|
3480
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
3481
|
-
});
|
|
3482
|
-
const match = result.match(/HEAD branch: (.+)/);
|
|
3483
|
-
if (match) {
|
|
3484
|
-
return match[1].trim();
|
|
3485
|
-
}
|
|
3486
|
-
} catch {
|
|
3487
|
-
}
|
|
3488
3766
|
return getCurrentBranch(repoRoot) ?? "main";
|
|
3489
3767
|
}
|
|
3490
3768
|
function getBranchOrDefault(repoRoot) {
|
|
@@ -3495,7 +3773,7 @@ function getBranchOrDefault(repoRoot) {
|
|
|
3495
3773
|
}
|
|
3496
3774
|
|
|
3497
3775
|
// src/indexer/index.ts
|
|
3498
|
-
var CALL_GRAPH_LANGUAGES = /* @__PURE__ */ new Set(["typescript", "tsx", "javascript", "jsx", "python", "go", "rust"]);
|
|
3776
|
+
var CALL_GRAPH_LANGUAGES = /* @__PURE__ */ new Set(["typescript", "tsx", "javascript", "jsx", "python", "go", "rust", "php"]);
|
|
3499
3777
|
var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
3500
3778
|
"function_declaration",
|
|
3501
3779
|
"function",
|
|
@@ -3516,7 +3794,8 @@ var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
|
3516
3794
|
"struct_item",
|
|
3517
3795
|
"enum_item",
|
|
3518
3796
|
"trait_item",
|
|
3519
|
-
"mod_item"
|
|
3797
|
+
"mod_item",
|
|
3798
|
+
"trait_declaration"
|
|
3520
3799
|
]);
|
|
3521
3800
|
function float32ArrayToBuffer(arr) {
|
|
3522
3801
|
const float32 = new Float32Array(arr);
|
|
@@ -3898,8 +4177,8 @@ function stripFilePathHint(query) {
|
|
|
3898
4177
|
const stripped = query.replace(FILE_PATH_HINT_SUFFIX_REGEX, "").trim();
|
|
3899
4178
|
return stripped.length > 0 ? stripped : query;
|
|
3900
4179
|
}
|
|
3901
|
-
function buildDeterministicIdentifierPass(query, candidates, limit) {
|
|
3902
|
-
if (
|
|
4180
|
+
function buildDeterministicIdentifierPass(query, candidates, limit, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
|
|
4181
|
+
if (!prioritizeSourcePaths) {
|
|
3903
4182
|
return [];
|
|
3904
4183
|
}
|
|
3905
4184
|
const primary = extractPrimaryIdentifierQueryHint(query);
|
|
@@ -4115,9 +4394,8 @@ function rankHybridResults(query, semanticResults, keywordResults, options) {
|
|
|
4115
4394
|
const fused = options.fusionStrategy === "rrf" ? fuseResultsRrf(semanticResults, keywordResults, options.rrfK, overfetchLimit) : fuseResultsWeighted(semanticResults, keywordResults, options.hybridWeight, overfetchLimit);
|
|
4116
4395
|
const rerankPoolLimit = Math.max(overfetchLimit, options.rerankTopN * 3, options.limit * 6);
|
|
4117
4396
|
const rerankPool = fused.slice(0, rerankPoolLimit);
|
|
4118
|
-
const intent = classifyQueryIntentRaw(query);
|
|
4119
4397
|
return rerankResults(query, rerankPool, options.rerankTopN, {
|
|
4120
|
-
prioritizeSourcePaths:
|
|
4398
|
+
prioritizeSourcePaths: options.prioritizeSourcePaths ?? classifyQueryIntentRaw(query) === "source"
|
|
4121
4399
|
});
|
|
4122
4400
|
}
|
|
4123
4401
|
function rankSemanticOnlyResults(query, semanticResults, options) {
|
|
@@ -4127,11 +4405,11 @@ function rankSemanticOnlyResults(query, semanticResults, options) {
|
|
|
4127
4405
|
prioritizeSourcePaths: options.prioritizeSourcePaths ?? false
|
|
4128
4406
|
});
|
|
4129
4407
|
}
|
|
4130
|
-
function promoteIdentifierMatches(query, combined, semanticCandidates, keywordCandidates, database, branchChunkIds) {
|
|
4408
|
+
function promoteIdentifierMatches(query, combined, semanticCandidates, keywordCandidates, database, branchChunkIds, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
|
|
4131
4409
|
if (combined.length === 0) {
|
|
4132
4410
|
return combined;
|
|
4133
4411
|
}
|
|
4134
|
-
if (
|
|
4412
|
+
if (!prioritizeSourcePaths) {
|
|
4135
4413
|
return combined;
|
|
4136
4414
|
}
|
|
4137
4415
|
const identifierHints = extractIdentifierHints(query);
|
|
@@ -4221,8 +4499,8 @@ function promoteIdentifierMatches(query, combined, semanticCandidates, keywordCa
|
|
|
4221
4499
|
const remainder = combined.filter((candidate) => !promotedIds.has(candidate.id));
|
|
4222
4500
|
return [...promoted, ...remainder];
|
|
4223
4501
|
}
|
|
4224
|
-
function buildSymbolDefinitionLane(query, database, branchChunkIds, limit, fallbackCandidates) {
|
|
4225
|
-
if (
|
|
4502
|
+
function buildSymbolDefinitionLane(query, database, branchChunkIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
|
|
4503
|
+
if (!prioritizeSourcePaths) {
|
|
4226
4504
|
return [];
|
|
4227
4505
|
}
|
|
4228
4506
|
const identifierHints = extractIdentifierHints(query);
|
|
@@ -4367,8 +4645,8 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, limit, fallb
|
|
|
4367
4645
|
const withFallback = Array.from(symbolCandidates.values()).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
|
4368
4646
|
return withFallback.slice(0, Math.max(limit * 2, limit));
|
|
4369
4647
|
}
|
|
4370
|
-
function buildIdentifierDefinitionLane(query, candidates, limit) {
|
|
4371
|
-
if (
|
|
4648
|
+
function buildIdentifierDefinitionLane(query, candidates, limit, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
|
|
4649
|
+
if (!prioritizeSourcePaths) {
|
|
4372
4650
|
return [];
|
|
4373
4651
|
}
|
|
4374
4652
|
const primaryHint = extractPrimaryIdentifierQueryHint(query);
|
|
@@ -4453,22 +4731,22 @@ var Indexer = class {
|
|
|
4453
4731
|
this.projectRoot = projectRoot;
|
|
4454
4732
|
this.config = config;
|
|
4455
4733
|
this.indexPath = this.getIndexPath();
|
|
4456
|
-
this.fileHashCachePath =
|
|
4457
|
-
this.failedBatchesPath =
|
|
4458
|
-
this.indexingLockPath =
|
|
4734
|
+
this.fileHashCachePath = path6.join(this.indexPath, "file-hashes.json");
|
|
4735
|
+
this.failedBatchesPath = path6.join(this.indexPath, "failed-batches.json");
|
|
4736
|
+
this.indexingLockPath = path6.join(this.indexPath, "indexing.lock");
|
|
4459
4737
|
this.logger = initializeLogger(config.debug);
|
|
4460
4738
|
}
|
|
4461
4739
|
getIndexPath() {
|
|
4462
4740
|
if (this.config.scope === "global") {
|
|
4463
4741
|
const homeDir = process.env.HOME || process.env.USERPROFILE || "";
|
|
4464
|
-
return
|
|
4742
|
+
return path6.join(homeDir, ".opencode", "global-index");
|
|
4465
4743
|
}
|
|
4466
|
-
return
|
|
4744
|
+
return path6.join(this.projectRoot, ".opencode", "index");
|
|
4467
4745
|
}
|
|
4468
4746
|
loadFileHashCache() {
|
|
4469
4747
|
try {
|
|
4470
4748
|
if (existsSync4(this.fileHashCachePath)) {
|
|
4471
|
-
const data =
|
|
4749
|
+
const data = readFileSync5(this.fileHashCachePath, "utf-8");
|
|
4472
4750
|
const parsed = JSON.parse(data);
|
|
4473
4751
|
this.fileHashCache = new Map(Object.entries(parsed));
|
|
4474
4752
|
}
|
|
@@ -4485,7 +4763,7 @@ var Indexer = class {
|
|
|
4485
4763
|
}
|
|
4486
4764
|
atomicWriteSync(targetPath, data) {
|
|
4487
4765
|
const tempPath = `${targetPath}.tmp`;
|
|
4488
|
-
|
|
4766
|
+
writeFileSync2(tempPath, data);
|
|
4489
4767
|
renameSync(tempPath, targetPath);
|
|
4490
4768
|
}
|
|
4491
4769
|
checkForInterruptedIndexing() {
|
|
@@ -4496,7 +4774,7 @@ var Indexer = class {
|
|
|
4496
4774
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4497
4775
|
pid: process.pid
|
|
4498
4776
|
};
|
|
4499
|
-
|
|
4777
|
+
writeFileSync2(this.indexingLockPath, JSON.stringify(lockData));
|
|
4500
4778
|
}
|
|
4501
4779
|
releaseIndexingLock() {
|
|
4502
4780
|
if (existsSync4(this.indexingLockPath)) {
|
|
@@ -4515,7 +4793,7 @@ var Indexer = class {
|
|
|
4515
4793
|
loadFailedBatches() {
|
|
4516
4794
|
try {
|
|
4517
4795
|
if (existsSync4(this.failedBatchesPath)) {
|
|
4518
|
-
const data =
|
|
4796
|
+
const data = readFileSync5(this.failedBatchesPath, "utf-8");
|
|
4519
4797
|
return JSON.parse(data);
|
|
4520
4798
|
}
|
|
4521
4799
|
} catch {
|
|
@@ -4531,7 +4809,7 @@ var Indexer = class {
|
|
|
4531
4809
|
}
|
|
4532
4810
|
return;
|
|
4533
4811
|
}
|
|
4534
|
-
|
|
4812
|
+
writeFileSync2(this.failedBatchesPath, JSON.stringify(batches, null, 2));
|
|
4535
4813
|
}
|
|
4536
4814
|
addFailedBatch(batch, error) {
|
|
4537
4815
|
const existing = this.loadFailedBatches();
|
|
@@ -4590,13 +4868,13 @@ var Indexer = class {
|
|
|
4590
4868
|
this.provider = createEmbeddingProvider(this.configuredProviderInfo);
|
|
4591
4869
|
await fsPromises2.mkdir(this.indexPath, { recursive: true });
|
|
4592
4870
|
const dimensions = this.configuredProviderInfo.modelInfo.dimensions;
|
|
4593
|
-
const storePath =
|
|
4871
|
+
const storePath = path6.join(this.indexPath, "vectors");
|
|
4594
4872
|
this.store = new VectorStore(storePath, dimensions);
|
|
4595
|
-
const indexFilePath =
|
|
4873
|
+
const indexFilePath = path6.join(this.indexPath, "vectors.usearch");
|
|
4596
4874
|
if (existsSync4(indexFilePath)) {
|
|
4597
4875
|
this.store.load();
|
|
4598
4876
|
}
|
|
4599
|
-
const invertedIndexPath =
|
|
4877
|
+
const invertedIndexPath = path6.join(this.indexPath, "inverted-index.json");
|
|
4600
4878
|
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
4601
4879
|
try {
|
|
4602
4880
|
this.invertedIndex.load();
|
|
@@ -4606,7 +4884,7 @@ var Indexer = class {
|
|
|
4606
4884
|
}
|
|
4607
4885
|
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
4608
4886
|
}
|
|
4609
|
-
const dbPath =
|
|
4887
|
+
const dbPath = path6.join(this.indexPath, "codebase.db");
|
|
4610
4888
|
const dbIsNew = !existsSync4(dbPath);
|
|
4611
4889
|
this.database = new Database(dbPath);
|
|
4612
4890
|
if (this.checkForInterruptedIndexing()) {
|
|
@@ -4885,7 +5163,7 @@ var Indexer = class {
|
|
|
4885
5163
|
for (const parsed of parsedFiles) {
|
|
4886
5164
|
currentFilePaths.add(parsed.path);
|
|
4887
5165
|
if (parsed.chunks.length === 0) {
|
|
4888
|
-
const relativePath =
|
|
5166
|
+
const relativePath = path6.relative(this.projectRoot, parsed.path);
|
|
4889
5167
|
stats.parseFailures.push(relativePath);
|
|
4890
5168
|
}
|
|
4891
5169
|
let fileChunkCount = 0;
|
|
@@ -5096,7 +5374,7 @@ var Indexer = class {
|
|
|
5096
5374
|
for (const batch of dynamicBatches) {
|
|
5097
5375
|
queue.add(async () => {
|
|
5098
5376
|
if (rateLimitBackoffMs > 0) {
|
|
5099
|
-
await new Promise((
|
|
5377
|
+
await new Promise((resolve5) => setTimeout(resolve5, rateLimitBackoffMs));
|
|
5100
5378
|
}
|
|
5101
5379
|
try {
|
|
5102
5380
|
const result = await pRetry(
|
|
@@ -5300,6 +5578,7 @@ var Indexer = class {
|
|
|
5300
5578
|
const rrfK = this.config.search.rrfK;
|
|
5301
5579
|
const rerankTopN = this.config.search.rerankTopN;
|
|
5302
5580
|
const filterByBranch = options?.filterByBranch ?? true;
|
|
5581
|
+
const sourceIntent = options?.definitionIntent === true || classifyQueryIntentRaw(query) === "source";
|
|
5303
5582
|
this.logger.search("debug", "Starting search", {
|
|
5304
5583
|
query,
|
|
5305
5584
|
maxResults,
|
|
@@ -5351,7 +5630,8 @@ var Indexer = class {
|
|
|
5351
5630
|
rrfK,
|
|
5352
5631
|
rerankTopN,
|
|
5353
5632
|
limit: maxResults,
|
|
5354
|
-
hybridWeight
|
|
5633
|
+
hybridWeight,
|
|
5634
|
+
prioritizeSourcePaths: sourceIntent
|
|
5355
5635
|
});
|
|
5356
5636
|
const fusionMs = performance2.now() - fusionStartTime;
|
|
5357
5637
|
const rescued = promoteIdentifierMatches(
|
|
@@ -5360,30 +5640,33 @@ var Indexer = class {
|
|
|
5360
5640
|
semanticCandidates,
|
|
5361
5641
|
keywordCandidates,
|
|
5362
5642
|
database,
|
|
5363
|
-
branchChunkIds
|
|
5643
|
+
branchChunkIds,
|
|
5644
|
+
sourceIntent
|
|
5364
5645
|
);
|
|
5365
5646
|
const union = unionCandidates(semanticCandidates, keywordCandidates);
|
|
5366
5647
|
const deterministicIdentifierLane = buildDeterministicIdentifierPass(
|
|
5367
5648
|
query,
|
|
5368
5649
|
union,
|
|
5369
|
-
maxResults
|
|
5650
|
+
maxResults,
|
|
5651
|
+
sourceIntent
|
|
5370
5652
|
);
|
|
5371
5653
|
const identifierLane = buildIdentifierDefinitionLane(
|
|
5372
5654
|
query,
|
|
5373
5655
|
union,
|
|
5374
|
-
maxResults
|
|
5656
|
+
maxResults,
|
|
5657
|
+
sourceIntent
|
|
5375
5658
|
);
|
|
5376
5659
|
const symbolLane = buildSymbolDefinitionLane(
|
|
5377
5660
|
query,
|
|
5378
5661
|
database,
|
|
5379
5662
|
branchChunkIds,
|
|
5380
5663
|
maxResults,
|
|
5381
|
-
union
|
|
5664
|
+
union,
|
|
5665
|
+
sourceIntent
|
|
5382
5666
|
);
|
|
5383
5667
|
const prePrimaryLane = mergeTieredResults(deterministicIdentifierLane, identifierLane, maxResults * 4);
|
|
5384
5668
|
const primaryLane = mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
5385
5669
|
const tiered = mergeTieredResults(primaryLane, rescued, maxResults * 4);
|
|
5386
|
-
const sourceIntent = classifyQueryIntentRaw(query) === "source";
|
|
5387
5670
|
const hasCodeHints = extractCodeTermHints(query).length > 0 || extractIdentifierHints(query).length > 0;
|
|
5388
5671
|
const baseFiltered = tiered.filter((r) => {
|
|
5389
5672
|
if (r.score < this.config.search.minScore) return false;
|
|
@@ -5740,7 +6023,896 @@ var Indexer = class {
|
|
|
5740
6023
|
}
|
|
5741
6024
|
};
|
|
5742
6025
|
|
|
6026
|
+
// src/eval/budget.ts
|
|
6027
|
+
function evaluateBudgetGate(budget, summary, comparison) {
|
|
6028
|
+
const BASELINE_P95_EPSILON_MS = 1e-3;
|
|
6029
|
+
const violations = [];
|
|
6030
|
+
const { thresholds } = budget;
|
|
6031
|
+
if (thresholds.minHitAt5 !== void 0 && summary.metrics.hitAt5 < thresholds.minHitAt5) {
|
|
6032
|
+
violations.push({
|
|
6033
|
+
metric: "minHitAt5",
|
|
6034
|
+
message: `Hit@5 ${summary.metrics.hitAt5.toFixed(4)} is below minimum ${thresholds.minHitAt5.toFixed(4)}`
|
|
6035
|
+
});
|
|
6036
|
+
}
|
|
6037
|
+
if (thresholds.minMrrAt10 !== void 0 && summary.metrics.mrrAt10 < thresholds.minMrrAt10) {
|
|
6038
|
+
violations.push({
|
|
6039
|
+
metric: "minMrrAt10",
|
|
6040
|
+
message: `MRR@10 ${summary.metrics.mrrAt10.toFixed(4)} is below minimum ${thresholds.minMrrAt10.toFixed(4)}`
|
|
6041
|
+
});
|
|
6042
|
+
}
|
|
6043
|
+
if (comparison) {
|
|
6044
|
+
if (thresholds.hitAt5MaxDrop !== void 0 && comparison.deltas.hitAt5.absolute < -thresholds.hitAt5MaxDrop) {
|
|
6045
|
+
violations.push({
|
|
6046
|
+
metric: "hitAt5MaxDrop",
|
|
6047
|
+
message: `Hit@5 drop ${comparison.deltas.hitAt5.absolute.toFixed(4)} exceeds allowed -${thresholds.hitAt5MaxDrop.toFixed(4)}`
|
|
6048
|
+
});
|
|
6049
|
+
}
|
|
6050
|
+
if (thresholds.mrrAt10MaxDrop !== void 0 && comparison.deltas.mrrAt10.absolute < -thresholds.mrrAt10MaxDrop) {
|
|
6051
|
+
violations.push({
|
|
6052
|
+
metric: "mrrAt10MaxDrop",
|
|
6053
|
+
message: `MRR@10 drop ${comparison.deltas.mrrAt10.absolute.toFixed(4)} exceeds allowed -${thresholds.mrrAt10MaxDrop.toFixed(4)}`
|
|
6054
|
+
});
|
|
6055
|
+
}
|
|
6056
|
+
if (thresholds.p95LatencyMaxMultiplier !== void 0) {
|
|
6057
|
+
const baselineP95 = comparison.deltas.latencyP95Ms.baseline;
|
|
6058
|
+
if (baselineP95 > BASELINE_P95_EPSILON_MS) {
|
|
6059
|
+
const allowed = baselineP95 * thresholds.p95LatencyMaxMultiplier;
|
|
6060
|
+
if (summary.metrics.latencyMs.p95 > allowed) {
|
|
6061
|
+
violations.push({
|
|
6062
|
+
metric: "p95LatencyMaxMultiplier",
|
|
6063
|
+
message: `p95 latency ${summary.metrics.latencyMs.p95.toFixed(3)}ms exceeds allowed ${allowed.toFixed(3)}ms (${thresholds.p95LatencyMaxMultiplier.toFixed(2)}x baseline)`
|
|
6064
|
+
});
|
|
6065
|
+
}
|
|
6066
|
+
}
|
|
6067
|
+
}
|
|
6068
|
+
}
|
|
6069
|
+
if (thresholds.p95LatencyMaxAbsoluteMs !== void 0 && summary.metrics.latencyMs.p95 > thresholds.p95LatencyMaxAbsoluteMs) {
|
|
6070
|
+
violations.push({
|
|
6071
|
+
metric: "p95LatencyMaxAbsoluteMs",
|
|
6072
|
+
message: `p95 latency ${summary.metrics.latencyMs.p95.toFixed(3)}ms exceeds absolute maximum ${thresholds.p95LatencyMaxAbsoluteMs.toFixed(3)}ms`
|
|
6073
|
+
});
|
|
6074
|
+
}
|
|
6075
|
+
return {
|
|
6076
|
+
passed: violations.length === 0,
|
|
6077
|
+
budgetName: budget.name,
|
|
6078
|
+
violations
|
|
6079
|
+
};
|
|
6080
|
+
}
|
|
6081
|
+
|
|
6082
|
+
// src/eval/metrics.ts
|
|
6083
|
+
function percentile(values, p) {
|
|
6084
|
+
if (values.length === 0) return 0;
|
|
6085
|
+
if (values.length === 1) return values[0];
|
|
6086
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
6087
|
+
const x = p * (sorted.length - 1);
|
|
6088
|
+
const lowerIndex = Math.floor(x);
|
|
6089
|
+
const upperIndex = Math.ceil(x);
|
|
6090
|
+
if (lowerIndex === upperIndex) {
|
|
6091
|
+
return sorted[lowerIndex];
|
|
6092
|
+
}
|
|
6093
|
+
const fraction = x - lowerIndex;
|
|
6094
|
+
return sorted[lowerIndex] + fraction * (sorted[upperIndex] - sorted[lowerIndex]);
|
|
6095
|
+
}
|
|
6096
|
+
function normalizePath(input) {
|
|
6097
|
+
return input.replace(/\\/g, "/");
|
|
6098
|
+
}
|
|
6099
|
+
function uniqueResultsByPath(results) {
|
|
6100
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6101
|
+
const unique = [];
|
|
6102
|
+
for (const result of results) {
|
|
6103
|
+
const normalized = normalizePath(result.filePath);
|
|
6104
|
+
if (seen.has(normalized)) continue;
|
|
6105
|
+
seen.add(normalized);
|
|
6106
|
+
unique.push(result);
|
|
6107
|
+
}
|
|
6108
|
+
return unique;
|
|
6109
|
+
}
|
|
6110
|
+
function pathMatchesExpected(actualPath, expectedPath) {
|
|
6111
|
+
const actual = normalizePath(actualPath);
|
|
6112
|
+
const expected = normalizePath(expectedPath);
|
|
6113
|
+
if (actual === expected) return true;
|
|
6114
|
+
return actual.endsWith(`/${expected}`) || expected.endsWith(`/${actual}`);
|
|
6115
|
+
}
|
|
6116
|
+
function getRelevantPaths(query) {
|
|
6117
|
+
const fromExact = query.expected.filePath ? [query.expected.filePath] : [];
|
|
6118
|
+
const fromAcceptable = query.expected.acceptableFiles ?? [];
|
|
6119
|
+
return Array.from(/* @__PURE__ */ new Set([...fromExact, ...fromAcceptable]));
|
|
6120
|
+
}
|
|
6121
|
+
function isRelevantResult(filePath, relevantPaths) {
|
|
6122
|
+
return relevantPaths.some((expected) => pathMatchesExpected(filePath, expected));
|
|
6123
|
+
}
|
|
6124
|
+
function reciprocalRankAtK(results, relevantPaths, k) {
|
|
6125
|
+
const top = uniqueResultsByPath(results).slice(0, k);
|
|
6126
|
+
for (let i = 0; i < top.length; i += 1) {
|
|
6127
|
+
if (isRelevantResult(top[i].filePath, relevantPaths)) {
|
|
6128
|
+
return 1 / (i + 1);
|
|
6129
|
+
}
|
|
6130
|
+
}
|
|
6131
|
+
return 0;
|
|
6132
|
+
}
|
|
6133
|
+
function ndcgAtK(results, relevantPaths, k) {
|
|
6134
|
+
const top = uniqueResultsByPath(results).slice(0, k);
|
|
6135
|
+
const dcg = top.reduce((sum, result, i) => {
|
|
6136
|
+
const rel = isRelevantResult(result.filePath, relevantPaths) ? 1 : 0;
|
|
6137
|
+
return sum + rel / Math.log2(i + 2);
|
|
6138
|
+
}, 0);
|
|
6139
|
+
const idealLen = Math.min(k, relevantPaths.length);
|
|
6140
|
+
const idcg = Array.from({ length: idealLen }, (_, i) => 1 / Math.log2(i + 2)).reduce(
|
|
6141
|
+
(sum, value) => sum + value,
|
|
6142
|
+
0
|
|
6143
|
+
);
|
|
6144
|
+
return idcg === 0 ? 0 : dcg / idcg;
|
|
6145
|
+
}
|
|
6146
|
+
function isDocsOrTestsPath(filePath) {
|
|
6147
|
+
const lowered = normalizePath(filePath).toLowerCase();
|
|
6148
|
+
return lowered.includes("/docs/") || lowered.includes("/test/") || lowered.includes("/tests/") || lowered.includes("readme") || lowered.includes("/benchmarks/");
|
|
6149
|
+
}
|
|
6150
|
+
function classifyFailureBucket(query, results, k) {
|
|
6151
|
+
const relevantPaths = getRelevantPaths(query);
|
|
6152
|
+
const top = uniqueResultsByPath(results).slice(0, k);
|
|
6153
|
+
const hasRelevantTopK = top.some((result) => isRelevantResult(result.filePath, relevantPaths));
|
|
6154
|
+
if (!hasRelevantTopK) {
|
|
6155
|
+
return "no-relevant-hit-top-k";
|
|
6156
|
+
}
|
|
6157
|
+
if (query.expected.symbol) {
|
|
6158
|
+
const hasSymbol = top.some(
|
|
6159
|
+
(result) => isRelevantResult(result.filePath, relevantPaths) && result.name === query.expected.symbol
|
|
6160
|
+
);
|
|
6161
|
+
if (!hasSymbol) return "wrong-symbol";
|
|
6162
|
+
}
|
|
6163
|
+
const top1 = top[0];
|
|
6164
|
+
if (top1 && !isRelevantResult(top1.filePath, relevantPaths) && isDocsOrTestsPath(top1.filePath)) {
|
|
6165
|
+
return "docs-tests-outranking-source";
|
|
6166
|
+
}
|
|
6167
|
+
if (top1 && !isRelevantResult(top1.filePath, relevantPaths)) {
|
|
6168
|
+
return "wrong-file";
|
|
6169
|
+
}
|
|
6170
|
+
return void 0;
|
|
6171
|
+
}
|
|
6172
|
+
function buildPerQueryResult(query, results, latencyMs, k) {
|
|
6173
|
+
const relevantPaths = getRelevantPaths(query);
|
|
6174
|
+
const deduped = uniqueResultsByPath(results);
|
|
6175
|
+
const hitAt = (cutoff) => deduped.slice(0, cutoff).some((result) => isRelevantResult(result.filePath, relevantPaths));
|
|
6176
|
+
const perQuery = {
|
|
6177
|
+
id: query.id,
|
|
6178
|
+
query: query.query,
|
|
6179
|
+
queryType: query.queryType,
|
|
6180
|
+
latencyMs,
|
|
6181
|
+
hitAt1: hitAt(1),
|
|
6182
|
+
hitAt3: hitAt(3),
|
|
6183
|
+
hitAt5: hitAt(5),
|
|
6184
|
+
hitAt10: hitAt(10),
|
|
6185
|
+
reciprocalRankAt10: reciprocalRankAtK(deduped, relevantPaths, 10),
|
|
6186
|
+
ndcgAt10: ndcgAtK(deduped, relevantPaths, 10),
|
|
6187
|
+
failureBucket: classifyFailureBucket(query, results, k),
|
|
6188
|
+
results: deduped
|
|
6189
|
+
};
|
|
6190
|
+
return perQuery;
|
|
6191
|
+
}
|
|
6192
|
+
function computeEvalMetrics(queries, perQuery, embeddingCallCount, embeddingTokensUsed, costPer1MTokensUsd) {
|
|
6193
|
+
const count = perQuery.length;
|
|
6194
|
+
const safeDiv = (value) => count === 0 ? 0 : value / count;
|
|
6195
|
+
const sum = {
|
|
6196
|
+
hitAt1: 0,
|
|
6197
|
+
hitAt3: 0,
|
|
6198
|
+
hitAt5: 0,
|
|
6199
|
+
hitAt10: 0,
|
|
6200
|
+
mrrAt10: 0,
|
|
6201
|
+
ndcgAt10: 0
|
|
6202
|
+
};
|
|
6203
|
+
const failureBuckets = {
|
|
6204
|
+
"wrong-file": 0,
|
|
6205
|
+
"wrong-symbol": 0,
|
|
6206
|
+
"docs-tests-outranking-source": 0,
|
|
6207
|
+
"no-relevant-hit-top-k": 0
|
|
6208
|
+
};
|
|
6209
|
+
const latencies = perQuery.map((item) => item.latencyMs);
|
|
6210
|
+
for (const query of perQuery) {
|
|
6211
|
+
if (query.hitAt1) sum.hitAt1 += 1;
|
|
6212
|
+
if (query.hitAt3) sum.hitAt3 += 1;
|
|
6213
|
+
if (query.hitAt5) sum.hitAt5 += 1;
|
|
6214
|
+
if (query.hitAt10) sum.hitAt10 += 1;
|
|
6215
|
+
sum.mrrAt10 += query.reciprocalRankAt10;
|
|
6216
|
+
sum.ndcgAt10 += query.ndcgAt10;
|
|
6217
|
+
if (query.failureBucket) {
|
|
6218
|
+
failureBuckets[query.failureBucket] += 1;
|
|
6219
|
+
}
|
|
6220
|
+
}
|
|
6221
|
+
const queryTokens = queries.reduce((acc, q) => acc + estimateTokens(q.query), 0);
|
|
6222
|
+
return {
|
|
6223
|
+
hitAt1: safeDiv(sum.hitAt1),
|
|
6224
|
+
hitAt3: safeDiv(sum.hitAt3),
|
|
6225
|
+
hitAt5: safeDiv(sum.hitAt5),
|
|
6226
|
+
hitAt10: safeDiv(sum.hitAt10),
|
|
6227
|
+
mrrAt10: safeDiv(sum.mrrAt10),
|
|
6228
|
+
ndcgAt10: safeDiv(sum.ndcgAt10),
|
|
6229
|
+
latencyMs: {
|
|
6230
|
+
p50: percentile(latencies, 0.5),
|
|
6231
|
+
p95: percentile(latencies, 0.95),
|
|
6232
|
+
p99: percentile(latencies, 0.99)
|
|
6233
|
+
},
|
|
6234
|
+
tokenEstimate: {
|
|
6235
|
+
queryTokens,
|
|
6236
|
+
embeddingTokensUsed
|
|
6237
|
+
},
|
|
6238
|
+
embedding: {
|
|
6239
|
+
callCount: embeddingCallCount,
|
|
6240
|
+
estimatedCostUsd: embeddingTokensUsed / 1e6 * costPer1MTokensUsd,
|
|
6241
|
+
costPer1MTokensUsd
|
|
6242
|
+
},
|
|
6243
|
+
failureBuckets
|
|
6244
|
+
};
|
|
6245
|
+
}
|
|
6246
|
+
|
|
6247
|
+
// src/eval/schema.ts
|
|
6248
|
+
import { readFileSync as readFileSync6 } from "fs";
|
|
6249
|
+
function parseJsonFile(filePath) {
|
|
6250
|
+
const content = readFileSync6(filePath, "utf-8");
|
|
6251
|
+
return JSON.parse(content);
|
|
6252
|
+
}
|
|
6253
|
+
function isRecord(value) {
|
|
6254
|
+
return typeof value === "object" && value !== null;
|
|
6255
|
+
}
|
|
6256
|
+
function isStringArray2(value) {
|
|
6257
|
+
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
6258
|
+
}
|
|
6259
|
+
function asPositiveNumber(value, path10) {
|
|
6260
|
+
if (typeof value !== "number" || Number.isNaN(value) || value < 0) {
|
|
6261
|
+
throw new Error(`${path10} must be a non-negative number`);
|
|
6262
|
+
}
|
|
6263
|
+
return value;
|
|
6264
|
+
}
|
|
6265
|
+
function parseQueryType(value, path10) {
|
|
6266
|
+
if (value === "definition" || value === "implementation-intent" || value === "similarity" || value === "keyword-heavy") {
|
|
6267
|
+
return value;
|
|
6268
|
+
}
|
|
6269
|
+
throw new Error(
|
|
6270
|
+
`${path10} must be one of: definition, implementation-intent, similarity, keyword-heavy`
|
|
6271
|
+
);
|
|
6272
|
+
}
|
|
6273
|
+
function parseExpected(input, path10) {
|
|
6274
|
+
if (!isRecord(input)) {
|
|
6275
|
+
throw new Error(`${path10} must be an object`);
|
|
6276
|
+
}
|
|
6277
|
+
const filePathRaw = input.filePath;
|
|
6278
|
+
const acceptableFilesRaw = input.acceptableFiles;
|
|
6279
|
+
const symbolRaw = input.symbol;
|
|
6280
|
+
const branchRaw = input.branch;
|
|
6281
|
+
const filePath = typeof filePathRaw === "string" ? filePathRaw : void 0;
|
|
6282
|
+
const acceptableFiles = isStringArray2(acceptableFilesRaw) ? acceptableFilesRaw : void 0;
|
|
6283
|
+
if (!filePath && (!acceptableFiles || acceptableFiles.length === 0)) {
|
|
6284
|
+
throw new Error(`${path10} must include either expected.filePath or expected.acceptableFiles`);
|
|
6285
|
+
}
|
|
6286
|
+
if (acceptableFilesRaw !== void 0 && !isStringArray2(acceptableFilesRaw)) {
|
|
6287
|
+
throw new Error(`${path10}.acceptableFiles must be an array of strings`);
|
|
6288
|
+
}
|
|
6289
|
+
if (symbolRaw !== void 0 && typeof symbolRaw !== "string") {
|
|
6290
|
+
throw new Error(`${path10}.symbol must be a string when provided`);
|
|
6291
|
+
}
|
|
6292
|
+
if (branchRaw !== void 0 && typeof branchRaw !== "string") {
|
|
6293
|
+
throw new Error(`${path10}.branch must be a string when provided`);
|
|
6294
|
+
}
|
|
6295
|
+
return {
|
|
6296
|
+
filePath,
|
|
6297
|
+
acceptableFiles,
|
|
6298
|
+
symbol: typeof symbolRaw === "string" ? symbolRaw : void 0,
|
|
6299
|
+
branch: typeof branchRaw === "string" ? branchRaw : void 0
|
|
6300
|
+
};
|
|
6301
|
+
}
|
|
6302
|
+
function parseQuery(input, index) {
|
|
6303
|
+
const path10 = `queries[${index}]`;
|
|
6304
|
+
if (!isRecord(input)) {
|
|
6305
|
+
throw new Error(`${path10} must be an object`);
|
|
6306
|
+
}
|
|
6307
|
+
const id = input.id;
|
|
6308
|
+
const query = input.query;
|
|
6309
|
+
const queryType = input.queryType;
|
|
6310
|
+
const expected = input.expected;
|
|
6311
|
+
if (typeof id !== "string" || id.trim().length === 0) {
|
|
6312
|
+
throw new Error(`${path10}.id must be a non-empty string`);
|
|
6313
|
+
}
|
|
6314
|
+
if (typeof query !== "string" || query.trim().length === 0) {
|
|
6315
|
+
throw new Error(`${path10}.query must be a non-empty string`);
|
|
6316
|
+
}
|
|
6317
|
+
return {
|
|
6318
|
+
id,
|
|
6319
|
+
query,
|
|
6320
|
+
queryType: parseQueryType(queryType, `${path10}.queryType`),
|
|
6321
|
+
expected: parseExpected(expected, `${path10}.expected`)
|
|
6322
|
+
};
|
|
6323
|
+
}
|
|
6324
|
+
function parseGoldenDataset(raw, sourceLabel) {
|
|
6325
|
+
if (!isRecord(raw)) {
|
|
6326
|
+
throw new Error(`${sourceLabel} must be a JSON object`);
|
|
6327
|
+
}
|
|
6328
|
+
const version = raw.version;
|
|
6329
|
+
const name = raw.name;
|
|
6330
|
+
const description = raw.description;
|
|
6331
|
+
const queriesRaw = raw.queries;
|
|
6332
|
+
if (typeof version !== "string" || version.trim().length === 0) {
|
|
6333
|
+
throw new Error(`${sourceLabel}.version must be a non-empty string`);
|
|
6334
|
+
}
|
|
6335
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
6336
|
+
throw new Error(`${sourceLabel}.name must be a non-empty string`);
|
|
6337
|
+
}
|
|
6338
|
+
if (description !== void 0 && typeof description !== "string") {
|
|
6339
|
+
throw new Error(`${sourceLabel}.description must be a string when provided`);
|
|
6340
|
+
}
|
|
6341
|
+
if (!Array.isArray(queriesRaw)) {
|
|
6342
|
+
throw new Error(`${sourceLabel}.queries must be an array`);
|
|
6343
|
+
}
|
|
6344
|
+
if (queriesRaw.length === 0) {
|
|
6345
|
+
throw new Error(`${sourceLabel}.queries must contain at least one query`);
|
|
6346
|
+
}
|
|
6347
|
+
const queries = queriesRaw.map((query, idx) => parseQuery(query, idx));
|
|
6348
|
+
const idSet = /* @__PURE__ */ new Set();
|
|
6349
|
+
for (const query of queries) {
|
|
6350
|
+
if (idSet.has(query.id)) {
|
|
6351
|
+
throw new Error(`${sourceLabel}.queries has duplicate id: ${query.id}`);
|
|
6352
|
+
}
|
|
6353
|
+
idSet.add(query.id);
|
|
6354
|
+
}
|
|
6355
|
+
return {
|
|
6356
|
+
version,
|
|
6357
|
+
name,
|
|
6358
|
+
description: typeof description === "string" ? description : void 0,
|
|
6359
|
+
queries
|
|
6360
|
+
};
|
|
6361
|
+
}
|
|
6362
|
+
function loadGoldenDataset(datasetPath) {
|
|
6363
|
+
const parsed = parseJsonFile(datasetPath);
|
|
6364
|
+
return parseGoldenDataset(parsed, datasetPath);
|
|
6365
|
+
}
|
|
6366
|
+
function parseBudget(raw, sourceLabel) {
|
|
6367
|
+
if (!isRecord(raw)) {
|
|
6368
|
+
throw new Error(`${sourceLabel} must be a JSON object`);
|
|
6369
|
+
}
|
|
6370
|
+
const name = raw.name;
|
|
6371
|
+
const baselinePath = raw.baselinePath;
|
|
6372
|
+
const failOnMissingBaseline = raw.failOnMissingBaseline;
|
|
6373
|
+
const thresholds = raw.thresholds;
|
|
6374
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
6375
|
+
throw new Error(`${sourceLabel}.name must be a non-empty string`);
|
|
6376
|
+
}
|
|
6377
|
+
if (baselinePath !== void 0 && typeof baselinePath !== "string") {
|
|
6378
|
+
throw new Error(`${sourceLabel}.baselinePath must be a string when provided`);
|
|
6379
|
+
}
|
|
6380
|
+
if (!isRecord(thresholds)) {
|
|
6381
|
+
throw new Error(`${sourceLabel}.thresholds must be an object`);
|
|
6382
|
+
}
|
|
6383
|
+
return {
|
|
6384
|
+
name,
|
|
6385
|
+
baselinePath: typeof baselinePath === "string" ? baselinePath : void 0,
|
|
6386
|
+
failOnMissingBaseline: typeof failOnMissingBaseline === "boolean" ? failOnMissingBaseline : true,
|
|
6387
|
+
thresholds: {
|
|
6388
|
+
hitAt5MaxDrop: thresholds.hitAt5MaxDrop === void 0 ? void 0 : asPositiveNumber(thresholds.hitAt5MaxDrop, `${sourceLabel}.thresholds.hitAt5MaxDrop`),
|
|
6389
|
+
mrrAt10MaxDrop: thresholds.mrrAt10MaxDrop === void 0 ? void 0 : asPositiveNumber(thresholds.mrrAt10MaxDrop, `${sourceLabel}.thresholds.mrrAt10MaxDrop`),
|
|
6390
|
+
p95LatencyMaxMultiplier: thresholds.p95LatencyMaxMultiplier === void 0 ? void 0 : asPositiveNumber(
|
|
6391
|
+
thresholds.p95LatencyMaxMultiplier,
|
|
6392
|
+
`${sourceLabel}.thresholds.p95LatencyMaxMultiplier`
|
|
6393
|
+
),
|
|
6394
|
+
p95LatencyMaxAbsoluteMs: thresholds.p95LatencyMaxAbsoluteMs === void 0 ? void 0 : asPositiveNumber(
|
|
6395
|
+
thresholds.p95LatencyMaxAbsoluteMs,
|
|
6396
|
+
`${sourceLabel}.thresholds.p95LatencyMaxAbsoluteMs`
|
|
6397
|
+
),
|
|
6398
|
+
minHitAt5: thresholds.minHitAt5 === void 0 ? void 0 : asPositiveNumber(thresholds.minHitAt5, `${sourceLabel}.thresholds.minHitAt5`),
|
|
6399
|
+
minMrrAt10: thresholds.minMrrAt10 === void 0 ? void 0 : asPositiveNumber(thresholds.minMrrAt10, `${sourceLabel}.thresholds.minMrrAt10`)
|
|
6400
|
+
}
|
|
6401
|
+
};
|
|
6402
|
+
}
|
|
6403
|
+
function loadBudget(budgetPath) {
|
|
6404
|
+
const parsed = parseJsonFile(budgetPath);
|
|
6405
|
+
return parseBudget(parsed, budgetPath);
|
|
6406
|
+
}
|
|
6407
|
+
|
|
6408
|
+
// src/eval/runner.ts
|
|
6409
|
+
function toAbsolute(projectRoot, maybeRelative) {
|
|
6410
|
+
return path7.isAbsolute(maybeRelative) ? maybeRelative : path7.join(projectRoot, maybeRelative);
|
|
6411
|
+
}
|
|
6412
|
+
function loadRawConfig(projectRoot, configPath) {
|
|
6413
|
+
const fromPath = configPath ? toAbsolute(projectRoot, configPath) : null;
|
|
6414
|
+
if (fromPath && existsSync5(fromPath)) {
|
|
6415
|
+
return JSON.parse(readFileSync7(fromPath, "utf-8"));
|
|
6416
|
+
}
|
|
6417
|
+
const projectConfig = path7.join(projectRoot, ".opencode", "codebase-index.json");
|
|
6418
|
+
if (existsSync5(projectConfig)) {
|
|
6419
|
+
return JSON.parse(readFileSync7(projectConfig, "utf-8"));
|
|
6420
|
+
}
|
|
6421
|
+
const globalConfig = path7.join(os3.homedir(), ".config", "opencode", "codebase-index.json");
|
|
6422
|
+
if (existsSync5(globalConfig)) {
|
|
6423
|
+
return JSON.parse(readFileSync7(globalConfig, "utf-8"));
|
|
6424
|
+
}
|
|
6425
|
+
return {};
|
|
6426
|
+
}
|
|
6427
|
+
function getIndexRootPath(projectRoot, scope) {
|
|
6428
|
+
if (scope === "global") {
|
|
6429
|
+
return path7.join(os3.homedir(), ".opencode", "global-index");
|
|
6430
|
+
}
|
|
6431
|
+
return path7.join(projectRoot, ".opencode", "index");
|
|
6432
|
+
}
|
|
6433
|
+
function clearIndexRoot(projectRoot, scope) {
|
|
6434
|
+
const indexRoot = getIndexRootPath(projectRoot, scope);
|
|
6435
|
+
if (existsSync5(indexRoot)) {
|
|
6436
|
+
rmSync(indexRoot, { recursive: true, force: true });
|
|
6437
|
+
}
|
|
6438
|
+
}
|
|
6439
|
+
function loadParsedConfig(projectRoot, configPath) {
|
|
6440
|
+
const raw = loadRawConfig(projectRoot, configPath);
|
|
6441
|
+
return parseConfig(raw);
|
|
6442
|
+
}
|
|
6443
|
+
function resolveSearchConfig(parsedConfig, overrides) {
|
|
6444
|
+
const nextSearch = {
|
|
6445
|
+
...parsedConfig.search
|
|
6446
|
+
};
|
|
6447
|
+
if (overrides?.fusionStrategy !== void 0) {
|
|
6448
|
+
nextSearch.fusionStrategy = overrides.fusionStrategy;
|
|
6449
|
+
}
|
|
6450
|
+
if (overrides?.hybridWeight !== void 0) {
|
|
6451
|
+
nextSearch.hybridWeight = overrides.hybridWeight;
|
|
6452
|
+
}
|
|
6453
|
+
if (overrides?.rrfK !== void 0) {
|
|
6454
|
+
nextSearch.rrfK = overrides.rrfK;
|
|
6455
|
+
}
|
|
6456
|
+
if (overrides?.rerankTopN !== void 0) {
|
|
6457
|
+
nextSearch.rerankTopN = overrides.rerankTopN;
|
|
6458
|
+
}
|
|
6459
|
+
return {
|
|
6460
|
+
...parsedConfig,
|
|
6461
|
+
search: nextSearch
|
|
6462
|
+
};
|
|
6463
|
+
}
|
|
6464
|
+
async function runEvaluation(options) {
|
|
6465
|
+
const datasetPath = toAbsolute(options.projectRoot, options.datasetPath);
|
|
6466
|
+
const againstPath = options.againstPath ? toAbsolute(options.projectRoot, options.againstPath) : void 0;
|
|
6467
|
+
const budgetPath = options.budgetPath ? toAbsolute(options.projectRoot, options.budgetPath) : void 0;
|
|
6468
|
+
const dataset = loadGoldenDataset(datasetPath);
|
|
6469
|
+
const parsedConfig = loadParsedConfig(options.projectRoot, options.configPath);
|
|
6470
|
+
const effectiveConfig = resolveSearchConfig(parsedConfig, options.searchOverrides);
|
|
6471
|
+
if (options.reindex) {
|
|
6472
|
+
clearIndexRoot(options.projectRoot, effectiveConfig.scope);
|
|
6473
|
+
}
|
|
6474
|
+
const indexer = new Indexer(options.projectRoot, effectiveConfig);
|
|
6475
|
+
await indexer.index();
|
|
6476
|
+
const perQuery = [];
|
|
6477
|
+
for (const query of dataset.queries) {
|
|
6478
|
+
if (query.expected.branch && query.expected.branch !== indexer.getCurrentBranch()) {
|
|
6479
|
+
throw new Error(
|
|
6480
|
+
`Query '${query.id}' expects branch '${query.expected.branch}', but current branch is '${indexer.getCurrentBranch()}'. Switch branch before running this dataset.`
|
|
6481
|
+
);
|
|
6482
|
+
}
|
|
6483
|
+
const start = performance3.now();
|
|
6484
|
+
const result = await indexer.search(query.query, 10, {
|
|
6485
|
+
metadataOnly: true,
|
|
6486
|
+
filterByBranch: query.expected.branch ? true : false
|
|
6487
|
+
});
|
|
6488
|
+
const elapsed = performance3.now() - start;
|
|
6489
|
+
const materialized = result.map((item) => ({
|
|
6490
|
+
filePath: item.filePath,
|
|
6491
|
+
startLine: item.startLine,
|
|
6492
|
+
endLine: item.endLine,
|
|
6493
|
+
score: item.score,
|
|
6494
|
+
chunkType: item.chunkType,
|
|
6495
|
+
name: item.name
|
|
6496
|
+
}));
|
|
6497
|
+
perQuery.push(buildPerQueryResult(query, materialized, elapsed, 10));
|
|
6498
|
+
}
|
|
6499
|
+
const logger = indexer.getLogger();
|
|
6500
|
+
const metricSnapshot = logger.getMetrics();
|
|
6501
|
+
const costPer1MTokensUsd = effectiveConfig.embeddingProvider === "custom" || effectiveConfig.embeddingProvider === "auto" ? 0 : getDefaultModelForProvider(effectiveConfig.embeddingProvider).costPer1MTokens;
|
|
6502
|
+
const summary = {
|
|
6503
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6504
|
+
projectRoot: options.projectRoot,
|
|
6505
|
+
datasetPath,
|
|
6506
|
+
datasetName: dataset.name,
|
|
6507
|
+
datasetVersion: dataset.version,
|
|
6508
|
+
queryCount: dataset.queries.length,
|
|
6509
|
+
topK: 10,
|
|
6510
|
+
searchConfig: {
|
|
6511
|
+
fusionStrategy: effectiveConfig.search.fusionStrategy,
|
|
6512
|
+
hybridWeight: effectiveConfig.search.hybridWeight,
|
|
6513
|
+
rrfK: effectiveConfig.search.rrfK,
|
|
6514
|
+
rerankTopN: effectiveConfig.search.rerankTopN
|
|
6515
|
+
},
|
|
6516
|
+
metrics: computeEvalMetrics(
|
|
6517
|
+
dataset.queries,
|
|
6518
|
+
perQuery,
|
|
6519
|
+
metricSnapshot.embeddingApiCalls,
|
|
6520
|
+
metricSnapshot.embeddingTokensUsed,
|
|
6521
|
+
costPer1MTokensUsd
|
|
6522
|
+
)
|
|
6523
|
+
};
|
|
6524
|
+
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
6525
|
+
const perQueryArtifact = buildPerQueryArtifact(perQuery);
|
|
6526
|
+
writeJson(path7.join(outputDir, "summary.json"), summary);
|
|
6527
|
+
writeJson(path7.join(outputDir, "per-query.json"), perQueryArtifact);
|
|
6528
|
+
let comparison;
|
|
6529
|
+
if (againstPath) {
|
|
6530
|
+
const baseline = loadSummary(againstPath);
|
|
6531
|
+
comparison = compareSummaries(summary, baseline, againstPath);
|
|
6532
|
+
writeJson(path7.join(outputDir, "compare.json"), comparison);
|
|
6533
|
+
}
|
|
6534
|
+
let gate;
|
|
6535
|
+
if (options.ciMode) {
|
|
6536
|
+
if (!budgetPath) {
|
|
6537
|
+
throw new Error("CI mode requires --budget path");
|
|
6538
|
+
}
|
|
6539
|
+
const budget = loadBudget(budgetPath);
|
|
6540
|
+
if (!comparison && budget.baselinePath) {
|
|
6541
|
+
const resolvedBaseline = toAbsolute(options.projectRoot, budget.baselinePath);
|
|
6542
|
+
if (existsSync5(resolvedBaseline)) {
|
|
6543
|
+
const baselineSummary = loadSummary(resolvedBaseline);
|
|
6544
|
+
comparison = compareSummaries(summary, baselineSummary, resolvedBaseline);
|
|
6545
|
+
writeJson(path7.join(outputDir, "compare.json"), comparison);
|
|
6546
|
+
} else if (budget.failOnMissingBaseline) {
|
|
6547
|
+
throw new Error(
|
|
6548
|
+
`Budget baseline is missing: ${resolvedBaseline}. Set failOnMissingBaseline=false to allow CI run without baseline.`
|
|
6549
|
+
);
|
|
6550
|
+
}
|
|
6551
|
+
}
|
|
6552
|
+
gate = evaluateBudgetGate(budget, summary, comparison);
|
|
6553
|
+
}
|
|
6554
|
+
const markdown = createSummaryMarkdown(summary, comparison, gate);
|
|
6555
|
+
writeText(path7.join(outputDir, "summary.md"), markdown);
|
|
6556
|
+
return { outputDir, summary, perQuery, comparison, gate };
|
|
6557
|
+
}
|
|
6558
|
+
async function runSweep(options, sweep) {
|
|
6559
|
+
const fusionValues = sweep.fusionStrategy && sweep.fusionStrategy.length > 0 ? [...sweep.fusionStrategy] : [void 0];
|
|
6560
|
+
const weightValues = sweep.hybridWeight && sweep.hybridWeight.length > 0 ? [...sweep.hybridWeight] : [void 0];
|
|
6561
|
+
const rrfValues = sweep.rrfK && sweep.rrfK.length > 0 ? [...sweep.rrfK] : [void 0];
|
|
6562
|
+
const rerankValues = sweep.rerankTopN && sweep.rerankTopN.length > 0 ? [...sweep.rerankTopN] : [void 0];
|
|
6563
|
+
const runs = [];
|
|
6564
|
+
for (const fusion of fusionValues) {
|
|
6565
|
+
for (const hybridWeight of weightValues) {
|
|
6566
|
+
for (const rrfK of rrfValues) {
|
|
6567
|
+
for (const rerankTopN of rerankValues) {
|
|
6568
|
+
const run = await runEvaluation({
|
|
6569
|
+
...options,
|
|
6570
|
+
searchOverrides: {
|
|
6571
|
+
...fusion !== void 0 ? { fusionStrategy: fusion } : {},
|
|
6572
|
+
...hybridWeight !== void 0 ? { hybridWeight } : {},
|
|
6573
|
+
...rrfK !== void 0 ? { rrfK } : {},
|
|
6574
|
+
...rerankTopN !== void 0 ? { rerankTopN } : {}
|
|
6575
|
+
}
|
|
6576
|
+
});
|
|
6577
|
+
runs.push({
|
|
6578
|
+
searchConfig: run.summary.searchConfig,
|
|
6579
|
+
summary: run.summary,
|
|
6580
|
+
comparison: run.comparison,
|
|
6581
|
+
gate: run.gate
|
|
6582
|
+
});
|
|
6583
|
+
}
|
|
6584
|
+
}
|
|
6585
|
+
}
|
|
6586
|
+
}
|
|
6587
|
+
const bestByHitAt5 = [...runs].sort(
|
|
6588
|
+
(a, b) => b.summary.metrics.hitAt5 - a.summary.metrics.hitAt5
|
|
6589
|
+
)[0];
|
|
6590
|
+
const bestByMrrAt10 = [...runs].sort(
|
|
6591
|
+
(a, b) => b.summary.metrics.mrrAt10 - a.summary.metrics.mrrAt10
|
|
6592
|
+
)[0];
|
|
6593
|
+
const bestByP95Latency = [...runs].sort(
|
|
6594
|
+
(a, b) => a.summary.metrics.latencyMs.p95 - b.summary.metrics.latencyMs.p95
|
|
6595
|
+
)[0];
|
|
6596
|
+
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
6597
|
+
const failedGateRuns = runs.filter((run) => run.gate && !run.gate.passed).length;
|
|
6598
|
+
const gatePassed = failedGateRuns === 0;
|
|
6599
|
+
const aggregate = {
|
|
6600
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6601
|
+
againstPath: options.againstPath,
|
|
6602
|
+
runCount: runs.length,
|
|
6603
|
+
runs,
|
|
6604
|
+
gatePassed,
|
|
6605
|
+
failedGateRuns,
|
|
6606
|
+
bestByHitAt5,
|
|
6607
|
+
bestByMrrAt10,
|
|
6608
|
+
bestByP95Latency
|
|
6609
|
+
};
|
|
6610
|
+
writeJson(path7.join(outputDir, "compare.json"), aggregate);
|
|
6611
|
+
const md = createSummaryMarkdown(
|
|
6612
|
+
bestByHitAt5?.summary ?? runs[0].summary,
|
|
6613
|
+
bestByHitAt5?.comparison,
|
|
6614
|
+
void 0,
|
|
6615
|
+
aggregate
|
|
6616
|
+
);
|
|
6617
|
+
writeText(path7.join(outputDir, "summary.md"), md);
|
|
6618
|
+
writeJson(path7.join(outputDir, "summary.json"), bestByHitAt5?.summary ?? runs[0].summary);
|
|
6619
|
+
return { outputDir, aggregate };
|
|
6620
|
+
}
|
|
6621
|
+
|
|
6622
|
+
// src/eval/cli.ts
|
|
6623
|
+
function printUsage() {
|
|
6624
|
+
console.log(`
|
|
6625
|
+
Usage:
|
|
6626
|
+
opencode-codebase-index-mcp eval run [options]
|
|
6627
|
+
opencode-codebase-index-mcp eval compare --against <summary.json> [options]
|
|
6628
|
+
opencode-codebase-index-mcp eval diff --current <summary.json> --against <summary.json> [options]
|
|
6629
|
+
|
|
6630
|
+
Options:
|
|
6631
|
+
--project <path> Project root (default: cwd)
|
|
6632
|
+
--config <path> Config JSON path
|
|
6633
|
+
--dataset <path> Golden dataset path (default: benchmarks/golden/small.json)
|
|
6634
|
+
--current <path> Current summary.json path (required for eval diff)
|
|
6635
|
+
--output <path> Output root dir (default: benchmarks/results)
|
|
6636
|
+
--against <path> Baseline summary.json to compare against
|
|
6637
|
+
--budget <path> Budget file for CI mode (default: benchmarks/budgets/default.json)
|
|
6638
|
+
--ci Enable CI gate mode
|
|
6639
|
+
--reindex Force reindex before eval
|
|
6640
|
+
|
|
6641
|
+
Search overrides:
|
|
6642
|
+
--fusionStrategy <rrf|weighted>
|
|
6643
|
+
--hybridWeight <0-1>
|
|
6644
|
+
--rrfK <number>
|
|
6645
|
+
--rerankTopN <number>
|
|
6646
|
+
|
|
6647
|
+
Sweep options (comma-separated values):
|
|
6648
|
+
--sweepFusionStrategy <rrf,weighted>
|
|
6649
|
+
--sweepHybridWeight <0.3,0.5,0.7>
|
|
6650
|
+
--sweepRrfK <30,60,90>
|
|
6651
|
+
--sweepRerankTopN <10,20,40>
|
|
6652
|
+
`);
|
|
6653
|
+
}
|
|
6654
|
+
function parseNumber(value, flag) {
|
|
6655
|
+
const parsed = Number(value);
|
|
6656
|
+
if (Number.isNaN(parsed)) {
|
|
6657
|
+
throw new Error(`${flag} must be a number`);
|
|
6658
|
+
}
|
|
6659
|
+
return parsed;
|
|
6660
|
+
}
|
|
6661
|
+
function parseCsvNumbers(value, flag) {
|
|
6662
|
+
return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0).map((item) => parseNumber(item, flag));
|
|
6663
|
+
}
|
|
6664
|
+
function parseCsvFusion(value) {
|
|
6665
|
+
const values = value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
|
|
6666
|
+
const parsed = [];
|
|
6667
|
+
for (const candidate of values) {
|
|
6668
|
+
if (candidate !== "rrf" && candidate !== "weighted") {
|
|
6669
|
+
throw new Error("--sweepFusionStrategy accepts only rrf,weighted");
|
|
6670
|
+
}
|
|
6671
|
+
parsed.push(candidate);
|
|
6672
|
+
}
|
|
6673
|
+
return parsed;
|
|
6674
|
+
}
|
|
6675
|
+
function hasSweepOptions(sweep) {
|
|
6676
|
+
return Boolean(
|
|
6677
|
+
sweep.fusionStrategy && sweep.fusionStrategy.length > 0 || sweep.hybridWeight && sweep.hybridWeight.length > 0 || sweep.rrfK && sweep.rrfK.length > 0 || sweep.rerankTopN && sweep.rerankTopN.length > 0
|
|
6678
|
+
);
|
|
6679
|
+
}
|
|
6680
|
+
function parseEvalArgs(argv, cwd) {
|
|
6681
|
+
const parsed = {
|
|
6682
|
+
projectRoot: cwd,
|
|
6683
|
+
datasetPath: "benchmarks/golden/small.json",
|
|
6684
|
+
outputRoot: "benchmarks/results",
|
|
6685
|
+
budgetPath: "benchmarks/budgets/default.json",
|
|
6686
|
+
ciMode: false,
|
|
6687
|
+
reindex: false,
|
|
6688
|
+
sweep: {}
|
|
6689
|
+
};
|
|
6690
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
6691
|
+
const arg = argv[i];
|
|
6692
|
+
const next = argv[i + 1];
|
|
6693
|
+
if (arg === "--project" && next) {
|
|
6694
|
+
parsed.projectRoot = path8.resolve(cwd, next);
|
|
6695
|
+
i += 1;
|
|
6696
|
+
continue;
|
|
6697
|
+
}
|
|
6698
|
+
if (arg === "--config" && next) {
|
|
6699
|
+
parsed.configPath = path8.resolve(cwd, next);
|
|
6700
|
+
i += 1;
|
|
6701
|
+
continue;
|
|
6702
|
+
}
|
|
6703
|
+
if (arg === "--dataset" && next) {
|
|
6704
|
+
parsed.datasetPath = next;
|
|
6705
|
+
i += 1;
|
|
6706
|
+
continue;
|
|
6707
|
+
}
|
|
6708
|
+
if (arg === "--current" && next) {
|
|
6709
|
+
parsed.currentPath = next;
|
|
6710
|
+
i += 1;
|
|
6711
|
+
continue;
|
|
6712
|
+
}
|
|
6713
|
+
if (arg === "--output" && next) {
|
|
6714
|
+
parsed.outputRoot = next;
|
|
6715
|
+
i += 1;
|
|
6716
|
+
continue;
|
|
6717
|
+
}
|
|
6718
|
+
if (arg === "--against" && next) {
|
|
6719
|
+
parsed.againstPath = next;
|
|
6720
|
+
i += 1;
|
|
6721
|
+
continue;
|
|
6722
|
+
}
|
|
6723
|
+
if (arg === "--budget" && next) {
|
|
6724
|
+
parsed.budgetPath = next;
|
|
6725
|
+
i += 1;
|
|
6726
|
+
continue;
|
|
6727
|
+
}
|
|
6728
|
+
if (arg === "--ci") {
|
|
6729
|
+
parsed.ciMode = true;
|
|
6730
|
+
continue;
|
|
6731
|
+
}
|
|
6732
|
+
if (arg === "--reindex") {
|
|
6733
|
+
parsed.reindex = true;
|
|
6734
|
+
continue;
|
|
6735
|
+
}
|
|
6736
|
+
if (arg === "--fusionStrategy" && next) {
|
|
6737
|
+
if (next !== "rrf" && next !== "weighted") {
|
|
6738
|
+
throw new Error("--fusionStrategy must be rrf or weighted");
|
|
6739
|
+
}
|
|
6740
|
+
parsed.fusionStrategy = next;
|
|
6741
|
+
i += 1;
|
|
6742
|
+
continue;
|
|
6743
|
+
}
|
|
6744
|
+
if (arg === "--hybridWeight" && next) {
|
|
6745
|
+
parsed.hybridWeight = parseNumber(next, "--hybridWeight");
|
|
6746
|
+
i += 1;
|
|
6747
|
+
continue;
|
|
6748
|
+
}
|
|
6749
|
+
if (arg === "--rrfK" && next) {
|
|
6750
|
+
parsed.rrfK = parseNumber(next, "--rrfK");
|
|
6751
|
+
i += 1;
|
|
6752
|
+
continue;
|
|
6753
|
+
}
|
|
6754
|
+
if (arg === "--rerankTopN" && next) {
|
|
6755
|
+
parsed.rerankTopN = parseNumber(next, "--rerankTopN");
|
|
6756
|
+
i += 1;
|
|
6757
|
+
continue;
|
|
6758
|
+
}
|
|
6759
|
+
if (arg === "--sweepFusionStrategy" && next) {
|
|
6760
|
+
parsed.sweep.fusionStrategy = parseCsvFusion(next);
|
|
6761
|
+
i += 1;
|
|
6762
|
+
continue;
|
|
6763
|
+
}
|
|
6764
|
+
if (arg === "--sweepHybridWeight" && next) {
|
|
6765
|
+
parsed.sweep.hybridWeight = parseCsvNumbers(next, "--sweepHybridWeight");
|
|
6766
|
+
i += 1;
|
|
6767
|
+
continue;
|
|
6768
|
+
}
|
|
6769
|
+
if (arg === "--sweepRrfK" && next) {
|
|
6770
|
+
parsed.sweep.rrfK = parseCsvNumbers(next, "--sweepRrfK");
|
|
6771
|
+
i += 1;
|
|
6772
|
+
continue;
|
|
6773
|
+
}
|
|
6774
|
+
if (arg === "--sweepRerankTopN" && next) {
|
|
6775
|
+
parsed.sweep.rerankTopN = parseCsvNumbers(next, "--sweepRerankTopN");
|
|
6776
|
+
i += 1;
|
|
6777
|
+
continue;
|
|
6778
|
+
}
|
|
6779
|
+
}
|
|
6780
|
+
return parsed;
|
|
6781
|
+
}
|
|
6782
|
+
function parseEvalSubcommandOptions(argv, cwd) {
|
|
6783
|
+
let explicitAgainst;
|
|
6784
|
+
const filtered = [];
|
|
6785
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
6786
|
+
const current = argv[i];
|
|
6787
|
+
const next = argv[i + 1];
|
|
6788
|
+
if (current === "--against" && next) {
|
|
6789
|
+
explicitAgainst = next;
|
|
6790
|
+
i += 1;
|
|
6791
|
+
continue;
|
|
6792
|
+
}
|
|
6793
|
+
filtered.push(current);
|
|
6794
|
+
}
|
|
6795
|
+
return {
|
|
6796
|
+
parsed: parseEvalArgs(filtered, cwd),
|
|
6797
|
+
explicitAgainst
|
|
6798
|
+
};
|
|
6799
|
+
}
|
|
6800
|
+
function toRunOptions(parsed) {
|
|
6801
|
+
return {
|
|
6802
|
+
projectRoot: parsed.projectRoot,
|
|
6803
|
+
configPath: parsed.configPath,
|
|
6804
|
+
datasetPath: parsed.datasetPath,
|
|
6805
|
+
outputRoot: parsed.outputRoot,
|
|
6806
|
+
againstPath: parsed.againstPath,
|
|
6807
|
+
budgetPath: parsed.budgetPath,
|
|
6808
|
+
ciMode: parsed.ciMode,
|
|
6809
|
+
reindex: parsed.reindex,
|
|
6810
|
+
searchOverrides: {
|
|
6811
|
+
...parsed.fusionStrategy !== void 0 ? { fusionStrategy: parsed.fusionStrategy } : {},
|
|
6812
|
+
...parsed.hybridWeight !== void 0 ? { hybridWeight: parsed.hybridWeight } : {},
|
|
6813
|
+
...parsed.rrfK !== void 0 ? { rrfK: parsed.rrfK } : {},
|
|
6814
|
+
...parsed.rerankTopN !== void 0 ? { rerankTopN: parsed.rerankTopN } : {}
|
|
6815
|
+
}
|
|
6816
|
+
};
|
|
6817
|
+
}
|
|
6818
|
+
async function handleEvalCommand(args, cwd) {
|
|
6819
|
+
const subcommand = args[0];
|
|
6820
|
+
if (!subcommand || subcommand === "--help" || subcommand === "-h") {
|
|
6821
|
+
printUsage();
|
|
6822
|
+
return 0;
|
|
6823
|
+
}
|
|
6824
|
+
if (subcommand === "run") {
|
|
6825
|
+
const { parsed, explicitAgainst } = parseEvalSubcommandOptions(args.slice(1), cwd);
|
|
6826
|
+
if (explicitAgainst) {
|
|
6827
|
+
parsed.againstPath = explicitAgainst;
|
|
6828
|
+
}
|
|
6829
|
+
const runOptions = toRunOptions(parsed);
|
|
6830
|
+
if (hasSweepOptions(parsed.sweep)) {
|
|
6831
|
+
const sweep = await runSweep(runOptions, parsed.sweep);
|
|
6832
|
+
console.log(`Eval sweep complete. Artifacts: ${sweep.outputDir}`);
|
|
6833
|
+
console.log(`Sweep runs: ${sweep.aggregate.runCount}`);
|
|
6834
|
+
if (parsed.ciMode && sweep.aggregate.gatePassed === false) {
|
|
6835
|
+
console.error(
|
|
6836
|
+
`[CI-GATE] Sweep failed: ${sweep.aggregate.failedGateRuns ?? 0} run(s) violated budget/baseline gates`
|
|
6837
|
+
);
|
|
6838
|
+
return 1;
|
|
6839
|
+
}
|
|
6840
|
+
return 0;
|
|
6841
|
+
}
|
|
6842
|
+
const result = await runEvaluation(runOptions);
|
|
6843
|
+
console.log(`Eval run complete. Artifacts: ${result.outputDir}`);
|
|
6844
|
+
console.log(
|
|
6845
|
+
`Hit@5=${(result.summary.metrics.hitAt5 * 100).toFixed(2)}% MRR@10=${result.summary.metrics.mrrAt10.toFixed(4)} p95=${result.summary.metrics.latencyMs.p95.toFixed(3)}ms`
|
|
6846
|
+
);
|
|
6847
|
+
if (result.gate && !result.gate.passed) {
|
|
6848
|
+
for (const violation of result.gate.violations) {
|
|
6849
|
+
console.error(`[CI-GATE] ${violation.metric}: ${violation.message}`);
|
|
6850
|
+
}
|
|
6851
|
+
return 1;
|
|
6852
|
+
}
|
|
6853
|
+
return 0;
|
|
6854
|
+
}
|
|
6855
|
+
if (subcommand === "compare") {
|
|
6856
|
+
const { parsed, explicitAgainst } = parseEvalSubcommandOptions(args.slice(1), cwd);
|
|
6857
|
+
if (!explicitAgainst) {
|
|
6858
|
+
throw new Error("eval compare requires --against <baseline summary.json>");
|
|
6859
|
+
}
|
|
6860
|
+
parsed.againstPath = explicitAgainst;
|
|
6861
|
+
const runOptions = toRunOptions(parsed);
|
|
6862
|
+
if (hasSweepOptions(parsed.sweep)) {
|
|
6863
|
+
const sweep = await runSweep(runOptions, parsed.sweep);
|
|
6864
|
+
console.log(`Eval compare sweep complete. Artifacts: ${sweep.outputDir}`);
|
|
6865
|
+
if (parsed.ciMode && sweep.aggregate.gatePassed === false) {
|
|
6866
|
+
console.error(
|
|
6867
|
+
`[CI-GATE] Sweep failed: ${sweep.aggregate.failedGateRuns ?? 0} run(s) violated budget/baseline gates`
|
|
6868
|
+
);
|
|
6869
|
+
return 1;
|
|
6870
|
+
}
|
|
6871
|
+
return 0;
|
|
6872
|
+
}
|
|
6873
|
+
const result = await runEvaluation(runOptions);
|
|
6874
|
+
console.log(`Eval compare complete. Artifacts: ${result.outputDir}`);
|
|
6875
|
+
return 0;
|
|
6876
|
+
}
|
|
6877
|
+
if (subcommand === "diff") {
|
|
6878
|
+
const { parsed, explicitAgainst } = parseEvalSubcommandOptions(args.slice(1), cwd);
|
|
6879
|
+
if (!explicitAgainst) {
|
|
6880
|
+
throw new Error("eval diff requires --against <baseline summary.json>");
|
|
6881
|
+
}
|
|
6882
|
+
if (!parsed.currentPath) {
|
|
6883
|
+
throw new Error("eval diff requires --current <current summary.json>");
|
|
6884
|
+
}
|
|
6885
|
+
parsed.againstPath = explicitAgainst;
|
|
6886
|
+
const currentPath = parsed.currentPath;
|
|
6887
|
+
if (!currentPath.endsWith(".json")) {
|
|
6888
|
+
throw new Error("eval diff --current must point to a summary JSON file");
|
|
6889
|
+
}
|
|
6890
|
+
if (!parsed.againstPath.endsWith(".json")) {
|
|
6891
|
+
throw new Error("eval diff --against must point to a summary JSON file");
|
|
6892
|
+
}
|
|
6893
|
+
const currentSummary = loadSummary(path8.resolve(parsed.projectRoot, currentPath));
|
|
6894
|
+
const baselineSummary = loadSummary(path8.resolve(parsed.projectRoot, parsed.againstPath));
|
|
6895
|
+
const comparison = compareSummaries(
|
|
6896
|
+
currentSummary,
|
|
6897
|
+
baselineSummary,
|
|
6898
|
+
path8.resolve(parsed.projectRoot, parsed.againstPath)
|
|
6899
|
+
);
|
|
6900
|
+
const outputDir = createRunDirectory(path8.resolve(parsed.projectRoot, parsed.outputRoot));
|
|
6901
|
+
const summaryMd = createSummaryMarkdown(currentSummary, comparison);
|
|
6902
|
+
writeJson(path8.join(outputDir, "compare.json"), comparison);
|
|
6903
|
+
writeText(path8.join(outputDir, "summary.md"), summaryMd);
|
|
6904
|
+
writeJson(path8.join(outputDir, "summary.json"), currentSummary);
|
|
6905
|
+
console.log(`Eval diff complete. Artifacts: ${outputDir}`);
|
|
6906
|
+
return 0;
|
|
6907
|
+
}
|
|
6908
|
+
throw new Error(`Unknown eval subcommand: ${subcommand}`);
|
|
6909
|
+
}
|
|
6910
|
+
|
|
5743
6911
|
// src/mcp-server.ts
|
|
6912
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
6913
|
+
import { z } from "zod";
|
|
6914
|
+
|
|
6915
|
+
// src/tools/utils.ts
|
|
5744
6916
|
var MAX_CONTENT_LINES = 30;
|
|
5745
6917
|
function truncateContent(content) {
|
|
5746
6918
|
const lines = content.split("\n");
|
|
@@ -5748,6 +6920,31 @@ function truncateContent(content) {
|
|
|
5748
6920
|
return lines.slice(0, MAX_CONTENT_LINES).join("\n") + `
|
|
5749
6921
|
// ... (${lines.length - MAX_CONTENT_LINES} more lines)`;
|
|
5750
6922
|
}
|
|
6923
|
+
function formatDefinitionLookup(results, query) {
|
|
6924
|
+
if (results.length === 0) {
|
|
6925
|
+
return `No definition found for "${query}". Try codebase_search for broader discovery, or verify the symbol name.`;
|
|
6926
|
+
}
|
|
6927
|
+
const formatted = results.map((r, idx) => {
|
|
6928
|
+
const header2 = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6929
|
+
return `${header2} (score: ${r.score.toFixed(2)})
|
|
6930
|
+
\`\`\`
|
|
6931
|
+
${truncateContent(r.content)}
|
|
6932
|
+
\`\`\``;
|
|
6933
|
+
});
|
|
6934
|
+
const header = results.length === 1 ? `Definition found for "${query}":` : `Found ${results.length} definition candidates for "${query}":`;
|
|
6935
|
+
return `${header}
|
|
6936
|
+
|
|
6937
|
+
${formatted.join("\n\n")}`;
|
|
6938
|
+
}
|
|
6939
|
+
|
|
6940
|
+
// src/mcp-server.ts
|
|
6941
|
+
var MAX_CONTENT_LINES2 = 30;
|
|
6942
|
+
function truncateContent2(content) {
|
|
6943
|
+
const lines = content.split("\n");
|
|
6944
|
+
if (lines.length <= MAX_CONTENT_LINES2) return content;
|
|
6945
|
+
return lines.slice(0, MAX_CONTENT_LINES2).join("\n") + `
|
|
6946
|
+
// ... (${lines.length - MAX_CONTENT_LINES2} more lines)`;
|
|
6947
|
+
}
|
|
5751
6948
|
function formatIndexStats(stats, verbose = false) {
|
|
5752
6949
|
const lines = [];
|
|
5753
6950
|
if (stats.indexedChunks === 0 && stats.removedChunks === 0) {
|
|
@@ -5861,7 +7058,7 @@ function createMcpServer(projectRoot, config) {
|
|
|
5861
7058
|
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
5862
7059
|
return `${header} (score: ${r.score.toFixed(2)})
|
|
5863
7060
|
\`\`\`
|
|
5864
|
-
${
|
|
7061
|
+
${truncateContent2(r.content)}
|
|
5865
7062
|
\`\`\``;
|
|
5866
7063
|
});
|
|
5867
7064
|
return { content: [{ type: "text", text: `Found ${results.length} results for "${args.query}":
|
|
@@ -6039,7 +7236,7 @@ Use Read tool to examine specific files.` }] };
|
|
|
6039
7236
|
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6040
7237
|
return `${header} (similarity: ${(r.score * 100).toFixed(1)}%)
|
|
6041
7238
|
\`\`\`
|
|
6042
|
-
${
|
|
7239
|
+
${truncateContent2(r.content)}
|
|
6043
7240
|
\`\`\``;
|
|
6044
7241
|
});
|
|
6045
7242
|
return { content: [{ type: "text", text: `Found ${results.length} similar code blocks:
|
|
@@ -6047,6 +7244,25 @@ ${truncateContent(r.content)}
|
|
|
6047
7244
|
${formatted.join("\n\n")}` }] };
|
|
6048
7245
|
}
|
|
6049
7246
|
);
|
|
7247
|
+
server.tool(
|
|
7248
|
+
"implementation_lookup",
|
|
7249
|
+
"Jump to symbol definition. Find WHERE something is defined. Returns the authoritative source location(s). Prefers real implementation files over tests, docs, examples, and fixtures.",
|
|
7250
|
+
{
|
|
7251
|
+
query: z.string().describe("Symbol name or natural language description (e.g., 'validateToken', 'where is the payment handler defined')"),
|
|
7252
|
+
limit: z.number().optional().default(5).describe("Maximum number of results"),
|
|
7253
|
+
fileType: z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py')"),
|
|
7254
|
+
directory: z.string().optional().describe("Filter by directory path (e.g., 'src/utils')")
|
|
7255
|
+
},
|
|
7256
|
+
async (args) => {
|
|
7257
|
+
await ensureInitialized();
|
|
7258
|
+
const results = await indexer.search(args.query, args.limit ?? 5, {
|
|
7259
|
+
fileType: args.fileType,
|
|
7260
|
+
directory: args.directory,
|
|
7261
|
+
definitionIntent: true
|
|
7262
|
+
});
|
|
7263
|
+
return { content: [{ type: "text", text: formatDefinitionLookup(results, args.query) }] };
|
|
7264
|
+
}
|
|
7265
|
+
);
|
|
6050
7266
|
server.tool(
|
|
6051
7267
|
"call_graph",
|
|
6052
7268
|
"Query the call graph to find callers or callees of a function/method. Use to understand code flow and dependencies.",
|
|
@@ -6153,14 +7369,30 @@ Use a hybrid approach:
|
|
|
6153
7369
|
}]
|
|
6154
7370
|
})
|
|
6155
7371
|
);
|
|
7372
|
+
server.prompt(
|
|
7373
|
+
"definition",
|
|
7374
|
+
"Find where a symbol is defined in the codebase",
|
|
7375
|
+
{ query: z.string().describe("Symbol name or description to find the definition of") },
|
|
7376
|
+
(args) => ({
|
|
7377
|
+
messages: [{
|
|
7378
|
+
role: "user",
|
|
7379
|
+
content: {
|
|
7380
|
+
type: "text",
|
|
7381
|
+
text: `Find the definition of: "${args.query}"
|
|
7382
|
+
|
|
7383
|
+
Use the implementation_lookup tool to find where this symbol is defined. This prioritizes real implementation files over tests, docs, and examples. If no definition is found, fall back to codebase_search for broader discovery.`
|
|
7384
|
+
}
|
|
7385
|
+
}]
|
|
7386
|
+
})
|
|
7387
|
+
);
|
|
6156
7388
|
return server;
|
|
6157
7389
|
}
|
|
6158
7390
|
|
|
6159
7391
|
// src/cli.ts
|
|
6160
7392
|
function loadJsonFile(filePath) {
|
|
6161
7393
|
try {
|
|
6162
|
-
if (
|
|
6163
|
-
const content =
|
|
7394
|
+
if (existsSync6(filePath)) {
|
|
7395
|
+
const content = readFileSync8(filePath, "utf-8");
|
|
6164
7396
|
return JSON.parse(content);
|
|
6165
7397
|
}
|
|
6166
7398
|
} catch {
|
|
@@ -6172,9 +7404,9 @@ function loadPluginConfig(projectRoot, configPath) {
|
|
|
6172
7404
|
const config = loadJsonFile(configPath);
|
|
6173
7405
|
if (config) return config;
|
|
6174
7406
|
}
|
|
6175
|
-
const projectConfig = loadJsonFile(
|
|
7407
|
+
const projectConfig = loadJsonFile(path9.join(projectRoot, ".opencode", "codebase-index.json"));
|
|
6176
7408
|
if (projectConfig) return projectConfig;
|
|
6177
|
-
const globalConfig = loadJsonFile(
|
|
7409
|
+
const globalConfig = loadJsonFile(path9.join(os4.homedir(), ".config", "opencode", "codebase-index.json"));
|
|
6178
7410
|
if (globalConfig) return globalConfig;
|
|
6179
7411
|
return {};
|
|
6180
7412
|
}
|
|
@@ -6183,14 +7415,18 @@ function parseArgs(argv) {
|
|
|
6183
7415
|
let config;
|
|
6184
7416
|
for (let i = 2; i < argv.length; i++) {
|
|
6185
7417
|
if (argv[i] === "--project" && argv[i + 1]) {
|
|
6186
|
-
project =
|
|
7418
|
+
project = path9.resolve(argv[++i]);
|
|
6187
7419
|
} else if (argv[i] === "--config" && argv[i + 1]) {
|
|
6188
|
-
config =
|
|
7420
|
+
config = path9.resolve(argv[++i]);
|
|
6189
7421
|
}
|
|
6190
7422
|
}
|
|
6191
7423
|
return { project, config };
|
|
6192
7424
|
}
|
|
6193
7425
|
async function main() {
|
|
7426
|
+
if (process.argv[2] === "eval") {
|
|
7427
|
+
const exitCode = await handleEvalCommand(process.argv.slice(3), process.cwd());
|
|
7428
|
+
process.exit(exitCode);
|
|
7429
|
+
}
|
|
6194
7430
|
const args = parseArgs(process.argv);
|
|
6195
7431
|
const rawConfig = loadPluginConfig(args.project, args.config);
|
|
6196
7432
|
const config = parseConfig(rawConfig);
|