kontext-engine 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -1
- package/dist/cli/index.js +372 -84
- package/dist/cli/index.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +319 -130
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -779,6 +779,25 @@ function prepareChunkText(filePath, parent, text) {
|
|
|
779
779
|
parts.push(text);
|
|
780
780
|
return parts.join("\n");
|
|
781
781
|
}
|
|
782
|
+
var MAX_RETRIES = 3;
|
|
783
|
+
var BASE_DELAY_MS = 500;
|
|
784
|
+
async function fetchWithRetry(url, init) {
|
|
785
|
+
let lastError = null;
|
|
786
|
+
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
787
|
+
const response = await fetch(url, init);
|
|
788
|
+
if (response.ok) return response;
|
|
789
|
+
if (response.status === 429 && attempt < MAX_RETRIES) {
|
|
790
|
+
const delay = BASE_DELAY_MS * Math.pow(2, attempt);
|
|
791
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
792
|
+
lastError = new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
793
|
+
continue;
|
|
794
|
+
}
|
|
795
|
+
throw new Error(
|
|
796
|
+
`Embedding API error: HTTP ${response.status} ${response.statusText}`
|
|
797
|
+
);
|
|
798
|
+
}
|
|
799
|
+
throw lastError ?? new Error("Embedding API request failed after retries");
|
|
800
|
+
}
|
|
782
801
|
var LOCAL_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
|
|
783
802
|
var LOCAL_DIMENSIONS = 384;
|
|
784
803
|
var LOCAL_BATCH_SIZE = 32;
|
|
@@ -836,6 +855,87 @@ async function createLocalEmbedder() {
|
|
|
836
855
|
}
|
|
837
856
|
};
|
|
838
857
|
}
|
|
858
|
+
var VOYAGE_API_URL = "https://api.voyageai.com/v1/embeddings";
|
|
859
|
+
var VOYAGE_MODEL = "voyage-code-3";
|
|
860
|
+
var VOYAGE_DEFAULT_DIMENSIONS = 1024;
|
|
861
|
+
var VOYAGE_BATCH_SIZE = 128;
|
|
862
|
+
function createVoyageEmbedder(apiKey, dimensions = VOYAGE_DEFAULT_DIMENSIONS) {
|
|
863
|
+
return {
|
|
864
|
+
name: VOYAGE_MODEL,
|
|
865
|
+
dimensions,
|
|
866
|
+
async embed(texts, onProgress) {
|
|
867
|
+
const results = [];
|
|
868
|
+
for (let i = 0; i < texts.length; i += VOYAGE_BATCH_SIZE) {
|
|
869
|
+
const batch = texts.slice(i, i + VOYAGE_BATCH_SIZE);
|
|
870
|
+
const vectors = await callVoyageAPI(apiKey, batch, "document", dimensions);
|
|
871
|
+
results.push(...vectors);
|
|
872
|
+
onProgress?.(Math.min(i + batch.length, texts.length), texts.length);
|
|
873
|
+
}
|
|
874
|
+
return results;
|
|
875
|
+
},
|
|
876
|
+
async embedSingle(text) {
|
|
877
|
+
const vectors = await callVoyageAPI(apiKey, [text], "query", dimensions);
|
|
878
|
+
return vectors[0];
|
|
879
|
+
}
|
|
880
|
+
};
|
|
881
|
+
}
|
|
882
|
+
async function callVoyageAPI(apiKey, texts, inputType, dimensions) {
|
|
883
|
+
const response = await fetchWithRetry(VOYAGE_API_URL, {
|
|
884
|
+
method: "POST",
|
|
885
|
+
headers: {
|
|
886
|
+
"Content-Type": "application/json",
|
|
887
|
+
Authorization: `Bearer ${apiKey}`
|
|
888
|
+
},
|
|
889
|
+
body: JSON.stringify({
|
|
890
|
+
model: VOYAGE_MODEL,
|
|
891
|
+
input: texts,
|
|
892
|
+
input_type: inputType,
|
|
893
|
+
output_dimension: dimensions
|
|
894
|
+
})
|
|
895
|
+
});
|
|
896
|
+
const json = await response.json();
|
|
897
|
+
return json.data.map((d) => normalizeVector(new Float32Array(d.embedding)));
|
|
898
|
+
}
|
|
899
|
+
var OPENAI_API_URL = "https://api.openai.com/v1/embeddings";
|
|
900
|
+
var OPENAI_MODEL = "text-embedding-3-large";
|
|
901
|
+
var OPENAI_DEFAULT_DIMENSIONS = 1024;
|
|
902
|
+
var OPENAI_BATCH_SIZE = 128;
|
|
903
|
+
function createOpenAIEmbedder(apiKey, dimensions = OPENAI_DEFAULT_DIMENSIONS) {
|
|
904
|
+
return {
|
|
905
|
+
name: OPENAI_MODEL,
|
|
906
|
+
dimensions,
|
|
907
|
+
async embed(texts, onProgress) {
|
|
908
|
+
const results = [];
|
|
909
|
+
for (let i = 0; i < texts.length; i += OPENAI_BATCH_SIZE) {
|
|
910
|
+
const batch = texts.slice(i, i + OPENAI_BATCH_SIZE);
|
|
911
|
+
const vectors = await callOpenAIAPI(apiKey, batch, dimensions);
|
|
912
|
+
results.push(...vectors);
|
|
913
|
+
onProgress?.(Math.min(i + batch.length, texts.length), texts.length);
|
|
914
|
+
}
|
|
915
|
+
return results;
|
|
916
|
+
},
|
|
917
|
+
async embedSingle(text) {
|
|
918
|
+
const vectors = await callOpenAIAPI(apiKey, [text], dimensions);
|
|
919
|
+
return vectors[0];
|
|
920
|
+
}
|
|
921
|
+
};
|
|
922
|
+
}
|
|
923
|
+
async function callOpenAIAPI(apiKey, texts, dimensions) {
|
|
924
|
+
const response = await fetchWithRetry(OPENAI_API_URL, {
|
|
925
|
+
method: "POST",
|
|
926
|
+
headers: {
|
|
927
|
+
"Content-Type": "application/json",
|
|
928
|
+
Authorization: `Bearer ${apiKey}`
|
|
929
|
+
},
|
|
930
|
+
body: JSON.stringify({
|
|
931
|
+
model: OPENAI_MODEL,
|
|
932
|
+
input: texts,
|
|
933
|
+
dimensions
|
|
934
|
+
})
|
|
935
|
+
});
|
|
936
|
+
const json = await response.json();
|
|
937
|
+
return json.data.map((d) => normalizeVector(new Float32Array(d.embedding)));
|
|
938
|
+
}
|
|
839
939
|
|
|
840
940
|
// src/utils/errors.ts
|
|
841
941
|
var ErrorCode = {
|
|
@@ -877,6 +977,12 @@ var ConfigError = class extends KontextError {
|
|
|
877
977
|
this.name = "ConfigError";
|
|
878
978
|
}
|
|
879
979
|
};
|
|
980
|
+
var DatabaseError = class extends KontextError {
|
|
981
|
+
constructor(message, code, cause) {
|
|
982
|
+
super(message, code, cause);
|
|
983
|
+
this.name = "DatabaseError";
|
|
984
|
+
}
|
|
985
|
+
};
|
|
880
986
|
|
|
881
987
|
// src/utils/error-boundary.ts
|
|
882
988
|
function handleCommandError(err, logger, verbose) {
|
|
@@ -1051,7 +1157,8 @@ function searchVectors(db, query, limit) {
|
|
|
1051
1157
|
|
|
1052
1158
|
// src/storage/db.ts
|
|
1053
1159
|
var DEFAULT_DIMENSIONS = 384;
|
|
1054
|
-
|
|
1160
|
+
var VECTOR_DIMENSIONS_META_KEY = "vector_dimensions";
|
|
1161
|
+
function createDatabase(dbPath, dimensions) {
|
|
1055
1162
|
const dir = path3.dirname(dbPath);
|
|
1056
1163
|
if (!fs4.existsSync(dir)) {
|
|
1057
1164
|
fs4.mkdirSync(dir, { recursive: true });
|
|
@@ -1060,7 +1167,8 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
|
|
|
1060
1167
|
db.pragma("journal_mode = WAL");
|
|
1061
1168
|
db.pragma("foreign_keys = ON");
|
|
1062
1169
|
sqliteVec.load(db);
|
|
1063
|
-
initializeSchema(db, dimensions);
|
|
1170
|
+
initializeSchema(db, dimensions ?? DEFAULT_DIMENSIONS);
|
|
1171
|
+
ensureVectorDimensions(db, dimensions);
|
|
1064
1172
|
const stmtUpsertFile = db.prepare(`
|
|
1065
1173
|
INSERT INTO files (path, language, hash, last_indexed, size)
|
|
1066
1174
|
VALUES (@path, @language, @hash, @lastIndexed, @size)
|
|
@@ -1069,6 +1177,7 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
|
|
|
1069
1177
|
hash = excluded.hash,
|
|
1070
1178
|
last_indexed = excluded.last_indexed,
|
|
1071
1179
|
size = excluded.size
|
|
1180
|
+
RETURNING id
|
|
1072
1181
|
`);
|
|
1073
1182
|
const stmtGetFile = db.prepare(
|
|
1074
1183
|
"SELECT id, path, language, hash, last_indexed as lastIndexed, size FROM files WHERE path = ?"
|
|
@@ -1112,18 +1221,20 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
|
|
|
1112
1221
|
);
|
|
1113
1222
|
return {
|
|
1114
1223
|
upsertFile(file) {
|
|
1115
|
-
const
|
|
1224
|
+
const row = stmtUpsertFile.get({
|
|
1116
1225
|
path: file.path,
|
|
1117
1226
|
language: file.language,
|
|
1118
1227
|
hash: file.hash,
|
|
1119
1228
|
lastIndexed: Date.now(),
|
|
1120
1229
|
size: file.size
|
|
1121
1230
|
});
|
|
1122
|
-
if (
|
|
1123
|
-
|
|
1231
|
+
if (!row?.id) {
|
|
1232
|
+
throw new DatabaseError(
|
|
1233
|
+
`Failed to upsert file: ${file.path}`,
|
|
1234
|
+
ErrorCode.DB_WRITE_FAILED
|
|
1235
|
+
);
|
|
1124
1236
|
}
|
|
1125
|
-
|
|
1126
|
-
return existing?.id ?? 0;
|
|
1237
|
+
return row.id;
|
|
1127
1238
|
},
|
|
1128
1239
|
getFile(filePath) {
|
|
1129
1240
|
const row = stmtGetFile.get(filePath);
|
|
@@ -1166,15 +1277,17 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
|
|
|
1166
1277
|
return row.lastIndexed;
|
|
1167
1278
|
},
|
|
1168
1279
|
deleteFile(filePath) {
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1280
|
+
db.transaction(() => {
|
|
1281
|
+
const file = stmtGetFile.get(filePath);
|
|
1282
|
+
if (file) {
|
|
1283
|
+
const chunkRows = stmtGetChunkIdsByFile.all(file.id);
|
|
1284
|
+
const chunkIds = chunkRows.map((r) => r.id);
|
|
1285
|
+
if (chunkIds.length > 0) {
|
|
1286
|
+
deleteVectorsByChunkIds(db, chunkIds);
|
|
1287
|
+
}
|
|
1175
1288
|
}
|
|
1176
|
-
|
|
1177
|
-
|
|
1289
|
+
stmtDeleteFile.run(filePath);
|
|
1290
|
+
})();
|
|
1178
1291
|
},
|
|
1179
1292
|
insertChunks(fileId, chunks) {
|
|
1180
1293
|
const ids = [];
|
|
@@ -1269,12 +1382,14 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
|
|
|
1269
1382
|
}));
|
|
1270
1383
|
},
|
|
1271
1384
|
deleteChunksByFile(fileId) {
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1385
|
+
db.transaction(() => {
|
|
1386
|
+
const chunkRows = stmtGetChunkIdsByFile.all(fileId);
|
|
1387
|
+
const chunkIds = chunkRows.map((r) => r.id);
|
|
1388
|
+
if (chunkIds.length > 0) {
|
|
1389
|
+
deleteVectorsByChunkIds(db, chunkIds);
|
|
1390
|
+
}
|
|
1391
|
+
stmtDeleteChunksByFile.run(fileId);
|
|
1392
|
+
})();
|
|
1278
1393
|
},
|
|
1279
1394
|
insertDependency(sourceChunkId, targetChunkId, type) {
|
|
1280
1395
|
stmtInsertDep.run(sourceChunkId, targetChunkId, type);
|
|
@@ -1336,6 +1451,59 @@ function getMetaVersion(db) {
|
|
|
1336
1451
|
return 0;
|
|
1337
1452
|
}
|
|
1338
1453
|
}
|
|
1454
|
+
function ensureVectorDimensions(db, expectedDimensions) {
|
|
1455
|
+
const actual = getExistingVectorDimensions(db);
|
|
1456
|
+
const stored = db.prepare("SELECT value FROM meta WHERE key = ?").get(VECTOR_DIMENSIONS_META_KEY);
|
|
1457
|
+
const storedValue = stored?.value;
|
|
1458
|
+
const storedDimensions = storedValue ? Number.parseInt(storedValue, 10) : void 0;
|
|
1459
|
+
if (storedDimensions !== void 0 && (!Number.isInteger(storedDimensions) || storedDimensions <= 0)) {
|
|
1460
|
+
throw new DatabaseError(
|
|
1461
|
+
`Invalid stored vector dimensions metadata: ${storedValue ?? "unknown"}`,
|
|
1462
|
+
ErrorCode.DB_CORRUPTED
|
|
1463
|
+
);
|
|
1464
|
+
}
|
|
1465
|
+
if (actual !== null && storedDimensions !== void 0 && storedDimensions !== actual) {
|
|
1466
|
+
throw new DatabaseError(
|
|
1467
|
+
`Vector dimensions metadata mismatch: meta=${storedDimensions}, table=${actual}.`,
|
|
1468
|
+
ErrorCode.DB_CORRUPTED
|
|
1469
|
+
);
|
|
1470
|
+
}
|
|
1471
|
+
if (expectedDimensions === void 0) {
|
|
1472
|
+
if (storedDimensions !== void 0) return;
|
|
1473
|
+
const dimensions = actual ?? DEFAULT_DIMENSIONS;
|
|
1474
|
+
db.prepare(
|
|
1475
|
+
"INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)"
|
|
1476
|
+
).run(VECTOR_DIMENSIONS_META_KEY, String(dimensions));
|
|
1477
|
+
return;
|
|
1478
|
+
}
|
|
1479
|
+
if (!stored) {
|
|
1480
|
+
const dimensions = actual ?? expectedDimensions;
|
|
1481
|
+
db.prepare(
|
|
1482
|
+
"INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)"
|
|
1483
|
+
).run(VECTOR_DIMENSIONS_META_KEY, String(dimensions));
|
|
1484
|
+
if (actual !== null && actual !== expectedDimensions) {
|
|
1485
|
+
throw new DatabaseError(
|
|
1486
|
+
`Vector dimension mismatch: index uses ${actual} dims, but config requests ${expectedDimensions} dims. Rebuild the index.`,
|
|
1487
|
+
ErrorCode.CONFIG_INVALID
|
|
1488
|
+
);
|
|
1489
|
+
}
|
|
1490
|
+
return;
|
|
1491
|
+
}
|
|
1492
|
+
if (storedDimensions !== expectedDimensions) {
|
|
1493
|
+
throw new DatabaseError(
|
|
1494
|
+
`Vector dimension mismatch: index uses ${storedDimensions} dims, but config requests ${expectedDimensions} dims. Rebuild the index.`,
|
|
1495
|
+
ErrorCode.CONFIG_INVALID
|
|
1496
|
+
);
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
function getExistingVectorDimensions(db) {
|
|
1500
|
+
const row = db.prepare("SELECT sql FROM sqlite_master WHERE name = 'chunk_vectors'").get();
|
|
1501
|
+
const sql = row?.sql;
|
|
1502
|
+
if (!sql) return null;
|
|
1503
|
+
const match = sql.match(/embedding\s+float\[(\d+)\]/i);
|
|
1504
|
+
if (!match) return null;
|
|
1505
|
+
return Number.parseInt(match[1], 10);
|
|
1506
|
+
}
|
|
1339
1507
|
|
|
1340
1508
|
// src/cli/commands/config.ts
|
|
1341
1509
|
import fs5 from "fs";
|
|
@@ -1451,6 +1619,20 @@ function setNestedValue(obj, key, value) {
|
|
|
1451
1619
|
}
|
|
1452
1620
|
current[parts[parts.length - 1]] = value;
|
|
1453
1621
|
}
|
|
1622
|
+
function hasNestedKey(obj, key) {
|
|
1623
|
+
const parts = key.split(".");
|
|
1624
|
+
let current = obj;
|
|
1625
|
+
for (const part of parts) {
|
|
1626
|
+
if (current === null || current === void 0 || typeof current !== "object") {
|
|
1627
|
+
return false;
|
|
1628
|
+
}
|
|
1629
|
+
if (!(part in current)) {
|
|
1630
|
+
return false;
|
|
1631
|
+
}
|
|
1632
|
+
current = current[part];
|
|
1633
|
+
}
|
|
1634
|
+
return true;
|
|
1635
|
+
}
|
|
1454
1636
|
function parseValue(rawValue) {
|
|
1455
1637
|
if (rawValue === "null") return null;
|
|
1456
1638
|
if (rawValue === "true") return true;
|
|
@@ -1489,9 +1671,26 @@ function runConfigSet(projectPath, key, rawValue) {
|
|
|
1489
1671
|
setNestedValue(config, key, value);
|
|
1490
1672
|
writeConfig(ctxDir, config);
|
|
1491
1673
|
}
|
|
1492
|
-
function runConfigReset(projectPath) {
|
|
1674
|
+
function runConfigReset(projectPath, key) {
|
|
1493
1675
|
const ctxDir = resolveCtxDir(projectPath);
|
|
1494
|
-
|
|
1676
|
+
if (!key) {
|
|
1677
|
+
writeConfig(ctxDir, structuredClone(DEFAULT_CONFIG));
|
|
1678
|
+
return;
|
|
1679
|
+
}
|
|
1680
|
+
if (!hasNestedKey(DEFAULT_CONFIG, key)) {
|
|
1681
|
+
throw new ConfigError(`Invalid config key: ${key}`, ErrorCode.CONFIG_INVALID);
|
|
1682
|
+
}
|
|
1683
|
+
const config = readConfig(ctxDir);
|
|
1684
|
+
const defaultValue = getNestedValue(
|
|
1685
|
+
DEFAULT_CONFIG,
|
|
1686
|
+
key
|
|
1687
|
+
);
|
|
1688
|
+
setNestedValue(
|
|
1689
|
+
config,
|
|
1690
|
+
key,
|
|
1691
|
+
structuredClone(defaultValue)
|
|
1692
|
+
);
|
|
1693
|
+
writeConfig(ctxDir, config);
|
|
1495
1694
|
}
|
|
1496
1695
|
function registerConfigCommand(program2) {
|
|
1497
1696
|
const cmd = program2.command("config").description("Show or modify configuration");
|
|
@@ -1526,16 +1725,69 @@ function registerConfigCommand(program2) {
|
|
|
1526
1725
|
configErrorHandler(err);
|
|
1527
1726
|
}
|
|
1528
1727
|
});
|
|
1529
|
-
cmd.command("reset").description("Reset configuration to defaults").action(() => {
|
|
1728
|
+
cmd.command("reset [key]").description("Reset configuration to defaults or reset a specific key").action((key) => {
|
|
1530
1729
|
try {
|
|
1531
|
-
runConfigReset(process.cwd());
|
|
1532
|
-
|
|
1730
|
+
runConfigReset(process.cwd(), key);
|
|
1731
|
+
if (key) {
|
|
1732
|
+
console.log(`Reset ${key} to default.`);
|
|
1733
|
+
} else {
|
|
1734
|
+
console.log("Configuration reset to defaults.");
|
|
1735
|
+
}
|
|
1533
1736
|
} catch (err) {
|
|
1534
1737
|
configErrorHandler(err);
|
|
1535
1738
|
}
|
|
1536
1739
|
});
|
|
1537
1740
|
}
|
|
1538
1741
|
|
|
1742
|
+
// src/cli/embedder.ts
|
|
1743
|
+
function getProjectEmbedderConfig(projectPath) {
|
|
1744
|
+
const { config } = runConfigShow(projectPath);
|
|
1745
|
+
return config.embedder;
|
|
1746
|
+
}
|
|
1747
|
+
async function createProjectEmbedder(projectPath) {
|
|
1748
|
+
const config = getProjectEmbedderConfig(projectPath);
|
|
1749
|
+
validateProjectEmbedderConfig(config);
|
|
1750
|
+
switch (config.provider) {
|
|
1751
|
+
case "local":
|
|
1752
|
+
return await createLocalEmbedder();
|
|
1753
|
+
case "voyage": {
|
|
1754
|
+
const apiKey = requireApiKey("CTX_VOYAGE_KEY", "voyage");
|
|
1755
|
+
return createVoyageEmbedder(apiKey, config.dimensions);
|
|
1756
|
+
}
|
|
1757
|
+
case "openai": {
|
|
1758
|
+
const apiKey = requireApiKey("CTX_OPENAI_KEY", "openai");
|
|
1759
|
+
return createOpenAIEmbedder(apiKey, config.dimensions);
|
|
1760
|
+
}
|
|
1761
|
+
default:
|
|
1762
|
+
throw new ConfigError(
|
|
1763
|
+
`Unsupported embedder provider "${config.provider}". Use local, voyage, or openai.`,
|
|
1764
|
+
ErrorCode.CONFIG_INVALID
|
|
1765
|
+
);
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
function requireApiKey(envVar, provider) {
|
|
1769
|
+
const value = process.env[envVar];
|
|
1770
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
1771
|
+
throw new ConfigError(
|
|
1772
|
+
`Embedder provider "${provider}" requires ${envVar}. Export ${envVar} before running this command.`,
|
|
1773
|
+
ErrorCode.CONFIG_INVALID
|
|
1774
|
+
);
|
|
1775
|
+
}
|
|
1776
|
+
function validateProjectEmbedderConfig(config) {
|
|
1777
|
+
if (!Number.isInteger(config.dimensions) || config.dimensions <= 0) {
|
|
1778
|
+
throw new ConfigError(
|
|
1779
|
+
`Invalid embedder.dimensions (${String(config.dimensions)}). Must be a positive integer.`,
|
|
1780
|
+
ErrorCode.CONFIG_INVALID
|
|
1781
|
+
);
|
|
1782
|
+
}
|
|
1783
|
+
if (config.provider === "local" && config.dimensions !== 384) {
|
|
1784
|
+
throw new ConfigError(
|
|
1785
|
+
'Local embedder requires "embedder.dimensions" = 384. Update config or switch provider.',
|
|
1786
|
+
ErrorCode.CONFIG_INVALID
|
|
1787
|
+
);
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1539
1791
|
// src/cli/commands/init.ts
|
|
1540
1792
|
var CTX_DIR2 = ".ctx";
|
|
1541
1793
|
var DB_FILENAME = "index.db";
|
|
@@ -1584,8 +1836,9 @@ async function runInit(projectPath, options = {}) {
|
|
|
1584
1836
|
if (!fs6.existsSync(ctxDir)) fs6.mkdirSync(ctxDir, { recursive: true });
|
|
1585
1837
|
ensureGitignore(absoluteRoot);
|
|
1586
1838
|
ensureConfig(ctxDir);
|
|
1839
|
+
const embedderConfig = getProjectEmbedderConfig(absoluteRoot);
|
|
1587
1840
|
const dbPath = path5.join(ctxDir, DB_FILENAME);
|
|
1588
|
-
const db = createDatabase(dbPath);
|
|
1841
|
+
const db = createDatabase(dbPath, embedderConfig.dimensions);
|
|
1589
1842
|
try {
|
|
1590
1843
|
const discovered = await discoverFiles({
|
|
1591
1844
|
root: absoluteRoot,
|
|
@@ -1673,7 +1926,7 @@ async function runInit(projectPath, options = {}) {
|
|
|
1673
1926
|
log(` ${allChunksWithMeta.length} chunks created`);
|
|
1674
1927
|
let vectorsCreated = 0;
|
|
1675
1928
|
if (!options.skipEmbedding && allChunksWithMeta.length > 0) {
|
|
1676
|
-
const embedder = await createEmbedder();
|
|
1929
|
+
const embedder = await createEmbedder(absoluteRoot);
|
|
1677
1930
|
const texts = allChunksWithMeta.map(
|
|
1678
1931
|
(cm) => prepareChunkText(cm.fileRelPath, cm.chunk.parent, cm.chunk.text)
|
|
1679
1932
|
);
|
|
@@ -1711,8 +1964,8 @@ async function runInit(projectPath, options = {}) {
|
|
|
1711
1964
|
db.close();
|
|
1712
1965
|
}
|
|
1713
1966
|
}
|
|
1714
|
-
async function createEmbedder() {
|
|
1715
|
-
return
|
|
1967
|
+
async function createEmbedder(projectPath) {
|
|
1968
|
+
return createProjectEmbedder(projectPath);
|
|
1716
1969
|
}
|
|
1717
1970
|
function registerInitCommand(program2) {
|
|
1718
1971
|
program2.command("init [path]").description("Index current directory or specified path").action(async (inputPath) => {
|
|
@@ -2340,9 +2593,13 @@ async function runQuery(projectPath, query, options) {
|
|
|
2340
2593
|
);
|
|
2341
2594
|
}
|
|
2342
2595
|
const start = performance.now();
|
|
2343
|
-
const
|
|
2596
|
+
const embedderConfig = getProjectEmbedderConfig(absoluteRoot);
|
|
2597
|
+
const db = createDatabase(dbPath, embedderConfig.dimensions);
|
|
2344
2598
|
try {
|
|
2345
|
-
const strategyResults = await runStrategies(db, query, {
|
|
2599
|
+
const strategyResults = await runStrategies(db, absoluteRoot, query, {
|
|
2600
|
+
...options,
|
|
2601
|
+
limit
|
|
2602
|
+
});
|
|
2346
2603
|
const pathBoostTerms = extractPathBoostTerms(query);
|
|
2347
2604
|
const fused = fusionMergeWithPathBoost(strategyResults, limit, pathBoostTerms);
|
|
2348
2605
|
const outputResults = fused.map(toOutputResult);
|
|
@@ -2362,7 +2619,7 @@ async function runQuery(projectPath, query, options) {
|
|
|
2362
2619
|
db.close();
|
|
2363
2620
|
}
|
|
2364
2621
|
}
|
|
2365
|
-
async function runStrategies(db, query, options) {
|
|
2622
|
+
async function runStrategies(db, projectPath, query, options) {
|
|
2366
2623
|
const results = [];
|
|
2367
2624
|
const filters = options.language ? { language: options.language } : void 0;
|
|
2368
2625
|
const limit = options.limit * 3;
|
|
@@ -2371,6 +2628,7 @@ async function runStrategies(db, query, options) {
|
|
|
2371
2628
|
const weight = effectiveWeights[strategy];
|
|
2372
2629
|
const searchResults = await executeStrategy(
|
|
2373
2630
|
db,
|
|
2631
|
+
projectPath,
|
|
2374
2632
|
strategy,
|
|
2375
2633
|
query,
|
|
2376
2634
|
limit,
|
|
@@ -2382,10 +2640,10 @@ async function runStrategies(db, query, options) {
|
|
|
2382
2640
|
}
|
|
2383
2641
|
return results;
|
|
2384
2642
|
}
|
|
2385
|
-
async function executeStrategy(db, strategy, query, limit, filters) {
|
|
2643
|
+
async function executeStrategy(db, projectPath, strategy, query, limit, filters) {
|
|
2386
2644
|
switch (strategy) {
|
|
2387
2645
|
case "vector": {
|
|
2388
|
-
const embedder = await loadEmbedder();
|
|
2646
|
+
const embedder = await loadEmbedder(projectPath);
|
|
2389
2647
|
return vectorSearch(db, embedder, query, limit, filters);
|
|
2390
2648
|
}
|
|
2391
2649
|
case "fts":
|
|
@@ -2418,9 +2676,16 @@ async function executeStrategy(db, strategy, query, limit, filters) {
|
|
|
2418
2676
|
}
|
|
2419
2677
|
}
|
|
2420
2678
|
var embedderInstance = null;
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2679
|
+
var embedderKey = null;
|
|
2680
|
+
function getCacheKey(projectPath) {
|
|
2681
|
+
const config = getProjectEmbedderConfig(projectPath);
|
|
2682
|
+
return `${projectPath}:${config.provider}:${config.model}:${config.dimensions}`;
|
|
2683
|
+
}
|
|
2684
|
+
async function loadEmbedder(projectPath) {
|
|
2685
|
+
const cacheKey = getCacheKey(projectPath);
|
|
2686
|
+
if (embedderInstance && embedderKey === cacheKey) return embedderInstance;
|
|
2687
|
+
embedderInstance = await createProjectEmbedder(projectPath);
|
|
2688
|
+
embedderKey = cacheKey;
|
|
2424
2689
|
return embedderInstance;
|
|
2425
2690
|
}
|
|
2426
2691
|
function registerQueryCommand(program2) {
|
|
@@ -2838,7 +3103,9 @@ var COMMON_STEMS = {
|
|
|
2838
3103
|
transformer: "transform",
|
|
2839
3104
|
transformation: "transform",
|
|
2840
3105
|
connection: "connect",
|
|
3106
|
+
connecting: "connect",
|
|
2841
3107
|
connector: "connect",
|
|
3108
|
+
migrating: "migrate",
|
|
2842
3109
|
migration: "migrate",
|
|
2843
3110
|
scheduling: "schedule",
|
|
2844
3111
|
scheduler: "schedule",
|
|
@@ -2847,7 +3114,8 @@ var COMMON_STEMS = {
|
|
|
2847
3114
|
routing: "route",
|
|
2848
3115
|
router: "route",
|
|
2849
3116
|
indexing: "index",
|
|
2850
|
-
indexer: "index"
|
|
3117
|
+
indexer: "index",
|
|
3118
|
+
subscribing: "subscribe"
|
|
2851
3119
|
};
|
|
2852
3120
|
var STEM_SUFFIXES = [
|
|
2853
3121
|
"tion",
|
|
@@ -3127,13 +3395,13 @@ function formatTextOutput2(output) {
|
|
|
3127
3395
|
);
|
|
3128
3396
|
return lines.join("\n");
|
|
3129
3397
|
}
|
|
3130
|
-
function createSearchExecutor(db, query) {
|
|
3398
|
+
function createSearchExecutor(db, projectPath, query) {
|
|
3131
3399
|
const pathBoostTerms = extractPathBoostTerms(query);
|
|
3132
3400
|
return async (strategies, limit) => {
|
|
3133
3401
|
const strategyResults = [];
|
|
3134
3402
|
const fetchLimit = limit * 3;
|
|
3135
3403
|
for (const plan of strategies) {
|
|
3136
|
-
const results = await executeStrategy2(db, plan, fetchLimit);
|
|
3404
|
+
const results = await executeStrategy2(db, projectPath, plan, fetchLimit);
|
|
3137
3405
|
if (results.length > 0) {
|
|
3138
3406
|
strategyResults.push({
|
|
3139
3407
|
strategy: plan.strategy,
|
|
@@ -3152,10 +3420,10 @@ function extractSymbolNames2(query) {
|
|
|
3152
3420
|
function isPathLike2(query) {
|
|
3153
3421
|
return query.includes("/") || query.includes("*") || query.includes(".");
|
|
3154
3422
|
}
|
|
3155
|
-
async function executeStrategy2(db, plan, limit) {
|
|
3423
|
+
async function executeStrategy2(db, projectPath, plan, limit) {
|
|
3156
3424
|
switch (plan.strategy) {
|
|
3157
3425
|
case "vector": {
|
|
3158
|
-
const embedder = await loadEmbedder2();
|
|
3426
|
+
const embedder = await loadEmbedder2(projectPath);
|
|
3159
3427
|
return vectorSearch(db, embedder, plan.query, limit);
|
|
3160
3428
|
}
|
|
3161
3429
|
case "fts":
|
|
@@ -3184,13 +3452,20 @@ async function executeStrategy2(db, plan, limit) {
|
|
|
3184
3452
|
}
|
|
3185
3453
|
}
|
|
3186
3454
|
var embedderInstance2 = null;
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3455
|
+
var embedderKey2 = null;
|
|
3456
|
+
function getCacheKey2(projectPath) {
|
|
3457
|
+
const config = getProjectEmbedderConfig(projectPath);
|
|
3458
|
+
return `${projectPath}:${config.provider}:${config.model}:${config.dimensions}`;
|
|
3459
|
+
}
|
|
3460
|
+
async function loadEmbedder2(projectPath) {
|
|
3461
|
+
const cacheKey = getCacheKey2(projectPath);
|
|
3462
|
+
if (embedderInstance2 && embedderKey2 === cacheKey) return embedderInstance2;
|
|
3463
|
+
embedderInstance2 = await createProjectEmbedder(projectPath);
|
|
3464
|
+
embedderKey2 = cacheKey;
|
|
3190
3465
|
return embedderInstance2;
|
|
3191
3466
|
}
|
|
3192
|
-
async function fallbackSearch(db, query, limit) {
|
|
3193
|
-
const executor = createSearchExecutor(db, query);
|
|
3467
|
+
async function fallbackSearch(db, projectPath, query, limit) {
|
|
3468
|
+
const executor = createSearchExecutor(db, projectPath, query);
|
|
3194
3469
|
const fallbackStrategies = buildFallbackStrategies(query);
|
|
3195
3470
|
const results = await executor(fallbackStrategies, limit);
|
|
3196
3471
|
return {
|
|
@@ -3217,18 +3492,19 @@ async function runAsk(projectPath, query, options) {
|
|
|
3217
3492
|
ErrorCode.NOT_INITIALIZED
|
|
3218
3493
|
);
|
|
3219
3494
|
}
|
|
3220
|
-
const
|
|
3495
|
+
const embedderConfig = getProjectEmbedderConfig(absoluteRoot);
|
|
3496
|
+
const db = createDatabase(dbPath, embedderConfig.dimensions);
|
|
3221
3497
|
try {
|
|
3222
3498
|
const provider = options.provider ?? null;
|
|
3223
3499
|
if (!provider) {
|
|
3224
|
-
const output = await fallbackSearch(db, query, limit);
|
|
3500
|
+
const output = await fallbackSearch(db, absoluteRoot, query, limit);
|
|
3225
3501
|
output.warning = FALLBACK_NOTICE;
|
|
3226
3502
|
if (options.format === "text") {
|
|
3227
3503
|
output.text = formatTextOutput2(output);
|
|
3228
3504
|
}
|
|
3229
3505
|
return output;
|
|
3230
3506
|
}
|
|
3231
|
-
const executor = createSearchExecutor(db, query);
|
|
3507
|
+
const executor = createSearchExecutor(db, absoluteRoot, query);
|
|
3232
3508
|
if (options.noExplain) {
|
|
3233
3509
|
return await runNoExplain(provider, query, limit, options, executor);
|
|
3234
3510
|
}
|
|
@@ -3430,8 +3706,8 @@ async function reindexChanges(db, changes, projectPath, options) {
|
|
|
3430
3706
|
const language = detectLanguage(change.path);
|
|
3431
3707
|
if (change.type === "unlink") {
|
|
3432
3708
|
log(`[${timestamp()}] Deleted: ${change.path}`);
|
|
3433
|
-
const
|
|
3434
|
-
if (
|
|
3709
|
+
const existingFile = db.getFile(change.path);
|
|
3710
|
+
if (existingFile) {
|
|
3435
3711
|
db.deleteFile(change.path);
|
|
3436
3712
|
}
|
|
3437
3713
|
filesProcessed++;
|
|
@@ -3441,10 +3717,6 @@ async function reindexChanges(db, changes, projectPath, options) {
|
|
|
3441
3717
|
if (!fs9.existsSync(absolutePath)) continue;
|
|
3442
3718
|
const label = change.type === "add" ? "Added" : "Changed";
|
|
3443
3719
|
log(`[${timestamp()}] ${label}: ${change.path}`);
|
|
3444
|
-
const existingFile = db.getFile(change.path);
|
|
3445
|
-
if (existingFile) {
|
|
3446
|
-
db.deleteChunksByFile(existingFile.id);
|
|
3447
|
-
}
|
|
3448
3720
|
let nodes;
|
|
3449
3721
|
try {
|
|
3450
3722
|
nodes = await parseFile(absolutePath, language);
|
|
@@ -3455,26 +3727,31 @@ async function reindexChanges(db, changes, projectPath, options) {
|
|
|
3455
3727
|
const chunks = chunkFile(nodes, change.path);
|
|
3456
3728
|
const hash = await hashFile(absolutePath);
|
|
3457
3729
|
const size = fs9.statSync(absolutePath).size;
|
|
3458
|
-
const
|
|
3459
|
-
|
|
3460
|
-
|
|
3461
|
-
|
|
3462
|
-
|
|
3730
|
+
const chunkRows = chunks.map((c) => ({
|
|
3731
|
+
lineStart: c.lineStart,
|
|
3732
|
+
lineEnd: c.lineEnd,
|
|
3733
|
+
type: c.type,
|
|
3734
|
+
name: c.name,
|
|
3735
|
+
parent: c.parent,
|
|
3736
|
+
text: c.text,
|
|
3737
|
+
imports: c.imports,
|
|
3738
|
+
exports: c.exports,
|
|
3739
|
+
hash: c.hash
|
|
3740
|
+
}));
|
|
3741
|
+
let chunkIds = [];
|
|
3742
|
+
db.transaction(() => {
|
|
3743
|
+
const existingFile = db.getFile(change.path);
|
|
3744
|
+
if (existingFile) {
|
|
3745
|
+
db.deleteChunksByFile(existingFile.id);
|
|
3746
|
+
}
|
|
3747
|
+
const fileId = db.upsertFile({
|
|
3748
|
+
path: change.path,
|
|
3749
|
+
language,
|
|
3750
|
+
hash,
|
|
3751
|
+
size
|
|
3752
|
+
});
|
|
3753
|
+
chunkIds = db.insertChunks(fileId, chunkRows);
|
|
3463
3754
|
});
|
|
3464
|
-
const chunkIds = db.insertChunks(
|
|
3465
|
-
fileId,
|
|
3466
|
-
chunks.map((c) => ({
|
|
3467
|
-
lineStart: c.lineStart,
|
|
3468
|
-
lineEnd: c.lineEnd,
|
|
3469
|
-
type: c.type,
|
|
3470
|
-
name: c.name,
|
|
3471
|
-
parent: c.parent,
|
|
3472
|
-
text: c.text,
|
|
3473
|
-
imports: c.imports,
|
|
3474
|
-
exports: c.exports,
|
|
3475
|
-
hash: c.hash
|
|
3476
|
-
}))
|
|
3477
|
-
);
|
|
3478
3755
|
for (let i = 0; i < chunks.length; i++) {
|
|
3479
3756
|
allChunksWithMeta.push({
|
|
3480
3757
|
fileRelPath: change.path,
|
|
@@ -3485,7 +3762,7 @@ async function reindexChanges(db, changes, projectPath, options) {
|
|
|
3485
3762
|
filesProcessed++;
|
|
3486
3763
|
}
|
|
3487
3764
|
if (!options.skipEmbedding && allChunksWithMeta.length > 0) {
|
|
3488
|
-
const embedder = await loadEmbedder3();
|
|
3765
|
+
const embedder = await loadEmbedder3(projectPath);
|
|
3489
3766
|
const texts = allChunksWithMeta.map(
|
|
3490
3767
|
(cm) => prepareChunkText(cm.fileRelPath, cm.chunk.parent, cm.chunk.text)
|
|
3491
3768
|
);
|
|
@@ -3501,9 +3778,16 @@ async function reindexChanges(db, changes, projectPath, options) {
|
|
|
3501
3778
|
return { filesProcessed, chunksUpdated, durationMs };
|
|
3502
3779
|
}
|
|
3503
3780
|
var embedderInstance3 = null;
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
|
|
3781
|
+
var embedderKey3 = null;
|
|
3782
|
+
function getCacheKey3(projectPath) {
|
|
3783
|
+
const config = getProjectEmbedderConfig(projectPath);
|
|
3784
|
+
return `${projectPath}:${config.provider}:${config.model}:${config.dimensions}`;
|
|
3785
|
+
}
|
|
3786
|
+
async function loadEmbedder3(projectPath) {
|
|
3787
|
+
const cacheKey = getCacheKey3(projectPath);
|
|
3788
|
+
if (embedderInstance3 && embedderKey3 === cacheKey) return embedderInstance3;
|
|
3789
|
+
embedderInstance3 = await createProjectEmbedder(projectPath);
|
|
3790
|
+
embedderKey3 = cacheKey;
|
|
3507
3791
|
return embedderInstance3;
|
|
3508
3792
|
}
|
|
3509
3793
|
async function runWatch(projectPath, options = {}) {
|
|
@@ -3520,8 +3804,10 @@ async function runWatch(projectPath, options = {}) {
|
|
|
3520
3804
|
);
|
|
3521
3805
|
}
|
|
3522
3806
|
await initParser();
|
|
3523
|
-
const
|
|
3807
|
+
const embedderConfig = getProjectEmbedderConfig(absoluteRoot);
|
|
3808
|
+
const db = createDatabase(dbPath, embedderConfig.dimensions);
|
|
3524
3809
|
let watcherHandle = null;
|
|
3810
|
+
let reindexQueue = Promise.resolve();
|
|
3525
3811
|
const watcher = createWatcher(
|
|
3526
3812
|
{
|
|
3527
3813
|
projectPath: absoluteRoot,
|
|
@@ -3530,7 +3816,7 @@ async function runWatch(projectPath, options = {}) {
|
|
|
3530
3816
|
},
|
|
3531
3817
|
{
|
|
3532
3818
|
onChange: (changes) => {
|
|
3533
|
-
|
|
3819
|
+
reindexQueue = reindexQueue.then(async () => {
|
|
3534
3820
|
try {
|
|
3535
3821
|
const result = await reindexChanges(db, changes, absoluteRoot, {
|
|
3536
3822
|
skipEmbedding: options.skipEmbedding,
|
|
@@ -3546,7 +3832,7 @@ async function runWatch(projectPath, options = {}) {
|
|
|
3546
3832
|
`[${timestamp()}] Error: ${err instanceof Error ? err.message : String(err)}`
|
|
3547
3833
|
);
|
|
3548
3834
|
}
|
|
3549
|
-
})
|
|
3835
|
+
});
|
|
3550
3836
|
},
|
|
3551
3837
|
onError: (err) => {
|
|
3552
3838
|
log(`[${timestamp()}] Watcher error: ${err.message}`);
|
|
@@ -3562,6 +3848,7 @@ async function runWatch(projectPath, options = {}) {
|
|
|
3562
3848
|
await watcherHandle.stop();
|
|
3563
3849
|
watcherHandle = null;
|
|
3564
3850
|
}
|
|
3851
|
+
await reindexQueue;
|
|
3565
3852
|
db.close();
|
|
3566
3853
|
log("Stopped watching. Database saved.");
|
|
3567
3854
|
}
|
|
@@ -3623,6 +3910,7 @@ function readConfig2(ctxDir) {
|
|
|
3623
3910
|
const parsed = JSON.parse(raw);
|
|
3624
3911
|
const embedder = parsed.embedder;
|
|
3625
3912
|
return {
|
|
3913
|
+
provider: embedder?.provider ?? parsed.provider ?? "unknown",
|
|
3626
3914
|
model: embedder?.model ?? parsed.model ?? "unknown",
|
|
3627
3915
|
dimensions: embedder?.dimensions ?? parsed.dimensions ?? 0
|
|
3628
3916
|
};
|
|
@@ -3666,7 +3954,7 @@ function formatStatus(projectPath, output) {
|
|
|
3666
3954
|
if (output.config) {
|
|
3667
3955
|
lines.push("");
|
|
3668
3956
|
lines.push(
|
|
3669
|
-
` Embedder:
|
|
3957
|
+
` Embedder: ${output.config.provider} (${output.config.model}, ${output.config.dimensions} dims)`
|
|
3670
3958
|
);
|
|
3671
3959
|
}
|
|
3672
3960
|
lines.push("");
|