@gzoo/cortex 0.5.11 → 0.5.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cortex.mjs +318 -147
- package/package.json +1 -1
- package/packages/cli/dist/commands/ingest.js +1 -0
- package/packages/cli/dist/commands/ingest.js.map +1 -1
- package/packages/cli/dist/commands/mcp.js +1 -1
- package/packages/cli/dist/commands/mcp.js.map +1 -1
- package/packages/cli/dist/commands/models.js +2 -1
- package/packages/cli/dist/commands/models.js.map +1 -1
- package/packages/cli/dist/commands/serve.js +1 -1
- package/packages/cli/dist/commands/serve.js.map +1 -1
- package/packages/cli/dist/commands/watch.js +1 -0
- package/packages/cli/dist/commands/watch.js.map +1 -1
- package/packages/cli/dist/index.js +1 -1
- package/packages/cli/dist/index.js.map +1 -1
- package/packages/graph/dist/migrations/002-add-indexes.d.ts +4 -0
- package/packages/graph/dist/migrations/002-add-indexes.d.ts.map +1 -0
- package/packages/graph/dist/migrations/002-add-indexes.js +30 -0
- package/packages/graph/dist/migrations/002-add-indexes.js.map +1 -0
- package/packages/graph/dist/query-engine.d.ts.map +1 -1
- package/packages/graph/dist/query-engine.js +16 -25
- package/packages/graph/dist/query-engine.js.map +1 -1
- package/packages/graph/dist/sqlite-store.d.ts +5 -1
- package/packages/graph/dist/sqlite-store.d.ts.map +1 -1
- package/packages/graph/dist/sqlite-store.js +152 -118
- package/packages/graph/dist/sqlite-store.js.map +1 -1
- package/packages/ingest/dist/pipeline.d.ts +7 -0
- package/packages/ingest/dist/pipeline.d.ts.map +1 -1
- package/packages/ingest/dist/pipeline.js +43 -5
- package/packages/ingest/dist/pipeline.js.map +1 -1
- package/packages/ingest/dist/watcher.d.ts +1 -0
- package/packages/ingest/dist/watcher.d.ts.map +1 -1
- package/packages/ingest/dist/watcher.js +13 -5
- package/packages/ingest/dist/watcher.js.map +1 -1
- package/packages/llm/dist/cache.js +5 -5
- package/packages/llm/dist/cache.js.map +1 -1
- package/packages/llm/dist/providers/ollama.d.ts +1 -0
- package/packages/llm/dist/providers/ollama.d.ts.map +1 -1
- package/packages/llm/dist/providers/ollama.js +68 -17
- package/packages/llm/dist/providers/ollama.js.map +1 -1
- package/packages/llm/dist/token-tracker.d.ts +1 -0
- package/packages/llm/dist/token-tracker.d.ts.map +1 -1
- package/packages/llm/dist/token-tracker.js +19 -0
- package/packages/llm/dist/token-tracker.js.map +1 -1
- package/packages/server/dist/index.d.ts.map +1 -1
- package/packages/server/dist/index.js +13 -1
- package/packages/server/dist/index.js.map +1 -1
- package/packages/server/dist/middleware/auth.d.ts.map +1 -1
- package/packages/server/dist/middleware/auth.js +4 -0
- package/packages/server/dist/middleware/auth.js.map +1 -1
- package/packages/server/dist/routes/status.js +1 -1
- package/packages/server/dist/routes/status.js.map +1 -1
package/dist/cortex.mjs
CHANGED
|
@@ -688,6 +688,43 @@ var init_initial = __esm({
|
|
|
688
688
|
}
|
|
689
689
|
});
|
|
690
690
|
|
|
691
|
+
// packages/graph/dist/migrations/002-add-indexes.js
|
|
692
|
+
function up2(db) {
|
|
693
|
+
const currentVersion = db.prepare("SELECT MAX(version) as v FROM schema_version").get()?.v ?? 0;
|
|
694
|
+
if (currentVersion >= MIGRATION_VERSION2)
|
|
695
|
+
return;
|
|
696
|
+
db.exec(`
|
|
697
|
+
-- Composite index for common entity queries (project + status + soft-delete filter)
|
|
698
|
+
CREATE INDEX IF NOT EXISTS idx_entities_project_status_deleted
|
|
699
|
+
ON entities(project_id, status, deleted_at);
|
|
700
|
+
|
|
701
|
+
-- Contradiction lookups by status and severity
|
|
702
|
+
CREATE INDEX IF NOT EXISTS idx_contradictions_status_severity
|
|
703
|
+
ON contradictions(status, severity);
|
|
704
|
+
|
|
705
|
+
-- Contradiction lookups by entity
|
|
706
|
+
CREATE INDEX IF NOT EXISTS idx_contradictions_entity_a
|
|
707
|
+
ON contradictions(entity_id_a);
|
|
708
|
+
|
|
709
|
+
CREATE INDEX IF NOT EXISTS idx_contradictions_entity_b
|
|
710
|
+
ON contradictions(entity_id_b);
|
|
711
|
+
|
|
712
|
+
-- Files by project + status (used during watch/ingest)
|
|
713
|
+
CREATE INDEX IF NOT EXISTS idx_files_project_status
|
|
714
|
+
ON files(project_id, status);
|
|
715
|
+
|
|
716
|
+
INSERT OR IGNORE INTO schema_version (version, applied_at)
|
|
717
|
+
VALUES (${MIGRATION_VERSION2}, datetime('now'));
|
|
718
|
+
`);
|
|
719
|
+
}
|
|
720
|
+
var MIGRATION_VERSION2;
|
|
721
|
+
var init_add_indexes = __esm({
|
|
722
|
+
"packages/graph/dist/migrations/002-add-indexes.js"() {
|
|
723
|
+
"use strict";
|
|
724
|
+
MIGRATION_VERSION2 = 2;
|
|
725
|
+
}
|
|
726
|
+
});
|
|
727
|
+
|
|
691
728
|
// packages/graph/dist/sqlite-store.js
|
|
692
729
|
import Database from "better-sqlite3";
|
|
693
730
|
import { randomUUID } from "node:crypto";
|
|
@@ -780,17 +817,22 @@ var init_sqlite_store = __esm({
|
|
|
780
817
|
"use strict";
|
|
781
818
|
init_dist();
|
|
782
819
|
init_initial();
|
|
820
|
+
init_add_indexes();
|
|
783
821
|
SQLiteStore = class {
|
|
784
822
|
db;
|
|
785
823
|
dbPath;
|
|
786
824
|
constructor(options = {}) {
|
|
787
825
|
const { dbPath = "~/.cortex/cortex.db", walMode = true, backupOnStartup = true } = options;
|
|
788
826
|
this.dbPath = resolveHomePath(dbPath);
|
|
789
|
-
mkdirSync3(dirname(this.dbPath), { recursive: true });
|
|
827
|
+
mkdirSync3(dirname(this.dbPath), { recursive: true, mode: 448 });
|
|
790
828
|
if (backupOnStartup) {
|
|
791
829
|
this.backupSync();
|
|
792
830
|
}
|
|
793
831
|
this.db = new Database(this.dbPath);
|
|
832
|
+
try {
|
|
833
|
+
chmodSync(this.dbPath, 384);
|
|
834
|
+
} catch {
|
|
835
|
+
}
|
|
794
836
|
if (walMode) {
|
|
795
837
|
this.db.pragma("journal_mode = WAL");
|
|
796
838
|
}
|
|
@@ -801,6 +843,7 @@ var init_sqlite_store = __esm({
|
|
|
801
843
|
migrate() {
|
|
802
844
|
try {
|
|
803
845
|
up(this.db);
|
|
846
|
+
up2(this.db);
|
|
804
847
|
} catch (err) {
|
|
805
848
|
throw new CortexError(GRAPH_DB_ERROR, "critical", "graph", `Migration failed: ${err instanceof Error ? err.message : String(err)}`, void 0, "Delete the database and restart.");
|
|
806
849
|
}
|
|
@@ -822,8 +865,17 @@ var init_sqlite_store = __esm({
|
|
|
822
865
|
close() {
|
|
823
866
|
this.db.close();
|
|
824
867
|
}
|
|
868
|
+
transaction(fn) {
|
|
869
|
+
return this.db.transaction(fn)();
|
|
870
|
+
}
|
|
825
871
|
// --- Entities ---
|
|
872
|
+
createEntitySync(entity) {
|
|
873
|
+
return this._createEntity(entity);
|
|
874
|
+
}
|
|
826
875
|
async createEntity(entity) {
|
|
876
|
+
return this._createEntity(entity);
|
|
877
|
+
}
|
|
878
|
+
_createEntity(entity) {
|
|
827
879
|
const id = randomUUID();
|
|
828
880
|
const ts = now();
|
|
829
881
|
this.db.prepare(`
|
|
@@ -903,13 +955,17 @@ var init_sqlite_store = __esm({
|
|
|
903
955
|
}
|
|
904
956
|
let sql;
|
|
905
957
|
if (query.search) {
|
|
958
|
+
const sanitizedSearch = query.search.replace(/[^a-zA-Z0-9\s]/g, " ").trim();
|
|
959
|
+
if (!sanitizedSearch) {
|
|
960
|
+
return [];
|
|
961
|
+
}
|
|
906
962
|
sql = `
|
|
907
963
|
SELECT e.* FROM entities e
|
|
908
964
|
JOIN entities_fts fts ON fts.rowid = e.rowid
|
|
909
965
|
WHERE fts.entities_fts MATCH ? AND ${conditions.join(" AND ")}
|
|
910
966
|
ORDER BY rank
|
|
911
967
|
`;
|
|
912
|
-
params.unshift(
|
|
968
|
+
params.unshift(sanitizedSearch);
|
|
913
969
|
} else {
|
|
914
970
|
sql = `
|
|
915
971
|
SELECT * FROM entities
|
|
@@ -945,22 +1001,35 @@ var init_sqlite_store = __esm({
|
|
|
945
1001
|
const row = this.db.prepare("SELECT * FROM relationships WHERE id = ?").get(id);
|
|
946
1002
|
return row ? rowToRelationship(row) : null;
|
|
947
1003
|
}
|
|
948
|
-
async getRelationshipsForEntity(entityId, direction = "both") {
|
|
1004
|
+
async getRelationshipsForEntity(entityId, direction = "both", limit = 200) {
|
|
949
1005
|
let sql;
|
|
950
1006
|
let params;
|
|
951
1007
|
if (direction === "out") {
|
|
952
|
-
sql = "SELECT * FROM relationships WHERE source_entity_id = ?";
|
|
953
|
-
params = [entityId];
|
|
1008
|
+
sql = "SELECT * FROM relationships WHERE source_entity_id = ? LIMIT ?";
|
|
1009
|
+
params = [entityId, limit];
|
|
954
1010
|
} else if (direction === "in") {
|
|
955
|
-
sql = "SELECT * FROM relationships WHERE target_entity_id = ?";
|
|
956
|
-
params = [entityId];
|
|
1011
|
+
sql = "SELECT * FROM relationships WHERE target_entity_id = ? LIMIT ?";
|
|
1012
|
+
params = [entityId, limit];
|
|
957
1013
|
} else {
|
|
958
|
-
sql = "SELECT * FROM relationships WHERE source_entity_id = ? OR target_entity_id = ?";
|
|
959
|
-
params = [entityId, entityId];
|
|
1014
|
+
sql = "SELECT * FROM relationships WHERE source_entity_id = ? OR target_entity_id = ? LIMIT ?";
|
|
1015
|
+
params = [entityId, entityId, limit];
|
|
960
1016
|
}
|
|
961
1017
|
const rows = this.db.prepare(sql).all(...params);
|
|
962
1018
|
return rows.map(rowToRelationship);
|
|
963
1019
|
}
|
|
1020
|
+
async getRelationshipsForEntities(entityIds) {
|
|
1021
|
+
if (entityIds.length === 0)
|
|
1022
|
+
return [];
|
|
1023
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
1024
|
+
const sql = `
|
|
1025
|
+
SELECT * FROM relationships
|
|
1026
|
+
WHERE source_entity_id IN (${placeholders})
|
|
1027
|
+
OR target_entity_id IN (${placeholders})
|
|
1028
|
+
LIMIT 2000
|
|
1029
|
+
`;
|
|
1030
|
+
const rows = this.db.prepare(sql).all(...entityIds, ...entityIds);
|
|
1031
|
+
return rows.map(rowToRelationship);
|
|
1032
|
+
}
|
|
964
1033
|
async deleteRelationship(id) {
|
|
965
1034
|
this.db.prepare("DELETE FROM relationships WHERE id = ?").run(id);
|
|
966
1035
|
}
|
|
@@ -1388,13 +1457,100 @@ var init_vector_store = __esm({
|
|
|
1388
1457
|
function estimateTokens(text) {
|
|
1389
1458
|
return Math.ceil(text.length / AVG_CHARS_PER_TOKEN);
|
|
1390
1459
|
}
|
|
1391
|
-
var logger2, AVG_CHARS_PER_TOKEN, QueryEngine;
|
|
1460
|
+
var logger2, AVG_CHARS_PER_TOKEN, FTS_STOP_WORDS, QueryEngine;
|
|
1392
1461
|
var init_query_engine = __esm({
|
|
1393
1462
|
"packages/graph/dist/query-engine.js"() {
|
|
1394
1463
|
"use strict";
|
|
1395
1464
|
init_dist();
|
|
1396
1465
|
logger2 = createLogger("graph:query-engine");
|
|
1397
1466
|
AVG_CHARS_PER_TOKEN = 4;
|
|
1467
|
+
FTS_STOP_WORDS = /* @__PURE__ */ new Set([
|
|
1468
|
+
"a",
|
|
1469
|
+
"an",
|
|
1470
|
+
"the",
|
|
1471
|
+
"and",
|
|
1472
|
+
"or",
|
|
1473
|
+
"but",
|
|
1474
|
+
"in",
|
|
1475
|
+
"on",
|
|
1476
|
+
"at",
|
|
1477
|
+
"to",
|
|
1478
|
+
"for",
|
|
1479
|
+
"of",
|
|
1480
|
+
"with",
|
|
1481
|
+
"by",
|
|
1482
|
+
"from",
|
|
1483
|
+
"is",
|
|
1484
|
+
"are",
|
|
1485
|
+
"was",
|
|
1486
|
+
"were",
|
|
1487
|
+
"be",
|
|
1488
|
+
"been",
|
|
1489
|
+
"being",
|
|
1490
|
+
"have",
|
|
1491
|
+
"has",
|
|
1492
|
+
"had",
|
|
1493
|
+
"do",
|
|
1494
|
+
"does",
|
|
1495
|
+
"did",
|
|
1496
|
+
"will",
|
|
1497
|
+
"would",
|
|
1498
|
+
"could",
|
|
1499
|
+
"should",
|
|
1500
|
+
"may",
|
|
1501
|
+
"might",
|
|
1502
|
+
"shall",
|
|
1503
|
+
"can",
|
|
1504
|
+
"need",
|
|
1505
|
+
"must",
|
|
1506
|
+
"what",
|
|
1507
|
+
"which",
|
|
1508
|
+
"who",
|
|
1509
|
+
"how",
|
|
1510
|
+
"why",
|
|
1511
|
+
"when",
|
|
1512
|
+
"where",
|
|
1513
|
+
"that",
|
|
1514
|
+
"this",
|
|
1515
|
+
"these",
|
|
1516
|
+
"those",
|
|
1517
|
+
"it",
|
|
1518
|
+
"its",
|
|
1519
|
+
"me",
|
|
1520
|
+
"my",
|
|
1521
|
+
"you",
|
|
1522
|
+
"your",
|
|
1523
|
+
"we",
|
|
1524
|
+
"our",
|
|
1525
|
+
"they",
|
|
1526
|
+
"their",
|
|
1527
|
+
"he",
|
|
1528
|
+
"she",
|
|
1529
|
+
"i",
|
|
1530
|
+
"all",
|
|
1531
|
+
"any",
|
|
1532
|
+
"each",
|
|
1533
|
+
"some",
|
|
1534
|
+
"no",
|
|
1535
|
+
"not",
|
|
1536
|
+
"so",
|
|
1537
|
+
"yet",
|
|
1538
|
+
"use",
|
|
1539
|
+
"used",
|
|
1540
|
+
"using",
|
|
1541
|
+
"about",
|
|
1542
|
+
"tell",
|
|
1543
|
+
"know",
|
|
1544
|
+
"get",
|
|
1545
|
+
"got",
|
|
1546
|
+
"make",
|
|
1547
|
+
"made",
|
|
1548
|
+
"see",
|
|
1549
|
+
"give",
|
|
1550
|
+
"go",
|
|
1551
|
+
"come",
|
|
1552
|
+
"take"
|
|
1553
|
+
]);
|
|
1398
1554
|
QueryEngine = class {
|
|
1399
1555
|
sqliteStore;
|
|
1400
1556
|
vectorStore;
|
|
@@ -1430,16 +1586,8 @@ var init_query_engine = __esm({
|
|
|
1430
1586
|
}
|
|
1431
1587
|
const privacyFiltered = await this.filterByPrivacy(contextEntities);
|
|
1432
1588
|
const entityIds = new Set(privacyFiltered.map((e) => e.id));
|
|
1433
|
-
const
|
|
1434
|
-
|
|
1435
|
-
const rels = await this.sqliteStore.getRelationshipsForEntity(entity.id);
|
|
1436
|
-
for (const rel of rels) {
|
|
1437
|
-
if (entityIds.has(rel.sourceEntityId) && entityIds.has(rel.targetEntityId)) {
|
|
1438
|
-
relationships.push(rel);
|
|
1439
|
-
}
|
|
1440
|
-
}
|
|
1441
|
-
}
|
|
1442
|
-
const uniqueRels = [...new Map(relationships.map((r) => [r.id, r])).values()];
|
|
1589
|
+
const allRels = await this.sqliteStore.getRelationshipsForEntities([...entityIds]);
|
|
1590
|
+
const uniqueRels = allRels.filter((r) => entityIds.has(r.sourceEntityId) && entityIds.has(r.targetEntityId));
|
|
1443
1591
|
const relTokens = uniqueRels.reduce((sum, r) => sum + estimateTokens(r.description ?? "") + 20, 0);
|
|
1444
1592
|
const filteredTokens = privacyFiltered.reduce((sum, e) => sum + estimateTokens(e.content) + estimateTokens(e.name), 0);
|
|
1445
1593
|
logger2.debug("Context assembled", {
|
|
@@ -1493,94 +1641,7 @@ var init_query_engine = __esm({
|
|
|
1493
1641
|
* entities matching ANY meaningful keyword are returned.
|
|
1494
1642
|
*/
|
|
1495
1643
|
buildFtsQuery(query) {
|
|
1496
|
-
const
|
|
1497
|
-
"a",
|
|
1498
|
-
"an",
|
|
1499
|
-
"the",
|
|
1500
|
-
"and",
|
|
1501
|
-
"or",
|
|
1502
|
-
"but",
|
|
1503
|
-
"in",
|
|
1504
|
-
"on",
|
|
1505
|
-
"at",
|
|
1506
|
-
"to",
|
|
1507
|
-
"for",
|
|
1508
|
-
"of",
|
|
1509
|
-
"with",
|
|
1510
|
-
"by",
|
|
1511
|
-
"from",
|
|
1512
|
-
"is",
|
|
1513
|
-
"are",
|
|
1514
|
-
"was",
|
|
1515
|
-
"were",
|
|
1516
|
-
"be",
|
|
1517
|
-
"been",
|
|
1518
|
-
"being",
|
|
1519
|
-
"have",
|
|
1520
|
-
"has",
|
|
1521
|
-
"had",
|
|
1522
|
-
"do",
|
|
1523
|
-
"does",
|
|
1524
|
-
"did",
|
|
1525
|
-
"will",
|
|
1526
|
-
"would",
|
|
1527
|
-
"could",
|
|
1528
|
-
"should",
|
|
1529
|
-
"may",
|
|
1530
|
-
"might",
|
|
1531
|
-
"shall",
|
|
1532
|
-
"can",
|
|
1533
|
-
"need",
|
|
1534
|
-
"must",
|
|
1535
|
-
"what",
|
|
1536
|
-
"which",
|
|
1537
|
-
"who",
|
|
1538
|
-
"how",
|
|
1539
|
-
"why",
|
|
1540
|
-
"when",
|
|
1541
|
-
"where",
|
|
1542
|
-
"that",
|
|
1543
|
-
"this",
|
|
1544
|
-
"these",
|
|
1545
|
-
"those",
|
|
1546
|
-
"it",
|
|
1547
|
-
"its",
|
|
1548
|
-
"me",
|
|
1549
|
-
"my",
|
|
1550
|
-
"you",
|
|
1551
|
-
"your",
|
|
1552
|
-
"we",
|
|
1553
|
-
"our",
|
|
1554
|
-
"they",
|
|
1555
|
-
"their",
|
|
1556
|
-
"he",
|
|
1557
|
-
"she",
|
|
1558
|
-
"i",
|
|
1559
|
-
"all",
|
|
1560
|
-
"any",
|
|
1561
|
-
"each",
|
|
1562
|
-
"some",
|
|
1563
|
-
"no",
|
|
1564
|
-
"not",
|
|
1565
|
-
"so",
|
|
1566
|
-
"yet",
|
|
1567
|
-
"use",
|
|
1568
|
-
"used",
|
|
1569
|
-
"using",
|
|
1570
|
-
"about",
|
|
1571
|
-
"tell",
|
|
1572
|
-
"know",
|
|
1573
|
-
"get",
|
|
1574
|
-
"got",
|
|
1575
|
-
"make",
|
|
1576
|
-
"made",
|
|
1577
|
-
"see",
|
|
1578
|
-
"give",
|
|
1579
|
-
"go",
|
|
1580
|
-
"come",
|
|
1581
|
-
"take"
|
|
1582
|
-
]);
|
|
1583
|
-
const keywords = query.replace(/[^a-zA-Z0-9\s]/g, " ").toLowerCase().split(/\s+/).filter((w) => w.length >= 3 && !stopWords.has(w));
|
|
1644
|
+
const keywords = query.replace(/[^a-zA-Z0-9\s]/g, " ").toLowerCase().split(/\s+/).filter((w) => w.length >= 3 && !FTS_STOP_WORDS.has(w));
|
|
1584
1645
|
if (keywords.length === 0) {
|
|
1585
1646
|
return query.replace(/[^a-zA-Z0-9\s]/g, " ").trim();
|
|
1586
1647
|
}
|
|
@@ -1796,12 +1857,38 @@ var init_anthropic = __esm({
|
|
|
1796
1857
|
});
|
|
1797
1858
|
|
|
1798
1859
|
// packages/llm/dist/providers/ollama.js
|
|
1799
|
-
|
|
1860
|
+
function validateOllamaHost(host) {
|
|
1861
|
+
let parsed;
|
|
1862
|
+
try {
|
|
1863
|
+
parsed = new URL(host);
|
|
1864
|
+
} catch {
|
|
1865
|
+
throw new CortexError(LLM_PROVIDER_UNAVAILABLE, "high", "llm", `Invalid Ollama host URL: ${host}`, { host }, "Set a valid URL like http://localhost:11434", false);
|
|
1866
|
+
}
|
|
1867
|
+
const hostname = parsed.hostname;
|
|
1868
|
+
for (const pattern of BLOCKED_HOST_PATTERNS) {
|
|
1869
|
+
if (pattern.test(hostname)) {
|
|
1870
|
+
throw new CortexError(LLM_PROVIDER_UNAVAILABLE, "high", "llm", `Ollama host "${hostname}" is blocked \u2014 it matches a link-local or cloud metadata IP range.`, { host }, "Use a non-link-local address for Ollama.", false);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
const localhostNames = /* @__PURE__ */ new Set(["localhost", "127.0.0.1", "::1", "0.0.0.0"]);
|
|
1874
|
+
if (!localhostNames.has(hostname)) {
|
|
1875
|
+
logger4.warn(`Ollama host is not localhost (${hostname}). Ensure the remote Ollama instance is trusted and network-secured.`);
|
|
1876
|
+
}
|
|
1877
|
+
}
|
|
1878
|
+
var logger4, BLOCKED_HOST_PATTERNS, OllamaProvider;
|
|
1800
1879
|
var init_ollama = __esm({
|
|
1801
1880
|
"packages/llm/dist/providers/ollama.js"() {
|
|
1802
1881
|
"use strict";
|
|
1803
1882
|
init_dist();
|
|
1804
1883
|
logger4 = createLogger("llm:ollama");
|
|
1884
|
+
BLOCKED_HOST_PATTERNS = [
|
|
1885
|
+
/^169\.254\./,
|
|
1886
|
+
// AWS/Azure metadata link-local
|
|
1887
|
+
/^fd[0-9a-f]{2}:/i,
|
|
1888
|
+
// IPv6 unique local (fd00::/8)
|
|
1889
|
+
/^fe80:/i
|
|
1890
|
+
// IPv6 link-local
|
|
1891
|
+
];
|
|
1805
1892
|
OllamaProvider = class {
|
|
1806
1893
|
name = "ollama";
|
|
1807
1894
|
type = "local";
|
|
@@ -1812,6 +1899,7 @@ var init_ollama = __esm({
|
|
|
1812
1899
|
numGpu;
|
|
1813
1900
|
timeoutMs;
|
|
1814
1901
|
keepAlive;
|
|
1902
|
+
streamInactivityTimeoutMs;
|
|
1815
1903
|
capabilities = {
|
|
1816
1904
|
supportedTasks: [
|
|
1817
1905
|
LLMTask.ENTITY_EXTRACTION,
|
|
@@ -1830,12 +1918,14 @@ var init_ollama = __esm({
|
|
|
1830
1918
|
};
|
|
1831
1919
|
constructor(options = {}) {
|
|
1832
1920
|
this.host = options.host ?? process.env["CORTEX_OLLAMA_HOST"] ?? "http://localhost:11434";
|
|
1921
|
+
validateOllamaHost(this.host);
|
|
1833
1922
|
this.model = options.model ?? "mistral:7b-instruct-q5_K_M";
|
|
1834
1923
|
this.embeddingModel = options.embeddingModel ?? "nomic-embed-text";
|
|
1835
1924
|
this.numCtx = options.numCtx ?? 8192;
|
|
1836
1925
|
this.numGpu = options.numGpu ?? -1;
|
|
1837
1926
|
this.timeoutMs = options.timeoutMs ?? 3e5;
|
|
1838
1927
|
this.keepAlive = options.keepAlive ?? "5m";
|
|
1928
|
+
this.streamInactivityTimeoutMs = 6e4;
|
|
1839
1929
|
this.capabilities.maxContextTokens = this.numCtx;
|
|
1840
1930
|
}
|
|
1841
1931
|
getModel() {
|
|
@@ -1942,25 +2032,43 @@ var init_ollama = __esm({
|
|
|
1942
2032
|
const decoder = new TextDecoder();
|
|
1943
2033
|
let inputTokens = 0;
|
|
1944
2034
|
let outputTokens = 0;
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
2035
|
+
const streamController = new AbortController();
|
|
2036
|
+
let inactivityTimer = setTimeout(() => streamController.abort(), this.streamInactivityTimeoutMs);
|
|
2037
|
+
try {
|
|
2038
|
+
while (true) {
|
|
2039
|
+
const readPromise = reader.read();
|
|
2040
|
+
const raceResult = await Promise.race([
|
|
2041
|
+
readPromise,
|
|
2042
|
+
new Promise((_, reject) => {
|
|
2043
|
+
streamController.signal.addEventListener("abort", () => reject(new Error("Stream inactivity timeout")), { once: true });
|
|
2044
|
+
if (streamController.signal.aborted) {
|
|
2045
|
+
reject(new Error("Stream inactivity timeout"));
|
|
2046
|
+
}
|
|
2047
|
+
})
|
|
2048
|
+
]);
|
|
2049
|
+
const { done, value } = raceResult;
|
|
2050
|
+
if (done)
|
|
2051
|
+
break;
|
|
2052
|
+
clearTimeout(inactivityTimer);
|
|
2053
|
+
inactivityTimer = setTimeout(() => streamController.abort(), this.streamInactivityTimeoutMs);
|
|
2054
|
+
const chunk = decoder.decode(value, { stream: true });
|
|
2055
|
+
const lines = chunk.split("\n").filter((line) => line.trim());
|
|
2056
|
+
for (const line of lines) {
|
|
2057
|
+
try {
|
|
2058
|
+
const data = JSON.parse(line);
|
|
2059
|
+
if (data.response) {
|
|
2060
|
+
yield data.response;
|
|
2061
|
+
}
|
|
2062
|
+
if (data.done) {
|
|
2063
|
+
inputTokens = data.prompt_eval_count ?? 0;
|
|
2064
|
+
outputTokens = data.eval_count ?? 0;
|
|
2065
|
+
}
|
|
2066
|
+
} catch {
|
|
1960
2067
|
}
|
|
1961
|
-
} catch {
|
|
1962
2068
|
}
|
|
1963
2069
|
}
|
|
2070
|
+
} finally {
|
|
2071
|
+
clearTimeout(inactivityTimer);
|
|
1964
2072
|
}
|
|
1965
2073
|
return {
|
|
1966
2074
|
inputTokens,
|
|
@@ -2252,7 +2360,7 @@ function estimateCost(model, inputTokens, outputTokens) {
|
|
|
2252
2360
|
const costs = MODEL_COSTS[model] ?? DEFAULT_COST;
|
|
2253
2361
|
return inputTokens / 1e6 * costs.input + outputTokens / 1e6 * costs.output;
|
|
2254
2362
|
}
|
|
2255
|
-
var logger6, MODEL_COSTS, DEFAULT_COST, TokenTracker;
|
|
2363
|
+
var logger6, MODEL_COSTS, DEFAULT_COST, MAX_IN_MEMORY_RECORDS, TokenTracker;
|
|
2256
2364
|
var init_token_tracker = __esm({
|
|
2257
2365
|
"packages/llm/dist/token-tracker.js"() {
|
|
2258
2366
|
"use strict";
|
|
@@ -2263,6 +2371,7 @@ var init_token_tracker = __esm({
|
|
|
2263
2371
|
"claude-haiku-4-5-20251001": { input: 0.8, output: 4 }
|
|
2264
2372
|
};
|
|
2265
2373
|
DEFAULT_COST = { input: 3, output: 15 };
|
|
2374
|
+
MAX_IN_MEMORY_RECORDS = 1e4;
|
|
2266
2375
|
TokenTracker = class {
|
|
2267
2376
|
records = [];
|
|
2268
2377
|
monthlyBudgetUsd;
|
|
@@ -2287,9 +2396,23 @@ var init_token_tracker = __esm({
|
|
|
2287
2396
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
2288
2397
|
};
|
|
2289
2398
|
this.records.push(record);
|
|
2399
|
+
this.trimOldRecords();
|
|
2290
2400
|
this.checkBudget();
|
|
2291
2401
|
return record;
|
|
2292
2402
|
}
|
|
2403
|
+
trimOldRecords() {
|
|
2404
|
+
if (this.records.length <= MAX_IN_MEMORY_RECORDS)
|
|
2405
|
+
return;
|
|
2406
|
+
const currentMonth = (/* @__PURE__ */ new Date()).toISOString().slice(0, 7);
|
|
2407
|
+
const currentMonthRecords = this.records.filter((r) => r.timestamp.startsWith(currentMonth));
|
|
2408
|
+
const priorRecords = this.records.filter((r) => !r.timestamp.startsWith(currentMonth));
|
|
2409
|
+
if (currentMonthRecords.length >= MAX_IN_MEMORY_RECORDS) {
|
|
2410
|
+
this.records = currentMonthRecords.slice(-MAX_IN_MEMORY_RECORDS);
|
|
2411
|
+
} else {
|
|
2412
|
+
const keepFromPrior = MAX_IN_MEMORY_RECORDS - currentMonthRecords.length;
|
|
2413
|
+
this.records = [...priorRecords.slice(-keepFromPrior), ...currentMonthRecords];
|
|
2414
|
+
}
|
|
2415
|
+
}
|
|
2293
2416
|
checkBudget() {
|
|
2294
2417
|
const spent = this.getCurrentMonthSpend();
|
|
2295
2418
|
const usedPercent = this.monthlyBudgetUsd > 0 ? spent / this.monthlyBudgetUsd : 0;
|
|
@@ -2402,9 +2525,9 @@ var init_cache = __esm({
|
|
|
2402
2525
|
if (!this.enabled)
|
|
2403
2526
|
return;
|
|
2404
2527
|
if (this.cache.size >= this.maxEntries) {
|
|
2405
|
-
const
|
|
2406
|
-
if (
|
|
2407
|
-
this.cache.delete(
|
|
2528
|
+
const firstKey = this.cache.keys().next().value;
|
|
2529
|
+
if (firstKey !== void 0) {
|
|
2530
|
+
this.cache.delete(firstKey);
|
|
2408
2531
|
}
|
|
2409
2532
|
}
|
|
2410
2533
|
const key = this.buildKey(contentHash, promptId, promptVersion);
|
|
@@ -4187,8 +4310,16 @@ var init_watcher = __esm({
|
|
|
4187
4310
|
options;
|
|
4188
4311
|
handler = null;
|
|
4189
4312
|
debounceTimers = /* @__PURE__ */ new Map();
|
|
4313
|
+
compiledExcludePatterns;
|
|
4190
4314
|
constructor(options) {
|
|
4191
4315
|
this.options = options;
|
|
4316
|
+
this.compiledExcludePatterns = options.exclude.map((pattern) => {
|
|
4317
|
+
if (pattern.includes("*")) {
|
|
4318
|
+
const re = new RegExp("^" + pattern.replace(/\\/g, "\\\\").replace(/\./g, "\\.").replace(/\*\*/g, ".*").replace(/\*/g, "[^/\\\\]*") + "$");
|
|
4319
|
+
return { pattern: re, isGlob: true };
|
|
4320
|
+
}
|
|
4321
|
+
return pattern;
|
|
4322
|
+
});
|
|
4192
4323
|
}
|
|
4193
4324
|
static fromConfig(config8) {
|
|
4194
4325
|
return new _FileWatcher({
|
|
@@ -4272,13 +4403,12 @@ var init_watcher = __esm({
|
|
|
4272
4403
|
}
|
|
4273
4404
|
isExcluded(filePath) {
|
|
4274
4405
|
const parts = filePath.split(/[\\/]/);
|
|
4275
|
-
for (const
|
|
4276
|
-
if (
|
|
4277
|
-
|
|
4278
|
-
if (parts.some((p) => re.test(p)))
|
|
4406
|
+
for (const compiled of this.compiledExcludePatterns) {
|
|
4407
|
+
if (typeof compiled === "string") {
|
|
4408
|
+
if (parts.some((p) => p === compiled))
|
|
4279
4409
|
return true;
|
|
4280
4410
|
} else {
|
|
4281
|
-
if (parts.some((p) => p
|
|
4411
|
+
if (parts.some((p) => compiled.pattern.test(p)))
|
|
4282
4412
|
return true;
|
|
4283
4413
|
}
|
|
4284
4414
|
}
|
|
@@ -4466,10 +4596,34 @@ var init_pipeline = __esm({
|
|
|
4466
4596
|
// Shared across all ingestFile calls — prevents the same entity pair from being
|
|
4467
4597
|
// evaluated twice when multiple files ingest in the same batch.
|
|
4468
4598
|
checkedContradictionPairs = /* @__PURE__ */ new Set();
|
|
4599
|
+
// Pre-compiled secret patterns for scrubbing before cloud LLM calls
|
|
4600
|
+
compiledSecretPatterns;
|
|
4469
4601
|
constructor(router, store, options) {
|
|
4470
4602
|
this.router = router;
|
|
4471
4603
|
this.store = store;
|
|
4472
4604
|
this.options = options;
|
|
4605
|
+
this.compiledSecretPatterns = (options.secretPatterns ?? []).map((pattern) => {
|
|
4606
|
+
try {
|
|
4607
|
+
return new RegExp(pattern, "g");
|
|
4608
|
+
} catch {
|
|
4609
|
+
logger11.warn("Invalid secret pattern, skipping", { pattern });
|
|
4610
|
+
return null;
|
|
4611
|
+
}
|
|
4612
|
+
}).filter((r) => r !== null);
|
|
4613
|
+
}
|
|
4614
|
+
/**
|
|
4615
|
+
* Scrub secrets from content before sending to cloud LLMs.
|
|
4616
|
+
* Only applied for standard privacy (sensitive/restricted use local provider).
|
|
4617
|
+
*/
|
|
4618
|
+
scrubSecrets(content) {
|
|
4619
|
+
if (this.compiledSecretPatterns.length === 0)
|
|
4620
|
+
return content;
|
|
4621
|
+
let scrubbed = content;
|
|
4622
|
+
for (const re of this.compiledSecretPatterns) {
|
|
4623
|
+
re.lastIndex = 0;
|
|
4624
|
+
scrubbed = scrubbed.replace(re, "[SECRET_REDACTED]");
|
|
4625
|
+
}
|
|
4626
|
+
return scrubbed;
|
|
4473
4627
|
}
|
|
4474
4628
|
async ingestFile(filePath) {
|
|
4475
4629
|
try {
|
|
@@ -4545,9 +4699,12 @@ var init_pipeline = __esm({
|
|
|
4545
4699
|
const deduped = this.deduplicateEntities(allEntities);
|
|
4546
4700
|
logger11.debug("Extracted entities", { filePath, raw: allEntities.length, deduped: deduped.length });
|
|
4547
4701
|
const storedEntities = [];
|
|
4548
|
-
|
|
4549
|
-
const
|
|
4550
|
-
|
|
4702
|
+
this.store.transaction(() => {
|
|
4703
|
+
for (const entity of deduped) {
|
|
4704
|
+
storedEntities.push(this.store.createEntitySync(entity));
|
|
4705
|
+
}
|
|
4706
|
+
});
|
|
4707
|
+
for (const stored of storedEntities) {
|
|
4551
4708
|
eventBus.emit({
|
|
4552
4709
|
type: "entity.created",
|
|
4553
4710
|
payload: { entity: stored },
|
|
@@ -4608,6 +4765,7 @@ var init_pipeline = __esm({
|
|
|
4608
4765
|
async extractEntities(chunk, filePath, fileType) {
|
|
4609
4766
|
const contentHash = createHash2("sha256").update(chunk.content).digest("hex");
|
|
4610
4767
|
const privacyOverride = this.options.projectPrivacyLevel !== "standard" ? { forceProvider: "local" } : {};
|
|
4768
|
+
const safeContent = this.options.projectPrivacyLevel === "standard" ? this.scrubSecrets(chunk.content) : chunk.content;
|
|
4611
4769
|
try {
|
|
4612
4770
|
const result = await this.router.completeStructured({
|
|
4613
4771
|
systemPrompt: entity_extraction_exports.systemPrompt,
|
|
@@ -4615,7 +4773,7 @@ var init_pipeline = __esm({
|
|
|
4615
4773
|
filePath,
|
|
4616
4774
|
projectName: this.options.projectName,
|
|
4617
4775
|
fileType,
|
|
4618
|
-
content:
|
|
4776
|
+
content: safeContent
|
|
4619
4777
|
}),
|
|
4620
4778
|
promptId: entity_extraction_exports.PROMPT_ID,
|
|
4621
4779
|
promptVersion: entity_extraction_exports.PROMPT_VERSION,
|
|
@@ -5153,7 +5311,7 @@ var init_status = __esm({
|
|
|
5153
5311
|
for (let i = 0; i < 6; i++) {
|
|
5154
5312
|
try {
|
|
5155
5313
|
const pkg = JSON.parse(readFileSync9(resolve19(dir, "package.json"), "utf-8"));
|
|
5156
|
-
if (pkg.name === "gzoo-cortex" && pkg.version) {
|
|
5314
|
+
if ((pkg.name === "@gzoo/cortex" || pkg.name === "gzoo-cortex") && pkg.version) {
|
|
5157
5315
|
_version = pkg.version;
|
|
5158
5316
|
break;
|
|
5159
5317
|
}
|
|
@@ -5385,7 +5543,7 @@ async function startServer(options) {
|
|
|
5385
5543
|
callback(new Error("CORS not allowed"));
|
|
5386
5544
|
}
|
|
5387
5545
|
}));
|
|
5388
|
-
app.use(express.json());
|
|
5546
|
+
app.use(express.json({ limit: "1mb" }));
|
|
5389
5547
|
const isLocal = host === "127.0.0.1" || host === "localhost" || host === "::1";
|
|
5390
5548
|
if (!isLocal && !config8.server.auth.enabled) {
|
|
5391
5549
|
logger34.warn("Server bound to non-localhost without auth enabled. Set server.auth.enabled=true and server.auth.token in config, or set CORTEX_SERVER_AUTH_TOKEN env var.");
|
|
@@ -5393,6 +5551,15 @@ async function startServer(options) {
|
|
|
5393
5551
|
const rateLimitWindow = 6e4;
|
|
5394
5552
|
const rateLimitMax = 30;
|
|
5395
5553
|
const rateLimitMap = /* @__PURE__ */ new Map();
|
|
5554
|
+
const rateLimitCleanupInterval = setInterval(() => {
|
|
5555
|
+
const now2 = Date.now();
|
|
5556
|
+
for (const [key, entry] of rateLimitMap) {
|
|
5557
|
+
if (now2 >= entry.resetAt) {
|
|
5558
|
+
rateLimitMap.delete(key);
|
|
5559
|
+
}
|
|
5560
|
+
}
|
|
5561
|
+
}, 6e4);
|
|
5562
|
+
rateLimitCleanupInterval.unref();
|
|
5396
5563
|
const rateLimiter = (req, res, next) => {
|
|
5397
5564
|
const key = req.ip ?? "unknown";
|
|
5398
5565
|
const now2 = Date.now();
|
|
@@ -5443,7 +5610,8 @@ async function startServer(options) {
|
|
|
5443
5610
|
maxFileSize: config8.ingest.maxFileSize,
|
|
5444
5611
|
batchSize: config8.ingest.batchSize,
|
|
5445
5612
|
projectPrivacyLevel: project.privacyLevel,
|
|
5446
|
-
mergeConfidenceThreshold: 0.85
|
|
5613
|
+
mergeConfidenceThreshold: 0.85,
|
|
5614
|
+
secretPatterns: config8.privacy.secretPatterns
|
|
5447
5615
|
});
|
|
5448
5616
|
const watcher = new FileWatcher2({
|
|
5449
5617
|
dirs: [project.rootPath],
|
|
@@ -5488,6 +5656,7 @@ async function startServer(options) {
|
|
|
5488
5656
|
});
|
|
5489
5657
|
const shutdown = () => {
|
|
5490
5658
|
logger34.info("Shutting down...");
|
|
5659
|
+
clearInterval(rateLimitCleanupInterval);
|
|
5491
5660
|
relay.close();
|
|
5492
5661
|
server.close();
|
|
5493
5662
|
bundle.store.close();
|
|
@@ -5852,7 +6021,8 @@ async function runWatch(projectName, opts, globals) {
|
|
|
5852
6021
|
maxFileSize: config8.ingest.maxFileSize,
|
|
5853
6022
|
batchSize: config8.ingest.batchSize,
|
|
5854
6023
|
projectPrivacyLevel: project.privacyLevel,
|
|
5855
|
-
mergeConfidenceThreshold: config8.graph.mergeConfidenceThreshold
|
|
6024
|
+
mergeConfidenceThreshold: config8.graph.mergeConfidenceThreshold,
|
|
6025
|
+
secretPatterns: config8.privacy.secretPatterns
|
|
5856
6026
|
});
|
|
5857
6027
|
const watcher = FileWatcher.fromConfig(config8.ingest);
|
|
5858
6028
|
let ingestedCount = 0;
|
|
@@ -7436,7 +7606,8 @@ Dry run complete: ~${totalSections * 3} entities estimated across ${filePaths.le
|
|
|
7436
7606
|
maxFileSize: config8.ingest.maxFileSize,
|
|
7437
7607
|
batchSize: config8.ingest.batchSize,
|
|
7438
7608
|
projectPrivacyLevel: project.privacyLevel,
|
|
7439
|
-
mergeConfidenceThreshold: config8.graph.mergeConfidenceThreshold
|
|
7609
|
+
mergeConfidenceThreshold: config8.graph.mergeConfidenceThreshold,
|
|
7610
|
+
secretPatterns: config8.privacy.secretPatterns
|
|
7440
7611
|
});
|
|
7441
7612
|
let totalEntities = 0;
|
|
7442
7613
|
let totalRelationships = 0;
|
|
@@ -7576,7 +7747,7 @@ async function runModelsPull(model, globals) {
|
|
|
7576
7747
|
Pulling model: ${chalk13.cyan(model)}`));
|
|
7577
7748
|
console.log(chalk13.dim("This may take several minutes for large models...\n"));
|
|
7578
7749
|
}
|
|
7579
|
-
const result = spawnSync("ollama", ["pull", model], { stdio: "inherit"
|
|
7750
|
+
const result = spawnSync("ollama", ["pull", model], { stdio: "inherit" });
|
|
7580
7751
|
if (result.status !== 0) {
|
|
7581
7752
|
console.error(chalk13.red(`
|
|
7582
7753
|
\u2717 Failed to pull model "${model}"`));
|
|
@@ -7722,7 +7893,7 @@ function findPackageRoot(startDir) {
|
|
|
7722
7893
|
try {
|
|
7723
7894
|
const pkgPath = resolve16(dir, "package.json");
|
|
7724
7895
|
const pkg = JSON.parse(readFileSync8(pkgPath, "utf-8"));
|
|
7725
|
-
if (pkg.name === "gzoo-cortex")
|
|
7896
|
+
if (pkg.name === "@gzoo/cortex" || pkg.name === "gzoo-cortex")
|
|
7726
7897
|
return dir;
|
|
7727
7898
|
} catch {
|
|
7728
7899
|
}
|
|
@@ -8018,7 +8189,7 @@ function findPkgRoot(startDir) {
|
|
|
8018
8189
|
try {
|
|
8019
8190
|
const pkgPath = resolve21(dir, "package.json");
|
|
8020
8191
|
const pkg = JSON.parse(readFileSync10(pkgPath, "utf-8"));
|
|
8021
|
-
if (pkg.name === "gzoo-cortex")
|
|
8192
|
+
if (pkg.name === "@gzoo/cortex" || pkg.name === "gzoo-cortex")
|
|
8022
8193
|
return dir;
|
|
8023
8194
|
} catch {
|
|
8024
8195
|
}
|
|
@@ -8180,7 +8351,7 @@ function getVersion() {
|
|
|
8180
8351
|
for (let i = 0; i < 6; i++) {
|
|
8181
8352
|
try {
|
|
8182
8353
|
const pkg = JSON.parse(readFileSync12(resolve23(dir, "package.json"), "utf-8"));
|
|
8183
|
-
if (pkg.name === "gzoo-cortex" && pkg.version)
|
|
8354
|
+
if ((pkg.name === "@gzoo/cortex" || pkg.name === "gzoo-cortex") && pkg.version)
|
|
8184
8355
|
return pkg.version;
|
|
8185
8356
|
} catch {
|
|
8186
8357
|
}
|