@piagent/platform 1.2.8 → 1.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +75 -0
- package/README.md +4 -4
- package/catalog/capabilities.json +2 -2
- package/docs/command-reference-vietnamese.md +7 -4
- package/docs/context-engine.md +30 -1
- package/docs/distribution-standard.md +7 -7
- package/docs/mcp-and-tools.md +7 -0
- package/docs/operator-manual-vietnamese.md +27 -11
- package/docs/publishing-for-teams.md +2 -0
- package/docs/release-install-policy.md +6 -6
- package/docs/security-threat-model.md +7 -1
- package/docs/subagents-and-multiagent.md +1 -1
- package/docs/team-onboarding.md +2 -2
- package/docs-site/architecture.html +1 -1
- package/docs-site/commands.html +1 -1
- package/docs-site/content/context-engine.html +8 -1
- package/docs-site/content/security.html +1 -0
- package/docs-site/context-engine.html +9 -2
- package/docs-site/index.html +1 -1
- package/docs-site/mcp-approval.html +1 -1
- package/docs-site/mcp-auth.html +1 -1
- package/docs-site/mcp-servers.html +1 -1
- package/docs-site/mcp.html +1 -1
- package/docs-site/permissions.html +1 -1
- package/docs-site/profiles.html +1 -1
- package/docs-site/quickstart.html +1 -1
- package/docs-site/runtime.html +1 -1
- package/docs-site/scope.html +1 -1
- package/docs-site/security.html +2 -1
- package/docs-site/team.html +1 -1
- package/docs-site/workflows.html +1 -1
- package/package.json +1 -1
- package/packages/piagent-core/README.md +6 -1
- package/packages/piagent-core/capabilities/capability-core.js +3 -0
- package/packages/piagent-core/extensions/context-engine.js +263 -120
- package/packages/piagent-core/extensions/context-index-policy.js +136 -0
- package/packages/piagent-core/extensions/piagent-guard.ts +258 -101
- package/packages/piagent-core/mcp/mcp-config-layers.js +52 -6
- package/packages/piagent-core/mcp/mcp-tool-naming.js +12 -1
- package/packages/piagent-core/package.json +1 -1
- package/packages/piagent-core/subagents/piagent-reviewer.md +3 -3
- package/scripts/context-engine.mjs +22 -20
- package/scripts/update-global.mjs +1 -1
- package/scripts/verify-local.sh +2 -0
|
@@ -3,6 +3,13 @@ import fs from "node:fs";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { execFileSync } from "node:child_process";
|
|
5
5
|
|
|
6
|
+
import {
|
|
7
|
+
contextIndexExcludeDigest,
|
|
8
|
+
contextIndexExcludePolicyVersion,
|
|
9
|
+
normalizeContextIndexExcludePatterns
|
|
10
|
+
} from "./context-index-policy.js";
|
|
11
|
+
import { matchesProtectedPath } from "./policy-core.js";
|
|
12
|
+
|
|
6
13
|
const INDEX_SCHEMA_VERSION = 2;
|
|
7
14
|
const TELEMETRY_SCHEMA_VERSION = 1;
|
|
8
15
|
const RRF_K = 60;
|
|
@@ -64,6 +71,13 @@ function clampInteger(value, fallback, min, max) {
|
|
|
64
71
|
return Math.max(min, Math.min(max, Math.trunc(number)));
|
|
65
72
|
}
|
|
66
73
|
|
|
74
|
+
function requireExplicitExcludePatterns(options, operation) {
|
|
75
|
+
if (!Array.isArray(options?.excludePatterns)) {
|
|
76
|
+
throw new TypeError(`${operation} requires an explicit excludePatterns array`);
|
|
77
|
+
}
|
|
78
|
+
return normalizeContextIndexExcludePatterns(options.excludePatterns);
|
|
79
|
+
}
|
|
80
|
+
|
|
67
81
|
export function estimateContextTokens(value) {
|
|
68
82
|
return Math.max(0, Math.ceil(String(value ?? "").length / 4));
|
|
69
83
|
}
|
|
@@ -78,6 +92,21 @@ export function contextEnginePaths(cwd) {
|
|
|
78
92
|
};
|
|
79
93
|
}
|
|
80
94
|
|
|
95
|
+
function hardenContextDatabaseArtifacts(cwd) {
|
|
96
|
+
const database = contextEnginePaths(cwd).database;
|
|
97
|
+
for (const artifact of [database, `${database}-wal`, `${database}-shm`]) {
|
|
98
|
+
try {
|
|
99
|
+
const stat = fs.lstatSync(artifact);
|
|
100
|
+
if (!stat.isFile()) {
|
|
101
|
+
throw new Error(`Context index storage path must be a regular file: ${artifact}`);
|
|
102
|
+
}
|
|
103
|
+
fs.chmodSync(artifact, 0o600);
|
|
104
|
+
} catch (error) {
|
|
105
|
+
if (error?.code !== "ENOENT") throw error;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
81
110
|
async function loadSqlite() {
|
|
82
111
|
if (!sqliteModulePromise) {
|
|
83
112
|
sqliteModulePromise = (async () => {
|
|
@@ -103,53 +132,85 @@ async function loadSqlite() {
|
|
|
103
132
|
async function openDatabase(cwd, { create = true } = {}) {
|
|
104
133
|
const paths = contextEnginePaths(cwd);
|
|
105
134
|
if (!create && !fs.existsSync(paths.database)) return undefined;
|
|
106
|
-
fs.mkdirSync(paths.root, { recursive: true });
|
|
135
|
+
fs.mkdirSync(paths.root, { recursive: true, mode: 0o700 });
|
|
136
|
+
if (!fs.lstatSync(paths.root).isDirectory()) {
|
|
137
|
+
throw new Error(`Context index storage root must be a directory: ${paths.root}`);
|
|
138
|
+
}
|
|
139
|
+
fs.chmodSync(paths.root, 0o700);
|
|
140
|
+
hardenContextDatabaseArtifacts(cwd);
|
|
141
|
+
const descriptor = fs.openSync(
|
|
142
|
+
paths.database,
|
|
143
|
+
fs.constants.O_RDWR
|
|
144
|
+
| (create ? fs.constants.O_CREAT : 0)
|
|
145
|
+
| (fs.constants.O_NOFOLLOW ?? 0),
|
|
146
|
+
0o600
|
|
147
|
+
);
|
|
148
|
+
try {
|
|
149
|
+
fs.fchmodSync(descriptor, 0o600);
|
|
150
|
+
} finally {
|
|
151
|
+
fs.closeSync(descriptor);
|
|
152
|
+
}
|
|
107
153
|
const { DatabaseSync } = await loadSqlite();
|
|
108
154
|
const db = new DatabaseSync(paths.database);
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
155
|
+
try {
|
|
156
|
+
db.exec(`
|
|
157
|
+
PRAGMA journal_mode = WAL;
|
|
158
|
+
PRAGMA synchronous = NORMAL;
|
|
159
|
+
PRAGMA foreign_keys = ON;
|
|
160
|
+
PRAGMA secure_delete = ON;
|
|
161
|
+
CREATE TABLE IF NOT EXISTS metadata (
|
|
162
|
+
key TEXT PRIMARY KEY,
|
|
163
|
+
value TEXT NOT NULL
|
|
164
|
+
);
|
|
165
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
166
|
+
path TEXT PRIMARY KEY,
|
|
167
|
+
hash TEXT NOT NULL,
|
|
168
|
+
bytes INTEGER NOT NULL,
|
|
169
|
+
mtime_ms REAL NOT NULL,
|
|
170
|
+
language TEXT NOT NULL,
|
|
171
|
+
indexed_at TEXT NOT NULL
|
|
172
|
+
);
|
|
173
|
+
CREATE TABLE IF NOT EXISTS symbols (
|
|
174
|
+
id INTEGER PRIMARY KEY,
|
|
175
|
+
file_path TEXT NOT NULL REFERENCES files(path) ON DELETE CASCADE,
|
|
176
|
+
name TEXT NOT NULL,
|
|
177
|
+
kind TEXT NOT NULL,
|
|
178
|
+
line INTEGER NOT NULL,
|
|
179
|
+
end_line INTEGER NOT NULL,
|
|
180
|
+
signature TEXT NOT NULL
|
|
181
|
+
);
|
|
182
|
+
CREATE INDEX IF NOT EXISTS symbols_name_idx ON symbols(name);
|
|
183
|
+
CREATE INDEX IF NOT EXISTS symbols_file_idx ON symbols(file_path);
|
|
184
|
+
CREATE TABLE IF NOT EXISTS imports (
|
|
185
|
+
id INTEGER PRIMARY KEY,
|
|
186
|
+
file_path TEXT NOT NULL REFERENCES files(path) ON DELETE CASCADE,
|
|
187
|
+
specifier TEXT NOT NULL,
|
|
188
|
+
target_path TEXT,
|
|
189
|
+
line INTEGER NOT NULL
|
|
190
|
+
);
|
|
191
|
+
CREATE INDEX IF NOT EXISTS imports_file_idx ON imports(file_path);
|
|
192
|
+
CREATE INDEX IF NOT EXISTS imports_target_idx ON imports(target_path);
|
|
193
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS file_fts USING fts5(
|
|
194
|
+
path UNINDEXED,
|
|
195
|
+
body,
|
|
196
|
+
symbol_names,
|
|
197
|
+
tokenize = 'unicode61 remove_diacritics 2'
|
|
198
|
+
);
|
|
199
|
+
`);
|
|
200
|
+
const ftsSecureDelete = db.prepare(`
|
|
201
|
+
SELECT v
|
|
202
|
+
FROM file_fts_config
|
|
203
|
+
WHERE k = 'secure-delete'
|
|
204
|
+
`).get();
|
|
205
|
+
if (Number(ftsSecureDelete?.v) !== 1) {
|
|
206
|
+
db.exec("INSERT INTO file_fts(file_fts, rank) VALUES('secure-delete', 1)");
|
|
207
|
+
}
|
|
208
|
+
hardenContextDatabaseArtifacts(cwd);
|
|
209
|
+
return db;
|
|
210
|
+
} catch (error) {
|
|
211
|
+
db.close();
|
|
212
|
+
throw error;
|
|
213
|
+
}
|
|
153
214
|
}
|
|
154
215
|
|
|
155
216
|
function metadataMap(db) {
|
|
@@ -184,41 +245,28 @@ function shouldIndexPath(relativePath, options) {
|
|
|
184
245
|
const segments = rel.split("/");
|
|
185
246
|
if (segments.some((segment) => IGNORED_SEGMENTS.has(segment))) return false;
|
|
186
247
|
if (SECRET_FILE_PATTERNS.some((pattern) => pattern.test(rel))) return false;
|
|
187
|
-
if ((options.excludePatterns ?? [])
|
|
248
|
+
if (matchesProtectedPath(rel, options.excludePatterns ?? [])) return false;
|
|
188
249
|
const name = path.posix.basename(rel);
|
|
189
250
|
return SOURCE_NAMES.has(name) || SOURCE_EXTENSIONS.has(path.posix.extname(name).toLowerCase());
|
|
190
251
|
}
|
|
191
252
|
|
|
192
|
-
function
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
let source = "";
|
|
196
|
-
for (let index = 0; index < normalized.length; index += 1) {
|
|
197
|
-
const char = normalized[index];
|
|
198
|
-
if (char === "*" && normalized[index + 1] === "*") {
|
|
199
|
-
if (normalized[index + 2] === "/") {
|
|
200
|
-
source += "(?:.*/)?";
|
|
201
|
-
index += 2;
|
|
202
|
-
} else {
|
|
203
|
-
source += ".*";
|
|
204
|
-
index += 1;
|
|
205
|
-
}
|
|
206
|
-
continue;
|
|
207
|
-
}
|
|
208
|
-
if (char === "*") {
|
|
209
|
-
source += "[^/]*";
|
|
210
|
-
continue;
|
|
211
|
-
}
|
|
212
|
-
if (char === "?") {
|
|
213
|
-
source += "[^/]";
|
|
214
|
-
continue;
|
|
215
|
-
}
|
|
216
|
-
source += /[.+^${}()|[\]\\]/.test(char) ? `\\${char}` : char;
|
|
217
|
-
}
|
|
253
|
+
function projectFileInfo(cwd, relativePath, root = fs.realpathSync.native(cwd), directoryCache = new Map()) {
|
|
254
|
+
const expected = path.resolve(root, relativePath);
|
|
255
|
+
if (expected !== root && !expected.startsWith(`${root}${path.sep}`)) return undefined;
|
|
218
256
|
try {
|
|
219
|
-
|
|
257
|
+
const directPath = path.resolve(cwd, relativePath);
|
|
258
|
+
const directParent = path.dirname(directPath);
|
|
259
|
+
let canonicalParent = directoryCache.get(directParent);
|
|
260
|
+
if (canonicalParent === undefined) {
|
|
261
|
+
canonicalParent = fs.realpathSync.native(directParent);
|
|
262
|
+
directoryCache.set(directParent, canonicalParent);
|
|
263
|
+
}
|
|
264
|
+
if (canonicalParent !== path.dirname(expected)) return undefined;
|
|
265
|
+
const direct = fs.lstatSync(directPath);
|
|
266
|
+
if (direct.isSymbolicLink() || !direct.isFile()) return undefined;
|
|
267
|
+
return { absolute: expected, stat: direct };
|
|
220
268
|
} catch {
|
|
221
|
-
return
|
|
269
|
+
return undefined;
|
|
222
270
|
}
|
|
223
271
|
}
|
|
224
272
|
|
|
@@ -463,13 +511,20 @@ export function classifyContextTask(prompt) {
|
|
|
463
511
|
}
|
|
464
512
|
|
|
465
513
|
export async function buildContextIndexV2(cwd, options = {}) {
|
|
514
|
+
const excludePatterns = requireExplicitExcludePatterns(options, "buildContextIndexV2");
|
|
515
|
+
const excludeDigest = contextIndexExcludeDigest(excludePatterns);
|
|
466
516
|
const startedAt = Date.now();
|
|
517
|
+
const projectRoot = fs.realpathSync.native(cwd);
|
|
518
|
+
const directoryCache = new Map();
|
|
467
519
|
const maxFiles = clampInteger(options.maxFiles, DEFAULT_MAX_FILES, 1, 50_000);
|
|
468
520
|
const maxFileBytes = clampInteger(options.maxFileBytes, DEFAULT_MAX_FILE_BYTES, 1_024, 2 * 1024 * 1024);
|
|
469
521
|
const listed = gitFileList(cwd) ?? walkedFileList(cwd, maxFiles);
|
|
470
|
-
const candidates = listed.filter((file) => shouldIndexPath(file, options)).slice(0, maxFiles).sort();
|
|
522
|
+
const candidates = listed.filter((file) => shouldIndexPath(file, { ...options, excludePatterns })).slice(0, maxFiles).sort();
|
|
471
523
|
const db = await openDatabase(cwd);
|
|
524
|
+
const previousMetadata = metadataMap(db);
|
|
472
525
|
const existing = new Map(db.prepare("SELECT path, hash, bytes, mtime_ms FROM files").all().map((row) => [row.path, row]));
|
|
526
|
+
const purgeStaleContent = Boolean(previousMetadata.builtAt || existing.size > 0)
|
|
527
|
+
&& (previousMetadata.purgePending === "1" || previousMetadata.excludeDigest !== excludeDigest);
|
|
473
528
|
const current = new Set();
|
|
474
529
|
const currentHashes = new Map();
|
|
475
530
|
const changed = [];
|
|
@@ -480,18 +535,12 @@ export async function buildContextIndexV2(cwd, options = {}) {
|
|
|
480
535
|
const skippedPaths = [];
|
|
481
536
|
|
|
482
537
|
for (const relativePath of candidates) {
|
|
483
|
-
const
|
|
484
|
-
if (!
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
if (stat.size > maxFileBytes) {
|
|
490
|
-
skippedLarge += 1;
|
|
491
|
-
skippedPaths.push(relativePath);
|
|
492
|
-
continue;
|
|
493
|
-
}
|
|
494
|
-
} catch {
|
|
538
|
+
const file = projectFileInfo(cwd, relativePath, projectRoot, directoryCache);
|
|
539
|
+
if (!file) continue;
|
|
540
|
+
const { absolute, stat } = file;
|
|
541
|
+
if (stat.size > maxFileBytes) {
|
|
542
|
+
skippedLarge += 1;
|
|
543
|
+
skippedPaths.push(relativePath);
|
|
495
544
|
continue;
|
|
496
545
|
}
|
|
497
546
|
const previous = existing.get(relativePath);
|
|
@@ -527,8 +576,10 @@ export async function buildContextIndexV2(cwd, options = {}) {
|
|
|
527
576
|
}
|
|
528
577
|
|
|
529
578
|
const removed = [...existing.keys()].filter((file) => !current.has(file));
|
|
530
|
-
|
|
579
|
+
let transactionOpen = false;
|
|
531
580
|
try {
|
|
581
|
+
db.exec("BEGIN IMMEDIATE");
|
|
582
|
+
transactionOpen = true;
|
|
532
583
|
const deleteFile = db.prepare("DELETE FROM files WHERE path = ?");
|
|
533
584
|
const deleteFts = db.prepare("DELETE FROM file_fts WHERE path = ?");
|
|
534
585
|
for (const relativePath of [...removed, ...changed.map((item) => item.relativePath)]) {
|
|
@@ -575,11 +626,29 @@ export async function buildContextIndexV2(cwd, options = {}) {
|
|
|
575
626
|
symbolCount: counts.symbols,
|
|
576
627
|
importCount: counts.imports,
|
|
577
628
|
maxFileBytes,
|
|
578
|
-
excludePatterns: JSON.stringify(
|
|
629
|
+
excludePatterns: JSON.stringify(excludePatterns),
|
|
630
|
+
excludeDigest,
|
|
631
|
+
excludePolicyVersion: contextIndexExcludePolicyVersion(),
|
|
632
|
+
purgePending: purgeStaleContent ? 1 : 0,
|
|
579
633
|
skippedPaths: JSON.stringify(skippedPaths)
|
|
580
634
|
});
|
|
581
635
|
db.exec("COMMIT");
|
|
582
|
-
|
|
636
|
+
transactionOpen = false;
|
|
637
|
+
if (purgeStaleContent) {
|
|
638
|
+
if (typeof options.onPurgeStart === "function") await options.onPurgeStart();
|
|
639
|
+
db.exec("INSERT INTO file_fts(file_fts) VALUES('rebuild')");
|
|
640
|
+
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
641
|
+
db.exec("VACUUM");
|
|
642
|
+
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
643
|
+
db.exec("BEGIN IMMEDIATE");
|
|
644
|
+
transactionOpen = true;
|
|
645
|
+
setMetadata(db, {
|
|
646
|
+
purgePending: 0,
|
|
647
|
+
purgedAt: nowIso()
|
|
648
|
+
});
|
|
649
|
+
db.exec("COMMIT");
|
|
650
|
+
transactionOpen = false;
|
|
651
|
+
}
|
|
583
652
|
return {
|
|
584
653
|
schemaVersion: INDEX_SCHEMA_VERSION,
|
|
585
654
|
builtAt: indexedAt,
|
|
@@ -590,19 +659,35 @@ export async function buildContextIndexV2(cwd, options = {}) {
|
|
|
590
659
|
removed: removed.length,
|
|
591
660
|
skippedLarge,
|
|
592
661
|
skippedBinary,
|
|
662
|
+
purgedStaleContent: purgeStaleContent,
|
|
593
663
|
...counts
|
|
594
664
|
};
|
|
595
665
|
} catch (error) {
|
|
596
|
-
db.exec("ROLLBACK");
|
|
597
|
-
db.close();
|
|
666
|
+
if (transactionOpen) db.exec("ROLLBACK");
|
|
598
667
|
throw error;
|
|
668
|
+
} finally {
|
|
669
|
+
db.close();
|
|
670
|
+
hardenContextDatabaseArtifacts(cwd);
|
|
599
671
|
}
|
|
600
672
|
}
|
|
601
673
|
|
|
602
|
-
export async function contextIndexV2Status(cwd) {
|
|
674
|
+
export async function contextIndexV2Status(cwd, options = {}) {
|
|
675
|
+
const expectedExcludePatterns = requireExplicitExcludePatterns(options, "contextIndexV2Status");
|
|
676
|
+
const expectedExcludeDigest = contextIndexExcludeDigest(expectedExcludePatterns);
|
|
603
677
|
const paths = contextEnginePaths(cwd);
|
|
678
|
+
const projectRoot = fs.realpathSync.native(cwd);
|
|
679
|
+
const directoryCache = new Map();
|
|
604
680
|
if (!fs.existsSync(paths.database)) {
|
|
605
|
-
return {
|
|
681
|
+
return {
|
|
682
|
+
schemaVersion: INDEX_SCHEMA_VERSION,
|
|
683
|
+
exists: false,
|
|
684
|
+
path: normalizeRelative(path.relative(cwd, paths.database)),
|
|
685
|
+
stale: false,
|
|
686
|
+
policyStale: false,
|
|
687
|
+
expectedExcludeDigest,
|
|
688
|
+
purgePending: false,
|
|
689
|
+
warnings: ["index-v2 missing"]
|
|
690
|
+
};
|
|
606
691
|
}
|
|
607
692
|
const db = await openDatabase(cwd, { create: false });
|
|
608
693
|
try {
|
|
@@ -612,24 +697,23 @@ export async function contextIndexV2Status(cwd) {
|
|
|
612
697
|
const symbols = Number(db.prepare("SELECT COUNT(*) count FROM symbols").get().count);
|
|
613
698
|
const imports = Number(db.prepare("SELECT COUNT(*) count FROM imports").get().count);
|
|
614
699
|
const indexedPaths = new Set(fileRows.map((row) => row.path));
|
|
700
|
+
const storedExcludeDigest = metadata.excludeDigest;
|
|
701
|
+
const purgePending = metadata.purgePending === "1";
|
|
702
|
+
const excludeDigestStale = typeof storedExcludeDigest !== "string"
|
|
703
|
+
|| storedExcludeDigest !== expectedExcludeDigest;
|
|
704
|
+
const policyStale = purgePending || excludeDigestStale;
|
|
615
705
|
const statChanged = [];
|
|
616
706
|
for (const row of fileRows) {
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
707
|
+
if (!shouldIndexPath(row.path, { excludePatterns: expectedExcludePatterns })) continue;
|
|
708
|
+
const file = projectFileInfo(cwd, row.path, projectRoot, directoryCache);
|
|
709
|
+
if (
|
|
710
|
+
!file
|
|
711
|
+
|| file.stat.size !== Number(row.bytes)
|
|
712
|
+
|| Math.abs(file.stat.mtimeMs - Number(row.mtime_ms)) >= 0.5
|
|
713
|
+
) {
|
|
623
714
|
statChanged.push(row.path);
|
|
624
715
|
}
|
|
625
716
|
}
|
|
626
|
-
let excludePatterns = [];
|
|
627
|
-
try {
|
|
628
|
-
const parsed = JSON.parse(metadata.excludePatterns ?? "[]");
|
|
629
|
-
if (Array.isArray(parsed)) excludePatterns = parsed.filter((value) => typeof value === "string");
|
|
630
|
-
} catch {
|
|
631
|
-
excludePatterns = [];
|
|
632
|
-
}
|
|
633
717
|
let skippedPaths = new Set();
|
|
634
718
|
try {
|
|
635
719
|
const parsed = JSON.parse(metadata.skippedPaths ?? "[]");
|
|
@@ -639,9 +723,13 @@ export async function contextIndexV2Status(cwd) {
|
|
|
639
723
|
}
|
|
640
724
|
const listed = gitFileList(cwd);
|
|
641
725
|
const newlyVisible = listed
|
|
642
|
-
? listed.filter((file) => !indexedPaths.has(file) && !skippedPaths.has(file) && shouldIndexPath(file, { excludePatterns }))
|
|
726
|
+
? listed.filter((file) => !indexedPaths.has(file) && !skippedPaths.has(file) && shouldIndexPath(file, { excludePatterns: expectedExcludePatterns }))
|
|
643
727
|
: [];
|
|
644
728
|
const stalePaths = [...new Set([...statChanged, ...newlyVisible])];
|
|
729
|
+
const warnings = [];
|
|
730
|
+
if (purgePending) warnings.push("context index storage purge pending");
|
|
731
|
+
if (excludeDigestStale) warnings.push("context index exclusion policy changed");
|
|
732
|
+
if (stalePaths.length > 0) warnings.push(`${stalePaths.length} changed file(s) since the last index build`);
|
|
645
733
|
return {
|
|
646
734
|
schemaVersion: Number(metadata.schemaVersion ?? INDEX_SCHEMA_VERSION),
|
|
647
735
|
exists: true,
|
|
@@ -650,15 +738,38 @@ export async function contextIndexV2Status(cwd) {
|
|
|
650
738
|
files,
|
|
651
739
|
symbols,
|
|
652
740
|
imports,
|
|
653
|
-
stale: stalePaths.length > 0,
|
|
741
|
+
stale: policyStale || stalePaths.length > 0,
|
|
742
|
+
policyStale,
|
|
743
|
+
purgePending,
|
|
744
|
+
excludePolicyVersion: metadata.excludePolicyVersion,
|
|
745
|
+
excludeDigest: storedExcludeDigest,
|
|
746
|
+
expectedExcludeDigest,
|
|
654
747
|
stalePaths: stalePaths.slice(0, 20),
|
|
655
|
-
warnings
|
|
748
|
+
warnings
|
|
656
749
|
};
|
|
657
750
|
} finally {
|
|
658
751
|
db.close();
|
|
659
752
|
}
|
|
660
753
|
}
|
|
661
754
|
|
|
755
|
+
export async function ensureContextIndexV2(cwd, options = {}) {
|
|
756
|
+
const excludePatterns = requireExplicitExcludePatterns(options, "ensureContextIndexV2");
|
|
757
|
+
let status = await contextIndexV2Status(cwd, { excludePatterns });
|
|
758
|
+
const reason = options.refresh
|
|
759
|
+
? "refresh"
|
|
760
|
+
: status.policyStale
|
|
761
|
+
? "exclusion-policy"
|
|
762
|
+
: !status.exists && options.rebuildMissing
|
|
763
|
+
? "missing"
|
|
764
|
+
: undefined;
|
|
765
|
+
let build;
|
|
766
|
+
if (reason) {
|
|
767
|
+
build = await buildContextIndexV2(cwd, { ...options, excludePatterns });
|
|
768
|
+
status = await contextIndexV2Status(cwd, { excludePatterns });
|
|
769
|
+
}
|
|
770
|
+
return { status, rebuilt: Boolean(reason), reason, build, excludePatterns };
|
|
771
|
+
}
|
|
772
|
+
|
|
662
773
|
function addRankedList(scores, rows, source, weight = 1) {
|
|
663
774
|
rows.forEach((row, index) => {
|
|
664
775
|
const filePath = row.path ?? row.file_path;
|
|
@@ -783,8 +894,11 @@ function retrievalFeedback(cwd) {
|
|
|
783
894
|
}
|
|
784
895
|
|
|
785
896
|
export async function searchContextIndexV2(cwd, query, options = {}) {
|
|
897
|
+
const excludePatterns = requireExplicitExcludePatterns(options, "searchContextIndexV2");
|
|
786
898
|
const limit = clampInteger(options.limit, 12, 1, 50);
|
|
787
|
-
const
|
|
899
|
+
const projectRoot = fs.realpathSync.native(cwd);
|
|
900
|
+
const directoryCache = new Map();
|
|
901
|
+
const status = await contextIndexV2Status(cwd, { excludePatterns });
|
|
788
902
|
if (!status.exists) return { query, terms: [], confidence: "none", results: [], status };
|
|
789
903
|
const terms = tokenizeQuery(query);
|
|
790
904
|
const explicitPaths = queryPathCandidates(query);
|
|
@@ -792,7 +906,8 @@ export async function searchContextIndexV2(cwd, query, options = {}) {
|
|
|
792
906
|
const db = await openDatabase(cwd, { create: false });
|
|
793
907
|
try {
|
|
794
908
|
const scores = new Map();
|
|
795
|
-
const
|
|
909
|
+
const pathAllowed = (filePath) => shouldIndexPath(filePath, { excludePatterns });
|
|
910
|
+
const allFiles = db.prepare("SELECT path, mtime_ms FROM files").all().filter((file) => pathAllowed(file.path));
|
|
796
911
|
const explicitRows = [];
|
|
797
912
|
for (const file of allFiles) {
|
|
798
913
|
const basename = path.posix.basename(file.path).toLowerCase();
|
|
@@ -809,7 +924,7 @@ export async function searchContextIndexV2(cwd, query, options = {}) {
|
|
|
809
924
|
lexicalRows = db.prepare(`
|
|
810
925
|
SELECT path, bm25(file_fts, 0.0, 1.0, 2.2) lexical_score
|
|
811
926
|
FROM file_fts WHERE file_fts MATCH ? ORDER BY lexical_score LIMIT ?
|
|
812
|
-
`).all(ftsQuery(terms), Math.max(limit * 4, 30));
|
|
927
|
+
`).all(ftsQuery(terms), Math.max(limit * 4, 30)).filter((row) => pathAllowed(row.path));
|
|
813
928
|
addRankedList(scores, lexicalRows, "lexical", 1.5);
|
|
814
929
|
}
|
|
815
930
|
|
|
@@ -820,7 +935,7 @@ export async function searchContextIndexV2(cwd, query, options = {}) {
|
|
|
820
935
|
ORDER BY CASE WHEN lower(name) = ? THEN 0 ELSE 1 END, length(name) LIMIT ?
|
|
821
936
|
`);
|
|
822
937
|
for (const term of terms.slice(0, 8)) {
|
|
823
|
-
symbolRows.push(...symbolStatement.all(term, `%${term}%`, term, Math.max(limit * 2, 20)));
|
|
938
|
+
symbolRows.push(...symbolStatement.all(term, `%${term}%`, term, Math.max(limit * 2, 20)).filter((row) => pathAllowed(row.file_path)));
|
|
824
939
|
}
|
|
825
940
|
const dedupedSymbols = [...new Map(symbolRows.map((row) => [`${row.file_path}:${row.line}`, row])).values()];
|
|
826
941
|
addRankedList(scores, dedupedSymbols, "symbol", 2.2);
|
|
@@ -841,11 +956,19 @@ export async function searchContextIndexV2(cwd, query, options = {}) {
|
|
|
841
956
|
addRankedList(scores, feedbackRows, "feedback", 0.45);
|
|
842
957
|
|
|
843
958
|
const graphSeeds = new Map([...scores.entries()].map(([filePath, value]) => [filePath, value.score]));
|
|
844
|
-
const importRows = db.prepare("SELECT file_path, target_path FROM imports WHERE target_path IS NOT NULL")
|
|
959
|
+
const importRows = db.prepare("SELECT file_path, target_path FROM imports WHERE target_path IS NOT NULL")
|
|
960
|
+
.all()
|
|
961
|
+
.filter((row) => pathAllowed(row.file_path) && pathAllowed(row.target_path));
|
|
845
962
|
const graphRows = personalizedPageRank(importRows, graphSeeds).slice(0, Math.max(limit * 3, 30));
|
|
846
963
|
addRankedList(scores, graphRows, "graph", 1);
|
|
847
964
|
|
|
848
|
-
const ranked = [...scores.values()]
|
|
965
|
+
const ranked = [...scores.values()]
|
|
966
|
+
.filter((candidate) => (
|
|
967
|
+
shouldIndexPath(candidate.path, { excludePatterns })
|
|
968
|
+
&& projectFileInfo(cwd, candidate.path, projectRoot, directoryCache)
|
|
969
|
+
))
|
|
970
|
+
.sort((left, right) => right.score - left.score)
|
|
971
|
+
.slice(0, limit);
|
|
849
972
|
const symbolLookup = db.prepare("SELECT name, kind, line, end_line, signature FROM symbols WHERE file_path = ? ORDER BY line LIMIT 12");
|
|
850
973
|
const results = ranked.map((candidate) => {
|
|
851
974
|
const matchingSymbols = candidate.rows
|
|
@@ -897,9 +1020,15 @@ function lineRangesForFile(text, result, terms, includeCode) {
|
|
|
897
1020
|
}
|
|
898
1021
|
|
|
899
1022
|
export async function buildContextPack(cwd, query, options = {}) {
|
|
1023
|
+
const excludePatterns = requireExplicitExcludePatterns(options, "buildContextPack");
|
|
900
1024
|
const budgetTokens = clampInteger(options.budgetTokens, DEFAULT_PACK_TOKENS, 200, 50_000);
|
|
901
1025
|
const includeCode = options.includeCode !== false;
|
|
902
|
-
const
|
|
1026
|
+
const projectRoot = fs.realpathSync.native(cwd);
|
|
1027
|
+
const directoryCache = new Map();
|
|
1028
|
+
const search = await searchContextIndexV2(cwd, query, {
|
|
1029
|
+
limit: clampInteger(options.limit, 16, 1, 50),
|
|
1030
|
+
excludePatterns
|
|
1031
|
+
});
|
|
903
1032
|
const selected = [];
|
|
904
1033
|
const mapBudget = Math.min(1_200, Math.max(40, Math.trunc(budgetTokens * 0.2)));
|
|
905
1034
|
const mapLines = [];
|
|
@@ -935,7 +1064,9 @@ export async function buildContextPack(cwd, query, options = {}) {
|
|
|
935
1064
|
for (const result of search.results) {
|
|
936
1065
|
let text;
|
|
937
1066
|
try {
|
|
938
|
-
|
|
1067
|
+
const file = projectFileInfo(cwd, result.path, projectRoot, directoryCache);
|
|
1068
|
+
if (!file) continue;
|
|
1069
|
+
text = fs.readFileSync(file.absolute, "utf8");
|
|
939
1070
|
} catch {
|
|
940
1071
|
continue;
|
|
941
1072
|
}
|
|
@@ -979,7 +1110,10 @@ export async function buildContextPack(cwd, query, options = {}) {
|
|
|
979
1110
|
}
|
|
980
1111
|
|
|
981
1112
|
export async function buildTestImpact(cwd, changedFiles = [], options = {}) {
|
|
982
|
-
const
|
|
1113
|
+
const excludePatterns = requireExplicitExcludePatterns(options, "buildTestImpact");
|
|
1114
|
+
const projectRoot = fs.realpathSync.native(cwd);
|
|
1115
|
+
const directoryCache = new Map();
|
|
1116
|
+
const status = await contextIndexV2Status(cwd, { excludePatterns });
|
|
983
1117
|
const normalizedChanged = [...new Set((changedFiles.length > 0 ? changedFiles : gitChangedFiles(cwd)).map(normalizeRelative).filter(Boolean))];
|
|
984
1118
|
if (!status.exists) return { changedFiles: normalizedChanged, impactedFiles: [], tests: [], status };
|
|
985
1119
|
const db = await openDatabase(cwd, { create: false });
|
|
@@ -993,13 +1127,22 @@ export async function buildTestImpact(cwd, changedFiles = [], options = {}) {
|
|
|
993
1127
|
const current = queue.shift();
|
|
994
1128
|
if (current.depth >= maxDepth) continue;
|
|
995
1129
|
for (const row of reverseStatement.all(current.filePath)) {
|
|
1130
|
+
if (
|
|
1131
|
+
!shouldIndexPath(row.file_path, { excludePatterns })
|
|
1132
|
+
|| !projectFileInfo(cwd, row.file_path, projectRoot, directoryCache)
|
|
1133
|
+
) continue;
|
|
996
1134
|
if (seen.has(row.file_path)) continue;
|
|
997
1135
|
seen.add(row.file_path);
|
|
998
1136
|
impacted.push({ path: row.file_path, via: current.filePath, depth: current.depth + 1 });
|
|
999
1137
|
queue.push({ filePath: row.file_path, depth: current.depth + 1 });
|
|
1000
1138
|
}
|
|
1001
1139
|
}
|
|
1002
|
-
const allFiles = db.prepare("SELECT path FROM files").all()
|
|
1140
|
+
const allFiles = db.prepare("SELECT path FROM files").all()
|
|
1141
|
+
.map((row) => row.path)
|
|
1142
|
+
.filter((filePath) => (
|
|
1143
|
+
shouldIndexPath(filePath, { excludePatterns })
|
|
1144
|
+
&& projectFileInfo(cwd, filePath, projectRoot, directoryCache)
|
|
1145
|
+
));
|
|
1003
1146
|
const stemTerms = normalizedChanged.flatMap((filePath) => {
|
|
1004
1147
|
const basename = path.posix.basename(filePath, path.posix.extname(filePath)).replace(/\.(?:test|spec)$/, "");
|
|
1005
1148
|
return basename.split(/[-_.]/).filter((term) => term.length >= 3);
|