code-guardian-ai 0.0.0-stage → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/.env.example +24 -0
  2. package/LICENSE +21 -0
  3. package/README.md +774 -2
  4. package/bin/cli.js +2 -0
  5. package/dist/build-manifest.json +136 -0
  6. package/dist/cli/commands/check.js +56 -0
  7. package/dist/cli/commands/doctor.js +55 -0
  8. package/dist/cli/commands/find.js +31 -0
  9. package/dist/cli/commands/init-files.js +13 -0
  10. package/dist/cli/commands/init.js +50 -0
  11. package/dist/cli/commands/optimize.js +53 -0
  12. package/dist/cli/commands/scan.js +47 -0
  13. package/dist/cli/index.js +62 -0
  14. package/dist/cli/output.js +4 -0
  15. package/dist/context.js +11 -0
  16. package/dist/core/ast/analysis.js +455 -0
  17. package/dist/core/ast/cached-info.js +46 -0
  18. package/dist/core/ast/discovery.js +485 -0
  19. package/dist/core/ast/extract-js.js +531 -0
  20. package/dist/core/ast/extract-py.js +298 -0
  21. package/dist/core/ast/graph.js +92 -0
  22. package/dist/core/ast/health.js +338 -0
  23. package/dist/core/ast/indexer.js +286 -0
  24. package/dist/core/ast/packages.js +343 -0
  25. package/dist/core/ast/parser.js +91 -0
  26. package/dist/core/ast/report.js +154 -0
  27. package/dist/core/ast/resolver.js +392 -0
  28. package/dist/core/ast/rules.js +37 -0
  29. package/dist/core/ast/scope-js.js +191 -0
  30. package/dist/core/ast/shared-index.js +131 -0
  31. package/dist/core/ast/tsconfig.js +301 -0
  32. package/dist/core/ast/types.js +1 -0
  33. package/dist/core/ast/util.js +156 -0
  34. package/dist/core/git/git.js +430 -0
  35. package/dist/core/llm/policy.js +35 -0
  36. package/dist/core/llm/provider.js +199 -0
  37. package/dist/core/prompt/compressor.js +118 -0
  38. package/dist/core/prompt/criteria.js +36 -0
  39. package/dist/core/prompt/find.js +47 -0
  40. package/dist/core/prompt/guidelines.js +22 -0
  41. package/dist/core/prompt/model-output.js +178 -0
  42. package/dist/core/prompt/refiner.js +194 -0
  43. package/dist/core/prompt/relevance.js +265 -0
  44. package/dist/core/prompt/untrusted.js +57 -0
  45. package/dist/core/security/masker.js +330 -0
  46. package/dist/core/security/outbound.js +40 -0
  47. package/dist/core/security/safe-file.js +396 -0
  48. package/dist/core/security/sandbox.js +181 -0
  49. package/dist/core/validation/baseline.js +121 -0
  50. package/dist/core/validation/delta.js +243 -0
  51. package/dist/core/validation/policy.js +123 -0
  52. package/dist/db/cache-key.js +221 -0
  53. package/dist/db/client.js +563 -0
  54. package/dist/db/schema-check.js +149 -0
  55. package/dist/db/schema.js +36 -0
  56. package/dist/db/windows-acl.js +150 -0
  57. package/dist/index.js +55 -0
  58. package/dist/mcp/server.js +20 -0
  59. package/dist/mcp/tools/find-reusable.js +22 -0
  60. package/dist/mcp/tools/get-context.js +39 -0
  61. package/dist/mcp/tools/optimize-prompt.js +31 -0
  62. package/dist/mcp/tools/result.js +31 -0
  63. package/dist/mcp/tools/validate-rules.js +28 -0
  64. package/dist/utils/config.js +117 -0
  65. package/dist/utils/errors.js +9 -0
  66. package/dist/utils/logger.js +19 -0
  67. package/dist/utils/pace.js +19 -0
  68. package/dist/utils/text.js +22 -0
  69. package/dist/utils/tokens.js +14 -0
  70. package/package.json +69 -4
@@ -0,0 +1,338 @@
1
+ const MAX_LINES = 5;
2
+ const MAX_VISITED = 5000;
3
+ // Per-file budgets for the syntax tree. Every extraction pass walks the whole tree, and several walk up the parent chain per
4
+ // reference, so cost grows with node count and with depth. The node budget sits well above the largest allowed source file
5
+ // (maxFileKb tops out at 2 MB, roughly one node per 2-3 bytes even when minified); real code is nowhere near the depth budget
6
+ // (long else-if and string-concatenation chains in generated files reach a few hundred). A file beyond either is reported as
7
+ // partial instead of analysed.
8
+ export const MAX_TREE_NODES = 750_000;
9
+ export const MAX_TREE_DEPTH = 1_000;
10
+ // Counts nodes and measures depth with an explicit stack, so a hostile file cannot exhaust the JavaScript call stack, and
11
+ // stops the moment a budget is crossed so the cost of the check itself is bounded. Returns the budget that was exceeded.
12
+ export function exceedsBudget(root, budget = { nodes: MAX_TREE_NODES, depth: MAX_TREE_DEPTH }) {
13
+ const stack = [{ node: root, depth: 1 }];
14
+ let nodes = 0;
15
+ for (let entry = stack.pop(); entry; entry = stack.pop()) {
16
+ if (++nodes > budget.nodes)
17
+ return 'nodes';
18
+ if (entry.depth > budget.depth)
19
+ return 'depth';
20
+ for (const child of entry.node.children)
21
+ if (child)
22
+ stack.push({ node: child, depth: entry.depth + 1 });
23
+ }
24
+ return undefined;
25
+ }
26
+ const STRUCTURAL = new Set(['import_statement', 'import_from_statement', 'future_import_statement', 'export_statement', 'import_require_clause']);
27
+ function flag(node, key) {
28
+ const value = node[key];
29
+ return typeof value === 'function' ? Boolean(value.call(node)) : Boolean(value);
30
+ }
31
+ export const hasParseError = (node) => (node ? flag(node, 'hasError') : false);
32
+ // Bodies of declarations: damage in one of these is contained, because the declaration's name, kind and position are
33
+ // decided before the body starts. `export function f() { <error> }` is still an exported function f.
34
+ const FUNCTION_LIKE = new Set([
35
+ 'function_declaration', 'function_expression', 'function', 'generator_function_declaration', 'generator_function',
36
+ 'arrow_function', 'method_definition', 'class_static_block', 'function_definition', 'class_definition',
37
+ ]);
38
+ const BODY_OF_CLASS_LIKE = new Set(['class_body', 'interface_body', 'enum_body']);
39
+ function isDeclarationBody(node) {
40
+ if (BODY_OF_CLASS_LIKE.has(node.type))
41
+ return true;
42
+ // A TypeScript namespace or `declare module` body is also a statement_block, but its parent is not function-like and
43
+ // legitimately holds exports, so it is not a declaration body.
44
+ return (node.type === 'statement_block' || node.type === 'block') && node.parent !== null && FUNCTION_LIKE.has(node.parent.type);
45
+ }
46
+ // An unclosed brace makes the parser keep reading the following source inside the damaged body. An export or import
47
+ // statement can never be valid inside a function or class body, so finding one there means declarations that really sit
48
+ // at the top level were absorbed into the body, and the damage is not contained after all. This is the JavaScript and
49
+ // TypeScript rule; Python allows imports inside functions and is judged by indentation instead (below).
50
+ function absorbsTopLevel(body) {
51
+ for (const child of body.children) {
52
+ if (!child)
53
+ continue;
54
+ if (STRUCTURAL.has(child.type))
55
+ return true;
56
+ if (child.type === 'ERROR') {
57
+ for (const inner of child.children)
58
+ if (inner && (STRUCTURAL.has(inner.type) || inner.type === 'export' || inner.type === 'import'))
59
+ return true;
60
+ }
61
+ }
62
+ return false;
63
+ }
64
+ // Python has no braces: what belongs to a body is decided by indentation, and a malformed expression (an unclosed parenthesis,
65
+ // a stray operator) can make the parser keep reading into the lines after it. Each body is judged against the column of the
66
+ // declaration that owns it: a function, class or import statement that starts at or to the left of that column can never belong to
67
+ // the body, so finding one inside means a later sibling was absorbed. For a method of a class that is the method's own `def`
68
+ // column, so a following method (same column, inside the same class) is caught; for a module-level function it is column 0. Imports
69
+ // and nested functions that are indented deeper than the owner are legitimate and are not matched.
70
+ const PYTHON_STATEMENT_START = new Set([
71
+ 'function_definition', 'class_definition', 'decorated_definition', 'import_statement', 'import_from_statement', 'future_import_statement',
72
+ 'def', 'class', 'import', 'from', '@',
73
+ ]);
74
+ const startsStatementAtOrLeftOf = (n, column) => PYTHON_STATEMENT_START.has(n.type) && n.startPosition.column !== undefined && n.startPosition.column <= column;
75
+ function absorbsPythonSibling(body, limitColumn) {
76
+ for (const child of body.children) {
77
+ if (!child)
78
+ continue;
79
+ if (startsStatementAtOrLeftOf(child, limitColumn))
80
+ return true;
81
+ if (child.type === 'ERROR')
82
+ for (const inner of child.children)
83
+ if (inner && startsStatementAtOrLeftOf(inner, limitColumn))
84
+ return true;
85
+ }
86
+ return false;
87
+ }
88
+ // An ERROR node that starts a statement at some column and then contains a def, class or import that starts on a LATER line at
89
+ // that column or further left has absorbed a sibling statement (the first line is the damaged statement itself, and its
90
+ // continuation lines are indented deeper). This also covers an ERROR placed directly in a class body, which has no body of its own.
91
+ function errorAbsorbsSibling(node) {
92
+ const column = node.startPosition.column;
93
+ if (column === undefined || (node.type !== 'ERROR' && !flag(node, 'isError')))
94
+ return false;
95
+ return node.children.some((inner) => inner !== null && inner.startPosition.row > node.startPosition.row && startsStatementAtOrLeftOf(inner, column));
96
+ }
97
+ // Does the damage at `node` threaten imports, exports or top-level declarations?
98
+ // - directly under the root, or inside an import/export header (specifiers, clause, signature): yes
99
+ // - inside the body of a function, method or class, even one wrapped in `export`: no, unless the body absorbed top-level code
100
+ function damagesStructure(node, lang) {
101
+ if (!node.parent || !node.parent.parent)
102
+ return true;
103
+ // Every block between the damage and the outermost declaration body that holds it. Swallowed top-level code lands in
104
+ // whichever block was still open, which may be a nested one, so all of them are inspected.
105
+ const blocks = [];
106
+ let outermost = -1;
107
+ for (let current = node.parent; current; current = current.parent) {
108
+ if (isDeclarationBody(current)) {
109
+ blocks.push(current);
110
+ outermost = blocks.length - 1;
111
+ continue;
112
+ }
113
+ // Only an import/export reached before any body is structural; one above a body is just the wrapper of the declaration.
114
+ if (outermost < 0 && STRUCTURAL.has(current.type))
115
+ return true;
116
+ if (outermost < 0 && (current.type === 'statement_block' || current.type === 'block'))
117
+ blocks.push(current);
118
+ }
119
+ if (outermost < 0) {
120
+ return lang === 'python' || errorAbsorbsSibling(node);
121
+ }
122
+ const inspected = blocks.slice(0, outermost + 1);
123
+ if (lang === 'python') {
124
+ // Every block is judged against its own owner (a method's body against the method, not against the class around it).
125
+ const swallowed = inspected.some((b) => {
126
+ const owner = b.parent?.startPosition.column;
127
+ return owner !== undefined && absorbsPythonSibling(b, owner);
128
+ });
129
+ return swallowed || errorAbsorbsSibling(node);
130
+ }
131
+ return inspected.some(absorbsTopLevel);
132
+ }
133
+ // ---- Python: invalid code the grammar accepts without reporting an error --------------------------------------------------
134
+ //
135
+ // The Python grammar is deliberately forgiving in two places, so a tree can have no ERROR or MISSING node and still describe
136
+ // code that Python itself rejects:
137
+ // - an empty suite: `def f():` or `class A:` with nothing indented under it parses with an empty block;
138
+ // - a trailing comma in an import that is not parenthesized: `import os,` and `from os import path,`.
139
+ // `assessParse` therefore cannot rely on `hasError` alone for Python. This pass looks for exactly these two shapes. A node whose
140
+ // subtree already reports an error is skipped here (the error walk counts it), so nothing is counted twice, and it only descends
141
+ // through statements, so a file costs one walk over its statements, not over every expression.
142
+ const PY_SUITE_OWNERS = new Set(['function_definition', 'class_definition']);
143
+ const PY_IMPORTS = new Set(['import_statement', 'import_from_statement', 'future_import_statement']);
144
+ const PY_CONTAINER = /^(module|block|decorated_definition|function_definition|class_definition|.+_statement|.+_clause)$/;
145
+ const isNamedNode = (node) => {
146
+ const value = node.isNamed;
147
+ return typeof value === 'function' ? Boolean(value.call(node)) : value !== false;
148
+ };
149
+ // A body counts as present when it holds at least one statement. Comments are extras and do not make a suite valid; a valid
150
+ // `pass`, `...`, docstring or one-line suite (`def f(): pass`) is a statement and does.
151
+ function hasEmptyBody(owner) {
152
+ const body = typeof owner.childForFieldName === 'function' ? owner.childForFieldName('body') : (owner.children.find((c) => c?.type === 'block') ?? null);
153
+ if (!body)
154
+ return true;
155
+ return !body.children.some((child) => child !== null && child.type !== 'comment' && isNamedNode(child));
156
+ }
157
+ // `import a,` and `from a import b,` end in a comma token; the parenthesized form ends in ")" and may legally have one inside.
158
+ function endsWithComma(statement) {
159
+ for (let i = statement.children.length - 1; i >= 0; i--) {
160
+ const child = statement.children[i];
161
+ if (!child || child.type === 'comment')
162
+ continue;
163
+ return child.type === ',';
164
+ }
165
+ return false;
166
+ }
167
+ function pythonStructure(root) {
168
+ const found = [];
169
+ const stack = [root];
170
+ while (stack.length > 0) {
171
+ const node = stack.pop();
172
+ if (!flag(node, 'hasError')) {
173
+ const line = node.startPosition.row + 1;
174
+ if (PY_SUITE_OWNERS.has(node.type) && hasEmptyBody(node))
175
+ found.push({ line, kind: 'empty-body' });
176
+ else if (PY_IMPORTS.has(node.type) && endsWithComma(node))
177
+ found.push({ line, kind: 'trailing-comma' });
178
+ }
179
+ for (const child of node.children)
180
+ if (child && PY_CONTAINER.test(child.type))
181
+ stack.push(child);
182
+ }
183
+ return found;
184
+ }
185
+ // Python indentation can be invalid even when Tree-sitter reports a clean tree.
186
+ // Check only direct statements in modules and suites, so multiline expression
187
+ // continuations are not mistaken for indentation errors.
188
+ function pythonIndentation(root, source) {
189
+ const found = [];
190
+ const lines = source.split(/\r?\n/);
191
+ const indentationAt = (row) => {
192
+ const line = lines[row];
193
+ if (line === undefined || line.trim().length === 0)
194
+ return undefined;
195
+ const prefix = line.match(/^[\t ]*/)?.[0] ?? '';
196
+ let visual = 0;
197
+ let alternate = 0;
198
+ for (const char of prefix) {
199
+ if (char === '\t') {
200
+ visual += 8 - (visual % 8);
201
+ alternate += 1;
202
+ }
203
+ else {
204
+ visual += 1;
205
+ alternate += 1;
206
+ }
207
+ }
208
+ return { visual, alternate };
209
+ };
210
+ const addIssue = (node) => {
211
+ const line = node.startPosition.row + 1;
212
+ if (!found.some((issue) => issue.line === line)) {
213
+ found.push({ line, kind: 'indentation' });
214
+ }
215
+ };
216
+ const statements = (node) => node.children.filter((child) => child !== null &&
217
+ child.type !== 'comment' &&
218
+ isNamedNode(child));
219
+ // Statements directly inside a Python module must start at column zero.
220
+ if (root.type === 'module') {
221
+ for (const statement of statements(root)) {
222
+ const indent = indentationAt(statement.startPosition.row);
223
+ if (indent && (indent.visual !== 0 || indent.alternate !== 0)) {
224
+ addIssue(statement);
225
+ }
226
+ }
227
+ }
228
+ // Statements inside one suite must use a consistent indentation level.
229
+ // Compare both normal tab expansion (8 columns) and alternate tab expansion
230
+ // (1 column) to catch mixed tabs/spaces that look visually aligned.
231
+ const stack = [root];
232
+ while (stack.length > 0) {
233
+ const node = stack.pop();
234
+ if (node.type === 'block') {
235
+ const owner = node.parent;
236
+ const ownerIndent = owner
237
+ ? indentationAt(owner.startPosition.row)
238
+ : undefined;
239
+ let expected;
240
+ for (const statement of statements(node)) {
241
+ // A one-line suite is legal and does not introduce a new indentation level.
242
+ if (owner && statement.startPosition.row <= owner.startPosition.row) {
243
+ continue;
244
+ }
245
+ const indent = indentationAt(statement.startPosition.row);
246
+ if (!indent)
247
+ continue;
248
+ if (!expected) {
249
+ expected = indent;
250
+ if (ownerIndent &&
251
+ (indent.visual <= ownerIndent.visual ||
252
+ indent.alternate <= ownerIndent.alternate)) {
253
+ addIssue(statement);
254
+ }
255
+ continue;
256
+ }
257
+ if (indent.visual !== expected.visual ||
258
+ indent.alternate !== expected.alternate) {
259
+ addIssue(statement);
260
+ }
261
+ }
262
+ }
263
+ for (const child of node.children) {
264
+ if (child)
265
+ stack.push(child);
266
+ }
267
+ }
268
+ return found;
269
+ }
270
+ // Looks for ERROR and MISSING nodes below `root`, descending only into subtrees that contain one.
271
+ // Returns undefined for a clean tree.
272
+ export function assessParse(root, lang, source = '') {
273
+ // Python can be invalid without the tree saying so (see above), so this pass runs whether or not the root reports an error.
274
+ const structure = lang === 'python'
275
+ ? [...pythonStructure(root), ...pythonIndentation(root, source)]
276
+ : [];
277
+ const reported = flag(root, 'hasError');
278
+ if (!reported && structure.length === 0)
279
+ return undefined;
280
+ let errors = 0;
281
+ let missing = 0;
282
+ let visited = 0;
283
+ let structural = false;
284
+ const lines = [];
285
+ // Explicit stack: error-bearing subtrees can be as deep as the file allows, and the call stack must not be what limits that.
286
+ const pending = reported ? [root] : [];
287
+ for (let node = pending.pop(); node; node = pending.pop()) {
288
+ if (++visited > MAX_VISITED) {
289
+ structural = true;
290
+ break;
291
+ }
292
+ const isMissing = flag(node, 'isMissing');
293
+ const isError = node.type === 'ERROR' || flag(node, 'isError');
294
+ if (isMissing || isError) {
295
+ if (isMissing)
296
+ missing++;
297
+ else
298
+ errors++;
299
+ const line = node.startPosition.row + 1;
300
+ if (lines.length < MAX_LINES && !lines.includes(line))
301
+ lines.push(line);
302
+ if (damagesStructure(node, lang))
303
+ structural = true;
304
+ continue;
305
+ }
306
+ // Pushed in reverse so subtrees are still visited in source order.
307
+ for (let i = node.children.length - 1; i >= 0; i--) {
308
+ const child = node.children[i];
309
+ if (child && flag(child, 'hasError'))
310
+ pending.push(child);
311
+ }
312
+ }
313
+ // A malformed import is not a dependency to rely on: the file's imports are degraded. An empty suite leaves the declaration
314
+ // itself (its name, kind and position) intact, so it is only recovered.
315
+ if (structure.some((issue) => issue.kind === 'trailing-comma' ||
316
+ issue.kind === 'indentation')) {
317
+ structural = true;
318
+ }
319
+ for (const issue of structure)
320
+ if (!lines.includes(issue.line))
321
+ lines.push(issue.line);
322
+ const total = errors + structure.length;
323
+ // A root that reports an error but yields no ERROR or MISSING node cannot be judged: treat it as degraded.
324
+ const found = reported ? errors + missing > 0 : true;
325
+ return { status: !found || structural ? 'degraded' : 'recovered', errors: total, missing, lines: lines.sort((a, b) => a - b).slice(0, MAX_LINES) };
326
+ }
327
+ export function describeHealth(health) {
328
+ if (health.limit) {
329
+ const what = health.limit === 'depth' ? `nested deeper than ${MAX_TREE_DEPTH} levels` : `larger than ${MAX_TREE_NODES} syntax nodes`;
330
+ return `the file is ${what}, so it was not analysed and nothing was extracted from it`;
331
+ }
332
+ const parts = [health.errors > 0 ? `${health.errors} syntax error(s)` : '', health.missing > 0 ? `${health.missing} missing token(s)` : ''].filter(Boolean);
333
+ const where = health.lines.length > 0 ? ` near line ${health.lines.join(', ')}` : '';
334
+ const effect = health.status === 'degraded'
335
+ ? 'imports, exports or declarations may be missing from the extraction'
336
+ : 'declarations were extracted, but code inside the damaged region may be missing';
337
+ return `${parts.join(' and ') || 'parse problems'}${where}: ${effect}`;
338
+ }
@@ -0,0 +1,286 @@
1
+ import { createHash } from 'node:crypto';
2
+ import path from 'node:path';
3
+ import { performance } from 'node:perf_hooks';
4
+ import ignore from 'ignore';
5
+ import { applyCacheWrites, loadCache } from '../../db/client.js';
6
+ import { parseCachedInfo } from './cached-info.js';
7
+ import { RESOURCE_CEILINGS, effectiveLimit } from '../../utils/config.js';
8
+ import { errorMessage } from '../../utils/errors.js';
9
+ import { log } from '../../utils/logger.js';
10
+ import { compareText } from '../../utils/text.js';
11
+ import { isSensitivePath, readInside, readOptionalInside } from '../security/sandbox.js';
12
+ import { createPacer } from '../../utils/pace.js';
13
+ import { FOREIGN_SOURCE, GENERATED, languageOf, walk } from './discovery.js';
14
+ import { describeHealth } from './health.js';
15
+ import { loadLanguages, parseFile } from './parser.js';
16
+ import { loadPackages, pythonConfigProblems } from './packages.js';
17
+ import { TSCONFIG_NAME, diskReader, loadTsConfigs } from './tsconfig.js';
18
+ export { FOREIGN_SOURCE, languageOf };
19
+ const DEFAULT_IGNORES = [
20
+ 'node_modules/', '.git/', '.guardian/', 'dist/', 'build/', 'out/', 'coverage/', '.next/', '.nuxt/',
21
+ '.turbo/', '.cache/', '.venv/', 'venv/', '__pycache__/', '.tox/', '.mypy_cache/', '.pytest_cache/',
22
+ '*.egg-info/', 'target/', 'vendor/',
23
+ ];
24
+ // The language of a file the indexer would analyse by name alone: generated and credential-like files never qualify.
25
+ export function sourceLanguage(rel) {
26
+ const name = path.posix.basename(rel);
27
+ if (GENERATED.test(name) || isSensitivePath(rel) || TSCONFIG_NAME.test(name))
28
+ return undefined;
29
+ return languageOf(name);
30
+ }
31
+ export const buildIgnoreFilter = (gitignore, configIgnore) => ignore().add(DEFAULT_IGNORES).add(gitignore).add([...configIgnore]);
32
+ export const contentHash = (data) => createHash('sha256').update(data).digest('hex');
33
+ const MAX_GITIGNORE_BYTES = 256 * 1024;
34
+ export function readGitignoreState(root) {
35
+ try {
36
+ const data = readOptionalInside(root, '.gitignore', MAX_GITIGNORE_BYTES, { rejectHardLinks: true });
37
+ return { text: data ? data.toString('utf8') : '', problem: null };
38
+ }
39
+ catch (err) {
40
+ return { text: '', problem: `.gitignore was not applied (${errorMessage(err)})` };
41
+ }
42
+ }
43
+ // The text of the .gitignore, or '' when there is none or it cannot be used (readGitignoreState says which).
44
+ export function readGitignore(root) {
45
+ return readGitignoreState(root).text;
46
+ }
47
+ function fromCache(raw) {
48
+ try {
49
+ const info = JSON.parse(raw);
50
+ const valid = [info.symbols, info.imports, info.reexports, info.unusedImports, info.suppressions, info.dynamicLoads].every(Array.isArray) && typeof info.decorated === 'boolean' && typeof info.script === 'boolean';
51
+ return valid ? info : null;
52
+ }
53
+ catch {
54
+ return null;
55
+ }
56
+ }
57
+ // A scan that a too small maxIndexMs keeps stopping can never finish on its own: handling a file means reading and hashing it (the
58
+ // hash is the only validity check, so a cached file is never taken on trust), and a call that cannot get through every file within
59
+ // its budget starts again at the same files next time. So after a call that was stopped by maxIndexMs, the next call for the same
60
+ // workspace in this process gets twice the budget, and so on, up to the same hard ceiling that bounds the setting itself
61
+ // (RESOURCE_CEILINGS.maxIndexMs). Nothing about validity is relaxed: every file is still read, hashed and compared before its cached
62
+ // result is used. The state is a few small numbers, held in memory only, bounded in size, and reset by any call that is not stopped
63
+ // by this limit. A deadline that is already expired (zero or negative) is never extended.
64
+ const MAX_TRACKED_ROOTS = 32;
65
+ const MAX_DOUBLINGS = 20;
66
+ const stoppedStreak = new Map();
67
+ export function indexBudget(root, configured) {
68
+ if (!(configured > 0))
69
+ return configured;
70
+ const doublings = stoppedStreak.get(root) ?? 0;
71
+ return Math.min(configured * 2 ** doublings, Math.max(configured, RESOURCE_CEILINGS.maxIndexMs));
72
+ }
73
+ export function rememberStop(root, stoppedByBudget) {
74
+ const before = stoppedStreak.get(root) ?? 0;
75
+ stoppedStreak.delete(root);
76
+ if (!stoppedByBudget)
77
+ return;
78
+ stoppedStreak.set(root, Math.min(before + 1, MAX_DOUBLINGS));
79
+ while (stoppedStreak.size > MAX_TRACKED_ROOTS) {
80
+ const oldest = stoppedStreak.keys().next();
81
+ if (oldest.done)
82
+ break;
83
+ stoppedStreak.delete(oldest.value);
84
+ }
85
+ }
86
+ // Hands the event loop back between slices of work, so MCP requests keep being served during a large scan.
87
+ // Pass a signal to cancel: the work stops at the next slice, keeps what was already cached, and throws the signal's reason.
88
+ //
89
+ // Time limits and cancellation are cooperative. They are checked at the boundaries between units of work (between
90
+ // directory batches, between entries, between files, between phases), never inside one: a file system call, one
91
+ // Tree-sitter parse, or the load of one configuration file runs to completion before the next check, so a limit can be
92
+ // overrun by the duration of the slowest single unit (the largest file's parse, or one slow stat). Nothing expensive is
93
+ // started once the deadline has passed or the signal has fired, and a stopped scan is always reported as incomplete.
94
+ //
95
+ // What maxIndexMs covers: the time from the end of discovery until the last file has been handled. That includes loading the grammars
96
+ // and the cache (neither can be interrupted, but their time counts against the budget, so a slow start leaves less for files), and,
97
+ // for every file, reading, hashing, the cache lookup and parsing: no file is started and no parse is begun after the deadline.
98
+ // What it does not cover: discovery (limited by maxDiscoveryMs), the first file of a call (always handled, so a call makes progress),
99
+ // and the steps after the last file: writing the cache, loading tsconfig, package and pyproject files (one small read per
100
+ // configuration file), and building the result. Those are bounded by the number of configuration files, not by the number of sources.
101
+ export async function indexProject(ctx, options = {}) {
102
+ const { signal } = options;
103
+ const pacer = createPacer({ signal });
104
+ signal?.throwIfAborted();
105
+ // Resource limits are read once, through the trusted ceilings, so every phase of this call works with the same bounded values.
106
+ const maxFiles = effectiveLimit(ctx.config, 'maxFiles');
107
+ const maxBytes = effectiveLimit(ctx.config, 'maxFileKb') * 1024;
108
+ const maxIndexMs = indexBudget(ctx.root, effectiveLimit(ctx.config, 'maxIndexMs'));
109
+ const gitignore = readGitignoreState(ctx.root);
110
+ const filter = buildIgnoreFilter(gitignore.text, ctx.config.ignore);
111
+ const discovered = await walk({ root: ctx.root, ignores: (rel) => filter.ignores(rel), maxBytes, isTsConfig: (name) => TSCONFIG_NAME.test(name), isSensitive: isSensitivePath }, {
112
+ maxEntries: effectiveLimit(ctx.config, 'maxDiscoveryEntries'),
113
+ maxMs: effectiveLimit(ctx.config, 'maxDiscoveryMs'),
114
+ maxMemoryBytes: effectiveLimit(ctx.config, 'maxDiscoveryMemoryMb') * 1024 * 1024,
115
+ signal,
116
+ });
117
+ // maxIndexMs starts here, when discovery (which has its own limit, maxDiscoveryMs) is over, so it covers loading the grammars and
118
+ // the cache as well as reading, hashing, cache lookups and parsing.
119
+ const indexDeadline = performance.now() + maxIndexMs;
120
+ // Cancelled during discovery or while waiting for it: nothing has been written yet, so the cache is untouched.
121
+ signal?.throwIfAborted();
122
+ const selected = discovered.eligible.slice(0, maxFiles);
123
+ const truncated = discovered.eligible.length > selected.length;
124
+ await loadLanguages(selected.map((f) => f.lang));
125
+ signal?.throwIfAborted();
126
+ const cache = loadCache(ctx.db);
127
+ const writes = [];
128
+ const files = [];
129
+ const skipped = [
130
+ ...discovered.oversized.map((p) => ({ path: p, reason: 'too-large' })),
131
+ ...discovered.unreadableDirs.map((p) => ({ path: p, reason: 'unreadable-directory' })),
132
+ // Could be a source file or a directory: the workspace was not fully discovered, so the scan is partial.
133
+ ...discovered.unclassified.map((p) => ({ path: p, reason: 'unclassified' })),
134
+ // Replaced by a link or another directory while the scan ran: whatever was inside was not used, so the scan is partial.
135
+ ...discovered.replacedDirs.map((p) => ({ path: p, reason: 'directory-replaced' })),
136
+ ];
137
+ let stopped = discovered.stopped;
138
+ // Files fully handled in this call (reused from the cache, parsed, or skipped). Everything after them is `unindexed`.
139
+ let done = 0;
140
+ // Rows are only pruned when the whole workspace was seen; a partial scan must never evict files it did not reach.
141
+ const persist = (prune) => {
142
+ try {
143
+ applyCacheWrites(ctx.db, writes, prune ? new Set(discovered.eligible.map((f) => f.rel)) : undefined);
144
+ }
145
+ catch (err) {
146
+ log.warn(`Cache not updated this run: ${errorMessage(err)}`);
147
+ }
148
+ };
149
+ // Files that are certain to need parsing (no cache row, or a row whose size or mtime differ) go first, in path order, then the rest in
150
+ // path order. size and mtime are only a hint for this ordering, never a validity check: the SHA-256 below decides. The order
151
+ // guarantees that a call stopped by maxIndexMs has spent its time on new work whenever there is any, so repeated calls always finish.
152
+ const stale = (f) => {
153
+ const row = cache.get(f.rel);
154
+ return !row || row.size !== f.size || row.mtime_ms !== f.mtimeMs;
155
+ };
156
+ const order = [...selected.filter(stale), ...selected.filter((f) => !stale(f))];
157
+ try {
158
+ for (const file of order) {
159
+ await pacer.tick();
160
+ // maxIndexMs covers the whole indexing phase: reading, hashing, the cache lookup and parsing. No further file is started once
161
+ // it has passed. The first file of every call is always handled, so a call never ends without having made progress. A read or
162
+ // a parse that has already started is not interrupted.
163
+ if (done > 0 && performance.now() > indexDeadline) {
164
+ stopped ??= 'index-time-limit';
165
+ break;
166
+ }
167
+ try {
168
+ const data = readInside(ctx.root, file.rel, maxBytes);
169
+ const hash = contentHash(data);
170
+ const cached = cache.get(file.rel);
171
+ // A cached result is reused only when its signature verified in loadCache and every nested field checks out here.
172
+ const reusable = cached && cached.hash === hash ? parseCachedInfo(cached.info, file.rel) : null;
173
+ if (reusable) {
174
+ if (cached && (cached.size !== file.size || cached.mtime_ms !== file.mtimeMs)) {
175
+ writes.push({ kind: 'touch', path: file.rel, size: file.size, mtime_ms: file.mtimeMs });
176
+ }
177
+ files.push(reusable);
178
+ done++;
179
+ continue;
180
+ }
181
+ // The file was read before the deadline; parsing it is the most expensive step, so it is not started after the deadline.
182
+ if (done > 0 && performance.now() > indexDeadline) {
183
+ stopped ??= 'index-time-limit';
184
+ break;
185
+ }
186
+ const info = parseFile(file.rel, file.lang, data.toString('utf8').replace(/^\uFEFF/, ''));
187
+ if (!info) {
188
+ skipped.push({ path: file.rel, reason: 'unparseable' });
189
+ done++;
190
+ continue;
191
+ }
192
+ const serialized = JSON.stringify(info);
193
+ if (typeof serialized !== 'string') {
194
+ skipped.push({ path: file.rel, reason: 'unparseable' });
195
+ done++;
196
+ continue;
197
+ }
198
+ const normalized = fromCache(serialized);
199
+ if (!normalized) {
200
+ skipped.push({ path: file.rel, reason: 'unparseable' });
201
+ done++;
202
+ continue;
203
+ }
204
+ writes.push({
205
+ kind: 'save',
206
+ row: {
207
+ path: file.rel,
208
+ hash,
209
+ size: file.size,
210
+ mtime_ms: file.mtimeMs,
211
+ info: serialized,
212
+ },
213
+ });
214
+ files.push(normalized);
215
+ done++;
216
+ }
217
+ catch (err) {
218
+ skipped.push({ path: file.rel, reason: 'unreadable' });
219
+ done++;
220
+ log.debug(`Skipping ${file.rel}: ${errorMessage(err)}`);
221
+ }
222
+ }
223
+ }
224
+ catch (err) {
225
+ // Cancelled mid-scan: keep the files already parsed so the next call does not redo them.
226
+ persist(false);
227
+ throw err;
228
+ }
229
+ persist(!discovered.stopped);
230
+ rememberStop(ctx.root, stopped === 'index-time-limit');
231
+ // The cache is saved; a request cancelled during the loop's last slice does not go on to load configuration files.
232
+ signal?.throwIfAborted();
233
+ // Files were handled stale-first; everything downstream sees them in path order, whatever order they were processed in.
234
+ files.sort((a, b) => compareText(a.path, b.path));
235
+ const unindexed = selected.length - done;
236
+ const reader = diskReader(ctx.root);
237
+ const tsconfigs = loadTsConfigs(reader, discovered.configs);
238
+ const packages = loadPackages(reader, discovered.manifests);
239
+ // A manifest or tsconfig that exists but did not load (unreadable, invalid JSON, an `extends` that cannot be found) means
240
+ // aliases, workspace packages, entry points or framework signals may be missing, so the analysis cannot claim to be complete.
241
+ const loadedConfigs = new Set([...tsconfigs.map((c) => c.file), ...packages.map((p) => p.file)]);
242
+ const configProblems = [...discovered.configs, ...discovered.manifests.filter((m) => m.endsWith('package.json'))]
243
+ .filter((file) => !loadedConfigs.has(file))
244
+ .sort(compareText)
245
+ .map((file) => `${file} could not be read or parsed`);
246
+ // Ignore rules that could not be applied change which files were analysed: that is a problem of the scan, not a note.
247
+ if (gitignore.problem)
248
+ configProblems.push(gitignore.problem);
249
+ // pyproject.toml (and the legacy setup files) only matter for entry-point detection, which only matters with Python source.
250
+ if (discovered.eligible.some((f) => f.lang === 'python')) {
251
+ configProblems.push(...pythonConfigProblems(reader, { pyproject: discovered.manifests.filter((m) => m.endsWith('pyproject.toml')), legacy: discovered.legacy }));
252
+ }
253
+ for (const config of tsconfigs)
254
+ if (config.incomplete)
255
+ configProblems.push(`${config.file} extends a configuration file that could not be loaded`);
256
+ const parseIssues = files
257
+ .flatMap((f) => (f.parse ? [{ path: f.path, ...f.parse }] : []))
258
+ .sort((a, b) => compareText(a.path, b.path));
259
+ const complete = !truncated && !stopped && skipped.length === 0 && configProblems.length === 0 && parseIssues.length === 0;
260
+ if (truncated)
261
+ log.warn(`Indexed ${selected.length} of ${discovered.eligible.length} eligible files (sorted by path); raise maxFiles in .guardianrc to see more`);
262
+ if (stopped)
263
+ log.warn(`Scan stopped early (${stopped}); the analysis is partial. Raise maxDiscoveryEntries, maxDiscoveryMemoryMb, maxDiscoveryMs or maxIndexMs in .guardianrc, or run the scan again to continue from the cache`);
264
+ if (skipped.length > 0)
265
+ log.info(`${skipped.length} file(s) were skipped, analysis is partial`);
266
+ if (parseIssues.length > 0)
267
+ log.warn(`${parseIssues.length} file(s) have syntax errors, e.g. ${parseIssues[0].path}: ${describeHealth(parseIssues[0])}`);
268
+ if (configProblems.length > 0)
269
+ log.warn(`Analysis is partial: ${configProblems.join('; ')}`);
270
+ return {
271
+ root: ctx.root,
272
+ files,
273
+ tsconfigs,
274
+ packages,
275
+ foreignExtensions: [...discovered.foreign].sort(),
276
+ knownPaths: discovered.eligible.map((f) => f.rel),
277
+ eligible: discovered.eligible.length,
278
+ truncated,
279
+ skipped: skipped.sort((a, b) => compareText(a.path, b.path)),
280
+ configProblems,
281
+ parseIssues,
282
+ stopped,
283
+ unindexed,
284
+ complete,
285
+ };
286
+ }