llmnav 0.6.1 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +2 -2
- package/docs/api.md +1 -1
- package/docs/architecture.md +2 -2
- package/docs/publishing.md +2 -2
- package/docs/quickstart.md +1 -1
- package/docs/spec.md +4 -0
- package/package.json +1 -1
- package/src/audit.js +1 -1
- package/src/boundaries.js +15 -3
- package/src/declaration.js +62 -14
- package/src/generator.js +1 -1
- package/src/incremental.js +14 -6
- package/src/index.d.ts +1 -1
- package/src/initializer.js +18 -4
- package/src/parser.js +48 -19
- package/src/project.js +3 -2
- package/src/prompt-bundle.js +7 -3
- package/src/registry.js +8 -2
- package/src/search.js +15 -6
- package/src/spec.js +1 -1
- package/src/transaction.js +9 -0
- package/src/util.js +19 -4
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,33 @@ The npm package follows Semantic Versioning. The `llmnav/N` source protocol is v
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.6.4] — 2026-08-11
|
|
10
|
+
|
|
11
|
+
### Fixed
|
|
12
|
+
|
|
13
|
+
* Rejected symlink and junction traversal across search caches, incremental state, and public registry reads and writes.
|
|
14
|
+
* Enforced parser card limits before materialization and replaced repeated interval and line-number scans with bounded indexes.
|
|
15
|
+
* Bounded aggregate declaration extraction work per source file, including malformed declarations and whitespace lookahead.
|
|
16
|
+
|
|
17
|
+
## [0.6.3] — 2026-08-11
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
|
|
21
|
+
* Rejected nested symlink and junction traversal before managed cache, state, schema, and prompt-bundle filesystem operations.
|
|
22
|
+
* Replaced per-marker lexical rescans with a bounded single-pass parser classification.
|
|
23
|
+
* Required npm release tags to resolve to commits on the protected `origin/main` lineage.
|
|
24
|
+
|
|
25
|
+
## [0.6.2] — 2026-08-11
|
|
26
|
+
|
|
27
|
+
### Added
|
|
28
|
+
|
|
29
|
+
* Added generated boundary signals for persistent dotted JSON filename protocols, versioned schema literals, and TypeScript Tauri invoke adapters.
|
|
30
|
+
|
|
31
|
+
### Fixed
|
|
32
|
+
|
|
33
|
+
* Stopped classifying high-fan-in domain contract hubs as broad utilities solely because they export many declarations.
|
|
34
|
+
* Made tag-driven publication create an idempotent GitHub Release after npm integrity verification or publication succeeds.
|
|
35
|
+
|
|
9
36
|
## [0.6.1] — 2026-08-10
|
|
10
37
|
|
|
11
38
|
### Added
|
package/README.md
CHANGED
|
@@ -18,7 +18,7 @@ LLMNav is not a documentation generator, an embedding database, or a reason to a
|
|
|
18
18
|
* Machine-readable changed-card, affected-boundary, and affected-catalog output
|
|
19
19
|
* Exported API and effective configuration contract fingerprints
|
|
20
20
|
* TypeScript and Go declaration enrichment with declaration-level body hashes
|
|
21
|
-
* Generated route, event, schema, migration, and command boundaries
|
|
21
|
+
* Generated artifact, route, event, schema, migration, runtime, and command boundaries
|
|
22
22
|
* SARIF 2.1.0 diagnostic output
|
|
23
23
|
* Optional deterministic card-range search shards for very large repositories
|
|
24
24
|
* Strict repository-local imports for generated definition and reference indexes
|
|
@@ -250,7 +250,7 @@ Do not annotate trivial getters, generated files, obvious wrappers, every test f
|
|
|
250
250
|
|
|
251
251
|
## Current implementation boundary
|
|
252
252
|
|
|
253
|
-
Version 0.6 adds a deterministic coverage audit that prioritizes root and nested-package entrypoints, structural boundaries, Rust/Tauri runtime signals, and high fan-in modules
|
|
253
|
+
Version 0.6 adds a deterministic coverage audit that prioritizes root and nested-package entrypoints, structural boundaries, persistent artifact filename protocols, versioned schema literals, Rust/Tauri runtime signals, TypeScript Tauri invoke adapters, and high fan-in modules. It suppresses declaration files, dependency caches, test support code, simple barrels, and low-fan-in broad utilities. It suggests narrow coverage rules but never writes cards or invents semantic roles. Use `llmnav audit --summary --json` for a compact automation result or `llmnav audit --json --output .llmnav/audit.json` to keep the full candidate report out of captured stdout.
|
|
254
254
|
|
|
255
255
|
LLMNav does not discover sibling repositories automatically and does not ship an MCP server, embedding database, hosted service, SCIP generator, or complete language-aware call graph. External tools may export the documented compact graph-input schema. Generated structure never writes derived edges into source cards.
|
|
256
256
|
|
package/docs/api.md
CHANGED
|
@@ -60,7 +60,7 @@ console.log(stats.parsedFiles, stats.reusedFiles);
|
|
|
60
60
|
|
|
61
61
|
Stat hints are an optimization, not deterministic output. The returned `fileState` contains only repository-relative generated data.
|
|
62
62
|
|
|
63
|
-
Attached symbol declarations expose generated `language`, `exported`, `visibility`, optional Go `receiver`, `endOffset`, and declaration `bodyHash` fields. Indexed cards also expose sorted route, event, schema, migration, and command `boundaries` with confidence and evidence.
|
|
63
|
+
Attached symbol declarations expose generated `language`, `exported`, `visibility`, optional Go `receiver`, `endOffset`, and declaration `bodyHash` fields. Indexed cards also expose sorted artifact, route, event, schema, migration, runtime, and command `boundaries` with confidence and evidence.
|
|
64
64
|
|
|
65
65
|
## Audit annotation coverage
|
|
66
66
|
|
package/docs/architecture.md
CHANGED
|
@@ -64,13 +64,13 @@ Each card carries three independent hashes.
|
|
|
64
64
|
|
|
65
65
|
Symbol attachment records the detected language, public/exported status, visibility, Go receiver when present, declaration span, and a declaration-level body hash. TypeScript and Go use dedicated deterministic declaration patterns; JavaScript, Rust, Python, and generic C-like declarations retain the compatible fallback patterns.
|
|
66
66
|
|
|
67
|
-
Generated cards may contain a sorted `boundaries` array. LLMNav detects `route`, `event`, `schema`, `migration`, and `command` boundaries from repository-relative paths
|
|
67
|
+
Generated cards may contain a sorted `boundaries` array. LLMNav detects `artifact`, `route`, `event`, `schema`, `migration`, `runtime`, and `command` boundaries from repository-relative paths, controlled semantic effects or risks, and narrow source evidence. Source evidence includes persistent dotted JSON filename builders, versioned schema literals, Tauri commands and invoke adapters, and platform lifecycle code. Each record includes `confidence` and explicit evidence. These hints are generated navigation data and are never copied into source comments.
|
|
68
68
|
|
|
69
69
|
Generation compares the previous and current primary indexes to emit `affectedBoundaries`. The report preserves the card change and hash dimensions while adding module IDs, structural boundary records, outbound relation targets, and reverse semantic dependents. It is returned through the API and `generate --json`; it is not stored in source cards.
|
|
70
70
|
|
|
71
71
|
## Annotation coverage audit
|
|
72
72
|
|
|
73
|
-
The read-only audit combines package entrypoints, public re-export reachability, generated structural boundaries, import fan-in, exported declaration counts, and source size. It lowers the priority of
|
|
73
|
+
The read-only audit combines package entrypoints, public re-export reachability, generated structural boundaries, import fan-in, exported declaration counts, and source size. It lowers the priority of named utilities and low-fan-in export hubs, declaration files, non-production support paths, and pure re-export barrels. High-fan-in domain contract hubs do not receive the broad-utility penalty merely because they export many declarations. Every candidate retains its score inputs and deterministic repository-relative path.
|
|
74
74
|
|
|
75
75
|
This analyzer deliberately stops before semantic generation. Structure can identify a file worth inspecting, but it cannot safely invent a durable ID, role, ownership statement, invariant, risk, or relation. Accepted boundaries become explicit source cards, path-specific coverage rules, and retrieval regression queries through normal review.
|
|
76
76
|
|
package/docs/publishing.md
CHANGED
|
@@ -31,7 +31,7 @@ npm run release:check
|
|
|
31
31
|
|
|
32
32
|
## 3. Configure npm authentication
|
|
33
33
|
|
|
34
|
-
The supplied release workflow uses GitHub's OIDC token through npm Trusted Publishers. The GitHub source repository is private, so the workflow explicitly disables provenance; npm accepts provenance only from public source repositories. If the repository becomes public, enable provenance in both `package.json` and the workflow after a successful release check.
|
|
34
|
+
The supplied release workflow uses GitHub's OIDC token through npm Trusted Publishers. The GitHub source repository is private, so the workflow explicitly disables provenance; npm accepts provenance only from public source repositories. If the repository becomes public, enable provenance in both `package.json` and the workflow after a successful release check. The same tag workflow creates an idempotent GitHub Release only after the npm registry check or publication succeeds.
|
|
35
35
|
|
|
36
36
|
The workflow references a GitHub environment named `npm`. Create that environment for release protection, or remove the `environment` line when no environment gate is desired.
|
|
37
37
|
|
|
@@ -65,7 +65,7 @@ git tag v0.6.0
|
|
|
65
65
|
git push origin v0.6.0
|
|
66
66
|
```
|
|
67
67
|
|
|
68
|
-
The release workflow rejects a tag that does not match `package.json`. If the exact version is already present in npm, the workflow succeeds only when the registry tarball integrity matches the tagged package; a mismatched package fails closed.
|
|
68
|
+
The release workflow rejects a tag that does not match `package.json`. If the exact version is already present in npm, the workflow succeeds only when the registry tarball integrity matches the tagged package; a mismatched package fails closed. It then reuses an existing GitHub Release for the tag or creates one with generated release notes. Registry, GitHub API, and Release creation errors other than an expected missing Release fail the workflow.
|
|
69
69
|
|
|
70
70
|
## 6. Verify from a clean directory
|
|
71
71
|
|
package/docs/quickstart.md
CHANGED
|
@@ -19,7 +19,7 @@ Source roots and evaluation files must remain inside the repository. The generat
|
|
|
19
19
|
npx llmnav audit
|
|
20
20
|
```
|
|
21
21
|
|
|
22
|
-
Start with high and medium candidates. The audit explains whether a file is a package entrypoint, public API, generated structural boundary, or high fan-in module. It does not write source or invent card contents. Declaration files and common non-production or utility shapes are suppressed so the result is a review queue, not a demand to annotate every file.
|
|
22
|
+
Start with high and medium candidates. The audit explains whether a file is a package entrypoint, public API, generated structural boundary, or high fan-in module. Narrow source signals also identify versioned schema literals, persistent dotted JSON filename protocols, and Tauri invoke adapters. It does not write source or invent card contents. Declaration files and common non-production or low-fan-in utility shapes are suppressed so the result is a review queue, not a demand to annotate every file.
|
|
23
23
|
|
|
24
24
|
After accepting a candidate, write its durable meaning by inspecting the source, add a narrow `coverageRules` entry for that exact boundary, and add a real task-language query to `.llmnav/eval/queries.jsonl`. Use `npx llmnav audit --fail-on high` in CI only after the initial review.
|
|
25
25
|
|
package/docs/spec.md
CHANGED
|
@@ -351,6 +351,10 @@ For repositories with at least 50,000 scanned source bytes, LLMNav comments SHOU
|
|
|
351
351
|
|
|
352
352
|
These are anti-bloat limits, not targets.
|
|
353
353
|
|
|
354
|
+
The reference parser also rejects a single source file larger than 16 MiB and more than 10,000
|
|
355
|
+
LLMNav blocks in one file. These resource budgets keep repository-controlled input from causing
|
|
356
|
+
unbounded parser memory or CPU work; they do not change the `llmnav/1` comment grammar.
|
|
357
|
+
|
|
354
358
|
## Canonicalization
|
|
355
359
|
|
|
356
360
|
A canonical formatter MUST refuse to erase malformed lines, unknown fields, overlapping blocks, or duplicate scalar values. Unsafe cards remain unchanged until the checker-reported issue is fixed.
|
package/package.json
CHANGED
package/src/audit.js
CHANGED
|
@@ -66,7 +66,7 @@ export async function auditProject(root) {
|
|
|
66
66
|
const reexportBarrel = isReexportBarrel(record.source ?? "", file);
|
|
67
67
|
const basename = path.posix.basename(file, path.posix.extname(file)).toLowerCase();
|
|
68
68
|
const broadUtility = UTILITY_NAME_PATTERN.test(basename) ||
|
|
69
|
-
(exportedDeclarations >= 10 && !entrypoint && !publicApi && boundaries.length === 0);
|
|
69
|
+
(exportedDeclarations >= 10 && importers <= 1 && !entrypoint && !publicApi && boundaries.length === 0);
|
|
70
70
|
const nonProduction = NON_PRODUCTION_PATH_PATTERN.test(file);
|
|
71
71
|
const largeSource = record.sourceBytes >= LARGE_SOURCE_BYTES;
|
|
72
72
|
const hasSignal = entrypoint || publicApi || boundaries.length > 0 || importers > 0 || exportedDeclarations > 0 || largeSource;
|
package/src/boundaries.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/* llmnav/1 module
|
|
2
2
|
id=llmnav.structure.boundaries
|
|
3
|
-
role=Detect route, event, schema, migration, and command boundaries from generated local evidence.
|
|
3
|
+
role=Detect durable artifact, route, event, schema, migration, runtime, and command boundaries from generated local evidence.
|
|
4
4
|
owns=boundary kinds|confidence assignment|boundary evidence
|
|
5
5
|
excludes=framework execution|source annotations
|
|
6
|
-
search=boundary detection|route schema migration|command event
|
|
6
|
+
search=boundary detection|artifact route schema migration|command event runtime
|
|
7
7
|
rel=workflow>llmnav.index.generate
|
|
8
8
|
stability=architecture
|
|
9
9
|
*/
|
|
@@ -11,7 +11,7 @@ stability=architecture
|
|
|
11
11
|
import path from "node:path";
|
|
12
12
|
import { compareText, toPosix } from "./util.js";
|
|
13
13
|
|
|
14
|
-
export const BOUNDARY_KINDS = Object.freeze(["command", "event", "migration", "route", "runtime", "schema"]);
|
|
14
|
+
export const BOUNDARY_KINDS = Object.freeze(["artifact", "command", "event", "migration", "route", "runtime", "schema"]);
|
|
15
15
|
|
|
16
16
|
export function detectBoundaries(record) {
|
|
17
17
|
const relativePath = toPosix(record.relativePath).toLowerCase();
|
|
@@ -40,6 +40,13 @@ export function detectBoundaries(record) {
|
|
|
40
40
|
/(?:schema\.(?:json|ya?ml)|\.(?:proto|graphql|gql))$/u.test(relativePath)) {
|
|
41
41
|
add("schema", "high", "path");
|
|
42
42
|
}
|
|
43
|
+
if (/\bschemaVersion\s*:\s*["'][^"'\r\n]+\/v\d+(?:\.\d+)*["']/u.test(source)) {
|
|
44
|
+
add("schema", "high", "versioned-schema-literal");
|
|
45
|
+
}
|
|
46
|
+
if (/\bexport\s+(?:async\s+)?function\s+[A-Za-z_$][\w$]*FileName\b/u.test(source) &&
|
|
47
|
+
/["'`][^"'`\r\n]*\.[a-z0-9-]+\.json["'`]/iu.test(source)) {
|
|
48
|
+
add("artifact", "medium", "persistent-json-filename");
|
|
49
|
+
}
|
|
43
50
|
if (risks.includes("migration") || /(?:^|\/)(?:migrations?|migrate)(?:\/|$)/u.test(relativePath)) {
|
|
44
51
|
add("migration", "high", risks.includes("migration") ? "risk" : "path");
|
|
45
52
|
}
|
|
@@ -49,6 +56,11 @@ export function detectBoundaries(record) {
|
|
|
49
56
|
if (/\.rs$/u.test(relativePath) && /#\[tauri::command\]|tauri::generate_handler!/u.test(source)) {
|
|
50
57
|
add("command", "high", "tauri-command");
|
|
51
58
|
}
|
|
59
|
+
if (/\.[cm]?[jt]sx?$/u.test(relativePath) &&
|
|
60
|
+
/\bgetTauriInvoke\s*\(/u.test(source) &&
|
|
61
|
+
/\binvoke(?:\s*<[^;\r\n]+?>)?\s*\(/u.test(source)) {
|
|
62
|
+
add("command", "high", "tauri-invoke");
|
|
63
|
+
}
|
|
52
64
|
if (/\.rs$/u.test(relativePath) &&
|
|
53
65
|
/#\[cfg\((?:windows|unix|target_(?:os|family))/u.test(source) &&
|
|
54
66
|
/\b(?:Drop|shutdown|terminate|kill|process_group|job_object)\b/iu.test(source)) {
|
package/src/declaration.js
CHANGED
|
@@ -10,7 +10,7 @@ stability=architecture
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import path from "node:path";
|
|
13
|
-
import {
|
|
13
|
+
import { buildLineStarts, lineAtOffsetFromStarts, normalizeNewlines, sha256 } from "./util.js";
|
|
14
14
|
|
|
15
15
|
const DECLARATION_PATTERNS = {
|
|
16
16
|
javascript: [
|
|
@@ -42,8 +42,18 @@ const DECLARATION_PATTERNS = {
|
|
|
42
42
|
{ kind: "function", pattern: /^(?:public\s+|private\s+|protected\s+|internal\s+|static\s+|final\s+|async\s+)*(?:[A-Za-z_][\w<>?\[\],.]*\s+)+([A-Za-z_][A-Za-z0-9_]*)\s*\(/u },
|
|
43
43
|
],
|
|
44
44
|
};
|
|
45
|
+
const JAVASCRIPT_REGEX_PREFIX_KEYWORDS = new Set([
|
|
46
|
+
"case", "delete", "do", "else", "in", "instanceof", "new", "return", "throw", "typeof", "void", "yield",
|
|
47
|
+
]);
|
|
45
48
|
|
|
46
|
-
export function
|
|
49
|
+
export function createDeclarationScanContext(source) {
|
|
50
|
+
return {
|
|
51
|
+
lineStarts: buildLineStarts(source),
|
|
52
|
+
remainingScanUnits: Math.max(1, source.length * 2),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function findAttachedDeclaration(source, block, filePath, scanContext = createDeclarationScanContext(source)) {
|
|
47
57
|
if (block.scope !== "symbol") return null;
|
|
48
58
|
const window = source.slice(block.end, block.end + 3000);
|
|
49
59
|
const skipped = skipTrivia(window);
|
|
@@ -61,11 +71,12 @@ export function findAttachedDeclaration(source, block, filePath) {
|
|
|
61
71
|
const declarationOffset = block.end + skipped;
|
|
62
72
|
const signature = extractSignature(collapsed);
|
|
63
73
|
const exported = isExportedDeclaration(language, match[1], signature);
|
|
64
|
-
const endOffset = findDeclarationEnd(source, declarationOffset, family);
|
|
74
|
+
const endOffset = findDeclarationEnd(source, declarationOffset, family, scanContext);
|
|
75
|
+
if (endOffset === null) return null;
|
|
65
76
|
return {
|
|
66
77
|
symbol: match[1],
|
|
67
78
|
kind: definition.kind,
|
|
68
|
-
line:
|
|
79
|
+
line: lineAtOffsetFromStarts(scanContext.lineStarts, declarationOffset),
|
|
69
80
|
signature,
|
|
70
81
|
language,
|
|
71
82
|
exported,
|
|
@@ -155,8 +166,8 @@ function extractGoReceiver(candidate) {
|
|
|
155
166
|
return (parts.at(-1) ?? "").replace(/^\*+/u, "") || null;
|
|
156
167
|
}
|
|
157
168
|
|
|
158
|
-
function findDeclarationEnd(source, start, family) {
|
|
159
|
-
if (family === "python") return findPythonDeclarationEnd(source, start);
|
|
169
|
+
function findDeclarationEnd(source, start, family, scanContext) {
|
|
170
|
+
if (family === "python") return findPythonDeclarationEnd(source, start, scanContext);
|
|
160
171
|
let state = "normal";
|
|
161
172
|
let escaped = false;
|
|
162
173
|
let regexCharacterClass = false;
|
|
@@ -166,6 +177,7 @@ function findDeclarationEnd(source, start, family) {
|
|
|
166
177
|
let openedBody = false;
|
|
167
178
|
|
|
168
179
|
for (let index = start; index < source.length; index += 1) {
|
|
180
|
+
if (!consumeDeclarationScan(scanContext)) return null;
|
|
169
181
|
const character = source[index];
|
|
170
182
|
const next = source[index + 1];
|
|
171
183
|
if (state === "line-comment") {
|
|
@@ -230,15 +242,16 @@ function findDeclarationEnd(source, start, family) {
|
|
|
230
242
|
} else if (character === ";" && !openedBody && parentheses === 0 && brackets === 0) {
|
|
231
243
|
return index + 1;
|
|
232
244
|
} else if (character === "\n" && !openedBody && parentheses === 0 && brackets === 0) {
|
|
233
|
-
const
|
|
234
|
-
const
|
|
235
|
-
if (
|
|
245
|
+
const continuation = declarationContinuesBefore(source, start, index, scanContext);
|
|
246
|
+
const next = nextSignificantCharacter(source, index + 1, scanContext);
|
|
247
|
+
if (continuation === null || next.exhausted) return null;
|
|
248
|
+
if (next.character !== "{" && !continuation) return index;
|
|
236
249
|
}
|
|
237
250
|
}
|
|
238
251
|
return source.length;
|
|
239
252
|
}
|
|
240
253
|
|
|
241
|
-
function findPythonDeclarationEnd(source, start) {
|
|
254
|
+
function findPythonDeclarationEnd(source, start, scanContext) {
|
|
242
255
|
const declarationLineStart = source.lastIndexOf("\n", start - 1) + 1;
|
|
243
256
|
const baseIndent = indentationWidth(source.slice(declarationLineStart, start));
|
|
244
257
|
let cursor = source.indexOf("\n", start);
|
|
@@ -248,6 +261,7 @@ function findPythonDeclarationEnd(source, start) {
|
|
|
248
261
|
while (cursor < source.length) {
|
|
249
262
|
const lineEnd = source.indexOf("\n", cursor);
|
|
250
263
|
const end = lineEnd < 0 ? source.length : lineEnd + 1;
|
|
264
|
+
if (!consumeDeclarationScan(scanContext, Math.max(1, end - cursor))) return null;
|
|
251
265
|
const line = source.slice(cursor, lineEnd < 0 ? source.length : lineEnd);
|
|
252
266
|
if (/^\s*(?:#.*)?$/u.test(line)) {
|
|
253
267
|
cursor = end;
|
|
@@ -261,6 +275,15 @@ function findPythonDeclarationEnd(source, start) {
|
|
|
261
275
|
return source.length;
|
|
262
276
|
}
|
|
263
277
|
|
|
278
|
+
function consumeDeclarationScan(scanContext, units = 1) {
|
|
279
|
+
if (scanContext.remainingScanUnits < units) {
|
|
280
|
+
scanContext.remainingScanUnits = 0;
|
|
281
|
+
return false;
|
|
282
|
+
}
|
|
283
|
+
scanContext.remainingScanUnits -= units;
|
|
284
|
+
return true;
|
|
285
|
+
}
|
|
286
|
+
|
|
264
287
|
function indentationWidth(value) {
|
|
265
288
|
let width = 0;
|
|
266
289
|
for (const character of value) width += character === "\t" ? 8 - (width % 8) : 1;
|
|
@@ -268,11 +291,36 @@ function indentationWidth(value) {
|
|
|
268
291
|
}
|
|
269
292
|
|
|
270
293
|
function canStartJavaScriptRegex(source, start, offset) {
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
294
|
+
let cursor = offset - 1;
|
|
295
|
+
while (cursor >= start && /\s/u.test(source[cursor])) cursor -= 1;
|
|
296
|
+
if (cursor < start) return true;
|
|
297
|
+
const previous = source[cursor];
|
|
274
298
|
if (/[=(:,!&|?{};\[]/u.test(previous)) return true;
|
|
275
|
-
|
|
299
|
+
if (!/[A-Za-z]/u.test(previous)) return false;
|
|
300
|
+
const wordEnd = cursor + 1;
|
|
301
|
+
while (cursor >= start && /[A-Za-z]/u.test(source[cursor])) cursor -= 1;
|
|
302
|
+
return JAVASCRIPT_REGEX_PREFIX_KEYWORDS.has(source.slice(cursor + 1, wordEnd));
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function declarationContinuesBefore(source, start, offset, scanContext) {
|
|
306
|
+
let cursor = offset - 1;
|
|
307
|
+
while (cursor >= start && /\s/u.test(source[cursor])) {
|
|
308
|
+
if (!consumeDeclarationScan(scanContext)) return null;
|
|
309
|
+
cursor -= 1;
|
|
310
|
+
}
|
|
311
|
+
if (cursor < start) return false;
|
|
312
|
+
const character = source[cursor];
|
|
313
|
+
if (/[=|&,([{]/u.test(character)) return true;
|
|
314
|
+
return character === ">" && source[cursor - 1] === "=";
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
function nextSignificantCharacter(source, offset, scanContext) {
|
|
318
|
+
let cursor = offset;
|
|
319
|
+
while (cursor < source.length && /\s/u.test(source[cursor])) {
|
|
320
|
+
if (!consumeDeclarationScan(scanContext)) return { character: "", exhausted: true };
|
|
321
|
+
cursor += 1;
|
|
322
|
+
}
|
|
323
|
+
return { character: source[cursor] ?? "", exhausted: false };
|
|
276
324
|
}
|
|
277
325
|
|
|
278
326
|
function looksLikeRustCharacterLiteral(source, offset) {
|
package/src/generator.js
CHANGED
|
@@ -217,7 +217,7 @@ async function generateProjectLocked(root, options) {
|
|
|
217
217
|
|
|
218
218
|
if (statHints && hintsPath) {
|
|
219
219
|
try {
|
|
220
|
-
await persistStatHints(hintsPath, statHints);
|
|
220
|
+
await persistStatHints(root, hintsPath, statHints);
|
|
221
221
|
statHintsPersisted = true;
|
|
222
222
|
} catch (error) {
|
|
223
223
|
statHintsError = error instanceof Error ? error.message : String(error);
|
package/src/incremental.js
CHANGED
|
@@ -11,11 +11,12 @@ stability=architecture
|
|
|
11
11
|
import { readFile, stat } from "node:fs/promises";
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
import { loadConfig } from "./config.js";
|
|
14
|
-
import { findAttachedDeclaration, extractImports } from "./declaration.js";
|
|
14
|
+
import { createDeclarationScanContext, findAttachedDeclaration, extractImports } from "./declaration.js";
|
|
15
15
|
import { collectSourceFiles } from "./files.js";
|
|
16
16
|
import { parseLlmnavBlocks } from "./parser.js";
|
|
17
17
|
import { loadRegistry } from "./registry.js";
|
|
18
18
|
import {
|
|
19
|
+
assertNoSymlinkTraversal,
|
|
19
20
|
atomicWrite,
|
|
20
21
|
compareText,
|
|
21
22
|
readJsonSafe,
|
|
@@ -25,15 +26,19 @@ import {
|
|
|
25
26
|
} from "./util.js";
|
|
26
27
|
|
|
27
28
|
export const FILE_STATE_SCHEMA_VERSION = 1;
|
|
28
|
-
export const SOURCE_INDEXER_VERSION =
|
|
29
|
+
export const SOURCE_INDEXER_VERSION = 5;
|
|
29
30
|
const STAT_HINTS_SCHEMA_VERSION = 1;
|
|
30
31
|
|
|
31
32
|
export async function scanProjectIncremental(root, options = {}) {
|
|
32
33
|
const { config, configPath } = await loadConfig(root);
|
|
33
34
|
const files = await collectSourceFiles(root, config, options.paths ?? []);
|
|
34
35
|
const cacheDirectory = path.join(root, config.generation.cacheDirectory);
|
|
35
|
-
|
|
36
|
+
await assertNoSymlinkTraversal(root, cacheDirectory, config.generation.cacheDirectory);
|
|
37
|
+
const fileStatePath = path.join(cacheDirectory, "file-state.json");
|
|
38
|
+
await assertNoSymlinkTraversal(root, fileStatePath, relativePosix(root, fileStatePath));
|
|
39
|
+
const previousState = options.previousState ?? await readJsonSafe(fileStatePath, null);
|
|
36
40
|
const hintsPath = path.join(root, ".llmnav", "state", "stat-hints.json");
|
|
41
|
+
await assertNoSymlinkTraversal(root, hintsPath, relativePosix(root, hintsPath));
|
|
37
42
|
const previousHints = options.useStatHints === false
|
|
38
43
|
? null
|
|
39
44
|
: await readJsonSafe(hintsPath, null);
|
|
@@ -133,8 +138,9 @@ export async function scanProjectIncremental(root, options = {}) {
|
|
|
133
138
|
export function buildFileStateFromProject(project) {
|
|
134
139
|
const files = project.fileRecords.map((fileRecord) => {
|
|
135
140
|
const source = fileRecord.source ?? "";
|
|
141
|
+
const declarationScanContext = createDeclarationScanContext(source);
|
|
136
142
|
const declarations = fileRecord.blocks.map((block) =>
|
|
137
|
-
findAttachedDeclaration(source, block, fileRecord.relativePath),
|
|
143
|
+
findAttachedDeclaration(source, block, fileRecord.relativePath, declarationScanContext),
|
|
138
144
|
);
|
|
139
145
|
return normalizeStateFile(
|
|
140
146
|
{
|
|
@@ -157,7 +163,8 @@ export function buildFileStateFromProject(project) {
|
|
|
157
163
|
};
|
|
158
164
|
}
|
|
159
165
|
|
|
160
|
-
export async function persistStatHints(hintsPath, statHints) {
|
|
166
|
+
export async function persistStatHints(root, hintsPath, statHints) {
|
|
167
|
+
await assertNoSymlinkTraversal(root, hintsPath, relativePosix(root, hintsPath));
|
|
161
168
|
await atomicWrite(hintsPath, stableStringify(statHints));
|
|
162
169
|
}
|
|
163
170
|
|
|
@@ -176,6 +183,7 @@ export function usableFileState(value) {
|
|
|
176
183
|
|
|
177
184
|
function analyzeFile(relativePath, source) {
|
|
178
185
|
const blocks = parseLlmnavBlocks(source, relativePath);
|
|
186
|
+
const declarationScanContext = createDeclarationScanContext(source);
|
|
179
187
|
return {
|
|
180
188
|
path: relativePath,
|
|
181
189
|
contentHash: sha256(source),
|
|
@@ -183,7 +191,7 @@ function analyzeFile(relativePath, source) {
|
|
|
183
191
|
semanticBytes: blocks.reduce((sum, block) => sum + Buffer.byteLength(block.raw), 0),
|
|
184
192
|
imports: extractImports(source, relativePath),
|
|
185
193
|
blocks,
|
|
186
|
-
declarations: blocks.map((block) => findAttachedDeclaration(source, block, relativePath)),
|
|
194
|
+
declarations: blocks.map((block) => findAttachedDeclaration(source, block, relativePath, declarationScanContext)),
|
|
187
195
|
};
|
|
188
196
|
}
|
|
189
197
|
|
package/src/index.d.ts
CHANGED
|
@@ -58,7 +58,7 @@ export interface Declaration {
|
|
|
58
58
|
}
|
|
59
59
|
|
|
60
60
|
export interface DetectedBoundary {
|
|
61
|
-
kind: "command" | "event" | "migration" | "route" | "schema";
|
|
61
|
+
kind: "artifact" | "command" | "event" | "migration" | "route" | "runtime" | "schema";
|
|
62
62
|
confidence: "high" | "medium";
|
|
63
63
|
evidence: string[];
|
|
64
64
|
}
|
package/src/initializer.js
CHANGED
|
@@ -20,9 +20,14 @@ import { assertNoSymlinkTraversal, atomicWrite, readJson, readText, stableString
|
|
|
20
20
|
const PACKAGE_ROOT = fileURLToPath(new URL("..", import.meta.url));
|
|
21
21
|
|
|
22
22
|
export async function initializeProject(root, options = {}) {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
const controlDirectory = path.join(root, ".llmnav");
|
|
24
|
+
const evalDirectory = path.join(controlDirectory, "eval");
|
|
25
|
+
const schemaDirectory = path.join(controlDirectory, "schema");
|
|
26
|
+
await assertNoSymlinkTraversal(root, controlDirectory, ".llmnav");
|
|
27
|
+
await assertNoSymlinkTraversal(root, evalDirectory, ".llmnav/eval");
|
|
28
|
+
await assertNoSymlinkTraversal(root, schemaDirectory, ".llmnav/schema");
|
|
29
|
+
await mkdir(evalDirectory, { recursive: true });
|
|
30
|
+
await mkdir(schemaDirectory, { recursive: true });
|
|
26
31
|
const repositoryId = await inferRepositoryId(root);
|
|
27
32
|
const changed = [];
|
|
28
33
|
|
|
@@ -30,6 +35,7 @@ export async function initializeProject(root, options = {}) {
|
|
|
30
35
|
config.repositoryId = repositoryId;
|
|
31
36
|
config.$schema = "./schema/config.schema.json";
|
|
32
37
|
await writeIfMissingOrForced(
|
|
38
|
+
root,
|
|
33
39
|
path.join(root, ".llmnav", "config.json"),
|
|
34
40
|
stableStringify(config),
|
|
35
41
|
options.force,
|
|
@@ -38,6 +44,7 @@ export async function initializeProject(root, options = {}) {
|
|
|
38
44
|
);
|
|
39
45
|
|
|
40
46
|
await writeIfMissingOrForced(
|
|
47
|
+
root,
|
|
41
48
|
path.join(root, ".llmnav", "lexicon.json"),
|
|
42
49
|
stableStringify({ version: 1, aliases: {} }),
|
|
43
50
|
false,
|
|
@@ -45,6 +52,7 @@ export async function initializeProject(root, options = {}) {
|
|
|
45
52
|
".llmnav/lexicon.json",
|
|
46
53
|
);
|
|
47
54
|
await writeIfMissingOrForced(
|
|
55
|
+
root,
|
|
48
56
|
path.join(root, ".llmnav", "ids.jsonl"),
|
|
49
57
|
"",
|
|
50
58
|
false,
|
|
@@ -52,6 +60,7 @@ export async function initializeProject(root, options = {}) {
|
|
|
52
60
|
".llmnav/ids.jsonl",
|
|
53
61
|
);
|
|
54
62
|
await writeIfMissingOrForced(
|
|
63
|
+
root,
|
|
55
64
|
path.join(root, ".llmnav", "order.lock"),
|
|
56
65
|
"",
|
|
57
66
|
false,
|
|
@@ -59,6 +68,7 @@ export async function initializeProject(root, options = {}) {
|
|
|
59
68
|
".llmnav/order.lock",
|
|
60
69
|
);
|
|
61
70
|
await writeIfMissingOrForced(
|
|
71
|
+
root,
|
|
62
72
|
path.join(root, ".llmnav", "eval", "queries.jsonl"),
|
|
63
73
|
'# One JSON object per line: {"query":"...","expected":["domain.feature.action"]}\n',
|
|
64
74
|
false,
|
|
@@ -66,6 +76,7 @@ export async function initializeProject(root, options = {}) {
|
|
|
66
76
|
".llmnav/eval/queries.jsonl",
|
|
67
77
|
);
|
|
68
78
|
await writeIfMissingOrForced(
|
|
79
|
+
root,
|
|
69
80
|
path.join(root, ".llmnav", ".gitignore"),
|
|
70
81
|
"tmp/\nstate/\n.transactions/\ngeneration-transaction.json\ngeneration.lock\ngeneration.lock.release-*\n*.tmp-*\n",
|
|
71
82
|
options.force,
|
|
@@ -75,6 +86,7 @@ export async function initializeProject(root, options = {}) {
|
|
|
75
86
|
|
|
76
87
|
const schemaSource = path.join(PACKAGE_ROOT, "schema", "config.schema.json");
|
|
77
88
|
const schemaTarget = path.join(root, ".llmnav", "schema", "config.schema.json");
|
|
89
|
+
await assertNoSymlinkTraversal(root, schemaTarget, ".llmnav/schema/config.schema.json");
|
|
78
90
|
if (options.force || (await readText(schemaTarget, null)) === null) {
|
|
79
91
|
await copyFile(schemaSource, schemaTarget);
|
|
80
92
|
changed.push(".llmnav/schema/config.schema.json");
|
|
@@ -106,7 +118,8 @@ async function inferRepositoryId(root) {
|
|
|
106
118
|
return normalized || "repository";
|
|
107
119
|
}
|
|
108
120
|
|
|
109
|
-
async function writeIfMissingOrForced(filePath, content, force, changed, displayPath) {
|
|
121
|
+
async function writeIfMissingOrForced(root, filePath, content, force, changed, displayPath) {
|
|
122
|
+
await assertNoSymlinkTraversal(root, filePath, displayPath);
|
|
110
123
|
const existing = await readText(filePath, null);
|
|
111
124
|
if (existing !== null && !force) return;
|
|
112
125
|
if (existing === content) return;
|
|
@@ -116,6 +129,7 @@ async function writeIfMissingOrForced(filePath, content, force, changed, display
|
|
|
116
129
|
|
|
117
130
|
async function addPackageScripts(root) {
|
|
118
131
|
const packagePath = path.join(root, "package.json");
|
|
132
|
+
await assertNoSymlinkTraversal(root, packagePath, "package.json");
|
|
119
133
|
const text = await readText(packagePath, null);
|
|
120
134
|
if (text === null) return false;
|
|
121
135
|
const parsed = JSON.parse(text);
|
package/src/parser.js
CHANGED
|
@@ -15,7 +15,13 @@ import {
|
|
|
15
15
|
SCOPES,
|
|
16
16
|
SPEC_VERSION,
|
|
17
17
|
} from "./spec.js";
|
|
18
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
buildLineStarts,
|
|
20
|
+
detectNewline,
|
|
21
|
+
lineAtOffsetFromStarts,
|
|
22
|
+
normalizeNewlines,
|
|
23
|
+
splitPipe,
|
|
24
|
+
} from "./util.js";
|
|
19
25
|
|
|
20
26
|
const BLOCK_PATTERNS = [
|
|
21
27
|
{
|
|
@@ -31,24 +37,33 @@ const BLOCK_PATTERNS = [
|
|
|
31
37
|
terminator: "-->",
|
|
32
38
|
},
|
|
33
39
|
];
|
|
40
|
+
const MAX_SOURCE_BYTES = 16 * 1024 * 1024;
|
|
41
|
+
const MAX_BLOCKS_PER_FILE = 10_000;
|
|
34
42
|
|
|
35
43
|
export function parseLlmnavBlocks(source, filePath = "<memory>") {
|
|
44
|
+
if (Buffer.byteLength(source) > MAX_SOURCE_BYTES) {
|
|
45
|
+
throw new Error(`${filePath} exceeds the ${MAX_SOURCE_BYTES}-byte parser byte limit.`);
|
|
46
|
+
}
|
|
47
|
+
const literalMask = buildLiteralMask(source, filePath);
|
|
48
|
+
const lineStarts = buildLineStarts(source);
|
|
49
|
+
const blockBudget = { count: 0 };
|
|
36
50
|
const blocks = [];
|
|
37
51
|
for (const definition of BLOCK_PATTERNS) {
|
|
38
52
|
definition.pattern.lastIndex = 0;
|
|
39
53
|
for (const match of source.matchAll(definition.pattern)) {
|
|
40
54
|
const matchedRaw = match[0];
|
|
41
55
|
const matchStart = match.index ?? 0;
|
|
42
|
-
if (
|
|
56
|
+
if (literalMask[matchStart] === 1) continue;
|
|
43
57
|
const lineStart = source.lastIndexOf("\n", matchStart - 1) + 1;
|
|
44
58
|
const leading = source.slice(lineStart, matchStart);
|
|
45
59
|
const start = /^\s*$/u.test(leading) ? lineStart : matchStart;
|
|
46
60
|
const end = matchStart + matchedRaw.length;
|
|
47
61
|
const indent = start === lineStart ? leading : "";
|
|
48
62
|
const raw = source.slice(start, end);
|
|
63
|
+
reserveBlock(blockBudget, filePath);
|
|
49
64
|
blocks.push(
|
|
50
65
|
createBlock({
|
|
51
|
-
|
|
66
|
+
lineStarts,
|
|
52
67
|
filePath,
|
|
53
68
|
raw,
|
|
54
69
|
body: match[2],
|
|
@@ -63,8 +78,8 @@ export function parseLlmnavBlocks(source, filePath = "<memory>") {
|
|
|
63
78
|
}
|
|
64
79
|
}
|
|
65
80
|
|
|
66
|
-
blocks.push(...parseLineBlocks(source, filePath));
|
|
67
|
-
blocks.push(...parseUnterminatedBlockComments(source, filePath, blocks));
|
|
81
|
+
blocks.push(...parseLineBlocks(source, filePath, literalMask, lineStarts, blockBudget));
|
|
82
|
+
blocks.push(...parseUnterminatedBlockComments(source, filePath, blocks, literalMask, lineStarts, blockBudget));
|
|
68
83
|
blocks.sort((left, right) => left.start - right.start);
|
|
69
84
|
|
|
70
85
|
const overlapping = [];
|
|
@@ -85,7 +100,7 @@ export function parseLlmnavBlocks(source, filePath = "<memory>") {
|
|
|
85
100
|
return blocks;
|
|
86
101
|
}
|
|
87
102
|
|
|
88
|
-
function parseLineBlocks(source, filePath) {
|
|
103
|
+
function parseLineBlocks(source, filePath, literalMask, lineStarts, blockBudget) {
|
|
89
104
|
const blocks = [];
|
|
90
105
|
const lines = source.split(/(?<=\n)/u);
|
|
91
106
|
const offsets = [];
|
|
@@ -101,7 +116,7 @@ function parseLineBlocks(source, filePath) {
|
|
|
101
116
|
if (!header) continue;
|
|
102
117
|
|
|
103
118
|
const start = offsets[index];
|
|
104
|
-
if (
|
|
119
|
+
if (literalMask[start] === 1) continue;
|
|
105
120
|
const indent = header[1];
|
|
106
121
|
const prefix = header[2];
|
|
107
122
|
const scope = header[3].toLowerCase();
|
|
@@ -127,8 +142,9 @@ function parseLineBlocks(source, filePath) {
|
|
|
127
142
|
}
|
|
128
143
|
|
|
129
144
|
const raw = source.slice(start, end);
|
|
145
|
+
reserveBlock(blockBudget, filePath);
|
|
130
146
|
const block = createBlock({
|
|
131
|
-
|
|
147
|
+
lineStarts,
|
|
132
148
|
filePath,
|
|
133
149
|
raw,
|
|
134
150
|
body: bodyLines.join("\n"),
|
|
@@ -154,24 +170,28 @@ function parseLineBlocks(source, filePath) {
|
|
|
154
170
|
return blocks;
|
|
155
171
|
}
|
|
156
172
|
|
|
157
|
-
function parseUnterminatedBlockComments(source, filePath, parsedBlocks) {
|
|
173
|
+
function parseUnterminatedBlockComments(source, filePath, parsedBlocks, literalMask, lineStarts, blockBudget) {
|
|
158
174
|
const blocks = [];
|
|
175
|
+
const parsedIntervals = [...parsedBlocks].sort((left, right) => left.start - right.start);
|
|
159
176
|
for (const definition of BLOCK_PATTERNS) {
|
|
177
|
+
let intervalIndex = 0;
|
|
160
178
|
definition.opening.lastIndex = 0;
|
|
161
179
|
for (const match of source.matchAll(definition.opening)) {
|
|
162
180
|
const tokenStart = match.index ?? 0;
|
|
163
|
-
if (
|
|
164
|
-
|
|
181
|
+
if (literalMask[tokenStart] === 1) continue;
|
|
182
|
+
while (parsedIntervals[intervalIndex]?.end <= tokenStart) intervalIndex += 1;
|
|
183
|
+
const interval = parsedIntervals[intervalIndex];
|
|
184
|
+
if (interval && tokenStart >= interval.start && tokenStart < interval.end) continue;
|
|
165
185
|
const bodyStart = tokenStart + match[0].length;
|
|
166
|
-
if (source.indexOf(definition.terminator, bodyStart) >= 0) continue;
|
|
167
186
|
|
|
168
187
|
const lineStart = source.lastIndexOf("\n", tokenStart - 1) + 1;
|
|
169
188
|
const leading = source.slice(lineStart, tokenStart);
|
|
170
189
|
const start = /^\s*$/u.test(leading) ? lineStart : tokenStart;
|
|
171
190
|
const indent = start === lineStart ? leading : "";
|
|
172
191
|
const end = source.length;
|
|
192
|
+
reserveBlock(blockBudget, filePath);
|
|
173
193
|
const block = createBlock({
|
|
174
|
-
|
|
194
|
+
lineStarts,
|
|
175
195
|
filePath,
|
|
176
196
|
raw: source.slice(start, end),
|
|
177
197
|
body: source.slice(bodyStart),
|
|
@@ -192,15 +212,17 @@ function parseUnterminatedBlockComments(source, filePath, parsedBlocks) {
|
|
|
192
212
|
return blocks;
|
|
193
213
|
}
|
|
194
214
|
|
|
195
|
-
function
|
|
215
|
+
function buildLiteralMask(source, filePath) {
|
|
196
216
|
const extension = filePath.toLowerCase().match(/\.[a-z0-9]+$/u)?.[0] ?? "";
|
|
197
217
|
const hashComments = [".py", ".rb", ".sh", ".bash", ".zsh"].includes(extension);
|
|
198
218
|
const dashComments = extension === ".sql";
|
|
199
219
|
const tripleQuotes = extension === ".py";
|
|
200
220
|
let state = "normal";
|
|
201
221
|
let escaped = false;
|
|
222
|
+
const mask = new Uint8Array(source.length);
|
|
202
223
|
|
|
203
|
-
for (let index = 0; index <
|
|
224
|
+
for (let index = 0; index < source.length; index += 1) {
|
|
225
|
+
mask[index] = state === "normal" ? 0 : 1;
|
|
204
226
|
const character = source[index];
|
|
205
227
|
const next = source[index + 1];
|
|
206
228
|
const nextTwo = source.slice(index, index + 3);
|
|
@@ -284,15 +306,22 @@ function isInsideStringLiteral(source, targetOffset, filePath) {
|
|
|
284
306
|
}
|
|
285
307
|
}
|
|
286
308
|
|
|
287
|
-
return
|
|
309
|
+
return mask;
|
|
288
310
|
}
|
|
289
311
|
|
|
290
312
|
function looksLikeRustCharacterLiteral(source, offset) {
|
|
291
313
|
return /^'(?:\\.|[^'\\\r\n])'/u.test(source.slice(offset));
|
|
292
314
|
}
|
|
293
315
|
|
|
294
|
-
function
|
|
295
|
-
|
|
316
|
+
function reserveBlock(blockBudget, filePath) {
|
|
317
|
+
blockBudget.count += 1;
|
|
318
|
+
if (blockBudget.count > MAX_BLOCKS_PER_FILE) {
|
|
319
|
+
throw new Error(`${filePath} exceeds the ${MAX_BLOCKS_PER_FILE}-block parser limit.`);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
function createBlock({ lineStarts, filePath, raw, body, scope, style, start, end, indent, prefix }) {
|
|
324
|
+
const startLine = lineAtOffsetFromStarts(lineStarts, start);
|
|
296
325
|
const bodyStartLine = style === "line" ? startLine + 1 : startLine;
|
|
297
326
|
const parsed = parseBody(body, bodyStartLine);
|
|
298
327
|
const card = materializeCard(scope, parsed.entries);
|
|
@@ -310,7 +339,7 @@ function createBlock({ source, filePath, raw, body, scope, style, start, end, in
|
|
|
310
339
|
start,
|
|
311
340
|
end,
|
|
312
341
|
startLine,
|
|
313
|
-
endLine:
|
|
342
|
+
endLine: lineAtOffsetFromStarts(lineStarts, Math.max(start, end - 1)),
|
|
314
343
|
newline: detectNewline(raw),
|
|
315
344
|
};
|
|
316
345
|
}
|
package/src/project.js
CHANGED
|
@@ -12,7 +12,7 @@ stability=architecture
|
|
|
12
12
|
|
|
13
13
|
import { readFile } from "node:fs/promises";
|
|
14
14
|
import { loadConfig } from "./config.js";
|
|
15
|
-
import { findAttachedDeclaration, extractImports } from "./declaration.js";
|
|
15
|
+
import { createDeclarationScanContext, findAttachedDeclaration, extractImports } from "./declaration.js";
|
|
16
16
|
import { collectSourceFiles } from "./files.js";
|
|
17
17
|
import { parseLlmnavBlocks } from "./parser.js";
|
|
18
18
|
import { loadRegistry } from "./registry.js";
|
|
@@ -34,6 +34,7 @@ export async function scanProject(root, options = {}) {
|
|
|
34
34
|
sourceBytes += Buffer.byteLength(source);
|
|
35
35
|
semanticBytes += blocks.reduce((sum, block) => sum + Buffer.byteLength(block.raw), 0);
|
|
36
36
|
const imports = extractImports(source, relativePath);
|
|
37
|
+
const declarationScanContext = createDeclarationScanContext(source);
|
|
37
38
|
const fileRecord = {
|
|
38
39
|
absolutePath,
|
|
39
40
|
relativePath,
|
|
@@ -47,7 +48,7 @@ export async function scanProject(root, options = {}) {
|
|
|
47
48
|
};
|
|
48
49
|
fileRecords.push(fileRecord);
|
|
49
50
|
for (const block of blocks) {
|
|
50
|
-
const declaration = findAttachedDeclaration(source, block, relativePath);
|
|
51
|
+
const declaration = findAttachedDeclaration(source, block, relativePath, declarationScanContext);
|
|
51
52
|
records.push({
|
|
52
53
|
root,
|
|
53
54
|
absolutePath,
|
package/src/prompt-bundle.js
CHANGED
|
@@ -12,7 +12,7 @@ stability=architecture
|
|
|
12
12
|
|
|
13
13
|
import path from "node:path";
|
|
14
14
|
import { loadConfig } from "./config.js";
|
|
15
|
-
import { approximateTokens, compareText, readText, sha256, stableJson, stableStringify, toPosix } from "./util.js";
|
|
15
|
+
import { approximateTokens, assertNoSymlinkTraversal, compareText, readText, sha256, stableJson, stableStringify, toPosix } from "./util.js";
|
|
16
16
|
|
|
17
17
|
export const PROMPT_BUNDLE_SCHEMA_VERSION = 1;
|
|
18
18
|
|
|
@@ -62,7 +62,11 @@ export function isCompatiblePromptPrefixBundle(bundle, repositoryId = undefined)
|
|
|
62
62
|
export async function loadPromptPrefixBundle(root) {
|
|
63
63
|
const { config } = await loadConfig(root);
|
|
64
64
|
const relativePath = `${toPosix(config.generation.cacheDirectory).replace(/\/+$/u, "")}/prompt-prefix.json`;
|
|
65
|
-
const
|
|
65
|
+
const bundlePath = path.join(root, relativePath);
|
|
66
|
+
const manifestPath = path.join(root, config.generation.cacheDirectory, "manifest.json");
|
|
67
|
+
await assertNoSymlinkTraversal(root, bundlePath, relativePath);
|
|
68
|
+
await assertNoSymlinkTraversal(root, manifestPath, `${config.generation.cacheDirectory}/manifest.json`);
|
|
69
|
+
const content = await readText(bundlePath, "");
|
|
66
70
|
if (!content) throw new Error(`Missing generated prompt bundle ${relativePath}.`);
|
|
67
71
|
let bundle;
|
|
68
72
|
try {
|
|
@@ -73,7 +77,7 @@ export async function loadPromptPrefixBundle(root) {
|
|
|
73
77
|
if (!isCompatiblePromptPrefixBundle(bundle, config.repositoryId)) {
|
|
74
78
|
throw new Error(`Incompatible generated prompt bundle ${relativePath}.`);
|
|
75
79
|
}
|
|
76
|
-
const manifestContent = await readText(
|
|
80
|
+
const manifestContent = await readText(manifestPath, "");
|
|
77
81
|
const manifest = manifestContent ? JSON.parse(manifestContent) : null;
|
|
78
82
|
if (manifest?.files?.[relativePath] !== sha256(content)) throw new Error(`Prompt bundle hash does not match manifest.json.`);
|
|
79
83
|
return bundle;
|
package/src/registry.js
CHANGED
|
@@ -11,13 +11,14 @@ stability=contract
|
|
|
11
11
|
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
import { ID_PATTERN } from "./spec.js";
|
|
14
|
-
import { atomicWrite, parseJsonLines, readText } from "./util.js";
|
|
14
|
+
import { assertNoSymlinkTraversal, atomicWrite, parseJsonLines, readText } from "./util.js";
|
|
15
15
|
|
|
16
16
|
const REGISTRY_STATES = new Set(["active", "redirect", "replaced", "retired"]);
|
|
17
17
|
const REGISTRY_KEYS = new Set(["id", "state", "to", "by"]);
|
|
18
18
|
|
|
19
19
|
export async function loadRegistry(root) {
|
|
20
20
|
const registryPath = path.join(root, ".llmnav", "ids.jsonl");
|
|
21
|
+
await assertNoSymlinkTraversal(root, registryPath, ".llmnav/ids.jsonl");
|
|
21
22
|
const text = await readText(registryPath, "");
|
|
22
23
|
const parsed = parseJsonLines(text, registryPath);
|
|
23
24
|
const byId = new Map();
|
|
@@ -72,8 +73,13 @@ export async function loadRegistry(root) {
|
|
|
72
73
|
}
|
|
73
74
|
|
|
74
75
|
export async function ensureActiveIds(root, registry, ids) {
|
|
76
|
+
const registryPath = path.join(root, ".llmnav", "ids.jsonl");
|
|
77
|
+
if (path.resolve(registry.registryPath) !== path.resolve(registryPath)) {
|
|
78
|
+
throw new Error("Registry path does not belong to the requested project root.");
|
|
79
|
+
}
|
|
80
|
+
await assertNoSymlinkTraversal(root, registryPath, ".llmnav/ids.jsonl");
|
|
75
81
|
const { records, changed } = mergeActiveIds(registry, ids);
|
|
76
|
-
if (changed) await atomicWrite(
|
|
82
|
+
if (changed) await atomicWrite(registryPath, renderRegistryRecords(records));
|
|
77
83
|
return { records, changed };
|
|
78
84
|
}
|
|
79
85
|
|
package/src/search.js
CHANGED
|
@@ -14,7 +14,7 @@ import path from "node:path";
|
|
|
14
14
|
import { loadConfig } from "./config.js";
|
|
15
15
|
import { resolveRegistryId, loadRegistry } from "./registry.js";
|
|
16
16
|
import { renderCompactCard } from "./generator.js";
|
|
17
|
-
import { approximateTokens, compareText, readJsonSafe, readText, sha256, toPosix, truncateToTokenBudget } from "./util.js";
|
|
17
|
+
import { assertNoSymlinkTraversal, approximateTokens, compareText, readJsonSafe, readText, sha256, toPosix, truncateToTokenBudget } from "./util.js";
|
|
18
18
|
import {
|
|
19
19
|
buildInvertedIndex,
|
|
20
20
|
isCompatibleSearchIndex,
|
|
@@ -33,12 +33,21 @@ export async function loadSearchData(root) {
|
|
|
33
33
|
const { config } = await loadConfig(root);
|
|
34
34
|
await recoverGenerationTransaction(root, { cacheDirectory: config.generation.cacheDirectory });
|
|
35
35
|
const cacheRoot = path.join(root, config.generation.cacheDirectory);
|
|
36
|
-
|
|
36
|
+
await assertNoSymlinkTraversal(root, cacheRoot, config.generation.cacheDirectory);
|
|
37
|
+
const indexPath = path.join(cacheRoot, "index.json");
|
|
38
|
+
const manifestPath = path.join(cacheRoot, "manifest.json");
|
|
39
|
+
const graphPath = path.join(cacheRoot, "graph.json");
|
|
40
|
+
const searchPath = path.join(cacheRoot, "search-index.json");
|
|
41
|
+
const lexiconPath = path.join(root, ".llmnav", "lexicon.json");
|
|
42
|
+
for (const managedPath of [indexPath, manifestPath, graphPath, searchPath, lexiconPath]) {
|
|
43
|
+
await assertNoSymlinkTraversal(root, managedPath, toPosix(path.relative(root, managedPath)));
|
|
44
|
+
}
|
|
45
|
+
const index = await readJsonSafe(indexPath, null);
|
|
37
46
|
if (!index) throw new Error("No generated index found. Run `llmnav generate` first.");
|
|
38
|
-
const lexicon = await readJsonSafe(
|
|
39
|
-
const manifest = await readJsonSafe(
|
|
47
|
+
const lexicon = await readJsonSafe(lexiconPath, { version: 1, aliases: {} });
|
|
48
|
+
const manifest = await readJsonSafe(manifestPath, null);
|
|
40
49
|
const graphRelative = `${toPosix(config.generation.cacheDirectory).replace(/\/+$/u, "")}/graph.json`;
|
|
41
|
-
const graphText = await readText(
|
|
50
|
+
const graphText = await readText(graphPath, null);
|
|
42
51
|
let graph = null;
|
|
43
52
|
if (graphText !== null) {
|
|
44
53
|
try {
|
|
@@ -54,7 +63,7 @@ export async function loadSearchData(root) {
|
|
|
54
63
|
);
|
|
55
64
|
if (!graphMatches) graph = null;
|
|
56
65
|
const searchRelative = `${toPosix(config.generation.cacheDirectory).replace(/\/+$/u, "")}/search-index.json`;
|
|
57
|
-
const searchText = await readText(
|
|
66
|
+
const searchText = await readText(searchPath, null);
|
|
58
67
|
let searchIndex = null;
|
|
59
68
|
if (searchText !== null) {
|
|
60
69
|
try {
|
package/src/spec.js
CHANGED
|
@@ -10,7 +10,7 @@ rel=workflow>llmnav.rules.validate
|
|
|
10
10
|
stability=contract
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
export const PACKAGE_VERSION = "0.6.
|
|
13
|
+
export const PACKAGE_VERSION = "0.6.4";
|
|
14
14
|
export const SPEC_VERSION = "1";
|
|
15
15
|
|
|
16
16
|
export const SCOPES = Object.freeze(["file", "module", "symbol"]);
|
package/src/transaction.js
CHANGED
|
@@ -47,6 +47,7 @@ export async function acquireGenerationLock(root, options = {}) {
|
|
|
47
47
|
const controlDirectory = path.join(root, ".llmnav");
|
|
48
48
|
const lockPath = path.join(controlDirectory, "generation.lock");
|
|
49
49
|
await assertNoSymlinkTraversal(root, controlDirectory, ".llmnav");
|
|
50
|
+
await assertNoSymlinkTraversal(root, lockPath, ".llmnav/generation.lock");
|
|
50
51
|
await mkdir(controlDirectory, { recursive: true });
|
|
51
52
|
const ownerId = options.ownerId ?? createTransactionId();
|
|
52
53
|
const timeoutMs = options.timeoutMs ?? DEFAULT_LOCK_TIMEOUT_MS;
|
|
@@ -95,6 +96,8 @@ export async function commitGeneratedCache(root, cacheDirectory, artifacts, opti
|
|
|
95
96
|
const transactionsDirectory = path.join(controlDirectory, ".transactions");
|
|
96
97
|
const journalPath = path.join(controlDirectory, "generation-transaction.json");
|
|
97
98
|
await assertNoSymlinkTraversal(root, controlDirectory, ".llmnav");
|
|
99
|
+
await assertNoSymlinkTraversal(root, transactionsDirectory, ".llmnav/.transactions");
|
|
100
|
+
await assertNoSymlinkTraversal(root, journalPath, ".llmnav/generation-transaction.json");
|
|
98
101
|
await assertNoSymlinkTraversal(root, cachePath, cacheRelative);
|
|
99
102
|
const recovery = await recoverGenerationTransaction(root, {
|
|
100
103
|
cacheDirectory: cacheRelative,
|
|
@@ -106,6 +109,7 @@ export async function commitGeneratedCache(root, cacheDirectory, artifacts, opti
|
|
|
106
109
|
const transactionPath = path.join(transactionsDirectory, transactionId);
|
|
107
110
|
const stagePath = path.join(transactionPath, "stage");
|
|
108
111
|
const backupPath = path.join(transactionPath, "backup");
|
|
112
|
+
await assertNoSymlinkTraversal(root, transactionPath, relativePosix(root, transactionPath));
|
|
109
113
|
const hadExistingCacheAtStart = await pathExists(cachePath);
|
|
110
114
|
let controlRecords = [];
|
|
111
115
|
await mkdir(stagePath, { recursive: true });
|
|
@@ -218,6 +222,11 @@ export async function recoverGenerationTransaction(root, options = {}) {
|
|
|
218
222
|
const controlDirectory = path.join(root, ".llmnav");
|
|
219
223
|
await assertNoSymlinkTraversal(root, controlDirectory, ".llmnav");
|
|
220
224
|
const journalPath = path.join(controlDirectory, "generation-transaction.json");
|
|
225
|
+
await assertNoSymlinkTraversal(root, journalPath, ".llmnav/generation-transaction.json");
|
|
226
|
+
if (options.cacheDirectory) {
|
|
227
|
+
const configuredCacheDirectory = projectRelativePath(options.cacheDirectory, "cacheDirectory");
|
|
228
|
+
await assertNoSymlinkTraversal(root, path.join(root, configuredCacheDirectory), configuredCacheDirectory);
|
|
229
|
+
}
|
|
221
230
|
const journal = await readJson(journalPath, null);
|
|
222
231
|
if (!journal) return { recovered: false, action: "none" };
|
|
223
232
|
validateJournal(root, journal, options.cacheDirectory);
|
package/src/util.js
CHANGED
|
@@ -77,11 +77,26 @@ export function relativePosix(root, absolutePath) {
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
export function lineAtOffset(source, offset) {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
80
|
+
return lineAtOffsetFromStarts(buildLineStarts(source), offset);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function buildLineStarts(source) {
|
|
84
|
+
const starts = [0];
|
|
85
|
+
for (let index = 0; index < source.length; index += 1) {
|
|
86
|
+
if (source.charCodeAt(index) === 10) starts.push(index + 1);
|
|
87
|
+
}
|
|
88
|
+
return starts;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export function lineAtOffsetFromStarts(lineStarts, offset) {
|
|
92
|
+
let low = 0;
|
|
93
|
+
let high = lineStarts.length;
|
|
94
|
+
while (low < high) {
|
|
95
|
+
const middle = Math.floor((low + high) / 2);
|
|
96
|
+
if (lineStarts[middle] <= offset) low = middle + 1;
|
|
97
|
+
else high = middle;
|
|
83
98
|
}
|
|
84
|
-
return
|
|
99
|
+
return Math.max(1, low);
|
|
85
100
|
}
|
|
86
101
|
|
|
87
102
|
export function offsetAtLine(source, targetLine) {
|