@danypops/lector 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -7
- package/package.json +5 -1
- package/src/adapters/fallback-code-intelligence-index.ts +73 -0
- package/src/adapters/find-source-files.ts +1 -1
- package/src/adapters/git-repo-fetcher.ts +154 -37
- package/src/adapters/lsp/json-rpc-stream.ts +52 -2
- package/src/adapters/lsp/language-server-process.ts +54 -10
- package/src/adapters/lsp/lsp-symbol-index.ts +133 -32
- package/src/adapters/normalize-npm-repository.ts +77 -0
- package/src/adapters/npm-lockfile-version-resolver.ts +519 -0
- package/src/adapters/npm-package-source-resolver.ts +322 -0
- package/src/adapters/npm-registry-client.ts +304 -0
- package/src/adapters/sqlite-symbol-graph.ts +47 -3
- package/src/adapters/tree-sitter/typescript-tree-sitter-symbol-index.ts +61 -11
- package/src/adapters/typescript-compiler-symbol-index.ts +129 -0
- package/src/cli.ts +87 -23
- package/src/daemon.ts +4 -0
- package/src/domain/find-workspace-symbols.ts +4 -3
- package/src/domain/installed-package-version.ts +66 -0
- package/src/domain/intelligence-provenance.ts +19 -0
- package/src/domain/language-server-descriptor.ts +18 -0
- package/src/domain/npm-package-metadata.ts +26 -0
- package/src/domain/package-source.ts +143 -0
- package/src/domain/repo-fetch-result.ts +33 -1
- package/src/domain/resolve-package-source.ts +204 -0
- package/src/domain/symbol-graph-generation.ts +3 -0
- package/src/domain/symbol-query.ts +16 -0
- package/src/domain/workspace-symbol.ts +8 -0
- package/src/index.ts +61 -4
- package/src/ports/installed-package-version-resolver-port.ts +5 -0
- package/src/ports/npm-registry-port.ts +5 -0
- package/src/ports/package-source-resolver-port.ts +5 -0
- package/src/ports/repo-fetcher-port.ts +2 -2
- package/src/ports/symbol-index-port.ts +4 -2
- package/src/service.ts +89 -25
package/README.md
CHANGED
|
@@ -42,6 +42,7 @@ lector workspace symbols <workspace-id> <query>
|
|
|
42
42
|
lector workspace populate-symbol-graph <workspace-id> --max-files <n> --max-symbols-per-file <n> --background --wait-ms 500
|
|
43
43
|
lector job status <job-id>
|
|
44
44
|
lector workspace cache-status <workspace-id> --max-files <n> --max-symbols-per-file <n>
|
|
45
|
+
lector package source <project-dir> <package-name> [--version <exact-version>] [--registry <url>] [--json]
|
|
45
46
|
```
|
|
46
47
|
|
|
47
48
|
Background jobs are process-lifetime and bounded. A daemon restart or retention
|
|
@@ -49,18 +50,19 @@ expiry makes an old id unavailable; `job status` reports that explicitly. A runn
|
|
|
49
50
|
scan returns its job id and an actionable still-loading state instead of blocking
|
|
50
51
|
the caller.
|
|
51
52
|
|
|
53
|
+
`package source` resolves the installed version from npm, pnpm, Yarn, or Bun lockfiles, then verifies registry repository metadata against an exact Git ref, commit, and source `package.json`. Missing, ambiguous, or mismatched source fails closed. Verified package directories are registered read-only. Private registry requests use `NPM_TOKEN`; output reports only that variable name when authentication is required.
|
|
54
|
+
|
|
52
55
|
`exactEdit` requires either `--create` (the entry must not already exist) or
|
|
53
56
|
`--expected-hash <hash>` (the entry must currently match that hash) — a write never
|
|
54
57
|
silently overwrites content it didn't know about.
|
|
55
58
|
|
|
56
59
|
## Symbol backends
|
|
57
60
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
+
Three independent `SymbolIndexPort` implementations use one result DTO with explicit provenance and truncation:
|
|
62
|
+
|
|
63
|
+
- **`typescript-language-server`** is semantic authority for identity, definitions, references, implementations, hover, diagnostics, symbols, and calls.
|
|
64
|
+
- **TypeScript compiler** extracts bounded structural declarations from cold, malformed, or partially configured projects. It does not claim cross-file identity.
|
|
65
|
+
- **tree-sitter** extracts bounded structural declarations and caches them by content hash. It does not claim type or language-server identity.
|
|
61
66
|
|
|
62
|
-
|
|
63
|
-
workspace, most accurate when it has a loaded project.
|
|
64
|
-
- **tree-sitter** (via `web-tree-sitter`, WASM — no native toolchain required) — no
|
|
65
|
-
subprocess, always current, results cached per file under a content-addressed key.
|
|
67
|
+
Symbol results report `fidelity`, `backend`, `authority`, `freshness`, `limitations`, and `truncated`. Language-server messages, pending requests, opened files, file bytes, settling, parser files, parser nodes, parser bytes, and returned symbols are bounded.
|
|
66
68
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@danypops/lector",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.8",
|
|
4
4
|
"description": "Platform-neutral filesystem & code-intelligence capability: core, service, and driven adapters",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -18,8 +18,11 @@
|
|
|
18
18
|
},
|
|
19
19
|
"dependencies": {
|
|
20
20
|
"@danypops/daemon-kit": "^0.3.2",
|
|
21
|
+
"@yarnpkg/parsers": "^3.0.3",
|
|
21
22
|
"bash-language-server": "^5.6.0",
|
|
23
|
+
"fetch-retry": "^6.0.0",
|
|
22
24
|
"graphology": "^0.26.0",
|
|
25
|
+
"jsonc-parser": "^3.3.1",
|
|
23
26
|
"lru-cache": "^11.5.2",
|
|
24
27
|
"pyright": "^1.1.411",
|
|
25
28
|
"simple-git": "^3.36.0",
|
|
@@ -27,6 +30,7 @@
|
|
|
27
30
|
"typescript": "^5.9.3",
|
|
28
31
|
"typescript-language-server": "^5.3.0",
|
|
29
32
|
"web-tree-sitter": "0.20.8",
|
|
33
|
+
"yaml": "^2.9.0",
|
|
30
34
|
"yaml-language-server": "^1.24.0"
|
|
31
35
|
},
|
|
32
36
|
"devDependencies": {
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { CallHierarchyEntry, IncomingCall, OutgoingCall } from "../domain/call-hierarchy.ts";
|
|
2
|
+
import type { Diagnostic } from "../domain/diagnostic.ts";
|
|
3
|
+
import type { DocumentSymbolEntry } from "../domain/document-symbol.ts";
|
|
4
|
+
import type { Hover } from "../domain/hover.ts";
|
|
5
|
+
import type { IntelligenceProvenance, SymbolSearchBounds } from "../domain/intelligence-provenance.ts";
|
|
6
|
+
import type { SymbolSearchResult, WorkspaceLocation } from "../domain/workspace-symbol.ts";
|
|
7
|
+
import type { CodeIntelligencePort } from "../ports/code-intelligence-port.ts";
|
|
8
|
+
import type { SymbolIndexPort } from "../ports/symbol-index-port.ts";
|
|
9
|
+
|
|
10
|
+
export type ClosableIntelligenceIndex = SymbolIndexPort & { close(): Promise<void> };
|
|
11
|
+
|
|
12
|
+
/** Structural fallbacks serve name discovery only; identity-aware operations always stay on the semantic primary. */
|
|
13
|
+
export class FallbackCodeIntelligenceIndex implements SymbolIndexPort, CodeIntelligencePort {
|
|
14
|
+
readonly provenance: IntelligenceProvenance;
|
|
15
|
+
|
|
16
|
+
constructor(
|
|
17
|
+
private readonly primary: ClosableIntelligenceIndex & CodeIntelligencePort,
|
|
18
|
+
private readonly fallbacks: readonly ClosableIntelligenceIndex[],
|
|
19
|
+
) {
|
|
20
|
+
this.provenance = primary.provenance;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async findSymbols(query: string, bounds?: SymbolSearchBounds): Promise<SymbolSearchResult> {
|
|
24
|
+
try {
|
|
25
|
+
return await this.primary.findSymbols(query, bounds);
|
|
26
|
+
} catch (primaryError) {
|
|
27
|
+
let emptyResult: SymbolSearchResult | undefined;
|
|
28
|
+
for (const fallback of this.fallbacks) {
|
|
29
|
+
try {
|
|
30
|
+
const result = await fallback.findSymbols(query, bounds);
|
|
31
|
+
if (result.symbols.length > 0) return result;
|
|
32
|
+
emptyResult ??= result;
|
|
33
|
+
} catch {
|
|
34
|
+
// Continue through the bounded fallback chain; the primary error remains authoritative if none can answer.
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
if (emptyResult) return emptyResult;
|
|
38
|
+
throw primaryError;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
goToDefinition(at: WorkspaceLocation) {
|
|
43
|
+
return this.primary.goToDefinition(at);
|
|
44
|
+
}
|
|
45
|
+
goToImplementation(at: WorkspaceLocation) {
|
|
46
|
+
return this.primary.goToImplementation(at);
|
|
47
|
+
}
|
|
48
|
+
findReferences(at: WorkspaceLocation, includeDeclaration: boolean) {
|
|
49
|
+
return this.primary.findReferences(at, includeDeclaration);
|
|
50
|
+
}
|
|
51
|
+
hover(at: WorkspaceLocation): Promise<Hover | undefined> {
|
|
52
|
+
return this.primary.hover(at);
|
|
53
|
+
}
|
|
54
|
+
documentSymbols(path: string): Promise<DocumentSymbolEntry[]> {
|
|
55
|
+
return this.primary.documentSymbols(path);
|
|
56
|
+
}
|
|
57
|
+
diagnostics(path: string): Promise<Diagnostic[]> {
|
|
58
|
+
return this.primary.diagnostics(path);
|
|
59
|
+
}
|
|
60
|
+
prepareCallHierarchy(at: WorkspaceLocation): Promise<CallHierarchyEntry[]> {
|
|
61
|
+
return this.primary.prepareCallHierarchy(at);
|
|
62
|
+
}
|
|
63
|
+
incomingCalls(at: WorkspaceLocation): Promise<IncomingCall[]> {
|
|
64
|
+
return this.primary.incomingCalls(at);
|
|
65
|
+
}
|
|
66
|
+
outgoingCalls(at: WorkspaceLocation): Promise<OutgoingCall[]> {
|
|
67
|
+
return this.primary.outgoingCalls(at);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async close(): Promise<void> {
|
|
71
|
+
await Promise.allSettled([this.primary.close(), ...this.fallbacks.map((fallback) => fallback.close())]);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -17,7 +17,7 @@ export function findSourceFiles(rootPath: string, isSourceExtension: (extension:
|
|
|
17
17
|
} catch {
|
|
18
18
|
return;
|
|
19
19
|
}
|
|
20
|
-
for (const entry of entries) {
|
|
20
|
+
for (const entry of [...entries].sort((left, right) => left.name.localeCompare(right.name))) {
|
|
21
21
|
if (scanned >= maxFiles) return;
|
|
22
22
|
const relativePath = relativeDir ? join(relativeDir, entry.name) : entry.name;
|
|
23
23
|
if (entry.isDirectory()) {
|
|
@@ -4,7 +4,7 @@ import { dirname, join } from "node:path";
|
|
|
4
4
|
import { LRUCache } from "lru-cache";
|
|
5
5
|
import simpleGit from "simple-git";
|
|
6
6
|
import { assertSafeRepoReference } from "../domain/assert-safe-repo-reference.ts";
|
|
7
|
-
import { RepoFetchFailed, type RepoFetchResult } from "../domain/repo-fetch-result.ts";
|
|
7
|
+
import { RepoFetchCapacityExceeded, RepoFetchFailed, RepoFetchLimitExceeded, type RepoFetchPolicy, type RepoFetchResult } from "../domain/repo-fetch-result.ts";
|
|
8
8
|
import type { RepoReference } from "../domain/repo-reference.ts";
|
|
9
9
|
import type { RepoFetcherPort } from "../ports/repo-fetcher-port.ts";
|
|
10
10
|
import { measureDirectorySizeBytes } from "./directory-size.ts";
|
|
@@ -12,12 +12,18 @@ import { measureDirectorySizeBytes } from "./directory-size.ts";
|
|
|
12
12
|
interface RepoCacheEntry {
|
|
13
13
|
readonly path: string;
|
|
14
14
|
readonly resolvedRef: string;
|
|
15
|
+
readonly commit: string;
|
|
16
|
+
readonly cloneSizeBytes: number;
|
|
17
|
+
readonly cacheSizeBytes: number;
|
|
15
18
|
readonly fetchedAt: number;
|
|
16
19
|
}
|
|
17
20
|
|
|
18
21
|
const INDEX_FILENAME = "index.json";
|
|
19
22
|
const DEFAULT_MAX_CACHE_BYTES = 5 * 1024 * 1024 * 1024;
|
|
20
23
|
const DEFAULT_MAX_ENTRIES = 500;
|
|
24
|
+
const DEFAULT_TIMEOUT_MS = 60_000;
|
|
25
|
+
const DEFAULT_MAX_QUEUED = 32;
|
|
26
|
+
const COMMIT_HASH = /^[0-9a-f]{40,64}$/i;
|
|
21
27
|
|
|
22
28
|
function cacheKey(reference: RepoReference): string {
|
|
23
29
|
return `${reference.host}/${reference.owner}/${reference.repo}/${reference.ref ?? "HEAD"}`;
|
|
@@ -27,6 +33,7 @@ export interface GitRepoFetcherOptions {
|
|
|
27
33
|
/** Disk budget in bytes across every cached clone. Least-recently-fetched clones are deleted once exceeded. */
|
|
28
34
|
readonly maxCacheBytes?: number;
|
|
29
35
|
readonly maxEntries?: number;
|
|
36
|
+
readonly maxQueued?: number;
|
|
30
37
|
/** Maps a reference to what git should clone from. Defaults to `https://<host>/<owner>/<repo>.git`. Overridable so tests can point at a real local bare-repo fixture instead of the network. */
|
|
31
38
|
readonly resolveCloneUrl?: (reference: RepoReference) => string;
|
|
32
39
|
}
|
|
@@ -35,31 +42,64 @@ function defaultCloneUrl(reference: RepoReference): string {
|
|
|
35
42
|
return `https://${reference.host}/${reference.owner}/${reference.repo}.git`;
|
|
36
43
|
}
|
|
37
44
|
|
|
45
|
+
function positiveLimit(value: number | undefined, fallback: number, field: string): number {
|
|
46
|
+
const result = value ?? fallback;
|
|
47
|
+
if (!Number.isSafeInteger(result) || result < 1) throw new TypeError(`${field} must be a positive safe integer`);
|
|
48
|
+
return result;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function nonNegativeLimit(value: number | undefined, fallback: number, field: string): number {
|
|
52
|
+
const result = value ?? fallback;
|
|
53
|
+
if (!Number.isSafeInteger(result) || result < 0) throw new TypeError(`${field} must be a non-negative safe integer`);
|
|
54
|
+
return result;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
58
|
+
return typeof value === "object" && value !== null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function validDump(value: unknown): value is Parameters<LRUCache<string, RepoCacheEntry>["load"]>[0] {
|
|
62
|
+
if (!Array.isArray(value)) return false;
|
|
63
|
+
return value.every((item) => {
|
|
64
|
+
if (!Array.isArray(item) || item.length !== 2 || typeof item[0] !== "string" || !isRecord(item[1])) return false;
|
|
65
|
+
const entry = item[1].value;
|
|
66
|
+
return (
|
|
67
|
+
isRecord(entry) &&
|
|
68
|
+
typeof entry.path === "string" &&
|
|
69
|
+
typeof entry.resolvedRef === "string" &&
|
|
70
|
+
typeof entry.commit === "string" &&
|
|
71
|
+
COMMIT_HASH.test(entry.commit) &&
|
|
72
|
+
typeof entry.cloneSizeBytes === "number" &&
|
|
73
|
+
typeof entry.cacheSizeBytes === "number" &&
|
|
74
|
+
typeof entry.fetchedAt === "number"
|
|
75
|
+
);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
38
79
|
/**
|
|
39
|
-
* RepoFetcherPort backed by a real `git
|
|
40
|
-
*
|
|
41
|
-
* way; evicting an entry deletes its directory, so the cache and the filesystem never disagree
|
|
42
|
-
* about what's actually on disk. Calls are serialized through an in-process queue -- this daemon
|
|
43
|
-
* is the sole writer of reposDir, so a promise chain is sufficient; no cross-process file lock
|
|
44
|
-
* is needed.
|
|
80
|
+
* RepoFetcherPort backed by a real bounded `git` checkout. Calls are serialized because this
|
|
81
|
+
* daemon is the sole writer of reposDir. Exact callers never fall back to HEAD.
|
|
45
82
|
*/
|
|
46
83
|
export class GitRepoFetcher implements RepoFetcherPort {
|
|
47
84
|
private readonly reposDir: string;
|
|
48
85
|
private readonly indexPath: string;
|
|
49
86
|
private readonly resolveCloneUrl: (reference: RepoReference) => string;
|
|
87
|
+
private readonly maxCacheBytes: number;
|
|
88
|
+
private readonly maxQueued: number;
|
|
50
89
|
private readonly cache: LRUCache<string, RepoCacheEntry>;
|
|
90
|
+
private pending = 0;
|
|
51
91
|
private queue: Promise<unknown> = Promise.resolve();
|
|
52
92
|
|
|
53
93
|
constructor(reposDir: string, options: GitRepoFetcherOptions = {}) {
|
|
54
94
|
this.reposDir = reposDir;
|
|
55
95
|
this.indexPath = join(reposDir, INDEX_FILENAME);
|
|
56
96
|
this.resolveCloneUrl = options.resolveCloneUrl ?? defaultCloneUrl;
|
|
97
|
+
this.maxCacheBytes = positiveLimit(options.maxCacheBytes, DEFAULT_MAX_CACHE_BYTES, "maxCacheBytes");
|
|
98
|
+
this.maxQueued = nonNegativeLimit(options.maxQueued, DEFAULT_MAX_QUEUED, "maxQueued");
|
|
57
99
|
this.cache = new LRUCache<string, RepoCacheEntry>({
|
|
58
|
-
maxSize:
|
|
59
|
-
// A single clone larger than the whole budget must still be stored (evicting everything
|
|
60
|
-
// else) rather than silently rejected -- lru-cache's own default couples this to maxSize.
|
|
100
|
+
maxSize: this.maxCacheBytes,
|
|
61
101
|
maxEntrySize: Number.MAX_SAFE_INTEGER,
|
|
62
|
-
max: options.maxEntries
|
|
102
|
+
max: positiveLimit(options.maxEntries, DEFAULT_MAX_ENTRIES, "maxEntries"),
|
|
63
103
|
dispose: (entry) => {
|
|
64
104
|
rmSync(entry.path, { recursive: true, force: true });
|
|
65
105
|
},
|
|
@@ -67,22 +107,63 @@ export class GitRepoFetcher implements RepoFetcherPort {
|
|
|
67
107
|
this.loadIndex();
|
|
68
108
|
}
|
|
69
109
|
|
|
70
|
-
async fetch(reference: RepoReference): Promise<RepoFetchResult> {
|
|
110
|
+
async fetch(reference: RepoReference, policy: RepoFetchPolicy = {}): Promise<RepoFetchResult> {
|
|
71
111
|
assertSafeRepoReference(reference);
|
|
72
|
-
|
|
112
|
+
if (this.pending > this.maxQueued) throw new RepoFetchCapacityExceeded(this.maxQueued);
|
|
113
|
+
const normalizedPolicy = {
|
|
114
|
+
exactRef: policy.exactRef ?? false,
|
|
115
|
+
maxCloneBytes: positiveLimit(policy.maxCloneBytes, Number.MAX_SAFE_INTEGER, "maxCloneBytes"),
|
|
116
|
+
maxCacheBytes: positiveLimit(policy.maxCacheBytes, this.maxCacheBytes, "maxCacheBytes"),
|
|
117
|
+
timeoutMs: positiveLimit(policy.timeoutMs, DEFAULT_TIMEOUT_MS, "timeoutMs"),
|
|
118
|
+
};
|
|
119
|
+
this.pending++;
|
|
120
|
+
const deadline = Date.now() + normalizedPolicy.timeoutMs;
|
|
121
|
+
const task = this.queue.then(() => {
|
|
122
|
+
const remaining = deadline - Date.now();
|
|
123
|
+
if (remaining <= 0) throw new RepoFetchFailed(reference.host, reference.owner, reference.repo, reference.ref, new Error("fetch timed out in queue"));
|
|
124
|
+
return this.fetchLocked(reference, { ...normalizedPolicy, timeoutMs: remaining });
|
|
125
|
+
});
|
|
73
126
|
this.queue = task.then(
|
|
74
127
|
() => undefined,
|
|
75
128
|
() => undefined,
|
|
76
129
|
);
|
|
77
|
-
return task
|
|
130
|
+
return task.finally(() => {
|
|
131
|
+
this.pending--;
|
|
132
|
+
});
|
|
78
133
|
}
|
|
79
134
|
|
|
80
|
-
private
|
|
81
|
-
|
|
135
|
+
private trimToCacheBound(maxCacheBytes: number, requiredBytes = 0): void {
|
|
136
|
+
while (this.cache.calculatedSize + requiredBytes > maxCacheBytes && this.cache.size > 0) this.cache.pop();
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
private cachedResult(key: string, reference: RepoReference, policy: Required<RepoFetchPolicy>): RepoFetchResult | null {
|
|
82
140
|
const cached = this.cache.get(key);
|
|
83
|
-
if (cached)
|
|
84
|
-
|
|
141
|
+
if (!cached) return null;
|
|
142
|
+
if (policy.exactRef && reference.ref !== null && cached.resolvedRef !== reference.ref) {
|
|
143
|
+
this.cache.delete(key);
|
|
144
|
+
return null;
|
|
85
145
|
}
|
|
146
|
+
if (cached.cloneSizeBytes > policy.maxCloneBytes) {
|
|
147
|
+
throw new RepoFetchLimitExceeded("clone-bytes", policy.maxCloneBytes, cached.cloneSizeBytes);
|
|
148
|
+
}
|
|
149
|
+
if (cached.cacheSizeBytes > policy.maxCacheBytes) {
|
|
150
|
+
throw new RepoFetchLimitExceeded("cache-bytes", policy.maxCacheBytes, cached.cacheSizeBytes);
|
|
151
|
+
}
|
|
152
|
+
this.trimToCacheBound(policy.maxCacheBytes);
|
|
153
|
+
if (!this.cache.has(key)) throw new RepoFetchLimitExceeded("cache-bytes", policy.maxCacheBytes, cached.cacheSizeBytes);
|
|
154
|
+
return {
|
|
155
|
+
path: cached.path,
|
|
156
|
+
fromCache: true,
|
|
157
|
+
resolvedRef: cached.resolvedRef,
|
|
158
|
+
refFallbackOccurred: false,
|
|
159
|
+
commit: cached.commit,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
private async fetchLocked(reference: RepoReference, policy: Required<RepoFetchPolicy>): Promise<RepoFetchResult> {
|
|
164
|
+
const key = cacheKey(reference);
|
|
165
|
+
const cached = this.cachedResult(key, reference, policy);
|
|
166
|
+
if (cached) return cached;
|
|
86
167
|
|
|
87
168
|
const targetDir = join(this.reposDir, reference.host, reference.owner, reference.repo, reference.ref ?? "HEAD");
|
|
88
169
|
await rm(targetDir, { recursive: true, force: true });
|
|
@@ -93,47 +174,83 @@ export class GitRepoFetcher implements RepoFetcherPort {
|
|
|
93
174
|
let resolvedRef = reference.ref ?? "HEAD";
|
|
94
175
|
let refFallbackOccurred = false;
|
|
95
176
|
try {
|
|
96
|
-
|
|
177
|
+
if (policy.exactRef && reference.ref !== null) {
|
|
178
|
+
await this.cloneExact(url, reference.ref, tmpDir, policy.timeoutMs);
|
|
179
|
+
} else {
|
|
180
|
+
await this.clone(url, reference.ref, tmpDir, policy.timeoutMs);
|
|
181
|
+
}
|
|
97
182
|
} catch (firstError) {
|
|
98
|
-
if (reference.ref === null) {
|
|
183
|
+
if (reference.ref === null || policy.exactRef) {
|
|
184
|
+
await rm(tmpDir, { recursive: true, force: true });
|
|
99
185
|
throw new RepoFetchFailed(reference.host, reference.owner, reference.repo, reference.ref, firstError);
|
|
100
186
|
}
|
|
101
187
|
await rm(tmpDir, { recursive: true, force: true });
|
|
102
188
|
try {
|
|
103
|
-
await this.clone(url, null, tmpDir);
|
|
189
|
+
await this.clone(url, null, tmpDir, policy.timeoutMs);
|
|
104
190
|
refFallbackOccurred = true;
|
|
105
191
|
resolvedRef = "HEAD";
|
|
106
192
|
} catch (secondError) {
|
|
193
|
+
await rm(tmpDir, { recursive: true, force: true });
|
|
107
194
|
throw new RepoFetchFailed(reference.host, reference.owner, reference.repo, reference.ref, secondError);
|
|
108
195
|
}
|
|
109
196
|
}
|
|
110
197
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
198
|
+
try {
|
|
199
|
+
const git = simpleGit({ baseDir: tmpDir, timeout: { block: policy.timeoutMs } });
|
|
200
|
+
const commit = (await git.revparse(["HEAD"])).trim();
|
|
201
|
+
if (!COMMIT_HASH.test(commit)) throw new Error("git returned an invalid commit id");
|
|
202
|
+
const cloneSizeBytes = await measureDirectorySizeBytes(tmpDir);
|
|
203
|
+
if (cloneSizeBytes > policy.maxCloneBytes) throw new RepoFetchLimitExceeded("clone-bytes", policy.maxCloneBytes, cloneSizeBytes);
|
|
204
|
+
await rm(join(tmpDir, ".git"), { recursive: true, force: true });
|
|
205
|
+
const cacheSizeBytes = await measureDirectorySizeBytes(tmpDir);
|
|
206
|
+
if (cacheSizeBytes > policy.maxCacheBytes) throw new RepoFetchLimitExceeded("cache-bytes", policy.maxCacheBytes, cacheSizeBytes);
|
|
207
|
+
this.trimToCacheBound(policy.maxCacheBytes, cacheSizeBytes);
|
|
208
|
+
await mkdir(dirname(targetDir), { recursive: true });
|
|
209
|
+
await rename(tmpDir, targetDir);
|
|
114
210
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
211
|
+
const entry: RepoCacheEntry = { path: targetDir, resolvedRef, commit, cloneSizeBytes, cacheSizeBytes, fetchedAt: Date.now() };
|
|
212
|
+
this.cache.set(key, entry, { size: cacheSizeBytes });
|
|
213
|
+
this.persistIndex();
|
|
214
|
+
return { path: targetDir, fromCache: false, resolvedRef, refFallbackOccurred, commit };
|
|
215
|
+
} catch (error) {
|
|
216
|
+
await rm(tmpDir, { recursive: true, force: true });
|
|
217
|
+
throw error;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
119
220
|
|
|
120
|
-
|
|
221
|
+
private async clone(url: string, ref: string | null, targetDir: string, timeoutMs: number): Promise<void> {
|
|
222
|
+
const controller = new AbortController();
|
|
223
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
224
|
+
try {
|
|
225
|
+
const options = ref ? ["--quiet", "--depth", "1", "--branch", ref] : ["--quiet", "--depth", "1"];
|
|
226
|
+
await simpleGit({ timeout: { block: timeoutMs }, abort: controller.signal }).clone(url, targetDir, options);
|
|
227
|
+
} finally {
|
|
228
|
+
clearTimeout(timer);
|
|
229
|
+
}
|
|
121
230
|
}
|
|
122
231
|
|
|
123
|
-
private async
|
|
124
|
-
const
|
|
125
|
-
|
|
232
|
+
private async cloneExact(url: string, ref: string, targetDir: string, timeoutMs: number): Promise<void> {
|
|
233
|
+
const controller = new AbortController();
|
|
234
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
235
|
+
try {
|
|
236
|
+
await mkdir(targetDir, { recursive: true });
|
|
237
|
+
const git = simpleGit({ baseDir: targetDir, timeout: { block: timeoutMs }, abort: controller.signal });
|
|
238
|
+
await git.init(["--quiet"]);
|
|
239
|
+
await git.addRemote("origin", url);
|
|
240
|
+
await git.raw(["fetch", "--quiet", "--depth", "1", "origin", ref]);
|
|
241
|
+
await git.raw(["checkout", "--quiet", "--detach", "FETCH_HEAD"]);
|
|
242
|
+
} finally {
|
|
243
|
+
clearTimeout(timer);
|
|
244
|
+
}
|
|
126
245
|
}
|
|
127
246
|
|
|
128
247
|
private loadIndex(): void {
|
|
129
248
|
if (!existsSync(this.indexPath)) return;
|
|
130
249
|
try {
|
|
131
|
-
const dumped = JSON.parse(readFileSync(this.indexPath, "utf8"))
|
|
132
|
-
this.cache.load(dumped);
|
|
250
|
+
const dumped: unknown = JSON.parse(readFileSync(this.indexPath, "utf8"));
|
|
251
|
+
if (validDump(dumped)) this.cache.load(dumped);
|
|
133
252
|
} catch {
|
|
134
|
-
//
|
|
135
|
-
// prior run are cleaned up lazily the next time their key is fetched again (fetchLocked
|
|
136
|
-
// always rm's targetDir before cloning into it).
|
|
253
|
+
// An invalid index is disposable; fetched repositories can be cloned again.
|
|
137
254
|
}
|
|
138
255
|
}
|
|
139
256
|
|
|
@@ -14,6 +14,36 @@ export interface JsonRpcMessage {
|
|
|
14
14
|
readonly error?: { readonly code: number; readonly message: string; readonly data?: unknown };
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
+
export class JsonRpcMessageLimitExceeded extends Error {
|
|
18
|
+
constructor(
|
|
19
|
+
readonly limit: "header-bytes" | "message-bytes" | "buffered-bytes",
|
|
20
|
+
readonly maxBytes: number,
|
|
21
|
+
readonly observedBytes: number,
|
|
22
|
+
) {
|
|
23
|
+
super(`JSON-RPC ${limit} exceeded ${maxBytes} bytes (observed ${observedBytes})`);
|
|
24
|
+
this.name = "JsonRpcMessageLimitExceeded";
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
29
|
+
return typeof value === "object" && value !== null;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function isJsonRpcMessage(value: unknown): value is JsonRpcMessage {
|
|
33
|
+
if (!isRecord(value) || value.jsonrpc !== "2.0") return false;
|
|
34
|
+
if (value.id !== undefined && typeof value.id !== "number" && typeof value.id !== "string") return false;
|
|
35
|
+
if (value.method !== undefined && typeof value.method !== "string") return false;
|
|
36
|
+
if (value.error !== undefined) {
|
|
37
|
+
if (!isRecord(value.error) || typeof value.error.code !== "number" || typeof value.error.message !== "string") return false;
|
|
38
|
+
}
|
|
39
|
+
return true;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface JsonRpcStreamDecoderOptions {
|
|
43
|
+
readonly maxHeaderBytes?: number;
|
|
44
|
+
readonly maxMessageBytes?: number;
|
|
45
|
+
}
|
|
46
|
+
|
|
17
47
|
export function encodeJsonRpcMessage(message: JsonRpcMessage): Buffer {
|
|
18
48
|
const body = Buffer.from(JSON.stringify(message), "utf-8");
|
|
19
49
|
const header = Buffer.from(`Content-Length: ${body.byteLength}\r\n\r\n`, "ascii");
|
|
@@ -28,15 +58,31 @@ export function encodeJsonRpcMessage(message: JsonRpcMessage): Buffer {
|
|
|
28
58
|
*/
|
|
29
59
|
export class JsonRpcStreamDecoder {
|
|
30
60
|
private buffer: Buffer = Buffer.alloc(0);
|
|
61
|
+
private readonly maxHeaderBytes: number;
|
|
62
|
+
private readonly maxMessageBytes: number;
|
|
63
|
+
|
|
64
|
+
constructor(options: JsonRpcStreamDecoderOptions = {}) {
|
|
65
|
+
this.maxHeaderBytes = options.maxHeaderBytes ?? 8 * 1024;
|
|
66
|
+
this.maxMessageBytes = options.maxMessageBytes ?? 8 * 1024 * 1024;
|
|
67
|
+
if (!Number.isSafeInteger(this.maxHeaderBytes) || this.maxHeaderBytes < 1) throw new TypeError("maxHeaderBytes must be a positive safe integer");
|
|
68
|
+
if (!Number.isSafeInteger(this.maxMessageBytes) || this.maxMessageBytes < 1) throw new TypeError("maxMessageBytes must be a positive safe integer");
|
|
69
|
+
}
|
|
31
70
|
|
|
32
71
|
/** Feed one chunk; returns every complete message it produced (zero or more, in order). */
|
|
33
72
|
push(chunk: Buffer): JsonRpcMessage[] {
|
|
73
|
+
const bufferedBytes = this.buffer.byteLength + chunk.byteLength;
|
|
74
|
+
const maxBufferedBytes = this.maxHeaderBytes + this.maxMessageBytes;
|
|
75
|
+
if (bufferedBytes > maxBufferedBytes) throw new JsonRpcMessageLimitExceeded("buffered-bytes", maxBufferedBytes, bufferedBytes);
|
|
34
76
|
this.buffer = Buffer.concat([this.buffer, chunk]);
|
|
35
77
|
const messages: JsonRpcMessage[] = [];
|
|
36
78
|
|
|
37
79
|
for (;;) {
|
|
38
80
|
const headerEnd = this.buffer.indexOf("\r\n\r\n");
|
|
39
|
-
if (headerEnd === -1)
|
|
81
|
+
if (headerEnd === -1) {
|
|
82
|
+
if (this.buffer.byteLength > this.maxHeaderBytes) throw new JsonRpcMessageLimitExceeded("header-bytes", this.maxHeaderBytes, this.buffer.byteLength);
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
if (headerEnd > this.maxHeaderBytes) throw new JsonRpcMessageLimitExceeded("header-bytes", this.maxHeaderBytes, headerEnd);
|
|
40
86
|
|
|
41
87
|
const header = this.buffer.subarray(0, headerEnd).toString("ascii");
|
|
42
88
|
const match = /Content-Length:\s*(\d+)/i.exec(header);
|
|
@@ -49,6 +95,9 @@ export class JsonRpcStreamDecoder {
|
|
|
49
95
|
}
|
|
50
96
|
|
|
51
97
|
const length = Number.parseInt(digits, 10);
|
|
98
|
+
if (!Number.isSafeInteger(length) || length > this.maxMessageBytes) {
|
|
99
|
+
throw new JsonRpcMessageLimitExceeded("message-bytes", this.maxMessageBytes, length);
|
|
100
|
+
}
|
|
52
101
|
const bodyStart = headerEnd + 4;
|
|
53
102
|
const bodyEnd = bodyStart + length;
|
|
54
103
|
if (this.buffer.byteLength < bodyEnd) break; // body split across chunks -- wait for more
|
|
@@ -56,7 +105,8 @@ export class JsonRpcStreamDecoder {
|
|
|
56
105
|
const body = this.buffer.subarray(bodyStart, bodyEnd).toString("utf-8");
|
|
57
106
|
this.buffer = this.buffer.subarray(bodyEnd);
|
|
58
107
|
try {
|
|
59
|
-
|
|
108
|
+
const value: unknown = JSON.parse(body);
|
|
109
|
+
if (isJsonRpcMessage(value)) messages.push(value);
|
|
60
110
|
} catch {
|
|
61
111
|
// An unparseable body must not crash the whole stream -- skip it and continue.
|
|
62
112
|
}
|