@cassiomc1/forgeloop 1.10.2 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursor/rules/project-loop.mdc +2 -2
- package/.forgeloop/forgeloop.gitignore +1 -0
- package/.github/copilot-instructions.md +2 -2
- package/AGENTS.md +1 -0
- package/AGENT_COMPATIBILITY.md +7 -0
- package/CLAUDE.md +1 -0
- package/DOCS_INDEX.md +10 -0
- package/LOOP_SYSTEM_DESIGN.md +12 -0
- package/ORCHESTRATOR_INTEGRATION.md +9 -0
- package/PROTOCOL_INTEGRATION.md +17 -0
- package/README.md +12 -10
- package/THIRD_PARTY_NOTICES.md +11 -0
- package/THREAT_MODEL.md +21 -0
- package/benchmarks/repository-index/README.md +73 -0
- package/benchmarks/repository-index/queries.json +12 -0
- package/benchmarks/repository-index/run-hot-path.mjs +142 -0
- package/benchmarks/repository-index/run-persistent-transport.mjs +169 -0
- package/completions/_forgeloop +7 -1
- package/completions/forgeloop.bash +13 -1
- package/completions/forgeloop.fish +40 -1
- package/docs/AGENT_PROTOCOL_SUMMARY.md +8 -1
- package/docs/CLI_REFERENCE.md +114 -2
- package/docs/DOCUMENTATION_GUIDE.md +10 -5
- package/docs/GETTING_STARTED.md +17 -0
- package/docs/MCP.md +16 -1
- package/docs/PACKAGE_CONTENTS.md +7 -1
- package/docs/PERSISTENT_SEARCH_TRANSPORT.md +289 -0
- package/docs/RECIPES.md +30 -0
- package/docs/RELEASE_CHECKLIST.md +34 -0
- package/docs/REPOSITORY_INDEX.md +553 -0
- package/docs/TROUBLESHOOTING.md +147 -0
- package/docs/UNIVERSAL_INTEGRATION.md +30 -0
- package/docs/diagrams/README.md +10 -0
- package/package.json +11 -2
- package/scripts/update-tgrep-manifest.mjs +86 -0
- package/scripts/verify-tgrep-manifest.mjs +17 -0
- package/src/cli.js +35 -0
- package/src/commands/doctor.js +76 -1
- package/src/commands/index-rebuild.js +1 -0
- package/src/commands/index-setup.js +1 -0
- package/src/commands/index-start.js +1 -0
- package/src/commands/index-status.js +1 -0
- package/src/commands/index-stop.js +1 -0
- package/src/commands/init.js +39 -1
- package/src/commands/repository-index.js +111 -0
- package/src/commands/search.js +1 -0
- package/src/commands/update.js +32 -4
- package/src/core/cli-command-definitions.js +97 -3
- package/src/core/command-executors.js +35 -2
- package/src/core/command-input.js +23 -0
- package/src/core/error-codes.js +195 -0
- package/src/core/filesystem.js +10 -1
- package/src/core/integration-invocation-policy.js +27 -0
- package/src/core/integration-resources.js +16 -1
- package/src/core/protocol-info.js +23 -0
- package/src/integration.d.ts +90 -0
- package/src/integration.js +21 -0
- package/src/persistent-transport/client.js +293 -0
- package/src/persistent-transport/constants.js +24 -0
- package/src/persistent-transport/errors.js +38 -0
- package/src/persistent-transport/framing.js +61 -0
- package/src/persistent-transport/lifecycle.js +116 -0
- package/src/persistent-transport/ownership.js +184 -0
- package/src/persistent-transport/paths.js +31 -0
- package/src/persistent-transport/protocol.js +95 -0
- package/src/persistent-transport/server.js +256 -0
- package/src/persistent-transport/state.js +49 -0
- package/src/repository-index/args.js +59 -0
- package/src/repository-index/binary-manager.js +413 -0
- package/src/repository-index/constants.js +45 -0
- package/src/repository-index/errors.js +38 -0
- package/src/repository-index/lifecycle.js +17 -0
- package/src/repository-index/lock.js +113 -0
- package/src/repository-index/manifest.js +132 -0
- package/src/repository-index/metrics.js +30 -0
- package/src/repository-index/normalize-json.js +187 -0
- package/src/repository-index/paths.js +39 -0
- package/src/repository-index/platform.js +20 -0
- package/src/repository-index/process.js +140 -0
- package/src/repository-index/readiness.js +62 -0
- package/src/repository-index/search.js +262 -0
- package/src/repository-index/server.js +432 -0
- package/src/repository-index/status.js +397 -0
- package/src/repository-index/tgrep-manifest.json +38 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
import {
|
|
5
|
+
REPOSITORY_INDEX_ENGINE,
|
|
6
|
+
REPOSITORY_INDEX_SUPPORTED_PLATFORMS,
|
|
7
|
+
} from "./constants.js";
|
|
8
|
+
import { REPOSITORY_INDEX_ERROR_CODES, repositoryIndexError } from "./errors.js";
|
|
9
|
+
import { getPackageRoot } from "../core/templates.js";
|
|
10
|
+
|
|
11
|
+
const SEMVER = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/;
|
|
12
|
+
const SHA256 = /^[a-f0-9]{64}$/;
|
|
13
|
+
const ARCHIVES = new Set(["tar.gz", "zip"]);
|
|
14
|
+
|
|
15
|
+
function manifestError(source, message, code = REPOSITORY_INDEX_ERROR_CODES.ENGINE_MISSING) {
|
|
16
|
+
throw repositoryIndexError(code, `${source} ${message}`);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function assertManifestObject(value, source) {
|
|
20
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
21
|
+
manifestError(source, "must be a JSON object");
|
|
22
|
+
}
|
|
23
|
+
if (value.schemaVersion !== 1) {
|
|
24
|
+
manifestError(source, "schemaVersion must be 1");
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function assertManifestIdentity(value, source) {
|
|
29
|
+
if (value.engine !== REPOSITORY_INDEX_ENGINE) {
|
|
30
|
+
manifestError(source, `engine must be ${REPOSITORY_INDEX_ENGINE}`);
|
|
31
|
+
}
|
|
32
|
+
if (typeof value.repository !== "string" || value.repository !== "microsoft/tgrep") {
|
|
33
|
+
manifestError(source, "repository must be microsoft/tgrep");
|
|
34
|
+
}
|
|
35
|
+
if (typeof value.version !== "string" || !SEMVER.test(value.version)) {
|
|
36
|
+
manifestError(source, "version is invalid");
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function assertManifestRelease(value, source) {
|
|
41
|
+
if (typeof value.releaseBaseUrl !== "string"
|
|
42
|
+
|| value.releaseBaseUrl !== `https://github.com/microsoft/tgrep/releases/download/v${value.version}`) {
|
|
43
|
+
manifestError(source, "releaseBaseUrl must be the pinned Microsoft tgrep release URL");
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function assertManifestAsset(asset, platformKey, source) {
|
|
48
|
+
if (!asset || typeof asset !== "object") manifestError(source, `is missing ${platformKey}`);
|
|
49
|
+
if (typeof asset.assetName !== "string" || asset.assetName.trim() === "") {
|
|
50
|
+
manifestError(source, `${platformKey} assetName is required`);
|
|
51
|
+
}
|
|
52
|
+
const assetName = asset.assetName.replaceAll("\\", "/");
|
|
53
|
+
if (path.posix.isAbsolute(assetName) || path.win32.isAbsolute(assetName)
|
|
54
|
+
|| path.posix.basename(assetName) !== assetName || path.posix.normalize(assetName) !== assetName
|
|
55
|
+
|| !/^[A-Za-z0-9._-]+$/u.test(assetName)) {
|
|
56
|
+
manifestError(source, `${platformKey} assetName must be a single safe filename`);
|
|
57
|
+
}
|
|
58
|
+
if (!ARCHIVES.has(asset.archive)) manifestError(source, `${platformKey} archive is unsupported`);
|
|
59
|
+
if (typeof asset.binaryName !== "string" || asset.binaryName.trim() === "") {
|
|
60
|
+
manifestError(source, `${platformKey} binaryName is required`);
|
|
61
|
+
}
|
|
62
|
+
const expectedBinaryName = platformKey === "windows-x64" ? "tgrep.exe" : "tgrep";
|
|
63
|
+
if (asset.binaryName !== expectedBinaryName) {
|
|
64
|
+
manifestError(source, `${platformKey} binaryName must be ${expectedBinaryName}`);
|
|
65
|
+
}
|
|
66
|
+
if (!SHA256.test(asset.sha256) || /^0+$/.test(asset.sha256) || /replace|todo|placeholder/i.test(asset.sha256)) {
|
|
67
|
+
manifestError(source, `${platformKey} checksum is not a verified SHA-256`, REPOSITORY_INDEX_ERROR_CODES.ENGINE_CHECKSUM_MISMATCH);
|
|
68
|
+
}
|
|
69
|
+
if (!SHA256.test(asset.binarySha256) || /^0+$/.test(asset.binarySha256) || /replace|todo|placeholder/i.test(asset.binarySha256)) {
|
|
70
|
+
manifestError(source, `${platformKey} binarySha256 is not a verified SHA-256`, REPOSITORY_INDEX_ERROR_CODES.ENGINE_BINARY_CHECKSUM_MISMATCH);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function assertManifestAssets(value, source) {
|
|
75
|
+
if (!value.assets || typeof value.assets !== "object" || Array.isArray(value.assets)) {
|
|
76
|
+
manifestError(source, "assets are required");
|
|
77
|
+
}
|
|
78
|
+
for (const key of REPOSITORY_INDEX_SUPPORTED_PLATFORMS) {
|
|
79
|
+
assertManifestAsset(value.assets[key], key, source);
|
|
80
|
+
}
|
|
81
|
+
const unsupportedKeys = Object.keys(value.assets).filter((key) => !REPOSITORY_INDEX_SUPPORTED_PLATFORMS.includes(key));
|
|
82
|
+
if (unsupportedKeys.length > 0) {
|
|
83
|
+
manifestError(source, `contains unsupported platforms: ${unsupportedKeys.join(", ")}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function assertManifest(value, source = "tgrep manifest") {
|
|
88
|
+
assertManifestObject(value, source);
|
|
89
|
+
assertManifestIdentity(value, source);
|
|
90
|
+
assertManifestRelease(value, source);
|
|
91
|
+
assertManifestAssets(value, source);
|
|
92
|
+
return Object.freeze({
|
|
93
|
+
...value,
|
|
94
|
+
assets: Object.freeze(Object.fromEntries(Object.entries(value.assets).map(([key, asset]) => [key, Object.freeze({ ...asset })]))),
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export async function loadTgrepManifest(packageRoot = getPackageRoot()) {
|
|
99
|
+
if (typeof packageRoot !== "string" || packageRoot.trim() === "") {
|
|
100
|
+
throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_MISSING, "ForgeLoop package root is required to load the tgrep manifest");
|
|
101
|
+
}
|
|
102
|
+
const manifestPath = path.join(packageRoot, "src", "repository-index", "tgrep-manifest.json");
|
|
103
|
+
let value;
|
|
104
|
+
try {
|
|
105
|
+
value = JSON.parse(await readFile(manifestPath, "utf8"));
|
|
106
|
+
} catch (cause) {
|
|
107
|
+
throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_MISSING, `Unable to read ${manifestPath}: ${cause.message}`, { cause, manifestPath });
|
|
108
|
+
}
|
|
109
|
+
return assertManifest(value, manifestPath);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function validateTgrepManifest(value) {
|
|
113
|
+
return assertManifest(value);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export async function getPinnedTgrepVersion(packageRoot) {
|
|
117
|
+
return (await loadTgrepManifest(packageRoot)).version;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export async function getTgrepAsset(platformKey, packageRoot) {
|
|
121
|
+
const manifest = await loadTgrepManifest(packageRoot);
|
|
122
|
+
const asset = manifest.assets[platformKey];
|
|
123
|
+
if (!asset) {
|
|
124
|
+
throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.PLATFORM_UNSUPPORTED, `No tgrep asset is available for ${platformKey}`);
|
|
125
|
+
}
|
|
126
|
+
return Object.freeze({
|
|
127
|
+
...asset,
|
|
128
|
+
url: `${manifest.releaseBaseUrl}/${asset.assetName}`,
|
|
129
|
+
version: manifest.version,
|
|
130
|
+
engine: manifest.engine,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Repository-index measurements are operational telemetry only. They never
|
|
3
|
+
* become protocol evidence, completion proof, or claims about token/cost
|
|
4
|
+
* savings.
|
|
5
|
+
*/
|
|
6
|
+
export function buildRepositorySearchMetrics({ durationMs, nativeDurationMs, result, engine, engineVersion, serverUsed }) {
|
|
7
|
+
return {
|
|
8
|
+
queryDurationMs: durationMs,
|
|
9
|
+
nativeDurationMs,
|
|
10
|
+
matchCount: result.matches.length,
|
|
11
|
+
matchedFileCount: result.files.length,
|
|
12
|
+
engine,
|
|
13
|
+
engineVersion,
|
|
14
|
+
serverUsed,
|
|
15
|
+
exitCode: result.exitCode,
|
|
16
|
+
ignoredNativeEvents: result.ignoredEvents,
|
|
17
|
+
...(result.stats.bytesSearched !== undefined ? { bytesSearched: result.stats.bytesSearched } : {}),
|
|
18
|
+
...(result.stats.matchedLines !== undefined ? { matchedLines: result.stats.matchedLines } : {}),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export function buildRepositoryIndexMetrics(status) {
|
|
22
|
+
return {
|
|
23
|
+
engine: status.engine,
|
|
24
|
+
engineVersion: status.engineVersion,
|
|
25
|
+
indexedFiles: status.index.files,
|
|
26
|
+
indexedTrigrams: status.index.trigrams,
|
|
27
|
+
health: status.health,
|
|
28
|
+
serverUsed: status.server.running,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import { realpath } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
import { REPOSITORY_INDEX_ERROR_CODES, repositoryIndexError } from "./errors.js";
|
|
5
|
+
|
|
6
|
+
function invalid(message, details = {}) {
|
|
7
|
+
return repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.OUTPUT_INVALID, message, details);
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function finiteInteger(value, label, { min = 0, optional = false } = {}) {
|
|
11
|
+
if (value === undefined || value === null) {
|
|
12
|
+
if (optional) return null;
|
|
13
|
+
throw invalid(`tgrep JSON field ${label} is required`);
|
|
14
|
+
}
|
|
15
|
+
if (!Number.isSafeInteger(value) || value < min) throw invalid(`tgrep JSON field ${label} is invalid`);
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function extractData(event) {
|
|
20
|
+
return event && typeof event === "object" && event.data && typeof event.data === "object"
|
|
21
|
+
? event.data
|
|
22
|
+
: event;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function usesWindowsPath(value, repositoryRoot, pathStyle) {
|
|
26
|
+
return pathStyle === "win32"
|
|
27
|
+
|| /^[A-Za-z]:[\\/]/u.test(value)
|
|
28
|
+
|| /^\\\\/u.test(value)
|
|
29
|
+
|| /^[A-Za-z]:[\\/]/u.test(repositoryRoot)
|
|
30
|
+
|| /^\\\\/u.test(repositoryRoot);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function normalizePath(value, repositoryRoot, { pathStyle = null } = {}) {
|
|
34
|
+
if (typeof value !== "string" || value.length === 0) throw invalid("tgrep JSON match path is missing");
|
|
35
|
+
const windows = usesWindowsPath(value, repositoryRoot, pathStyle);
|
|
36
|
+
const pathApi = windows ? path.win32 : path;
|
|
37
|
+
const normalizedRoot = pathApi.normalize(repositoryRoot);
|
|
38
|
+
const normalizedValue = value.replaceAll("/", pathApi.sep);
|
|
39
|
+
const candidate = pathApi.isAbsolute(normalizedValue)
|
|
40
|
+
? pathApi.normalize(normalizedValue)
|
|
41
|
+
: pathApi.resolve(normalizedRoot, normalizedValue);
|
|
42
|
+
const comparisonRoot = windows ? normalizedRoot.toLowerCase() : normalizedRoot;
|
|
43
|
+
const comparisonCandidate = windows ? candidate.toLowerCase() : candidate;
|
|
44
|
+
const relative = pathApi.relative(comparisonRoot, comparisonCandidate);
|
|
45
|
+
if (relative === ".." || relative.startsWith(`..${pathApi.sep}`) || pathApi.isAbsolute(relative)) {
|
|
46
|
+
throw invalid(`tgrep JSON path escapes the repository root: ${value}`);
|
|
47
|
+
}
|
|
48
|
+
return relative ? relative.split(pathApi.sep).join("/") : ".";
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function pathText(data) {
|
|
52
|
+
if (typeof data.path === "string") return data.path;
|
|
53
|
+
if (data.path && typeof data.path.text === "string") return data.path.text;
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function lineText(data) {
|
|
58
|
+
if (typeof data.text === "string") return data.text;
|
|
59
|
+
if (data.lines && typeof data.lines.text === "string") return data.lines.text;
|
|
60
|
+
return "";
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function normalizeSubmatches(data) {
|
|
64
|
+
if (data.submatches === undefined) return [];
|
|
65
|
+
if (!Array.isArray(data.submatches)) throw invalid("tgrep JSON submatches must be an array");
|
|
66
|
+
return data.submatches.map((submatch, index) => {
|
|
67
|
+
if (!submatch || typeof submatch !== "object") throw invalid(`tgrep JSON submatch ${index} is invalid`);
|
|
68
|
+
const start = finiteInteger(submatch.start, `submatches[${index}].start`);
|
|
69
|
+
const end = finiteInteger(submatch.end, `submatches[${index}].end`);
|
|
70
|
+
if (end < start) throw invalid(`tgrep JSON submatch ${index} has a reversed range`);
|
|
71
|
+
const match = typeof submatch.match === "string" ? submatch.match : submatch.match?.text;
|
|
72
|
+
return { start, end, match: typeof match === "string" ? match : null };
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async function normalizeMatch(event, repositoryRoot, options) {
|
|
77
|
+
const data = extractData(event);
|
|
78
|
+
const relativePath = await normalizePath(pathText(data), repositoryRoot, options);
|
|
79
|
+
const lineNumber = finiteInteger(data.line_number ?? data.lineNumber, "line_number", { min: 1 });
|
|
80
|
+
const absoluteOffset = finiteInteger(data.absolute_offset ?? data.absoluteOffset, "absolute_offset", { optional: true });
|
|
81
|
+
return {
|
|
82
|
+
path: relativePath,
|
|
83
|
+
line: lineNumber,
|
|
84
|
+
column: finiteInteger(data.column, "column", { min: 1, optional: true }),
|
|
85
|
+
offset: absoluteOffset,
|
|
86
|
+
text: lineText(data),
|
|
87
|
+
submatches: normalizeSubmatches(data),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function addStats(target, data) {
|
|
92
|
+
const stats = data?.stats && typeof data.stats === "object" ? data.stats : data;
|
|
93
|
+
if (!stats || typeof stats !== "object") return;
|
|
94
|
+
const values = {
|
|
95
|
+
bytesSearched: stats.bytes_searched,
|
|
96
|
+
bytesPrinted: stats.bytes_printed,
|
|
97
|
+
matchedLines: stats.matched_lines,
|
|
98
|
+
matches: stats.matches,
|
|
99
|
+
searches: stats.searches,
|
|
100
|
+
searchesWithMatch: stats.searches_with_match,
|
|
101
|
+
elapsedNanos: stats.elapsed?.nanos,
|
|
102
|
+
};
|
|
103
|
+
for (const [key, value] of Object.entries(values)) {
|
|
104
|
+
if (Number.isSafeInteger(value) && value >= 0) target[key] = value;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function looksLikeStatsLine(line) {
|
|
109
|
+
return /^(?:Query plan:|Search completed in |Walking |Found \d+ text files|Extracting trigrams|Writing index|Index built successfully|Indexed in )/i.test(line.trim());
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async function parseFileLines(lines, repositoryRoot, options) {
|
|
113
|
+
const files = [];
|
|
114
|
+
for (const line of lines) {
|
|
115
|
+
if (!line.trim()) continue;
|
|
116
|
+
if (looksLikeStatsLine(line)) continue;
|
|
117
|
+
let value;
|
|
118
|
+
try { value = JSON.parse(line); } catch {
|
|
119
|
+
files.push(await normalizePath(line.trim(), repositoryRoot, options));
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
const text = pathText(extractData(value));
|
|
123
|
+
if (value.type === "match" || value.type === "context") {
|
|
124
|
+
files.push(await normalizePath(text, repositoryRoot, options));
|
|
125
|
+
} else if (text) {
|
|
126
|
+
files.push(await normalizePath(text, repositoryRoot));
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return [...new Set(files)].sort();
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Converts tgrep's versioned JSON-lines stream into ForgeLoop's stable,
|
|
134
|
+
* provider-neutral repository-search result. Native output is never treated
|
|
135
|
+
* as evidence or lifecycle state; malformed lines fail closed.
|
|
136
|
+
*/
|
|
137
|
+
export async function normalizeTgrepJson(stdout, {
|
|
138
|
+
repositoryRoot,
|
|
139
|
+
filesWithMatches = false,
|
|
140
|
+
realpathImpl = realpath,
|
|
141
|
+
pathStyle = null,
|
|
142
|
+
} = {}) {
|
|
143
|
+
if (typeof stdout !== "string" || typeof repositoryRoot !== "string") throw invalid("tgrep JSON output requires text and repositoryRoot");
|
|
144
|
+
const canonicalRoot = await realpathImpl(repositoryRoot);
|
|
145
|
+
const pathOptions = { pathStyle };
|
|
146
|
+
const lines = stdout.split(/\r?\n/).filter((line) => line.trim());
|
|
147
|
+
if (filesWithMatches) {
|
|
148
|
+
return {
|
|
149
|
+
matches: [],
|
|
150
|
+
contexts: [],
|
|
151
|
+
files: await parseFileLines(lines, canonicalRoot, pathOptions),
|
|
152
|
+
stats: {},
|
|
153
|
+
events: { begin: 0, match: 0, context: 0, end: 0, summary: 0 },
|
|
154
|
+
ignoredEvents: 0,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const matches = [];
|
|
159
|
+
const contexts = [];
|
|
160
|
+
const stats = {};
|
|
161
|
+
const files = new Set();
|
|
162
|
+
const events = { begin: 0, match: 0, context: 0, end: 0, summary: 0 };
|
|
163
|
+
let ignoredEvents = 0;
|
|
164
|
+
for (const line of lines) {
|
|
165
|
+
let event;
|
|
166
|
+
try { event = JSON.parse(line); } catch {
|
|
167
|
+
if (looksLikeStatsLine(line)) continue;
|
|
168
|
+
throw invalid("tgrep emitted a non-JSON line in JSON search mode", { line: line.slice(0, 500) });
|
|
169
|
+
}
|
|
170
|
+
if (!event || typeof event !== "object" || typeof event.type !== "string") {
|
|
171
|
+
throw invalid("tgrep emitted a JSON value without an event type");
|
|
172
|
+
}
|
|
173
|
+
if (!Object.hasOwn(events, event.type)) {
|
|
174
|
+
ignoredEvents += 1;
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
events[event.type] += 1;
|
|
178
|
+
if (event.type === "match" || event.type === "context") {
|
|
179
|
+
const normalized = await normalizeMatch(event, canonicalRoot, pathOptions);
|
|
180
|
+
files.add(normalized.path);
|
|
181
|
+
(event.type === "match" ? matches : contexts).push(normalized);
|
|
182
|
+
} else if (event.type === "end" || event.type === "summary") {
|
|
183
|
+
addStats(stats, extractData(event));
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return { matches, contexts, files: [...files].sort(), stats, events, ignoredEvents };
|
|
187
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import os from "node:os";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
import { REPOSITORY_INDEX_RELATIVE_PATH, REPOSITORY_INDEX_STATE_FILE } from "./constants.js";
|
|
5
|
+
|
|
6
|
+
export function getEngineHome({ homeDirectory = os.homedir() } = {}) {
|
|
7
|
+
return path.join(homeDirectory, ".forgeloop", "engines", "tgrep");
|
|
8
|
+
}
|
|
9
|
+
export function getTgrepVersionDirectory(version, options = {}) {
|
|
10
|
+
return path.join(getEngineHome(options), version);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function getManagedTgrepDirectory(version, platformKey, options = {}) {
|
|
14
|
+
return path.join(getTgrepVersionDirectory(version, options), platformKey);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function getManagedTgrepBinaryPath(version, platformKey, { windows = platformKey.startsWith("windows-") || process.platform === "win32", ...options } = {}) {
|
|
18
|
+
return path.join(getManagedTgrepDirectory(version, platformKey, options), windows ? "tgrep.exe" : "tgrep");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function getRepositoryIndexRoot(repoRoot) {
|
|
22
|
+
return path.join(repoRoot, REPOSITORY_INDEX_RELATIVE_PATH);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function getTgrepIndexPath(repoRoot) {
|
|
26
|
+
return path.join(getRepositoryIndexRoot(repoRoot), "tgrep");
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function getRepositoryIndexStatePath(repoRoot) {
|
|
30
|
+
return path.join(getRepositoryIndexRoot(repoRoot), REPOSITORY_INDEX_STATE_FILE);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function getRepositoryIndexMetricsPath(repoRoot) {
|
|
34
|
+
return path.join(getRepositoryIndexRoot(repoRoot), "metrics");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function getRepositoryIndexLockPath(repoRoot) {
|
|
38
|
+
return path.join(getRepositoryIndexRoot(repoRoot), "operation.lock");
|
|
39
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { REPOSITORY_INDEX_ERROR_CODES, repositoryIndexError } from "./errors.js";
|
|
2
|
+
|
|
3
|
+
const SUPPORTED = new Map([
|
|
4
|
+
["darwin:arm64", "darwin-arm64"],
|
|
5
|
+
["darwin:x64", "darwin-x64"],
|
|
6
|
+
["linux:x64", "linux-x64"],
|
|
7
|
+
["win32:x64", "windows-x64"],
|
|
8
|
+
]);
|
|
9
|
+
|
|
10
|
+
export function getRepositoryIndexPlatform({ platform = process.platform, arch = process.arch } = {}) {
|
|
11
|
+
const key = SUPPORTED.get(`${platform}:${arch}`);
|
|
12
|
+
if (!key) {
|
|
13
|
+
throw repositoryIndexError(
|
|
14
|
+
REPOSITORY_INDEX_ERROR_CODES.PLATFORM_UNSUPPORTED,
|
|
15
|
+
`Repository Index does not support platform ${platform}/${arch}`,
|
|
16
|
+
{ platform, arch },
|
|
17
|
+
);
|
|
18
|
+
}
|
|
19
|
+
return Object.freeze({ platform, arch, key });
|
|
20
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { clearInterval, setInterval } from "node:timers";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
|
|
5
|
+
import { REPOSITORY_INDEX_DEFAULTS } from "./constants.js";
|
|
6
|
+
import { REPOSITORY_INDEX_ERROR_CODES, repositoryIndexError } from "./errors.js";
|
|
7
|
+
|
|
8
|
+
const MAX_COMMAND_TIMEOUT_MS = 120_000;
|
|
9
|
+
const PROCESS_TIMER_INTERVAL_MS = 50;
|
|
10
|
+
const trustedBinaryPaths = new WeakMap();
|
|
11
|
+
|
|
12
|
+
function assertArgs(args) {
|
|
13
|
+
if (!Array.isArray(args) || args.some((arg) => typeof arg !== "string" || arg.length === 0)) {
|
|
14
|
+
throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_EXECUTION_FAILED, "tgrep arguments must be a non-empty string array");
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function assertExecutablePath(binaryPath) {
|
|
19
|
+
if (typeof binaryPath !== "string" || binaryPath.trim() === "") {
|
|
20
|
+
throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_EXECUTION_FAILED, "A verified managed tgrep binary path is required");
|
|
21
|
+
}
|
|
22
|
+
if (!path.isAbsolute(binaryPath) || !binaryPath.match(/^[\w./\\: @%+=~-]+$/u)) {
|
|
23
|
+
throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_EXECUTION_FAILED, "The managed tgrep binary path contains unsafe command characters");
|
|
24
|
+
}
|
|
25
|
+
return binaryPath;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function createTgrepBinaryHandle(binaryPath) {
|
|
29
|
+
const handle = Object.freeze({});
|
|
30
|
+
trustedBinaryPaths.set(handle, assertExecutablePath(binaryPath));
|
|
31
|
+
return handle;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function resolveTgrepBinaryHandle(handle) {
|
|
35
|
+
const binaryPath = trustedBinaryPaths.get(handle);
|
|
36
|
+
if (!binaryPath) {
|
|
37
|
+
throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_EXECUTION_FAILED, "A validated managed tgrep binary handle is required");
|
|
38
|
+
}
|
|
39
|
+
return binaryPath;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function runTgrep({ binary, repoRoot, args, stdin = null, timeoutMs = REPOSITORY_INDEX_DEFAULTS.commandTimeoutMs, env = {}, spawnImpl = spawn, maxOutputBytes = REPOSITORY_INDEX_DEFAULTS.maxProcessOutputBytes } = {}) {
|
|
43
|
+
const binaryPath = resolveTgrepBinaryHandle(binary);
|
|
44
|
+
if (typeof repoRoot !== "string" || repoRoot.trim() === "") {
|
|
45
|
+
throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_EXECUTION_FAILED, "A repository root is required to execute tgrep");
|
|
46
|
+
}
|
|
47
|
+
assertArgs(args);
|
|
48
|
+
if (!Number.isInteger(timeoutMs) || timeoutMs <= 0) {
|
|
49
|
+
throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_EXECUTION_FAILED, "tgrep timeoutMs must be a positive integer");
|
|
50
|
+
}
|
|
51
|
+
const effectiveTimeoutMs = Math.min(timeoutMs, MAX_COMMAND_TIMEOUT_MS);
|
|
52
|
+
|
|
53
|
+
const startedAt = Date.now();
|
|
54
|
+
return new Promise((resolve, reject) => {
|
|
55
|
+
let child;
|
|
56
|
+
try {
|
|
57
|
+
child = spawnImpl(binaryPath, args, {
|
|
58
|
+
cwd: repoRoot,
|
|
59
|
+
env: { ...process.env, ...env },
|
|
60
|
+
shell: false,
|
|
61
|
+
windowsHide: true,
|
|
62
|
+
});
|
|
63
|
+
} catch (cause) {
|
|
64
|
+
reject(repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_EXECUTION_FAILED, `Unable to launch managed tgrep: ${cause.message}`, { cause }));
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
let stdout = "";
|
|
69
|
+
let stderr = "";
|
|
70
|
+
let settled = false;
|
|
71
|
+
let timedOut = false;
|
|
72
|
+
const timeoutDeadline = startedAt + effectiveTimeoutMs;
|
|
73
|
+
const timer = setInterval(() => {
|
|
74
|
+
if (Date.now() < timeoutDeadline) return;
|
|
75
|
+
timedOut = true;
|
|
76
|
+
clearInterval(timer);
|
|
77
|
+
try { child.kill("SIGTERM"); } catch { /* process may already be gone */ }
|
|
78
|
+
}, PROCESS_TIMER_INTERVAL_MS);
|
|
79
|
+
timer.unref?.();
|
|
80
|
+
|
|
81
|
+
const finish = (fn, value) => {
|
|
82
|
+
if (settled) return;
|
|
83
|
+
settled = true;
|
|
84
|
+
clearInterval(timer);
|
|
85
|
+
fn(value);
|
|
86
|
+
};
|
|
87
|
+
const append = (current, chunk) => {
|
|
88
|
+
const next = `${current}${chunk.toString("utf8")}`;
|
|
89
|
+
if (Buffer.byteLength(next, "utf8") > maxOutputBytes) {
|
|
90
|
+
try { child.kill("SIGTERM"); } catch { /* process may already be gone */ }
|
|
91
|
+
finish(reject, repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_OUTPUT_LIMIT, `tgrep output exceeded ${maxOutputBytes} bytes`));
|
|
92
|
+
return current;
|
|
93
|
+
}
|
|
94
|
+
return next;
|
|
95
|
+
};
|
|
96
|
+
child.stdout?.on("data", (chunk) => { stdout = append(stdout, chunk); });
|
|
97
|
+
child.stderr?.on("data", (chunk) => { stderr = append(stderr, chunk); });
|
|
98
|
+
child.once("error", (cause) => finish(reject, repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_EXECUTION_FAILED, `Managed tgrep failed to start: ${cause.message}`, { cause })));
|
|
99
|
+
child.once("close", (exitCode, signal) => {
|
|
100
|
+
if (settled) return;
|
|
101
|
+
const result = {
|
|
102
|
+
exitCode: Number.isInteger(exitCode) ? exitCode : null,
|
|
103
|
+
signal: signal ?? null,
|
|
104
|
+
stdout,
|
|
105
|
+
stderr,
|
|
106
|
+
durationMs: Date.now() - startedAt,
|
|
107
|
+
timedOut,
|
|
108
|
+
};
|
|
109
|
+
finish(resolve, result);
|
|
110
|
+
});
|
|
111
|
+
if (stdin !== null && stdin !== undefined) {
|
|
112
|
+
child.stdin?.end(stdin);
|
|
113
|
+
} else {
|
|
114
|
+
child.stdin?.end();
|
|
115
|
+
}
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function spawnTgrepServer({ binary, repoRoot, args, env = {}, spawnImpl = spawn, stdio = "ignore" } = {}) {
|
|
120
|
+
const binaryPath = resolveTgrepBinaryHandle(binary);
|
|
121
|
+
if (typeof repoRoot !== "string" || !repoRoot) {
|
|
122
|
+
throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.SERVER_START_FAILED, "A verified managed tgrep binary and repository root are required");
|
|
123
|
+
}
|
|
124
|
+
assertArgs(args);
|
|
125
|
+
let child;
|
|
126
|
+
try {
|
|
127
|
+
child = spawnImpl(binaryPath, args, {
|
|
128
|
+
cwd: repoRoot,
|
|
129
|
+
env: { ...process.env, ...env },
|
|
130
|
+
shell: false,
|
|
131
|
+
detached: true,
|
|
132
|
+
stdio,
|
|
133
|
+
windowsHide: true,
|
|
134
|
+
});
|
|
135
|
+
} catch (cause) {
|
|
136
|
+
throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.SERVER_START_FAILED, `Unable to launch managed tgrep server: ${cause.message}`, { cause });
|
|
137
|
+
}
|
|
138
|
+
child.unref?.();
|
|
139
|
+
return child;
|
|
140
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
|
|
3
|
+
import { processIsAlive } from "./status.js";
|
|
4
|
+
|
|
5
|
+
const readinessByRepository = new Map();
|
|
6
|
+
|
|
7
|
+
function normalize(value, platform = process.platform) {
|
|
8
|
+
if (typeof value !== "string" || value.length === 0) return value;
|
|
9
|
+
const pathApi = platform === "win32" ? path.win32 : path;
|
|
10
|
+
const normalized = pathApi.normalize(value);
|
|
11
|
+
return platform === "win32" ? normalized.toLowerCase() : normalized;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function rememberRepositoryIndexReadiness({ canonicalRoot, status, platform } = {}) {
|
|
15
|
+
if (typeof canonicalRoot !== "string" || !status?.indexPath || !status?.binaryPath || status.server?.running !== true) return null;
|
|
16
|
+
const selectedPlatform = platform ?? status.platform?.platform ?? process.platform;
|
|
17
|
+
const readiness = Object.freeze({
|
|
18
|
+
canonicalRoot,
|
|
19
|
+
indexPath: status.indexPath,
|
|
20
|
+
binaryPath: status.binaryPath,
|
|
21
|
+
engineVersion: status.engineVersion ?? null,
|
|
22
|
+
serverRunning: true,
|
|
23
|
+
pid: Number.isInteger(status.server?.pid) ? status.server.pid : null,
|
|
24
|
+
platform: selectedPlatform,
|
|
25
|
+
status,
|
|
26
|
+
lastSuccessfulSearchAt: null,
|
|
27
|
+
});
|
|
28
|
+
readinessByRepository.set(normalize(canonicalRoot, selectedPlatform), readiness);
|
|
29
|
+
return readiness;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function getRepositoryIndexReadiness(canonicalRoot, platform = process.platform) {
|
|
33
|
+
return readinessByRepository.get(normalize(canonicalRoot, platform)) ?? null;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function invalidateRepositoryIndexReadiness(canonicalRoot, platform = process.platform) {
|
|
37
|
+
if (typeof canonicalRoot !== "string") return;
|
|
38
|
+
readinessByRepository.delete(normalize(canonicalRoot, platform));
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function clearRepositoryIndexReadiness() {
|
|
42
|
+
readinessByRepository.clear();
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function markRepositoryIndexSearchSucceeded(canonicalRoot, platform = process.platform) {
|
|
46
|
+
const current = getRepositoryIndexReadiness(canonicalRoot, platform);
|
|
47
|
+
if (!current) return null;
|
|
48
|
+
const next = Object.freeze({ ...current, lastSuccessfulSearchAt: new Date().toISOString() });
|
|
49
|
+
readinessByRepository.set(normalize(canonicalRoot, platform), next);
|
|
50
|
+
return next;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function isRepositoryIndexReadinessUsable(readiness, expected = {}, { isAlive = processIsAlive } = {}) {
|
|
54
|
+
if (!readiness || readiness.serverRunning !== true || typeof readiness.canonicalRoot !== "string") return false;
|
|
55
|
+
const platform = expected.platform ?? readiness.platform ?? process.platform;
|
|
56
|
+
if (expected.canonicalRoot && normalize(readiness.canonicalRoot, platform) !== normalize(expected.canonicalRoot, platform)) return false;
|
|
57
|
+
if (expected.indexPath && normalize(readiness.indexPath, platform) !== normalize(expected.indexPath, platform)) return false;
|
|
58
|
+
if (expected.binaryPath && normalize(readiness.binaryPath, platform) !== normalize(expected.binaryPath, platform)) return false;
|
|
59
|
+
if (expected.engineVersion && readiness.engineVersion !== expected.engineVersion) return false;
|
|
60
|
+
if (!Number.isInteger(readiness.pid) || !isAlive(readiness.pid)) return false;
|
|
61
|
+
return true;
|
|
62
|
+
}
|