@openclaw/plugin-inspector 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/package.json +1 -1
- package/src/advanced.js +5 -0
- package/src/api.js +4 -0
- package/src/artifacts.js +1 -1
- package/src/ci-policy.js +3 -1
- package/src/ci-summary.js +50 -3
- package/src/compatibility-report.js +39 -3
- package/src/config.js +35 -5
- package/src/contract-capture.js +1 -0
- package/src/contract-probes.js +25 -0
- package/src/fixture-summary.js +406 -0
- package/src/import-loop-profile.js +262 -8
- package/src/index.js +7 -0
- package/src/inspector.js +49 -4
- package/src/issues.js +66 -1
- package/src/openclaw-target.js +79 -2
- package/src/platform-probes.js +29 -6
- package/src/process-profile.js +40 -19
- package/src/prune-workspace-dev-deps-cli.js +23 -0
- package/src/report.js +19 -1
- package/src/runtime-profile.js +67 -15
- package/src/runtime-reconciliation.js +124 -0
- package/src/sdk-mock.js +64 -6
- package/src/synthetic-probes.js +51 -0
- package/src/workspace-plan.js +19 -4
package/src/openclaw-target.js
CHANGED
|
@@ -106,12 +106,89 @@ export function openClawTargetPathCandidates(manifest, configuredPath) {
|
|
|
106
106
|
|
|
107
107
|
export function parseCompatRecordEntries(source) {
|
|
108
108
|
const entries = [];
|
|
109
|
-
|
|
110
|
-
|
|
109
|
+
let cursor = 0;
|
|
110
|
+
while (cursor < source.length) {
|
|
111
|
+
const codeProperty = readStringProperty(source, "code", cursor);
|
|
112
|
+
if (!codeProperty) {
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const statusProperty = readStringProperty(source, "status", codeProperty.end);
|
|
117
|
+
if (statusProperty) {
|
|
118
|
+
entries.push({ code: codeProperty.value, status: statusProperty.value });
|
|
119
|
+
cursor = statusProperty.end;
|
|
120
|
+
} else {
|
|
121
|
+
cursor = codeProperty.end;
|
|
122
|
+
}
|
|
111
123
|
}
|
|
112
124
|
return dedupeBy(entries, (entry) => entry.code).sort((left, right) => left.code.localeCompare(right.code));
|
|
113
125
|
}
|
|
114
126
|
|
|
127
|
+
function readStringProperty(source, property, fromIndex) {
|
|
128
|
+
const propertyIndex = findProperty(source, property, fromIndex);
|
|
129
|
+
if (propertyIndex === -1) {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
const colonIndex = source.indexOf(":", propertyIndex + property.length);
|
|
133
|
+
if (colonIndex === -1) {
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
let quoteIndex = colonIndex + 1;
|
|
137
|
+
while (quoteIndex < source.length && isWhitespace(source[quoteIndex])) {
|
|
138
|
+
quoteIndex += 1;
|
|
139
|
+
}
|
|
140
|
+
if (!isQuote(source[quoteIndex])) {
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
return readQuotedValue(source, quoteIndex);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function findProperty(source, property, fromIndex) {
|
|
147
|
+
let index = source.indexOf(property, fromIndex);
|
|
148
|
+
while (index !== -1) {
|
|
149
|
+
const previous = index === 0 ? "" : source[index - 1];
|
|
150
|
+
const next = source[index + property.length] ?? "";
|
|
151
|
+
if (!isIdentifierChar(previous) && !isIdentifierChar(next)) {
|
|
152
|
+
return index;
|
|
153
|
+
}
|
|
154
|
+
index = source.indexOf(property, index + property.length);
|
|
155
|
+
}
|
|
156
|
+
return -1;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function readQuotedValue(source, quoteIndex) {
|
|
160
|
+
const quote = source[quoteIndex];
|
|
161
|
+
let value = "";
|
|
162
|
+
for (let index = quoteIndex + 1; index < source.length; index += 1) {
|
|
163
|
+
const char = source[index];
|
|
164
|
+
if (char === "\\") {
|
|
165
|
+
value += source[index + 1] ?? "";
|
|
166
|
+
index += 1;
|
|
167
|
+
} else if (char === quote) {
|
|
168
|
+
return { value, end: index + 1 };
|
|
169
|
+
} else {
|
|
170
|
+
value += char;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function isQuote(char) {
|
|
177
|
+
return char === '"' || char === "'" || char === "`";
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function isIdentifierChar(char) {
|
|
181
|
+
if (char === "_" || char === "$") {
|
|
182
|
+
return true;
|
|
183
|
+
}
|
|
184
|
+
const code = char.charCodeAt(0);
|
|
185
|
+
return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function isWhitespace(char) {
|
|
189
|
+
return char === " " || char === "\n" || char === "\r" || char === "\t";
|
|
190
|
+
}
|
|
191
|
+
|
|
115
192
|
export function parsePluginSdkExports(packageJson) {
|
|
116
193
|
return Object.keys(packageJson.exports ?? {})
|
|
117
194
|
.filter((specifier) => specifier === "./plugin-sdk" || specifier.startsWith("./plugin-sdk/"))
|
package/src/platform-probes.js
CHANGED
|
@@ -55,8 +55,11 @@ export function validatePlatformProbes(report, options = {}) {
|
|
|
55
55
|
errors.push("all TypeScript loader entrypoints must track a Jiti fallback candidate");
|
|
56
56
|
}
|
|
57
57
|
for (const entrypoint of report.entrypoints) {
|
|
58
|
-
if (
|
|
59
|
-
|
|
58
|
+
if (
|
|
59
|
+
entrypoint.loaderPrimary === "tsx" &&
|
|
60
|
+
(!entrypoint.captureUsesTypeScriptLoader || !entrypoint.syntheticUsesTypeScriptLoader)
|
|
61
|
+
) {
|
|
62
|
+
errors.push(`${entrypoint.id}: TypeScript loader strategy is not reflected in capture and synthetic commands`);
|
|
60
63
|
}
|
|
61
64
|
}
|
|
62
65
|
return errors;
|
|
@@ -94,9 +97,21 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
|
|
|
94
97
|
entrypoint.loaderAlternatives.join(", ") || "-",
|
|
95
98
|
entrypoint.captureUsesTsx ? "yes" : "no",
|
|
96
99
|
entrypoint.syntheticUsesTsx ? "yes" : "no",
|
|
100
|
+
entrypoint.captureUsesMockSdk ? "yes" : "no",
|
|
101
|
+
entrypoint.syntheticUsesMockSdk ? "yes" : "no",
|
|
97
102
|
entrypoint.entrypoint,
|
|
98
103
|
]),
|
|
99
|
-
[
|
|
104
|
+
[
|
|
105
|
+
"Fixture",
|
|
106
|
+
"Status",
|
|
107
|
+
"Primary",
|
|
108
|
+
"Alternatives",
|
|
109
|
+
"Capture TSX",
|
|
110
|
+
"Synthetic TSX",
|
|
111
|
+
"Capture Mock SDK",
|
|
112
|
+
"Synthetic Mock SDK",
|
|
113
|
+
"Entrypoint",
|
|
114
|
+
],
|
|
100
115
|
),
|
|
101
116
|
"",
|
|
102
117
|
"## Portability Findings",
|
|
@@ -137,6 +152,10 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
|
|
|
137
152
|
function summarizeEntrypoint(fixtureId, entrypoint) {
|
|
138
153
|
const captureStep = entrypoint.steps.find((step) => step.kind === "capture");
|
|
139
154
|
const syntheticStep = entrypoint.steps.find((step) => step.kind === "synthetic-probe");
|
|
155
|
+
const captureUsesTsx = Boolean(captureStep?.command.includes("--import tsx"));
|
|
156
|
+
const syntheticUsesTsx = Boolean(syntheticStep?.command.includes("--import tsx"));
|
|
157
|
+
const captureUsesMockSdk = Boolean(captureStep?.command.includes("--mock-sdk"));
|
|
158
|
+
const syntheticUsesMockSdk = Boolean(syntheticStep?.command.includes("--mock-sdk"));
|
|
140
159
|
return {
|
|
141
160
|
fixture: fixtureId,
|
|
142
161
|
id: entrypoint.id,
|
|
@@ -148,8 +167,12 @@ function summarizeEntrypoint(fixtureId, entrypoint) {
|
|
|
148
167
|
loaderAlternatives: entrypoint.loaderStrategy.alternatives,
|
|
149
168
|
capturePlanned: Boolean(captureStep),
|
|
150
169
|
syntheticProbePlanned: Boolean(syntheticStep),
|
|
151
|
-
captureUsesTsx
|
|
152
|
-
syntheticUsesTsx
|
|
170
|
+
captureUsesTsx,
|
|
171
|
+
syntheticUsesTsx,
|
|
172
|
+
captureUsesMockSdk,
|
|
173
|
+
syntheticUsesMockSdk,
|
|
174
|
+
captureUsesTypeScriptLoader: captureUsesTsx || captureUsesMockSdk,
|
|
175
|
+
syntheticUsesTypeScriptLoader: syntheticUsesTsx || syntheticUsesMockSdk,
|
|
153
176
|
};
|
|
154
177
|
}
|
|
155
178
|
|
|
@@ -260,7 +283,7 @@ function buildRecommendations(portabilityFindings, entrypoints) {
|
|
|
260
283
|
if (entrypoints.some((entrypoint) => entrypoint.loaderPrimary === "tsx")) {
|
|
261
284
|
recommendations.push({
|
|
262
285
|
area: "loader",
|
|
263
|
-
action: "keep
|
|
286
|
+
action: "keep mock-SDK TypeScript capture green, add a real host-loader/Jiti lane before treating TS plugin source compatibility as covered",
|
|
264
287
|
});
|
|
265
288
|
}
|
|
266
289
|
if (portabilityFindings.some((finding) => finding.riskCodes.includes("rsync-required"))) {
|
package/src/process-profile.js
CHANGED
|
@@ -7,8 +7,12 @@ export async function runProfiledProcess(options) {
|
|
|
7
7
|
let firstRssKb = 0;
|
|
8
8
|
let peakRssKb = 0;
|
|
9
9
|
let peakCpuPercent = 0;
|
|
10
|
+
let statSampleCount = 0;
|
|
11
|
+
let rssSampleCount = 0;
|
|
12
|
+
let cpuSampleCount = 0;
|
|
10
13
|
const cpuSamples = [];
|
|
11
14
|
let pollInFlight = false;
|
|
15
|
+
const pendingStats = new Set();
|
|
12
16
|
|
|
13
17
|
const child = spawn(options.command, options.args ?? [], {
|
|
14
18
|
cwd: options.cwd,
|
|
@@ -21,27 +25,43 @@ export async function runProfiledProcess(options) {
|
|
|
21
25
|
child.stderr?.on("data", (chunk) => stderr.push(chunk));
|
|
22
26
|
|
|
23
27
|
const recordStats = (stats) => {
|
|
24
|
-
if (stats.
|
|
28
|
+
if (stats.rssAvailable || stats.cpuAvailable) {
|
|
29
|
+
statSampleCount += 1;
|
|
30
|
+
}
|
|
31
|
+
if (stats.rssAvailable) {
|
|
32
|
+
rssSampleCount += 1;
|
|
33
|
+
}
|
|
34
|
+
if (stats.cpuAvailable) {
|
|
35
|
+
cpuSampleCount += 1;
|
|
36
|
+
}
|
|
37
|
+
if (stats.rssAvailable && stats.rssKb > 0 && firstRssKb === 0) {
|
|
25
38
|
firstRssKb = stats.rssKb;
|
|
26
39
|
}
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
40
|
+
if (stats.rssAvailable) {
|
|
41
|
+
peakRssKb = Math.max(peakRssKb, stats.rssKb);
|
|
42
|
+
}
|
|
43
|
+
if (stats.cpuAvailable) {
|
|
44
|
+
peakCpuPercent = Math.max(peakCpuPercent, stats.cpuPercent);
|
|
30
45
|
cpuSamples.push(stats.cpuPercent);
|
|
31
46
|
}
|
|
32
47
|
};
|
|
33
48
|
|
|
34
|
-
const
|
|
49
|
+
const sampleStats = () => {
|
|
35
50
|
if (pollInFlight) {
|
|
36
51
|
return;
|
|
37
52
|
}
|
|
38
53
|
pollInFlight = true;
|
|
39
|
-
readProcessStats(child.pid)
|
|
54
|
+
const pending = readProcessStats(child.pid)
|
|
40
55
|
.then(recordStats)
|
|
41
56
|
.finally(() => {
|
|
42
57
|
pollInFlight = false;
|
|
58
|
+
pendingStats.delete(pending);
|
|
43
59
|
});
|
|
44
|
-
|
|
60
|
+
pendingStats.add(pending);
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
sampleStats();
|
|
64
|
+
const poll = setInterval(sampleStats, options.pollMs ?? 25);
|
|
45
65
|
|
|
46
66
|
const exitCode = await new Promise((resolve, reject) => {
|
|
47
67
|
child.on("error", (error) => {
|
|
@@ -51,16 +71,10 @@ export async function runProfiledProcess(options) {
|
|
|
51
71
|
child.on("exit", (code) => resolve(code ?? 1));
|
|
52
72
|
});
|
|
53
73
|
clearInterval(poll);
|
|
74
|
+
await Promise.allSettled([...pendingStats]);
|
|
54
75
|
|
|
55
76
|
const finalStats = await readProcessStats(child.pid);
|
|
56
|
-
|
|
57
|
-
firstRssKb = finalStats.rssKb;
|
|
58
|
-
}
|
|
59
|
-
peakRssKb = Math.max(peakRssKb, finalStats.rssKb);
|
|
60
|
-
peakCpuPercent = Math.max(peakCpuPercent, finalStats.cpuPercent);
|
|
61
|
-
if (finalStats.cpuPercent > 0) {
|
|
62
|
-
cpuSamples.push(finalStats.cpuPercent);
|
|
63
|
-
}
|
|
77
|
+
recordStats(finalStats);
|
|
64
78
|
|
|
65
79
|
const wallMs = Math.round(performance.now() - start);
|
|
66
80
|
const averageCpuPercent =
|
|
@@ -79,6 +93,9 @@ export async function runProfiledProcess(options) {
|
|
|
79
93
|
peakCpuPercent: Math.round(peakCpuPercent * 10) / 10,
|
|
80
94
|
cpuMsEstimate: Math.round((wallMs * cpuPercentForEstimate) / 100),
|
|
81
95
|
harnessHeapDeltaMb: Math.round((heapUsedMb() - heapStartMb) * 10) / 10,
|
|
96
|
+
statSampleCount,
|
|
97
|
+
rssSampleCount,
|
|
98
|
+
cpuSampleCount,
|
|
82
99
|
exitCode,
|
|
83
100
|
stdoutPreview: previewLines(stdout),
|
|
84
101
|
stderrPreview: previewLines(stderr),
|
|
@@ -87,7 +104,7 @@ export async function runProfiledProcess(options) {
|
|
|
87
104
|
|
|
88
105
|
async function readProcessStats(pid) {
|
|
89
106
|
if (!pid || process.platform === "win32") {
|
|
90
|
-
return { rssKb: 0, cpuPercent: 0 };
|
|
107
|
+
return { rssAvailable: false, rssKb: 0, cpuAvailable: false, cpuPercent: 0 };
|
|
91
108
|
}
|
|
92
109
|
return new Promise((resolve) => {
|
|
93
110
|
const ps = spawn("ps", ["-o", "rss=", "-o", "%cpu=", "-p", String(pid)], {
|
|
@@ -95,14 +112,18 @@ async function readProcessStats(pid) {
|
|
|
95
112
|
});
|
|
96
113
|
const chunks = [];
|
|
97
114
|
ps.stdout.on("data", (chunk) => chunks.push(chunk));
|
|
98
|
-
ps.on("error", () => resolve({ rssKb: 0, cpuPercent: 0 }));
|
|
115
|
+
ps.on("error", () => resolve({ rssAvailable: false, rssKb: 0, cpuAvailable: false, cpuPercent: 0 }));
|
|
99
116
|
ps.on("exit", () => {
|
|
100
117
|
const [rssRaw, cpuRaw] = Buffer.concat(chunks).toString("utf8").trim().split(/\s+/);
|
|
101
118
|
const rssKb = Number.parseInt(rssRaw, 10);
|
|
102
119
|
const cpuPercent = Number.parseFloat(cpuRaw);
|
|
120
|
+
const rssAvailable = Number.isFinite(rssKb);
|
|
121
|
+
const cpuAvailable = Number.isFinite(cpuPercent);
|
|
103
122
|
resolve({
|
|
104
|
-
|
|
105
|
-
|
|
123
|
+
rssAvailable,
|
|
124
|
+
rssKb: rssAvailable ? rssKb : 0,
|
|
125
|
+
cpuAvailable,
|
|
126
|
+
cpuPercent: cpuAvailable ? cpuPercent : 0,
|
|
106
127
|
});
|
|
107
128
|
});
|
|
108
129
|
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
|
|
5
|
+
const packageJsonPath = path.resolve(process.cwd(), "package.json");
|
|
6
|
+
const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8"));
|
|
7
|
+
let changed = false;
|
|
8
|
+
|
|
9
|
+
for (const [name, specifier] of Object.entries(packageJson.devDependencies ?? {})) {
|
|
10
|
+
if (typeof specifier === "string" && specifier.startsWith("workspace:")) {
|
|
11
|
+
delete packageJson.devDependencies[name];
|
|
12
|
+
changed = true;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
if (packageJson.devDependencies && Object.keys(packageJson.devDependencies).length === 0) {
|
|
17
|
+
delete packageJson.devDependencies;
|
|
18
|
+
changed = true;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
if (changed) {
|
|
22
|
+
await writeFile(packageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`, "utf8");
|
|
23
|
+
}
|
package/src/report.js
CHANGED
|
@@ -5,6 +5,7 @@ import { buildContractProbes } from "./contract-probes.js";
|
|
|
5
5
|
import { classifyCompatibilityFixture } from "./fixture-summary.js";
|
|
6
6
|
import { buildIssues, summarizeIssueClasses } from "./issues.js";
|
|
7
7
|
import { sanitizeReportArtifact } from "./report-sanitizer.js";
|
|
8
|
+
import { applyRuntimeExecutionCoverage } from "./runtime-reconciliation.js";
|
|
8
9
|
|
|
9
10
|
export function buildReport({ config, inspections, failures = [], generatedAt = "deterministic" }) {
|
|
10
11
|
const inspectionById = new Map(inspections.map((inspection) => [inspection.id, inspection]));
|
|
@@ -140,6 +141,10 @@ export async function buildCompatibilityReport(options = {}) {
|
|
|
140
141
|
decisions,
|
|
141
142
|
});
|
|
142
143
|
|
|
144
|
+
const runtimeCoverage = applyRuntimeExecutionCoverage({
|
|
145
|
+
findings: [...warnings, ...suggestions],
|
|
146
|
+
executionResults: options.executionResults,
|
|
147
|
+
});
|
|
143
148
|
const issues = buildIssues({
|
|
144
149
|
breakages,
|
|
145
150
|
warnings,
|
|
@@ -149,6 +154,8 @@ export async function buildCompatibilityReport(options = {}) {
|
|
|
149
154
|
});
|
|
150
155
|
const contractProbes = buildContractProbes({ warnings, suggestions, fixtures: fixtureReports });
|
|
151
156
|
const issueSummary = summarizeIssueClasses(issues);
|
|
157
|
+
const openIssues = issues.filter((issue) => issue.status !== "runtime-covered");
|
|
158
|
+
const openIssueSummary = summarizeIssueClasses(openIssues);
|
|
152
159
|
|
|
153
160
|
return {
|
|
154
161
|
generatedAt: options.generatedAt ?? "deterministic",
|
|
@@ -163,8 +170,11 @@ export async function buildCompatibilityReport(options = {}) {
|
|
|
163
170
|
decisionCount: decisions.length,
|
|
164
171
|
logCount: logs.length,
|
|
165
172
|
issueCount: issues.length,
|
|
173
|
+
openIssueCount: openIssues.length,
|
|
166
174
|
p0IssueCount: issues.filter((issue) => issue.severity === "P0").length,
|
|
167
175
|
p1IssueCount: issues.filter((issue) => issue.severity === "P1").length,
|
|
176
|
+
openP0IssueCount: openIssues.filter((issue) => issue.severity === "P0").length,
|
|
177
|
+
openP1IssueCount: openIssues.filter((issue) => issue.severity === "P1").length,
|
|
168
178
|
liveIssueCount: issueSummary["live-issue"],
|
|
169
179
|
liveP0IssueCount: issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity === "P0").length,
|
|
170
180
|
compatGapCount: issueSummary["compat-gap"],
|
|
@@ -172,6 +182,10 @@ export async function buildCompatibilityReport(options = {}) {
|
|
|
172
182
|
inspectorGapCount: issueSummary["inspector-gap"],
|
|
173
183
|
upstreamIssueCount: issueSummary["upstream-metadata"],
|
|
174
184
|
fixtureRegressionCount: issueSummary["fixture-regression"],
|
|
185
|
+
openInspectorGapCount: openIssueSummary["inspector-gap"],
|
|
186
|
+
runtimeCoveredIssueCount: runtimeCoverage.coveredFindingCount,
|
|
187
|
+
runtimePartiallyCoveredIssueCount: runtimeCoverage.partiallyCoveredFindingCount,
|
|
188
|
+
runtimeCoverageArtifactCount: runtimeCoverage.coverage.artifactCount,
|
|
175
189
|
contractProbeCount: contractProbes.length,
|
|
176
190
|
},
|
|
177
191
|
fixtures: fixtureReports,
|
|
@@ -308,7 +322,11 @@ function topTextFindings(report, limit) {
|
|
|
308
322
|
return [
|
|
309
323
|
...(report.breakages ?? []).map((finding) => formatTextFinding(finding, "breakage")),
|
|
310
324
|
...(report.issues ?? [])
|
|
311
|
-
.filter(
|
|
325
|
+
.filter(
|
|
326
|
+
(issue) =>
|
|
327
|
+
issue.status !== "runtime-covered" &&
|
|
328
|
+
(issue.status === "blocking" || issue.severity === "P0" || issue.severity === "P1"),
|
|
329
|
+
)
|
|
312
330
|
.map((issue) => formatTextFinding(issue, issue.severity ?? "issue")),
|
|
313
331
|
...(report.warnings ?? []).map((finding) => formatTextFinding(finding, "warning")),
|
|
314
332
|
].slice(0, limit);
|
package/src/runtime-profile.js
CHANGED
|
@@ -46,8 +46,8 @@ export async function buildRuntimeProfile(options = {}) {
|
|
|
46
46
|
os: process.platform,
|
|
47
47
|
arch: process.arch,
|
|
48
48
|
node: process.version,
|
|
49
|
-
rssSampler: process.platform === "win32" ? "unavailable" : "ps",
|
|
50
|
-
cpuSampler: process.platform === "win32" ? "unavailable" : "ps-percent",
|
|
49
|
+
rssSampler: process.platform === "win32" ? "unavailable" : "ps-immediate-25ms",
|
|
50
|
+
cpuSampler: process.platform === "win32" ? "unavailable" : "ps-percent-immediate-25ms",
|
|
51
51
|
},
|
|
52
52
|
summary: summarizeProfile(commands),
|
|
53
53
|
groups: summarizeCommandGroups(commands),
|
|
@@ -65,8 +65,8 @@ export function validateRuntimeProfile(profile) {
|
|
|
65
65
|
errors.push(`${command.id}: missing wall time`);
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
|
-
if (profile.platform?.rssSampler !== "unavailable" && profile.commands.every((command) => command
|
|
69
|
-
errors.push("all commands are missing peak RSS");
|
|
68
|
+
if (profile.platform?.rssSampler !== "unavailable" && profile.commands.every((command) => !hasRssSample(command))) {
|
|
69
|
+
errors.push("all commands are missing peak RSS samples");
|
|
70
70
|
}
|
|
71
71
|
return errors;
|
|
72
72
|
}
|
|
@@ -98,10 +98,17 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
|
|
|
98
98
|
[
|
|
99
99
|
["Commands", profile.summary.commandCount],
|
|
100
100
|
["P50 wall time", `${profile.summary.p50WallMs} ms`],
|
|
101
|
-
["P95 wall time", `${profile.summary.p95WallMs} ms`],
|
|
102
|
-
["
|
|
103
|
-
["
|
|
104
|
-
["
|
|
101
|
+
["Command P95 wall time", `${profile.summary.p95WallMs} ms`],
|
|
102
|
+
["Wall time basis", profile.summary.wallTimeBasis ?? "command-median-p95"],
|
|
103
|
+
["Profile samples", profile.summary.sampleCount ?? sampleCount(profile.commands)],
|
|
104
|
+
["RSS samples", profile.summary.rssSampleCount ?? rssSampleCount(profile.commands)],
|
|
105
|
+
["CPU samples", profile.summary.cpuSampleCount ?? cpuSampleCount(profile.commands)],
|
|
106
|
+
["Max peak RSS", formatSampledMetric(profile.summary.maxPeakRssMb, profile.summary.rssSampleCount ?? rssSampleCount(profile.commands))],
|
|
107
|
+
["Max RSS delta", formatSampledMetric(profile.summary.maxRssDeltaMb, profile.summary.rssSampleCount ?? rssSampleCount(profile.commands))],
|
|
108
|
+
[
|
|
109
|
+
"Max CPU estimate",
|
|
110
|
+
formatSampledMetric(profile.summary.maxCpuMsEstimate, profile.summary.cpuSampleCount ?? cpuSampleCount(profile.commands), "ms"),
|
|
111
|
+
],
|
|
105
112
|
["Max harness heap delta", `${profile.summary.maxHarnessHeapDeltaMb} MB`],
|
|
106
113
|
],
|
|
107
114
|
["Metric", "Value"],
|
|
@@ -129,13 +136,14 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
|
|
|
129
136
|
command.label,
|
|
130
137
|
`${command.wallMs.median} ms`,
|
|
131
138
|
`${command.wallMs.max} ms`,
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
139
|
+
formatSampledMetric(command.peakRssMb.max, command.rssSampleCount),
|
|
140
|
+
formatSampledMetric(command.rssDeltaMb.max, command.rssSampleCount),
|
|
141
|
+
formatSampledMetric(command.cpuMsEstimate.max, command.cpuSampleCount, "ms"),
|
|
135
142
|
`${command.harnessHeapDeltaMb.max} MB`,
|
|
143
|
+
`${command.rssSampleCount ?? 0}/${command.cpuSampleCount ?? 0}`,
|
|
136
144
|
command.exitCodes.join(", "),
|
|
137
145
|
]),
|
|
138
|
-
["ID", "Label", "Median wall", "Max wall", "Max peak RSS", "Max RSS delta", "CPU estimate", "Heap delta", "Exit codes"],
|
|
146
|
+
["ID", "Label", "Median wall", "Max wall", "Max peak RSS", "Max RSS delta", "CPU estimate", "Heap delta", "RSS/CPU samples", "Exit codes"],
|
|
139
147
|
),
|
|
140
148
|
"",
|
|
141
149
|
"## Category Rollups",
|
|
@@ -146,11 +154,12 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
|
|
|
146
154
|
group.commandCount,
|
|
147
155
|
`${group.p50WallMs} ms`,
|
|
148
156
|
`${group.p95WallMs} ms`,
|
|
149
|
-
|
|
150
|
-
|
|
157
|
+
formatSampledMetric(group.maxPeakRssMb, group.rssSampleCount),
|
|
158
|
+
formatSampledMetric(group.maxCpuMsEstimate, group.cpuSampleCount, "ms"),
|
|
159
|
+
`${group.rssSampleCount ?? 0}/${group.cpuSampleCount ?? 0}`,
|
|
151
160
|
group.commands.join(", "),
|
|
152
161
|
]),
|
|
153
|
-
["Category", "Commands", "P50 wall", "P95 wall", "Max peak RSS", "CPU estimate", "Command IDs"],
|
|
162
|
+
["Category", "Commands", "P50 wall", "P95 wall", "Max peak RSS", "CPU estimate", "RSS/CPU samples", "Command IDs"],
|
|
154
163
|
),
|
|
155
164
|
].join("\n");
|
|
156
165
|
}
|
|
@@ -189,8 +198,15 @@ function summarizeProfile(commands) {
|
|
|
189
198
|
const maxRssDeltaMb = Math.max(0, ...commands.map((command) => command.rssDeltaMb.max));
|
|
190
199
|
const maxCpuMsEstimate = Math.max(0, ...commands.map((command) => command.cpuMsEstimate.max));
|
|
191
200
|
const maxHarnessHeapDeltaMb = Math.max(0, ...commands.map((command) => command.harnessHeapDeltaMb.max));
|
|
201
|
+
const totalSampleCount = sampleCount(commands);
|
|
202
|
+
const totalRssSampleCount = rssSampleCount(commands);
|
|
203
|
+
const totalCpuSampleCount = cpuSampleCount(commands);
|
|
192
204
|
return {
|
|
193
205
|
commandCount: commands.length,
|
|
206
|
+
sampleCount: totalSampleCount,
|
|
207
|
+
rssSampleCount: totalRssSampleCount,
|
|
208
|
+
cpuSampleCount: totalCpuSampleCount,
|
|
209
|
+
wallTimeBasis: "command-median-p95",
|
|
194
210
|
p50WallMs: percentile(wallTimes, 0.5),
|
|
195
211
|
p95WallMs: percentile(wallTimes, 0.95),
|
|
196
212
|
maxPeakRssMb,
|
|
@@ -206,6 +222,12 @@ function summarizeCommand(command, samples) {
|
|
|
206
222
|
const rssDeltaMb = samples.map((sample) => sample.rssDeltaMb).sort((left, right) => left - right);
|
|
207
223
|
const peakCpuPercent = samples.map((sample) => sample.peakCpuPercent).sort((left, right) => left - right);
|
|
208
224
|
const cpuMsEstimate = samples.map((sample) => sample.cpuMsEstimate).sort((left, right) => left - right);
|
|
225
|
+
const statSampleCount = samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0);
|
|
226
|
+
const rssSampleTotal = samples.reduce(
|
|
227
|
+
(sum, sample) => sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0)),
|
|
228
|
+
0,
|
|
229
|
+
);
|
|
230
|
+
const cpuSampleTotal = samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0);
|
|
209
231
|
const harnessHeapDeltaMb = samples
|
|
210
232
|
.map((sample) => sample.harnessHeapDeltaMb)
|
|
211
233
|
.sort((left, right) => left - right);
|
|
@@ -222,6 +244,9 @@ function summarizeCommand(command, samples) {
|
|
|
222
244
|
peakCpuPercent: summarizeNumbers(peakCpuPercent),
|
|
223
245
|
cpuMsEstimate: summarizeNumbers(cpuMsEstimate),
|
|
224
246
|
harnessHeapDeltaMb: summarizeNumbers(harnessHeapDeltaMb),
|
|
247
|
+
statSampleCount,
|
|
248
|
+
rssSampleCount: rssSampleTotal,
|
|
249
|
+
cpuSampleCount: cpuSampleTotal,
|
|
225
250
|
exitCodes: [...new Set(samples.map((sample) => sample.exitCode))].sort(),
|
|
226
251
|
};
|
|
227
252
|
}
|
|
@@ -244,6 +269,8 @@ function summarizeCommandGroups(commands) {
|
|
|
244
269
|
const cpuMs = categoryCommands
|
|
245
270
|
.flatMap((command) => command.samples.map((sample) => sample.cpuMsEstimate))
|
|
246
271
|
.sort((left, right) => left - right);
|
|
272
|
+
const groupRssSampleCount = rssSampleCount(categoryCommands);
|
|
273
|
+
const groupCpuSampleCount = cpuSampleCount(categoryCommands);
|
|
247
274
|
return {
|
|
248
275
|
category,
|
|
249
276
|
commandCount: categoryCommands.length,
|
|
@@ -251,11 +278,36 @@ function summarizeCommandGroups(commands) {
|
|
|
251
278
|
p95WallMs: percentile(wallTimes, 0.95),
|
|
252
279
|
maxPeakRssMb: peakRss.at(-1) ?? 0,
|
|
253
280
|
maxCpuMsEstimate: cpuMs.at(-1) ?? 0,
|
|
281
|
+
rssSampleCount: groupRssSampleCount,
|
|
282
|
+
cpuSampleCount: groupCpuSampleCount,
|
|
254
283
|
commands: categoryCommands.map((command) => command.id),
|
|
255
284
|
};
|
|
256
285
|
});
|
|
257
286
|
}
|
|
258
287
|
|
|
288
|
+
function hasRssSample(command) {
|
|
289
|
+
return (command.rssSampleCount ?? (command.peakRssMb?.max > 0 ? 1 : 0)) > 0;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
function sampleCount(commands) {
|
|
293
|
+
return commands.reduce((sum, command) => sum + (command.samples?.length ?? 0), 0);
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function rssSampleCount(commands) {
|
|
297
|
+
return commands.reduce((sum, command) => sum + (command.rssSampleCount ?? (command.peakRssMb?.max > 0 ? 1 : 0)), 0);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function cpuSampleCount(commands) {
|
|
301
|
+
return commands.reduce((sum, command) => sum + (command.cpuSampleCount ?? 0), 0);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function formatSampledMetric(value, count, unit = "MB") {
|
|
305
|
+
if ((count ?? 0) <= 0) {
|
|
306
|
+
return "n/a";
|
|
307
|
+
}
|
|
308
|
+
return `${value} ${unit}`;
|
|
309
|
+
}
|
|
310
|
+
|
|
259
311
|
function summarizeNumbers(values) {
|
|
260
312
|
return {
|
|
261
313
|
min: values[0],
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
export function applyRuntimeExecutionCoverage({ findings = [], executionResults } = {}) {
|
|
2
|
+
const coverage = buildRuntimeExecutionCoverage(executionResults);
|
|
3
|
+
let coveredFindingCount = 0;
|
|
4
|
+
let partiallyCoveredFindingCount = 0;
|
|
5
|
+
|
|
6
|
+
for (const finding of findings) {
|
|
7
|
+
const findingCoverage = runtimeCoverageForFinding(finding, coverage);
|
|
8
|
+
if (!findingCoverage) {
|
|
9
|
+
continue;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
finding.runtimeCoverage = findingCoverage;
|
|
13
|
+
if (findingCoverage.status === "covered") {
|
|
14
|
+
finding.status = "runtime-covered";
|
|
15
|
+
coveredFindingCount += 1;
|
|
16
|
+
} else {
|
|
17
|
+
partiallyCoveredFindingCount += 1;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
return {
|
|
22
|
+
coverage,
|
|
23
|
+
coveredFindingCount,
|
|
24
|
+
partiallyCoveredFindingCount,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function buildRuntimeExecutionCoverage(executionResults) {
|
|
29
|
+
const fixtures = new Map();
|
|
30
|
+
for (const artifact of executionResults?.artifacts ?? []) {
|
|
31
|
+
if (artifact.kind !== "capture") {
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const fixture = String(artifact.fixture ?? "unknown");
|
|
36
|
+
const fixtureCoverage = ensureFixtureCoverage(fixtures, fixture);
|
|
37
|
+
if (artifact.artifactPath) {
|
|
38
|
+
fixtureCoverage.artifacts.add(artifact.artifactPath);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
for (const captured of normalizeCaptured(artifact.captured)) {
|
|
42
|
+
fixtureCoverage.captured.add(captured);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return {
|
|
47
|
+
fixtures,
|
|
48
|
+
artifactCount: [...fixtures.values()].reduce((sum, fixture) => sum + fixture.artifacts.size, 0),
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function runtimeCoverageForFinding(finding, coverage) {
|
|
53
|
+
const fixtureCoverage = coverage.fixtures.get(finding.fixture);
|
|
54
|
+
if (!fixtureCoverage || fixtureCoverage.captured.size === 0) {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const expected = expectedRuntimeCaptureKeys(finding);
|
|
59
|
+
if (expected.length === 0) {
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const captured = expected.filter((item) => fixtureCoverage.captured.has(item));
|
|
64
|
+
if (captured.length === 0) {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return {
|
|
69
|
+
status: captured.length === expected.length ? "covered" : "partial",
|
|
70
|
+
captured,
|
|
71
|
+
expected,
|
|
72
|
+
artifacts: [...fixtureCoverage.artifacts].sort(),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function expectedRuntimeCaptureKeys(finding) {
|
|
77
|
+
const names = evidenceNames(finding.evidence);
|
|
78
|
+
if (finding.code === "registration-capture-gap") {
|
|
79
|
+
return names.map((name) => `registration:${name}`);
|
|
80
|
+
}
|
|
81
|
+
if (finding.code === "runtime-tool-capture") {
|
|
82
|
+
return ["registration:registerTool"];
|
|
83
|
+
}
|
|
84
|
+
if (finding.code === "conversation-access-hook") {
|
|
85
|
+
return names.map((name) => `hook:${name}`);
|
|
86
|
+
}
|
|
87
|
+
return [];
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function normalizeCaptured(captured) {
|
|
91
|
+
return (captured ?? [])
|
|
92
|
+
.map((item) => {
|
|
93
|
+
if (typeof item === "string") {
|
|
94
|
+
return item;
|
|
95
|
+
}
|
|
96
|
+
if (item && typeof item === "object" && item.kind && item.name) {
|
|
97
|
+
return `${item.kind}:${item.name}`;
|
|
98
|
+
}
|
|
99
|
+
return "";
|
|
100
|
+
})
|
|
101
|
+
.filter(Boolean);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function evidenceNames(evidence) {
|
|
105
|
+
return [
|
|
106
|
+
...new Set(
|
|
107
|
+
(evidence ?? [])
|
|
108
|
+
.map((item) => String(item).split(" @ ")[0]?.trim())
|
|
109
|
+
.filter(Boolean),
|
|
110
|
+
),
|
|
111
|
+
];
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function ensureFixtureCoverage(fixtures, fixture) {
|
|
115
|
+
let fixtureCoverage = fixtures.get(fixture);
|
|
116
|
+
if (!fixtureCoverage) {
|
|
117
|
+
fixtureCoverage = {
|
|
118
|
+
artifacts: new Set(),
|
|
119
|
+
captured: new Set(),
|
|
120
|
+
};
|
|
121
|
+
fixtures.set(fixture, fixtureCoverage);
|
|
122
|
+
}
|
|
123
|
+
return fixtureCoverage;
|
|
124
|
+
}
|