@openclaw/plugin-inspector 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/LICENSE +21 -0
- package/README.md +249 -2
- package/examples/github-actions-plugin-inspector.yml +24 -0
- package/examples/plugin-inspector.config.json +15 -0
- package/package.json +55 -3
- package/src/artifacts.js +113 -0
- package/src/capture-api.js +115 -0
- package/src/ci-policy.js +259 -0
- package/src/ci-summary.js +223 -0
- package/src/cli.js +130 -0
- package/src/cold-import-readiness.js +271 -0
- package/src/compatibility-report.js +356 -0
- package/src/config.js +182 -0
- package/src/contract-capture.js +288 -0
- package/src/contract-coverage.js +167 -0
- package/src/contract-probes.js +156 -0
- package/src/execution-results.js +297 -0
- package/src/fixture-summary.js +780 -0
- package/src/import-loop-profile.js +169 -0
- package/src/index.js +186 -0
- package/src/inspector.js +405 -0
- package/src/issues.js +366 -0
- package/src/json-file.js +10 -0
- package/src/mock-sdk-capture-runner.js +69 -0
- package/src/openclaw-target.js +179 -0
- package/src/path-utils.js +28 -0
- package/src/platform-probes.js +238 -0
- package/src/process-profile.js +117 -0
- package/src/profile-diff.js +222 -0
- package/src/ref-diff.js +335 -0
- package/src/report.js +435 -0
- package/src/runtime-capture-report.js +134 -0
- package/src/runtime-profile.js +289 -0
- package/src/sdk-mock.js +61 -0
- package/src/stats.js +13 -0
- package/src/synthetic-probes.js +544 -0
- package/src/workspace-plan.js +496 -0
package/src/ci-policy.js
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
|
|
3
|
+
import { resolveFromRoot } from "./path-utils.js";
|
|
4
|
+
|
|
5
|
+
export const defaultCiPolicyReportOptions = {
|
|
6
|
+
generatedAt: "deterministic",
|
|
7
|
+
jsonPath: "reports/plugin-inspector-ci-policy.json",
|
|
8
|
+
markdownPath: "reports/plugin-inspector-ci-policy.md",
|
|
9
|
+
reportTitle: "Plugin Inspector CI Policy",
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
export function buildCiPolicyReport(options = {}) {
|
|
13
|
+
const policy = options.policy;
|
|
14
|
+
validateCiPolicy(policy);
|
|
15
|
+
|
|
16
|
+
const checks = [
|
|
17
|
+
...compatibilityChecks(options.compatibilityReport, { strict: options.strict }),
|
|
18
|
+
...refDiffChecks(options.refDiff, { strict: options.strict }),
|
|
19
|
+
...executionChecks(options.executionResults, policy, { strict: options.strict }),
|
|
20
|
+
].sort((left, right) => actionRank(left.action) - actionRank(right.action) || left.id.localeCompare(right.id));
|
|
21
|
+
|
|
22
|
+
return {
|
|
23
|
+
generatedAt: options.generatedAt ?? defaultCiPolicyReportOptions.generatedAt,
|
|
24
|
+
status: checks.some((check) => check.action === "fail") ? "fail" : "pass",
|
|
25
|
+
strict: Boolean(options.strict),
|
|
26
|
+
policy: {
|
|
27
|
+
allowedBlocked: policy.allowedBlocked.length,
|
|
28
|
+
expectedWarnings: policy.expectedWarnings.length,
|
|
29
|
+
fixtureSets: Object.keys(policy.fixtureSets).sort(),
|
|
30
|
+
thresholds: policy.thresholds,
|
|
31
|
+
},
|
|
32
|
+
summary: {
|
|
33
|
+
checkCount: checks.length,
|
|
34
|
+
failCount: checks.filter((check) => check.action === "fail").length,
|
|
35
|
+
warnCount: checks.filter((check) => check.action === "warn").length,
|
|
36
|
+
passCount: checks.filter((check) => check.action === "pass").length,
|
|
37
|
+
},
|
|
38
|
+
checks,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function validateCiPolicy(policy) {
|
|
43
|
+
const errors = [];
|
|
44
|
+
if (policy?.version !== 1) {
|
|
45
|
+
errors.push("ci policy version must be 1");
|
|
46
|
+
}
|
|
47
|
+
for (const key of ["allowedBlocked", "expectedWarnings"]) {
|
|
48
|
+
if (!Array.isArray(policy?.[key])) {
|
|
49
|
+
errors.push(`ci policy ${key} must be an array`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
if (!policy?.thresholds || typeof policy.thresholds !== "object") {
|
|
53
|
+
errors.push("ci policy thresholds are required");
|
|
54
|
+
}
|
|
55
|
+
if (!policy?.fixtureSets || typeof policy.fixtureSets !== "object") {
|
|
56
|
+
errors.push("ci policy fixtureSets are required");
|
|
57
|
+
}
|
|
58
|
+
if (errors.length > 0) {
|
|
59
|
+
throw new Error(errors.join("\n"));
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function validateCiPolicyReport(report) {
|
|
64
|
+
return report.checks
|
|
65
|
+
.filter((check) => check.action === "fail")
|
|
66
|
+
.map((check) => `${check.id}: ${check.message}: ${check.evidence.join(", ")}`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export async function writeCiPolicyReport(report, options = {}) {
|
|
70
|
+
const rootDir = path.resolve(options.rootDir ?? process.cwd());
|
|
71
|
+
const jsonPath = resolveFromRoot(rootDir, options.jsonPath ?? defaultCiPolicyReportOptions.jsonPath);
|
|
72
|
+
const markdownPath = resolveFromRoot(rootDir, options.markdownPath ?? defaultCiPolicyReportOptions.markdownPath);
|
|
73
|
+
return writeJsonMarkdownArtifacts({
|
|
74
|
+
jsonPath,
|
|
75
|
+
markdownPath,
|
|
76
|
+
json: report,
|
|
77
|
+
markdown: renderCiPolicyMarkdown(report, options),
|
|
78
|
+
check: options.check,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function renderCiPolicyMarkdown(report, options = {}) {
|
|
83
|
+
const title = options.title ?? options.reportTitle ?? defaultCiPolicyReportOptions.reportTitle;
|
|
84
|
+
return [
|
|
85
|
+
`# ${title}`,
|
|
86
|
+
"",
|
|
87
|
+
`Generated: ${report.generatedAt}`,
|
|
88
|
+
`Status: ${report.status.toUpperCase()}`,
|
|
89
|
+
`Strict: ${report.strict}`,
|
|
90
|
+
"",
|
|
91
|
+
"## Summary",
|
|
92
|
+
"",
|
|
93
|
+
markdownTable(
|
|
94
|
+
[
|
|
95
|
+
["Checks", report.summary.checkCount],
|
|
96
|
+
["Fail", report.summary.failCount],
|
|
97
|
+
["Warn", report.summary.warnCount],
|
|
98
|
+
["Pass", report.summary.passCount],
|
|
99
|
+
["Allowed blocked rules", report.policy.allowedBlocked],
|
|
100
|
+
["Expected warning rules", report.policy.expectedWarnings],
|
|
101
|
+
["Fixture sets", report.policy.fixtureSets.join(", ")],
|
|
102
|
+
],
|
|
103
|
+
["Metric", "Value"],
|
|
104
|
+
),
|
|
105
|
+
"",
|
|
106
|
+
"## Checks",
|
|
107
|
+
"",
|
|
108
|
+
markdownTable(
|
|
109
|
+
report.checks.map((check) => [
|
|
110
|
+
check.action,
|
|
111
|
+
check.id,
|
|
112
|
+
check.message,
|
|
113
|
+
check.evidence.join(", ") || "-",
|
|
114
|
+
]),
|
|
115
|
+
["Action", "ID", "Message", "Evidence"],
|
|
116
|
+
),
|
|
117
|
+
].join("\n");
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function compatibilityChecks(report, options) {
|
|
121
|
+
const checks = [];
|
|
122
|
+
if (!report) {
|
|
123
|
+
checks.push({
|
|
124
|
+
id: "compatibility-report.missing",
|
|
125
|
+
action: "fail",
|
|
126
|
+
message: "compatibility report is missing",
|
|
127
|
+
evidence: [],
|
|
128
|
+
});
|
|
129
|
+
return checks;
|
|
130
|
+
}
|
|
131
|
+
checks.push({
|
|
132
|
+
id: "compatibility-report.breakages",
|
|
133
|
+
action: report.summary.breakageCount > 0 ? "fail" : "pass",
|
|
134
|
+
message: `${report.summary.breakageCount} hard breakages`,
|
|
135
|
+
evidence: (report.breakages ?? []).map((finding) => `${finding.fixture}:${finding.code}`),
|
|
136
|
+
});
|
|
137
|
+
checks.push({
|
|
138
|
+
id: "compatibility-report.p1-issues",
|
|
139
|
+
action: "pass",
|
|
140
|
+
message: `${report.summary.p1IssueCount} P1 issues tracked`,
|
|
141
|
+
evidence: (report.issues ?? [])
|
|
142
|
+
.filter((issue) => issue.severity === "P1")
|
|
143
|
+
.map((issue) => `${issue.fixture}:${issue.code}`),
|
|
144
|
+
});
|
|
145
|
+
const issues = report.issues ?? [];
|
|
146
|
+
const liveP0Issues = issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity === "P0");
|
|
147
|
+
const deprecationWarnings = issues.filter((issue) => issue.issueClass === "deprecation-warning");
|
|
148
|
+
const inspectorGaps = issues.filter((issue) => issue.issueClass === "inspector-gap");
|
|
149
|
+
checks.push({
|
|
150
|
+
id: "compatibility-report.live-p0-issues",
|
|
151
|
+
action: liveP0Issues.length > 0 ? (options.strict ? "fail" : "warn") : "pass",
|
|
152
|
+
message: `${liveP0Issues.length} live P0 issues tracked`,
|
|
153
|
+
evidence: liveP0Issues.map((issue) => `${issue.fixture}:${issue.code}:${issue.compatStatus ?? "none"}`),
|
|
154
|
+
});
|
|
155
|
+
checks.push({
|
|
156
|
+
id: "compatibility-report.deprecation-warnings",
|
|
157
|
+
action: "pass",
|
|
158
|
+
message: `${deprecationWarnings.length} deprecated compat seams tracked`,
|
|
159
|
+
evidence: deprecationWarnings.map((issue) => `${issue.fixture}:${issue.code}`),
|
|
160
|
+
});
|
|
161
|
+
checks.push({
|
|
162
|
+
id: "compatibility-report.inspector-gaps",
|
|
163
|
+
action: "pass",
|
|
164
|
+
message: `${inspectorGaps.length} inspector proof gaps tracked`,
|
|
165
|
+
evidence: inspectorGaps.map((issue) => `${issue.fixture}:${issue.code}`),
|
|
166
|
+
});
|
|
167
|
+
return checks;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function refDiffChecks(refDiff, options) {
|
|
171
|
+
if (!refDiff) {
|
|
172
|
+
return [
|
|
173
|
+
{
|
|
174
|
+
id: "ref-diff.not-run",
|
|
175
|
+
action: "pass",
|
|
176
|
+
message: "ref diff artifact was not present for this CI mode",
|
|
177
|
+
evidence: [],
|
|
178
|
+
},
|
|
179
|
+
];
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
return (refDiff.regressions ?? []).map((regression) => ({
|
|
183
|
+
id: `ref-diff.${regression.code}`,
|
|
184
|
+
action: regression.action === "fail" || (options.strict && regression.action === "warn") ? "fail" : "warn",
|
|
185
|
+
message: regression.message,
|
|
186
|
+
evidence: regression.evidence ?? [],
|
|
187
|
+
}));
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function executionChecks(executionResults, policy, options) {
|
|
191
|
+
if (!executionResults) {
|
|
192
|
+
return [
|
|
193
|
+
{
|
|
194
|
+
id: "execution-results.not-run",
|
|
195
|
+
action: "pass",
|
|
196
|
+
message: "isolated execution artifact was not present for this CI mode",
|
|
197
|
+
evidence: [],
|
|
198
|
+
},
|
|
199
|
+
];
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const checks = [
|
|
203
|
+
{
|
|
204
|
+
id: "execution-results.failures",
|
|
205
|
+
action: executionResults.summary.failCount > 0 ? "fail" : "pass",
|
|
206
|
+
message: `${executionResults.summary.failCount} failed synthetic probes`,
|
|
207
|
+
evidence: failedExecutionEvidence(executionResults),
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
id: "execution-results.audit-findings",
|
|
211
|
+
action: executionResults.summary.auditFindingCount > 0 ? "warn" : "pass",
|
|
212
|
+
message: `${executionResults.summary.auditFindingCount ?? 0} package audit findings`,
|
|
213
|
+
evidence: executionResults.artifacts
|
|
214
|
+
.filter((artifact) => artifact.kind === "audit" && artifact.findingCount > 0)
|
|
215
|
+
.map((artifact) => `${artifact.fixture}:${artifact.findingCount}`),
|
|
216
|
+
},
|
|
217
|
+
];
|
|
218
|
+
|
|
219
|
+
const blocked = executionResults.artifacts.flatMap((artifact) =>
|
|
220
|
+
(artifact.blocked ?? []).map((item) => ({ artifact, item })),
|
|
221
|
+
);
|
|
222
|
+
for (const blockedItem of blocked) {
|
|
223
|
+
const expectedWarning = findPolicyMatch(policy.expectedWarnings, blockedItem.item);
|
|
224
|
+
const allowedBlocked = findPolicyMatch(policy.allowedBlocked, blockedItem.item);
|
|
225
|
+
const match = expectedWarning ?? allowedBlocked;
|
|
226
|
+
checks.push({
|
|
227
|
+
id: `execution-results.blocked.${blockedItem.artifact.fixture}.${blockedItem.item.seam}.${blockedItem.item.captureIndex}`,
|
|
228
|
+
action: match ? (options.strict ? "fail" : "warn") : "fail",
|
|
229
|
+
message: match
|
|
230
|
+
? `${match.decision}: ${blockedItem.item.reason}`
|
|
231
|
+
: `unknown blocked synthetic probe: ${blockedItem.item.reason}`,
|
|
232
|
+
evidence: [
|
|
233
|
+
blockedItem.artifact.artifactPath,
|
|
234
|
+
blockedItem.item.seam,
|
|
235
|
+
blockedItem.item.reason,
|
|
236
|
+
match?.id ?? "unclassified",
|
|
237
|
+
],
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
return checks;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function findPolicyMatch(rules, item) {
|
|
244
|
+
return rules.find((rule) => item.seam === rule.seam && item.reason?.includes(rule.reasonIncludes));
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function failedExecutionEvidence(executionResults) {
|
|
248
|
+
return executionResults.artifacts.flatMap((artifact) =>
|
|
249
|
+
(artifact.failures ?? []).map((failure) => `${artifact.fixture}:${failure.seam}:${failure.error}`),
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function actionRank(value) {
|
|
254
|
+
return { fail: 0, warn: 1, pass: 2 }[value] ?? 3;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function markdownTable(rows, headers) {
|
|
258
|
+
return renderPaddedMarkdownTable(rows, headers);
|
|
259
|
+
}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
|
|
4
|
+
import { readOptionalJsonFile } from "./json-file.js";
|
|
5
|
+
|
|
6
|
+
export const defaultCiReportPaths = {
|
|
7
|
+
compatibility: "reports/plugin-inspector-report.json",
|
|
8
|
+
capture: "reports/plugin-inspector-capture.json",
|
|
9
|
+
synthetic: "reports/plugin-inspector-synthetic-probes.json",
|
|
10
|
+
coldImport: "reports/plugin-inspector-cold-import.json",
|
|
11
|
+
workspace: "reports/plugin-inspector-workspace-plan.json",
|
|
12
|
+
platform: "reports/plugin-inspector-platform-probes.json",
|
|
13
|
+
importLoop: "reports/plugin-inspector-import-loop-profile.json",
|
|
14
|
+
execution: "reports/plugin-inspector-execution-results.json",
|
|
15
|
+
runtimeProfile: "reports/plugin-inspector-runtime-profile.json",
|
|
16
|
+
refDiff: "reports/plugin-inspector-ref-diff.json",
|
|
17
|
+
profileDiff: "reports/plugin-inspector-profile-diff.json",
|
|
18
|
+
ciPolicy: "reports/plugin-inspector-ci-policy.json",
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export async function buildCiSummary(options = {}) {
|
|
22
|
+
const reportPaths = options.reportPaths ?? defaultCiReportPaths;
|
|
23
|
+
const reports = options.reports ?? (await readCiReports(options.reportsDir ?? "reports", reportPaths));
|
|
24
|
+
const artifactBaseDir = options.artifactBaseDir ?? process.cwd();
|
|
25
|
+
|
|
26
|
+
return {
|
|
27
|
+
generatedAt: options.generatedAt ?? "deterministic",
|
|
28
|
+
title: options.title ?? "Plugin Inspector CI Summary",
|
|
29
|
+
mode: options.mode ?? "local",
|
|
30
|
+
openclawLabel: options.openclawLabel ?? "",
|
|
31
|
+
status: deriveCiStatus(reports),
|
|
32
|
+
summary: {
|
|
33
|
+
breakages: reports.compatibility?.summary?.breakageCount ?? 0,
|
|
34
|
+
warnings: reports.compatibility?.summary?.warningCount ?? 0,
|
|
35
|
+
suggestions: reports.compatibility?.summary?.suggestionCount ?? 0,
|
|
36
|
+
issues: reports.compatibility?.summary?.issueCount ?? 0,
|
|
37
|
+
p0Issues: reports.compatibility?.summary?.p0IssueCount ?? 0,
|
|
38
|
+
p1Issues: reports.compatibility?.summary?.p1IssueCount ?? 0,
|
|
39
|
+
liveIssues: reports.compatibility?.summary?.liveIssueCount ?? 0,
|
|
40
|
+
liveP0Issues: reports.compatibility?.summary?.liveP0IssueCount ?? 0,
|
|
41
|
+
compatGaps: reports.compatibility?.summary?.compatGapCount ?? 0,
|
|
42
|
+
deprecationWarnings: reports.compatibility?.summary?.deprecationWarningCount ?? 0,
|
|
43
|
+
inspectorGaps: reports.compatibility?.summary?.inspectorGapCount ?? 0,
|
|
44
|
+
upstreamIssues: reports.compatibility?.summary?.upstreamIssueCount ?? 0,
|
|
45
|
+
refDiffFailures: reports.refDiff?.summary?.hardRegressionCount ?? 0,
|
|
46
|
+
refDiffWarnings: reports.refDiff?.summary?.warningRegressionCount ?? 0,
|
|
47
|
+
policyFailures: reports.ciPolicy?.summary?.failCount ?? 0,
|
|
48
|
+
policyWarnings: reports.ciPolicy?.summary?.warnCount ?? 0,
|
|
49
|
+
profileFailures: reports.profileDiff?.summary?.failCount ?? 0,
|
|
50
|
+
profileWarnings: reports.profileDiff?.summary?.warnCount ?? 0,
|
|
51
|
+
executionPass: reports.execution?.summary?.passCount ?? 0,
|
|
52
|
+
executionFail: reports.execution?.summary?.failCount ?? 0,
|
|
53
|
+
executionBlocked: reports.execution?.summary?.blockedCount ?? 0,
|
|
54
|
+
platformWindowsRisks: reports.platform?.summary?.windowsRiskStepCount ?? 0,
|
|
55
|
+
platformContainerRisks: reports.platform?.summary?.containerRiskStepCount ?? 0,
|
|
56
|
+
loaderJitiCandidates: reports.platform?.summary?.jitiAlternativeCount ?? 0,
|
|
57
|
+
importLoopP50Ms: reports.importLoop?.summary?.p50WallMs ?? 0,
|
|
58
|
+
importLoopP95Ms: reports.importLoop?.summary?.p95WallMs ?? 0,
|
|
59
|
+
importLoopMaxRssMb: reports.importLoop?.summary?.maxPeakRssMb ?? 0,
|
|
60
|
+
importLoopMaxCpuMs: reports.importLoop?.summary?.maxCpuMsEstimate ?? 0,
|
|
61
|
+
},
|
|
62
|
+
topIssues: topIssues(reports.compatibility),
|
|
63
|
+
refRegressions: (reports.refDiff?.regressions ?? []).slice(0, 20),
|
|
64
|
+
policyFindings: (reports.ciPolicy?.checks ?? []).filter((check) => check.action !== "pass").slice(0, 20),
|
|
65
|
+
profileFindings: (reports.profileDiff?.checks ?? []).filter((check) => check.action !== "pass").slice(0, 20),
|
|
66
|
+
artifacts: Object.fromEntries(
|
|
67
|
+
Object.entries(reportPaths).map(([key, value]) => [key, existsSync(path.join(artifactBaseDir, value)) ? value : null]),
|
|
68
|
+
),
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export async function readCiReports(reportsDir, reportPaths = defaultCiReportPaths) {
|
|
73
|
+
const reports = {};
|
|
74
|
+
for (const [key, defaultPath] of Object.entries(reportPaths)) {
|
|
75
|
+
const reportPath = path.join(reportsDir, path.basename(defaultPath));
|
|
76
|
+
reports[key] = await readOptionalJsonFile(reportPath);
|
|
77
|
+
}
|
|
78
|
+
return reports;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function deriveCiStatus(reports) {
|
|
82
|
+
if ((reports.compatibility?.summary?.breakageCount ?? 0) > 0) {
|
|
83
|
+
return "fail";
|
|
84
|
+
}
|
|
85
|
+
if ((reports.refDiff?.summary?.hardRegressionCount ?? 0) > 0) {
|
|
86
|
+
return "fail";
|
|
87
|
+
}
|
|
88
|
+
if ((reports.ciPolicy?.summary?.failCount ?? 0) > 0) {
|
|
89
|
+
return "fail";
|
|
90
|
+
}
|
|
91
|
+
if ((reports.profileDiff?.summary?.failCount ?? 0) > 0) {
|
|
92
|
+
return "fail";
|
|
93
|
+
}
|
|
94
|
+
if ((reports.execution?.summary?.failCount ?? 0) > 0) {
|
|
95
|
+
return "fail";
|
|
96
|
+
}
|
|
97
|
+
return "pass";
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export async function writeCiSummary(summary, options = {}) {
|
|
101
|
+
const jsonPath = options.jsonPath ?? path.join(process.cwd(), "reports/plugin-inspector-ci-summary.json");
|
|
102
|
+
const markdownPath = options.markdownPath ?? path.join(process.cwd(), "reports/plugin-inspector-ci-summary.md");
|
|
103
|
+
return writeJsonMarkdownArtifacts({
|
|
104
|
+
jsonPath,
|
|
105
|
+
markdownPath,
|
|
106
|
+
json: summary,
|
|
107
|
+
markdown: renderCiSummaryMarkdown(summary),
|
|
108
|
+
check: options.check,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function renderCiSummaryMarkdown(summary) {
|
|
113
|
+
return [
|
|
114
|
+
`# ${summary.title ?? "Plugin Inspector CI Summary"}`,
|
|
115
|
+
"",
|
|
116
|
+
`Generated: ${summary.generatedAt}`,
|
|
117
|
+
`Mode: ${summary.mode}`,
|
|
118
|
+
`OpenClaw: ${summary.openclawLabel || "-"}`,
|
|
119
|
+
`Status: ${summary.status.toUpperCase()}`,
|
|
120
|
+
"",
|
|
121
|
+
"## Counts",
|
|
122
|
+
"",
|
|
123
|
+
markdownTable(
|
|
124
|
+
[
|
|
125
|
+
["Breakages", summary.summary.breakages],
|
|
126
|
+
["Warnings", summary.summary.warnings],
|
|
127
|
+
["Suggestions", summary.summary.suggestions],
|
|
128
|
+
["Issues", summary.summary.issues],
|
|
129
|
+
["P0 issues", summary.summary.p0Issues],
|
|
130
|
+
["P1 issues", summary.summary.p1Issues],
|
|
131
|
+
["Live issues", summary.summary.liveIssues],
|
|
132
|
+
["Live P0 issues", summary.summary.liveP0Issues],
|
|
133
|
+
["Compat gaps", summary.summary.compatGaps],
|
|
134
|
+
["Deprecation warnings", summary.summary.deprecationWarnings],
|
|
135
|
+
["Inspector gaps", summary.summary.inspectorGaps],
|
|
136
|
+
["Upstream metadata", summary.summary.upstreamIssues],
|
|
137
|
+
["Ref diff failures", summary.summary.refDiffFailures],
|
|
138
|
+
["Ref diff warnings", summary.summary.refDiffWarnings],
|
|
139
|
+
["Policy failures", summary.summary.policyFailures],
|
|
140
|
+
["Policy warnings", summary.summary.policyWarnings],
|
|
141
|
+
["Profile failures", summary.summary.profileFailures],
|
|
142
|
+
["Profile warnings", summary.summary.profileWarnings],
|
|
143
|
+
["Execution pass", summary.summary.executionPass],
|
|
144
|
+
["Execution fail", summary.summary.executionFail],
|
|
145
|
+
["Execution blocked", summary.summary.executionBlocked],
|
|
146
|
+
["Windows portability risks", summary.summary.platformWindowsRisks],
|
|
147
|
+
["Container portability risks", summary.summary.platformContainerRisks],
|
|
148
|
+
["Jiti loader candidates", summary.summary.loaderJitiCandidates],
|
|
149
|
+
[
|
|
150
|
+
"Import loop",
|
|
151
|
+
`p50 ${summary.summary.importLoopP50Ms} ms / p95 ${summary.summary.importLoopP95Ms} ms / max RSS ${summary.summary.importLoopMaxRssMb} MB / CPU ${summary.summary.importLoopMaxCpuMs} ms`,
|
|
152
|
+
],
|
|
153
|
+
],
|
|
154
|
+
["Metric", "Value"],
|
|
155
|
+
),
|
|
156
|
+
"",
|
|
157
|
+
"## Top Issues",
|
|
158
|
+
"",
|
|
159
|
+
markdownTable(
|
|
160
|
+
summary.topIssues.map((issue) => [issue.severity, issue.issueClass ?? "-", issue.fixture, issue.code, issue.decision, issue.title]),
|
|
161
|
+
["Severity", "Class", "Fixture", "Code", "Decision", "Title"],
|
|
162
|
+
),
|
|
163
|
+
"",
|
|
164
|
+
"## Ref Regressions",
|
|
165
|
+
"",
|
|
166
|
+
markdownTable(
|
|
167
|
+
summary.refRegressions.map((regression) => [
|
|
168
|
+
regression.action,
|
|
169
|
+
regression.severity,
|
|
170
|
+
regression.dimension,
|
|
171
|
+
regression.code,
|
|
172
|
+
regression.message,
|
|
173
|
+
]),
|
|
174
|
+
["Action", "Severity", "Surface", "Code", "Message"],
|
|
175
|
+
),
|
|
176
|
+
"",
|
|
177
|
+
"## Policy Findings",
|
|
178
|
+
"",
|
|
179
|
+
markdownTable(
|
|
180
|
+
summary.policyFindings.map((finding) => [finding.action, finding.id, finding.message, finding.evidence.join(", ")]),
|
|
181
|
+
["Action", "ID", "Message", "Evidence"],
|
|
182
|
+
),
|
|
183
|
+
"",
|
|
184
|
+
"## Profile Findings",
|
|
185
|
+
"",
|
|
186
|
+
markdownTable(
|
|
187
|
+
summary.profileFindings.map((finding) => [
|
|
188
|
+
finding.action,
|
|
189
|
+
finding.id,
|
|
190
|
+
finding.metric,
|
|
191
|
+
finding.baseline ?? "-",
|
|
192
|
+
finding.current ?? "-",
|
|
193
|
+
finding.message,
|
|
194
|
+
]),
|
|
195
|
+
["Action", "ID", "Metric", "Baseline", "Current", "Message"],
|
|
196
|
+
),
|
|
197
|
+
"",
|
|
198
|
+
"## Artifacts",
|
|
199
|
+
"",
|
|
200
|
+
markdownTable(
|
|
201
|
+
Object.entries(summary.artifacts).map(([key, value]) => [key, value ?? "-"]),
|
|
202
|
+
["Artifact", "Path"],
|
|
203
|
+
),
|
|
204
|
+
].join("\n");
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function topIssues(report) {
|
|
208
|
+
return (report?.issues ?? [])
|
|
209
|
+
.filter((issue) => ["P0", "P1"].includes(issue.severity))
|
|
210
|
+
.slice(0, 20)
|
|
211
|
+
.map((issue) => ({
|
|
212
|
+
severity: issue.severity,
|
|
213
|
+
issueClass: issue.issueClass,
|
|
214
|
+
fixture: issue.fixture,
|
|
215
|
+
code: issue.code,
|
|
216
|
+
title: issue.title,
|
|
217
|
+
decision: issue.decision,
|
|
218
|
+
}));
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function markdownTable(rows, headers) {
|
|
222
|
+
return renderPaddedMarkdownTable(rows, headers, { nullValue: "-" });
|
|
223
|
+
}
|
package/src/cli.js
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
buildRuntimeCaptureReport,
|
|
4
|
+
captureEntrypoint,
|
|
5
|
+
inspectCompatibilityFixtureSet,
|
|
6
|
+
inspectFixtureSet,
|
|
7
|
+
loadInspectorConfig,
|
|
8
|
+
loadPluginRootConfig,
|
|
9
|
+
renderTextSummary,
|
|
10
|
+
writeArtifacts,
|
|
11
|
+
writeCompatibilityReport,
|
|
12
|
+
writeReport,
|
|
13
|
+
writeRuntimeCaptureReport,
|
|
14
|
+
} from "./index.js";
|
|
15
|
+
|
|
16
|
+
const args = process.argv.slice(2);
|
|
17
|
+
const command = args[0]?.startsWith("-") ? "check" : (args[0] ?? "check");
|
|
18
|
+
const commandArgs = args[0]?.startsWith("-") ? args : args.slice(1);
|
|
19
|
+
|
|
20
|
+
try {
|
|
21
|
+
if (args.includes("--help") || args.includes("-h")) {
|
|
22
|
+
printHelp();
|
|
23
|
+
} else if (command === "check") {
|
|
24
|
+
await runCheck(commandArgs);
|
|
25
|
+
} else if (command === "inspect" || command === "report" || command === "ci") {
|
|
26
|
+
await runReport(command, commandArgs);
|
|
27
|
+
} else if (command === "capture") {
|
|
28
|
+
await runCapture(commandArgs);
|
|
29
|
+
} else {
|
|
30
|
+
throw new Error(`unknown command: ${command}`);
|
|
31
|
+
}
|
|
32
|
+
} catch (error) {
|
|
33
|
+
console.error(error.message);
|
|
34
|
+
process.exitCode = 1;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async function runCheck(commandArgs) {
|
|
38
|
+
const configPath = readFlag(commandArgs, "--config");
|
|
39
|
+
const outDir = readFlag(commandArgs, "--out") ?? "reports";
|
|
40
|
+
const openclawPath = commandArgs.includes("--no-openclaw") ? false : readFlag(commandArgs, "--openclaw");
|
|
41
|
+
const json = commandArgs.includes("--json");
|
|
42
|
+
const capture = commandArgs.includes("--capture");
|
|
43
|
+
const config = configPath ? await loadInspectorConfig(configPath) : await loadPluginRootConfig();
|
|
44
|
+
const report = await inspectCompatibilityFixtureSet(config, { openclawPath });
|
|
45
|
+
await writeCompatibilityReport(report, { outDir });
|
|
46
|
+
if (capture) {
|
|
47
|
+
if (process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED !== "1") {
|
|
48
|
+
throw new Error("check --capture imports plugin code; rerun with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 in an isolated workspace");
|
|
49
|
+
}
|
|
50
|
+
const captureReport = await buildRuntimeCaptureReport({ report, rootDir: config.rootDir, mockSdk: true });
|
|
51
|
+
await writeRuntimeCaptureReport(captureReport, {
|
|
52
|
+
jsonPath: `${outDir}/plugin-inspector-runtime-capture.json`,
|
|
53
|
+
markdownPath: `${outDir}/plugin-inspector-runtime-capture.md`,
|
|
54
|
+
});
|
|
55
|
+
if (captureReport.summary.failedCount > 0) {
|
|
56
|
+
throw new Error(`plugin-inspector runtime capture failed for ${captureReport.summary.failedCount} entrypoints`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (json) {
|
|
61
|
+
console.log(JSON.stringify(report, null, 2));
|
|
62
|
+
} else {
|
|
63
|
+
console.log(renderTextSummary(report));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (report.status !== "pass") {
|
|
67
|
+
throw new Error(`plugin-inspector found ${report.summary.breakageCount} breakages`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function runReport(command, commandArgs) {
|
|
72
|
+
const configPath = readFlag(commandArgs, "--config");
|
|
73
|
+
const outDir = readFlag(commandArgs, "--out") ?? "reports";
|
|
74
|
+
const check = commandArgs.includes("--check") || command === "ci";
|
|
75
|
+
const json = commandArgs.includes("--json");
|
|
76
|
+
const config = await loadInspectorConfig(configPath);
|
|
77
|
+
const report = await inspectFixtureSet(config);
|
|
78
|
+
await writeReport(report, { outDir });
|
|
79
|
+
|
|
80
|
+
if (json) {
|
|
81
|
+
console.log(JSON.stringify(report, null, 2));
|
|
82
|
+
} else {
|
|
83
|
+
console.log(renderTextSummary(report));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
if (check && report.status !== "pass") {
|
|
87
|
+
throw new Error(`plugin-inspector found ${report.summary.breakageCount} breakages`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function runCapture(commandArgs) {
|
|
92
|
+
const entrypoint = commandArgs.find((arg) => !arg.startsWith("-"));
|
|
93
|
+
const outputPath = readFlag(commandArgs, "--output");
|
|
94
|
+
const pluginRoot = readFlag(commandArgs, "--plugin-root");
|
|
95
|
+
const mockSdk = commandArgs.includes("--mock-sdk");
|
|
96
|
+
if (!entrypoint) {
|
|
97
|
+
throw new Error("capture requires an entrypoint path");
|
|
98
|
+
}
|
|
99
|
+
if (process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED !== "1") {
|
|
100
|
+
throw new Error("capture imports plugin code; rerun with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 in an isolated workspace");
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const result = await captureEntrypoint(entrypoint, { mockSdk, pluginRoot });
|
|
104
|
+
const json = `${JSON.stringify(result, null, 2)}\n`;
|
|
105
|
+
if (outputPath) {
|
|
106
|
+
await writeArtifacts([{ path: outputPath, content: json }]);
|
|
107
|
+
} else {
|
|
108
|
+
process.stdout.write(json);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function readFlag(commandArgs, name) {
|
|
113
|
+
const index = commandArgs.indexOf(name);
|
|
114
|
+
if (index === -1) {
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
return commandArgs[index + 1] ?? null;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function printHelp() {
|
|
121
|
+
console.log(`plugin-inspector
|
|
122
|
+
|
|
123
|
+
Usage:
|
|
124
|
+
plugin-inspector check [--config <path>] [--out <dir>] [--openclaw <path>] [--no-openclaw] [--capture] [--json]
|
|
125
|
+
plugin-inspector report --config <path> [--out <dir>] [--check] [--json]
|
|
126
|
+
plugin-inspector inspect --config <path> [--out <dir>] [--check] [--json]
|
|
127
|
+
plugin-inspector ci --config <path> [--out <dir>]
|
|
128
|
+
PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector capture <entrypoint> [--mock-sdk] [--plugin-root <path>] [--output <path>]
|
|
129
|
+
`);
|
|
130
|
+
}
|