@openclaw/plugin-inspector 0.0.0 → 0.1.0

package/src/contract-probes.js

@@ -0,0 +1,156 @@
+export const contractProbeRules = {
+  "before-tool-call-probe": {
+    id: "hook.before_tool_call.terminal-block-approval",
+    contract: "Hook returns preserve terminal, block, and approval semantics.",
+    target: "hook-runner",
+  },
+  "channel-contract-probe": {
+    id: "channel.runtime.envelope-config-metadata",
+    contract: "Channel setup, message envelope, sender metadata, and config schema remain stable.",
+    target: "channel-runtime",
+  },
+  "conversation-access-hook": {
+    id: "hook.llm-observer.privacy-payload",
+    contract: "LLM observer hooks receive documented prompt/output fields with expected redaction behavior.",
+    target: "hook-runner",
+  },
+  "legacy-root-sdk-import": {
+    id: "sdk.import.root-barrel-cold-import",
+    contract: "Root plugin SDK barrel remains importable or has a machine-readable migration path.",
+    target: "sdk-alias",
+  },
+  "legacy-before-agent-start": {
+    id: "hook.compat.before-agent-start-migration",
+    contract: "Legacy before_agent_start remains wired until plugins migrate to before_model_resolve and before_prompt_build.",
+    target: "hook-runner",
+  },
+  "sdk-export-missing": {
+    id: "sdk.import.package-export-cold-import",
+    contract: "Every observed OpenClaw plugin SDK import remains exported by the target OpenClaw package.",
+    target: "sdk-alias",
+  },
+  "provider-auth-env-vars": {
+    id: "manifest.compat.provider-auth-env-vars",
+    contract: "Legacy provider auth env metadata continues to map into config/help surfaces.",
+    target: "manifest-loader",
+  },
+  "channel-env-vars": {
+    id: "manifest.compat.channel-env-vars",
+    contract: "Legacy channel env metadata continues to map into channel setup/help surfaces.",
+    target: "manifest-loader",
+  },
+  "manifest-unknown-contracts": {
+    id: "manifest.schema.contract-keys",
+    contract: "Manifest contract keys are represented in target OpenClaw PluginManifestContracts.",
+    target: "manifest-loader",
+  },
+  "manifest-unknown-fields": {
+    id: "manifest.schema.top-level-fields",
+    contract: "Manifest top-level fields are represented in target OpenClaw PluginManifest.",
+    target: "manifest-loader",
+  },
+  "registration-capture-gap": {
+    id: "api.capture.runtime-registrars",
+    contract: "External inspector capture records service, route, gateway, command, and interactive registrations.",
+    target: "inspector-capture-api",
+  },
+  "package-build-artifact-entrypoint": {
+    id: "package.entrypoint.build-before-cold-import",
+    contract: "Inspector can build or resolve source aliases before cold importing package entrypoints.",
+    target: "package-loader",
+  },
+  "package-dependency-install-required": {
+    id: "package.entrypoint.isolated-dependency-install",
+    contract: "Inspector installs package dependencies in an isolated workspace before cold import.",
+    target: "package-loader",
+  },
+  "package-entrypoint-missing": {
+    id: "package.entrypoint.exists",
+    contract: "OpenClaw package entrypoints resolve to files in the published or built plugin package.",
+    target: "package-loader",
+  },
+  "package-openclaw-entry-missing": {
+    id: "package.entrypoint.openclaw-metadata",
+    contract: "OpenClaw package metadata declares entrypoints for cold import and registration capture.",
+    target: "package-loader",
+  },
+  "package-openclaw-metadata-missing": {
+    id: "package.metadata.openclaw",
+    contract: "Plugins that register OpenClaw APIs declare OpenClaw install and entrypoint metadata.",
+    target: "package-loader",
+  },
+  "package-manifest-version-drift": {
+    id: "package.metadata.version-alignment",
+    contract: "Package and OpenClaw manifest versions stay aligned for release compatibility reporting.",
+    target: "package-loader",
+  },
+  "package-plugin-api-compat-missing": {
+    id: "package.compat.plugin-api-range",
+    contract: "Package metadata declares the OpenClaw plugin API range used by the plugin.",
+    target: "package-loader",
+  },
+  "package-typescript-source-entrypoint": {
+    id: "package.entrypoint.typescript-loader",
+    contract: "Inspector can compile or load TypeScript source entrypoints before registration capture.",
+    target: "package-loader",
+  },
+  "runtime-tool-capture": {
+    id: "tool.registration.schema-capture",
+    contract: "Registered runtime tools expose stable names, input schemas, and result metadata.",
+    target: "tool-runtime",
+  },
+};
+
+export function buildContractProbes({ warnings = [], suggestions = [], fixtures = [] }) {
+  const fixtureById = new Map(fixtures.map((fixture) => [fixture.id, fixture]));
+  const probes = [];
+
+  for (const finding of [...warnings, ...suggestions]) {
+    const rule = contractProbeRules[finding.code];
+    if (!rule) {
+      continue;
+    }
+    probes.push({
+      id: `${rule.id}:${finding.fixture}`,
+      fixture: finding.fixture,
+      priority: probePriority(finding.code, fixtureById.get(finding.fixture)?.priority),
+      target: rule.target,
+      contract: rule.contract,
+      evidence: finding.evidence ?? [],
+    });
+  }
+
+  return dedupeBy(probes, (probe) => probe.id).sort(
+    (left, right) => priorityRank(left.priority) - priorityRank(right.priority) || left.id.localeCompare(right.id),
+  );
+}
+
+export function probePriority(code, fixturePriority) {
+  if (
+    [
+      "before-tool-call-probe",
+      "conversation-access-hook",
+      "missing-compat-record",
+      "registration-capture-gap",
+      "sdk-export-missing",
+    ].includes(code)
+  ) {
+    return "P1";
+  }
+  if (fixturePriority === "high") {
+    return "P2";
+  }
+  return "P3";
+}
+
+function dedupeBy(values, keyForValue) {
+  const output = new Map();
+  for (const value of values) {
+    output.set(keyForValue(value), value);
+  }
+  return [...output.values()];
+}
+
+function priorityRank(priority) {
+  return { P0: 0, P1: 1, P2: 2, P3: 3 }[priority] ?? 4;
+}

package/src/execution-results.js

@@ -0,0 +1,297 @@
+import { existsSync } from "node:fs";
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
+import { resolveFromRoot, toRepoPath } from "./path-utils.js";
+
+export const defaultExecutionResultsOptions = {
+  generatedAt: "deterministic",
+  markdownPath: "reports/plugin-execution-results.md",
+  reportTitle: "Plugin Execution Results",
+  resultsDir: ".plugin-inspector/results",
+  jsonPath: "reports/plugin-execution-results.json",
+};
+
+export async function buildExecutionResultsReport(options = {}) {
+  const rootDir = path.resolve(options.rootDir ?? process.cwd());
+  const resultsDir = resolveFromRoot(rootDir, options.resultsDir ?? defaultExecutionResultsOptions.resultsDir);
+  const artifacts = existsSync(resultsDir) ? await readArtifacts(resultsDir, { rootDir }) : [];
+  const syntheticArtifacts = artifacts.filter((artifact) => artifact.kind === "synthetic");
+  const captureArtifacts = artifacts.filter((artifact) => artifact.kind === "capture");
+  const auditArtifacts = artifacts.filter((artifact) => artifact.kind === "audit");
+  const profileArtifacts = artifacts.filter((artifact) => artifact.kind === "profile");
+
+  return {
+    generatedAt: options.generatedAt ?? defaultExecutionResultsOptions.generatedAt,
+    resultsDir: repoRelative(resultsDir, { rootDir }),
+    summary: {
+      artifactCount: artifacts.length,
+      captureArtifactCount: captureArtifacts.length,
+      syntheticArtifactCount: syntheticArtifacts.length,
+      auditArtifactCount: auditArtifacts.length,
+      profileArtifactCount: profileArtifacts.length,
+      capturedRegistrationCount: captureArtifacts.reduce(
+        (sum, artifact) => sum + (artifact.capturedCount ?? 0),
+        0,
+      ),
+      auditFindingCount: auditArtifacts.reduce((sum, artifact) => sum + artifact.findingCount, 0),
+      executionWallMs: profileArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.totalWallMs ?? 0), 0),
+      maxPeakRssMb: Math.max(0, ...profileArtifacts.map((artifact) => artifact.summary?.maxPeakRssMb ?? 0)),
+      maxCpuMsEstimate: Math.max(0, ...profileArtifacts.map((artifact) => artifact.summary?.maxCpuMsEstimate ?? 0)),
+      passCount: syntheticArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.passCount ?? 0), 0),
+      failCount: syntheticArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.failCount ?? 0), 0),
+      blockedCount: syntheticArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.blockedCount ?? 0), 0),
+    },
+    artifacts,
+  };
+}
+
+export async function writeExecutionResultsReport(report, options = {}) {
+  const rootDir = path.resolve(options.rootDir ?? process.cwd());
+  const jsonPath = resolveFromRoot(rootDir, options.jsonPath ?? defaultExecutionResultsOptions.jsonPath);
+  const markdownPath = resolveFromRoot(rootDir, options.markdownPath ?? defaultExecutionResultsOptions.markdownPath);
+  return writeJsonMarkdownArtifacts({
+    jsonPath,
+    markdownPath,
+    json: report,
+    markdown: renderExecutionResultsMarkdown(report, options),
+    check: options.check,
+  });
+}
+
+export function renderExecutionResultsMarkdown(report, options = {}) {
+  const title = options.title ?? options.reportTitle ?? defaultExecutionResultsOptions.reportTitle;
+  return [
+    `# ${title}`,
+    "",
+    `Generated: ${report.generatedAt}`,
+    `Results dir: ${report.resultsDir}`,
+    "",
+    "## Summary",
+    "",
+    markdownTable(
+      [
+        ["Artifacts", report.summary.artifactCount],
+        ["Capture artifacts", report.summary.captureArtifactCount],
+        ["Synthetic artifacts", report.summary.syntheticArtifactCount],
+        ["Audit artifacts", report.summary.auditArtifactCount],
+        ["Profile artifacts", report.summary.profileArtifactCount],
+        ["Captured registrations/hooks", report.summary.capturedRegistrationCount],
+        ["Audit findings", report.summary.auditFindingCount],
+        ["Execution wall", `${report.summary.executionWallMs} ms`],
+        ["Max peak RSS", `${report.summary.maxPeakRssMb} MB`],
+        ["Max CPU estimate", `${report.summary.maxCpuMsEstimate} ms`],
+        ["Pass", report.summary.passCount],
+        ["Fail", report.summary.failCount],
+        ["Blocked", report.summary.blockedCount],
+      ],
+      ["Metric", "Value"],
+    ),
+    "",
+    "## Artifacts",
+    "",
+    markdownTable(
+      report.artifacts.map((artifact) => [
+        artifact.fixture,
+        artifact.kind,
+        artifact.status,
+        artifact.entrypoint,
+        summarizeArtifactResult(artifact),
+        artifact.artifactPath,
+      ]),
+      ["Fixture", "Kind", "Status", "Entrypoint", "Result", "Artifact"],
+    ),
+    "",
+    "## Blocked Synthetic Probes",
+    "",
+    markdownTable(
+      report.artifacts.flatMap((artifact) =>
+        (artifact.blocked ?? []).map((item) => [
+          artifact.fixture,
+          item.kind,
+          item.seam,
+          item.label,
+          item.reason,
+          artifact.artifactPath,
+        ]),
+      ),
+      ["Fixture", "Kind", "Seam", "Label", "Reason", "Artifact"],
+    ),
+    "",
+    "## Failed Synthetic Probes",
+    "",
+    markdownTable(
+      report.artifacts.flatMap((artifact) =>
+        (artifact.failures ?? []).map((item) => [
+          artifact.fixture,
+          item.kind,
+          item.seam,
+          item.label,
+          item.error,
+          artifact.artifactPath,
+        ]),
+      ),
+      ["Fixture", "Kind", "Seam", "Label", "Error", "Artifact"],
+    ),
+    "",
+    "## Dependency Audit Artifacts",
+    "",
+    markdownTable(
+      report.artifacts
+        .filter((artifact) => artifact.kind === "audit")
+        .map((artifact) => [
+          artifact.fixture,
+          artifact.findingCount,
+          artifact.vulnerabilities ? JSON.stringify(artifact.vulnerabilities) : "-",
+          artifact.artifactPath,
+        ]),
+      ["Fixture", "Findings", "Vulnerabilities", "Artifact"],
+    ),
+    "",
+    "## Execution Profiles",
+    "",
+    markdownTable(
+      report.artifacts.flatMap((artifact) =>
+        (artifact.slowestSteps ?? []).map((step) => [
+          artifact.fixture,
+          step.kind,
+          `${step.wallMs} ms`,
+          `${step.peakRssMb} MB`,
+          `${step.cpuMsEstimate} ms`,
+          step.command,
+        ]),
+      ),
+      ["Fixture", "Step", "Wall", "Peak RSS", "CPU Estimate", "Command"],
+    ),
+  ].join("\n");
+}
+
+async function readArtifacts(resultsDir, options) {
+  const paths = await listJsonFiles(resultsDir);
+  const artifacts = [];
+  for (const artifactPath of paths) {
+    const parsed = JSON.parse(await readFile(artifactPath, "utf8"));
+    const relativePath = repoRelative(artifactPath, options);
+    artifacts.push(summarizeArtifact({ artifactPath: relativePath, parsed, rootDir: options.rootDir }));
+  }
+  return artifacts.sort((left, right) => left.artifactPath.localeCompare(right.artifactPath));
+}
+
+async function listJsonFiles(dir) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const files = [];
+  for (const entry of entries) {
+    const entryPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...(await listJsonFiles(entryPath)));
+      continue;
+    }
+    if (entry.isFile() && entry.name.endsWith(".json")) {
+      files.push(entryPath);
+    }
+  }
+  return files;
+}
+
+function summarizeArtifact({ artifactPath, parsed, rootDir }) {
+  const normalizedArtifactPath = toRepoPath(artifactPath);
+  const kind = normalizedArtifactPath.endsWith(".synthetic.json")
+    ? "synthetic"
+    : normalizedArtifactPath.endsWith("package-audit.json")
+      ? "audit"
+      : normalizedArtifactPath.endsWith("execution-profile.json")
+        ? "profile"
+        : "capture";
+  const fixture = normalizedArtifactPath.split("/").at(-2) ?? "unknown";
+  if (kind === "synthetic") {
+    return {
+      artifactPath: normalizedArtifactPath,
+      fixture,
+      kind,
+      entrypoint: scrubPath(parsed.entrypoint, { rootDir }),
+      status: parsed.status,
+      summary: parsed.summary,
+      failures: (parsed.results ?? []).filter((result) => result.status === "fail"),
+      blocked: (parsed.results ?? []).filter((result) => result.status === "blocked"),
+    };
+  }
+  if (kind === "audit") {
+    return {
+      artifactPath: normalizedArtifactPath,
+      fixture,
+      kind,
+      entrypoint: "-",
+      status: "warning",
+      findingCount: auditFindingCount(parsed),
+      vulnerabilities: parsed.metadata?.vulnerabilities ?? null,
+    };
+  }
+  if (kind === "profile") {
+    return {
+      artifactPath: normalizedArtifactPath,
+      fixture,
+      kind,
+      entrypoint: "-",
+      status: parsed.summary?.failCount > 0 ? "fail" : "pass",
+      summary: parsed.summary,
+      slowestSteps: [...(parsed.steps ?? [])].sort((left, right) => right.wallMs - left.wallMs).slice(0, 5),
+    };
+  }
+  return {
+    artifactPath: normalizedArtifactPath,
+    fixture,
+    kind,
+    entrypoint: scrubPath(parsed.entrypoint, { rootDir }),
+    status: parsed.status,
+    capturedCount: parsed.captured?.length ?? 0,
+    captured: (parsed.captured ?? []).map((item) => `${item.kind}:${item.name}`),
+  };
+}
+
+function summarizeArtifactResult(artifact) {
+  if (artifact.kind === "audit") {
+    return `${artifact.findingCount} audit findings`;
+  }
+  if (artifact.kind === "profile") {
+    return `${artifact.summary?.stepCount ?? 0} steps / ${artifact.summary?.totalWallMs ?? 0} ms / ${artifact.summary?.maxPeakRssMb ?? 0} MB`;
+  }
+  if (artifact.summary) {
+    return `${artifact.summary.passCount} pass / ${artifact.summary.failCount} fail / ${artifact.summary.blockedCount} blocked`;
+  }
+  return `${artifact.capturedCount} captured`;
+}
+
+function auditFindingCount(parsed) {
+  const vulnerabilities = parsed.metadata?.vulnerabilities;
+  if (vulnerabilities && typeof vulnerabilities === "object") {
+    const severityTotal = Object.entries(vulnerabilities)
+      .filter(([key]) => key !== "total")
+      .reduce((sum, [, value]) => sum + (Number(value) || 0), 0);
+    return severityTotal || Number(vulnerabilities.total) || 0;
+  }
+  if (Array.isArray(parsed.vulnerabilities)) {
+    return parsed.vulnerabilities.length;
+  }
+  if (parsed.vulnerabilities && typeof parsed.vulnerabilities === "object") {
+    return Object.keys(parsed.vulnerabilities).length;
+  }
+  return 0;
+}
+
+function scrubPath(value, options) {
+  return typeof value === "string" ? repoRelative(value, options) : value;
+}
+
+function repoRelative(value, options = {}) {
+  const rootDir = path.resolve(options.rootDir ?? process.cwd());
+  const absolute = path.resolve(value);
+  const relative = path.relative(rootDir, absolute);
+  if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) {
+    return toRepoPath(relative || ".");
+  }
+  return toRepoPath(value);
+}
+
+function markdownTable(rows, headers) {
+  return renderPaddedMarkdownTable(rows, headers);
+}