@kungfu-tech/buildchain 2.3.1-alpha.0 → 2.3.1-alpha.1

package/docs/toolkit-observability.md

@@ -60,6 +60,20 @@ await logger.span("native.compile", {
   await compile();
 });
 
+logger.spanSync("native.archive", {
+  phase: "build",
+  attributes: { tool: "libtool" },
+}, () => {
+  archiveStaticLibraries();
+});
+
+logger.spawnSync("native.build", "make", ["-j20"], {
+  stdio: "inherit",
+}, {
+  phase: "build",
+  attributes: { requestedJobs: 20 },
+});
+
 const report = verifyBuildchainLogEvents({
   path: logger.path,
   minEvents: 3,
@@ -80,6 +94,227 @@ Use the API when a build script has internal stages that are invisible to the
 outer workflow. Keep secret values out of attributes; known sensitive keys are
 redacted, but callers should still avoid logging private material.
 
+CommonJS scripts should import Buildchain's ESM surfaces dynamically:
+
+```js
+const { createBuildchainLogger } = await import("@kungfu-tech/buildchain/logging");
+const { collectRunnerDiagnostics } = await import("@kungfu-tech/buildchain/diagnostics");
+```
+
+## Diagnostics API
+
+The diagnostics surface collects local, non-telemetry build facts that are
+useful when a native build is slow or flaky:
+
+```js
+import {
+  collectBuildchainDiagnostics,
+  collectCacheDiagnostics,
+  collectCompilerCacheDiagnostics,
+  collectRunnerDiagnostics,
+  collectToolDiagnostics,
+  detectRequestedParallelism,
+  startProcessSampler,
+  summarizeDiagnosticsArtifacts,
+  summarizeLifecycleObservability,
+  summarizeProcessSamples,
+  validateAnchoredPackageRelease,
+  writeDiagnosticsArtifact,
+} from "@kungfu-tech/buildchain/diagnostics";
+
+const lifecycleObservability = summarizeLifecycleObservability({
+  logPath: ".buildchain/logs/events.jsonl",
+});
+const buildCommand = "make";
+const buildArgs = ["-j20"];
+const requestedParallelism = detectRequestedParallelism({
+  command: buildCommand,
+  args: buildArgs,
+});
+const processSampler = startProcessSampler({
+  intervalMs: 15000,
+  label: "native-build",
+  command: buildCommand,
+  args: buildArgs,
+});
+// Run the long native build while the sampler is active.
+const processSummary = summarizeProcessSamples({
+  requestedParallelism: requestedParallelism.value,
+  samples: processSampler.stop(),
+});
+const cacheDiagnostics = collectCacheDiagnostics({ cwd: process.cwd() });
+
+writeDiagnosticsArtifact(".buildchain/artifacts/diagnostics.json", {
+  contract: "consumer-build-diagnostics",
+  buildchain: collectBuildchainDiagnostics({ cwd: process.cwd() }),
+  runner: collectRunnerDiagnostics(),
+  tools: collectToolDiagnostics({ cwd: process.cwd() }),
+  cache: cacheDiagnostics,
+  lifecycleObservability,
+  process: processSummary,
+});
+```
+
+`collectCacheDiagnostics()` includes package-manager/workspace context, selected
+cache directory stats, and compiler-cache stats from `ccache --show-stats
+--json` plus `sccache --show-stats --stats-format json` when those tools are
+present. Missing cache tools are recorded as unavailable instead of failing the
+diagnostics artifact. Call `collectCompilerCacheDiagnostics()` directly when a
+consumer script only needs compiler cache data.
+
+Process samples are intentionally summarized before they become long-lived
+artifacts. The summary records requested parallelism, the source of that value
+(`command`, `env:MAKEFLAGS`, `env:CMAKE_BUILD_PARALLEL_LEVEL`, or `explicit`),
+observed active process concurrency, elapsed sample time, total sampled CPU,
+and conservative command categories such as `compiler`, `archive`, `linker`,
+`build-tool`, and `cache`. This lets native projects distinguish "we asked for
+`make -j20`" from "the build graph only kept two active compiler or archive
+children busy during the sampled window" without storing full command lines.
+
+Native repositories can opt into a reusable diagnostics profile in
+`buildchain.toml`:
+
+```toml
+[diagnostics.native]
+enabled = true
+sample_process_tree = true
+compiler_cache = "auto"
+expected_tools = ["ccache", "sccache", "clang", "cl", "cmake", "ninja"]
+artifact_dirs = ["build", "dist", "build/Release"]
+cache_dirs = [".ccache", ".sccache"]
+```
+
+When enabled, diagnostics artifacts include the normalized profile, selected
+tool versions, compiler-cache stats, and configured artifact/cache directory
+stats. The profile is data-driven: Buildchain does not assume a specific
+project such as libnode. The reusable build workflow also exposes
+`sample-process-tree`, which wraps the build lifecycle with the process sampler
+and carries the generated summary into the final diagnostics artifact. Custom
+workflows can call `buildchain sample process-tree` directly when they need a
+different command boundary.
+
+Anchored/manual package projects can also run one higher-level release-shape
+check instead of assembling lower-level config calls:
+
+```js
+const anchoredReport = validateAnchoredPackageRelease({
+  cwd: process.cwd(),
+  requireManifest: true,
+  requirePackageSetOrder: "platforms-first-main-last",
+  requireTrustedPublishing: true,
+});
+
+if (!anchoredReport.ok) {
+  throw new Error("Anchored package release contract failed");
+}
+```
+
+In an actual publish job, make that check source-lock aware:
+
+```js
+const publishReady = validateAnchoredPackageRelease({
+  cwd: process.cwd(),
+  requirePublishGateSourceLock: true,
+});
+```
+
+The source-lock inputs are read from `BUILDCHAIN_PUBLISH_SOURCE_REF`,
+`BUILDCHAIN_PUBLISH_SOURCE_SHA`, and `BUILDCHAIN_PUBLISH_SOURCE_LOCKED`, which
+the reusable build workflow emits after resolving `publish-source-ref`. That
+turns direct channel-branch publication from `alpha/*` or `release/*` into a
+hard failure, while `publish-gate/alpha/<line>/<version>` and
+`publish-gate/release/<line>/<version>` also validate the requested consumer
+version against configured version files and the anchor manifest.
+
+`buildchain lifecycle run` writes this small diagnostics artifact next to the
+platform manifest by default. The per-platform diagnostics upload includes the
+compact `diagnostics.json`, `diagnostics-manifest.json`, lifecycle
+`events.jsonl`, and, when process sampling is enabled, copied
+`process-summary.json` and `process-samples.jsonl` sidecars. The sidecar
+manifest records each uploaded diagnostics file with its relative path, byte
+count, and sha256 hash. It is intended to stay small enough to download without
+fetching large binary packages, and it should not include full environment dumps
+or secret-looking values.
+
+Use `summarizeDiagnosticsArtifacts()` when a matrix build uploads one
+diagnostics artifact per platform. The summary keeps each platform's lifecycle
+stage table, adds a lifecycle total duration, carries the top slow spans, and
+aggregates warning/error counts plus the slowest platforms. Per-platform rows
+also include compact runner facts, checked tool versions/missing tools, package
+manager/cache directory details, compiler cache availability, and a compact
+process sampler view with requested parallelism, observed max active processes,
+the ratio between them, sample count, process categories, and top sampled
+command basenames. That gives release reviewers a small cross-platform timing,
+runner, tool, cache, and concurrency view without downloading full build outputs
+or process sidecars.
+
+The reusable build workflow writes that rollup as `diagnostics-summary.json` and
+uploads it in a separate aggregate diagnostics summary artifact. Consumers can
+read the `build-diagnostics-summary-artifact` output when they need only timing,
+warning/error, runner, cache, and process-sampler context instead of the build
+summary or binary artifacts. Per-platform rows keep the diagnostics `links`
+object, including the binary artifact name, manifest artifact name, diagnostics
+artifact name, platform id, diagnostics sidecar manifest path, manifest path,
+summary path, and process sidecar paths when present. They also keep compact
+runner/tool/cache summaries so reviewers can tell whether a slow row ran on the
+expected runner, missed an expected tool, or lacked useful compiler-cache stats.
+When the downloaded platform diagnostics include a sibling
+`diagnostics-manifest.json`, `summarizeDiagnosticsArtifacts()` carries a compact
+`diagnosticsManifest` section for that platform and verifies the manifest's
+`diagnostics.json` byte count and sha256. Missing or mismatched sidecar manifests
+increment `diagnosticsManifestWarningCount`, so reviewers can distinguish
+diagnostics sidecar drift from lifecycle warnings or build failures.
+The same summary carries `diagnosticsContract` per platform and aggregates
+`diagnosticsContractWarningCount` when a downloaded `diagnostics.json` does not
+match `BUILDCHAIN_DIAGNOSTICS_CONTRACT`.
+
+The diagnostics SDK also exports stable contract constants such as
+`BUILDCHAIN_DIAGNOSTICS_SUMMARY_CONTRACT`,
+`BUILDCHAIN_DIAGNOSTICS_MANIFEST_CONTRACT`,
+`BUILDCHAIN_PROCESS_SAMPLE_REPORT_CONTRACT`,
+`BUILDCHAIN_PROCESS_SAMPLE_SUMMARY_CONTRACT`, and
+`BUILDCHAIN_ANCHORED_PACKAGE_RELEASE_VALIDATION_CONTRACT` from
+`@kungfu-tech/buildchain/diagnostics`; consumers should compare against those
+constants instead of hardcoding contract strings.
+
+The CLI exposes the same aggregation for shell and workflow steps:
+
+```bash
+buildchain diagnostics summary \
+  .buildchain/artifacts/linux-x64/diagnostics.json \
+  .buildchain/artifacts/macos-arm64/diagnostics.json \
+  --output .buildchain/artifacts/diagnostics-summary.json \
+  --json
+```
+
+Omit `--json` when the workflow log should show a compact platform table with
+lifecycle stages, artifact scan/upload time, total time, requested jobs,
+observed active processes, warnings, and errors.
+
+For long native build commands, the CLI can also sample the child process tree
+while preserving the wrapped command's exit code:
+
+```bash
+buildchain sample process-tree \
+  --label native-build \
+  --interval-ms 15000 \
+  --output .buildchain/diagnostics/process-samples.jsonl \
+  --summary-output .buildchain/diagnostics/process-summary.json \
+  -- \
+  make -j20
+```
+
+The JSONL sample file stores timestamped process-tree snapshots with command
+basenames, CPU percentages, elapsed time, and requested parallelism context. The
+summary file records observed concurrency, total sampled CPU, command
+categories, and top command basenames. This is intended for diagnosing
+low-utilization tails such as archive/link phases without logging full command
+lines or environment dumps.
+
+The lifecycle observability summary is stage-wide, not just final-step timing:
+when install and build write to the same Buildchain log, the final platform
+manifest can show both stages and the slowest spans.
+
 ## CLI Logging
 
 ```bash

package/docs/web-surface-deployments.md

@@ -356,7 +356,6 @@ jobs:
   web-surface:
     uses: kungfu-systems/buildchain/.github/workflows/.web-surface.yml@v2
     with:
-      buildchain-ref: v2
       build-command: npm run build
       verify-command: npm run check
       artifact-path: dist
@@ -371,6 +370,41 @@ The reusable workflow maps GitHub events to Buildchain web-surface semantics:
 | `push` to `main` | validate, build, verify, and plan `staging` from the merged `main` SHA |
 | `workflow_dispatch` with `production-approved = true` | plan `production` and enter the configured GitHub Environment gate |
 
+The optional `buildchain-ref` input is empty by default. Empty keeps the
+web-surface run on the stable Buildchain runtime selected by the reusable
+workflow ref, normally `@v2`. A trusted maintainer can expose a
+`workflow_dispatch` input and pass it through for one-off train validation.
+See [`runtime-train-validation.md`](runtime-train-validation.md) for the shared
+train protocol and notification template:
+
+```yaml
+on:
+  workflow_dispatch:
+    inputs:
+      buildchain-ref:
+        description: "Temporary Buildchain runtime ref for trusted manual validation"
+        required: false
+        default: ""
+
+jobs:
+  web-surface:
+    uses: kungfu-systems/buildchain/.github/workflows/.web-surface.yml@v2
+    with:
+      buildchain-ref: ${{ inputs.buildchain-ref || '' }}
+      build-command: pnpm run build
+      verify-command: pnpm run check
+      artifact-path: dist
+```
+
+Only trusted `workflow_dispatch` runs by repository actors with write,
+maintain, or admin permission may use a non-empty runtime override. Train refs
+such as `train/v2/v2.3/site-source-of-truth` are temporary validation refs, not
+stable production dependencies or pending merge targets. They may remain for a
+retention window after release as a fast-use and rollback channel, with old
+trains handled by periodic Buildchain cleanup. The web-surface deployment
+manifest records the resolved runtime SHA as `runtimeId` and the stable
+rollback ref as `rollbackPointer`.
+
 The workflow deliberately plans and emits manifests by default. Live mutation is
 opt-in per channel:
 
@@ -384,7 +418,6 @@ jobs:
   web-surface:
     uses: kungfu-systems/buildchain/.github/workflows/.web-surface.yml@v2
     with:
-      buildchain-ref: v2
       build-command: pnpm run build
       verify-command: pnpm run check
       artifact-path: dist

package/fixtures/libnode-shaped/README.md

@@ -7,7 +7,8 @@ It is intentionally small, but it keeps the important shape:
 
 - `package.json` is the package version-state file;
 - `libnode.release.json` is the explicit upstream anchor manifest;
-- `buildchain.toml` declares `install`, `build`, and `verify` lifecycle stages;
+- `buildchain.toml` declares `install`, `build`, `verify`, and `publish`
+  lifecycle stages;
 - `version.strategy = "anchored"` and `version.next = "manual"` tell
   Buildchain to validate the current anchor instead of deriving the next Node
   anchor automatically;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kungfu-tech/buildchain",
-  "version": "2.3.1-alpha.0",
+  "version": "2.3.1-alpha.1",
   "private": false,
   "description": "Buildchain Release Passport, release governance, CLI toolkit, and site facts.",
   "repository": "https://github.com/kungfu-systems/buildchain",
@@ -13,6 +13,7 @@
   "exports": {
     ".": "./packages/core/index.js",
     "./core": "./packages/core/index.js",
+    "./diagnostics": "./packages/core/diagnostics.js",
     "./logging": "./packages/core/logging.js",
     "./release-passport": "./packages/core/release-passport.js",
     "./site/buildchain-site.json": "./dist/site/buildchain-site.json",

package/packages/core/README.md

@@ -10,6 +10,8 @@ Current shared surfaces:
 - lifecycle stage normalization and execution;
 - config validation for release-package and `web-surface` projects.
 - toolkit observability logging through `@kungfu-tech/buildchain/logging`;
+- toolkit diagnostics and native profile collection through
+  `@kungfu-tech/buildchain/diagnostics`;
 - release passport creation and verification through
   `@kungfu-tech/buildchain/release-passport`.
 
@@ -28,6 +30,26 @@ await logger.span("native.package", { phase: "package" }, packageArtifacts);
 The CLI remains the right surface for GitHub Actions steps, shell scripts, and
 non-JavaScript build tools.
 
+Diagnostics consumers should import the published subpath and compare stable
+contracts through the exported constants instead of hardcoding JSON contract
+names:
+
+```js
+import {
+  BUILDCHAIN_DIAGNOSTICS_CONTRACT,
+  BUILDCHAIN_DIAGNOSTICS_SUMMARY_CONTRACT,
+  collectRunnerDiagnostics,
+  summarizeDiagnosticsArtifacts,
+} from "@kungfu-tech/buildchain/diagnostics";
+```
+
+CommonJS scripts can use dynamic imports for the same package surfaces:
+
+```js
+const { createBuildchainLogger } = await import("@kungfu-tech/buildchain/logging");
+const { collectRunnerDiagnostics } = await import("@kungfu-tech/buildchain/diagnostics");
+```
+
 Web-surface validation stays in core because both local scripts and GitHub
 Actions need the same fail-closed interpretation of project, channel, deploy,
 retention, and staging security declarations.

package/packages/core/buildchain-config.js

@@ -13,6 +13,7 @@ const SUPPORTED_PROJECT_TYPES = new Set(["package", "web-surface"]);
 const SUPPORTED_PUBLISH_MODES = new Set(["publish-final-version", "promote-existing-version"]);
 const SUPPORTED_PUBLISH_AUTH = new Set(["trusted-publishing", "npm-token"]);
 const SUPPORTED_PACKAGE_SET_ORDER = new Set(["as-provided", "platforms-first-main-last"]);
+const SUPPORTED_NATIVE_COMPILER_CACHE = new Set(["auto", "ccache", "sccache", "none"]);
 const WEB_SURFACE_CHANNELS = ["preview", "staging", "production"];
 const SUPPORTED_CHANNEL_VISIBILITY = new Set(["ephemeral", "protected", "public", "internal"]);
 const SUPPORTED_ACCESS_CONTROL = new Set(["none", "managed-network", "edge-basic-auth", "oidc", "app-auth"]);
@@ -125,10 +126,51 @@ export function normalizeBuildchainConfig(config) {
   if (normalized.security !== undefined) {
     normalized.security = normalizeSecuritySection(normalized.security);
   }
+  if (normalized.diagnostics !== undefined) {
+    normalized.diagnostics = normalizeDiagnosticsSection(normalized.diagnostics);
+  }
   validateWebSurfaceConfig(normalized);
   return normalized;
 }
 
+function normalizeDiagnosticsSection(diagnostics) {
+  assertPlainObject(diagnostics, "diagnostics");
+  const normalized = { ...diagnostics };
+  if (diagnostics.native !== undefined) {
+    normalized.native = normalizeNativeDiagnosticsProfile(diagnostics.native);
+  }
+  return normalized;
+}
+
+function normalizeNativeDiagnosticsProfile(native) {
+  assertPlainObject(native, "diagnostics.native");
+  const compilerCache = native.compiler_cache === undefined
+    ? "auto"
+    : assertString(native.compiler_cache, "diagnostics.native.compiler_cache");
+  if (!SUPPORTED_NATIVE_COMPILER_CACHE.has(compilerCache)) {
+    throw new Error("diagnostics.native.compiler_cache must be one of auto, ccache, sccache, or none");
+  }
+  return {
+    enabled: optionalBoolean(native.enabled, false),
+    sampleProcessTree: optionalBoolean(native.sample_process_tree, false),
+    compilerCache,
+    expectedTools: normalizeStringArray(native.expected_tools, "diagnostics.native.expected_tools"),
+    artifactDirs: normalizeStringArray(native.artifact_dirs, "diagnostics.native.artifact_dirs").map(posixPath),
+    cacheDirs: normalizeStringArray(native.cache_dirs, "diagnostics.native.cache_dirs").map(posixPath),
+  };
+}
+
+export function getNativeDiagnosticsProfile(loadedConfig) {
+  return loadedConfig?.config?.diagnostics?.native || {
+    enabled: false,
+    sampleProcessTree: false,
+    compilerCache: "auto",
+    expectedTools: [],
+    artifactDirs: [],
+    cacheDirs: [],
+  };
+}
+
 function normalizePublishSection(publish) {
   assertPlainObject(publish, "publish");
   const mode = publish.mode === undefined