@tahminator/pipeline 1.0.90-beta.gd62102f7 → 1.0.91

package/README.md

@@ -312,24 +312,22 @@ const flux = new FluxClient(gh, {
   isLocalSource: (ref) => ref.kind === "GitRepository" && ref.name === "flux-system",
 });
 
-// renders every Flux Kustomization at baseRef and in the working tree
-const result = await flux.diff({ baseRef: process.env.BASE_SHA! });
-// result.changed: [{ name, namespace, path, status, diff, additions, deletions, notes }]
-// result.skipped: Kustomizations whose source is not this repository
-
-// posts a new PR comment. If it would exceed GitHub's comment size limit, the full
+// renders every Flux Kustomization at baseRef and in the working tree, then posts the
+// changed ones as a new PR comment. If comment would exceed GitHub's comment size limit, the full
 // comment is uploaded as a workflow run artifact and the PR comment links to it instead.
-await flux.commentDiff({
+await flux.diff({
+  baseRef: process.env.BASE_SHA!,
   owner: "tahminator",
   repository: "infra",
   prId: 123,
-  result,
   title: "Rendered manifest diff", // optional
   description: "Secrets are shown SOPS-encrypted.", // optional
 });
 ```
 
-Each Kustomization is rendered as `kustomize build` of `spec.path` with `targetNamespace`, `namePrefix`, `nameSuffix`, `components`, `images`, `patches` and `commonMetadata` applied on top, followed by `postBuild.substitute`. Paths without a `kustomization.yaml` get one generated, as Flux does. Values from `postBuild.substituteFrom` live in the cluster, so those variables are left as-is and listed in the diff's notes. A build failure is shown in the diff instead of being skipped. `HelmRelease` objects are diffed as manifests; charts are not templated.
+Each Kustomization is rendered as `kustomize build` of `spec.path` with `targetNamespace`, `namePrefix`, `nameSuffix`, `components`, `images`, `patches` and `commonMetadata` applied on top, followed by `postBuild.substitute`. Paths without a `kustomization.yaml` get one generated, as Flux does. Values from `postBuild.substituteFrom` live in the cluster, so those variables are left as-is and listed in the diff's notes. A build failure is shown in the comment instead of being skipped. `HelmRelease` objects are diffed as manifests; charts are not templated.
+
+The diff is split per Kubernetes object (matched by API group, kind, namespace and name), and each object is labelled with the file that defines it, taken from kustomize's `originAnnotations`. Generated ConfigMaps and Secrets point at the `kustomization.yaml` whose generator creates them. Files that only patch an object are not recorded, so a change made by a patch is attributed to the object's original file. A renamed object (e.g. a new generator hash suffix) shows as one removal and one addition.
 
 ### `DockerClient`
 

package/dist/flux/client.d.ts

@@ -1,6 +1,6 @@
 import type { GitHubClient } from "../gh";
 import { type FormatFluxDiffOpts } from "./format";
-import { type FluxClientOpts, type FluxDiffResult, type FluxKustomization } from "./types";
+import { type FluxClientOpts, type FluxKustomization } from "./types";
 /**
  * Works with a repository of Flux manifests: finds its Flux `Kustomization`s, renders them the
  * way kustomize-controller would, and reports how a change affects the rendered output.
@@ -16,27 +16,18 @@ export declare class FluxClient {
     findKustomizations(root?: string): Promise<FluxKustomization[]>;
     /**
      * Renders every Flux `Kustomization` at `baseRef` and in the working tree at `root`,
-     * returning the ones whose output differs.
+     * and posts the ones whose output differs as a new PR comment.
      *
      * `baseRef` is fetched from `origin` if it is not available locally (e.g. shallow CI clones).
-     */
-    diff({ baseRef, root, }: {
-        baseRef: string;
-        root?: string;
-    }): Promise<FluxDiffResult>;
-    /**
-     * Posts a {@link diff} result as a new PR comment.
-     *
      * If the comment would exceed GitHub's size limit, the full comment is uploaded as an
      * artifact of the current workflow run and the PR comment links to it instead.
      */
-    commentDiff({ owner, repository, prId, result, ...formatOpts }: {
+    diff({ baseRef, root, owner, repository, prId, ...formatOpts }: {
+        baseRef: string;
+        root?: string;
         owner: string;
         repository: string;
         prId: number;
-        result: FluxDiffResult;
-    } & Omit<FormatFluxDiffOpts, "artifactUrl">): Promise<{
-        message: string;
-        artifactUrl?: string;
-    }>;
+    } & Omit<FormatFluxDiffOpts, "artifactUrl">): Promise<void>;
+    private collectDiff;
 }

package/dist/flux/client.js

@@ -31,11 +31,45 @@ export class FluxClient {
     }
     /**
      * Renders every Flux `Kustomization` at `baseRef` and in the working tree at `root`,
-     * returning the ones whose output differs.
+     * and posts the ones whose output differs as a new PR comment.
      *
      * `baseRef` is fetched from `origin` if it is not available locally (e.g. shallow CI clones).
+     * If the comment would exceed GitHub's size limit, the full comment is uploaded as an
+     * artifact of the current workflow run and the PR comment links to it instead.
      */
-    async diff({ baseRef, root = ".", }) {
+    async diff({ baseRef, root = ".", owner, repository, prId, ...formatOpts }) {
+        const result = await this.collectDiff(baseRef, root);
+        const full = formatFluxDiff(result, formatOpts);
+        if (full.length <= MAX_COMMENT_LENGTH) {
+            await this.ghClient.sendPrMessage({
+                owner,
+                repository,
+                prId,
+                message: full,
+            });
+            return;
+        }
+        const dir = await mkdtemp(path.join(tmpdir(), "flux-diff-artifact-"));
+        try {
+            const file = path.join(dir, "flux-diff.md");
+            await Bun.write(file, full);
+            const attempt = process.env.GITHUB_RUN_ATTEMPT ?? "1";
+            const { url } = await this.ghClient.uploadArtifact({
+                name: `flux-diff-pr-${prId}-attempt-${attempt}`,
+                files: [file],
+                rootDirectory: dir,
+            });
+            const message = formatFluxDiff(result, {
+                ...formatOpts,
+                artifactUrl: url,
+            });
+            await this.ghClient.sendPrMessage({ owner, repository, prId, message });
+        }
+        finally {
+            await rm(dir, { recursive: true, force: true });
+        }
+    }
+    async collectDiff(baseRef, root) {
         const workdir = await mkdtemp(path.join(tmpdir(), "flux-diff-"));
         const baseRoot = path.join(workdir, "base");
         await $ `git -C ${root} fetch --depth=1 origin ${baseRef}`.quiet().nothrow();
@@ -55,6 +89,8 @@ export class FluxClient {
             }
             const changed = [];
             const skipped = [];
+            let nextDiffId = 0;
+            const nextDiffDir = () => path.join(workdir, "diffs", String(nextDiffId++));
             await Promise.all([...pairs.values()].map(async ({ base, head }) => {
                 const current = (head ?? base);
                 const { name, namespace } = current;
@@ -73,13 +109,23 @@ export class FluxClient {
                     base ? renderKustomization(baseRoot, base) : undefined,
                     head ? renderKustomization(root, head) : undefined,
                 ]);
-                const baseOutput = baseRender?.output ?? "";
-                const headOutput = headRender?.output ?? "";
-                if (baseOutput === headOutput) {
+                const buildError = headRender?.error;
+                const objects = buildError ?
+                    []
+                    : await diffObjects(baseRender?.objects ?? [], headRender?.objects ?? [], nextDiffDir);
+                const unchanged = buildError ?
+                    buildError === baseRender?.error
+                    : objects.length === 0;
+                if (unchanged) {
                     return;
                 }
-                const diff = await unifiedDiff(path.join(workdir, "diffs", namespace, name), current.path, baseOutput, headOutput);
-                const diffLines = diff.split("\n");
+                const notes = new Set([
+                    ...(baseRender?.notes ?? []),
+                    ...(headRender?.notes ?? []),
+                ]);
+                if (baseRender?.error && !buildError) {
+                    notes.add("`kustomize build` fails at the base commit, so every object is shown as added.");
+                }
                 changed.push({
                     name,
                     namespace,
@@ -87,15 +133,11 @@ export class FluxClient {
                     status: !base ? "added"
                         : !head ? "removed"
                             : "changed",
-                    diff,
-                    additions: diffLines.filter((line) => line.startsWith("+") && !line.startsWith("+++")).length,
-                    deletions: diffLines.filter((line) => line.startsWith("-") && !line.startsWith("---")).length,
-                    notes: [
-                        ...new Set([
-                            ...(baseRender?.notes ?? []),
-                            ...(headRender?.notes ?? []),
-                        ]),
-                    ],
+                    objects,
+                    ...(buildError ? { buildError } : {}),
+                    additions: objects.reduce((sum, o) => sum + o.additions, 0),
+                    deletions: objects.reduce((sum, o) => sum + o.deletions, 0),
+                    notes: [...notes],
                 });
             }));
             const byLabel = (a, b) => label(a).localeCompare(label(b));
@@ -112,44 +154,54 @@ export class FluxClient {
             await rm(workdir, { recursive: true, force: true });
         }
     }
-    /**
-     * Posts a {@link diff} result as a new PR comment.
-     *
-     * If the comment would exceed GitHub's size limit, the full comment is uploaded as an
-     * artifact of the current workflow run and the PR comment links to it instead.
-     */
-    async commentDiff({ owner, repository, prId, result, ...formatOpts }) {
-        const full = formatFluxDiff(result, formatOpts);
-        if (full.length <= MAX_COMMENT_LENGTH) {
-            await this.ghClient.sendPrMessage({
-                owner,
-                repository,
-                prId,
-                message: full,
-            });
-            return { message: full };
-        }
-        const dir = await mkdtemp(path.join(tmpdir(), "flux-diff-artifact-"));
-        try {
-            const file = path.join(dir, "flux-diff.md");
-            await Bun.write(file, full);
-            const attempt = process.env.GITHUB_RUN_ATTEMPT ?? "1";
-            const { url } = await this.ghClient.uploadArtifact({
-                name: `flux-diff-pr-${prId}-attempt-${attempt}`,
-                files: [file],
-                rootDirectory: dir,
-            });
-            const message = formatFluxDiff(result, {
-                ...formatOpts,
-                artifactUrl: url,
-            });
-            await this.ghClient.sendPrMessage({ owner, repository, prId, message });
-            return { message, artifactUrl: url };
-        }
-        finally {
-            await rm(dir, { recursive: true, force: true });
-        }
+}
+/**
+ * Pairs base and head objects by API group, kind, namespace and name, and diffs the ones whose
+ * rendered YAML differs. An apiVersion bump within a group is a change, not a remove + add.
+ */
+async function diffObjects(baseObjects, headObjects, nextDir) {
+    const key = (object) => {
+        const group = object.apiVersion.includes("/") ? object.apiVersion.split("/")[0] : "";
+        return [group, object.kind, object.namespace, object.name].join("\0");
+    };
+    const pairs = new Map();
+    for (const object of baseObjects) {
+        pairs.set(key(object), { base: object });
     }
+    for (const object of headObjects) {
+        const k = key(object);
+        pairs.set(k, { ...pairs.get(k), head: object });
+    }
+    const diffs = await Promise.all([...pairs.values()].map(async ({ base, head }) => {
+        if (base?.yaml === head?.yaml) {
+            return null;
+        }
+        const current = (head ?? base);
+        const { kind, namespace, name, file } = current;
+        const diff = await unifiedDiff(nextDir(), [kind, namespace, name].filter(Boolean).join("/"), base?.yaml ?? "", head?.yaml ?? "");
+        const lines = diff.split("\n");
+        const objectDiff = {
+            kind,
+            namespace,
+            name,
+            ...(file ? { file } : {}),
+            status: !base ? "added"
+                : !head ? "removed"
+                    : "changed",
+            diff,
+            additions: lines.filter((line) => line.startsWith("+") && !line.startsWith("+++")).length,
+            deletions: lines.filter((line) => line.startsWith("-") && !line.startsWith("---")).length,
+        };
+        return objectDiff;
+    }));
+    return diffs
+        .filter((diff) => diff !== null)
+        .sort((a, b) => 
+    // objects without a recorded file sort last
+    (a.file ?? "\uffff").localeCompare(b.file ?? "\uffff") ||
+        a.kind.localeCompare(b.kind) ||
+        a.namespace.localeCompare(b.namespace) ||
+        a.name.localeCompare(b.name));
 }
 async function unifiedDiff(dir, targetPath, baseOutput, headOutput) {
     await mkdir(dir, { recursive: true });

package/dist/flux/format.js

@@ -9,11 +9,26 @@ export function formatFluxDiff(result, { title = "Flux diff", description, artif
         `No Flux Kustomization renders differently against ${base}.`
         : `${result.changed.length} Flux Kustomization(s) render differently against ${base}.`, "");
     if (artifactUrl) {
-        lines.push(`> [!WARNING]`, `> The rendered diff is too long for a GitHub comment. [Download the full diff from the workflow run](${artifactUrl}).`, "", ...result.changed.map((change) => `- ${summary(change)}`), "");
+        lines.push(`> [!WARNING]`, `> The rendered diff is too long for a GitHub comment. [Download the full diff from the workflow run](${artifactUrl}).`, "");
+        for (const change of result.changed) {
+            lines.push(`- ${summary(change)}`);
+            if (change.buildError) {
+                lines.push("  - `kustomize build` failed");
+            }
+            lines.push(...change.objects.map((object) => `  - ${objectSummary(object)}`));
+        }
+        lines.push("");
     }
     else {
         for (const change of result.changed) {
-            lines.push(`<details><summary>${summary(change)}</summary>`, "", ...change.notes.map((note) => `> ${note}\n`), "````diff", change.diff.trimEnd(), "````", "", "</details>", "");
+            lines.push(`<details><summary>${summary(change)}</summary>`, "", ...change.notes.map((note) => `> ${note}\n`));
+            if (change.buildError) {
+                lines.push("`kustomize build` failed:", "", "````text", change.buildError, "````", "");
+            }
+            for (const object of change.objects) {
+                lines.push(objectSummary(object), "", "````diff", object.diff.trimEnd(), "````", "");
+            }
+            lines.push("</details>", "");
         }
     }
     if (result.skipped.length > 0) {
@@ -25,5 +40,11 @@ export function label({ name, namespace, }) {
     return namespace ? `${namespace}/${name}` : name;
 }
 function summary(change) {
-    return `<code>${label(change)}</code> — <code>${change.path}</code> (${change.status}, +${change.additions} −${change.deletions})`;
+    const counts = change.buildError ? "build failed" : (`${change.objects.length} object(s), +${change.additions} −${change.deletions}`);
+    return `<code>${label(change)}</code> — <code>${change.path}</code> (${change.status}, ${counts})`;
+}
+function objectSummary(object) {
+    const name = [object.namespace, object.name].filter(Boolean).join("/");
+    const file = object.file ? ` — \`${object.file}\`` : "";
+    return `**\`${object.kind}\` \`${name}\`**${file} (${object.status}, +${object.additions} −${object.deletions})`;
 }

package/dist/flux/index.d.ts

@@ -1,4 +1,2 @@
 export * from "./client";
-export { formatFluxDiff, type FormatFluxDiffOpts } from "./format";
-export { substituteVariables } from "./render";
-export * from "./types";
+export type { FluxClientOpts, FluxKustomization, FluxSourceRef } from "./types";

package/dist/flux/index.js

@@ -1,4 +1 @@
 export * from "./client";
-export { formatFluxDiff } from "./format";
-export { substituteVariables } from "./render";
-export * from "./types";

package/dist/flux/render.d.ts

@@ -1,6 +1,22 @@
 import type { FluxKustomization } from "./types";
-export type RenderedKustomization = {
-    output: string;
+export type RenderedObject = {
+    apiVersion: string;
+    kind: string;
+    namespace: string;
+    name: string;
+    /**
+     * repository-relative file that defines the object (for generated objects, the
+     * kustomization.yaml whose generator creates it); `undefined` if kustomize did not record one
+     */
+    file?: string;
+    /** the object's rendered YAML, without the origin annotation */
+    yaml: string;
+};
+type RenderedKustomization = {
+    /** rendered objects, in kustomize's output order; empty when `error` is set */
+    objects: RenderedObject[];
+    /** `kustomize build` error output, when the build failed */
+    error?: string;
     notes: string[];
 };
 /**
@@ -9,7 +25,7 @@ export type RenderedKustomization = {
  * `nameSuffix`, `components`, `images`, `patches`, `commonMetadata`) applied on top,
  * followed by `postBuild` variable substitution.
  *
- * A failed build is returned as commented-out output so it shows up in a diff.
+ * Each rendered object records the file that defines it.
  */
 export declare function renderKustomization(root: string, kustomization: FluxKustomization): Promise<RenderedKustomization>;
 /**
@@ -25,3 +41,4 @@ export declare function substituteVariables(input: string, vars: Record<string,
     output: string;
     unresolved: string[];
 };
+export {};

package/dist/flux/render.js

@@ -10,13 +10,15 @@ const KUSTOMIZATION_FILES = [
 const SUBSTITUTE_DISABLED = /kustomize\.toolkit\.fluxcd\.io\/substitute:\s*["']?disabled["']?/;
 /** `${VAR}`, `${VAR:-default}`, `${VAR-default}`, `${VAR:=default}`, `${VAR=default}`; `$${VAR}` escapes. */
 const VARIABLE = /\$(\$)?\{([A-Za-z_][A-Za-z0-9_]*)(?:(:?[-=])([^}]*))?\}/g;
+/** kustomize's `originAnnotations` build metadata; records the file that defines each object. */
+const ORIGIN_ANNOTATION = "config.kubernetes.io/origin";
 /**
  * Renders a Flux `Kustomization` the way kustomize-controller does: `kustomize build` of
  * `spec.path` with the overlay fields from the spec (`targetNamespace`, `namePrefix`,
  * `nameSuffix`, `components`, `images`, `patches`, `commonMetadata`) applied on top,
  * followed by `postBuild` variable substitution.
  *
- * A failed build is returned as commented-out output so it shows up in a diff.
+ * Each rendered object records the file that defines it.
  */
 export async function renderKustomization(root, kustomization) {
     const notes = [];
@@ -45,19 +47,14 @@ export async function renderKustomization(root, kustomization) {
                 [{ pairs: spec.commonMetadata.labels, includeSelectors: false }]
                 : undefined,
             commonAnnotations: spec.commonMetadata?.annotations,
+            buildMetadata: ["originAnnotations"],
         };
         await Bun.write(path.join(wrapper, "kustomization.yaml"), yaml.stringify(wrapperKustomization));
         const result = await $ `kustomize build --load-restrictor LoadRestrictionsNone ${wrapper}`
             .quiet()
             .nothrow();
         if (result.exitCode !== 0) {
-            const stderr = result.stderr
-                .toString()
-                .trim()
-                .split("\n")
-                .map((line) => `# ${line}`)
-                .join("\n");
-            return { output: `# kustomize build failed\n${stderr}\n`, notes };
+            return { objects: [], error: result.stderr.toString().trim(), notes };
         }
         let output = result.stdout.toString();
         if (spec.postBuild) {
@@ -70,12 +67,56 @@ export async function renderKustomization(root, kustomization) {
                     .join(", ")}.`);
             }
         }
-        return { output, notes };
+        const docs = yaml.parseAllDocuments(output);
+        const invalid = docs.find((doc) => doc.errors.length > 0);
+        if (invalid) {
+            return {
+                objects: [],
+                error: `rendered output is not valid YAML: ${invalid.errors[0]?.message}`,
+                notes,
+            };
+        }
+        const objects = docs
+            .filter((doc) => doc.contents !== null)
+            .map((doc) => toRenderedObject(doc, root, wrapper));
+        return { objects, notes };
     }
     finally {
         await rm(wrapper, { recursive: true, force: true });
     }
 }
+function toRenderedObject(doc, root, wrapper) {
+    const origin = doc.getIn(["metadata", "annotations", ORIGIN_ANNOTATION]);
+    doc.deleteIn(["metadata", "annotations", ORIGIN_ANNOTATION]);
+    const annotations = doc.getIn(["metadata", "annotations"]);
+    if (yaml.isMap(annotations) && annotations.items.length === 0) {
+        doc.deleteIn(["metadata", "annotations"]);
+    }
+    const value = (doc.toJS() ?? {});
+    return {
+        apiVersion: String(value.apiVersion ?? ""),
+        kind: String(value.kind ?? ""),
+        namespace: String(value.metadata?.namespace ?? ""),
+        name: String(value.metadata?.name ?? ""),
+        file: typeof origin === "string" ?
+            originFile(origin, root, wrapper)
+            : undefined,
+        yaml: doc.toString({ lineWidth: 0 }).replace(/^---\n/, ""),
+    };
+}
+/**
+ * Converts an origin annotation (`path`, or `configuredIn` for generated objects, relative to
+ * the wrapper kustomization) to a repository-relative path. Remote origins are returned as
+ * `<repo>//<path>?ref=<ref>`.
+ */
+function originFile(origin, root, wrapper) {
+    const parsed = yaml.parse(origin);
+    const file = parsed?.path ?? parsed?.configuredIn;
+    if (parsed?.repo) {
+        return `${parsed.repo}//${file ?? ""}${parsed.ref ? `?ref=${parsed.ref}` : ""}`;
+    }
+    return file ? path.relative(root, path.resolve(wrapper, file)) : undefined;
+}
 /**
  * Applies Flux `postBuild` substitution to rendered multi-document YAML. Documents annotated
  * with `kustomize.toolkit.fluxcd.io/substitute: disabled` are left untouched.

package/dist/flux/types.d.ts

@@ -40,7 +40,7 @@ export declare const fluxKustomizationSchema: z.ZodObject<{
         }, z.core.$strip>>;
     }, z.core.$strip>;
 }, z.core.$strip>;
-export type FluxKustomizationSpec = z.infer<typeof fluxKustomizationSchema>["spec"];
+type FluxKustomizationSpec = z.infer<typeof fluxKustomizationSchema>["spec"];
 /** A Flux `Kustomization` found in the repository. */
 export type FluxKustomization = {
     /** repository-relative file the Kustomization is declared in */
@@ -53,13 +53,35 @@ export type FluxKustomization = {
     spec: FluxKustomizationSpec;
 };
 export type FluxSourceRef = FluxKustomizationSpec["sourceRef"];
+type FluxDiffStatus = "added" | "changed" | "removed";
+/** A rendered Kubernetes object whose output differs between base and head. */
+export type FluxObjectDiff = {
+    kind: string;
+    /** `""` for cluster-scoped objects */
+    namespace: string;
+    name: string;
+    /**
+     * repository-relative file that defines the object, from kustomize's origin annotations.
+     * Generated objects (ConfigMap/Secret generators) point at the kustomization.yaml that
+     * generates them. Files that only patch the object are not recorded.
+     */
+    file?: string;
+    status: FluxDiffStatus;
+    /** unified diff of the object's rendered YAML */
+    diff: string;
+    additions: number;
+    deletions: number;
+};
 export type FluxKustomizationDiff = {
     name: string;
     namespace: string;
     path: string;
-    status: "added" | "changed" | "removed";
-    /** unified diff of the rendered output */
-    diff: string;
+    status: FluxDiffStatus;
+    /** changed objects, sorted by file then kind, namespace and name */
+    objects: FluxObjectDiff[];
+    /** `kustomize build` error at head; `objects` is empty when set */
+    buildError?: string;
+    /** totals across `objects` */
     additions: number;
     deletions: number;
     /** caveats about how faithful the render is, e.g. unresolvable `substituteFrom` variables */
@@ -92,3 +114,4 @@ export type FluxClientOpts = {
      */
     isLocalSource?: (sourceRef: FluxSourceRef) => boolean;
 };
+export {};

package/package.json

@@ -3,7 +3,7 @@
   "type": "module",
   "author": "Tahmid Ahmed",
   "description": "A collection of Bun shell scripts that can be re-used in various CICD pipelines.",
-  "version": "1.0.90-beta.gd62102f7",
+  "version": "1.0.91",
   "repository": {
     "url": "git+https://github.com/tahminator/pipeline.git"
   },