@intentius/chant-lexicon-grafana 0.97.0 → 0.99.0

package/src/observe.e2e.test.ts

@@ -0,0 +1,255 @@
+/**
+ * "A dashboard edited in Grafana's UI shows as drift in `chant lifecycle
+ * diff --live`" (#2946, #2960), against Grafana itself, 12.4.11 and 13.2.2.
+ *
+ * Per image: a small project (a datasource, a dashboard with a variable and
+ * a row of panels, a provider that allows UI saves) is built, its
+ * provisioning files and dashboards are mounted into a fresh container, and
+ * this checkout's `chant lifecycle diff <env> --live --json` is run in the
+ * project:
+ *
+ * 1. Untouched: every declared entity present, the dashboard `owned` (its
+ *    provider is chant's), no drift and nothing unclaimed.
+ * 2. Then the dashboard is saved through `POST /api/dashboards/db`, the call
+ *    the dashboard editor's Save makes, with one query and one panel title
+ *    changed. The same command reports exactly those two paths as drift.
+ * 3. Live export of the environment generates TypeScript that carries the
+ *    edit, and the generated project builds.
+ * 4. On 13.x only (#2947): the dashboard is recreated through the v2 API,
+ *    so Grafana stores it as v2 and a classic read of it is a lossy
+ *    down-conversion. (A v2 PUT over a dashboard stored as v0 keeps it at
+ *    v0, converting v2-only layout away on write, so the test deletes it and
+ *    creates it at v2.) The diff reads it at v2 instead. With a panel renamed
+ *    the drift is exactly the classic-edit paths, so the v2 read adds no
+ *    noise; with its row moved into a tab, the tab is a row the declaration
+ *    does not have. Export names the tab.
+ *
+ * Containers come from the shared helpers in ../test/e2e/containers.ts
+ * (unique names, random host ports, removed in afterAll even on failure;
+ * `CHANT_GRAFANA_IMAGES` overrides the image pair). Skipped, with the reason
+ * in the test name, when Docker is not running. Run through the slot lock:
+ * `~/checkouts/intentius/chant-worktrees/.docker-slot.sh <label> -- npx vitest run lexicons/grafana/src/observe.e2e.test.ts`
+ */
+
+import { execSync, spawn } from "node:child_process";
+import { mkdirSync, symlinkSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { build } from "@intentius/chant/build";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { grafanaSerializer } from "./serializer";
+import { exportResources } from "./export-resources";
+import { GrafanaGenerator } from "./import/generator";
+import { DockerScope, GRAFANA_IMAGES, dockerAvailable, waitFor, type GrafanaContainer } from "../test/e2e/containers";
+
+const hasDocker = dockerAvailable();
+const skipReason = hasDocker ? "" : " (skipped: Docker is not running)";
+const repoRoot = resolve(import.meta.dirname, "..", "..", "..");
+
+const PROJECT: Record<string, string> = {
+  "src/datasources.ts": `import { Datasource } from "@intentius/chant-lexicon-grafana";
+
+export const prometheus = new Datasource({ name: "Prometheus", type: "prometheus", url: "http://prometheus:9090", isDefault: true });
+`,
+  "src/panels.ts": `import { PromQuery, Row, StatPanel, TimeSeriesPanel, CustomVariable } from "@intentius/chant-lexicon-grafana";
+import { prometheus } from "./datasources";
+
+export const job = new CustomVariable({ name: "job", values: ["api", "worker"] });
+export const rate = new PromQuery({ expr: 'sum(rate(http_requests_total{job="$job"}[5m]))', datasource: prometheus });
+export const errors = new PromQuery({ expr: 'sum(rate(http_requests_total{job="$job", code=~"5.."}[5m]))', datasource: prometheus });
+const rateUnit = { defaults: { unit: "reqps" } };
+export const ratePanel = new TimeSeriesPanel({ title: "Request rate", targets: [rate], fieldConfig: rateUnit });
+export const errorPanel = new StatPanel({ title: "Errors", targets: [errors] });
+export const traffic = new Row({ title: "Traffic", panels: [ratePanel, errorPanel] });
+`,
+  "src/dashboard.ts": `import { Dashboard, DashboardProvider } from "@intentius/chant-lexicon-grafana";
+import { job, traffic } from "./panels";
+
+const lastHour = { from: "now-1h", to: "now" };
+export const apiOverview = new Dashboard({ title: "API overview", tags: ["api"], time: lastHour, variables: [job], panels: [traffic] });
+export const provider = new DashboardProvider({ name: "chant", allowUiUpdates: true, updateIntervalSeconds: 3600 });
+`,
+};
+
+/** This checkout's chant, run in `cwd`. */
+function chant(cwd: string, env: Record<string, string>, ...args: string[]): Promise<{ status: number | null; stdout: string; stderr: string }> {
+  return new Promise((done) => {
+    const child = spawn(
+      process.execPath,
+      ["--import", pathToFileURL(join(repoRoot, "node_modules/tsx/dist/loader.mjs")).href, join(repoRoot, "packages/core/src/cli/main.ts"), ...args],
+      { cwd, env: { ...process.env, NO_COLOR: "1", ...env } },
+    );
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (d) => (stdout += d));
+    child.stderr.on("data", (d) => (stderr += d));
+    const timer = setTimeout(() => child.kill(), 240_000);
+    child.on("close", (status) => {
+      clearTimeout(timer);
+      done({ status, stdout, stderr });
+    });
+  });
+}
+
+interface LiveDiffJson {
+  lexicons: {
+    grafana: {
+      resources: { missing: string[]; unobserved: Array<{ name: string; reason: string }> };
+      observed: Record<string, { ownership?: string; status: string }>;
+      deep: {
+        drifted: Array<{ name: string; changes: Array<{ path: string; kind: string; declared?: unknown; live?: unknown }> }>;
+        unclaimed: Array<{ name: string; fields: Array<{ path: string }> }>;
+        unchanged: string[];
+      };
+    };
+  };
+}
+
+describe.skipIf(!hasDocker).each(GRAFANA_IMAGES)(`a dashboard edited in Grafana shows as drift in chant lifecycle diff --live: %s${skipReason}`, (image) => {
+  const scope = new DockerScope("grafana-observe");
+  let dir = "";
+  let project = "";
+  let grafana: GrafanaContainer;
+  const env = { E2E_GRAFANA_USER: "admin", E2E_GRAFANA_PASSWORD: "admin", GRAFANA_URL: "", GRAFANA_TOKEN: "" };
+  const profile = () => ({ url: grafana.base, basicAuth: { user: { env: "E2E_GRAFANA_USER" }, password: { env: "E2E_GRAFANA_PASSWORD" } } });
+
+  const diff = async (): Promise<LiveDiffJson> => {
+    const r = await chant(project, env, "lifecycle", "diff", "e2e", "--live", "--json");
+    const line = r.stdout.trim().split("\n").filter((l) => l.startsWith("{")).pop();
+    if (!line) throw new Error(`no JSON from chant lifecycle diff (exit ${r.status}):\n${r.stdout}\n${r.stderr}`);
+    return JSON.parse(line) as LiveDiffJson;
+  };
+
+  beforeAll(async () => {
+    dir = scope.tempDir();
+    project = join(dir, "project");
+    const mounted = join(dir, "mounted");
+    for (const [file, content] of Object.entries(PROJECT)) {
+      mkdirSync(dirname(join(project, file)), { recursive: true });
+      writeFileSync(join(project, file), content);
+    }
+    writeFileSync(join(project, "package.json"), JSON.stringify({ name: "grafana-observe-e2e", private: true, type: "module" }));
+    symlinkSync(join(repoRoot, "node_modules"), join(project, "node_modules"), "dir");
+    execSync("git init -q && git -c user.email=e2e@example.com -c user.name=e2e commit -q --allow-empty -m init", { cwd: project });
+
+    const result = await build(join(project, "src"), [grafanaSerializer]);
+    expect(result.errors).toEqual([]);
+    const out = result.outputs.get("grafana") as SerializerResult;
+    for (const [file, content] of Object.entries(out.files ?? {})) {
+      mkdirSync(dirname(join(mounted, file)), { recursive: true });
+      writeFileSync(join(mounted, file), content);
+    }
+
+    grafana = await scope.grafana({ image, provisioningDir: join(mounted, "provisioning"), dashboardsDir: join(mounted, "dashboards") });
+    writeFileSync(join(project, "chant.config.ts"), `export default { lexicons: ["grafana"], grafana: { profiles: { e2e: ${JSON.stringify(profile())} } } };\n`);
+
+    await waitFor("the dashboard to be provisioned", async () => ((await grafana.api("/api/dashboards/uid/api-overview")).status === 200 ? true : undefined));
+    await waitFor("the datasource to be provisioned", async () => ((await grafana.api("/api/datasources/uid/prometheus")).status === 200 ? true : undefined));
+  }, 300_000);
+
+  afterAll(() => scope.cleanup());
+
+  it("an untouched dashboard: present, chant's, no drift, nothing unclaimed", { timeout: 300_000 }, async () => {
+    const report = (await diff()).lexicons.grafana;
+    expect(report.resources.missing).toEqual([]);
+    expect(report.observed.apiOverview).toMatchObject({ status: "PRESENT", ownership: "owned" });
+    expect(report.observed.prometheus).toMatchObject({ status: "PRESENT" });
+    // Only the provider is not observed: Grafana serves no API for it.
+    expect(report.resources.unobserved.map((u) => `${u.name}:${u.reason}`)).toEqual(["provider:unsupported-kind"]);
+    expect(report.deep.drifted).toEqual([]);
+    expect(report.deep.unclaimed).toEqual([]);
+    expect(report.deep.unchanged).toEqual(expect.arrayContaining(["apiOverview", "prometheus"]));
+  });
+
+  it("the same dashboard saved from the editor with a changed query and title: exactly those paths drift", { timeout: 300_000 }, async () => {
+    const { body } = await grafana.api("/api/dashboards/uid/api-overview");
+    const dashboard = body.dashboard as { panels: Array<{ title: string; targets?: Array<{ expr?: string }> }> };
+    const rate = dashboard.panels.find((p) => p.title === "Request rate")!;
+    rate.targets![0].expr = "sum(rate(http_requests_total[1m]))";
+    dashboard.panels.find((p) => p.title === "Errors")!.title = "5xx";
+    const saved = await grafana.api("/api/dashboards/db", { method: "POST", body: JSON.stringify({ dashboard, overwrite: true, message: "edited in the UI" }) });
+    expect(saved.status).toBe(200);
+
+    const report = (await diff()).lexicons.grafana;
+    const changes = report.deep.drifted.find((d) => d.name === "apiOverview")?.changes ?? [];
+    expect(changes.map((c) => ({ path: c.path, kind: c.kind, live: c.live })).sort((a, b) => a.path.localeCompare(b.path))).toEqual([
+      { path: "panels[0].panels[0].targets[0].expr", kind: "changed", live: "sum(rate(http_requests_total[1m]))" },
+      { path: "panels[0].panels[1].title", kind: "changed", live: "5xx" },
+    ]);
+    expect(report.deep.drifted.map((d) => d.name)).toEqual(["apiOverview"]);
+  });
+
+  it("live export generates TypeScript that carries the edit and builds", { timeout: 240_000 }, async () => {
+    const ir = await exportResources({
+      environment: "e2e",
+      config: { grafana: { profiles: { e2e: profile() } } },
+      env,
+    });
+    const files = new GrafanaGenerator().generate(ir);
+    const source = files.map((f) => f.content).join("\n");
+    expect(source).toContain("sum(rate(http_requests_total[1m]))");
+    expect(source).toContain('"5xx"');
+
+    // The generated project builds: the exported Datasource and the dashboard's reference to it do not collide.
+    const exported = join(dir, "exported");
+    for (const f of files) {
+      mkdirSync(dirname(join(exported, "src", f.path)), { recursive: true });
+      writeFileSync(join(exported, "src", f.path), f.content);
+    }
+    writeFileSync(join(exported, "package.json"), JSON.stringify({ name: "grafana-export-e2e", private: true, type: "module" }));
+    writeFileSync(join(exported, "chant.config.ts"), 'export default { lexicons: ["grafana"] };\n');
+    symlinkSync(join(repoRoot, "node_modules"), join(exported, "node_modules"), "dir");
+    const built = await chant(exported, {}, "build", "src", "--lexicon", "grafana", "-o", join(exported, "dist", "index.json"));
+    expect(built.status, `${built.stdout}\n${built.stderr}`).toBe(0);
+  });
+
+  it.runIf(/:13\./.test(image))("stored as v2: read at v2, a rename is drift at the classic path, and a tab is a row the declaration lacks", { timeout: 300_000 }, async () => {
+    const path = "/apis/dashboard.grafana.app/v2/namespaces/default/dashboards/api-overview";
+    type V2Spec = { elements: Record<string, { spec: { title: string } }>; layout: unknown };
+    /** Delete the dashboard and create it again at v2 with `edit` applied, so Grafana stores it as v2. */
+    const recreateAtV2 = async (edit: (spec: V2Spec) => void) => {
+      const { status, body: resource } = await grafana.api(path);
+      expect(status).toBe(200);
+      edit(resource.spec as V2Spec);
+      const deleted = await grafana.api(path, { method: "DELETE" });
+      expect(deleted.status, JSON.stringify(deleted.body)).toBe(200);
+      const { name, labels, annotations } = resource.metadata;
+      const created = await grafana.api("/apis/dashboard.grafana.app/v2/namespaces/default/dashboards", {
+        method: "POST",
+        body: JSON.stringify({ apiVersion: resource.apiVersion, kind: "Dashboard", metadata: { name, labels, annotations }, spec: resource.spec }),
+      });
+      expect(created.status, JSON.stringify(created.body)).toBeLessThan(300);
+      // Grafana stores it as v2 now: its v1 read says so, and is the lossy copy chant must not read.
+      const v1 = await grafana.api("/apis/dashboard.grafana.app/v1/namespaces/default/dashboards/api-overview");
+      expect(v1.body.status?.conversion?.storedVersion).toBe("v2");
+    };
+    const driftOf = async () => {
+      const report = (await diff()).lexicons.grafana;
+      expect(report.resources.unobserved.map((u) => `${u.name}:${u.reason}`)).toEqual(["provider:unsupported-kind"]);
+      // Observed, not unsupported-kind. Created through the API it is no longer the file provider's: Grafana does not
+      // keep the provisioning manager annotations a client sends, so the v2 resource's own metadata makes it foreign.
+      expect(report.observed.apiOverview).toMatchObject({ status: "PRESENT", ownership: "foreign" });
+      return (report.deep.drifted.find((d) => d.name === "apiOverview")?.changes ?? []).map((c) => ({ path: c.path, kind: c.kind, live: c.live }));
+    };
+
+    // Only a rename: the v2 read adds no noise, so the drift is the earlier edit plus the new title, at the paths a classic read gives.
+    await recreateAtV2((spec) => {
+      Object.values(spec.elements).find((e) => e.spec.title === "5xx")!.spec.title = "Errors (v2)";
+    });
+    expect((await driftOf()).sort((a, b) => a.path.localeCompare(b.path))).toEqual([
+      { path: "panels[0].panels[0].targets[0].expr", kind: "changed", live: "sum(rate(http_requests_total[1m]))" },
+      { path: "panels[0].panels[1].title", kind: "changed", live: "Errors (v2)" },
+    ]);
+
+    // The row moved into a tab: the tab is an expanded row ahead of it, which the declaration does not have.
+    await recreateAtV2((spec) => {
+      spec.layout = { kind: "TabsLayout", spec: { tabs: [{ kind: "TabsLayoutTab", spec: { title: "Everything", layout: spec.layout } }] } };
+    });
+    expect(await driftOf()).toContainEqual({ path: "panels[0].title", kind: "changed", live: "Everything" });
+
+    const ir = await exportResources({ environment: "e2e", config: { grafana: { profiles: { e2e: profile() } } }, env, selector: { name: "api-overview" } });
+    expect(ir.warnings).toContainEqual(expect.stringMatching(/^dashboard api-overview is stored as v2 and exported in the classic model: layout: the dashboard's tabs "Everything" become expanded rows/));
+    expect(new GrafanaGenerator().generate(ir).map((f) => f.content).join("\n")).toContain('"Errors (v2)"');
+  });
+});

package/src/op/activities/grafana-apply.e2e.test.ts

@@ -0,0 +1,175 @@
+/**
+ * The API applier (#2948) against Grafana itself, 12.4.11 and 13.2.2.
+ *
+ * Per image, one project is built and applied with `grafanaApply`, the Op
+ * activity, through its build output (the index and the dashboard files):
+ *
+ * 1. Apply: two folders, a library panel and three dashboards are created,
+ *    each dashboard and folder labelled with the project's marker.
+ * 2. Re-apply: nothing is created, and nothing is written.
+ * 3. A panel title is edited in the source and applied: that dashboard is
+ *    updated, the rest unchanged.
+ * 4. A dashboard is removed from the source and applied with `prune`: it is
+ *    deleted; a dashboard created directly in Grafana survives, and so does
+ *    the removed dashboard's folder, since somebody saved a dashboard of
+ *    their own into it.
+ * 5. Once that dashboard is gone, the next prune deletes the empty folder.
+ *
+ * The build does not write `__elements` yet (library panels are not
+ * declared in the lexicon), so the library panel is added to one built
+ * dashboard's file the way an exported dashboard carries it.
+ *
+ * Containers come from ../../../test/e2e/containers.ts (unique names,
+ * random host ports, removed in afterAll even on failure;
+ * `CHANT_GRAFANA_IMAGES` overrides the image pair). Skipped, with the reason
+ * in the test name, when Docker is not running. Run through the slot lock:
+ * `~/checkouts/intentius/chant-worktrees/.docker-slot.sh <label> -- npx vitest run lexicons/grafana/src/op/activities/grafana-apply.e2e.test.ts`
+ */
+
+import { mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { build } from "@intentius/chant/build";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { grafanaSerializer } from "../../serializer";
+import { grafanaApply, type GrafanaApplyDeps, type GrafanaApplyOutcome } from "./grafana-apply";
+import { DockerScope, GRAFANA_IMAGES, dockerAvailable, type GrafanaContainer } from "../../../test/e2e/containers";
+
+const hasDocker = dockerAvailable();
+const skipReason = hasDocker ? "" : " (skipped: Docker is not running)";
+const repoRoot = resolve(import.meta.dirname, "..", "..", "..", "..", "..");
+
+const MARKER_LABELS = { "app.kubernetes.io/managed-by": "chant", "chant.intentius.io/stack": "e2e-shop", "chant.intentius.io/env": "e2e" };
+const LIBRARY_PANEL = { uid: "chant-e2e-burn", name: "Error budget burn", kind: 1, model: { type: "stat", title: "Error budget burn", description: "shared" } };
+
+function source(opts: { requestsTitle: string; withErrors: boolean }): string {
+  return `import { Dashboard, StatPanel } from "@intentius/chant-lexicon-grafana";
+
+export const requests = new StatPanel({ title: ${JSON.stringify(opts.requestsTitle)} });
+export const latency = new StatPanel({ title: "Latency" });
+export const fiveHundreds = new StatPanel({ title: "5xx" });
+export const apiOverview = new Dashboard({ title: "API overview", folder: "Team A", tags: ["api"], panels: [requests, latency] });
+${opts.withErrors ? `export const errors = new Dashboard({ title: "Errors", folder: "Team B", panels: [fiveHundreds] });\n` : ""}export const home = new Dashboard({ title: "Home" });
+`;
+}
+
+const summary = (o: GrafanaApplyOutcome) => o.applied.map((a) => `${a.action} ${a.kind}/${a.name}`).sort();
+
+describe.skipIf(!hasDocker).each(GRAFANA_IMAGES)(`grafanaApply applies, re-applies, updates and prunes against %s${skipReason}`, (image) => {
+  const scope = new DockerScope("grafana-apply");
+  let project = "";
+  let indexPath = "";
+  let grafana: GrafanaContainer;
+  let deps: GrafanaApplyDeps;
+
+  /**
+   * Build the project as `opts` declares it, into `dist/`, with the library
+   * panel added to api-overview's file. Each build is a fresh project
+   * directory: the build imports the source modules, and a module path
+   * already imported in this process would be served from the cache.
+   */
+  const buildProject = async (opts: { requestsTitle: string; withErrors: boolean }) => {
+    project = join(scope.tempDir(), "project");
+    mkdirSync(join(project, "src"), { recursive: true });
+    writeFileSync(join(project, "package.json"), JSON.stringify({ name: "grafana-apply-e2e", private: true, type: "module" }));
+    symlinkSync(join(repoRoot, "node_modules"), join(project, "node_modules"), "dir");
+    writeFileSync(join(project, "src", "dashboards.ts"), source(opts));
+    const result = await build(join(project, "src"), [grafanaSerializer]);
+    expect(result.errors).toEqual([]);
+    const out = result.outputs.get("grafana") as SerializerResult;
+    const dist = join(project, "dist");
+    rmSync(dist, { recursive: true, force: true });
+    for (const [file, content] of Object.entries(out.files ?? {})) {
+      mkdirSync(dirname(join(dist, file)), { recursive: true });
+      writeFileSync(join(dist, file), content);
+    }
+    indexPath = join(dist, "grafana.json");
+    writeFileSync(indexPath, out.primary);
+    const index = JSON.parse(out.primary) as { dashboards: Array<{ uid: string; file: string }> };
+    const file = join(dist, index.dashboards.find((d) => d.uid === "api-overview")!.file);
+    const json = JSON.parse(readFileSync(file, "utf8")) as { panels: unknown[] } & Record<string, unknown>;
+    json.__elements = { [LIBRARY_PANEL.uid]: LIBRARY_PANEL };
+    json.panels.push({ id: 99, gridPos: { x: 0, y: 20, w: 12, h: 6 }, libraryPanel: { uid: LIBRARY_PANEL.uid, name: LIBRARY_PANEL.name } });
+    writeFileSync(file, JSON.stringify(json, null, 2));
+  };
+
+  const apply = (prune = false) => grafanaApply({ indexPath, environment: "e2e", ...(prune ? { prune: true } : {}) }, undefined, deps);
+  const dashboard = (uid: string) => grafana.api(`/api/dashboards/uid/${uid}`);
+  const saveForeign = (uid: string, folderUid = "") =>
+    grafana.api("/api/dashboards/db", { method: "POST", body: JSON.stringify({ dashboard: { uid, title: `Saved in the UI (${uid})` }, folderUid, overwrite: true }) });
+
+  beforeAll(async () => {
+    grafana = await scope.grafana({ image });
+    deps = {
+      config: {
+        grafana: { profiles: { e2e: { url: grafana.base, basicAuth: { user: { env: "E2E_GRAFANA_USER" }, password: { env: "E2E_GRAFANA_PASSWORD" } } } } },
+        ownership: { stack: "e2e-shop", env: "e2e" },
+      },
+      env: { E2E_GRAFANA_USER: "admin", E2E_GRAFANA_PASSWORD: "admin" },
+    };
+    expect((await saveForeign("foreign-home")).status).toBe(200);
+  }, 300_000);
+
+  afterAll(() => scope.cleanup());
+
+  it("apply: creates the folders, the library panel and the dashboards, labelled with the marker", { timeout: 240_000 }, async () => {
+    await buildProject({ requestsTitle: "Requests", withErrors: true });
+    const out = await apply();
+    expect(out.api).toMatch(/^apis\/v1(beta1)?$/);
+    expect(out.notAttempted).toEqual([]);
+    expect(summary(out)).toEqual([
+      "created Dashboard/api-overview",
+      "created Dashboard/errors",
+      "created Dashboard/home",
+      "created Folder/team-a",
+      "created Folder/team-b",
+      "created LibraryPanel/chant-e2e-burn",
+    ]);
+
+    const path = out.applied.find((a) => a.name === "api-overview")!.address;
+    const live = (await grafana.api(path)).body as { metadata: { labels: Record<string, string>; annotations: Record<string, string> } };
+    expect(live.metadata.labels).toMatchObject(MARKER_LABELS);
+    expect(live.metadata.annotations["grafana.app/folder"]).toBe("team-a");
+    const folder = (await grafana.api(out.applied.find((a) => a.name === "team-a")!.address)).body as { metadata: { labels: Record<string, string> }; spec: { title: string } };
+    expect(folder.metadata.labels).toMatchObject(MARKER_LABELS);
+    expect(folder.spec.title).toBe("Team A");
+    const connections = (await grafana.api(`/api/library-elements/${LIBRARY_PANEL.uid}/connections`)).body as { result: Array<{ connectionUid: string }> };
+    expect(connections.result.map((c) => c.connectionUid)).toEqual(["api-overview"]);
+  });
+
+  it("re-apply: nothing is created and nothing changes", { timeout: 240_000 }, async () => {
+    const out = await apply();
+    expect(out.applied.filter((a) => a.action === "created")).toEqual([]);
+    expect(out.applied.filter((a) => a.action !== "unchanged")).toEqual([]);
+  });
+
+  it("an edited declaration: that dashboard is updated, the rest unchanged", { timeout: 240_000 }, async () => {
+    await buildProject({ requestsTitle: "Requests per second", withErrors: true });
+    const out = await apply();
+    expect(out.applied.filter((a) => a.action !== "unchanged").map((a) => `${a.action} ${a.kind}/${a.name}`)).toEqual(["updated Dashboard/api-overview"]);
+    const body = (await dashboard("api-overview")).body as { dashboard: { panels: Array<{ title?: string }> } };
+    expect(body.dashboard.panels.map((p) => p.title)).toContain("Requests per second");
+  });
+
+  it("a removed declaration is pruned; what Grafana users made survives, and so does the folder they saved into", { timeout: 240_000 }, async () => {
+    expect((await saveForeign("foreign-in-b", "team-b")).status).toBe(200);
+    await buildProject({ requestsTitle: "Requests per second", withErrors: false });
+    const out = await apply(true);
+    expect(out.pruned.map((p) => `${p.kind}/${p.name}:${p.deleted}`)).toEqual(["Dashboard/errors:true"]);
+    expect(out.notAttempted).toEqual([expect.objectContaining({ kind: "Folder", name: "team-b", reason: "not-prunable" })]);
+    expect((await dashboard("errors")).status).toBe(404);
+    expect((await dashboard("foreign-home")).status).toBe(200);
+    expect((await dashboard("foreign-in-b")).status).toBe(200);
+    expect((await dashboard("api-overview")).status).toBe(200);
+    expect((await dashboard("home")).status).toBe(200);
+  });
+
+  it("once the folder is empty, the next prune deletes it", { timeout: 240_000 }, async () => {
+    expect((await grafana.api("/api/dashboards/uid/foreign-in-b", { method: "DELETE" })).status).toBe(200);
+    const out = await apply(true);
+    expect(out.pruned.map((p) => `${p.kind}/${p.name}:${p.deleted}`)).toEqual(["Folder/team-b:true"]);
+    expect(out.notAttempted).toEqual([]);
+    expect((await grafana.api("/api/folders/team-b")).status).toBe(404);
+    expect((await grafana.api("/api/folders/team-a")).status).toBe(200);
+  });
+});