@intentius/chant-lexicon-grafana 0.98.0 → 0.100.0

package/src/datasource.ts

@@ -10,9 +10,14 @@
 import { createResource } from "@intentius/chant/runtime";
 import type { Declarable } from "@intentius/chant/declarable";
 import { slugUid } from "./util";
+import type { DatasourceJsonData, DatasourceSecureJsonData, TypedDatasourceType } from "./datasource-settings";
 
-/** Datasource plugin types this lexicon types queries for. Any other plugin id is accepted as a string. */
-export type KnownDatasourceType = "prometheus" | "tempo" | "loki";
+/**
+ * Datasource plugin types this lexicon types queries and settings for
+ * (`postgres` is the PostgreSQL plugin's old id). Any other plugin id is
+ * accepted as a string.
+ */
+export type KnownDatasourceType = TypedDatasourceType;
 
 /** How Grafana reaches the datasource: through its backend (`proxy`) or from the browser (`direct`). */
 export type DatasourceAccess = "proxy" | "direct";
@@ -20,7 +25,7 @@ export type DatasourceAccess = "proxy" | "direct";
 export interface DatasourceProps<T extends string = string> {
   /** Display name, unique within the Grafana organisation. */
   name: string;
-  /** The datasource plugin id, e.g. `prometheus`, `tempo`, `loki`. */
+  /** The datasource plugin id, e.g. `prometheus`, `tempo`, `loki`, `grafana-postgresql-datasource`. */
   type: T;
   /** Stable id panels refer to. Defaults to the name as a uid (`Prometheus` becomes `prometheus`). */
   uid?: string;
@@ -33,17 +38,20 @@ export interface DatasourceProps<T extends string = string> {
   database?: string;
   withCredentials?: boolean;
   /**
-   * Plugin settings. A declared `Datasource` anywhere in here is written as
-   * its uid, so `tracesToLogsV2: { datasourceUid: loki }` links Tempo to a
-   * declared Loki.
+   * Plugin settings, typed for the plugins in `DatasourceJsonDataTypes`
+   * (`./datasource-settings.ts`) and any object for others. A declared
+   * `Datasource` anywhere in here is written as its uid, so
+   * `tracesToLogsV2: { datasourceUid: loki }` links Tempo to a declared Loki;
+   * a typed field that names a datasource accepts only the plugin types
+   * Grafana offers there.
    */
-  jsonData?: Record<string, unknown>;
+  jsonData?: DatasourceJsonData<T>;
   /**
-   * Secrets. Write them as Grafana provisioning expands them
-   * (`$__env{NAME}`, `$__file{/path}`, `${NAME}`), not as literals: GRAF002
-   * flags a literal.
+   * Secrets, by the keys the plugin reads. Write them as Grafana provisioning
+   * expands them (`$__env{NAME}`, `$__file{/path}`, `${NAME}`), not as
+   * literals: GRAF002 flags a literal.
    */
-  secureJsonData?: Record<string, string>;
+  secureJsonData?: DatasourceSecureJsonData<T>;
   /** Whether users may edit the provisioned datasource in the UI. Defaults to false. */
   editable?: boolean;
   orgId?: number;
@@ -83,6 +91,58 @@ export function isDatasourceEntity(value: unknown): value is DatasourceEntity {
   );
 }
 
+export const EXTERNAL_DATASOURCE_TYPE = "Grafana::ExternalDatasource";
+
+export interface ExternalDatasourceProps<T extends string = string> {
+  /** The datasource plugin id, e.g. `prometheus`. */
+  type: T;
+  /** The uid the datasource already has in Grafana. */
+  uid: string;
+  /** Its display name in Grafana, for messages and for a `DatasourceVariable` `regex`. */
+  name?: string;
+}
+
+export interface ExternalDatasourceEntity<T extends string = string> extends Declarable {
+  readonly props: ExternalDatasourceProps<T>;
+  /** The plugin id. */
+  readonly datasourceType: T;
+  /** The uid panels will reference. */
+  readonly uid: string;
+}
+
+const ExternalBase = createResource(EXTERNAL_DATASOURCE_TYPE, "grafana", {}) as unknown as (this: object, props: Record<string, unknown>) => void;
+
+export interface ExternalDatasourceConstructor {
+  new <T extends string>(props: ExternalDatasourceProps<T>): ExternalDatasourceEntity<T>;
+}
+
+/**
+ * A datasource that already exists in Grafana: provisioned by hand, by
+ * another build root, or by another tool. It is never written to the
+ * provisioning file. Panels, queries and variables use it like a
+ * `Datasource`, and GRAF101/GRAF102 check references against it.
+ */
+export const ExternalDatasource = function (this: object, props: ExternalDatasourceProps) {
+  ExternalBase.call(this, props as unknown as Record<string, unknown>);
+  Object.defineProperty(this, "datasourceType", { value: props.type, enumerable: false });
+  Object.defineProperty(this, "uid", { value: props.uid, enumerable: false });
+} as unknown as ExternalDatasourceConstructor;
+Object.defineProperty(ExternalDatasource, "name", { value: "ExternalDatasource" });
+
+export function isExternalDatasource(value: unknown): value is ExternalDatasourceEntity {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    (value as Declarable).entityType === EXTERNAL_DATASOURCE_TYPE &&
+    (value as Declarable).lexicon === "grafana"
+  );
+}
+
+/** A declared or an external datasource: anything a panel can hold that has a fixed uid. */
+export function isDatasourceDeclaration(value: unknown): value is DatasourceEntity | ExternalDatasourceEntity {
+  return isDatasourceEntity(value) || isExternalDatasource(value);
+}
+
 /** A datasource declared somewhere this build can't see, named by its Grafana ref. */
 export interface DatasourceRef<T extends string = string> {
   type: T;
@@ -91,3 +151,55 @@ export interface DatasourceRef<T extends string = string> {
 
 /** Grafana's own pseudo-datasources, which need no declaration. */
 export const BUILTIN_DATASOURCE_UIDS: ReadonlySet<string> = new Set(["grafana", "-- Grafana --", "-- Mixed --", "-- Dashboard --"]);
+
+export const DATASOURCE_PROVISIONING_TYPE = "Grafana::DatasourceProvisioning";
+
+/** A datasource the provisioning file deletes, by its name in an organisation. */
+export interface DeletedDatasource {
+  name: string;
+  /** Defaults to 1, as in Grafana. */
+  orgId?: number;
+}
+
+export interface DatasourceProvisioningProps {
+  /**
+   * Written as the file's `prune`. Grafana marks every datasource this file
+   * provisions as prunable, and deletes a prunable datasource once no
+   * provisioning file lists it any more, so removing a `Datasource` removes
+   * it from Grafana on the next provisioning run. Datasources provisioned
+   * without it, or made in the UI or API, are never pruned. Defaults to
+   * true; set false to keep a removed datasource in Grafana.
+   */
+  prune?: boolean;
+  /**
+   * Written as the file's `deleteDatasources`: Grafana deletes these by name
+   * before it provisions the file's datasources. Name one the file also
+   * provisions to have Grafana re-create it from the file on every run.
+   */
+  deleteDatasources?: DeletedDatasource[];
+}
+
+export interface DatasourceProvisioningEntity extends Declarable {
+  readonly props: DatasourceProvisioningProps;
+}
+
+const ProvisioningBase = createResource(DATASOURCE_PROVISIONING_TYPE, "grafana", {}) as unknown as (this: object, props: Record<string, unknown>) => void;
+
+/**
+ * The settings of the datasource provisioning file itself: `prune` and
+ * `deleteDatasources`. Optional: without one the file prunes. Declare at
+ * most one per build.
+ */
+export const DatasourceProvisioning = function (this: object, props: DatasourceProvisioningProps) {
+  ProvisioningBase.call(this, props as unknown as Record<string, unknown>);
+} as unknown as new (props: DatasourceProvisioningProps) => DatasourceProvisioningEntity;
+Object.defineProperty(DatasourceProvisioning, "name", { value: "DatasourceProvisioning" });
+
+export function isDatasourceProvisioningEntity(value: unknown): value is DatasourceProvisioningEntity {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    (value as Declarable).entityType === DATASOURCE_PROVISIONING_TYPE &&
+    (value as Declarable).lexicon === "grafana"
+  );
+}

package/src/deep-observe-hooks.ts

@@ -0,0 +1,267 @@
+/**
+ * grafana deep-observation noise rules (#2946).
+ *
+ * Plain data on the plugin, because core applies the same rules to the
+ * declared tree (a `Dashboard`'s props, panels and queries inlined), which
+ * no reader touches. The live tree is the stored dashboard put back into the
+ * same vocabulary by the importer (./import/live-export.ts), so what is left
+ * to name here is what the translation cannot decide: values the build
+ * wrote that the author did not, values Grafana adds or drops on save, and
+ * values an author wrote out that are also what happens when they are left
+ * out.
+ *
+ * Measured against Grafana 12.4.11 and 13.2.2 (the e2e in
+ * ./observe.e2e.test.ts): read over `/api/dashboards/uid`, a stored dashboard
+ * differs from the built one only by the `id` and `version` Grafana adds
+ * (../test/e2e/stored-model.ts). Read over `/apis/dashboard.grafana.app`, its
+ * spec has no `id` or `version`, but it carries the built-in "Annotations &
+ * Alerts" annotation, and empty `options` and `fieldConfig` and every `null`
+ * are gone. The rest of this table is the build's own choices: panel ids,
+ * grid positions, query refIds, a panel's datasource taken from its queries.
+ *
+ * Every rule is one of four kinds, and they are in this order below:
+ *
+ * 1. **Never compared**, on either side: Grafana's bookkeeping.
+ * 2. **Empty is absent**, on both sides: `null`, `{}` and `[]` mean the same
+ *    as leaving a key out, everywhere in a dashboard or datasource. The cost
+ *    is that values added where the declaration wrote `[]` are reported as
+ *    unclaimed rather than as drift.
+ * 3. **Build-derived**, on the live side where the declaration has nothing:
+ *    the build filled it in, so it cannot disagree with what was declared.
+ * 4. **Default written out**, on the declared side where the live side has
+ *    nothing: the author wrote the value Grafana assumes, which the
+ *    translation leaves out.
+ *
+ * Rules 3 and 4 are gated on `counterpart === "absent"`, so a value the
+ * author did declare is always compared, and a change away from it is drift.
+ * (That relies on core walking a panel's props when it lists the declared
+ * paths, which #2946 fixed in `deepPathSet`.)
+ */
+
+import type { DeepArrayElement, DeepNode, DeepNormalizationHooks } from "@intentius/chant/deep-observation";
+import { DASHBOARD_TYPE } from "./dashboard";
+import { DATASOURCE_TYPE, EXTERNAL_DATASOURCE_TYPE } from "./datasource";
+import { FOLDER_TYPE } from "./folder";
+import { registeredQueries } from "./query";
+import { DASHBOARD_SCHEMA_VERSION } from "./schema/dashboard.gen";
+import { BOOKKEEPING_KEYS, LINK_DEFAULTS, deepEqual, isBuiltinAnnotation } from "./import/normalize";
+import { DATASOURCE_API_DEFAULTS } from "./import/live-export";
+
+type Json = Record<string, unknown>;
+
+/** Where a node sits in a dashboard tree, from its index-erased pattern. */
+type Level = "dashboard" | "panel" | "target" | "variable" | "link" | "annotation";
+
+const SEGMENT = "([^.\\[\\]]+)";
+const PANEL = "panels\\[\\](?:\\.panels\\[\\])?";
+const LEVELS: ReadonlyArray<[Level, RegExp]> = [
+  ["dashboard", new RegExp(`^${SEGMENT}$`)],
+  ["panel", new RegExp(`^${PANEL}\\.${SEGMENT}$`)],
+  ["target", new RegExp(`^${PANEL}\\.targets\\[\\]\\.${SEGMENT}$`)],
+  ["variable", new RegExp(`^variables\\[\\]\\.${SEGMENT}$`)],
+  ["link", new RegExp(`^(?:${PANEL}\\.)?links\\[\\]\\.${SEGMENT}$`)],
+  ["annotation", new RegExp(`^annotations\\[\\]\\.${SEGMENT}$`)],
+];
+
+function levelOf(pattern: string): { level: Level; key: string } | undefined {
+  for (const [level, re] of LEVELS) {
+    const m = re.exec(pattern);
+    if (m) return { level, key: m[1] };
+  }
+  return undefined;
+}
+
+/**
+ * Defaults in the props vocabulary: a key an author may write at this value,
+ * and the translation leaves out because Grafana assumes it. The dashboard
+ * and panel entries mirror `DASHBOARD_DEFAULTS` and `PANEL_DEFAULTS` in
+ * ./import/normalize.ts, with `graphTooltip` spelled as `Dashboard` takes it.
+ */
+export const PROP_DEFAULTS: Readonly<Record<Level, Readonly<Json>>> = {
+  dashboard: {
+    editable: true,
+    fiscalYearStartMonth: 0,
+    graphTooltip: "default",
+    links: [],
+    tags: [],
+    time: { from: "now-6h", to: "now" },
+    timepicker: {},
+    timezone: "",
+    weekStart: "",
+    refresh: "",
+    liveNow: false,
+    description: "",
+    schemaVersion: DASHBOARD_SCHEMA_VERSION,
+  },
+  panel: {
+    title: "",
+    description: "",
+    options: {},
+    targets: [],
+    transformations: [],
+    links: [],
+    transparent: false,
+    hideTimeOverride: false,
+    repeatDirection: "h",
+    // A row's.
+    collapsed: false,
+    panels: [],
+  },
+  target: {},
+  variable: {
+    label: "",
+    description: "",
+    hide: "label",
+    skipUrlSync: false,
+    multi: false,
+    includeAll: false,
+    allValue: "",
+    regex: "",
+    sort: 0,
+    refresh: "onLoad",
+    auto: false,
+    autoCount: 30,
+    autoMin: "10s",
+  },
+  link: LINK_DEFAULTS,
+  // Grafana 13.2 drops `hide: false` from an annotation it stores.
+  annotation: { hide: false },
+};
+
+/** What the build writes into an annotation that leaves them out (annotations.ts `annotationJson`). */
+const ANNOTATION_BUILD_DEFAULTS: Readonly<Json> = { enable: true, iconColor: "red" };
+
+/**
+ * Keys the build fills in when a declaration leaves them out, by level:
+ * the dashboard's uid (from its export name) and schema version, a panel's
+ * id, grid position and datasource (from its queries), a query's refId.
+ * Grafana keeps what the build wrote, so undeclared, they read back as the
+ * build's choice rather than as anybody's edit.
+ *
+ * `timezone` is here as a value, not a key: chant writes `browser` for a
+ * dashboard that names none, where Grafana's own default is `""`.
+ */
+export const BUILD_DERIVED: Readonly<Record<Level, ReadonlySet<string>>> = {
+  dashboard: new Set(["uid", "schemaVersion"]),
+  panel: new Set(["id", "gridPos", "datasource", "pluginVersion"]),
+  target: new Set(["refId", "datasource"]),
+  // A query variable's selection is whatever the viewer last saved; the build writes one only when it is declared.
+  variable: new Set(["current", "datasource"]),
+  link: new Set(),
+  annotation: new Set(),
+};
+
+/** Defaults the build merges into a query from its class (`queryType: "traceql"`, `filters: []`): any key some class defaults, at that value. */
+function queryDefault(key: string, value: unknown): boolean {
+  for (const def of registeredQueries()) {
+    const defaults = (def as { defaults?: Json }).defaults;
+    if (defaults && key in defaults && deepEqual(defaults[key], value)) return true;
+  }
+  return false;
+}
+
+/** `[]` or a plain `{}`. Not a declarable, whose own keys are not enumerable. */
+function isEmptyContainer(value: unknown): boolean {
+  if (Array.isArray(value)) return value.length === 0;
+  if (typeof value !== "object" || value === null) return false;
+  const proto = Object.getPrototypeOf(value) as unknown;
+  return (proto === Object.prototype || proto === null) && Object.keys(value).length === 0;
+}
+
+function isEmptyFieldConfig(value: unknown): boolean {
+  return isEmptyContainer(value) || deepEqual(value, { defaults: {}, overrides: [] });
+}
+
+/** Any node at or under a `datasource` key: a reference, whichever form (ref, variable, entity) it takes. */
+function underDatasource(pattern: string): boolean {
+  return /(^|\.)datasource(\.|$)/.test(pattern);
+}
+
+function isDatasourceType(entityType: string): boolean {
+  return entityType === DATASOURCE_TYPE || entityType === EXTERNAL_DATASOURCE_TYPE;
+}
+
+function pruneDatasource(node: DeepNode): boolean {
+  // 1. The organisation is where it lives, not a property of it; the version is Grafana's save counter.
+  if (node.pattern === "orgId" || node.pattern === "version") return true;
+  // 2.
+  if (node.value === null || isEmptyContainer(node.value)) return true;
+  if (node.counterpart !== "absent") return false;
+  if (node.side === "live") {
+    // 3. The uid comes from the name when not declared; `editable` is `!readOnly`, true for one created in the UI or API.
+    return node.pattern === "uid" || node.pattern === "editable";
+  }
+  // 4.
+  if (node.pattern === "editable") return node.value === false;
+  return node.pattern in DATASOURCE_API_DEFAULTS && deepEqual(DATASOURCE_API_DEFAULTS[node.pattern], node.value);
+}
+
+function pruneDashboard(node: DeepNode): boolean {
+  const at = levelOf(node.pattern);
+
+  // 1. Bookkeeping and Grafana's own additions, whatever either side says.
+  if (at?.level === "dashboard" && BOOKKEEPING_KEYS.includes(at.key)) return true;
+  if (/^annotations(\.list)?\[\]$/.test(node.pattern) && isBuiltinAnnotation(node.value)) return true;
+  if (at?.level === "panel" && (at.key === "options" || at.key === "fieldConfig") && isEmptyFieldConfig(node.value)) return true;
+
+  // 2. Empty is absent, on both sides whatever the other has. Grafana drops
+  // every `null` when it stores a dashboard through /apis, and an empty list
+  // or object means what leaving the key out means. Not gated on the
+  // counterpart, which is matched by pattern: a threshold's first step has
+  // `value: null` where the others have a number, and one panel's
+  // `overrides: []` sits beside another's list of overrides.
+  if (node.value === null || isEmptyContainer(node.value)) return true;
+  if (node.counterpart !== "absent") return false;
+
+  if (node.side === "live") {
+    // 3. Build-derived.
+    if (underDatasource(node.pattern)) return true;
+    if (/^panels\[\](?:\.panels\[\])?\.gridPos\./.test(node.pattern)) return true;
+    if (!at) return false;
+    if (BUILD_DERIVED[at.level].has(at.key)) return true;
+    if (at.level === "dashboard" && at.key === "timezone" && node.value === "browser") return true;
+    if (at.level === "target" && queryDefault(at.key, node.value)) return true;
+    if (at.level === "link" && at.key in LINK_DEFAULTS && deepEqual(LINK_DEFAULTS[at.key], node.value)) return true;
+    if (at.level === "annotation" && at.key in ANNOTATION_BUILD_DEFAULTS && deepEqual(ANNOTATION_BUILD_DEFAULTS[at.key], node.value)) return true;
+    return false;
+  }
+
+  // 4. A default written out.
+  if (!at) return false;
+  const defaults = PROP_DEFAULTS[at.level];
+  return at.key in defaults && deepEqual(defaults[at.key], node.value);
+}
+
+/** A folder's uid comes from its path when it is not declared. */
+function pruneFolder(node: DeepNode): boolean {
+  return node.side === "live" && node.counterpart === "absent" && node.pattern === "uid";
+}
+
+export const grafanaDeepNormalizationHooks: DeepNormalizationHooks = {
+  prune(node: DeepNode): boolean {
+    if (node.entityType === DASHBOARD_TYPE) return pruneDashboard(node);
+    if (node.entityType === FOLDER_TYPE) return pruneFolder(node);
+    if (isDatasourceType(node.entityType)) return pruneDatasource(node);
+    return false;
+  },
+
+  /**
+   * A datasource's secrets are compared by key, never by value: Grafana
+   * returns only which ones are set, the reader writes those as
+   * `[REDACTED]`, and the declared values (`$__env{...}` references, or
+   * worse) are masked the same way.
+   */
+  mask(node: DeepNode): boolean {
+    return isDatasourceType(node.entityType) && /^secureJsonData\./.test(node.pattern);
+  },
+
+  /**
+   * Dashboard tags are a set. Nothing else is reordered: panel order is
+   * layout order, and variable and link order is the order Grafana shows
+   * them in.
+   */
+  orderKey(element: DeepArrayElement): string | undefined {
+    if (element.entityType === DASHBOARD_TYPE && element.pattern === "tags" && typeof element.element === "string") return element.element;
+    return undefined;
+  },
+};