@intentius/chant 0.124.46 → 0.124.49

package/src/workspace/kinds.ts

@@ -17,7 +17,7 @@
  * fails ({@link resolveKind}).
  */
 
-import { readFileSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
 import { isAbsolute, join, relative, resolve } from "node:path";
 import schema from "./workspace-kinds.schema.json" with { type: "json" };
 import type { WorkspaceTree } from "./tree";
@@ -25,6 +25,7 @@ import { joinPath, skippedDir } from "./tree";
 import { checkPinIntegrity } from "./pin-integrity";
 import { dataExportTarget, findInstalledPackage } from "./class-data";
 import { loadAjv2020 } from "./ajv2020";
+import yaml from "js-yaml";
 
 export const KINDS_SCHEMA_ID = schema.$id;
 
@@ -34,7 +35,9 @@ export const KINDS_SUBPATH = "./workspace-kinds";
 /**
  * The probe a package's kind carries (#2535, #2545). It passes when any of
  * its clauses does. A name in either clause may use `*`, which matches any
- * run of characters in one file name, so `*.tf` is every Terraform file.
+ * run of characters in one file name, so `*.tf` is every Terraform file, and
+ * `[0-9]` matches one digit (#4400), so `[0-9]*_*.sql` is a file that opens
+ * with a version number.
  */
 export interface FileProbe {
   /** A file matching one of these names sits directly in the directory. */
@@ -53,6 +56,25 @@ export interface FileProbe {
    * without claiming every Node package (#3151).
    */
   anyJsonKey?: { in: string[]; pointers: string[] };
+  /**
+   * A file directly in the directory whose name matches one of `in` has a
+   * line that starts, after leading space and ignoring case, with one of
+   * `starts` (#4400). This is how a goose directory is told from any other
+   * directory of `.sql` files: its files carry `-- +goose Up`.
+   */
+  anyLine?: { in: string[]; starts: string[] };
+  /**
+   * A file directly in the directory whose name matches one of `in` parses
+   * as YAML or JSON (every document of a multi-document file is read), and
+   * one document satisfies one of the `match` entries. An entry maps JSON
+   * Pointers to the values allowed there, and a document satisfies it when
+   * every pointer holds a value that matches one of its patterns. A `*` in
+   * a pattern matches any run of characters, so `["*"]` means "has a
+   * value". This is how an Argo Application, a Flux Kustomization and a
+   * CloudFormation template are found by what they say, not by their file
+   * names (#3179). A file that doesn't parse is no evidence either way.
+   */
+  anyDocument?: { in: string[]; match: Record<string, string[]>[] };
 }
 
 /** A string, integer or boolean field a kind declares for its members (#3151). */
@@ -81,7 +103,9 @@ export type KindProbe =
   /** The directory exists; nothing else is checked (`other`). */
   | { directory: true }
   /** The directory holds a chant project, as {@link holdsChantProject} says (`examples`). */
-  | { chantProject: true };
+  | { chantProject: true }
+  /** Nothing local: the member names another repository's workspace (`repo`, #3168). */
+  | { repo: true };
 
 /**
  * Which outputs a member of a kind exposes as link targets (#2524 D6,
@@ -101,7 +125,15 @@ export type KindOutputs =
  * chant writes itself. See {@link substituteGraphConfig} for the placeholders.
  */
 export interface KindGraph {
-  lexicon: string;
+  /** The lexicon that reads the member; unset when `reader` is. */
+  lexicon?: string;
+  /**
+   * A graph reader module (#4413), as the kinds file names it: a path inside
+   * the supplying package, which may be any package. Unset when `lexicon` is.
+   */
+  reader?: string;
+  /** The reader module, absolute; set once the package directory is known. */
+  readerFile?: string;
   config: Record<string, unknown>;
 }
 
@@ -135,15 +167,38 @@ export interface MemberKind {
    * entry states.
    */
   reader?: KindLinkReaderName;
+  /** A file every member of the kind has, and the JSON Schema it must pass (#3672); unset, the kind checks no member file. */
+  memberFile?: KindMemberFile;
+}
+
+/**
+ * A kind's member file (#3672): `path` in each member's directory must pass
+ * the JSON Schema at `schema` in the supplying package. `chant workspace
+ * check` reads and validates it (WSP012). The schema is read as data when the
+ * package's kinds are read, so a schema that is missing or doesn't compile is
+ * a load problem (WSP002), and `validate` is unset.
+ */
+export interface KindMemberFile {
+  /** The file, relative to the member's directory. */
+  path: string;
+  /** The schema, as the kinds file names it, relative to the package. */
+  schema: string;
+  /** The schema, absolute; unset until the package directory is known. */
+  schemaFile?: string;
+  /** The compiled schema; unset when it could not be read or compiled. */
+  validate?: MemberFileValidate;
 }
 
+/** A compiled member-file schema, in Ajv's shape. */
+export type MemberFileValidate = ((data: unknown) => boolean) & { errors?: { instancePath: string; message?: string; params?: Record<string, unknown> }[] | null };
+
 /**
  * The link readers chant ships (#3932). A kind names one in its kinds file as
  * `reader`. The readers are chant's own code, not the package's: a kinds file
  * stays data, and chant never imports the package that supplies it. Closed:
  * a kind naming another reader is kept without one, and the load reports it.
  */
-export const KIND_LINK_READERS = ["dbt-project"] as const;
+export const KIND_LINK_READERS = ["dbt-project", "goose-dir", "golang-migrate-dir", "clickhouse-migrations-dir", "flyway-dir"] as const;
 export type KindLinkReaderName = (typeof KIND_LINK_READERS)[number];
 
 export interface KindRegistry {
@@ -157,6 +212,9 @@ export interface KindRegistry {
 /** The group kind (ws-051). An entry of this kind is a group, not a member. */
 export const EXAMPLES_KIND = "examples";
 
+/** The member kind that names another repository's workspace by URL and pinned revision (#3168). */
+export const REPO_KIND = "repo";
+
 /** The data member kind that holds design artifacts (#2549, ws-062). */
 export const DESIGN_KIND = "design";
 
@@ -179,6 +237,15 @@ export const BUILTIN_KINDS: readonly MemberKind[] = [
     outputs: { from: "none" },
     source: "builtin",
   },
+  {
+    name: REPO_KIND,
+    description: "another repository's workspace, named by URL and pinned revision; chant reads it through that repository's own chant workspace graph and never writes there",
+    probe: { repo: true },
+    precedence: 0,
+    shape: "member",
+    outputs: { from: "none" },
+    source: "builtin",
+  },
   {
     name: "other",
     description: "a directory chant does not read; the entry says why in `because`",
@@ -232,7 +299,8 @@ export function builtinKindRegistry(): KindRegistry {
 export function probeKind(kind: MemberKind, tree: WorkspaceTree, dir: string): boolean {
   if ("directory" in kind.probe) return true;
   if ("chantProject" in kind.probe) return holdsChantProject(tree, dir);
-  const { anyFile, anyBlock, anyJsonKey } = kind.probe;
+  if ("repo" in kind.probe) return true;
+  const { anyFile, anyBlock, anyJsonKey, anyLine, anyDocument } = kind.probe;
   if (anyFile && filesMatching(tree, dir, anyFile).length > 0) return true;
   if (anyJsonKey) {
     for (const file of filesMatching(tree, dir, anyJsonKey.in)) {
@@ -244,6 +312,25 @@ export function probeKind(kind: MemberKind, tree: WorkspaceTree, dir: string): b
       }
     }
   }
+  if (anyLine) {
+    const starts = anyLine.starts.map((t) => t.toLowerCase());
+    for (const file of filesMatching(tree, dir, anyLine.in)) {
+      try {
+        if (tree.read(file).split(/\r?\n/).some((l) => starts.some((t) => l.trimStart().toLowerCase().startsWith(t)))) return true;
+      } catch {
+        // An unreadable file is no evidence either way.
+      }
+    }
+  }
+  if (anyDocument) {
+    for (const file of filesMatching(tree, dir, anyDocument.in)) {
+      try {
+        if (documentsOf(tree.read(file)).some((doc) => anyDocument.match.some((m) => documentMatches(doc, m)))) return true;
+      } catch {
+        // An unreadable file is no evidence either way.
+      }
+    }
+  }
   if (anyBlock) {
     const opener = new RegExp(`^\\s*(?:${anyBlock.blocks.map(escapeRegExp).join("|")})\\s*\\{`, "m");
     for (const file of filesMatching(tree, dir, anyBlock.in)) {
@@ -259,7 +346,7 @@ export function probeKind(kind: MemberKind, tree: WorkspaceTree, dir: string): b
 
 /** Whether a probe looks at the directory's files, and so can claim it. */
 function isFileProbe(probe: KindProbe): probe is FileProbe {
-  return !("directory" in probe) && !("chantProject" in probe);
+  return !("directory" in probe) && !("chantProject" in probe) && !("repo" in probe);
 }
 
 /**
@@ -274,7 +361,7 @@ function filesMatching(tree: WorkspaceTree, dir: string, names: readonly string[
     if (!name.includes("*") && tree.stat(joinPath(dir, name)) === "file") out.push(joinPath(dir, name));
   }
   if (globs.length > 0) {
-    const patterns = globs.map((g) => new RegExp(`^${g.split("*").map(escapeRegExp).join("[^/]*")}$`));
+    const patterns = globs.map((g) => new RegExp(`^${g.split("*").map((part) => escapeRegExp(part).replace(/\\\[0-9\\\]/g, "[0-9]")).join("[^/]*")}$`));
     for (const e of tree.list(dir) ?? []) {
       if (e.type === "file" && patterns.some((p) => p.test(e.name))) out.push(joinPath(dir, e.name));
     }
@@ -282,6 +369,37 @@ function filesMatching(tree: WorkspaceTree, dir: string, names: readonly string[
   return out.sort();
 }
 
+/**
+ * The documents in `text`, read as YAML (which holds JSON). A CloudFormation
+ * short-form tag such as `!Ref` or `!Sub` is not YAML js-yaml knows, so a
+ * file that fails to parse is read again with those tags dropped. Text that
+ * still doesn't parse has no documents.
+ */
+function documentsOf(text: string): unknown[] {
+  try {
+    return yaml.loadAll(text);
+  } catch {
+    try {
+      return yaml.loadAll(text.replace(/(^|[\s:\-[,{])!(?:[A-Za-z][\w]*:?)+/gm, "$1"));
+    } catch {
+      return [];
+    }
+  }
+}
+
+/** Whether every pointer in `match` holds a value matching one of its patterns. */
+function documentMatches(doc: unknown, match: Record<string, string[]>): boolean {
+  const pointers = Object.entries(match);
+  return (
+    pointers.length > 0 &&
+    pointers.every(([pointer, patterns]) => {
+      const value = atPointer(doc, pointer);
+      if (value === undefined) return false;
+      return patterns.some((pattern) => new RegExp(`^${pattern.split("*").map(escapeRegExp).join(".*")}$`).test(String(value)));
+    })
+  );
+}
+
 /** The value at JSON Pointer `pointer` in `json`, or undefined when there is none. */
 function atPointer(json: unknown, pointer: string): unknown {
   let at: unknown = json;
@@ -400,7 +518,7 @@ export function parseKindData(text: string, source: string): KindData {
   const problems: string[] = [];
   const kinds: MemberKind[] = [];
   const seen = new Set<string>();
-  for (const k of (raw as { kinds: { name: string; description: string; precedence: number; probe: FileProbe; outputs?: string[]; graph?: KindGraph; fields?: Record<string, KindField>; reader?: string }[] }).kinds) {
+  for (const k of (raw as { kinds: { name: string; description: string; precedence: number; probe: FileProbe; outputs?: string[]; graph?: KindGraph; fields?: Record<string, KindField>; reader?: string; memberFile?: { path: string; schema: string } }[] }).kinds) {
     if (BUILTIN_KIND_NAMES.includes(k.name)) {
       problems.push(`${source}: kind ${k.name} is built in and can't be supplied by a package`);
       continue;
@@ -417,14 +535,19 @@ export function parseKindData(text: string, source: string): KindData {
         ...(k.probe.anyFile ? { anyFile: [...k.probe.anyFile] } : {}),
         ...(k.probe.anyBlock ? { anyBlock: { in: [...k.probe.anyBlock.in], blocks: [...k.probe.anyBlock.blocks] } } : {}),
         ...(k.probe.anyJsonKey ? { anyJsonKey: { in: [...k.probe.anyJsonKey.in], pointers: [...k.probe.anyJsonKey.pointers] } } : {}),
+        ...(k.probe.anyLine ? { anyLine: { in: [...k.probe.anyLine.in], starts: [...k.probe.anyLine.starts] } } : {}),
+        ...(k.probe.anyDocument ? { anyDocument: { in: [...k.probe.anyDocument.in], match: k.probe.anyDocument.match.map((m) => structuredClone(m)) } } : {}),
       },
       precedence: k.precedence,
       shape: "member",
       outputs: { from: "declared", names: [...(k.outputs ?? [])] },
       source,
-      ...(k.graph ? { graph: { lexicon: k.graph.lexicon, config: structuredClone(k.graph.config) } } : {}),
+      ...(k.graph
+        ? { graph: { ...(k.graph.lexicon ? { lexicon: k.graph.lexicon } : {}), ...(k.graph.reader ? { reader: k.graph.reader } : {}), config: structuredClone(k.graph.config ?? {}) } }
+        : {}),
       ...(k.fields ? { fields: structuredClone(k.fields) } : {}),
       ...(k.reader && (KIND_LINK_READERS as readonly string[]).includes(k.reader) ? { reader: k.reader as KindLinkReaderName } : {}),
+      ...(k.memberFile ? { memberFile: { path: k.memberFile.path, schema: k.memberFile.schema } } : {}),
     });
     if (k.reader && !(KIND_LINK_READERS as readonly string[]).includes(k.reader)) {
       problems.push(`${source}: kind ${k.name} names link reader ${k.reader}, which this chant does not have (it has ${KIND_LINK_READERS.join(", ")}); the kind is kept, and its members' links are only those their entries state`);
@@ -477,7 +600,24 @@ export function readPackageKinds(packageDir: string, source?: string): PackageKi
   const packageName = typeof pkg.name === "string" ? pkg.name : undefined;
   for (const k of data.kinds) {
     k.packageDir = packageDir;
-    if (k.graph && packageName !== lexiconPackageName(k.graph.lexicon)) {
+    if (k.memberFile) {
+      const problem = loadMemberFileSchema(k.memberFile, packageDir);
+      if (problem) data.problems.push(`${name}: kind ${k.name} ${problem}; members of the kind fail WSP012 until it is fixed`);
+    }
+    if (k.graph?.reader) {
+      // A reader module is any package's own file (#4413): it must sit inside the package and exist. Nothing is imported here.
+      const file = resolve(packageDir, ...k.graph.reader.split("/"));
+      const inside = relative(packageDir, file);
+      if (inside.startsWith("..") || isAbsolute(inside)) {
+        data.problems.push(`${name}: kind ${k.name} names graph reader ${k.graph.reader}, which is outside the package; the kind is kept, and workspace graph lists its members with kind-not-run`);
+        delete k.graph;
+      } else if (!existsSync(file)) {
+        data.problems.push(`${name}: kind ${k.name} names graph reader ${k.graph.reader}, which does not exist in the package; the kind is kept, and workspace graph lists its members with kind-not-run`);
+        delete k.graph;
+      } else {
+        k.graph.readerFile = file;
+      }
+    } else if (k.graph?.lexicon && packageName !== lexiconPackageName(k.graph.lexicon)) {
       data.problems.push(
         `${name}: kind ${k.name} reads its members with lexicon ${k.graph.lexicon}, which is ${lexiconPackageName(k.graph.lexicon)}, ` +
           `and this package is ${packageName ?? "unnamed"}; the kind is kept, and workspace graph lists its members with kind-not-run`,
@@ -488,6 +628,67 @@ export function readPackageKinds(packageDir: string, source?: string): PackageKi
   return { ...data, file };
 }
 
+/**
+ * Read and compile a kind's member-file schema from `packageDir` (#3672),
+ * setting `schemaFile` and `validate`. Returns why it can't be used, or
+ * undefined. The schema is a JSON file inside the package; it is data, so
+ * nothing is imported, and a `$ref` outside the file is not followed.
+ */
+export function loadMemberFileSchema(memberFile: KindMemberFile, packageDir: string): string | undefined {
+  const file = resolve(packageDir, ...memberFile.schema.split("/"));
+  memberFile.schemaFile = file;
+  const inside = relative(packageDir, file);
+  if (inside.startsWith("..") || isAbsolute(inside)) return `names member-file schema ${memberFile.schema}, which is outside the package`;
+  let doc: unknown;
+  try {
+    doc = JSON.parse(readFileSync(file, "utf-8"));
+  } catch (err) {
+    const missing = (err as NodeJS.ErrnoException).code === "ENOENT";
+    return missing ? `names member-file schema ${memberFile.schema}, which does not exist` : `names member-file schema ${memberFile.schema}, which is not JSON (${(err as Error).message})`;
+  }
+  if (doc === null || typeof doc !== "object" || Array.isArray(doc)) return `names member-file schema ${memberFile.schema}, which is not a JSON Schema object`;
+  try {
+    const Ajv = loadAjv2020() as new (opts: object) => { compile(s: object): MemberFileValidate };
+    memberFile.validate = new Ajv({ allErrors: true, strict: false }).compile(doc);
+  } catch (err) {
+    return `names member-file schema ${memberFile.schema}, which does not compile as a draft 2020-12 schema (${(err as Error).message})`;
+  }
+  return undefined;
+}
+
+/** One reason a member's file fails its kind's schema (#3672): a JSON Pointer into the file, and why. */
+export interface MemberFileProblem {
+  pointer: string;
+  message: string;
+}
+
+/**
+ * Check a member's file text against its kind's schema (#3672). `file` is the
+ * file's name, which picks the parser: YAML for `.yml` and `.yaml`, JSON
+ * otherwise. The kind's schema must have loaded; when it didn't, WSP002 has
+ * already said why, and this returns nothing.
+ */
+export function checkMemberFile(memberFile: KindMemberFile, file: string, text: string): MemberFileProblem[] {
+  if (!memberFile.validate) return [];
+  let data: unknown;
+  try {
+    data = /\.ya?ml$/i.test(file) ? yaml.load(text) : JSON.parse(text);
+  } catch (err) {
+    return [{ pointer: "", message: `is not ${/\.ya?ml$/i.test(file) ? "YAML" : "JSON"} (${(err as Error).message.split("\n")[0]})` }];
+  }
+  if (memberFile.validate(data)) return [];
+  const seen = new Set<string>();
+  const out: MemberFileProblem[] = [];
+  for (const e of memberFile.validate.errors ?? []) {
+    const extra = e.params && typeof e.params.additionalProperty === "string" ? ` (${e.params.additionalProperty})` : "";
+    const message = `${e.instancePath || "the file"} ${e.message ?? "is invalid"}${extra}`;
+    if (seen.has(message)) continue;
+    seen.add(message);
+    out.push({ pointer: e.instancePath, message });
+  }
+  return out;
+}
+
 /** The package a lexicon name resolves to. */
 export function lexiconPackageName(lexicon: string): string {
   return `@intentius/chant-lexicon-${lexicon}`;

package/src/workspace/ledger.schema.json

@@ -0,0 +1,158 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://intentius.io/chant/schemas/workspace/ledger/v1/ledger.schema.json",
+  "title": "chant ledger output",
+  "description": "What `chant ledger <table> --json` prints (#4363): the rows of one ledger on the local chant/lifecycle branch, filtered with typed filters instead of SQL. The tables and columns are the ones `chant query` loads (schema version 1 of its tables); a column chant query keeps as JSON text is an object or array here, and `signed` and `empty` are booleans. It reads the local branch and never fetches. Version 1 of the read contract. Readers ignore fields they do not know; a field is only ever added within a version. The codes are a closed list, each in the one closed list of `reason-codes.ts`.",
+  "oneOf": [
+    {
+      "$ref": "#/$defs/result"
+    },
+    {
+      "$ref": "#/$defs/failure"
+    }
+  ],
+  "$defs": {
+    "result": {
+      "type": "object",
+      "description": "The rows that every filter matched. Exit code 0.",
+      "required": [
+        "$schema",
+        "contract",
+        "table",
+        "columns",
+        "at",
+        "filter",
+        "total",
+        "rows",
+        "malformed"
+      ],
+      "properties": {
+        "$schema": {
+          "const": "https://intentius.io/chant/schemas/workspace/ledger/v1/ledger.schema.json"
+        },
+        "contract": {
+          "const": 1
+        },
+        "table": {
+          "enum": [
+            "resources",
+            "snapshots",
+            "runs",
+            "gates",
+            "deploys",
+            "releases",
+            "cuts"
+          ],
+          "description": "The table read: resources and snapshots from each lexicon's snapshots, runs from the Op run ledgers, gates from _gates/, deploys from <env>/releases.jsonl, releases and cuts from _releases/."
+        },
+        "columns": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "The table's columns, in order. Every row has each of them."
+        },
+        "at": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "description": "The revision of chant/lifecycle read (--at), as given, or null for the branch tip."
+        },
+        "filter": {
+          "type": "object",
+          "required": [
+            "where",
+            "since"
+          ],
+          "properties": {
+            "where": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Each --where clause, <field><op><value>, as given. Every one held for each row listed."
+            },
+            "since": {
+              "type": [
+                "string",
+                "null"
+              ],
+              "description": "--since as the ISO instant it was read as, or null. The rows whose time column (observed_at, started or timestamp) is at or after it are listed."
+            }
+          }
+        },
+        "total": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Rows in the table before the filter."
+        },
+        "rows": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/row"
+          }
+        },
+        "malformed": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Lines and files of this table that could not be read as their kind, left out of total and rows."
+        }
+      }
+    },
+    "row": {
+      "type": "object",
+      "description": "One row, keyed by column. Every row starts with member: the workspace member whose ledger it came from, or null outside a workspace. A value is a string, a number, a boolean, an object or array (a JSON column), or null.",
+      "required": [
+        "member"
+      ],
+      "additionalProperties": {
+        "type": [
+          "string",
+          "number",
+          "boolean",
+          "object",
+          "array",
+          "null"
+        ]
+      }
+    },
+    "failure": {
+      "type": "object",
+      "description": "Nothing was read. Exit code 1.",
+      "required": [
+        "$schema",
+        "contract",
+        "error"
+      ],
+      "properties": {
+        "$schema": {
+          "const": "https://intentius.io/chant/schemas/workspace/ledger/v1/ledger.schema.json"
+        },
+        "contract": {
+          "const": 1
+        },
+        "error": {
+          "type": "object",
+          "required": [
+            "code",
+            "message"
+          ],
+          "properties": {
+            "code": {
+              "enum": [
+                "filter-invalid",
+                "table-unknown",
+                "revision-unknown",
+                "ledger-unreadable"
+              ]
+            },
+            "message": {
+              "type": "string"
+            }
+          }
+        }
+      }
+    }
+  }
+}