@milaboratories/pl-middle-layer 1.66.19 → 1.67.0

package/src/model/template_serializer.test.ts

@@ -0,0 +1,398 @@
+import { describe, expect, test } from "vitest";
+import YAML from "yaml";
+import type { BlockKindReference } from "@milaboratories/pl-model-common";
+import {
+  createPlRef,
+  kindReferenceToSelectorReference,
+  parseProjectTemplateV1,
+} from "@milaboratories/pl-model-common";
+import type { BlockPackSpec } from "@milaboratories/pl-model-middle-layer";
+import type { ProjectStructure } from "./project_model";
+import type { TemplateParamsResult } from "./template_export";
+import {
+  assembleProjectTemplateV1,
+  exportProjectAsTemplateV1,
+  locationOf,
+  stringifyProjectTemplateV1,
+} from "./template_serializer";
+
+function simpleStructure(...ids: string[]): ProjectStructure {
+  return {
+    groups: [
+      {
+        id: "g1",
+        label: "G1",
+        blocks: ids.map((id) => ({ id, label: id, renderingMode: "Heavy" })),
+      },
+    ],
+  };
+}
+
+const ok = (value: unknown): TemplateParamsResult => ({ value });
+
+const kindOf = (name: string, version = "1.4.2") =>
+  `@platforma-open/milaboratories.${name}.kind@${version}` as BlockKindReference;
+
+/** Every block gets a kind derived from its own id. */
+const kindPerBlock = (blockId: string) => kindOf(blockId);
+
+/** A registry-installed block: found by name, so it needs no locator. */
+const registrySpec: BlockPackSpec = {
+  type: "from-registry-v2",
+  registryUrl: "https://block.registry.platforma.bio/releases",
+  id: { organization: "milaboratories", name: "demo", version: "1.4.2" },
+  channel: "stable",
+};
+
+const devSpec = (folder: string): BlockPackSpec => ({ type: "dev-v2", folder });
+
+function exportOf(
+  structure: ProjectStructure,
+  params: Record<string, TemplateParamsResult>,
+  kinds: (blockId: string) => BlockKindReference | undefined = kindPerBlock,
+  specs: (blockId: string) => BlockPackSpec | undefined = () => registrySpec,
+) {
+  return exportProjectAsTemplateV1(structure, (id) => params[id], kinds, specs);
+}
+
+describe("the document", () => {
+  test("an entry is the block id, its exact kind, and its params", () => {
+    const result = exportOf(simpleStructure("samples"), {
+      samples: ok({ dataset: "bulk-rna" }),
+    });
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect(result.document).toEqual({
+      schema: "template-v1",
+      blocks: [
+        {
+          id: "samples",
+          kind: "@platforma-open/milaboratories.samples.kind@1.4.2",
+          params: { dataset: "bulk-rna" },
+        },
+      ],
+    });
+  });
+
+  test("the kind is emitted at the exact tier, never widened to a range", () => {
+    // A block implements exactly one kind version, so pinning it is the whole
+    // point; a `~` or `^` tier would let apply pick a different params contract.
+    const result = exportOf(simpleStructure("a"), { a: ok({}) });
+
+    expect(result.ok && result.document.blocks[0].kind).toBe(
+      "@platforma-open/milaboratories.a.kind@1.4.2",
+    );
+  });
+
+  test("no `block` override is emitted", () => {
+    // The override pins an implementation against a kind version *range*. Export
+    // writes the exact version, so there is nothing left for it to pin.
+    const result = exportOf(simpleStructure("a"), { a: ok({}) });
+
+    expect(result.ok && "block" in result.document.blocks[0]).toBe(false);
+  });
+
+  test("empty params are written as `params: {}`, never as `params: null`", () => {
+    // Every entry carries a `params` key, because every block projects one. `null` would
+    // be a third thing the schema does not define, and a reader would have to guess.
+    const result = exportOf(simpleStructure("empty"), { empty: ok({}) });
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect(result.document.blocks[0].params).toEqual({});
+    expect(result.yaml).toContain("params: {}");
+    expect(result.yaml).not.toContain("params: null");
+  });
+
+  test("entry order is structure order", () => {
+    const result = exportOf(simpleStructure("samples", "mixcr", "browser"), {
+      samples: ok({}),
+      mixcr: ok({ input: createPlRef("samples", "reads") }),
+      browser: ok({ clones: createPlRef("mixcr", "clonotypes") }),
+    });
+
+    expect(result.ok && result.document.blocks.map((b) => b.id)).toEqual([
+      "samples",
+      "mixcr",
+      "browser",
+    ]);
+  });
+});
+
+describe("the YAML", () => {
+  test("round-trips through the import-side parser unchanged", () => {
+    // The one property that matters: export emits exactly what import parses.
+    const result = exportOf(simpleStructure("samples", "mixcr"), {
+      samples: ok({ dataset: "bulk-rna", replicates: [1, 2, 3] }),
+      mixcr: ok({
+        input: createPlRef("samples", "reads"),
+        species: "hsa",
+        nested: { deep: { flag: true, absent: null } },
+      }),
+    });
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect(parseProjectTemplateV1(YAML.parse(result.yaml))).toEqual(result.document);
+  });
+
+  test("opens with the schema marker", () => {
+    const result = exportOf(simpleStructure("a"), { a: ok({}) });
+
+    expect(result.ok && result.yaml.startsWith("schema: template-v1\n")).toBe(true);
+  });
+
+  test("nothing is line-folded", () => {
+    // A folded scalar still parses, but it makes a diff between two exported
+    // templates unreadable, which is most of the reason to emit YAML at all.
+    const long = "x".repeat(400);
+    const yaml = stringifyProjectTemplateV1({
+      schema: "template-v1",
+      blocks: [
+        {
+          id: "a",
+          kind: kindReferenceToSelectorReference(kindOf("a")),
+          params: { note: long },
+        },
+      ],
+    });
+
+    expect(yaml).toContain(long);
+  });
+
+  test("a reference is written exactly as the block holds it, with no marker", () => {
+    // Nothing in the file says this value is a reference, because nothing downstream needs
+    // told: the block that receives these params is the one that recognizes them. So the
+    // rendering is the block's own value and nothing else.
+    const result = exportOf(simpleStructure("samples", "mixcr"), {
+      samples: ok({}),
+      mixcr: ok({ input: createPlRef("samples", "reads") }),
+    });
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect(result.yaml).not.toContain("$ref");
+    expect(result.yaml).toContain("__isRef: true");
+    expect(result.yaml).toContain("blockId: samples");
+    expect(result.document.blocks[1].params).toEqual({
+      input: createPlRef("samples", "reads"),
+    });
+  });
+});
+
+describe("problems", () => {
+  test("a kind-less block fails the export and is named", () => {
+    // Every block published before kinds existed is in this state, so this is the
+    // common case today rather than an edge one.
+    const result = exportOf(
+      simpleStructure("modern", "legacy"),
+      { modern: ok({}), legacy: ok({}) },
+      (id) => (id === "legacy" ? undefined : kindOf(id)),
+    );
+
+    expect(result.ok).toBe(false);
+    if (result.ok) return;
+    expect(result.problems).toHaveLength(1);
+    expect(result.problems[0].blockId).toBe("legacy");
+    expect(result.problems[0].error).toContain("declares no kind");
+  });
+
+  test("a malformed stored kind reference is a problem, not a throw", () => {
+    const result = exportOf(
+      simpleStructure("a"),
+      { a: ok({}) },
+      () => "no-version-here" as BlockKindReference,
+    );
+
+    expect(result.ok).toBe(false);
+    if (result.ok) return;
+    expect(result.problems[0].blockId).toBe("a");
+    expect(result.problems[0].error).toContain("malformed");
+  });
+
+  test("a reference to a block that is not in the project is NOT caught", () => {
+    // Deleting a block only removes it from the structure and does not rewrite downstream
+    // args, so a live project holds such references routinely. Recognizing one would mean
+    // knowing which values are identifiers, which the engine deliberately does not — so the
+    // reference is written out and surfaces on apply as a block wired to nothing.
+    const result = exportOf(simpleStructure("survivor"), {
+      survivor: ok({ input: createPlRef("deleted-upstream", "reads") }),
+    });
+
+    expect(result.ok).toBe(true);
+  });
+
+  test("a forward reference is not caught here, and cannot be", () => {
+    // Reading which entries an entry references means reading its params, which only the
+    // block that wrote them can do. A project's structure is topological by construction, so
+    // this state is not reachable from a real project — it is pinned to say that the export
+    // asserts nothing about it rather than to bless it.
+    const result = exportOf(simpleStructure("downstream", "upstream"), {
+      downstream: ok({ input: createPlRef("upstream", "reads") }),
+      upstream: ok({}),
+    });
+
+    expect(result.ok).toBe(true);
+  });
+
+  test("walk problems and assembly problems are reported together, in one pass", () => {
+    // Fixing an export should take one round, not one round per broken block.
+    const result = exportOf(
+      simpleStructure("unreadable", "kindless", "fine"),
+      { kindless: ok({}), fine: ok({}) },
+      (id) => (id === "kindless" ? undefined : kindOf(id)),
+    );
+
+    expect(result.ok).toBe(false);
+    if (result.ok) return;
+    expect(result.problems.map((p) => p.blockId).sort()).toEqual(["kindless", "unreadable"]);
+  });
+
+  test("no partial YAML is produced when anything is wrong", () => {
+    // All-or-nothing: a template missing blocks the user never chose to leave out
+    // would still look like a successful export.
+    const result = exportOf(simpleStructure("a", "b"), { a: ok({}), b: ok({}) }, (id) =>
+      id === "b" ? undefined : kindOf(id),
+    );
+
+    expect(result).not.toHaveProperty("yaml");
+  });
+
+  test("an empty project exports an empty template", () => {
+    const result = exportOf(simpleStructure(), {});
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect(result.document.blocks).toEqual([]);
+    expect(parseProjectTemplateV1(YAML.parse(result.yaml))).toEqual(result.document);
+  });
+});
+
+describe("locationOf", () => {
+  test("a dev block's folder becomes a file URL", () => {
+    expect(locationOf(devSpec("/Users/dev/blocks/enter-numbers/block"))).toBe(
+      "file:///Users/dev/blocks/enter-numbers/block",
+    );
+  });
+
+  test("a path with a space is encoded, not written raw", () => {
+    // A raw space makes the value not a URI at all, so it would be rejected by the
+    // document parser on the way back in.
+    expect(locationOf(devSpec("/Users/dev/my blocks/x"))).toBe("file:///Users/dev/my%20blocks/x");
+  });
+
+  test("an npm-consumed pack's own URL is passed through untouched", () => {
+    // The block emitted this locator itself; rebuilding one from it could only lose
+    // information, and the pack directory is not derivable from the package root.
+    expect(
+      locationOf({
+        type: "from-pack-v2",
+        packUrl: "file:///repo/node_modules/@o/x/block-pack",
+        rootUrl: "file:///repo/node_modules/@o/x",
+      }),
+    ).toBe("file:///repo/node_modules/@o/x/block-pack");
+  });
+
+  test("a registry block gets no locator, which is what keeps it portable", () => {
+    expect(locationOf(registrySpec)).toBeUndefined();
+    expect(
+      locationOf({
+        type: "from-registry-v1",
+        registryUrl: "https://old",
+        id: { organization: "o", name: "n", version: "1.0.0" },
+      }),
+    ).toBeUndefined();
+  });
+
+  test("a legacy dev block gets none either", () => {
+    // It predates kinds, so it has no kind reference and the assembler refuses it
+    // before a locator would matter.
+    expect(locationOf({ type: "dev-v1", folder: "/Users/dev/old" })).toBeUndefined();
+  });
+});
+
+describe("locating a block installed from a folder", () => {
+  test("the entry carries the folder, and keeps the kind alongside it", () => {
+    const result = exportOf(
+      simpleStructure("samples"),
+      { samples: ok({ dataset: "bulk-rna" }) },
+      kindPerBlock,
+      () => devSpec("/Users/dev/blocks/samples/block"),
+    );
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect(result.document.blocks[0].location).toBe("file:///Users/dev/blocks/samples/block");
+    // The kind stays alongside: it is the params contract, not the locator.
+    expect(result.document.blocks[0].kind).toBe(
+      kindReferenceToSelectorReference(kindOf("samples")),
+    );
+  });
+
+  test("what is written survives the round trip through YAML", () => {
+    const result = exportOf(simpleStructure("samples"), { samples: ok({}) }, kindPerBlock, () =>
+      devSpec("/Users/dev/blocks/samples/block"),
+    );
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect(parseProjectTemplateV1(YAML.parse(result.yaml))).toEqual(result.document);
+  });
+
+  test("a registry-only project is portable and says nothing", () => {
+    const result = exportOf(simpleStructure("samples"), { samples: ok({}) });
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect("location" in result.document.blocks[0]).toBe(false);
+  });
+
+  test("a block whose origin is unknown is written without a locator", () => {
+    // Not a failure: an entry with no locator is the normal, portable form, and the
+    // kind is what the importer resolves.
+    const result = exportOf(
+      simpleStructure("samples"),
+      { samples: ok({}) },
+      kindPerBlock,
+      () => undefined,
+    );
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect("location" in result.document.blocks[0]).toBe(false);
+  });
+
+  test("a locator is written per block, and only for the located ones", () => {
+    const result = exportOf(
+      simpleStructure("a", "b", "c"),
+      { a: ok({}), b: ok({}), c: ok({}) },
+      kindPerBlock,
+      (id) => (id === "b" ? registrySpec : devSpec(`/Users/dev/blocks/${id}/block`)),
+    );
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect(result.document.blocks.map((e) => e.location)).toEqual([
+      "file:///Users/dev/blocks/a/block",
+      undefined,
+      "file:///Users/dev/blocks/c/block",
+    ]);
+  });
+});
+
+describe("assembleProjectTemplateV1", () => {
+  test("carries the walk's problems through unchanged", () => {
+    const { document, problems } = assembleProjectTemplateV1(
+      {
+        entries: [{ blockId: "a", params: {} }],
+        problems: [{ blockId: "ghost", error: "state unavailable" }],
+      },
+      kindPerBlock,
+      () => registrySpec,
+    );
+
+    expect(problems).toEqual([{ blockId: "ghost", error: "state unavailable" }]);
+    expect(document.blocks.map((b) => b.id)).toEqual(["a"]);
+  });
+});

package/src/model/template_serializer.ts

@@ -0,0 +1,232 @@
+import YAML from "yaml";
+import { pathToFileURL } from "node:url";
+import type {
+  BlockKindReference,
+  BlockKindSelectorReference,
+  BlockPackLocationReference,
+  ProjectTemplateV1,
+  ProjectTemplateV1Entry,
+} from "@milaboratories/pl-model-common";
+import {
+  PROJECT_TEMPLATE_SCHEMA_V1,
+  kindReferenceToSelectorReference,
+  parseProjectTemplateV1,
+} from "@milaboratories/pl-model-common";
+import type { BlockPackSpec } from "@milaboratories/pl-model-middle-layer";
+import type { ProjectStructure } from "./project_model";
+import type {
+  TemplateExportProblem,
+  TemplateExportWalk,
+  TemplateParamsResult,
+} from "./template_export";
+import { walkProjectForTemplateExport } from "./template_export";
+
+/** A block's exact kind reference, or `undefined` for a block that declares no kind. */
+export type BlockKindProvider = (blockId: string) => BlockKindReference | undefined;
+
+/**
+ * A block's origin spec — where the installed block came from — or `undefined` when
+ * it is not known for that block.
+ *
+ * The project stores this next to the kind reference, so both are read from the same
+ * place and neither costs an extra round-trip.
+ */
+export type BlockPackSpecProvider = (blockId: string) => BlockPackSpec | undefined;
+
+/** What the caller gets back for a whole project. */
+export type ProjectTemplateExportOutcome =
+  | {
+      readonly ok: true;
+      readonly yaml: string;
+      /** The document the YAML was rendered from, already validated. */
+      readonly document: ProjectTemplateV1;
+    }
+  | {
+      readonly ok: false;
+      /** Every block that stands in the way, not just the first. */
+      readonly problems: readonly TemplateExportProblem[];
+    };
+
+/**
+ * The `location` to write for a block installed from the filesystem, or `undefined`
+ * for one that came from a registry and therefore needs no locator.
+ *
+ * Both filesystem spec shapes are emitted, and they anchor at different directories
+ * — a dev block at its facade package, an npm-consumed one at its block-pack folder.
+ * The document does not distinguish them: one URI is written either way, and telling
+ * the two layouts apart is done by looking at what is actually there, by the side
+ * that has the filesystem anyway. Encoding the layout in the file instead would
+ * freeze today's two shapes into the format.
+ *
+ * A dev spec carries an OS path and is converted here, which also percent-encodes a
+ * path containing spaces. An npm-consumed spec already carries a `file:` URL and is
+ * passed through: it is the locator the block itself emitted, and reconstructing one
+ * from it could only lose information.
+ */
+export function locationOf(spec: BlockPackSpec): BlockPackLocationReference | undefined {
+  switch (spec.type) {
+    case "dev-v2":
+      return pathToFileURL(spec.folder).href as BlockPackLocationReference;
+    case "from-pack-v2":
+      return spec.packUrl as BlockPackLocationReference;
+    // A registry block is found by name, which is what makes the entry portable —
+    // writing where this machine happened to cache it would take that away. `dev-v1`
+    // predates kinds entirely, so such a block has no kind and never reaches here.
+    case "dev-v1":
+    case "from-registry-v1":
+    case "from-registry-v2":
+      return undefined;
+  }
+}
+
+/**
+ * Turn a project into a template document.
+ *
+ * Assembly is deliberately dull — the entry is the block's id, its widened kind
+ * reference, and the params the walk already collected. The interesting decisions
+ * were made upstream; what is left here is the two things only this layer can
+ * check, both of which produce problems rather than a broken file:
+ *
+ * - **A block with no kind cannot be written.** An entry's `kind` is required — it
+ *   is the params contract the entry is typed against — while a block's kind is
+ *   optional, so a block that predates kinds, or that uses the deprecated
+ *   kind-less model overload, has no legal entry. Reported per block. This is not
+ *   an edge case today: it is what most existing projects will hit until their
+ *   blocks are republished.
+ * - **References must point at an entry declared earlier.** Verbatim id reuse
+ *   means a reference to a deleted block survives into the file naming nothing:
+ *   deleting a block only removes it from the structure and does not rewrite
+ *   downstream args, so a live project holds such references routinely.
+ *
+ * `block` is never emitted. That override exists to pin an implementation against
+ * a kind's version range, and export always writes the exact version the block
+ * implements, so there is nothing left for it to pin.
+ *
+ * `location` IS emitted, for every block that was installed from the filesystem. Such
+ * a block is not in any registry, so the kind reference alone names nothing the
+ * importer could find, and a file that omitted the one usable answer would describe a
+ * project that cannot be recreated. It costs portability, and nothing says so: such a
+ * file is the debugging path, read by the developer who wrote it on the machine that
+ * wrote it.
+ *
+ * Problems from `walk` are carried through, so a caller can hand a walk straight
+ * in and get one combined list.
+ */
+export function assembleProjectTemplateV1(
+  walk: TemplateExportWalk,
+  kindProvider: BlockKindProvider,
+  specProvider: BlockPackSpecProvider,
+): { document: ProjectTemplateV1; problems: readonly TemplateExportProblem[] } {
+  const problems: TemplateExportProblem[] = [...walk.problems];
+  const blocks: ProjectTemplateV1Entry[] = [];
+
+  for (const entry of walk.entries) {
+    const kind = kindProvider(entry.blockId);
+
+    if (kind === undefined) {
+      problems.push({
+        blockId: entry.blockId,
+        error:
+          "Block declares no kind, so it cannot be written to a template: an entry's kind " +
+          "carries the params contract the entry is typed against",
+      });
+      continue;
+    }
+
+    let selector: BlockKindSelectorReference;
+    try {
+      // Widening validates, and therefore throws — which is why it happens here
+      // and not where the reference is read: every read site sits inside a
+      // recomputed project overview, where one malformed stored reference must not
+      // be able to break unrelated blocks.
+      selector = kindReferenceToSelectorReference(kind);
+    } catch (e) {
+      problems.push({
+        blockId: entry.blockId,
+        error: `Block's stored kind reference is malformed: ${e instanceof Error ? e.message : String(e)}`,
+      });
+      continue;
+    }
+
+    const spec = specProvider(entry.blockId);
+    const location = spec === undefined ? undefined : locationOf(spec);
+
+    blocks.push({
+      id: entry.blockId,
+      kind: selector,
+      params: entry.params,
+      ...(location !== undefined ? { location } : {}),
+    });
+  }
+
+  // References are not examined. A project's structure is topological by construction, so an
+  // entry cannot legally reference one below it — and checking would mean reading the params,
+  // which only the block that wrote them can do.
+  const document: ProjectTemplateV1 = { schema: PROJECT_TEMPLATE_SCHEMA_V1, blocks };
+
+  return { document, problems };
+}
+
+/**
+ * Render a template document to YAML text.
+ *
+ * Two non-default emitter settings, both about the file being read by someone
+ * else's code:
+ *
+ * - **No line folding.** A wrapped scalar still parses, but it makes a diff between
+ *   two exported templates unreadable, which is most of the reason to prefer YAML
+ *   over JSON here.
+ * - **Quote as if the reader were YAML 1.1**, while still parsing as 1.2. YAML 1.2
+ *   dropped `yes`/`no`/`on`/`off`/`y`/`n` as booleans and dropped sexagesimal
+ *   integers, so a 1.2 emitter leaves a params value of `"yes"` or `"1:30"` bare —
+ *   which a 1.1 reader (PyYAML's default, and Go's yaml.v2) turns into `true` and
+ *   `90`. A template is a contract for a second implementation, so the safe
+ *   combination is to quote against the stricter ruleset and read with the looser
+ *   one: a quoted scalar means the same thing under both. This adds no `%YAML`
+ *   directive — it only changes which scalars get quotes.
+ */
+export function stringifyProjectTemplateV1(document: ProjectTemplateV1): string {
+  return YAML.stringify(document, { lineWidth: 0, version: "1.1" });
+}
+
+/**
+ * Export a project as `template-v1` YAML, or report every reason it cannot be.
+ *
+ * All-or-nothing on purpose. A partial template silently drops blocks and the
+ * surviving entries may reference the dropped ones, so what looks like a
+ * successful export would produce a project missing pieces the user never chose
+ * to leave out. Reporting everything at once instead of failing on the first
+ * problem is the other half of that: fixing an export should take one pass.
+ *
+ * @param structure The project structure, which supplies both membership and order
+ * @param paramsProvider A block's derived template params, in live form
+ * @param kindProvider A block's exact kind reference, read from its stored config
+ * @param specProvider A block's origin spec, read from the same stored container
+ */
+export function exportProjectAsTemplateV1(
+  structure: ProjectStructure,
+  paramsProvider: (blockId: string) => TemplateParamsResult | undefined,
+  kindProvider: BlockKindProvider,
+  specProvider: BlockPackSpecProvider,
+): ProjectTemplateExportOutcome {
+  const walk = walkProjectForTemplateExport(structure, paramsProvider);
+  const { document, problems } = assembleProjectTemplateV1(walk, kindProvider, specProvider);
+
+  if (problems.length > 0) return { ok: false, problems };
+
+  // Export must emit exactly what import parses, so that is asserted on every
+  // export rather than only in tests — running the import-side parser over the
+  // document we are about to write is the cheapest possible proof of it. Nothing
+  // user-facing is expected to fail here: the kind grammar was checked by the
+  // widening above, params were checked to be a mapping by the walk, and the
+  // reference rules by the assembler. A throw means a bug in the assembler, with
+  // one known exception: a project structure holding two blocks with the same id,
+  // which is reachable through the mutator and produces duplicate entry ids.
+  parseProjectTemplateV1(document);
+
+  return {
+    ok: true,
+    yaml: stringifyProjectTemplateV1(document),
+    document,
+  };
+}