@milaboratories/pl-middle-layer 1.66.18 → 1.67.0

package/src/model/template_serializer_fixtures.test.ts

@@ -0,0 +1,274 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, test } from "vitest";
+import YAML from "yaml";
+import type { BlockKindReference, ColumnUniversalId } from "@milaboratories/pl-model-common";
+import {
+  createColumnFilteredId,
+  createGlobalPObjectId,
+  createPlRef,
+  parseProjectTemplateV1,
+} from "@milaboratories/pl-model-common";
+import type { ProjectStructure } from "./project_model";
+import type { TemplateParamsResult } from "./template_export";
+import { exportProjectAsTemplateV1 } from "./template_serializer";
+
+/**
+ * Golden-file tests for the exported document's TEXT.
+ *
+ * The sibling `template_serializer.test.ts` asserts the document field by field,
+ * which says nothing about how it is rendered. These pin the bytes: key order,
+ * indentation, how `params` nests, which scalars get quoted, and that nothing is
+ * line-folded. That is what "export emits exactly what import parses" rests on,
+ * and what makes a diff between two exported templates readable.
+ *
+ * Each expected file lives on disk rather than in a snapshot, deliberately. There
+ * is no snapshot infrastructure in this repo to fit into, and a snapshot carries an
+ * `-u` flag that rewrites the expectation without anyone reading it — for a file
+ * format promised to a second implementation, changing the expectation should be a
+ * deliberate edit that shows up in review. To update one, run the test, read the
+ * diff, and write the new content in by hand.
+ *
+ * Every fixture is also parsed back with the import-side parser, so a golden file
+ * can never be updated to something import cannot read.
+ */
+const FIXTURE_DIR = join(import.meta.dirname, "..", "..", "test_fixtures", "template-v1");
+
+/**
+ * Params as the block's callback hands them over — live, and untouched on the way out. What a
+ * block projects is what the file holds, so there is nothing between these values and the
+ * rendered bytes.
+ */
+const ok = (value: unknown): TemplateParamsResult => ({ value });
+
+const kind = (name: string, version: string) =>
+  `@platforma-open/milaboratories.${name}.kind@${version}` as BlockKindReference;
+
+function structureOf(...ids: string[]): ProjectStructure {
+  return {
+    groups: [
+      {
+        id: "g1",
+        label: "Main",
+        blocks: ids.map((id) => ({ id, label: id, renderingMode: "Heavy" })),
+      },
+    ],
+  };
+}
+
+type Fixture = {
+  /** File under `test_fixtures/template-v1/`, and the test's name. */
+  readonly file: string;
+  /** What each fixture is here to pin — one property per fixture, not a grab bag. */
+  readonly pins: string;
+  readonly structure: ProjectStructure;
+  readonly params: Record<string, TemplateParamsResult>;
+  readonly kinds: Record<string, BlockKindReference>;
+};
+
+const FIXTURES: readonly Fixture[] = [
+  {
+    file: "empty-project.yaml",
+    pins: "an empty project is still a valid document, not an empty file",
+    structure: structureOf(),
+    params: {},
+    kinds: {},
+  },
+  {
+    file: "minimal.yaml",
+    pins: "the schema marker comes first, and a block with nothing to project renders `params: {}`",
+    structure: structureOf("11111111-1111-4111-8111-111111111111"),
+    params: { "11111111-1111-4111-8111-111111111111": ok({}) },
+    kinds: { "11111111-1111-4111-8111-111111111111": kind("pool-explorer", "1.0.0") },
+  },
+  {
+    file: "linear-chain.yaml",
+    pins: "the canonical shape — three blocks wired in a chain, ids reused verbatim",
+    structure: structureOf(
+      "aaaaaaaa-0000-4000-8000-000000000001",
+      "bbbbbbbb-0000-4000-8000-000000000002",
+      "cccccccc-0000-4000-8000-000000000003",
+    ),
+    params: {
+      "aaaaaaaa-0000-4000-8000-000000000001": ok({ dataset: "bulk-rna" }),
+      "bbbbbbbb-0000-4000-8000-000000000002": ok({
+        input: createPlRef("aaaaaaaa-0000-4000-8000-000000000001", "reads"),
+        species: "hsa",
+      }),
+      "cccccccc-0000-4000-8000-000000000003": ok({
+        clonotypes: createPlRef("bbbbbbbb-0000-4000-8000-000000000002", "clonotypes"),
+      }),
+    },
+    kinds: {
+      "aaaaaaaa-0000-4000-8000-000000000001": kind("import-fastq", "2.1.0"),
+      "bbbbbbbb-0000-4000-8000-000000000002": kind("mixcr-clonotyping", "3.0.4"),
+      "cccccccc-0000-4000-8000-000000000003": kind("clonotype-browser", "1.2.10"),
+    },
+  },
+  {
+    file: "nested-params.yaml",
+    pins: "how nesting renders — objects in arrays, references at depth, an empty object",
+    structure: structureOf(
+      "dddddddd-0000-4000-8000-000000000004",
+      "ffffffff-0000-4000-8000-000000000006",
+    ),
+    params: {
+      "dddddddd-0000-4000-8000-000000000004": ok({}),
+      "ffffffff-0000-4000-8000-000000000006": ok({
+        thresholds: { min: 0.01, max: 1, exact: 0.5 },
+        steps: [
+          { name: "filter", enabled: true },
+          { name: "cluster", enabled: false, seed: 42 },
+        ],
+        // Two references to the same upstream, inside an array.
+        inputs: [
+          createPlRef("dddddddd-0000-4000-8000-000000000004", "a"),
+          createPlRef("dddddddd-0000-4000-8000-000000000004", "b"),
+        ],
+        advanced: {},
+        tags: [],
+        note: null,
+      }),
+    },
+    kinds: {
+      "dddddddd-0000-4000-8000-000000000004": kind("import-fastq", "2.1.0"),
+      "ffffffff-0000-4000-8000-000000000006": kind("clonotype-clustering", "0.4.1"),
+    },
+  },
+  {
+    file: "column-ids.yaml",
+    pins: "an identifier is written verbatim whatever its form — key object or canonical string",
+    structure: structureOf(
+      "aaaaaaaa-0000-4000-8000-000000000001",
+      "bbbbbbbb-0000-4000-8000-000000000002",
+    ),
+    params: {
+      "aaaaaaaa-0000-4000-8000-000000000001": ok({}),
+      "bbbbbbbb-0000-4000-8000-000000000002": ok({
+        // A filtered column id — a canonical JSON string with the block id buried inside it.
+        // It goes into the file as it came: repointing it is the receiving block's business,
+        // in its own bundle, and nothing on the way out parses it.
+        anchor: createColumnFilteredId({
+          source: createGlobalPObjectId(
+            "aaaaaaaa-0000-4000-8000-000000000001",
+            "clonotypes",
+          ) as ColumnUniversalId,
+          axisFilters: [[0, "IGH"]],
+        }),
+        // The same column in its object spelling. Both survive as written — no marker, no
+        // normalization, so a round trip cannot change which of the two a block gets back.
+        upstream: createPlRef("aaaaaaaa-0000-4000-8000-000000000001", "clonotypes"),
+      }),
+    },
+    kinds: {
+      "aaaaaaaa-0000-4000-8000-000000000001": kind("import-fastq", "2.1.0"),
+      "bbbbbbbb-0000-4000-8000-000000000002": kind("clonotype-browser", "1.2.10"),
+    },
+  },
+  {
+    file: "scalar-quoting.yaml",
+    pins: "strings that YAML would otherwise read back as something else stay strings",
+    structure: structureOf("eeeeeeee-0000-4000-8000-000000000005"),
+    params: {
+      "eeeeeeee-0000-4000-8000-000000000005": ok({
+        // Each of these is a real hazard: unquoted, some YAML reader turns them
+        // into a boolean, a number, a null, an alias, or a nested mapping.
+        // The first group is 1.1-only — a 1.2 emitter leaves them bare.
+        looksBoolean: "yes",
+        looksBooleanNo: "no",
+        looksBooleanOn: "on",
+        looksBooleanOff: "off",
+        looksBooleanY: "y",
+        looksBooleanN: "n",
+        looksSexagesimal: "1:30",
+        looksBooleanToo: "true",
+        looksNumeric: "1.0",
+        looksOctal: "0755",
+        looksNull: "null",
+        looksEmpty: "",
+        hasColon: "label: value",
+        startsWithAsterisk: "*anchor",
+        startsWithHash: "#not-a-comment",
+        startsWithDash: "-dash",
+        multiline: "first\nsecond",
+        unicode: "受容体 — αβ",
+        // Long enough to be folded at any default width; folding is switched off.
+        long: `${"x".repeat(200)} ${"y".repeat(200)}`,
+        actuallyBoolean: true,
+        actuallyNumeric: 1,
+      }),
+    },
+    kinds: { "eeeeeeee-0000-4000-8000-000000000005": kind("enter-numbers", "1.0.0") },
+  },
+];
+
+function exportFixture(fixture: Fixture) {
+  const result = exportProjectAsTemplateV1(
+    fixture.structure,
+    (blockId) => fixture.params[blockId],
+    (blockId) => fixture.kinds[blockId],
+    // The golden files describe portable templates, so every block is a registry one:
+    // a locator would pin them to whichever machine produced the fixture.
+    () => ({
+      type: "from-registry-v2",
+      registryUrl: "https://block.registry.platforma.bio/releases",
+      id: { organization: "milaboratories", name: "demo", version: "1.0.0" },
+      channel: "stable",
+    }),
+  );
+  if (!result.ok)
+    throw new Error(
+      `fixture '${fixture.file}' failed to export: ${result.problems.map((p) => p.error).join("; ")}`,
+    );
+  return result;
+}
+
+describe.each(FIXTURES)("$file", (fixture) => {
+  test(`renders as the golden file — ${fixture.pins}`, () => {
+    const expected = readFileSync(join(FIXTURE_DIR, fixture.file), "utf-8");
+
+    // Compared as text, not as a parsed value: the parsed comparison is the test
+    // below, and it cannot see formatting at all.
+    expect(exportFixture(fixture).yaml).toBe(expected);
+  });
+
+  test("the golden file parses back to the document it was rendered from", () => {
+    const result = exportFixture(fixture);
+    const fromDisk = readFileSync(join(FIXTURE_DIR, fixture.file), "utf-8");
+
+    expect(parseProjectTemplateV1(YAML.parse(fromDisk))).toEqual(result.document);
+  });
+});
+
+describe("what the golden files guard", () => {
+  test("no scalar is line-folded, however long", () => {
+    const yaml = exportFixture(FIXTURES.find((f) => f.file === "scalar-quoting.yaml")!).yaml;
+
+    // A folded scalar shows up as a continuation line; the value must stay on one.
+    const long = `${"x".repeat(200)} ${"y".repeat(200)}`;
+    expect(yaml).toContain(long);
+  });
+
+  test("scalars only YAML 1.1 would misread are quoted", () => {
+    // The emitter quotes against 1.1 rules even though we parse as 1.2, because the
+    // file is read by other implementations: PyYAML and Go's yaml.v2 default to 1.1,
+    // where a bare `yes` is `true` and a bare `1:30` is `90`. A 1.2 emitter leaves
+    // both bare, so this is the one formatting choice that is an interop bug rather
+    // than a matter of taste — and a round trip through our own parser cannot catch
+    // it, since 1.2 reads them back as strings either way.
+    const yaml = exportFixture(FIXTURES.find((f) => f.file === "scalar-quoting.yaml")!).yaml;
+
+    for (const bare of ["yes", "no", "on", "off", "y", "n", "1:30"]) {
+      expect(yaml).toContain(`: "${bare}"`);
+      expect(yaml).not.toContain(`: ${bare}\n`);
+    }
+  });
+
+  test("a document with no blocks is an empty sequence, not a null", () => {
+    // `blocks:` alone would parse as null and fail the schema, turning an empty
+    // project into an unreadable file.
+    const yaml = exportFixture(FIXTURES.find((f) => f.file === "empty-project.yaml")!).yaml;
+
+    expect(yaml).toContain("blocks: []");
+  });
+});

package/src/mutator/project-v3.test.ts

@@ -14,13 +14,13 @@ import path from "node:path";
 // These blocks use the new unified state format (state instead of args+uiState)
 const BPSpecEnterV3: BlockPackSpec = {
   type: "dev-v2",
-  // Navigate from lib/node/pl-middle-layer/src/mutator to etc/blocks/enter-numbers-v3/block
-  folder: path.resolve(__dirname, "../../../../../etc/blocks/enter-numbers-v3/block"),
+  // Navigate from lib/node/pl-middle-layer/src/mutator to etc/blocks/enter-numbers/block
+  folder: path.resolve(__dirname, "../../../../../etc/blocks/enter-numbers/block"),
 };
 
 const BPSpecSumV3: BlockPackSpec = {
   type: "dev-v2",
-  folder: path.resolve(__dirname, "../../../../../etc/blocks/sum-numbers-v3/block"),
+  folder: path.resolve(__dirname, "../../../../../etc/blocks/sum-numbers/block"),
 };
 
 test("v3 blocks: basic test with unified state", async () => {
@@ -34,7 +34,7 @@ test("v3 blocks: basic test with unified state", async () => {
       return await toGlobalResourceId(prjRef);
     });
 
-    // Add enter-numbers-v3 block with storageMode: 'fromModel'
+    // Add enter-numbers block with storageMode: 'fromModel'
     // Initial storage comes from VM, then we set desired state via setStates
     await pl.withWriteTx("AddEnterNumbersV3Block", async (tx) => {
       const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
@@ -68,7 +68,7 @@ test("v3 blocks: basic test with unified state", async () => {
       expect(JSON.parse(stateData).__data).toStrictEqual({ numbers: [1, 2, 3] });
     });
 
-    // Add second enter-numbers-v3 block
+    // Add second enter-numbers block
     await pl.withWriteTx("AddEnterNumbersV3Block2", async (tx) => {
       const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.addBlock(
@@ -93,7 +93,7 @@ test("v3 blocks: basic test with unified state", async () => {
       await tx.commit();
     });
 
-    // Add sum-numbers-v3 block that references both enter blocks
+    // Add sum-numbers block that references both enter blocks
     await pl.withWriteTx("AddSumNumbersV3Block", async (tx) => {
       const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.addBlock(
@@ -126,7 +126,7 @@ test("v3 blocks: basic test with unified state", async () => {
         .get(projectFieldName("enter1", "stagingOutput"))
         .then((r) => r.final());
       const all = await stagingOutput.getAllFinal();
-      // V3 enter-numbers-v3 block has prerun with numbersCount output
+      // V3 enter-numbers block has prerun with numbersCount output
       expect(Object.keys(all)).toContain("numbersCount");
     });
 
@@ -167,7 +167,7 @@ test("v3 blocks: prerunArgs skip test", async () => {
       return await toGlobalResourceId(prjRef);
     });
 
-    // Add enter-numbers-v3 block
+    // Add enter-numbers block
     await pl.withWriteTx("AddEnterNumbersV3Block", async (tx) => {
       const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.addBlock(
@@ -296,7 +296,7 @@ test("v3 blocks: migrateBlockPack preserves state and re-derives args and prerun
       return await toGlobalResourceId(prjRef);
     });
 
-    // Add enter-numbers-v3 block and set data with even numbers
+    // Add enter-numbers block and set data with even numbers
     await pl.withWriteTx("AddBlock", async (tx) => {
       const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.addBlock(
@@ -368,7 +368,7 @@ test("v3 blocks: migrateBlockPack with storage migration re-derives args and pre
       return await toGlobalResourceId(prjRef);
     });
 
-    // Add enter-numbers-v3 block with initial data
+    // Add enter-numbers block with initial data
     await pl.withWriteTx("AddBlock", async (tx) => {
       const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.addBlock(
@@ -559,7 +559,7 @@ test("v3 blocks: migrateBlockPack assigns author marker", async () => {
 // return undefined`) must not crash. Previously applyStorageAndDeriveArgs passed the
 // undefined value straight to createJsonFieldValue -> Buffer.from(undefined) -> ERR_INVALID_ARG_TYPE,
 // aborting the update-block-pack task (reported by AstraZeneca on mixcr-clonotyping 2.14 -> 2.20).
-// The sentinel input [777] makes the enter-numbers-v3 model's args() return undefined without throwing.
+// The sentinel input [777] makes the enter-numbers model's args() return undefined without throwing.
 test("v3 blocks: migrateBlockPack does not crash when args() returns undefined", async () => {
   const quickJs = await getQuickJS();
 

package/src/mutator/project.ts

@@ -82,7 +82,11 @@ import {
   UiError,
   BLOCK_STORAGE_FACADE_VERSION,
   type BlockConfig,
+  type BlockKindReference,
 } from "@platforma-sdk/model";
+import type { ProjectTemplateExportOutcome } from "../model/template_serializer";
+import { exportProjectAsTemplateV1 } from "../model/template_serializer";
+import type { TemplateParamsResult } from "../model/template_export";
 import { getDebugFlags } from "../debug";
 import type { BlockPackInfo } from "../model/block_pack";
 
@@ -112,6 +116,12 @@ interface BlockInfoState {
   readonly id: string;
   readonly fields: BlockFieldStates;
   blockConfig?: BlockConfig;
+  /**
+   * The block's kind reference, kept separately because `blockConfig` cannot carry
+   * it: `extractConfig` normalizes the render envelope and the kind sits at the
+   * container level, one level above. Undefined for a block that declares no kind.
+   */
+  blockKind?: BlockKindReference;
   blockPack?: BlockPackSpec;
 }
 
@@ -141,6 +151,8 @@ class BlockInfo {
     public readonly fields: BlockFieldStates,
     public readonly config: BlockConfig,
     public readonly source: BlockPackSpec,
+    /** See {@link BlockInfoState.blockKind}. */
+    public readonly kind: BlockKindReference | undefined,
     private readonly logger: MiLogger = new ConsoleLoggerAdapter(),
   ) {}
 
@@ -303,7 +315,25 @@ class BlockInfo {
  */
 /** Specification for creating a new block. Discriminated union based on `storageMode`. */
 export type NewBlockSpec =
-  | { storageMode: "fromModel"; blockPack: BlockPackSpecPrepared }
+  | {
+      storageMode: "fromModel";
+      blockPack: BlockPackSpecPrepared;
+      /**
+       * Storage to start the block with, instead of the model's own default.
+       *
+       * Only for a caller that produced it through the block's own model, which today
+       * means one seeding a block from a template entry's params: the model's
+       * params-to-storage initializer can reject those params, and it has to do so
+       * before anything in the transaction changes. Computing the storage first turns
+       * that rejection into a value the caller can report, and leaves this method with
+       * no failure path it did not already have.
+       *
+       * The result is the same storage the block would have written itself, so args
+       * derivation below is unchanged — which is why this is the `fromModel` mode with
+       * one input filled in rather than a mode of its own.
+       */
+      initialStorage?: string;
+    }
   | { storageMode: "legacy"; blockPack: BlockPackSpecPrepared; legacyState: string };
 
 const NoNewBlocks = (blockId: string) => {
@@ -516,6 +546,48 @@ export class ProjectMutator {
       };
   }
 
+  //
+  // Template export
+  //
+
+  /**
+   * Render this project as a `template-v1` YAML document, or report every reason it
+   * cannot be.
+   *
+   * Read-only: it derives params in the VM and touches no field, so a `withProject`
+   * wrapper sees `wasModified === false` and skips the commit.
+   *
+   * This is where the three providers the serializer needs come from, and all are
+   * only reachable here. A block's storage lives behind `BlockInfo`, which is
+   * populated by the loader's batched round-trips, and both its kind reference and
+   * its origin spec are read off the block-pack container during that same load —
+   * `config` cannot carry the kind, since `extractConfig` normalizes the render
+   * envelope one level below it, and the spec is not part of the config at all.
+   */
+  public exportAsTemplateV1(): ProjectTemplateExportOutcome {
+    return exportProjectAsTemplateV1(
+      this.struct,
+      (blockId) => this.deriveTemplateParams(blockId),
+      (blockId) => this.blockInfos.get(blockId)?.kind,
+      (blockId) => this.blockInfos.get(blockId)?.source,
+    );
+  }
+
+  private deriveTemplateParams(blockId: string): TemplateParamsResult | undefined {
+    const info = this.blockInfos.get(blockId);
+    // A block in the structure with no loaded info, or with no storage yet, has no
+    // state to project into params. Returning undefined makes the walk report it
+    // rather than quietly leaving the block out of the template.
+    if (info === undefined) return undefined;
+
+    const storageJson = info.blockStorageJson;
+    if (storageJson === undefined) return undefined;
+
+    const derived = this.projectHelper.deriveTemplateParamsFromStorage(info.config, storageJson);
+    if (derived.error !== undefined) return { error: derived.error.message };
+    return { value: derived.value };
+  }
+
   private getPendingProductionGraph(): BlockGraph {
     if (this.pendingProductionGraph === undefined)
       this.pendingProductionGraph = productionGraph(this.struct, (blockId) =>
@@ -1142,6 +1214,7 @@ export class ProjectMutator {
       {},
       extractConfig(spec.blockPack.config),
       spec.blockPack.source,
+      spec.blockPack.config.kind,
       this.projectHelper.logger,
     );
     this.blockInfos.set(blockId, info);
@@ -1164,8 +1237,11 @@ export class ProjectMutator {
     let storageToWrite: string;
 
     if (spec.storageMode === "fromModel") {
-      // Model API v2+: get initial storage and derive args from it
-      storageToWrite = this.projectHelper.getInitialStorageInVM(blockConfig);
+      // Model API v2+: get initial storage and derive args from it. A caller that
+      // already produced the storage — a template seeding a block from its params —
+      // passes it in, precisely so the VM call that could reject those params happens
+      // before the transaction is touched; see `initialStorage`.
+      storageToWrite = spec.initialStorage ?? this.projectHelper.getInitialStorageInVM(blockConfig);
 
       // Derive prerunArgs first — always derived independently of args validation
       prerunArgs = this.projectHelper.derivePrerunArgsFromStorage(blockConfig, storageToWrite);
@@ -1251,6 +1327,7 @@ export class ProjectMutator {
       {},
       originalBlockInfo.config,
       originalBlockInfo.source,
+      originalBlockInfo.kind,
       this.projectHelper.logger,
     );
 
@@ -1895,6 +1972,9 @@ export class ProjectMutator {
       const result = await response;
       const bpInfo = cachedDeserialize<BlockPackInfo>(notEmpty(result.data));
       info.blockConfig = extractConfig(bpInfo.config);
+      // Read off the container before `extractConfig`'s result replaces it as the
+      // only thing we keep. Costs no round-trip — `bpInfo` is already in hand.
+      info.blockKind = bpInfo.config.kind;
       info.blockPack = bpInfo.source;
     }
 
@@ -1926,10 +2006,17 @@ export class ProjectMutator {
     const blocksInLimboSet = new Set(blocksInLimbo);
 
     const blockInfos = new Map<string, BlockInfo>();
-    blockInfoStates.forEach(({ id, fields, blockConfig, blockPack }) =>
+    blockInfoStates.forEach(({ id, fields, blockConfig, blockKind, blockPack }) =>
       blockInfos.set(
         id,
-        new BlockInfo(id, fields, notEmpty(blockConfig), notEmpty(blockPack), projectHelper.logger),
+        new BlockInfo(
+          id,
+          fields,
+          notEmpty(blockConfig),
+          notEmpty(blockPack),
+          blockKind,
+          projectHelper.logger,
+        ),
       ),
     );