@platforma-sdk/tengo-builder 2.5.30 → 3.0.1

package/src/compiler/pack_limits.test.ts

@@ -0,0 +1,173 @@
+import type { CompiledTemplateV3 } from "@milaboratories/pl-model-backend";
+import { describe, expect, it } from "vitest";
+
+import type { FullArtifactName } from "./package";
+import {
+  MAX_TEMPLATE_PACK_BYTES_GZIPPED,
+  MAX_WASM_FILE_BYTES,
+  assertTemplatePackSize,
+  assertWasmFileSize,
+  collectWasmContributions,
+} from "./pack_limits";
+
+const wasmName: FullArtifactName = {
+  type: "wasm",
+  pkg: "@test/pkg",
+  id: "main",
+  version: "1.0.0",
+};
+
+describe("assertWasmFileSize", () => {
+  it("passes at exactly the cap", () => {
+    expect(() => assertWasmFileSize("/p/a.wasm", MAX_WASM_FILE_BYTES, wasmName)).not.toThrow();
+  });
+
+  it("throws above the cap with file path, full name, and shrink hint", () => {
+    let err: unknown;
+    try {
+      assertWasmFileSize("/p/a.wasm", MAX_WASM_FILE_BYTES + 1, wasmName);
+    } catch (e) {
+      err = e;
+    }
+    expect(err).toBeInstanceOf(Error);
+    const msg = (err as Error).message;
+    expect(msg).toContain("/p/a.wasm");
+    expect(msg).toContain("@test/pkg:main");
+    expect(msg).toContain("wasm-opt -Oz");
+    // points the reader at the backend source-of-truth
+    expect(msg).toContain("assertions.go");
+  });
+});
+
+describe("assertTemplatePackSize", () => {
+  it("passes at exactly the cap", () => {
+    expect(() =>
+      assertTemplatePackSize("@t/pkg:main 1.0.0", MAX_TEMPLATE_PACK_BYTES_GZIPPED, []),
+    ).not.toThrow();
+  });
+
+  it("lists WASM contributions sorted by size (largest first)", () => {
+    let err: unknown;
+    try {
+      assertTemplatePackSize("@t/pkg:main 1.0.0", MAX_TEMPLATE_PACK_BYTES_GZIPPED + 1, [
+        { name: "@small/wasm:main", rawBytes: 100 * 1024 },
+        { name: "@big/wasm:main", rawBytes: 1.5 * 1024 * 1024 },
+        { name: "@mid/wasm:main", rawBytes: 800 * 1024 },
+      ]);
+    } catch (e) {
+      err = e;
+    }
+    expect(err).toBeInstanceOf(Error);
+    const msg = (err as Error).message;
+    const big = msg.indexOf("@big/wasm:main");
+    const mid = msg.indexOf("@mid/wasm:main");
+    const small = msg.indexOf("@small/wasm:main");
+    // all three present
+    expect(big).toBeGreaterThan(-1);
+    expect(mid).toBeGreaterThan(-1);
+    expect(small).toBeGreaterThan(-1);
+    // ordered big > mid > small
+    expect(big).toBeLessThan(mid);
+    expect(mid).toBeLessThan(small);
+    expect(msg).toContain("@t/pkg:main 1.0.0");
+    expect(msg).toContain("config.go");
+  });
+
+  it("includes a fallback hint when no WASM artefacts contributed", () => {
+    let err: unknown;
+    try {
+      assertTemplatePackSize("@t/pkg:main 1.0.0", MAX_TEMPLATE_PACK_BYTES_GZIPPED + 1, []);
+    } catch (e) {
+      err = e;
+    }
+    expect((err as Error).message).toContain("no WASM artefacts");
+  });
+});
+
+describe("collectWasmContributions", () => {
+  // base64 length L → ~L * 3 / 4 raw bytes. Build a base64 string of the
+  // requested length so the test is independent of any specific WASM payload.
+  function base64OfLength(len: number): string {
+    return "A".repeat(len);
+  }
+
+  it("reports direct WASM dependencies with raw byte size derived from base64 length", () => {
+    const base64 = base64OfLength(400); // 400 base64 chars → 300 raw bytes
+    const tpl: CompiledTemplateV3 = {
+      type: "pl.tengo-template.v3",
+      hashToSource: { "hash-a": base64 },
+      template: {
+        name: "@t/pkg:main",
+        version: "1.0.0",
+        sourceHash: "tpl-hash",
+        libs: {},
+        templates: {},
+        software: {},
+        assets: {},
+        wasm: {
+          "@w/wasm:a": { name: "@w/wasm:a", version: "1.0.0", sourceHash: "hash-a" },
+        },
+      },
+    };
+    const got = collectWasmContributions(tpl);
+    expect(got).toEqual([{ name: "@w/wasm:a", rawBytes: 300 }]);
+  });
+
+  it("recurses into sub-templates and deduplicates by artefact name", () => {
+    const base64a = base64OfLength(800); // 600 raw
+    const base64b = base64OfLength(400); // 300 raw
+    const tpl: CompiledTemplateV3 = {
+      type: "pl.tengo-template.v3",
+      hashToSource: { "hash-a": base64a, "hash-b": base64b },
+      template: {
+        name: "@t/pkg:root",
+        version: "1.0.0",
+        sourceHash: "root-hash",
+        libs: {},
+        software: {},
+        assets: {},
+        wasm: {
+          "@w/wasm:a": { name: "@w/wasm:a", version: "1.0.0", sourceHash: "hash-a" },
+        },
+        templates: {
+          "@t/pkg:child": {
+            name: "@t/pkg:child",
+            version: "1.0.0",
+            sourceHash: "child-hash",
+            libs: {},
+            templates: {},
+            software: {},
+            assets: {},
+            wasm: {
+              // duplicate of the parent's WASM — must not be counted twice
+              "@w/wasm:a": { name: "@w/wasm:a", version: "1.0.0", sourceHash: "hash-a" },
+              "@w/wasm:b": { name: "@w/wasm:b", version: "1.0.0", sourceHash: "hash-b" },
+            },
+          },
+        },
+      },
+    };
+    const got = collectWasmContributions(tpl).sort((a, b) => a.name.localeCompare(b.name));
+    expect(got).toEqual([
+      { name: "@w/wasm:a", rawBytes: 600 },
+      { name: "@w/wasm:b", rawBytes: 300 },
+    ]);
+  });
+
+  it("returns an empty list when the template tree contains no WASM", () => {
+    const tpl: CompiledTemplateV3 = {
+      type: "pl.tengo-template.v3",
+      hashToSource: {},
+      template: {
+        name: "@t/pkg:main",
+        version: "1.0.0",
+        sourceHash: "tpl-hash",
+        libs: {},
+        templates: {},
+        software: {},
+        assets: {},
+      },
+    };
+    expect(collectWasmContributions(tpl)).toEqual([]);
+  });
+});

package/src/compiler/pack_limits.ts

@@ -0,0 +1,103 @@
+import type { CompiledTemplateV3, TemplateDataV3 } from "@milaboratories/pl-model-backend";
+
+import type { FullArtifactName } from "./package";
+import { fullNameToString } from "./package";
+
+// Build-time size guards mirroring the two backend caps that gate template
+// ingestion. Hardcoded because tengo-builder runs at block build time and has
+// no connection to a running backend; if pl ever changes the caps, update the
+// constants here in lockstep with the source-of-truth lines pointed at below.
+//
+// Source of truth (`core/pl` repo):
+//  - platform/core/transaction/assertions.go  : `maxResourceDataSize`
+//      3 MiB per value resource, checked on every CreateValue marshal.
+//      Hits the WasmV1 resource after the controller unpacks a template pack
+//      into its constituent libs/software/wasm resources.
+//  - controllers/workflow/pkg/cfg/config.go   : `TemplatePackSizeLimit`
+//      3.5 MiB per gzipped template pack, checked before the pack itself is
+//      stored as a TengoTemplatePackV1 value resource.
+
+// 2 MiB raw → ~2.67 MiB base64 + JSON wrapper → comfortably under the 3 MiB
+// per-value-resource cap with ~330 KiB headroom for the WasmV1Data wrapper
+// (Name, Version, Runtime, DefaultMemoryLimit, JSON syntax). WasmV1Data.Code
+// is a Go `[]byte`, which encoding/json serialises as base64 — that's where
+// the 4/3 expansion comes from.
+export const MAX_WASM_FILE_BYTES = 2 * 1024 * 1024;
+
+// 100 KiB below the backend's 3.5 MiB so blocks don't break on small
+// platform-side metadata overhead, or if pl tightens the limit slightly.
+export const MAX_TEMPLATE_PACK_BYTES_GZIPPED = 3.5 * 1024 * 1024 - 100 * 1024;
+
+function formatBytes(n: number): string {
+  if (n < 1024) return `${n} B`;
+  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KiB`;
+  return `${(n / 1024 / 1024).toFixed(2)} MiB`;
+}
+
+export function assertWasmFileSize(
+  file: string,
+  byteLength: number,
+  fullName: FullArtifactName,
+): void {
+  if (byteLength <= MAX_WASM_FILE_BYTES) return;
+  throw new Error(
+    `WASM artefact ${fullNameToString(fullName)} from ${file} is ${formatBytes(byteLength)}, ` +
+      `which exceeds tengo-builder's ${formatBytes(MAX_WASM_FILE_BYTES)} per-WASM cap. ` +
+      `The backend stores each WASM as a single value resource (3 MiB hard cap after base64+JSON ` +
+      `marshal — see maxResourceDataSize in pl: platform/core/transaction/assertions.go). ` +
+      `Strip debug info or run \`wasm-opt -Oz\` to shrink it.`,
+  );
+}
+
+export interface WasmContribution {
+  /** Artefact name as stored under TemplateDataV3.wasm (e.g. "@pkg:id"). */
+  readonly name: string;
+  /** Raw WASM bytes — decoded from the base64-encoded source in hashToSource. */
+  readonly rawBytes: number;
+}
+
+/**
+ * Walks a compiled template tree and reports every WASM artefact embedded
+ * directly or transitively. Used only by the too-large-pack error message —
+ * never on the success path. Deduplicates by artefact name (sub-templates can
+ * import the same WASM via different aliases and we want to attribute weight
+ * only once).
+ */
+export function collectWasmContributions(tpl: CompiledTemplateV3): WasmContribution[] {
+  const seen = new Map<string, WasmContribution>();
+  function walk(t: TemplateDataV3) {
+    for (const [name, ref] of Object.entries(t.wasm ?? {})) {
+      if (seen.has(name)) continue;
+      const base64 = tpl.hashToSource[ref.sourceHash];
+      // base64 encodes 3 raw bytes per 4 chars; the off-by-padding is at most
+      // 2 bytes per artefact — fine for a human-readable size breakdown.
+      const rawBytes = base64 != null ? Math.floor((base64.length * 3) / 4) : 0;
+      seen.set(name, { name, rawBytes });
+    }
+    for (const sub of Object.values(t.templates ?? {})) walk(sub);
+  }
+  walk(tpl.template);
+  return [...seen.values()];
+}
+
+export function assertTemplatePackSize(
+  templateName: string,
+  gzippedByteLength: number,
+  wasmContributions: readonly WasmContribution[],
+): void {
+  if (gzippedByteLength <= MAX_TEMPLATE_PACK_BYTES_GZIPPED) return;
+  const ranked = [...wasmContributions].sort((a, b) => b.rawBytes - a.rawBytes);
+  const breakdown =
+    ranked.length === 0
+      ? "\n  (no WASM artefacts in this pack — heavy libs, software descriptors, or sub-templates are the likely cause.)"
+      : "\nWASM artefacts in this pack (raw bytes):\n" +
+        ranked.map((w) => `  - ${w.name} — ${formatBytes(w.rawBytes)}`).join("\n");
+  throw new Error(
+    `Template pack ${templateName} is ${formatBytes(gzippedByteLength)} gzipped, ` +
+      `which exceeds tengo-builder's ${formatBytes(MAX_TEMPLATE_PACK_BYTES_GZIPPED)} per-pack cap. ` +
+      `The backend caps gzipped packs at 3.5 MiB (see TemplatePackSizeLimit in pl: ` +
+      `controllers/workflow/pkg/cfg/config.go).` +
+      breakdown +
+      "\nShrink WASM artefacts (wasm-opt -Oz), or split this template into smaller pieces.",
+  );
+}

package/src/compiler/package.ts

@@ -27,7 +27,7 @@ export type CompilerOption = {
   args: string[];
 };
 
-export type ArtifactType = "library" | "template" | "test" | "software" | "asset";
+export type ArtifactType = "library" | "template" | "test" | "software" | "asset" | "wasm";
 
 /** Artifact Name including package version */
 export interface FullArtifactName {

package/src/compiler/source.test.ts

@@ -126,6 +126,36 @@ describe("import statements", () => {
     const trickyCasesSrc = parseSource(logger, "dist", testTrickyCasesSrc, stubTplName, true);
     expect(trickyCasesSrc.src).toEqual(testTrickyCasesNormalized);
   });
+
+  test("assets.importWasm produces a wasm dependency", () => {
+    const logger = createLogger("error");
+
+    const wasmImport = `
+      assets := import("@platforma-sdk/workflow-tengo:assets")
+      wasm := assets.importWasm("@milaboratories/pframes-rs-wasip2:main")
+    `;
+
+    const parsed = parseSource(logger, "dist", wasmImport, stubTplName, true);
+    expect(parsed.dependencies).toContainEqual({
+      type: "wasm",
+      pkg: "@milaboratories/pframes-rs-wasip2",
+      id: "main",
+    });
+  });
+
+  test("assets.importWasm with variable id is rejected", () => {
+    const logger = createLogger("info");
+
+    const wasmByVar = `
+      assets := import("@platforma-sdk/workflow-tengo:assets")
+      wasmId := "@milaboratories/pframes-rs-wasip2:main"
+      wasm := assets.importWasm(wasmId)
+    `;
+
+    expect(() => parseSource(logger, "dist", wasmByVar, stubTplName, true)).toThrow(
+      "variables are not allowed",
+    );
+  });
 });
 
 describe("parseSingleSourceLine", () => {

package/src/compiler/source.ts

@@ -42,6 +42,8 @@ const newImportSoftwareDetector = (moduleName: string) =>
 const newImportAssetRE = (moduleName: string) => functionCallRE(moduleName, "importAsset");
 const newImportAssetDetector = (moduleName: string) =>
   functionCallLikeRE(moduleName, "importAsset");
+const newImportWasmRE = (moduleName: string) => functionCallRE(moduleName, "importWasm");
+const newImportWasmDetector = (moduleName: string) => functionCallLikeRE(moduleName, "importWasm");
 
 const emptyLineRE = /^\s*$/;
 const compilerOptionRE = /^\/\/tengo:[\w]/;
@@ -369,11 +371,13 @@ function processModuleImport(
         ["template", newImportTemplateRE(iInfo.alias)],
         ["software", newImportSoftwareRE(iInfo.alias)],
         ["asset", newImportAssetRE(iInfo.alias)],
+        ["wasm", newImportWasmRE(iInfo.alias)],
       ]);
       context.importLikeREs.set(iInfo.module, [
         ["template", newImportTemplateDetector(iInfo.alias)],
         ["software", newImportSoftwareDetector(iInfo.alias)],
         ["asset", newImportAssetDetector(iInfo.alias)],
+        ["wasm", newImportWasmDetector(iInfo.alias)],
       ]);
     }
   }

package/src/shared/basecmd.ts

@@ -31,7 +31,7 @@ export const ArtifactTypeFlag = {
   type: Flags.string({
     aliases: ["t"],
     description: "artifact type",
-    options: ["all", "library", "template", "test", "software", "asset"],
+    options: ["all", "library", "template", "test", "software", "asset", "wasm"],
     default: "all",
   }),
 };

package/src/shared/dump.ts

@@ -88,6 +88,15 @@ export function dumpArtifacts(
     }
   }
 
+  // Wasm
+
+  if (!aType || aType === "wasm") {
+    for (const wasm of compiler.allWasm()) {
+      logger.debug(`Dumping to pl-tester: ${typedArtifactNameToString(wasm.fullName)}`);
+      stream.write(JSON.stringify(wasm) + "\n");
+    }
+  }
+
   // Tests
 
   if (!aType || aType === "test") {