@platforma-sdk/tengo-builder 2.5.30 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/compiler/compiler.cjs +48 -1
- package/dist/compiler/compiler.cjs.map +1 -1
- package/dist/compiler/compiler.js +47 -1
- package/dist/compiler/compiler.js.map +1 -1
- package/dist/compiler/main.cjs +132 -6
- package/dist/compiler/main.cjs.map +1 -1
- package/dist/compiler/main.js +132 -6
- package/dist/compiler/main.js.map +1 -1
- package/dist/compiler/pack_limits.cjs +49 -0
- package/dist/compiler/pack_limits.cjs.map +1 -0
- package/dist/compiler/pack_limits.js +47 -0
- package/dist/compiler/pack_limits.js.map +1 -0
- package/dist/compiler/package.cjs.map +1 -1
- package/dist/compiler/package.js.map +1 -1
- package/dist/compiler/source.cjs +6 -2
- package/dist/compiler/source.cjs.map +1 -1
- package/dist/compiler/source.js +6 -2
- package/dist/compiler/source.js.map +1 -1
- package/dist/shared/basecmd.cjs +2 -1
- package/dist/shared/basecmd.cjs.map +1 -1
- package/dist/shared/basecmd.js +2 -1
- package/dist/shared/basecmd.js.map +1 -1
- package/dist/shared/dump.cjs +4 -0
- package/dist/shared/dump.cjs.map +1 -1
- package/dist/shared/dump.js +4 -0
- package/dist/shared/dump.js.map +1 -1
- package/package.json +2 -2
- package/src/compiler/compiler.test.ts +91 -0
- package/src/compiler/compiler.ts +72 -1
- package/src/compiler/main.ts +217 -11
- package/src/compiler/pack_limits.test.ts +173 -0
- package/src/compiler/pack_limits.ts +103 -0
- package/src/compiler/package.ts +1 -1
- package/src/compiler/source.test.ts +30 -0
- package/src/compiler/source.ts +4 -0
- package/src/shared/basecmd.ts +1 -1
- package/src/shared/dump.ts +9 -0
package/src/compiler/main.ts
CHANGED
|
@@ -13,6 +13,18 @@ import { newTemplateFromContent, templateToSource } from "./template";
|
|
|
13
13
|
import type winston from "winston";
|
|
14
14
|
import { tryResolve, tryResolveOrError } from "@milaboratories/resolve-helper";
|
|
15
15
|
import { serializeTemplate } from "@milaboratories/pl-model-backend";
|
|
16
|
+
import {
|
|
17
|
+
assertTemplatePackSize,
|
|
18
|
+
assertWasmFileSize,
|
|
19
|
+
collectWasmContributions,
|
|
20
|
+
} from "./pack_limits";
|
|
21
|
+
|
|
22
|
+
// Convention: wasm component packages declare their wasm under a "wasm" condition
|
|
23
|
+
// in package.json `exports`. Subpath "." maps to the tengo id "main"; subpath
|
|
24
|
+
// "./foo" maps to id "foo". Other shapes of `exports` (string-valued, condition-only)
|
|
25
|
+
// don't produce wasm artifacts.
|
|
26
|
+
const wasmExportCondition = "wasm";
|
|
27
|
+
const defaultWasmExportId = "main";
|
|
16
28
|
|
|
17
29
|
interface PackageId {
|
|
18
30
|
/** Package name from package.json */
|
|
@@ -44,6 +56,7 @@ const compiledTplSuffix = ".plj.gz";
|
|
|
44
56
|
const compiledLibSuffix = ".lib.tengo";
|
|
45
57
|
const compiledSoftwareSuffix = ".sw.json";
|
|
46
58
|
const compiledAssetSuffix = ".as.json";
|
|
59
|
+
const compiledWasmSuffix = ".wasm";
|
|
47
60
|
|
|
48
61
|
// We need to keep track of dependencies for correct tgo-test CLI utility configuraiton.
|
|
49
62
|
// It is much simpler to do this here, than duplicate all tle logic regarding dependencies
|
|
@@ -79,15 +92,32 @@ function resolvePackageJsonPackage(root: string, depPackageName: string): string
|
|
|
79
92
|
|
|
80
93
|
// Second approach: trying to find package.json in the package dir.
|
|
81
94
|
const resolved2 = tryResolveOrError(root, `${depPackageName}/package.json`);
|
|
82
|
-
if (resolved2.result
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
95
|
+
if (resolved2.result !== undefined) {
|
|
96
|
+
return resolved2.result;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Third approach: some packages (wasm-only components) don't expose any
|
|
100
|
+
// import-resolvable entry point — neither a default export nor `./package.json`.
|
|
101
|
+
// pnpm still installs them at <root>/node_modules/<depPackageName>/, so walk
|
|
102
|
+
// the parent chain and check that path directly.
|
|
103
|
+
if (resolved2.err === "ERR_PACKAGE_PATH_NOT_EXPORTED") {
|
|
104
|
+
const fromNodeModules = findInNodeModules(root, depPackageName);
|
|
105
|
+
if (fromNodeModules !== undefined) return fromNodeModules;
|
|
106
|
+
// tolerating not-exported package.json for dev dependencies even if no
|
|
107
|
+
// physical node_modules copy is found.
|
|
108
|
+
return undefined;
|
|
89
109
|
}
|
|
90
|
-
|
|
110
|
+
throw new Error(
|
|
111
|
+
`Can't resolve package.json for package ${depPackageName ?? "."} relative to ${root}`,
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function findInNodeModules(root: string, depPackageName: string): string | undefined {
|
|
116
|
+
const candidate = path.join(root, "node_modules", depPackageName, "package.json");
|
|
117
|
+
if (pathType(candidate) === "file") return candidate;
|
|
118
|
+
const parent = path.dirname(root);
|
|
119
|
+
if (parent === root) return undefined;
|
|
120
|
+
return findInNodeModules(parent, depPackageName);
|
|
91
121
|
}
|
|
92
122
|
|
|
93
123
|
export function resolvePackageJsonRoot(root: string): string {
|
|
@@ -165,6 +195,10 @@ function resolveAssetsDst(mode: CompileMode, root: string) {
|
|
|
165
195
|
return path.resolve(root, mode, "tengo", "asset");
|
|
166
196
|
}
|
|
167
197
|
|
|
198
|
+
function resolveWasmDst(mode: CompileMode, root: string) {
|
|
199
|
+
return path.resolve(root, mode, "tengo", "wasm");
|
|
200
|
+
}
|
|
201
|
+
|
|
168
202
|
function loadDependencies(
|
|
169
203
|
logger: winston.Logger,
|
|
170
204
|
compiler: TengoTemplateCompiler,
|
|
@@ -181,13 +215,23 @@ function loadDependencies(
|
|
|
181
215
|
const tplDistFolder = resolveTemplatesDst("dist", packageInfo.root);
|
|
182
216
|
const softwareDistFolder = resolveSoftwareDst("dist", packageInfo.root);
|
|
183
217
|
const assetDistFolder = resolveAssetsDst("dist", packageInfo.root);
|
|
218
|
+
const wasmDistFolder = resolveWasmDst("dist", packageInfo.root);
|
|
184
219
|
|
|
185
220
|
const libDistExists = pathType(libDistFolder) === "dir";
|
|
186
221
|
const tplDistExists = pathType(tplDistFolder) === "dir";
|
|
187
222
|
const softwareDistExists = pathType(softwareDistFolder) === "dir";
|
|
188
223
|
const assetDistExists = pathType(assetDistFolder) === "dir";
|
|
189
|
-
|
|
190
|
-
|
|
224
|
+
const wasmDistExists = pathType(wasmDistFolder) === "dir";
|
|
225
|
+
const wasmExports = collectWasmExportPaths(packageInfo.root);
|
|
226
|
+
|
|
227
|
+
if (
|
|
228
|
+
!libDistExists &&
|
|
229
|
+
!tplDistExists &&
|
|
230
|
+
!softwareDistExists &&
|
|
231
|
+
!assetDistExists &&
|
|
232
|
+
!wasmDistExists &&
|
|
233
|
+
wasmExports.length === 0
|
|
234
|
+
)
|
|
191
235
|
// if neither of tengo-specific folders detected, skipping package
|
|
192
236
|
return;
|
|
193
237
|
|
|
@@ -208,6 +252,14 @@ function loadDependencies(
|
|
|
208
252
|
if (assetDistExists) {
|
|
209
253
|
loadAssetsFromDir(logger, packageId, "dist", assetDistFolder, compiler);
|
|
210
254
|
}
|
|
255
|
+
|
|
256
|
+
if (wasmDistExists) {
|
|
257
|
+
loadWasmFromDir(logger, packageId, "dist", wasmDistFolder, compiler);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
for (const wasmExport of wasmExports) {
|
|
261
|
+
loadWasmFromExport(logger, packageId, "dist", packageInfo.root, wasmExport, compiler);
|
|
262
|
+
}
|
|
211
263
|
}
|
|
212
264
|
|
|
213
265
|
function loadLibsFromDir(
|
|
@@ -313,6 +365,142 @@ function loadAssetsFromDir(
|
|
|
313
365
|
}
|
|
314
366
|
}
|
|
315
367
|
|
|
368
|
+
function loadWasmFromDir(
|
|
369
|
+
logger: winston.Logger,
|
|
370
|
+
packageId: PackageId,
|
|
371
|
+
mode: CompileMode,
|
|
372
|
+
folder: string,
|
|
373
|
+
compiler: TengoTemplateCompiler,
|
|
374
|
+
) {
|
|
375
|
+
for (const f of fs.readdirSync(folder)) {
|
|
376
|
+
const file = path.resolve(folder, f);
|
|
377
|
+
if (!f.endsWith(compiledWasmSuffix))
|
|
378
|
+
throw new Error(`unexpected file in 'wasm' folder: ${file}`);
|
|
379
|
+
const fullName: FullArtifactName = {
|
|
380
|
+
type: "wasm",
|
|
381
|
+
pkg: packageId.name,
|
|
382
|
+
id: f.slice(0, f.length - compiledWasmSuffix.length),
|
|
383
|
+
version: packageId.version,
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
addWasmFile(logger, mode, file, fullName, compiler);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
interface WasmExport {
|
|
391
|
+
id: string;
|
|
392
|
+
/** wasm file path, relative to the package root */
|
|
393
|
+
relPath: string;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
function loadWasmFromExport(
|
|
397
|
+
logger: winston.Logger,
|
|
398
|
+
packageId: PackageId,
|
|
399
|
+
mode: CompileMode,
|
|
400
|
+
packageRoot: string,
|
|
401
|
+
wasmExport: WasmExport,
|
|
402
|
+
compiler: TengoTemplateCompiler,
|
|
403
|
+
) {
|
|
404
|
+
const file = path.resolve(packageRoot, wasmExport.relPath);
|
|
405
|
+
if (pathType(file) !== "file") {
|
|
406
|
+
throw new Error(
|
|
407
|
+
`wasm export "${wasmExport.id}" points to missing file: ${file} (package ${packageId.name})`,
|
|
408
|
+
);
|
|
409
|
+
}
|
|
410
|
+
const fullName: FullArtifactName = {
|
|
411
|
+
type: "wasm",
|
|
412
|
+
pkg: packageId.name,
|
|
413
|
+
id: wasmExport.id,
|
|
414
|
+
version: packageId.version,
|
|
415
|
+
};
|
|
416
|
+
|
|
417
|
+
addWasmFile(logger, mode, file, fullName, compiler);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
function addWasmFile(
|
|
421
|
+
logger: winston.Logger,
|
|
422
|
+
mode: CompileMode,
|
|
423
|
+
file: string,
|
|
424
|
+
fullName: FullArtifactName,
|
|
425
|
+
compiler: TengoTemplateCompiler,
|
|
426
|
+
) {
|
|
427
|
+
// wasm bytes are binary, but hashToSource carries strings only. We base64-encode
|
|
428
|
+
// the bytes; the sourceHash is sha256 of the base64 string so it stays consistent
|
|
429
|
+
// with the existing `hashToSource[hash] === stored value` invariant.
|
|
430
|
+
const bytes = fs.readFileSync(file);
|
|
431
|
+
assertWasmFileSize(file, bytes.length, fullName);
|
|
432
|
+
const source = bytes.toString("base64");
|
|
433
|
+
const wasm = new ArtifactSource(mode, fullName, getSha256(source), source, file, [], []);
|
|
434
|
+
|
|
435
|
+
logger.debug(`Adding dependency ${fullNameToString(fullName)} from ${file}`);
|
|
436
|
+
compiler.addWasm(wasm);
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* Discover wasm artifacts declared by a package's `package.json` via the
|
|
441
|
+
* node `exports` conditional-exports map.
|
|
442
|
+
*
|
|
443
|
+
* Convention: a package that wants to ship one or more wasm components to
|
|
444
|
+
* tengo-builder publishes them under a `"wasm"` condition keyed by an
|
|
445
|
+
* `exports` subpath. The subpath becomes the wasm artifact id (`.` →
|
|
446
|
+
* `main`, `./foo` → `foo`). Example shape we look for:
|
|
447
|
+
*
|
|
448
|
+
* {
|
|
449
|
+
* "exports": {
|
|
450
|
+
* ".": {
|
|
451
|
+
* "wasm": "./dist/pframes_rs_wasip2.wasm", // ← discovered
|
|
452
|
+
* "wit": "./wit/world.wit"
|
|
453
|
+
* },
|
|
454
|
+
* "./extra": {
|
|
455
|
+
* "wasm": "./dist/extra.wasm" // ← discovered as id "extra"
|
|
456
|
+
* }
|
|
457
|
+
* }
|
|
458
|
+
* }
|
|
459
|
+
*
|
|
460
|
+
* Why raw traversal instead of Zod / a typed schema:
|
|
461
|
+
* 1. `package.json#exports` is a community convention with no fixed shape
|
|
462
|
+
* — any combination of strings, objects, arrays, and conditions is
|
|
463
|
+
* legal node syntax. A strict schema would either reject valid
|
|
464
|
+
* packages or have to admit `unknown` for half of it anyway.
|
|
465
|
+
* 2. We care about exactly one condition (`wasm`) on subpath entries,
|
|
466
|
+
* not the rest of the exports tree. A targeted walk reads more
|
|
467
|
+
* directly than a schema that re-describes the conditional-exports
|
|
468
|
+
* grammar.
|
|
469
|
+
* 3. Errors here should always mean "this package contributes no wasm";
|
|
470
|
+
* never crash the build. Every check returns `[]` or `continue`s.
|
|
471
|
+
*
|
|
472
|
+
* Returns the discovered wasm exports; empty when the file is missing,
|
|
473
|
+
* malformed, has no `exports`, or has no subpath with a `"wasm"`
|
|
474
|
+
* condition pointing at a string path.
|
|
475
|
+
*/
|
|
476
|
+
function collectWasmExportPaths(packageRoot: string): WasmExport[] {
|
|
477
|
+
const packageJsonPath = path.join(packageRoot, "package.json");
|
|
478
|
+
if (pathType(packageJsonPath) !== "file") return [];
|
|
479
|
+
|
|
480
|
+
let raw: unknown;
|
|
481
|
+
try {
|
|
482
|
+
raw = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8"));
|
|
483
|
+
} catch {
|
|
484
|
+
return [];
|
|
485
|
+
}
|
|
486
|
+
if (typeof raw !== "object" || raw === null) return [];
|
|
487
|
+
|
|
488
|
+
const exportsField = (raw as { exports?: unknown }).exports;
|
|
489
|
+
if (typeof exportsField !== "object" || exportsField === null) return [];
|
|
490
|
+
|
|
491
|
+
const result: WasmExport[] = [];
|
|
492
|
+
for (const [subpath, value] of Object.entries(exportsField as Record<string, unknown>)) {
|
|
493
|
+
if (!subpath.startsWith(".")) continue; // skip condition-only entries
|
|
494
|
+
if (typeof value !== "object" || value === null) continue;
|
|
495
|
+
const wasmPath = (value as Record<string, unknown>)[wasmExportCondition];
|
|
496
|
+
if (typeof wasmPath !== "string") continue;
|
|
497
|
+
const id = subpath === "." ? defaultWasmExportId : subpath.replace(/^\.\/?/, "");
|
|
498
|
+
if (!id) continue;
|
|
499
|
+
result.push({ id, relPath: wasmPath });
|
|
500
|
+
}
|
|
501
|
+
return result;
|
|
502
|
+
}
|
|
503
|
+
|
|
316
504
|
export function parseSources(
|
|
317
505
|
logger: winston.Logger,
|
|
318
506
|
packageId: PackageId,
|
|
@@ -468,8 +656,14 @@ export function savePacks(logger: winston.Logger, compiled: TemplatesAndLibs, mo
|
|
|
468
656
|
fs.mkdirSync(tplOutput, { recursive: true });
|
|
469
657
|
for (const tpl of compiled.templates) {
|
|
470
658
|
const file = path.resolve(tplOutput, tpl.fullName.id + compiledTplSuffix);
|
|
659
|
+
const packed = serializeTemplate(tpl.data);
|
|
660
|
+
assertTemplatePackSize(
|
|
661
|
+
fullNameToString(tpl.fullName),
|
|
662
|
+
packed.byteLength,
|
|
663
|
+
collectWasmContributions(tpl.data),
|
|
664
|
+
);
|
|
471
665
|
logger.info(` - writing ${file}`);
|
|
472
|
-
fs.writeFileSync(file,
|
|
666
|
+
fs.writeFileSync(file, packed);
|
|
473
667
|
}
|
|
474
668
|
}
|
|
475
669
|
|
|
@@ -494,4 +688,16 @@ export function savePacks(logger: winston.Logger, compiled: TemplatesAndLibs, mo
|
|
|
494
688
|
fs.writeFileSync(file, sw.src);
|
|
495
689
|
}
|
|
496
690
|
}
|
|
691
|
+
|
|
692
|
+
// writing wasm
|
|
693
|
+
if (compiled.wasm.length > 0) {
|
|
694
|
+
const wasmOutput = resolveWasmDst(mode, ".");
|
|
695
|
+
fs.mkdirSync(wasmOutput, { recursive: true });
|
|
696
|
+
for (const wasm of compiled.wasm) {
|
|
697
|
+
const file = path.resolve(wasmOutput, wasm.fullName.id + compiledWasmSuffix);
|
|
698
|
+
logger.info(` - writing ${file}`);
|
|
699
|
+
// wasm.src holds base64-encoded bytes; decode back to raw bytes on disk
|
|
700
|
+
fs.writeFileSync(file, Buffer.from(wasm.src, "base64"));
|
|
701
|
+
}
|
|
702
|
+
}
|
|
497
703
|
}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import type { CompiledTemplateV3 } from "@milaboratories/pl-model-backend";
|
|
2
|
+
import { describe, expect, it } from "vitest";
|
|
3
|
+
|
|
4
|
+
import type { FullArtifactName } from "./package";
|
|
5
|
+
import {
|
|
6
|
+
MAX_TEMPLATE_PACK_BYTES_GZIPPED,
|
|
7
|
+
MAX_WASM_FILE_BYTES,
|
|
8
|
+
assertTemplatePackSize,
|
|
9
|
+
assertWasmFileSize,
|
|
10
|
+
collectWasmContributions,
|
|
11
|
+
} from "./pack_limits";
|
|
12
|
+
|
|
13
|
+
const wasmName: FullArtifactName = {
|
|
14
|
+
type: "wasm",
|
|
15
|
+
pkg: "@test/pkg",
|
|
16
|
+
id: "main",
|
|
17
|
+
version: "1.0.0",
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
describe("assertWasmFileSize", () => {
|
|
21
|
+
it("passes at exactly the cap", () => {
|
|
22
|
+
expect(() => assertWasmFileSize("/p/a.wasm", MAX_WASM_FILE_BYTES, wasmName)).not.toThrow();
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it("throws above the cap with file path, full name, and shrink hint", () => {
|
|
26
|
+
let err: unknown;
|
|
27
|
+
try {
|
|
28
|
+
assertWasmFileSize("/p/a.wasm", MAX_WASM_FILE_BYTES + 1, wasmName);
|
|
29
|
+
} catch (e) {
|
|
30
|
+
err = e;
|
|
31
|
+
}
|
|
32
|
+
expect(err).toBeInstanceOf(Error);
|
|
33
|
+
const msg = (err as Error).message;
|
|
34
|
+
expect(msg).toContain("/p/a.wasm");
|
|
35
|
+
expect(msg).toContain("@test/pkg:main");
|
|
36
|
+
expect(msg).toContain("wasm-opt -Oz");
|
|
37
|
+
// points the reader at the backend source-of-truth
|
|
38
|
+
expect(msg).toContain("assertions.go");
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
describe("assertTemplatePackSize", () => {
|
|
43
|
+
it("passes at exactly the cap", () => {
|
|
44
|
+
expect(() =>
|
|
45
|
+
assertTemplatePackSize("@t/pkg:main 1.0.0", MAX_TEMPLATE_PACK_BYTES_GZIPPED, []),
|
|
46
|
+
).not.toThrow();
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it("lists WASM contributions sorted by size (largest first)", () => {
|
|
50
|
+
let err: unknown;
|
|
51
|
+
try {
|
|
52
|
+
assertTemplatePackSize("@t/pkg:main 1.0.0", MAX_TEMPLATE_PACK_BYTES_GZIPPED + 1, [
|
|
53
|
+
{ name: "@small/wasm:main", rawBytes: 100 * 1024 },
|
|
54
|
+
{ name: "@big/wasm:main", rawBytes: 1.5 * 1024 * 1024 },
|
|
55
|
+
{ name: "@mid/wasm:main", rawBytes: 800 * 1024 },
|
|
56
|
+
]);
|
|
57
|
+
} catch (e) {
|
|
58
|
+
err = e;
|
|
59
|
+
}
|
|
60
|
+
expect(err).toBeInstanceOf(Error);
|
|
61
|
+
const msg = (err as Error).message;
|
|
62
|
+
const big = msg.indexOf("@big/wasm:main");
|
|
63
|
+
const mid = msg.indexOf("@mid/wasm:main");
|
|
64
|
+
const small = msg.indexOf("@small/wasm:main");
|
|
65
|
+
// all three present
|
|
66
|
+
expect(big).toBeGreaterThan(-1);
|
|
67
|
+
expect(mid).toBeGreaterThan(-1);
|
|
68
|
+
expect(small).toBeGreaterThan(-1);
|
|
69
|
+
// ordered big > mid > small
|
|
70
|
+
expect(big).toBeLessThan(mid);
|
|
71
|
+
expect(mid).toBeLessThan(small);
|
|
72
|
+
expect(msg).toContain("@t/pkg:main 1.0.0");
|
|
73
|
+
expect(msg).toContain("config.go");
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("includes a fallback hint when no WASM artefacts contributed", () => {
|
|
77
|
+
let err: unknown;
|
|
78
|
+
try {
|
|
79
|
+
assertTemplatePackSize("@t/pkg:main 1.0.0", MAX_TEMPLATE_PACK_BYTES_GZIPPED + 1, []);
|
|
80
|
+
} catch (e) {
|
|
81
|
+
err = e;
|
|
82
|
+
}
|
|
83
|
+
expect((err as Error).message).toContain("no WASM artefacts");
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
describe("collectWasmContributions", () => {
|
|
88
|
+
// base64 length L → ~L * 3 / 4 raw bytes. Build a base64 string of the
|
|
89
|
+
// requested length so the test is independent of any specific WASM payload.
|
|
90
|
+
function base64OfLength(len: number): string {
|
|
91
|
+
return "A".repeat(len);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
it("reports direct WASM dependencies with raw byte size derived from base64 length", () => {
|
|
95
|
+
const base64 = base64OfLength(400); // 400 base64 chars → 300 raw bytes
|
|
96
|
+
const tpl: CompiledTemplateV3 = {
|
|
97
|
+
type: "pl.tengo-template.v3",
|
|
98
|
+
hashToSource: { "hash-a": base64 },
|
|
99
|
+
template: {
|
|
100
|
+
name: "@t/pkg:main",
|
|
101
|
+
version: "1.0.0",
|
|
102
|
+
sourceHash: "tpl-hash",
|
|
103
|
+
libs: {},
|
|
104
|
+
templates: {},
|
|
105
|
+
software: {},
|
|
106
|
+
assets: {},
|
|
107
|
+
wasm: {
|
|
108
|
+
"@w/wasm:a": { name: "@w/wasm:a", version: "1.0.0", sourceHash: "hash-a" },
|
|
109
|
+
},
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
const got = collectWasmContributions(tpl);
|
|
113
|
+
expect(got).toEqual([{ name: "@w/wasm:a", rawBytes: 300 }]);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it("recurses into sub-templates and deduplicates by artefact name", () => {
|
|
117
|
+
const base64a = base64OfLength(800); // 600 raw
|
|
118
|
+
const base64b = base64OfLength(400); // 300 raw
|
|
119
|
+
const tpl: CompiledTemplateV3 = {
|
|
120
|
+
type: "pl.tengo-template.v3",
|
|
121
|
+
hashToSource: { "hash-a": base64a, "hash-b": base64b },
|
|
122
|
+
template: {
|
|
123
|
+
name: "@t/pkg:root",
|
|
124
|
+
version: "1.0.0",
|
|
125
|
+
sourceHash: "root-hash",
|
|
126
|
+
libs: {},
|
|
127
|
+
software: {},
|
|
128
|
+
assets: {},
|
|
129
|
+
wasm: {
|
|
130
|
+
"@w/wasm:a": { name: "@w/wasm:a", version: "1.0.0", sourceHash: "hash-a" },
|
|
131
|
+
},
|
|
132
|
+
templates: {
|
|
133
|
+
"@t/pkg:child": {
|
|
134
|
+
name: "@t/pkg:child",
|
|
135
|
+
version: "1.0.0",
|
|
136
|
+
sourceHash: "child-hash",
|
|
137
|
+
libs: {},
|
|
138
|
+
templates: {},
|
|
139
|
+
software: {},
|
|
140
|
+
assets: {},
|
|
141
|
+
wasm: {
|
|
142
|
+
// duplicate of the parent's WASM — must not be counted twice
|
|
143
|
+
"@w/wasm:a": { name: "@w/wasm:a", version: "1.0.0", sourceHash: "hash-a" },
|
|
144
|
+
"@w/wasm:b": { name: "@w/wasm:b", version: "1.0.0", sourceHash: "hash-b" },
|
|
145
|
+
},
|
|
146
|
+
},
|
|
147
|
+
},
|
|
148
|
+
},
|
|
149
|
+
};
|
|
150
|
+
const got = collectWasmContributions(tpl).sort((a, b) => a.name.localeCompare(b.name));
|
|
151
|
+
expect(got).toEqual([
|
|
152
|
+
{ name: "@w/wasm:a", rawBytes: 600 },
|
|
153
|
+
{ name: "@w/wasm:b", rawBytes: 300 },
|
|
154
|
+
]);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it("returns an empty list when the template tree contains no WASM", () => {
|
|
158
|
+
const tpl: CompiledTemplateV3 = {
|
|
159
|
+
type: "pl.tengo-template.v3",
|
|
160
|
+
hashToSource: {},
|
|
161
|
+
template: {
|
|
162
|
+
name: "@t/pkg:main",
|
|
163
|
+
version: "1.0.0",
|
|
164
|
+
sourceHash: "tpl-hash",
|
|
165
|
+
libs: {},
|
|
166
|
+
templates: {},
|
|
167
|
+
software: {},
|
|
168
|
+
assets: {},
|
|
169
|
+
},
|
|
170
|
+
};
|
|
171
|
+
expect(collectWasmContributions(tpl)).toEqual([]);
|
|
172
|
+
});
|
|
173
|
+
});
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import type { CompiledTemplateV3, TemplateDataV3 } from "@milaboratories/pl-model-backend";
|
|
2
|
+
|
|
3
|
+
import type { FullArtifactName } from "./package";
|
|
4
|
+
import { fullNameToString } from "./package";
|
|
5
|
+
|
|
6
|
+
// Build-time size guards mirroring the two backend caps that gate template
|
|
7
|
+
// ingestion. Hardcoded because tengo-builder runs at block build time and has
|
|
8
|
+
// no connection to a running backend; if pl ever changes the caps, update the
|
|
9
|
+
// constants here in lockstep with the source-of-truth lines pointed at below.
|
|
10
|
+
//
|
|
11
|
+
// Source of truth (`core/pl` repo):
|
|
12
|
+
// - platform/core/transaction/assertions.go : `maxResourceDataSize`
|
|
13
|
+
// 3 MiB per value resource, checked on every CreateValue marshal.
|
|
14
|
+
// Hits the WasmV1 resource after the controller unpacks a template pack
|
|
15
|
+
// into its constituent libs/software/wasm resources.
|
|
16
|
+
// - controllers/workflow/pkg/cfg/config.go : `TemplatePackSizeLimit`
|
|
17
|
+
// 3.5 MiB per gzipped template pack, checked before the pack itself is
|
|
18
|
+
// stored as a TengoTemplatePackV1 value resource.
|
|
19
|
+
|
|
20
|
+
// 2 MiB raw → ~2.67 MiB base64 + JSON wrapper → comfortably under the 3 MiB
|
|
21
|
+
// per-value-resource cap with ~330 KiB headroom for the WasmV1Data wrapper
|
|
22
|
+
// (Name, Version, Runtime, DefaultMemoryLimit, JSON syntax). WasmV1Data.Code
|
|
23
|
+
// is a Go `[]byte`, which encoding/json serialises as base64 — that's where
|
|
24
|
+
// the 4/3 expansion comes from.
|
|
25
|
+
export const MAX_WASM_FILE_BYTES = 2 * 1024 * 1024;
|
|
26
|
+
|
|
27
|
+
// 100 KiB below the backend's 3.5 MiB so blocks don't break on small
|
|
28
|
+
// platform-side metadata overhead, or if pl tightens the limit slightly.
|
|
29
|
+
export const MAX_TEMPLATE_PACK_BYTES_GZIPPED = 3.5 * 1024 * 1024 - 100 * 1024;
|
|
30
|
+
|
|
31
|
+
function formatBytes(n: number): string {
|
|
32
|
+
if (n < 1024) return `${n} B`;
|
|
33
|
+
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KiB`;
|
|
34
|
+
return `${(n / 1024 / 1024).toFixed(2)} MiB`;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function assertWasmFileSize(
|
|
38
|
+
file: string,
|
|
39
|
+
byteLength: number,
|
|
40
|
+
fullName: FullArtifactName,
|
|
41
|
+
): void {
|
|
42
|
+
if (byteLength <= MAX_WASM_FILE_BYTES) return;
|
|
43
|
+
throw new Error(
|
|
44
|
+
`WASM artefact ${fullNameToString(fullName)} from ${file} is ${formatBytes(byteLength)}, ` +
|
|
45
|
+
`which exceeds tengo-builder's ${formatBytes(MAX_WASM_FILE_BYTES)} per-WASM cap. ` +
|
|
46
|
+
`The backend stores each WASM as a single value resource (3 MiB hard cap after base64+JSON ` +
|
|
47
|
+
`marshal — see maxResourceDataSize in pl: platform/core/transaction/assertions.go). ` +
|
|
48
|
+
`Strip debug info or run \`wasm-opt -Oz\` to shrink it.`,
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface WasmContribution {
|
|
53
|
+
/** Artefact name as stored under TemplateDataV3.wasm (e.g. "@pkg:id"). */
|
|
54
|
+
readonly name: string;
|
|
55
|
+
/** Raw WASM bytes — decoded from the base64-encoded source in hashToSource. */
|
|
56
|
+
readonly rawBytes: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Walks a compiled template tree and reports every WASM artefact embedded
|
|
61
|
+
* directly or transitively. Used only by the too-large-pack error message —
|
|
62
|
+
* never on the success path. Deduplicates by artefact name (sub-templates can
|
|
63
|
+
* import the same WASM via different aliases and we want to attribute weight
|
|
64
|
+
* only once).
|
|
65
|
+
*/
|
|
66
|
+
export function collectWasmContributions(tpl: CompiledTemplateV3): WasmContribution[] {
|
|
67
|
+
const seen = new Map<string, WasmContribution>();
|
|
68
|
+
function walk(t: TemplateDataV3) {
|
|
69
|
+
for (const [name, ref] of Object.entries(t.wasm ?? {})) {
|
|
70
|
+
if (seen.has(name)) continue;
|
|
71
|
+
const base64 = tpl.hashToSource[ref.sourceHash];
|
|
72
|
+
// base64 encodes 3 raw bytes per 4 chars; the off-by-padding is at most
|
|
73
|
+
// 2 bytes per artefact — fine for a human-readable size breakdown.
|
|
74
|
+
const rawBytes = base64 != null ? Math.floor((base64.length * 3) / 4) : 0;
|
|
75
|
+
seen.set(name, { name, rawBytes });
|
|
76
|
+
}
|
|
77
|
+
for (const sub of Object.values(t.templates ?? {})) walk(sub);
|
|
78
|
+
}
|
|
79
|
+
walk(tpl.template);
|
|
80
|
+
return [...seen.values()];
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function assertTemplatePackSize(
|
|
84
|
+
templateName: string,
|
|
85
|
+
gzippedByteLength: number,
|
|
86
|
+
wasmContributions: readonly WasmContribution[],
|
|
87
|
+
): void {
|
|
88
|
+
if (gzippedByteLength <= MAX_TEMPLATE_PACK_BYTES_GZIPPED) return;
|
|
89
|
+
const ranked = [...wasmContributions].sort((a, b) => b.rawBytes - a.rawBytes);
|
|
90
|
+
const breakdown =
|
|
91
|
+
ranked.length === 0
|
|
92
|
+
? "\n (no WASM artefacts in this pack — heavy libs, software descriptors, or sub-templates are the likely cause.)"
|
|
93
|
+
: "\nWASM artefacts in this pack (raw bytes):\n" +
|
|
94
|
+
ranked.map((w) => ` - ${w.name} — ${formatBytes(w.rawBytes)}`).join("\n");
|
|
95
|
+
throw new Error(
|
|
96
|
+
`Template pack ${templateName} is ${formatBytes(gzippedByteLength)} gzipped, ` +
|
|
97
|
+
`which exceeds tengo-builder's ${formatBytes(MAX_TEMPLATE_PACK_BYTES_GZIPPED)} per-pack cap. ` +
|
|
98
|
+
`The backend caps gzipped packs at 3.5 MiB (see TemplatePackSizeLimit in pl: ` +
|
|
99
|
+
`controllers/workflow/pkg/cfg/config.go).` +
|
|
100
|
+
breakdown +
|
|
101
|
+
"\nShrink WASM artefacts (wasm-opt -Oz), or split this template into smaller pieces.",
|
|
102
|
+
);
|
|
103
|
+
}
|
package/src/compiler/package.ts
CHANGED
|
@@ -27,7 +27,7 @@ export type CompilerOption = {
|
|
|
27
27
|
args: string[];
|
|
28
28
|
};
|
|
29
29
|
|
|
30
|
-
export type ArtifactType = "library" | "template" | "test" | "software" | "asset";
|
|
30
|
+
export type ArtifactType = "library" | "template" | "test" | "software" | "asset" | "wasm";
|
|
31
31
|
|
|
32
32
|
/** Artifact Name including package version */
|
|
33
33
|
export interface FullArtifactName {
|
|
@@ -126,6 +126,36 @@ describe("import statements", () => {
|
|
|
126
126
|
const trickyCasesSrc = parseSource(logger, "dist", testTrickyCasesSrc, stubTplName, true);
|
|
127
127
|
expect(trickyCasesSrc.src).toEqual(testTrickyCasesNormalized);
|
|
128
128
|
});
|
|
129
|
+
|
|
130
|
+
test("assets.importWasm produces a wasm dependency", () => {
|
|
131
|
+
const logger = createLogger("error");
|
|
132
|
+
|
|
133
|
+
const wasmImport = `
|
|
134
|
+
assets := import("@platforma-sdk/workflow-tengo:assets")
|
|
135
|
+
wasm := assets.importWasm("@milaboratories/pframes-rs-wasip2:main")
|
|
136
|
+
`;
|
|
137
|
+
|
|
138
|
+
const parsed = parseSource(logger, "dist", wasmImport, stubTplName, true);
|
|
139
|
+
expect(parsed.dependencies).toContainEqual({
|
|
140
|
+
type: "wasm",
|
|
141
|
+
pkg: "@milaboratories/pframes-rs-wasip2",
|
|
142
|
+
id: "main",
|
|
143
|
+
});
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("assets.importWasm with variable id is rejected", () => {
|
|
147
|
+
const logger = createLogger("info");
|
|
148
|
+
|
|
149
|
+
const wasmByVar = `
|
|
150
|
+
assets := import("@platforma-sdk/workflow-tengo:assets")
|
|
151
|
+
wasmId := "@milaboratories/pframes-rs-wasip2:main"
|
|
152
|
+
wasm := assets.importWasm(wasmId)
|
|
153
|
+
`;
|
|
154
|
+
|
|
155
|
+
expect(() => parseSource(logger, "dist", wasmByVar, stubTplName, true)).toThrow(
|
|
156
|
+
"variables are not allowed",
|
|
157
|
+
);
|
|
158
|
+
});
|
|
129
159
|
});
|
|
130
160
|
|
|
131
161
|
describe("parseSingleSourceLine", () => {
|
package/src/compiler/source.ts
CHANGED
|
@@ -42,6 +42,8 @@ const newImportSoftwareDetector = (moduleName: string) =>
|
|
|
42
42
|
const newImportAssetRE = (moduleName: string) => functionCallRE(moduleName, "importAsset");
|
|
43
43
|
const newImportAssetDetector = (moduleName: string) =>
|
|
44
44
|
functionCallLikeRE(moduleName, "importAsset");
|
|
45
|
+
const newImportWasmRE = (moduleName: string) => functionCallRE(moduleName, "importWasm");
|
|
46
|
+
const newImportWasmDetector = (moduleName: string) => functionCallLikeRE(moduleName, "importWasm");
|
|
45
47
|
|
|
46
48
|
const emptyLineRE = /^\s*$/;
|
|
47
49
|
const compilerOptionRE = /^\/\/tengo:[\w]/;
|
|
@@ -369,11 +371,13 @@ function processModuleImport(
|
|
|
369
371
|
["template", newImportTemplateRE(iInfo.alias)],
|
|
370
372
|
["software", newImportSoftwareRE(iInfo.alias)],
|
|
371
373
|
["asset", newImportAssetRE(iInfo.alias)],
|
|
374
|
+
["wasm", newImportWasmRE(iInfo.alias)],
|
|
372
375
|
]);
|
|
373
376
|
context.importLikeREs.set(iInfo.module, [
|
|
374
377
|
["template", newImportTemplateDetector(iInfo.alias)],
|
|
375
378
|
["software", newImportSoftwareDetector(iInfo.alias)],
|
|
376
379
|
["asset", newImportAssetDetector(iInfo.alias)],
|
|
380
|
+
["wasm", newImportWasmDetector(iInfo.alias)],
|
|
377
381
|
]);
|
|
378
382
|
}
|
|
379
383
|
}
|
package/src/shared/basecmd.ts
CHANGED
|
@@ -31,7 +31,7 @@ export const ArtifactTypeFlag = {
|
|
|
31
31
|
type: Flags.string({
|
|
32
32
|
aliases: ["t"],
|
|
33
33
|
description: "artifact type",
|
|
34
|
-
options: ["all", "library", "template", "test", "software", "asset"],
|
|
34
|
+
options: ["all", "library", "template", "test", "software", "asset", "wasm"],
|
|
35
35
|
default: "all",
|
|
36
36
|
}),
|
|
37
37
|
};
|
package/src/shared/dump.ts
CHANGED
|
@@ -88,6 +88,15 @@ export function dumpArtifacts(
|
|
|
88
88
|
}
|
|
89
89
|
}
|
|
90
90
|
|
|
91
|
+
// Wasm
|
|
92
|
+
|
|
93
|
+
if (!aType || aType === "wasm") {
|
|
94
|
+
for (const wasm of compiler.allWasm()) {
|
|
95
|
+
logger.debug(`Dumping to pl-tester: ${typedArtifactNameToString(wasm.fullName)}`);
|
|
96
|
+
stream.write(JSON.stringify(wasm) + "\n");
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
91
100
|
// Tests
|
|
92
101
|
|
|
93
102
|
if (!aType || aType === "test") {
|