@koda-sl/baker-cli 0.122.0 → 0.122.1-dev.57a9836c5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -25
- package/dist/{chunk-IWPAXJC3.js → chunk-T6HBTZOO.js} +339 -76
- package/dist/chunk-T6HBTZOO.js.map +1 -0
- package/dist/cli.js +1391 -1141
- package/dist/cli.js.map +1 -1
- package/dist/engine/index.d.ts +58 -0
- package/dist/engine/index.js +1 -1
- package/package.json +1 -1
- package/dist/chunk-IWPAXJC3.js.map +0 -1
|
@@ -3,9 +3,9 @@ import {
|
|
|
3
3
|
__toESM
|
|
4
4
|
} from "./chunk-5WRI5ZAA.js";
|
|
5
5
|
|
|
6
|
-
//
|
|
6
|
+
// ../../.pnpm-store/v10/links/@/safe-stable-stringify/2.5.0/810146e81bae4e3a061fe487864f2fde80c4b03b886877dc0f1fffbc6480b67e/node_modules/safe-stable-stringify/index.js
|
|
7
7
|
var require_safe_stable_stringify = __commonJS({
|
|
8
|
-
"
|
|
8
|
+
"../../.pnpm-store/v10/links/@/safe-stable-stringify/2.5.0/810146e81bae4e3a061fe487864f2fde80c4b03b886877dc0f1fffbc6480b67e/node_modules/safe-stable-stringify/index.js"(exports, module) {
|
|
9
9
|
"use strict";
|
|
10
10
|
var { hasOwnProperty } = Object.prototype;
|
|
11
11
|
var stringify = configure2();
|
|
@@ -138,9 +138,9 @@ var require_safe_stable_stringify = __commonJS({
|
|
|
138
138
|
}
|
|
139
139
|
if (value) {
|
|
140
140
|
return (value2) => {
|
|
141
|
-
let
|
|
142
|
-
if (typeof value2 !== "function")
|
|
143
|
-
throw new Error(
|
|
141
|
+
let message2 = `Object can not safely be stringified. Received type ${typeof value2}`;
|
|
142
|
+
if (typeof value2 !== "function") message2 += ` (${value2.toString()})`;
|
|
143
|
+
throw new Error(message2);
|
|
144
144
|
};
|
|
145
145
|
}
|
|
146
146
|
}
|
|
@@ -652,6 +652,9 @@ var HttpClient = class {
|
|
|
652
652
|
async postJson(path16, body, signal) {
|
|
653
653
|
return await this.requestJson("POST", path16, body, signal);
|
|
654
654
|
}
|
|
655
|
+
async putJson(path16, body, signal) {
|
|
656
|
+
return await this.requestJson("PUT", path16, body, signal);
|
|
657
|
+
}
|
|
655
658
|
async getJson(path16, signal) {
|
|
656
659
|
return await this.requestJson("GET", path16, void 0, signal);
|
|
657
660
|
}
|
|
@@ -679,8 +682,8 @@ var HttpClient = class {
|
|
|
679
682
|
try {
|
|
680
683
|
const res = await this.fetchFn(url, {
|
|
681
684
|
method,
|
|
682
|
-
headers: method === "
|
|
683
|
-
body: method === "
|
|
685
|
+
headers: method === "GET" ? { Authorization: `Bearer ${this.apiKey}` } : { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` },
|
|
686
|
+
body: method === "GET" ? void 0 : JSON.stringify(body),
|
|
684
687
|
signal: controller.signal
|
|
685
688
|
});
|
|
686
689
|
if (res.ok) return { kind: "value", value: await res.json() };
|
|
@@ -717,33 +720,33 @@ async function parseErrorBody(res) {
|
|
|
717
720
|
const errObj = body.error ?? {};
|
|
718
721
|
return classifyHttpError(res.status, errObj, errObj.message ?? `HTTP ${res.status}`);
|
|
719
722
|
}
|
|
720
|
-
function classifyHttpError(status, errObj,
|
|
723
|
+
function classifyHttpError(status, errObj, message2) {
|
|
721
724
|
if (errObj.code === CONTENT_POLICY_CODE) {
|
|
722
|
-
return { kind: "content_policy", status, provider: errObj.provider, message };
|
|
725
|
+
return { kind: "content_policy", status, provider: errObj.provider, message: message2 };
|
|
723
726
|
}
|
|
724
727
|
if (status === 401 || status === 403) {
|
|
725
|
-
return { kind: "unauthorized", status, message };
|
|
728
|
+
return { kind: "unauthorized", status, message: message2 };
|
|
726
729
|
}
|
|
727
730
|
if (status === 400 || status === 422) {
|
|
728
|
-
return { kind: "validation", status, message, details: errObj.details };
|
|
731
|
+
return { kind: "validation", status, message: message2, details: errObj.details };
|
|
729
732
|
}
|
|
730
733
|
if (status === 502 || status === 504) {
|
|
731
734
|
if (errObj.code === "provider_timeout" || status === 504) {
|
|
732
|
-
return { kind: "timeout", provider: errObj.provider, message };
|
|
735
|
+
return { kind: "timeout", provider: errObj.provider, message: message2 };
|
|
733
736
|
}
|
|
734
737
|
return {
|
|
735
738
|
kind: "provider",
|
|
736
739
|
status,
|
|
737
740
|
provider: errObj.provider,
|
|
738
741
|
code: errObj.code ?? "provider_error",
|
|
739
|
-
message,
|
|
742
|
+
message: message2,
|
|
740
743
|
retryable: errObj.retryable ?? true
|
|
741
744
|
};
|
|
742
745
|
}
|
|
743
746
|
if (status >= 500 || status === 429) {
|
|
744
|
-
return { kind: "server", status, message };
|
|
747
|
+
return { kind: "server", status, message: message2 };
|
|
745
748
|
}
|
|
746
|
-
return { kind: "validation", status, message, details: errObj.details };
|
|
749
|
+
return { kind: "validation", status, message: message2, details: errObj.details };
|
|
747
750
|
}
|
|
748
751
|
function backoffMs(attempt) {
|
|
749
752
|
return 1e3 * 2 ** attempt;
|
|
@@ -814,6 +817,27 @@ var BackendClient = class {
|
|
|
814
817
|
signal
|
|
815
818
|
);
|
|
816
819
|
}
|
|
820
|
+
/** Remote cache lookup. A miss (404) — or an old backend without the route — returns null. */
|
|
821
|
+
async getCacheEntry(cacheKey, signal) {
|
|
822
|
+
try {
|
|
823
|
+
const res = await this.http.getJson(`/api/canvas/cache/${encodeURIComponent(cacheKey)}`, signal);
|
|
824
|
+
return res.entry;
|
|
825
|
+
} catch (e) {
|
|
826
|
+
if (e instanceof BackendHttpError && "status" in e.detail && e.detail.status === 404) return null;
|
|
827
|
+
throw e;
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
async putCacheEntry(entry, signal) {
|
|
831
|
+
await this.http.putJson(
|
|
832
|
+
`/api/canvas/cache/${encodeURIComponent(entry.cacheKey)}`,
|
|
833
|
+
entry,
|
|
834
|
+
signal
|
|
835
|
+
);
|
|
836
|
+
}
|
|
837
|
+
/** Durable run-history record — POST /api/canvas/runs (idempotent server-side on runId). */
|
|
838
|
+
async recordRun(payload, signal) {
|
|
839
|
+
await this.http.postJson("/api/canvas/runs", payload, signal);
|
|
840
|
+
}
|
|
817
841
|
getArtifact(kind, name, version, signal) {
|
|
818
842
|
const path16 = version ? `/api/canvas/artifacts/${encodeURIComponent(kind)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}` : `/api/canvas/artifacts/${encodeURIComponent(kind)}/${encodeURIComponent(name)}`;
|
|
819
843
|
return this.http.getJson(path16, signal);
|
|
@@ -837,14 +861,17 @@ function requireCredentialsFromEnv(env = process.env) {
|
|
|
837
861
|
}
|
|
838
862
|
return c;
|
|
839
863
|
}
|
|
864
|
+
function remoteCacheEnabledFromEnv(env = process.env) {
|
|
865
|
+
return env.BAKER_CANVAS_REMOTE_CACHE !== "off";
|
|
866
|
+
}
|
|
840
867
|
|
|
841
868
|
// src/engine/engine/errors.ts
|
|
842
869
|
function isBlocking(issue) {
|
|
843
870
|
return issue.severity !== "warning";
|
|
844
871
|
}
|
|
845
872
|
var CanvasError = class extends Error {
|
|
846
|
-
constructor(
|
|
847
|
-
super(
|
|
873
|
+
constructor(message2) {
|
|
874
|
+
super(message2);
|
|
848
875
|
this.name = "CanvasError";
|
|
849
876
|
}
|
|
850
877
|
};
|
|
@@ -901,7 +928,7 @@ function describeCause(c) {
|
|
|
901
928
|
}
|
|
902
929
|
}
|
|
903
930
|
|
|
904
|
-
//
|
|
931
|
+
// ../../.pnpm-store/v10/links/@/safe-stable-stringify/2.5.0/810146e81bae4e3a061fe487864f2fde80c4b03b886877dc0f1fffbc6480b67e/node_modules/safe-stable-stringify/esm/wrapper.js
|
|
905
932
|
var import__ = __toESM(require_safe_stable_stringify(), 1);
|
|
906
933
|
var configure = import__.default.configure;
|
|
907
934
|
var wrapper_default = import__.default;
|
|
@@ -1562,6 +1589,160 @@ function encodeRandom() {
|
|
|
1562
1589
|
return out;
|
|
1563
1590
|
}
|
|
1564
1591
|
|
|
1592
|
+
// src/engine/storage/remote-cache-store.ts
|
|
1593
|
+
var CANVAS_ASSETS_URL_SEGMENT = "/canvas-assets/";
|
|
1594
|
+
function isPersistedAssetRef(ref) {
|
|
1595
|
+
const { url, sha256 } = ref;
|
|
1596
|
+
if (typeof url !== "string" || typeof sha256 !== "string") return false;
|
|
1597
|
+
return url.includes(`${CANVAS_ASSETS_URL_SEGMENT}${sha256.slice(0, 2)}/${sha256}`) || url.includes(`${CANVAS_ASSETS_URL_SEGMENT}${sha256}`);
|
|
1598
|
+
}
|
|
1599
|
+
function isAssetRefLike(value) {
|
|
1600
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) && typeof value.sha256 === "string" && typeof value.mime === "string";
|
|
1601
|
+
}
|
|
1602
|
+
function collectAssetRefLikes(value, out = []) {
|
|
1603
|
+
if (Array.isArray(value)) {
|
|
1604
|
+
for (const item of value) collectAssetRefLikes(item, out);
|
|
1605
|
+
return out;
|
|
1606
|
+
}
|
|
1607
|
+
if (typeof value !== "object" || value === null) return out;
|
|
1608
|
+
if (isAssetRefLike(value)) {
|
|
1609
|
+
out.push(value);
|
|
1610
|
+
}
|
|
1611
|
+
for (const item of Object.values(value)) collectAssetRefLikes(item, out);
|
|
1612
|
+
return out;
|
|
1613
|
+
}
|
|
1614
|
+
function entryFullyPersisted(entry) {
|
|
1615
|
+
return collectAssetRefLikes(entry.outputs).every((ref) => isPersistedAssetRef(ref));
|
|
1616
|
+
}
|
|
1617
|
+
function stripLocalFields(entry) {
|
|
1618
|
+
const clone = JSON.parse(JSON.stringify(entry));
|
|
1619
|
+
for (const ref of collectAssetRefLikes(clone.outputs)) {
|
|
1620
|
+
delete ref.path;
|
|
1621
|
+
delete ref.bytes;
|
|
1622
|
+
}
|
|
1623
|
+
return clone;
|
|
1624
|
+
}
|
|
1625
|
+
var RemoteCacheStore = class {
|
|
1626
|
+
client;
|
|
1627
|
+
log;
|
|
1628
|
+
constructor(client, log) {
|
|
1629
|
+
this.client = client;
|
|
1630
|
+
this.log = log ?? (() => void 0);
|
|
1631
|
+
}
|
|
1632
|
+
async get(cacheKey) {
|
|
1633
|
+
return await this.client.getCacheEntry(cacheKey);
|
|
1634
|
+
}
|
|
1635
|
+
async put(entry) {
|
|
1636
|
+
if (!entryFullyPersisted(entry)) {
|
|
1637
|
+
this.log(`[cache ] ${entry.cacheKey.slice(0, 12)}\u2026 has local-only assets, kept local`);
|
|
1638
|
+
return;
|
|
1639
|
+
}
|
|
1640
|
+
const stripped = stripLocalFields(entry);
|
|
1641
|
+
if (stripped.refs.length > MAX_REMOTE_REFS) {
|
|
1642
|
+
stripped.refs = stripped.refs.slice(0, MAX_REMOTE_REFS);
|
|
1643
|
+
}
|
|
1644
|
+
await this.client.putCacheEntry(stripped);
|
|
1645
|
+
}
|
|
1646
|
+
};
|
|
1647
|
+
var MAX_REMOTE_REFS = 512;
|
|
1648
|
+
var LayeredCacheStore = class {
|
|
1649
|
+
rootDir;
|
|
1650
|
+
local;
|
|
1651
|
+
remote;
|
|
1652
|
+
assets;
|
|
1653
|
+
log;
|
|
1654
|
+
constructor(opts) {
|
|
1655
|
+
this.local = opts.local;
|
|
1656
|
+
this.remote = opts.remote;
|
|
1657
|
+
this.assets = opts.assets;
|
|
1658
|
+
this.rootDir = opts.local.rootDir;
|
|
1659
|
+
this.log = opts.log ?? (() => void 0);
|
|
1660
|
+
}
|
|
1661
|
+
async get(cacheKey) {
|
|
1662
|
+
const localHit = await this.local.get(cacheKey);
|
|
1663
|
+
if (localHit) return localHit;
|
|
1664
|
+
let remoteEntry;
|
|
1665
|
+
try {
|
|
1666
|
+
remoteEntry = await this.remote.get(cacheKey);
|
|
1667
|
+
} catch (e) {
|
|
1668
|
+
this.log(`[cache ] remote lookup failed (${message(e)}) \u2014 treating as miss`);
|
|
1669
|
+
return null;
|
|
1670
|
+
}
|
|
1671
|
+
if (!remoteEntry) return null;
|
|
1672
|
+
let rehydrated;
|
|
1673
|
+
try {
|
|
1674
|
+
rehydrated = await this.rehydrate(remoteEntry);
|
|
1675
|
+
} catch (e) {
|
|
1676
|
+
this.log(`[cache ] ${cacheKey.slice(0, 12)}\u2026 rehydration failed (${message(e)}) \u2014 treating as miss`);
|
|
1677
|
+
return null;
|
|
1678
|
+
}
|
|
1679
|
+
await this.local.put(rehydrated);
|
|
1680
|
+
return rehydrated;
|
|
1681
|
+
}
|
|
1682
|
+
async put(entry) {
|
|
1683
|
+
await this.local.put(entry);
|
|
1684
|
+
try {
|
|
1685
|
+
await this.remote.put(entry);
|
|
1686
|
+
} catch (e) {
|
|
1687
|
+
this.log(`[cache ] remote write failed (${message(e)}) \u2014 entry kept local`);
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
/**
|
|
1691
|
+
* Download every referenced asset into the local content-addressed store
|
|
1692
|
+
* (sha-verified) and stamp fresh local paths. Any ref that cannot be
|
|
1693
|
+
* rehydrated fails the WHOLE entry — a partially-hydrated cache hit would
|
|
1694
|
+
* crash materialization later with a far less actionable error.
|
|
1695
|
+
*/
|
|
1696
|
+
async rehydrate(entry) {
|
|
1697
|
+
const clone = JSON.parse(JSON.stringify(entry));
|
|
1698
|
+
for (const ref of collectAssetRefLikes(clone.outputs)) {
|
|
1699
|
+
if (!isPersistedAssetRef(ref)) {
|
|
1700
|
+
throw new Error(`ref ${ref.sha256.slice(0, 12)}\u2026 has no persisted url`);
|
|
1701
|
+
}
|
|
1702
|
+
const ingested = await this.assets.ingestRemote({
|
|
1703
|
+
kind: typeof ref.kind === "string" ? ref.kind : "json",
|
|
1704
|
+
url: ref.url,
|
|
1705
|
+
sha256: ref.sha256,
|
|
1706
|
+
mime: ref.mime,
|
|
1707
|
+
metadata: ref.metadata ?? void 0
|
|
1708
|
+
});
|
|
1709
|
+
ref.path = ingested.path;
|
|
1710
|
+
}
|
|
1711
|
+
return clone;
|
|
1712
|
+
}
|
|
1713
|
+
};
|
|
1714
|
+
function message(e) {
|
|
1715
|
+
return e instanceof Error ? e.message : String(e);
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
// src/engine/nodes/remote/upload.ts
|
|
1719
|
+
async function presignAndPut(args) {
|
|
1720
|
+
const { putUrl, publicUrl } = await args.ctx.client.presignAssetUpload(args.sha256, args.mime, args.ctx.signal);
|
|
1721
|
+
const putRes = await fetch(putUrl, {
|
|
1722
|
+
method: "PUT",
|
|
1723
|
+
body: new Uint8Array(args.bytes),
|
|
1724
|
+
headers: { "Content-Type": args.mime },
|
|
1725
|
+
signal: args.ctx.signal
|
|
1726
|
+
});
|
|
1727
|
+
if (!putRes.ok) {
|
|
1728
|
+
throw new Error(`upload: presigned PUT failed ${putRes.status} ${putRes.statusText}`);
|
|
1729
|
+
}
|
|
1730
|
+
return publicUrl;
|
|
1731
|
+
}
|
|
1732
|
+
async function ensureUploaded(ref, ctx) {
|
|
1733
|
+
if (ref.url) return ref;
|
|
1734
|
+
const bytes = await ctx.assets.readBytes(ref.sha256, ref.mime);
|
|
1735
|
+
const url = await presignAndPut({ bytes, sha256: ref.sha256, mime: ref.mime, ctx });
|
|
1736
|
+
return { ...ref, url };
|
|
1737
|
+
}
|
|
1738
|
+
async function persistOutputAssetUrls(outputs, ctx) {
|
|
1739
|
+
for (const ref of collectAssetRefLikes(outputs)) {
|
|
1740
|
+
if (isPersistedAssetRef(ref)) continue;
|
|
1741
|
+
const bytes = await ctx.assets.readBytes(ref.sha256, ref.mime);
|
|
1742
|
+
ref.url = await presignAndPut({ bytes, sha256: ref.sha256, mime: ref.mime, ctx });
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
|
|
1565
1746
|
// src/engine/schema/canvas.ts
|
|
1566
1747
|
import { z } from "zod";
|
|
1567
1748
|
var REF_PREFIX = "$ref:";
|
|
@@ -2841,6 +3022,7 @@ var Engine = class {
|
|
|
2841
3022
|
cache;
|
|
2842
3023
|
outputsDir;
|
|
2843
3024
|
log;
|
|
3025
|
+
persistAssets;
|
|
2844
3026
|
constructor(opts) {
|
|
2845
3027
|
this.registry = opts.registry;
|
|
2846
3028
|
this.client = opts.client;
|
|
@@ -2848,6 +3030,7 @@ var Engine = class {
|
|
|
2848
3030
|
this.cache = opts.cache;
|
|
2849
3031
|
this.outputsDir = opts.outputsDir;
|
|
2850
3032
|
this.log = opts.log ?? (() => void 0);
|
|
3033
|
+
this.persistAssets = opts.persistAssets ?? false;
|
|
2851
3034
|
}
|
|
2852
3035
|
validate(canvas) {
|
|
2853
3036
|
return validateCanvas(canvas, this.registry);
|
|
@@ -2871,7 +3054,15 @@ var Engine = class {
|
|
|
2871
3054
|
const outputs = {};
|
|
2872
3055
|
const counters = { cachedNodes: 0, totalCredits: 0 };
|
|
2873
3056
|
const nodeRuns = [];
|
|
2874
|
-
|
|
3057
|
+
const graph = this.pruneToOutput(canvas, buildGraph(canvas));
|
|
3058
|
+
this.emitProgress(opts, {
|
|
3059
|
+
kind: "plan",
|
|
3060
|
+
nodes: [...graph.entries()].map(([id, deps]) => {
|
|
3061
|
+
const node = canvas.nodes.find((n) => n.id === id);
|
|
3062
|
+
return { node_id: id, node_type: node?.type ?? "unknown", deps: [...deps], params: node?.params };
|
|
3063
|
+
})
|
|
3064
|
+
});
|
|
3065
|
+
await this.runLayers(canvas, graph, outputs, runId, writer, opts, counters, nodeRuns);
|
|
2875
3066
|
const output = pickFinalOutput(canvas, outputs);
|
|
2876
3067
|
const stats = {
|
|
2877
3068
|
total_nodes: canvas.nodes.length,
|
|
@@ -2894,39 +3085,51 @@ var Engine = class {
|
|
|
2894
3085
|
`[done ] ${stats.cached_nodes}/${stats.total_nodes} cached, ${stats.total_credits} credits, ${stats.duration_ms}ms`
|
|
2895
3086
|
);
|
|
2896
3087
|
this.log(`outputs in: ${writer.runDir}`);
|
|
2897
|
-
return { run_id: runId, output, outputs_by_node: outputs, stats, outputs_dir: writer.runDir };
|
|
3088
|
+
return { run_id: runId, output, outputs_by_node: outputs, stats, outputs_dir: writer.runDir, node_runs: nodeRuns };
|
|
2898
3089
|
}
|
|
2899
|
-
async runLayers(canvas, outputs, runId, writer, opts, counters, nodeRuns) {
|
|
2900
|
-
const layers = topologicalLayers(
|
|
3090
|
+
async runLayers(canvas, graph, outputs, runId, writer, opts, counters, nodeRuns) {
|
|
3091
|
+
const layers = topologicalLayers(graph);
|
|
2901
3092
|
const limit = resolveConcurrency(opts.concurrency);
|
|
2902
3093
|
for (const layer of layers) {
|
|
2903
|
-
const settled = await mapWithConcurrency(
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
(nodeId) => this.executeOne(canvas, nodeId, outputs, runId, writer, opts).then((r) => {
|
|
3094
|
+
const settled = await mapWithConcurrency(layer, limit, (nodeId) => {
|
|
3095
|
+
this.emitProgress(opts, { kind: "node_start", node_id: nodeId });
|
|
3096
|
+
return this.executeOne(canvas, nodeId, outputs, runId, writer, opts).then((r) => {
|
|
2907
3097
|
if (r.cached) counters.cachedNodes++;
|
|
2908
3098
|
counters.totalCredits += r.credits;
|
|
2909
3099
|
const node = canvas.nodes.find((n) => n.id === nodeId);
|
|
2910
3100
|
if (node) {
|
|
2911
|
-
|
|
3101
|
+
const run = {
|
|
2912
3102
|
node_id: nodeId,
|
|
2913
3103
|
node_type: node.type,
|
|
2914
3104
|
cached: r.cached,
|
|
2915
3105
|
duration_ms: r.durationMs,
|
|
2916
3106
|
credits: r.credits
|
|
2917
|
-
}
|
|
3107
|
+
};
|
|
3108
|
+
nodeRuns.push(run);
|
|
3109
|
+
this.emitProgress(opts, { kind: "node_settled", run, outputs: outputs[nodeId] ?? {} });
|
|
2918
3110
|
}
|
|
2919
|
-
})
|
|
2920
|
-
);
|
|
3111
|
+
});
|
|
3112
|
+
});
|
|
2921
3113
|
const failures = [];
|
|
2922
3114
|
settled.forEach((result, i) => {
|
|
2923
3115
|
const nodeId = layer[i];
|
|
2924
|
-
if (result.status === "rejected" && nodeId)
|
|
3116
|
+
if (result.status === "rejected" && nodeId) {
|
|
3117
|
+
failures.push({ nodeId, reason: result.reason });
|
|
3118
|
+
this.emitProgress(opts, { kind: "node_failed", node_id: nodeId });
|
|
3119
|
+
}
|
|
2925
3120
|
});
|
|
2926
3121
|
if (failures.length === 1 && failures[0]) throw failures[0].reason;
|
|
2927
3122
|
if (failures.length > 1) throw new LayerExecutionError(failures);
|
|
2928
3123
|
}
|
|
2929
3124
|
}
|
|
3125
|
+
/** Progress consumers are observers only — an exception there must never fail the run. */
|
|
3126
|
+
emitProgress(opts, event) {
|
|
3127
|
+
if (!opts.onProgress) return;
|
|
3128
|
+
try {
|
|
3129
|
+
opts.onProgress(event);
|
|
3130
|
+
} catch {
|
|
3131
|
+
}
|
|
3132
|
+
}
|
|
2930
3133
|
/**
|
|
2931
3134
|
* Dead-node elimination: when the canvas declares an `output`, execute only the
|
|
2932
3135
|
* nodes that output transitively depends on. Orphaned nodes (left by an edit or
|
|
@@ -2991,6 +3194,14 @@ var Engine = class {
|
|
|
2991
3194
|
const credits = def.cost ? def.cost({ params: parsedParams }).credits : 0;
|
|
2992
3195
|
const outputsObj = result;
|
|
2993
3196
|
outputs[node.id] = outputsObj;
|
|
3197
|
+
if (this.persistAssets) {
|
|
3198
|
+
try {
|
|
3199
|
+
await persistOutputAssetUrls(outputsObj, ctx);
|
|
3200
|
+
} catch (e) {
|
|
3201
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
3202
|
+
this.log(`[warn ] ${node.id}: asset persistence failed (${msg}) \u2014 outputs stay local-only`);
|
|
3203
|
+
}
|
|
3204
|
+
}
|
|
2994
3205
|
if (policy === "read_write") {
|
|
2995
3206
|
await this.cache.put({
|
|
2996
3207
|
cacheKey: prepared.cacheKey,
|
|
@@ -3309,27 +3520,6 @@ var FontRef = BaseAssetRef.extend({
|
|
|
3309
3520
|
});
|
|
3310
3521
|
var AssetRef = z4.discriminatedUnion("kind", [ImageRef, VideoRef, AudioRef, JsonRef, TextRef, FontRef]);
|
|
3311
3522
|
|
|
3312
|
-
// src/engine/nodes/remote/upload.ts
|
|
3313
|
-
async function presignAndPut(args) {
|
|
3314
|
-
const { putUrl, publicUrl } = await args.ctx.client.presignAssetUpload(args.sha256, args.mime, args.ctx.signal);
|
|
3315
|
-
const putRes = await fetch(putUrl, {
|
|
3316
|
-
method: "PUT",
|
|
3317
|
-
body: new Uint8Array(args.bytes),
|
|
3318
|
-
headers: { "Content-Type": args.mime },
|
|
3319
|
-
signal: args.ctx.signal
|
|
3320
|
-
});
|
|
3321
|
-
if (!putRes.ok) {
|
|
3322
|
-
throw new Error(`upload: presigned PUT failed ${putRes.status} ${putRes.statusText}`);
|
|
3323
|
-
}
|
|
3324
|
-
return publicUrl;
|
|
3325
|
-
}
|
|
3326
|
-
async function ensureUploaded(ref, ctx) {
|
|
3327
|
-
if (ref.url) return ref;
|
|
3328
|
-
const bytes = await ctx.assets.readBytes(ref.sha256, ref.mime);
|
|
3329
|
-
const url = await presignAndPut({ bytes, sha256: ref.sha256, mime: ref.mime, ctx });
|
|
3330
|
-
return { ...ref, url };
|
|
3331
|
-
}
|
|
3332
|
-
|
|
3333
3523
|
// src/engine/nodes/remote/delegate.ts
|
|
3334
3524
|
function delegated(spec) {
|
|
3335
3525
|
return {
|
|
@@ -3523,7 +3713,7 @@ function safePathname(rawUrl) {
|
|
|
3523
3713
|
}
|
|
3524
3714
|
var ingestNode = defineNode({
|
|
3525
3715
|
id: "ingest",
|
|
3526
|
-
version: "1.
|
|
3716
|
+
version: "1.2.0",
|
|
3527
3717
|
category: "io",
|
|
3528
3718
|
location: "local",
|
|
3529
3719
|
summary: "Ingest an external URL or a local file into the asset store. Declare the kind you expect (image/video/audio/text/json/font); the node picks the strategy. For source=url: yt-dlp for video/audio (YouTube/TikTok/Vimeo/etc. and direct file URLs), Handinger for HTML/PDF pages \u2192 markdown, direct HTTP fetch for binary URLs (images, fonts) and raw .txt/.md. For source=path: read from the local filesystem and upload to R2.",
|
|
@@ -3566,6 +3756,9 @@ function runStrategy(strategy, params, ctx) {
|
|
|
3566
3756
|
}
|
|
3567
3757
|
}
|
|
3568
3758
|
async function execDirectFetch(params, ctx) {
|
|
3759
|
+
if (params.expect === "image") {
|
|
3760
|
+
return ingestImageUrl(params.url, ctx);
|
|
3761
|
+
}
|
|
3569
3762
|
const result = await callBackendExec({
|
|
3570
3763
|
nodeType: "ingest",
|
|
3571
3764
|
nodeVersion: ingestNode.version,
|
|
@@ -3576,6 +3769,37 @@ async function execDirectFetch(params, ctx) {
|
|
|
3576
3769
|
});
|
|
3577
3770
|
return assertAssetOutput(result, params.expect);
|
|
3578
3771
|
}
|
|
3772
|
+
async function ingestImageUrl(url, ctx) {
|
|
3773
|
+
const res = await fetch(url);
|
|
3774
|
+
if (!res.ok) {
|
|
3775
|
+
throw localExecError(ctx, `fetch ${url} \u2192 ${res.status}`);
|
|
3776
|
+
}
|
|
3777
|
+
const ab = await res.arrayBuffer();
|
|
3778
|
+
if (ab.byteLength > MAX_ASSET_BYTES) {
|
|
3779
|
+
throw localExecError(ctx, `file_too_large: ${url} is ${ab.byteLength} bytes (limit ${MAX_ASSET_BYTES})`);
|
|
3780
|
+
}
|
|
3781
|
+
let normalized;
|
|
3782
|
+
try {
|
|
3783
|
+
normalized = await toModelSafeImage(Buffer.from(ab));
|
|
3784
|
+
} catch (e) {
|
|
3785
|
+
throw localExecError(ctx, `${url}: ${e.message}`);
|
|
3786
|
+
}
|
|
3787
|
+
if (normalized.rasterizedFrom) {
|
|
3788
|
+
ctx.log(`ingest: normalized ${normalized.rasterizedFrom} URL -> PNG (${normalized.bytes.length}B)`);
|
|
3789
|
+
}
|
|
3790
|
+
return uploadAndIngest({
|
|
3791
|
+
bytes: normalized.bytes,
|
|
3792
|
+
kind: "image",
|
|
3793
|
+
mime: normalized.mime,
|
|
3794
|
+
metadata: {
|
|
3795
|
+
source_url: url,
|
|
3796
|
+
strategy: "direct_fetch",
|
|
3797
|
+
ingested_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3798
|
+
...normalized.rasterizedFrom ? { rasterized_from: normalized.rasterizedFrom } : {}
|
|
3799
|
+
},
|
|
3800
|
+
ctx
|
|
3801
|
+
});
|
|
3802
|
+
}
|
|
3579
3803
|
async function execHandinger(params, ctx) {
|
|
3580
3804
|
const result = await callBackendExec({
|
|
3581
3805
|
nodeType: "ingest",
|
|
@@ -3663,6 +3887,25 @@ async function rasterizeSvgToPng(bytes) {
|
|
|
3663
3887
|
}
|
|
3664
3888
|
return await sharp(bytes, { density }).png({ force: true, palette: false }).toBuffer();
|
|
3665
3889
|
}
|
|
3890
|
+
var MODEL_SAFE_IMAGE_MIMES = /* @__PURE__ */ new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
|
|
3891
|
+
async function toModelSafeImage(bytes) {
|
|
3892
|
+
const safe = sniffImageMime(bytes);
|
|
3893
|
+
if (safe && MODEL_SAFE_IMAGE_MIMES.has(safe)) {
|
|
3894
|
+
return { bytes, mime: safe };
|
|
3895
|
+
}
|
|
3896
|
+
if (sniffSvg(bytes)) {
|
|
3897
|
+
return { bytes: await rasterizeSvgToPng(bytes), mime: "image/png", rasterizedFrom: "svg" };
|
|
3898
|
+
}
|
|
3899
|
+
const { default: sharp } = await import("sharp");
|
|
3900
|
+
try {
|
|
3901
|
+
const img = sharp(bytes);
|
|
3902
|
+
const format = (await img.metadata()).format;
|
|
3903
|
+
const png = await img.png({ force: true }).toBuffer();
|
|
3904
|
+
return { bytes: png, mime: "image/png", rasterizedFrom: format ?? "unknown" };
|
|
3905
|
+
} catch (e) {
|
|
3906
|
+
throw new Error(`bytes are not a decodable image (${e.message})`);
|
|
3907
|
+
}
|
|
3908
|
+
}
|
|
3666
3909
|
function sniffImageMime(buf) {
|
|
3667
3910
|
if (buf.length < 4) return null;
|
|
3668
3911
|
if (buf[0] === 137 && buf[1] === 80 && buf[2] === 78 && buf[3] === 71) return "image/png";
|
|
@@ -3724,10 +3967,10 @@ function inferKindFromMime(mime) {
|
|
|
3724
3967
|
if (mime.startsWith("font/")) return "font";
|
|
3725
3968
|
return null;
|
|
3726
3969
|
}
|
|
3727
|
-
function localExecError(ctx,
|
|
3970
|
+
function localExecError(ctx, message2) {
|
|
3728
3971
|
return new NodeExecutionError(ctx.nodeId, ctx.nodeType, {
|
|
3729
3972
|
kind: "local",
|
|
3730
|
-
cause: new Error(`ingest: ${
|
|
3973
|
+
cause: new Error(`ingest: ${message2}`)
|
|
3731
3974
|
});
|
|
3732
3975
|
}
|
|
3733
3976
|
async function execLocalFile(params, ctx) {
|
|
@@ -3774,17 +4017,20 @@ async function execLocalFile(params, ctx) {
|
|
|
3774
4017
|
ctx.log(`ingest: local file ${stats.size}B mime=${mime}`);
|
|
3775
4018
|
let outBytes = bytes;
|
|
3776
4019
|
let outMime = mime;
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
4020
|
+
let rasterizedFrom;
|
|
4021
|
+
if (kind === "image") {
|
|
4022
|
+
const normalized = await toModelSafeImage(bytes);
|
|
4023
|
+
outBytes = normalized.bytes;
|
|
4024
|
+
outMime = normalized.mime;
|
|
4025
|
+
rasterizedFrom = normalized.rasterizedFrom;
|
|
4026
|
+
if (rasterizedFrom) ctx.log(`ingest: normalized ${rasterizedFrom} -> PNG (${outBytes.length}B)`);
|
|
3781
4027
|
}
|
|
3782
4028
|
const durationMs = probeVideoDurationMs(params.expect, outBytes, ctx);
|
|
3783
4029
|
const ref = await uploadAndIngest({
|
|
3784
4030
|
bytes: outBytes,
|
|
3785
4031
|
kind: params.expect,
|
|
3786
4032
|
mime: outMime,
|
|
3787
|
-
metadata: localFileMetadata({ absPath, fileSize: stats.size, mime, durationMs }),
|
|
4033
|
+
metadata: localFileMetadata({ absPath, fileSize: stats.size, mime, durationMs, rasterizedFrom }),
|
|
3788
4034
|
ctx
|
|
3789
4035
|
});
|
|
3790
4036
|
return withProbedDuration(ref, durationMs);
|
|
@@ -3802,7 +4048,7 @@ function localFileMetadata(args) {
|
|
|
3802
4048
|
ingested_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3803
4049
|
file_size: args.fileSize,
|
|
3804
4050
|
original_filename: path3.basename(args.absPath),
|
|
3805
|
-
...args.
|
|
4051
|
+
...args.rasterizedFrom ? { rasterized_from: args.rasterizedFrom } : {},
|
|
3806
4052
|
...args.durationMs !== void 0 ? { duration_ms: args.durationMs } : {}
|
|
3807
4053
|
};
|
|
3808
4054
|
}
|
|
@@ -4394,7 +4640,7 @@ async function refToUrl(ref) {
|
|
|
4394
4640
|
return `data:${ref.mime};base64,${bytes.toString("base64")}`;
|
|
4395
4641
|
}
|
|
4396
4642
|
var ASSET_KINDS = /* @__PURE__ */ new Set(["image", "video", "audio", "json", "text", "font"]);
|
|
4397
|
-
function
|
|
4643
|
+
function isAssetRefLike2(value) {
|
|
4398
4644
|
if (!value || typeof value !== "object") return false;
|
|
4399
4645
|
const v = value;
|
|
4400
4646
|
return typeof v.kind === "string" && ASSET_KINDS.has(v.kind) && typeof v.mime === "string" && typeof v.sha256 === "string" && (typeof v.url === "string" || typeof v.path === "string");
|
|
@@ -4766,8 +5012,8 @@ var NEVER_BLOCK = [
|
|
|
4766
5012
|
/text[_-]?occluded/i
|
|
4767
5013
|
];
|
|
4768
5014
|
var UNAVAILABLE = /unknown command|command not found|not found|Did you mean|Unknown argument|ENOENT/i;
|
|
4769
|
-
function isAdvisory(code,
|
|
4770
|
-
const hay = `${code} ${
|
|
5015
|
+
function isAdvisory(code, message2) {
|
|
5016
|
+
const hay = `${code} ${message2}`;
|
|
4771
5017
|
return NEVER_BLOCK.some((re) => re.test(hay));
|
|
4772
5018
|
}
|
|
4773
5019
|
function parseCheckJson(raw) {
|
|
@@ -4795,10 +5041,10 @@ function classifyLint(json) {
|
|
|
4795
5041
|
for (const f of findings) {
|
|
4796
5042
|
const rec = f;
|
|
4797
5043
|
const code = String(rec?.code ?? "");
|
|
4798
|
-
const
|
|
5044
|
+
const message2 = String(rec?.message ?? "");
|
|
4799
5045
|
const severity = String(rec?.severity ?? "info");
|
|
4800
|
-
const blocking = severity === "error" && !isAdvisory(code,
|
|
4801
|
-
out.push({ source: "lint", code, message, severity: blocking ? "blocking" : "warning" });
|
|
5046
|
+
const blocking = severity === "error" && !isAdvisory(code, message2);
|
|
5047
|
+
out.push({ source: "lint", code, message: message2, severity: blocking ? "blocking" : "warning" });
|
|
4802
5048
|
}
|
|
4803
5049
|
return out;
|
|
4804
5050
|
}
|
|
@@ -4810,9 +5056,9 @@ function classifyInspect(json) {
|
|
|
4810
5056
|
for (const iss of issues) {
|
|
4811
5057
|
const rec = iss;
|
|
4812
5058
|
const code = String(rec?.code ?? rec?.type ?? "overflow");
|
|
4813
|
-
const
|
|
5059
|
+
const message2 = String(rec?.message ?? rec?.detail ?? JSON.stringify(iss));
|
|
4814
5060
|
const severity = rec?.severity ? String(rec.severity) : obj?.ok === false ? "error" : "warning";
|
|
4815
|
-
out.push({ source: "inspect", code, message, severity: severity === "error" ? "blocking" : "warning" });
|
|
5061
|
+
out.push({ source: "inspect", code, message: message2, severity: severity === "error" ? "blocking" : "warning" });
|
|
4816
5062
|
}
|
|
4817
5063
|
return out;
|
|
4818
5064
|
}
|
|
@@ -5257,7 +5503,7 @@ async function buildSubstitutionValues(compositionParams, meta, duration) {
|
|
|
5257
5503
|
}
|
|
5258
5504
|
function coerceImageParam(value) {
|
|
5259
5505
|
if (typeof value === "string") return Promise.resolve(value);
|
|
5260
|
-
if (
|
|
5506
|
+
if (isAssetRefLike2(value)) return refToUrl(value);
|
|
5261
5507
|
throw new Error("hyperframe_render: image param must be a URL string or AssetRef");
|
|
5262
5508
|
}
|
|
5263
5509
|
async function substituteCompositionFiles(tmp, values) {
|
|
@@ -5487,7 +5733,7 @@ async function buildSubstitutionValues2(compositionParams, meta) {
|
|
|
5487
5733
|
}
|
|
5488
5734
|
function coerceImageParam2(value) {
|
|
5489
5735
|
if (typeof value === "string") return Promise.resolve(value);
|
|
5490
|
-
if (
|
|
5736
|
+
if (isAssetRefLike2(value)) return refToUrl(value);
|
|
5491
5737
|
throw new Error("hyperframe_snapshot: image param must be a URL string or AssetRef");
|
|
5492
5738
|
}
|
|
5493
5739
|
async function substituteCompositionFiles2(tmp, values) {
|
|
@@ -6513,17 +6759,29 @@ function createEngineFromEnv(opts = {}) {
|
|
|
6513
6759
|
const cacheDir = opts.cacheDir ?? path15.join(cwd, "canvas", ".cache");
|
|
6514
6760
|
const outputsDir = opts.outputsDir ?? path15.join(cwd, "canvas");
|
|
6515
6761
|
const creds = requireCredentialsFromEnv();
|
|
6762
|
+
const client = new BackendClient({ baseUrl: creds.url, apiKey: creds.apiKey });
|
|
6763
|
+
const assets = new LocalAssetStore(path15.join(cacheDir, "assets"));
|
|
6764
|
+
const localCache = new LocalCacheStore(path15.join(cacheDir, "index"));
|
|
6765
|
+
const remoteCacheEnabled = opts.remoteCache ?? remoteCacheEnabledFromEnv();
|
|
6766
|
+
const cache = remoteCacheEnabled ? new LayeredCacheStore({
|
|
6767
|
+
local: localCache,
|
|
6768
|
+
remote: new RemoteCacheStore(client, opts.log),
|
|
6769
|
+
assets,
|
|
6770
|
+
log: opts.log
|
|
6771
|
+
}) : localCache;
|
|
6516
6772
|
return new Engine({
|
|
6517
6773
|
registry: defaultRegistry(),
|
|
6518
|
-
client
|
|
6519
|
-
assets
|
|
6520
|
-
cache
|
|
6774
|
+
client,
|
|
6775
|
+
assets,
|
|
6776
|
+
cache,
|
|
6521
6777
|
outputsDir,
|
|
6522
|
-
log: opts.log
|
|
6778
|
+
log: opts.log,
|
|
6779
|
+
persistAssets: remoteCacheEnabled
|
|
6523
6780
|
});
|
|
6524
6781
|
}
|
|
6525
6782
|
|
|
6526
6783
|
export {
|
|
6784
|
+
requireCredentialsFromEnv,
|
|
6527
6785
|
LayerExecutionError,
|
|
6528
6786
|
describeFailureReason,
|
|
6529
6787
|
SEEDANCE_DURATIONS,
|
|
@@ -6531,7 +6789,12 @@ export {
|
|
|
6531
6789
|
IMAGE_GENERATE_MODELS,
|
|
6532
6790
|
MODEL_REGISTRY,
|
|
6533
6791
|
resolveConcurrency,
|
|
6792
|
+
ulid,
|
|
6793
|
+
isPersistedAssetRef,
|
|
6794
|
+
collectAssetRefLikes,
|
|
6795
|
+
sha256Hex,
|
|
6534
6796
|
elementMentionKeywords,
|
|
6797
|
+
toModelSafeImage,
|
|
6535
6798
|
BackendClient2 as BackendClient,
|
|
6536
6799
|
Engine2 as Engine,
|
|
6537
6800
|
LocalAssetStore2 as LocalAssetStore,
|
|
@@ -6542,4 +6805,4 @@ export {
|
|
|
6542
6805
|
defaultRegistry,
|
|
6543
6806
|
createEngineFromEnv
|
|
6544
6807
|
};
|
|
6545
|
-
//# sourceMappingURL=chunk-
|
|
6808
|
+
//# sourceMappingURL=chunk-T6HBTZOO.js.map
|