@koda-sl/baker-cli 0.122.0-dev.57a9836c5 → 0.122.0-dev.fff192e73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -39
- package/dist/{chunk-SH6L4BCQ.js → chunk-MWFJ5NOP.js} +73 -335
- package/dist/chunk-MWFJ5NOP.js.map +1 -0
- package/dist/cli.js +1811 -1745
- package/dist/cli.js.map +1 -1
- package/dist/engine/index.d.ts +0 -58
- package/dist/engine/index.js +1 -1
- package/package.json +1 -1
- package/dist/chunk-SH6L4BCQ.js.map +0 -1
|
@@ -138,9 +138,9 @@ var require_safe_stable_stringify = __commonJS({
|
|
|
138
138
|
}
|
|
139
139
|
if (value) {
|
|
140
140
|
return (value2) => {
|
|
141
|
-
let
|
|
142
|
-
if (typeof value2 !== "function")
|
|
143
|
-
throw new Error(
|
|
141
|
+
let message = `Object can not safely be stringified. Received type ${typeof value2}`;
|
|
142
|
+
if (typeof value2 !== "function") message += ` (${value2.toString()})`;
|
|
143
|
+
throw new Error(message);
|
|
144
144
|
};
|
|
145
145
|
}
|
|
146
146
|
}
|
|
@@ -652,9 +652,6 @@ var HttpClient = class {
|
|
|
652
652
|
async postJson(path16, body, signal) {
|
|
653
653
|
return await this.requestJson("POST", path16, body, signal);
|
|
654
654
|
}
|
|
655
|
-
async putJson(path16, body, signal) {
|
|
656
|
-
return await this.requestJson("PUT", path16, body, signal);
|
|
657
|
-
}
|
|
658
655
|
async getJson(path16, signal) {
|
|
659
656
|
return await this.requestJson("GET", path16, void 0, signal);
|
|
660
657
|
}
|
|
@@ -682,8 +679,8 @@ var HttpClient = class {
|
|
|
682
679
|
try {
|
|
683
680
|
const res = await this.fetchFn(url, {
|
|
684
681
|
method,
|
|
685
|
-
headers: method === "
|
|
686
|
-
body: method === "
|
|
682
|
+
headers: method === "POST" ? { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` } : { Authorization: `Bearer ${this.apiKey}` },
|
|
683
|
+
body: method === "POST" ? JSON.stringify(body) : void 0,
|
|
687
684
|
signal: controller.signal
|
|
688
685
|
});
|
|
689
686
|
if (res.ok) return { kind: "value", value: await res.json() };
|
|
@@ -720,33 +717,33 @@ async function parseErrorBody(res) {
|
|
|
720
717
|
const errObj = body.error ?? {};
|
|
721
718
|
return classifyHttpError(res.status, errObj, errObj.message ?? `HTTP ${res.status}`);
|
|
722
719
|
}
|
|
723
|
-
function classifyHttpError(status, errObj,
|
|
720
|
+
function classifyHttpError(status, errObj, message) {
|
|
724
721
|
if (errObj.code === CONTENT_POLICY_CODE) {
|
|
725
|
-
return { kind: "content_policy", status, provider: errObj.provider, message
|
|
722
|
+
return { kind: "content_policy", status, provider: errObj.provider, message };
|
|
726
723
|
}
|
|
727
724
|
if (status === 401 || status === 403) {
|
|
728
|
-
return { kind: "unauthorized", status, message
|
|
725
|
+
return { kind: "unauthorized", status, message };
|
|
729
726
|
}
|
|
730
727
|
if (status === 400 || status === 422) {
|
|
731
|
-
return { kind: "validation", status, message
|
|
728
|
+
return { kind: "validation", status, message, details: errObj.details };
|
|
732
729
|
}
|
|
733
730
|
if (status === 502 || status === 504) {
|
|
734
731
|
if (errObj.code === "provider_timeout" || status === 504) {
|
|
735
|
-
return { kind: "timeout", provider: errObj.provider, message
|
|
732
|
+
return { kind: "timeout", provider: errObj.provider, message };
|
|
736
733
|
}
|
|
737
734
|
return {
|
|
738
735
|
kind: "provider",
|
|
739
736
|
status,
|
|
740
737
|
provider: errObj.provider,
|
|
741
738
|
code: errObj.code ?? "provider_error",
|
|
742
|
-
message
|
|
739
|
+
message,
|
|
743
740
|
retryable: errObj.retryable ?? true
|
|
744
741
|
};
|
|
745
742
|
}
|
|
746
743
|
if (status >= 500 || status === 429) {
|
|
747
|
-
return { kind: "server", status, message
|
|
744
|
+
return { kind: "server", status, message };
|
|
748
745
|
}
|
|
749
|
-
return { kind: "validation", status, message
|
|
746
|
+
return { kind: "validation", status, message, details: errObj.details };
|
|
750
747
|
}
|
|
751
748
|
function backoffMs(attempt) {
|
|
752
749
|
return 1e3 * 2 ** attempt;
|
|
@@ -817,27 +814,6 @@ var BackendClient = class {
|
|
|
817
814
|
signal
|
|
818
815
|
);
|
|
819
816
|
}
|
|
820
|
-
/** Remote cache lookup. A miss (404) — or an old backend without the route — returns null. */
|
|
821
|
-
async getCacheEntry(cacheKey, signal) {
|
|
822
|
-
try {
|
|
823
|
-
const res = await this.http.getJson(`/api/canvas/cache/${encodeURIComponent(cacheKey)}`, signal);
|
|
824
|
-
return res.entry;
|
|
825
|
-
} catch (e) {
|
|
826
|
-
if (e instanceof BackendHttpError && "status" in e.detail && e.detail.status === 404) return null;
|
|
827
|
-
throw e;
|
|
828
|
-
}
|
|
829
|
-
}
|
|
830
|
-
async putCacheEntry(entry, signal) {
|
|
831
|
-
await this.http.putJson(
|
|
832
|
-
`/api/canvas/cache/${encodeURIComponent(entry.cacheKey)}`,
|
|
833
|
-
entry,
|
|
834
|
-
signal
|
|
835
|
-
);
|
|
836
|
-
}
|
|
837
|
-
/** Durable run-history record — POST /api/canvas/runs (idempotent server-side on runId). */
|
|
838
|
-
async recordRun(payload, signal) {
|
|
839
|
-
await this.http.postJson("/api/canvas/runs", payload, signal);
|
|
840
|
-
}
|
|
841
817
|
getArtifact(kind, name, version, signal) {
|
|
842
818
|
const path16 = version ? `/api/canvas/artifacts/${encodeURIComponent(kind)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}` : `/api/canvas/artifacts/${encodeURIComponent(kind)}/${encodeURIComponent(name)}`;
|
|
843
819
|
return this.http.getJson(path16, signal);
|
|
@@ -861,17 +837,14 @@ function requireCredentialsFromEnv(env = process.env) {
|
|
|
861
837
|
}
|
|
862
838
|
return c;
|
|
863
839
|
}
|
|
864
|
-
function remoteCacheEnabledFromEnv(env = process.env) {
|
|
865
|
-
return env.BAKER_CANVAS_REMOTE_CACHE !== "off";
|
|
866
|
-
}
|
|
867
840
|
|
|
868
841
|
// src/engine/engine/errors.ts
|
|
869
842
|
function isBlocking(issue) {
|
|
870
843
|
return issue.severity !== "warning";
|
|
871
844
|
}
|
|
872
845
|
var CanvasError = class extends Error {
|
|
873
|
-
constructor(
|
|
874
|
-
super(
|
|
846
|
+
constructor(message) {
|
|
847
|
+
super(message);
|
|
875
848
|
this.name = "CanvasError";
|
|
876
849
|
}
|
|
877
850
|
};
|
|
@@ -1589,160 +1562,6 @@ function encodeRandom() {
|
|
|
1589
1562
|
return out;
|
|
1590
1563
|
}
|
|
1591
1564
|
|
|
1592
|
-
// src/engine/storage/remote-cache-store.ts
|
|
1593
|
-
var CANVAS_ASSETS_URL_SEGMENT = "/canvas-assets/";
|
|
1594
|
-
function isPersistedAssetRef(ref) {
|
|
1595
|
-
const { url, sha256 } = ref;
|
|
1596
|
-
if (typeof url !== "string" || typeof sha256 !== "string") return false;
|
|
1597
|
-
return url.includes(`${CANVAS_ASSETS_URL_SEGMENT}${sha256.slice(0, 2)}/${sha256}`) || url.includes(`${CANVAS_ASSETS_URL_SEGMENT}${sha256}`);
|
|
1598
|
-
}
|
|
1599
|
-
function isAssetRefLike(value) {
|
|
1600
|
-
return typeof value === "object" && value !== null && !Array.isArray(value) && typeof value.sha256 === "string" && typeof value.mime === "string";
|
|
1601
|
-
}
|
|
1602
|
-
function collectAssetRefLikes(value, out = []) {
|
|
1603
|
-
if (Array.isArray(value)) {
|
|
1604
|
-
for (const item of value) collectAssetRefLikes(item, out);
|
|
1605
|
-
return out;
|
|
1606
|
-
}
|
|
1607
|
-
if (typeof value !== "object" || value === null) return out;
|
|
1608
|
-
if (isAssetRefLike(value)) {
|
|
1609
|
-
out.push(value);
|
|
1610
|
-
}
|
|
1611
|
-
for (const item of Object.values(value)) collectAssetRefLikes(item, out);
|
|
1612
|
-
return out;
|
|
1613
|
-
}
|
|
1614
|
-
function entryFullyPersisted(entry) {
|
|
1615
|
-
return collectAssetRefLikes(entry.outputs).every((ref) => isPersistedAssetRef(ref));
|
|
1616
|
-
}
|
|
1617
|
-
function stripLocalFields(entry) {
|
|
1618
|
-
const clone = JSON.parse(JSON.stringify(entry));
|
|
1619
|
-
for (const ref of collectAssetRefLikes(clone.outputs)) {
|
|
1620
|
-
delete ref.path;
|
|
1621
|
-
delete ref.bytes;
|
|
1622
|
-
}
|
|
1623
|
-
return clone;
|
|
1624
|
-
}
|
|
1625
|
-
var RemoteCacheStore = class {
|
|
1626
|
-
client;
|
|
1627
|
-
log;
|
|
1628
|
-
constructor(client, log) {
|
|
1629
|
-
this.client = client;
|
|
1630
|
-
this.log = log ?? (() => void 0);
|
|
1631
|
-
}
|
|
1632
|
-
async get(cacheKey) {
|
|
1633
|
-
return await this.client.getCacheEntry(cacheKey);
|
|
1634
|
-
}
|
|
1635
|
-
async put(entry) {
|
|
1636
|
-
if (!entryFullyPersisted(entry)) {
|
|
1637
|
-
this.log(`[cache ] ${entry.cacheKey.slice(0, 12)}\u2026 has local-only assets, kept local`);
|
|
1638
|
-
return;
|
|
1639
|
-
}
|
|
1640
|
-
const stripped = stripLocalFields(entry);
|
|
1641
|
-
if (stripped.refs.length > MAX_REMOTE_REFS) {
|
|
1642
|
-
stripped.refs = stripped.refs.slice(0, MAX_REMOTE_REFS);
|
|
1643
|
-
}
|
|
1644
|
-
await this.client.putCacheEntry(stripped);
|
|
1645
|
-
}
|
|
1646
|
-
};
|
|
1647
|
-
var MAX_REMOTE_REFS = 512;
|
|
1648
|
-
var LayeredCacheStore = class {
|
|
1649
|
-
rootDir;
|
|
1650
|
-
local;
|
|
1651
|
-
remote;
|
|
1652
|
-
assets;
|
|
1653
|
-
log;
|
|
1654
|
-
constructor(opts) {
|
|
1655
|
-
this.local = opts.local;
|
|
1656
|
-
this.remote = opts.remote;
|
|
1657
|
-
this.assets = opts.assets;
|
|
1658
|
-
this.rootDir = opts.local.rootDir;
|
|
1659
|
-
this.log = opts.log ?? (() => void 0);
|
|
1660
|
-
}
|
|
1661
|
-
async get(cacheKey) {
|
|
1662
|
-
const localHit = await this.local.get(cacheKey);
|
|
1663
|
-
if (localHit) return localHit;
|
|
1664
|
-
let remoteEntry;
|
|
1665
|
-
try {
|
|
1666
|
-
remoteEntry = await this.remote.get(cacheKey);
|
|
1667
|
-
} catch (e) {
|
|
1668
|
-
this.log(`[cache ] remote lookup failed (${message(e)}) \u2014 treating as miss`);
|
|
1669
|
-
return null;
|
|
1670
|
-
}
|
|
1671
|
-
if (!remoteEntry) return null;
|
|
1672
|
-
let rehydrated;
|
|
1673
|
-
try {
|
|
1674
|
-
rehydrated = await this.rehydrate(remoteEntry);
|
|
1675
|
-
} catch (e) {
|
|
1676
|
-
this.log(`[cache ] ${cacheKey.slice(0, 12)}\u2026 rehydration failed (${message(e)}) \u2014 treating as miss`);
|
|
1677
|
-
return null;
|
|
1678
|
-
}
|
|
1679
|
-
await this.local.put(rehydrated);
|
|
1680
|
-
return rehydrated;
|
|
1681
|
-
}
|
|
1682
|
-
async put(entry) {
|
|
1683
|
-
await this.local.put(entry);
|
|
1684
|
-
try {
|
|
1685
|
-
await this.remote.put(entry);
|
|
1686
|
-
} catch (e) {
|
|
1687
|
-
this.log(`[cache ] remote write failed (${message(e)}) \u2014 entry kept local`);
|
|
1688
|
-
}
|
|
1689
|
-
}
|
|
1690
|
-
/**
|
|
1691
|
-
* Download every referenced asset into the local content-addressed store
|
|
1692
|
-
* (sha-verified) and stamp fresh local paths. Any ref that cannot be
|
|
1693
|
-
* rehydrated fails the WHOLE entry — a partially-hydrated cache hit would
|
|
1694
|
-
* crash materialization later with a far less actionable error.
|
|
1695
|
-
*/
|
|
1696
|
-
async rehydrate(entry) {
|
|
1697
|
-
const clone = JSON.parse(JSON.stringify(entry));
|
|
1698
|
-
for (const ref of collectAssetRefLikes(clone.outputs)) {
|
|
1699
|
-
if (!isPersistedAssetRef(ref)) {
|
|
1700
|
-
throw new Error(`ref ${ref.sha256.slice(0, 12)}\u2026 has no persisted url`);
|
|
1701
|
-
}
|
|
1702
|
-
const ingested = await this.assets.ingestRemote({
|
|
1703
|
-
kind: typeof ref.kind === "string" ? ref.kind : "json",
|
|
1704
|
-
url: ref.url,
|
|
1705
|
-
sha256: ref.sha256,
|
|
1706
|
-
mime: ref.mime,
|
|
1707
|
-
metadata: ref.metadata ?? void 0
|
|
1708
|
-
});
|
|
1709
|
-
ref.path = ingested.path;
|
|
1710
|
-
}
|
|
1711
|
-
return clone;
|
|
1712
|
-
}
|
|
1713
|
-
};
|
|
1714
|
-
function message(e) {
|
|
1715
|
-
return e instanceof Error ? e.message : String(e);
|
|
1716
|
-
}
|
|
1717
|
-
|
|
1718
|
-
// src/engine/nodes/remote/upload.ts
|
|
1719
|
-
async function presignAndPut(args) {
|
|
1720
|
-
const { putUrl, publicUrl } = await args.ctx.client.presignAssetUpload(args.sha256, args.mime, args.ctx.signal);
|
|
1721
|
-
const putRes = await fetch(putUrl, {
|
|
1722
|
-
method: "PUT",
|
|
1723
|
-
body: new Uint8Array(args.bytes),
|
|
1724
|
-
headers: { "Content-Type": args.mime },
|
|
1725
|
-
signal: args.ctx.signal
|
|
1726
|
-
});
|
|
1727
|
-
if (!putRes.ok) {
|
|
1728
|
-
throw new Error(`upload: presigned PUT failed ${putRes.status} ${putRes.statusText}`);
|
|
1729
|
-
}
|
|
1730
|
-
return publicUrl;
|
|
1731
|
-
}
|
|
1732
|
-
async function ensureUploaded(ref, ctx) {
|
|
1733
|
-
if (ref.url) return ref;
|
|
1734
|
-
const bytes = await ctx.assets.readBytes(ref.sha256, ref.mime);
|
|
1735
|
-
const url = await presignAndPut({ bytes, sha256: ref.sha256, mime: ref.mime, ctx });
|
|
1736
|
-
return { ...ref, url };
|
|
1737
|
-
}
|
|
1738
|
-
async function persistOutputAssetUrls(outputs, ctx) {
|
|
1739
|
-
for (const ref of collectAssetRefLikes(outputs)) {
|
|
1740
|
-
if (isPersistedAssetRef(ref)) continue;
|
|
1741
|
-
const bytes = await ctx.assets.readBytes(ref.sha256, ref.mime);
|
|
1742
|
-
ref.url = await presignAndPut({ bytes, sha256: ref.sha256, mime: ref.mime, ctx });
|
|
1743
|
-
}
|
|
1744
|
-
}
|
|
1745
|
-
|
|
1746
1565
|
// src/engine/schema/canvas.ts
|
|
1747
1566
|
import { z } from "zod";
|
|
1748
1567
|
var REF_PREFIX = "$ref:";
|
|
@@ -3022,7 +2841,6 @@ var Engine = class {
|
|
|
3022
2841
|
cache;
|
|
3023
2842
|
outputsDir;
|
|
3024
2843
|
log;
|
|
3025
|
-
persistAssets;
|
|
3026
2844
|
constructor(opts) {
|
|
3027
2845
|
this.registry = opts.registry;
|
|
3028
2846
|
this.client = opts.client;
|
|
@@ -3030,7 +2848,6 @@ var Engine = class {
|
|
|
3030
2848
|
this.cache = opts.cache;
|
|
3031
2849
|
this.outputsDir = opts.outputsDir;
|
|
3032
2850
|
this.log = opts.log ?? (() => void 0);
|
|
3033
|
-
this.persistAssets = opts.persistAssets ?? false;
|
|
3034
2851
|
}
|
|
3035
2852
|
validate(canvas) {
|
|
3036
2853
|
return validateCanvas(canvas, this.registry);
|
|
@@ -3054,15 +2871,7 @@ var Engine = class {
|
|
|
3054
2871
|
const outputs = {};
|
|
3055
2872
|
const counters = { cachedNodes: 0, totalCredits: 0 };
|
|
3056
2873
|
const nodeRuns = [];
|
|
3057
|
-
|
|
3058
|
-
this.emitProgress(opts, {
|
|
3059
|
-
kind: "plan",
|
|
3060
|
-
nodes: [...graph.entries()].map(([id, deps]) => {
|
|
3061
|
-
const node = canvas.nodes.find((n) => n.id === id);
|
|
3062
|
-
return { node_id: id, node_type: node?.type ?? "unknown", deps: [...deps], params: node?.params };
|
|
3063
|
-
})
|
|
3064
|
-
});
|
|
3065
|
-
await this.runLayers(canvas, graph, outputs, runId, writer, opts, counters, nodeRuns);
|
|
2874
|
+
await this.runLayers(canvas, outputs, runId, writer, opts, counters, nodeRuns);
|
|
3066
2875
|
const output = pickFinalOutput(canvas, outputs);
|
|
3067
2876
|
const stats = {
|
|
3068
2877
|
total_nodes: canvas.nodes.length,
|
|
@@ -3085,51 +2894,39 @@ var Engine = class {
|
|
|
3085
2894
|
`[done ] ${stats.cached_nodes}/${stats.total_nodes} cached, ${stats.total_credits} credits, ${stats.duration_ms}ms`
|
|
3086
2895
|
);
|
|
3087
2896
|
this.log(`outputs in: ${writer.runDir}`);
|
|
3088
|
-
return { run_id: runId, output, outputs_by_node: outputs, stats, outputs_dir: writer.runDir
|
|
2897
|
+
return { run_id: runId, output, outputs_by_node: outputs, stats, outputs_dir: writer.runDir };
|
|
3089
2898
|
}
|
|
3090
|
-
async runLayers(canvas,
|
|
3091
|
-
const layers = topologicalLayers(
|
|
2899
|
+
async runLayers(canvas, outputs, runId, writer, opts, counters, nodeRuns) {
|
|
2900
|
+
const layers = topologicalLayers(this.pruneToOutput(canvas, buildGraph(canvas)));
|
|
3092
2901
|
const limit = resolveConcurrency(opts.concurrency);
|
|
3093
2902
|
for (const layer of layers) {
|
|
3094
|
-
const settled = await mapWithConcurrency(
|
|
3095
|
-
|
|
3096
|
-
|
|
2903
|
+
const settled = await mapWithConcurrency(
|
|
2904
|
+
layer,
|
|
2905
|
+
limit,
|
|
2906
|
+
(nodeId) => this.executeOne(canvas, nodeId, outputs, runId, writer, opts).then((r) => {
|
|
3097
2907
|
if (r.cached) counters.cachedNodes++;
|
|
3098
2908
|
counters.totalCredits += r.credits;
|
|
3099
2909
|
const node = canvas.nodes.find((n) => n.id === nodeId);
|
|
3100
2910
|
if (node) {
|
|
3101
|
-
|
|
2911
|
+
nodeRuns.push({
|
|
3102
2912
|
node_id: nodeId,
|
|
3103
2913
|
node_type: node.type,
|
|
3104
2914
|
cached: r.cached,
|
|
3105
2915
|
duration_ms: r.durationMs,
|
|
3106
2916
|
credits: r.credits
|
|
3107
|
-
};
|
|
3108
|
-
nodeRuns.push(run);
|
|
3109
|
-
this.emitProgress(opts, { kind: "node_settled", run, outputs: outputs[nodeId] ?? {} });
|
|
2917
|
+
});
|
|
3110
2918
|
}
|
|
3111
|
-
})
|
|
3112
|
-
|
|
2919
|
+
})
|
|
2920
|
+
);
|
|
3113
2921
|
const failures = [];
|
|
3114
2922
|
settled.forEach((result, i) => {
|
|
3115
2923
|
const nodeId = layer[i];
|
|
3116
|
-
if (result.status === "rejected" && nodeId) {
|
|
3117
|
-
failures.push({ nodeId, reason: result.reason });
|
|
3118
|
-
this.emitProgress(opts, { kind: "node_failed", node_id: nodeId });
|
|
3119
|
-
}
|
|
2924
|
+
if (result.status === "rejected" && nodeId) failures.push({ nodeId, reason: result.reason });
|
|
3120
2925
|
});
|
|
3121
2926
|
if (failures.length === 1 && failures[0]) throw failures[0].reason;
|
|
3122
2927
|
if (failures.length > 1) throw new LayerExecutionError(failures);
|
|
3123
2928
|
}
|
|
3124
2929
|
}
|
|
3125
|
-
/** Progress consumers are observers only — an exception there must never fail the run. */
|
|
3126
|
-
emitProgress(opts, event) {
|
|
3127
|
-
if (!opts.onProgress) return;
|
|
3128
|
-
try {
|
|
3129
|
-
opts.onProgress(event);
|
|
3130
|
-
} catch {
|
|
3131
|
-
}
|
|
3132
|
-
}
|
|
3133
2930
|
/**
|
|
3134
2931
|
* Dead-node elimination: when the canvas declares an `output`, execute only the
|
|
3135
2932
|
* nodes that output transitively depends on. Orphaned nodes (left by an edit or
|
|
@@ -3194,14 +2991,6 @@ var Engine = class {
|
|
|
3194
2991
|
const credits = def.cost ? def.cost({ params: parsedParams }).credits : 0;
|
|
3195
2992
|
const outputsObj = result;
|
|
3196
2993
|
outputs[node.id] = outputsObj;
|
|
3197
|
-
if (this.persistAssets) {
|
|
3198
|
-
try {
|
|
3199
|
-
await persistOutputAssetUrls(outputsObj, ctx);
|
|
3200
|
-
} catch (e) {
|
|
3201
|
-
const msg = e instanceof Error ? e.message : String(e);
|
|
3202
|
-
this.log(`[warn ] ${node.id}: asset persistence failed (${msg}) \u2014 outputs stay local-only`);
|
|
3203
|
-
}
|
|
3204
|
-
}
|
|
3205
2994
|
if (policy === "read_write") {
|
|
3206
2995
|
await this.cache.put({
|
|
3207
2996
|
cacheKey: prepared.cacheKey,
|
|
@@ -3520,6 +3309,27 @@ var FontRef = BaseAssetRef.extend({
|
|
|
3520
3309
|
});
|
|
3521
3310
|
var AssetRef = z4.discriminatedUnion("kind", [ImageRef, VideoRef, AudioRef, JsonRef, TextRef, FontRef]);
|
|
3522
3311
|
|
|
3312
|
+
// src/engine/nodes/remote/upload.ts
|
|
3313
|
+
async function presignAndPut(args) {
|
|
3314
|
+
const { putUrl, publicUrl } = await args.ctx.client.presignAssetUpload(args.sha256, args.mime, args.ctx.signal);
|
|
3315
|
+
const putRes = await fetch(putUrl, {
|
|
3316
|
+
method: "PUT",
|
|
3317
|
+
body: new Uint8Array(args.bytes),
|
|
3318
|
+
headers: { "Content-Type": args.mime },
|
|
3319
|
+
signal: args.ctx.signal
|
|
3320
|
+
});
|
|
3321
|
+
if (!putRes.ok) {
|
|
3322
|
+
throw new Error(`upload: presigned PUT failed ${putRes.status} ${putRes.statusText}`);
|
|
3323
|
+
}
|
|
3324
|
+
return publicUrl;
|
|
3325
|
+
}
|
|
3326
|
+
async function ensureUploaded(ref, ctx) {
|
|
3327
|
+
if (ref.url) return ref;
|
|
3328
|
+
const bytes = await ctx.assets.readBytes(ref.sha256, ref.mime);
|
|
3329
|
+
const url = await presignAndPut({ bytes, sha256: ref.sha256, mime: ref.mime, ctx });
|
|
3330
|
+
return { ...ref, url };
|
|
3331
|
+
}
|
|
3332
|
+
|
|
3523
3333
|
// src/engine/nodes/remote/delegate.ts
|
|
3524
3334
|
function delegated(spec) {
|
|
3525
3335
|
return {
|
|
@@ -3713,7 +3523,7 @@ function safePathname(rawUrl) {
|
|
|
3713
3523
|
}
|
|
3714
3524
|
var ingestNode = defineNode({
|
|
3715
3525
|
id: "ingest",
|
|
3716
|
-
version: "1.
|
|
3526
|
+
version: "1.1.0",
|
|
3717
3527
|
category: "io",
|
|
3718
3528
|
location: "local",
|
|
3719
3529
|
summary: "Ingest an external URL or a local file into the asset store. Declare the kind you expect (image/video/audio/text/json/font); the node picks the strategy. For source=url: yt-dlp for video/audio (YouTube/TikTok/Vimeo/etc. and direct file URLs), Handinger for HTML/PDF pages \u2192 markdown, direct HTTP fetch for binary URLs (images, fonts) and raw .txt/.md. For source=path: read from the local filesystem and upload to R2.",
|
|
@@ -3756,9 +3566,6 @@ function runStrategy(strategy, params, ctx) {
|
|
|
3756
3566
|
}
|
|
3757
3567
|
}
|
|
3758
3568
|
async function execDirectFetch(params, ctx) {
|
|
3759
|
-
if (params.expect === "image") {
|
|
3760
|
-
return ingestImageUrl(params.url, ctx);
|
|
3761
|
-
}
|
|
3762
3569
|
const result = await callBackendExec({
|
|
3763
3570
|
nodeType: "ingest",
|
|
3764
3571
|
nodeVersion: ingestNode.version,
|
|
@@ -3769,37 +3576,6 @@ async function execDirectFetch(params, ctx) {
|
|
|
3769
3576
|
});
|
|
3770
3577
|
return assertAssetOutput(result, params.expect);
|
|
3771
3578
|
}
|
|
3772
|
-
async function ingestImageUrl(url, ctx) {
|
|
3773
|
-
const res = await fetch(url);
|
|
3774
|
-
if (!res.ok) {
|
|
3775
|
-
throw localExecError(ctx, `fetch ${url} \u2192 ${res.status}`);
|
|
3776
|
-
}
|
|
3777
|
-
const ab = await res.arrayBuffer();
|
|
3778
|
-
if (ab.byteLength > MAX_ASSET_BYTES) {
|
|
3779
|
-
throw localExecError(ctx, `file_too_large: ${url} is ${ab.byteLength} bytes (limit ${MAX_ASSET_BYTES})`);
|
|
3780
|
-
}
|
|
3781
|
-
let normalized;
|
|
3782
|
-
try {
|
|
3783
|
-
normalized = await toModelSafeImage(Buffer.from(ab));
|
|
3784
|
-
} catch (e) {
|
|
3785
|
-
throw localExecError(ctx, `${url}: ${e.message}`);
|
|
3786
|
-
}
|
|
3787
|
-
if (normalized.rasterizedFrom) {
|
|
3788
|
-
ctx.log(`ingest: normalized ${normalized.rasterizedFrom} URL -> PNG (${normalized.bytes.length}B)`);
|
|
3789
|
-
}
|
|
3790
|
-
return uploadAndIngest({
|
|
3791
|
-
bytes: normalized.bytes,
|
|
3792
|
-
kind: "image",
|
|
3793
|
-
mime: normalized.mime,
|
|
3794
|
-
metadata: {
|
|
3795
|
-
source_url: url,
|
|
3796
|
-
strategy: "direct_fetch",
|
|
3797
|
-
ingested_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3798
|
-
...normalized.rasterizedFrom ? { rasterized_from: normalized.rasterizedFrom } : {}
|
|
3799
|
-
},
|
|
3800
|
-
ctx
|
|
3801
|
-
});
|
|
3802
|
-
}
|
|
3803
3579
|
async function execHandinger(params, ctx) {
|
|
3804
3580
|
const result = await callBackendExec({
|
|
3805
3581
|
nodeType: "ingest",
|
|
@@ -3887,25 +3663,6 @@ async function rasterizeSvgToPng(bytes) {
|
|
|
3887
3663
|
}
|
|
3888
3664
|
return await sharp(bytes, { density }).png({ force: true, palette: false }).toBuffer();
|
|
3889
3665
|
}
|
|
3890
|
-
var MODEL_SAFE_IMAGE_MIMES = /* @__PURE__ */ new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
|
|
3891
|
-
async function toModelSafeImage(bytes) {
|
|
3892
|
-
const safe = sniffImageMime(bytes);
|
|
3893
|
-
if (safe && MODEL_SAFE_IMAGE_MIMES.has(safe)) {
|
|
3894
|
-
return { bytes, mime: safe };
|
|
3895
|
-
}
|
|
3896
|
-
if (sniffSvg(bytes)) {
|
|
3897
|
-
return { bytes: await rasterizeSvgToPng(bytes), mime: "image/png", rasterizedFrom: "svg" };
|
|
3898
|
-
}
|
|
3899
|
-
const { default: sharp } = await import("sharp");
|
|
3900
|
-
try {
|
|
3901
|
-
const img = sharp(bytes);
|
|
3902
|
-
const format = (await img.metadata()).format;
|
|
3903
|
-
const png = await img.png({ force: true }).toBuffer();
|
|
3904
|
-
return { bytes: png, mime: "image/png", rasterizedFrom: format ?? "unknown" };
|
|
3905
|
-
} catch (e) {
|
|
3906
|
-
throw new Error(`bytes are not a decodable image (${e.message})`);
|
|
3907
|
-
}
|
|
3908
|
-
}
|
|
3909
3666
|
function sniffImageMime(buf) {
|
|
3910
3667
|
if (buf.length < 4) return null;
|
|
3911
3668
|
if (buf[0] === 137 && buf[1] === 80 && buf[2] === 78 && buf[3] === 71) return "image/png";
|
|
@@ -3967,10 +3724,10 @@ function inferKindFromMime(mime) {
|
|
|
3967
3724
|
if (mime.startsWith("font/")) return "font";
|
|
3968
3725
|
return null;
|
|
3969
3726
|
}
|
|
3970
|
-
function localExecError(ctx,
|
|
3727
|
+
function localExecError(ctx, message) {
|
|
3971
3728
|
return new NodeExecutionError(ctx.nodeId, ctx.nodeType, {
|
|
3972
3729
|
kind: "local",
|
|
3973
|
-
cause: new Error(`ingest: ${
|
|
3730
|
+
cause: new Error(`ingest: ${message}`)
|
|
3974
3731
|
});
|
|
3975
3732
|
}
|
|
3976
3733
|
async function execLocalFile(params, ctx) {
|
|
@@ -4017,20 +3774,17 @@ async function execLocalFile(params, ctx) {
|
|
|
4017
3774
|
ctx.log(`ingest: local file ${stats.size}B mime=${mime}`);
|
|
4018
3775
|
let outBytes = bytes;
|
|
4019
3776
|
let outMime = mime;
|
|
4020
|
-
|
|
4021
|
-
|
|
4022
|
-
|
|
4023
|
-
|
|
4024
|
-
outMime = normalized.mime;
|
|
4025
|
-
rasterizedFrom = normalized.rasterizedFrom;
|
|
4026
|
-
if (rasterizedFrom) ctx.log(`ingest: normalized ${rasterizedFrom} -> PNG (${outBytes.length}B)`);
|
|
3777
|
+
if (mime === SVG_MIME) {
|
|
3778
|
+
outBytes = await rasterizeSvgToPng(bytes);
|
|
3779
|
+
outMime = "image/png";
|
|
3780
|
+
ctx.log(`ingest: rasterized SVG -> PNG (${outBytes.length}B)`);
|
|
4027
3781
|
}
|
|
4028
3782
|
const durationMs = probeVideoDurationMs(params.expect, outBytes, ctx);
|
|
4029
3783
|
const ref = await uploadAndIngest({
|
|
4030
3784
|
bytes: outBytes,
|
|
4031
3785
|
kind: params.expect,
|
|
4032
3786
|
mime: outMime,
|
|
4033
|
-
metadata: localFileMetadata({ absPath, fileSize: stats.size, mime, durationMs
|
|
3787
|
+
metadata: localFileMetadata({ absPath, fileSize: stats.size, mime, durationMs }),
|
|
4034
3788
|
ctx
|
|
4035
3789
|
});
|
|
4036
3790
|
return withProbedDuration(ref, durationMs);
|
|
@@ -4048,7 +3802,7 @@ function localFileMetadata(args) {
|
|
|
4048
3802
|
ingested_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4049
3803
|
file_size: args.fileSize,
|
|
4050
3804
|
original_filename: path3.basename(args.absPath),
|
|
4051
|
-
...args.
|
|
3805
|
+
...args.mime === SVG_MIME ? { rasterized_from: "svg" } : {},
|
|
4052
3806
|
...args.durationMs !== void 0 ? { duration_ms: args.durationMs } : {}
|
|
4053
3807
|
};
|
|
4054
3808
|
}
|
|
@@ -4640,7 +4394,7 @@ async function refToUrl(ref) {
|
|
|
4640
4394
|
return `data:${ref.mime};base64,${bytes.toString("base64")}`;
|
|
4641
4395
|
}
|
|
4642
4396
|
var ASSET_KINDS = /* @__PURE__ */ new Set(["image", "video", "audio", "json", "text", "font"]);
|
|
4643
|
-
function
|
|
4397
|
+
function isAssetRefLike(value) {
|
|
4644
4398
|
if (!value || typeof value !== "object") return false;
|
|
4645
4399
|
const v = value;
|
|
4646
4400
|
return typeof v.kind === "string" && ASSET_KINDS.has(v.kind) && typeof v.mime === "string" && typeof v.sha256 === "string" && (typeof v.url === "string" || typeof v.path === "string");
|
|
@@ -5012,8 +4766,8 @@ var NEVER_BLOCK = [
|
|
|
5012
4766
|
/text[_-]?occluded/i
|
|
5013
4767
|
];
|
|
5014
4768
|
var UNAVAILABLE = /unknown command|command not found|not found|Did you mean|Unknown argument|ENOENT/i;
|
|
5015
|
-
function isAdvisory(code,
|
|
5016
|
-
const hay = `${code} ${
|
|
4769
|
+
function isAdvisory(code, message) {
|
|
4770
|
+
const hay = `${code} ${message}`;
|
|
5017
4771
|
return NEVER_BLOCK.some((re) => re.test(hay));
|
|
5018
4772
|
}
|
|
5019
4773
|
function parseCheckJson(raw) {
|
|
@@ -5041,10 +4795,10 @@ function classifyLint(json) {
|
|
|
5041
4795
|
for (const f of findings) {
|
|
5042
4796
|
const rec = f;
|
|
5043
4797
|
const code = String(rec?.code ?? "");
|
|
5044
|
-
const
|
|
4798
|
+
const message = String(rec?.message ?? "");
|
|
5045
4799
|
const severity = String(rec?.severity ?? "info");
|
|
5046
|
-
const blocking = severity === "error" && !isAdvisory(code,
|
|
5047
|
-
out.push({ source: "lint", code, message
|
|
4800
|
+
const blocking = severity === "error" && !isAdvisory(code, message);
|
|
4801
|
+
out.push({ source: "lint", code, message, severity: blocking ? "blocking" : "warning" });
|
|
5048
4802
|
}
|
|
5049
4803
|
return out;
|
|
5050
4804
|
}
|
|
@@ -5056,9 +4810,9 @@ function classifyInspect(json) {
|
|
|
5056
4810
|
for (const iss of issues) {
|
|
5057
4811
|
const rec = iss;
|
|
5058
4812
|
const code = String(rec?.code ?? rec?.type ?? "overflow");
|
|
5059
|
-
const
|
|
4813
|
+
const message = String(rec?.message ?? rec?.detail ?? JSON.stringify(iss));
|
|
5060
4814
|
const severity = rec?.severity ? String(rec.severity) : obj?.ok === false ? "error" : "warning";
|
|
5061
|
-
out.push({ source: "inspect", code, message
|
|
4815
|
+
out.push({ source: "inspect", code, message, severity: severity === "error" ? "blocking" : "warning" });
|
|
5062
4816
|
}
|
|
5063
4817
|
return out;
|
|
5064
4818
|
}
|
|
@@ -5503,7 +5257,7 @@ async function buildSubstitutionValues(compositionParams, meta, duration) {
|
|
|
5503
5257
|
}
|
|
5504
5258
|
function coerceImageParam(value) {
|
|
5505
5259
|
if (typeof value === "string") return Promise.resolve(value);
|
|
5506
|
-
if (
|
|
5260
|
+
if (isAssetRefLike(value)) return refToUrl(value);
|
|
5507
5261
|
throw new Error("hyperframe_render: image param must be a URL string or AssetRef");
|
|
5508
5262
|
}
|
|
5509
5263
|
async function substituteCompositionFiles(tmp, values) {
|
|
@@ -5733,7 +5487,7 @@ async function buildSubstitutionValues2(compositionParams, meta) {
|
|
|
5733
5487
|
}
|
|
5734
5488
|
function coerceImageParam2(value) {
|
|
5735
5489
|
if (typeof value === "string") return Promise.resolve(value);
|
|
5736
|
-
if (
|
|
5490
|
+
if (isAssetRefLike(value)) return refToUrl(value);
|
|
5737
5491
|
throw new Error("hyperframe_snapshot: image param must be a URL string or AssetRef");
|
|
5738
5492
|
}
|
|
5739
5493
|
async function substituteCompositionFiles2(tmp, values) {
|
|
@@ -6759,29 +6513,17 @@ function createEngineFromEnv(opts = {}) {
|
|
|
6759
6513
|
const cacheDir = opts.cacheDir ?? path15.join(cwd, "canvas", ".cache");
|
|
6760
6514
|
const outputsDir = opts.outputsDir ?? path15.join(cwd, "canvas");
|
|
6761
6515
|
const creds = requireCredentialsFromEnv();
|
|
6762
|
-
const client = new BackendClient({ baseUrl: creds.url, apiKey: creds.apiKey });
|
|
6763
|
-
const assets = new LocalAssetStore(path15.join(cacheDir, "assets"));
|
|
6764
|
-
const localCache = new LocalCacheStore(path15.join(cacheDir, "index"));
|
|
6765
|
-
const remoteCacheEnabled = opts.remoteCache ?? remoteCacheEnabledFromEnv();
|
|
6766
|
-
const cache = remoteCacheEnabled ? new LayeredCacheStore({
|
|
6767
|
-
local: localCache,
|
|
6768
|
-
remote: new RemoteCacheStore(client, opts.log),
|
|
6769
|
-
assets,
|
|
6770
|
-
log: opts.log
|
|
6771
|
-
}) : localCache;
|
|
6772
6516
|
return new Engine({
|
|
6773
6517
|
registry: defaultRegistry(),
|
|
6774
|
-
client,
|
|
6775
|
-
assets,
|
|
6776
|
-
cache,
|
|
6518
|
+
client: new BackendClient({ baseUrl: creds.url, apiKey: creds.apiKey }),
|
|
6519
|
+
assets: new LocalAssetStore(path15.join(cacheDir, "assets")),
|
|
6520
|
+
cache: new LocalCacheStore(path15.join(cacheDir, "index")),
|
|
6777
6521
|
outputsDir,
|
|
6778
|
-
log: opts.log
|
|
6779
|
-
persistAssets: remoteCacheEnabled
|
|
6522
|
+
log: opts.log
|
|
6780
6523
|
});
|
|
6781
6524
|
}
|
|
6782
6525
|
|
|
6783
6526
|
export {
|
|
6784
|
-
requireCredentialsFromEnv,
|
|
6785
6527
|
LayerExecutionError,
|
|
6786
6528
|
describeFailureReason,
|
|
6787
6529
|
SEEDANCE_DURATIONS,
|
|
@@ -6789,10 +6531,6 @@ export {
|
|
|
6789
6531
|
IMAGE_GENERATE_MODELS,
|
|
6790
6532
|
MODEL_REGISTRY,
|
|
6791
6533
|
resolveConcurrency,
|
|
6792
|
-
ulid,
|
|
6793
|
-
isPersistedAssetRef,
|
|
6794
|
-
collectAssetRefLikes,
|
|
6795
|
-
sha256Hex,
|
|
6796
6534
|
elementMentionKeywords,
|
|
6797
6535
|
BackendClient2 as BackendClient,
|
|
6798
6536
|
Engine2 as Engine,
|
|
@@ -6804,4 +6542,4 @@ export {
|
|
|
6804
6542
|
defaultRegistry,
|
|
6805
6543
|
createEngineFromEnv
|
|
6806
6544
|
};
|
|
6807
|
-
//# sourceMappingURL=chunk-
|
|
6545
|
+
//# sourceMappingURL=chunk-MWFJ5NOP.js.map
|