@moldesk/core 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cloud/accounts.d.ts +69 -0
- package/dist/cloud/accounts.js +113 -0
- package/dist/cloud/accounts.test.d.ts +1 -0
- package/dist/cloud/accounts.test.js +188 -0
- package/dist/cloud/backend.test.d.ts +1 -0
- package/dist/cloud/backend.test.js +339 -0
- package/dist/cloud/deps.d.ts +15 -0
- package/dist/cloud/deps.js +30 -0
- package/dist/cloud/manage.d.ts +57 -0
- package/dist/cloud/manage.js +210 -0
- package/dist/cloud/manage.test.d.ts +1 -0
- package/dist/cloud/manage.test.js +105 -0
- package/dist/cloud/quote.d.ts +131 -0
- package/dist/cloud/quote.js +225 -0
- package/dist/cloud/quote.test.d.ts +1 -0
- package/dist/cloud/quote.test.js +264 -0
- package/dist/cloud/run.d.ts +114 -0
- package/dist/cloud/run.js +498 -0
- package/dist/cloud/run.test.d.ts +1 -0
- package/dist/cloud/run.test.js +597 -0
- package/dist/cloud/state.d.ts +159 -0
- package/dist/cloud/state.js +395 -0
- package/dist/cloud/state.test.d.ts +1 -0
- package/dist/cloud/state.test.js +199 -0
- package/dist/cloud/worker/cache.d.ts +75 -0
- package/dist/cloud/worker/cache.js +152 -0
- package/dist/cloud/worker/cache.test.d.ts +1 -0
- package/dist/cloud/worker/cache.test.js +123 -0
- package/dist/cloud/worker/main.d.ts +2 -0
- package/dist/cloud/worker/main.js +34 -0
- package/dist/cloud/worker/pipeline.d.ts +26 -0
- package/dist/cloud/worker/pipeline.js +160 -0
- package/dist/cloud/worker/pipeline.test.d.ts +1 -0
- package/dist/cloud/worker/pipeline.test.js +298 -0
- package/dist/cloud/worker/server.d.ts +38 -0
- package/dist/cloud/worker/server.js +287 -0
- package/dist/cloud/worker/server.test.d.ts +1 -0
- package/dist/cloud/worker/server.test.js +391 -0
- package/dist/cloud/worker/watchdog.d.ts +27 -0
- package/dist/cloud/worker/watchdog.js +46 -0
- package/dist/cloud/worker-release.d.ts +12 -0
- package/dist/cloud/worker-release.js +25 -0
- package/dist/cloud/worker-release.test.d.ts +1 -0
- package/dist/cloud/worker-release.test.js +21 -0
- package/dist/index.d.ts +17 -3
- package/dist/index.js +9 -2
- package/dist/run-lifecycle.d.ts +74 -0
- package/dist/run-lifecycle.js +178 -0
- package/dist/run.d.ts +61 -1
- package/dist/run.js +54 -125
- package/dist/run.test.js +47 -1
- package/package.json +10 -6
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { parseManifest } from "@moldesk/registry";
|
|
3
|
+
import { LIFETIME_RISK_WARNING, STORAGE_CHARGES_WARNING, assertApprovalValid, buildQuote } from "./quote.js";
|
|
4
|
+
const DIGEST = "a".repeat(64);
|
|
5
|
+
function manifest() {
|
|
6
|
+
return parseManifest({
|
|
7
|
+
schemaVersion: 1,
|
|
8
|
+
name: "test-model",
|
|
9
|
+
displayName: "Test",
|
|
10
|
+
modelVersion: "1.0.0",
|
|
11
|
+
adapterVersion: "0.1.0",
|
|
12
|
+
description: "t",
|
|
13
|
+
category: "other",
|
|
14
|
+
status: "planned",
|
|
15
|
+
runtimes: [{ kind: "python", python: "3.11", installer: "uv", platforms: ["linux-x64"], requirements: [{ name: "torch" }] }],
|
|
16
|
+
hardware: { platforms: ["linux-x64"] },
|
|
17
|
+
input: { formats: [".pdb"], required: true },
|
|
18
|
+
outputs: [{ id: "o", glob: "*.fa", required: true }],
|
|
19
|
+
cloud: {
|
|
20
|
+
schemaVersion: 1,
|
|
21
|
+
platform: "linux-x64",
|
|
22
|
+
images: { gpu: `registry.example/w/gpu@sha256:${DIGEST}`, cpu: `registry.example/w/cpu@sha256:${DIGEST}` },
|
|
23
|
+
minVramGb: 24,
|
|
24
|
+
minRamGb: 32,
|
|
25
|
+
ephemeralDiskGb: 40,
|
|
26
|
+
volumeGb: 40,
|
|
27
|
+
assetLayoutRevision: "1",
|
|
28
|
+
cpuAssetPreparation: true,
|
|
29
|
+
},
|
|
30
|
+
}, "x/manifest.yaml");
|
|
31
|
+
}
|
|
32
|
+
function catalog() {
|
|
33
|
+
return {
|
|
34
|
+
retrievedAt: "2026-10-13T12:00:00.000Z",
|
|
35
|
+
gpus: [
|
|
36
|
+
{ id: "small", name: "Small", memoryGb: 16, priceUsdPerHour: 0.2, availability: "HIGH", dataCenters: [{ id: "DC-A", availability: "HIGH" }], cudaVersions: [] },
|
|
37
|
+
{ id: "a40", name: "A40", memoryGb: 48, priceUsdPerHour: 0.6, availability: "LOW", dataCenters: [{ id: "DC-NOVOL", availability: "LOW" }, { id: "DC-VOL", availability: "LOW" }], cudaVersions: [] },
|
|
38
|
+
{ id: "a5000", name: "A5000", memoryGb: 24, priceUsdPerHour: 0.3, availability: "NONE", dataCenters: [{ id: "DC-VOL", availability: "NONE" }], cudaVersions: [] },
|
|
39
|
+
{ id: "l40", name: "L40", memoryGb: 48, priceUsdPerHour: 0.9, availability: "MEDIUM", dataCenters: [{ id: "DC-VOL", availability: "MEDIUM" }], cudaVersions: [] },
|
|
40
|
+
],
|
|
41
|
+
cpus: [{ id: "cpu3c", name: "c", priceUsdPerVcpuHour: 0.03, ramGbPerVcpu: 2, minVcpu: 2, maxVcpu: 32 }],
|
|
42
|
+
dataCenters: [
|
|
43
|
+
{ id: "DC-NOVOL", region: "EU", networkVolumeTypes: [] },
|
|
44
|
+
{ id: "DC-VOL", region: "EU", networkVolumeTypes: ["STANDARD"] },
|
|
45
|
+
{ id: "DC-A", region: "EU", networkVolumeTypes: ["STANDARD"] },
|
|
46
|
+
],
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
function inputs(overrides = {}) {
|
|
50
|
+
const m = manifest();
|
|
51
|
+
return {
|
|
52
|
+
provider: "runpod",
|
|
53
|
+
manifest: m,
|
|
54
|
+
cloud: m.cloud,
|
|
55
|
+
catalog: catalog(),
|
|
56
|
+
cachePolicy: "ephemeral",
|
|
57
|
+
maxBudgetUsd: 5,
|
|
58
|
+
maxLifetimeSeconds: 3600,
|
|
59
|
+
inputs: [{ role: "job", sha256: "b".repeat(64), sizeBytes: 100 }],
|
|
60
|
+
...overrides,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
describe("buildQuote", () => {
|
|
64
|
+
it("picks the cheapest GPU that meets VRAM and has capacity, skipping unavailable ones", () => {
|
|
65
|
+
const quote = buildQuote(inputs());
|
|
66
|
+
expect(quote.gpu.id).toBe("a40");
|
|
67
|
+
expect(quote.datacenterId).toBe("DC-NOVOL");
|
|
68
|
+
expect(quote.cost.worstCaseComputeUsd).toBe(0.6);
|
|
69
|
+
expect(quote.timing).toEqual({ preparation: null, inference: null, transfer: null });
|
|
70
|
+
expect(quote.cost.recurringStorage).toEqual({ recurring: false, volumeGb: null, usdPerMonth: null });
|
|
71
|
+
expect(quote.cpu).toBeUndefined();
|
|
72
|
+
});
|
|
73
|
+
it("persistent policy needs a volume-capable datacenter, adds the CPU preparer and discloses recurring storage", () => {
|
|
74
|
+
const quote = buildQuote(inputs({ cachePolicy: "persistent" }));
|
|
75
|
+
expect(quote.datacenterId).toBe("DC-VOL");
|
|
76
|
+
expect(quote.cpu).toEqual({ id: "cpu3c", vcpuCount: 2, priceUsdPerHour: 0.06 });
|
|
77
|
+
expect(quote.cost.worstCaseComputeUsd).toBe(0.66);
|
|
78
|
+
expect(quote.cost.recurringStorage).toEqual({ recurring: true, volumeGb: 40, usdPerMonth: null });
|
|
79
|
+
expect(quote.disclosures.join(" ")).toMatch(/billed until you delete/);
|
|
80
|
+
});
|
|
81
|
+
it("pins persistent runs to an existing volume's datacenter or fails with capacity guidance", () => {
|
|
82
|
+
expect(buildQuote(inputs({ cachePolicy: "persistent", existingVolumeDatacenterId: "DC-VOL" })).datacenterId).toBe("DC-VOL");
|
|
83
|
+
expect(() => buildQuote(inputs({ cachePolicy: "persistent", existingVolumeDatacenterId: "DC-A" }))).toThrow(/No eligible GPU/);
|
|
84
|
+
});
|
|
85
|
+
it("validates an explicit GPU instead of silently replacing it", () => {
|
|
86
|
+
expect(() => buildQuote(inputs({ gpuId: "nope" }))).toThrow(/not in the provider catalog/);
|
|
87
|
+
expect(() => buildQuote(inputs({ gpuId: "small" }))).toThrow(/needs at least 24 GB/);
|
|
88
|
+
expect(() => buildQuote(inputs({ gpuId: "a5000" }))).toThrow(/No eligible GPU/);
|
|
89
|
+
expect(buildQuote(inputs({ gpuId: "l40" })).gpu.id).toBe("l40");
|
|
90
|
+
expect(() => buildQuote(inputs({ gpuId: "a40", datacenterId: "DC-ELSEWHERE" }))).toThrow(/No eligible GPU/);
|
|
91
|
+
});
|
|
92
|
+
it("rejects over-budget quotes before anything is provisioned", () => {
|
|
93
|
+
expect(() => buildQuote(inputs({ maxBudgetUsd: 0.5 }))).toThrow(expect.objectContaining({ code: "CLOUD_BUDGET_EXCEEDED" }));
|
|
94
|
+
expect(() => buildQuote(inputs({ maxBudgetUsd: 0 }))).toThrow(expect.objectContaining({ code: "CLOUD_BUDGET_REQUIRED" }));
|
|
95
|
+
expect(() => buildQuote(inputs({ maxLifetimeSeconds: 0 }))).toThrow(expect.objectContaining({ code: "CLOUD_LIFETIME_REQUIRED" }));
|
|
96
|
+
expect(buildQuote(inputs({ maxBudgetUsd: 0.3, maxLifetimeSeconds: 1800 })).cost.worstCaseComputeUsd).toBe(0.3);
|
|
97
|
+
});
|
|
98
|
+
it("derives a stable id bound to inputs, policy, location and limits but not to price", () => {
|
|
99
|
+
const base = buildQuote(inputs());
|
|
100
|
+
expect(buildQuote(inputs()).id).toBe(base.id);
|
|
101
|
+
const cheaper = catalog();
|
|
102
|
+
cheaper.gpus[1].priceUsdPerHour = 0.5;
|
|
103
|
+
expect(buildQuote(inputs({ catalog: cheaper })).id).toBe(base.id);
|
|
104
|
+
expect(buildQuote(inputs({ inputs: [{ role: "job", sha256: "c".repeat(64), sizeBytes: 100 }] })).id).not.toBe(base.id);
|
|
105
|
+
expect(buildQuote(inputs({ cachePolicy: "persistent" })).id).not.toBe(base.id);
|
|
106
|
+
expect(buildQuote(inputs({ maxLifetimeSeconds: 1800 })).id).not.toBe(base.id);
|
|
107
|
+
expect(buildQuote(inputs({ gpuId: "l40" })).id).not.toBe(base.id);
|
|
108
|
+
});
|
|
109
|
+
});
|
|
110
|
+
describe("storage costs and budget coverage", () => {
|
|
111
|
+
const withRates = () => ({ ...catalog(), storageRates: { containerDiskUsdPerGbMonth: 0.1, networkVolumeUsdPerGbMonth: 0.07 } });
|
|
112
|
+
it("states storage as unknown, not counted, when the provider publishes no rates", () => {
|
|
113
|
+
const eph = buildQuote(inputs());
|
|
114
|
+
expect(eph.cost.temporaryStorage).toEqual({ diskGb: 40, machines: 1, worstCaseUsd: null });
|
|
115
|
+
expect(eph.cost.worstCaseBudgetedUsd).toBe(eph.cost.worstCaseComputeUsd);
|
|
116
|
+
expect(eph.budgetCoverage.unknown).toEqual(["container disk rate is not published by the provider's catalog API"]);
|
|
117
|
+
expect(eph.budgetCoverage.notCounted.join(" | ")).toMatch(/container disk charges \(40 GB x 1 machine\)/);
|
|
118
|
+
expect(eph.budgetCoverage.notCounted.join(" | ")).toMatch(/network transfer or any other provider fees/);
|
|
119
|
+
expect(eph.budgetCoverage.counted).toHaveLength(1);
|
|
120
|
+
const per = buildQuote(inputs({ cachePolicy: "persistent" }));
|
|
121
|
+
expect(per.cost.temporaryStorage.machines).toBe(2);
|
|
122
|
+
expect(per.budgetCoverage.unknown).toHaveLength(2);
|
|
123
|
+
expect(per.budgetCoverage.notCounted.join(" | ")).toMatch(/retained network volume storage \(40 GB\), billed monthly until deleted(?!:)/);
|
|
124
|
+
});
|
|
125
|
+
it("counts temporary storage in the budget and reports recurring storage separately when rates are published", () => {
|
|
126
|
+
const eph = buildQuote(inputs({ catalog: withRates() }));
|
|
127
|
+
expect(eph.cost.temporaryStorage.worstCaseUsd).toBeCloseTo(40 * 0.1 * (1 / 730), 4);
|
|
128
|
+
expect(eph.cost.worstCaseBudgetedUsd).toBeCloseTo(0.6 + eph.cost.temporaryStorage.worstCaseUsd, 4);
|
|
129
|
+
expect(eph.budgetCoverage.unknown).toEqual([]);
|
|
130
|
+
expect(eph.budgetCoverage.counted.join(" | ")).toMatch(/container disk/);
|
|
131
|
+
const per = buildQuote(inputs({ cachePolicy: "persistent", catalog: withRates() }));
|
|
132
|
+
expect(per.cost.recurringStorage).toEqual({ recurring: true, volumeGb: 40, usdPerMonth: 2.8 });
|
|
133
|
+
expect(per.budgetCoverage.notCounted.join(" | ")).toMatch(/billed monthly until deleted: about \$2\.8\/month/);
|
|
134
|
+
expect(per.budgetCoverage.unknown).toEqual([]);
|
|
135
|
+
// Recurring storage is disclosed, never silently folded into the one-run budget.
|
|
136
|
+
expect(per.cost.worstCaseBudgetedUsd).toBeCloseTo(per.cost.worstCaseComputeUsd + per.cost.temporaryStorage.worstCaseUsd, 4);
|
|
137
|
+
});
|
|
138
|
+
it("rejects when known storage pushes the counted worst case over the budget", () => {
|
|
139
|
+
expect(buildQuote(inputs({ maxBudgetUsd: 0.6 })).cost.worstCaseBudgetedUsd).toBe(0.6);
|
|
140
|
+
expect(() => buildQuote(inputs({ maxBudgetUsd: 0.6, catalog: withRates() }))).toThrow(expect.objectContaining({ code: "CLOUD_BUDGET_EXCEEDED" }));
|
|
141
|
+
});
|
|
142
|
+
it("does not change the approval id", () => {
|
|
143
|
+
expect(buildQuote(inputs({ catalog: withRates() })).id).toBe(buildQuote(inputs()).id);
|
|
144
|
+
});
|
|
145
|
+
});
|
|
146
|
+
describe("assertApprovalValid", () => {
|
|
147
|
+
const now = new Date("2026-10-13T12:05:00.000Z");
|
|
148
|
+
const approval = (quote, approvedAt = "2026-10-13T12:00:00.000Z") => ({ quoteId: quote.id, approvedAt, acknowledged: quote.warnings.map((w) => w.id) });
|
|
149
|
+
it("accepts a matching, fresh approval", () => {
|
|
150
|
+
const quote = buildQuote(inputs());
|
|
151
|
+
expect(() => assertApprovalValid(quote, approval(quote), buildQuote(inputs()), now)).not.toThrow();
|
|
152
|
+
});
|
|
153
|
+
it("rejects mismatched, expired and future-dated approvals", () => {
|
|
154
|
+
const quote = buildQuote(inputs());
|
|
155
|
+
expect(() => assertApprovalValid(quote, { quoteId: "other", approvedAt: "2026-10-13T12:00:00.000Z", acknowledged: ["lifetime-risk"] }, quote, now)).toThrow(expect.objectContaining({ code: "CLOUD_APPROVAL_MISMATCH" }));
|
|
156
|
+
expect(() => assertApprovalValid(quote, approval(quote, "2026-10-13T11:00:00.000Z"), quote, now)).toThrow(expect.objectContaining({ code: "CLOUD_APPROVAL_EXPIRED" }));
|
|
157
|
+
expect(() => assertApprovalValid(quote, approval(quote, "2026-10-13T13:00:00.000Z"), quote, now)).toThrow(expect.objectContaining({ code: "CLOUD_APPROVAL_EXPIRED" }));
|
|
158
|
+
expect(() => assertApprovalValid(quote, approval(quote, "garbage"), quote, now)).toThrow(expect.objectContaining({ code: "CLOUD_APPROVAL_EXPIRED" }));
|
|
159
|
+
});
|
|
160
|
+
it("requires a new approval when location, inputs or a material price change since", () => {
|
|
161
|
+
const quote = buildQuote(inputs());
|
|
162
|
+
const moved = catalog();
|
|
163
|
+
moved.gpus[1].dataCenters = [{ id: "DC-VOL", availability: "LOW" }];
|
|
164
|
+
moved.dataCenters = moved.dataCenters.filter((dc) => dc.id !== "DC-NOVOL");
|
|
165
|
+
expect(() => assertApprovalValid(quote, approval(quote), buildQuote(inputs({ catalog: moved })), now)).toThrow(expect.objectContaining({ code: "CLOUD_REQUOTE_REQUIRED" }));
|
|
166
|
+
const pricier = catalog();
|
|
167
|
+
pricier.gpus[1].priceUsdPerHour = 0.7;
|
|
168
|
+
const fresh = buildQuote(inputs({ catalog: pricier }));
|
|
169
|
+
expect(fresh.id).toBe(quote.id);
|
|
170
|
+
expect(() => assertApprovalValid(quote, approval(quote), fresh, now)).toThrow(/price rose/);
|
|
171
|
+
const slightly = catalog();
|
|
172
|
+
slightly.gpus[1].priceUsdPerHour = 0.63;
|
|
173
|
+
expect(() => assertApprovalValid(quote, approval(quote), buildQuote(inputs({ catalog: slightly })), now)).not.toThrow();
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
describe("approval revalidation of storage cost and budget coverage", () => {
|
|
177
|
+
const now = new Date("2026-10-13T12:05:00.000Z");
|
|
178
|
+
const rates = (disk, volume) => ({
|
|
179
|
+
...catalog(),
|
|
180
|
+
...(disk !== undefined || volume !== undefined ? { storageRates: { ...(disk !== undefined ? { containerDiskUsdPerGbMonth: disk } : {}), ...(volume !== undefined ? { networkVolumeUsdPerGbMonth: volume } : {}) } } : {}),
|
|
181
|
+
});
|
|
182
|
+
const approve = (quote) => ({ quoteId: quote.id, approvedAt: "2026-10-13T12:00:00.000Z", acknowledged: quote.warnings.map((w) => w.id) });
|
|
183
|
+
const check = (approved, fresh) => assertApprovalValid(approved, approve(approved), fresh, now);
|
|
184
|
+
const requote = expect.objectContaining({ code: "CLOUD_REQUOTE_REQUIRED" });
|
|
185
|
+
it("passes when storage figures and coverage are identical", () => {
|
|
186
|
+
const a = buildQuote(inputs({ cachePolicy: "persistent", catalog: rates(5, 0.07) }));
|
|
187
|
+
expect(() => check(a, buildQuote(inputs({ cachePolicy: "persistent", catalog: rates(5, 0.07) })))).not.toThrow();
|
|
188
|
+
const unknown = buildQuote(inputs({ cachePolicy: "persistent" }));
|
|
189
|
+
expect(() => check(unknown, buildQuote(inputs({ cachePolicy: "persistent" })))).not.toThrow();
|
|
190
|
+
});
|
|
191
|
+
it("requires fresh approval when an unknown rate becomes known, even if the amount is tiny", () => {
|
|
192
|
+
const approved = buildQuote(inputs());
|
|
193
|
+
expect(approved.cost.temporaryStorage.worstCaseUsd).toBeNull();
|
|
194
|
+
const fresh = buildQuote(inputs({ catalog: rates(0.0001, undefined) }));
|
|
195
|
+
expect(fresh.cost.temporaryStorage.worstCaseUsd).not.toBeNull();
|
|
196
|
+
expect(fresh.id).toBe(approved.id);
|
|
197
|
+
expect(() => check(approved, fresh)).toThrow(requote);
|
|
198
|
+
expect(() => check(approved, fresh)).toThrow(/coverage changed.*diskCounted/);
|
|
199
|
+
});
|
|
200
|
+
it("requires fresh approval when a known rate becomes unknown (coverage shrinks)", () => {
|
|
201
|
+
const approved = buildQuote(inputs({ catalog: rates(0.1, undefined) }));
|
|
202
|
+
expect(() => check(approved, buildQuote(inputs()))).toThrow(requote);
|
|
203
|
+
});
|
|
204
|
+
it("requires fresh approval when the recurring volume rate becomes known or unknown", () => {
|
|
205
|
+
const unknownVolume = buildQuote(inputs({ cachePolicy: "persistent" }));
|
|
206
|
+
const knownVolume = buildQuote(inputs({ cachePolicy: "persistent", catalog: rates(undefined, 0.07) }));
|
|
207
|
+
expect(() => check(unknownVolume, knownVolume)).toThrow(/volumeRateKnown/);
|
|
208
|
+
expect(() => check(knownVolume, unknownVolume)).toThrow(requote);
|
|
209
|
+
});
|
|
210
|
+
it("requires fresh approval for a material rise in container disk, recurring storage or the counted total, but not for noise or drops", () => {
|
|
211
|
+
const a = buildQuote(inputs({ cachePolicy: "persistent", catalog: rates(10, 0.07), maxBudgetUsd: 50 }));
|
|
212
|
+
// Disk rate doubles: temporary disk cost rises well past tolerance and the noise floor.
|
|
213
|
+
expect(() => check(a, buildQuote(inputs({ cachePolicy: "persistent", catalog: rates(20, 0.07), maxBudgetUsd: 50 })))).toThrow(/Container disk cost rose/);
|
|
214
|
+
// Volume rate doubles: recurring cost rises 2.8 -> 5.6 per month.
|
|
215
|
+
expect(() => check(a, buildQuote(inputs({ cachePolicy: "persistent", catalog: rates(10, 0.14), maxBudgetUsd: 50 })))).toThrow(/Recurring volume storage rose/);
|
|
216
|
+
// Within tolerance: allowed.
|
|
217
|
+
expect(() => check(a, buildQuote(inputs({ cachePolicy: "persistent", catalog: rates(10.5, 0.074), maxBudgetUsd: 50 })))).not.toThrow();
|
|
218
|
+
// Cheaper with the same coverage: allowed.
|
|
219
|
+
expect(() => check(a, buildQuote(inputs({ cachePolicy: "persistent", catalog: rates(5, 0.03), maxBudgetUsd: 50 })))).not.toThrow();
|
|
220
|
+
});
|
|
221
|
+
it("ignores sub-floor absolute changes so rounding noise never forces re-approval", () => {
|
|
222
|
+
const a = buildQuote(inputs({ catalog: rates(0.1, undefined) })); // ~ $0.0055 disk
|
|
223
|
+
const tiny = buildQuote(inputs({ catalog: rates(0.15, undefined) })); // ~ $0.0082: +50% but only ~$0.003
|
|
224
|
+
expect(() => check(a, tiny)).not.toThrow();
|
|
225
|
+
});
|
|
226
|
+
it("a changed disk size or machine count (persistent vs not) is a coverage change and already a different quote id", () => {
|
|
227
|
+
const eph = buildQuote(inputs());
|
|
228
|
+
const per = buildQuote(inputs({ cachePolicy: "persistent" }));
|
|
229
|
+
expect(() => check(eph, per)).toThrow(requote);
|
|
230
|
+
});
|
|
231
|
+
});
|
|
232
|
+
describe("explicit risk acknowledgment", () => {
|
|
233
|
+
const now = new Date("2026-10-13T12:05:00.000Z");
|
|
234
|
+
const base = { approvedAt: "2026-10-13T12:00:00.000Z" };
|
|
235
|
+
const needs = expect.objectContaining({ code: "CLOUD_ACKNOWLEDGMENT_REQUIRED" });
|
|
236
|
+
it("every quote carries the exact lifetime warning", () => {
|
|
237
|
+
const quote = buildQuote(inputs());
|
|
238
|
+
expect(quote.warnings).toEqual([{ id: "lifetime-risk", text: LIFETIME_RISK_WARNING }]);
|
|
239
|
+
expect(LIFETIME_RISK_WARNING).toBe("If MoleculeDesk closes or loses connection, your Runpod Pod may keep running and charging. Automatic shutdown is not guaranteed. Check your Runpod dashboard and terminate any remaining Pod.");
|
|
240
|
+
expect(quote.disclosures.join(" ")).toMatch(/only while it is running and connected/);
|
|
241
|
+
expect(quote.disclosures.join(" ")).not.toMatch(/are terminated after the run or at the/);
|
|
242
|
+
});
|
|
243
|
+
it("persistent quotes also carry the storage-charges warning; ephemeral ones do not", () => {
|
|
244
|
+
expect(buildQuote(inputs({ cachePolicy: "persistent" })).warnings.map((w) => w.id)).toEqual(["lifetime-risk", "storage-charges"]);
|
|
245
|
+
expect(buildQuote(inputs({ cachePolicy: "persistent" })).warnings[1].text).toBe(STORAGE_CHARGES_WARNING);
|
|
246
|
+
expect(STORAGE_CHARGES_WARNING).toMatch(/Storage charges continue until you delete it/);
|
|
247
|
+
expect(buildQuote(inputs()).warnings.map((w) => w.id)).toEqual(["lifetime-risk"]);
|
|
248
|
+
});
|
|
249
|
+
it("an approval without the lifetime acknowledgment is refused", () => {
|
|
250
|
+
const quote = buildQuote(inputs());
|
|
251
|
+
expect(() => assertApprovalValid(quote, { quoteId: quote.id, ...base, acknowledged: [] }, quote, now)).toThrow(needs);
|
|
252
|
+
expect(() => assertApprovalValid(quote, { quoteId: quote.id, ...base, acknowledged: ["lifetime-risk"] }, quote, now)).not.toThrow();
|
|
253
|
+
});
|
|
254
|
+
it("persistent approvals need both acknowledgments; acknowledging only one is refused", () => {
|
|
255
|
+
const quote = buildQuote(inputs({ cachePolicy: "persistent" }));
|
|
256
|
+
expect(() => assertApprovalValid(quote, { quoteId: quote.id, ...base, acknowledged: ["lifetime-risk"] }, quote, now)).toThrow(/Storage charges continue/);
|
|
257
|
+
expect(() => assertApprovalValid(quote, { quoteId: quote.id, ...base, acknowledged: ["storage-charges"] }, quote, now)).toThrow(needs);
|
|
258
|
+
expect(() => assertApprovalValid(quote, { quoteId: quote.id, ...base, acknowledged: ["lifetime-risk", "storage-charges"] }, quote, now)).not.toThrow();
|
|
259
|
+
});
|
|
260
|
+
it("a missing acknowledgments field (old or hand-built approval) is refused, not assumed", () => {
|
|
261
|
+
const quote = buildQuote(inputs());
|
|
262
|
+
expect(() => assertApprovalValid(quote, { quoteId: quote.id, ...base }, quote, now)).toThrow(needs);
|
|
263
|
+
});
|
|
264
|
+
});
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { type ModelAdapterDefinition } from "@moldesk/adapters";
|
|
2
|
+
import { type CloudSpec, type ModelManifestV1 } from "@moldesk/registry";
|
|
3
|
+
import { type CacheLedger, type CloudProvider, type ExecutionBackend, type JobSnapshot, type JobStage, type MoldeskPaths } from "@moldesk/runtime";
|
|
4
|
+
import type { RunCompute, RunModelResult, RunRecord } from "../run.js";
|
|
5
|
+
import { type CloudApproval, type CloudQuote } from "./quote.js";
|
|
6
|
+
import { CloudJournal, type CloudCachePolicy } from "./state.js";
|
|
7
|
+
export interface CloudQuoteOptions {
|
|
8
|
+
cachePolicy: CloudCachePolicy;
|
|
9
|
+
gpuId?: string;
|
|
10
|
+
datacenterId?: string;
|
|
11
|
+
maxBudgetUsd: number;
|
|
12
|
+
maxLifetimeSeconds: number;
|
|
13
|
+
}
|
|
14
|
+
export interface CloudRunOptions extends CloudQuoteOptions {
|
|
15
|
+
/** The exact quote the user saw, and their approval of it. */
|
|
16
|
+
quote: CloudQuote;
|
|
17
|
+
approval: CloudApproval;
|
|
18
|
+
}
|
|
19
|
+
export interface CloudRunModelOptions {
|
|
20
|
+
params?: Record<string, string>;
|
|
21
|
+
outputDir?: string;
|
|
22
|
+
signal?: AbortSignal;
|
|
23
|
+
onLog?: (stream: "stdout" | "stderr", chunk: string) => void;
|
|
24
|
+
onProgress?: (event: {
|
|
25
|
+
stage: JobStage | "quote" | "cleanup";
|
|
26
|
+
message?: string;
|
|
27
|
+
}) => void;
|
|
28
|
+
visualize?: boolean;
|
|
29
|
+
cloud: CloudRunOptions;
|
|
30
|
+
}
|
|
31
|
+
export interface ResolvedModel {
|
|
32
|
+
manifest: ModelManifestV1;
|
|
33
|
+
adapter: ModelAdapterDefinition;
|
|
34
|
+
}
|
|
35
|
+
export interface CloudRunDeps {
|
|
36
|
+
provider: CloudProvider;
|
|
37
|
+
paths?: MoldeskPaths;
|
|
38
|
+
/** Required by `runCloudModel`; `planCloudRun` does not provision. */
|
|
39
|
+
backend?: ExecutionBackend;
|
|
40
|
+
/** Non-secret scope identifying the provider account (credential fingerprint or account id). */
|
|
41
|
+
accountScope?: string;
|
|
42
|
+
journal?: CloudJournal;
|
|
43
|
+
resolveModel?: (name: string) => ResolvedModel;
|
|
44
|
+
now?: () => Date;
|
|
45
|
+
sleep?: (ms: number) => Promise<void>;
|
|
46
|
+
pollIntervalMs?: number;
|
|
47
|
+
/** How long a failed result download may be retried while the machine is kept alive. */
|
|
48
|
+
recoveryWindowSeconds?: number;
|
|
49
|
+
cleanupAttempts?: number;
|
|
50
|
+
}
|
|
51
|
+
export interface CloudPlan {
|
|
52
|
+
quote: CloudQuote;
|
|
53
|
+
manifest: ModelManifestV1;
|
|
54
|
+
cloud: CloudSpec;
|
|
55
|
+
adapter: ModelAdapterDefinition;
|
|
56
|
+
resolvedInputPath: string;
|
|
57
|
+
input: {
|
|
58
|
+
fileName: string;
|
|
59
|
+
sha256: string;
|
|
60
|
+
sizeBytes: number;
|
|
61
|
+
};
|
|
62
|
+
suppliedParams: Record<string, string>;
|
|
63
|
+
effectiveParams: Record<string, unknown>;
|
|
64
|
+
}
|
|
65
|
+
export declare function defaultResolveModelForRecovery(name: string): ResolvedModel;
|
|
66
|
+
/**
|
|
67
|
+
* Validates everything that can be validated without provisioning and builds the quote. Rejects
|
|
68
|
+
* (before any run directory or resource exists) for unknown/ineligible models, bad input or params,
|
|
69
|
+
* an unaffordable or unavailable quote.
|
|
70
|
+
*/
|
|
71
|
+
export declare function planCloudRun(modelName: string, inputPath: string, options: {
|
|
72
|
+
params?: Record<string, string>;
|
|
73
|
+
cloud: CloudQuoteOptions;
|
|
74
|
+
}, deps: Pick<CloudRunDeps, "provider" | "resolveModel" | "journal" | "paths" | "accountScope">): Promise<CloudPlan>;
|
|
75
|
+
/**
|
|
76
|
+
* The worker keeps a bounded log tail. When it reports chunks evicted before we fetched them, say so
|
|
77
|
+
* plainly: a marker in the local log, a warning on the run, and `compute.logs.complete = false`.
|
|
78
|
+
*/
|
|
79
|
+
export declare const LOGS_INCOMPLETE_CODE = "CLOUD_LOGS_INCOMPLETE";
|
|
80
|
+
export declare function recordLostLogs(snapshot: JobSnapshot, target: {
|
|
81
|
+
stdoutPath: string;
|
|
82
|
+
stderrPath: string;
|
|
83
|
+
compute: RunCompute;
|
|
84
|
+
record: RunRecord;
|
|
85
|
+
onLost?: (n: number) => void;
|
|
86
|
+
}): void;
|
|
87
|
+
/** Bridges a backend's cache lifecycle onto the journal for one job. */
|
|
88
|
+
export declare function cacheLedgerFor(journal: CloudJournal, params: {
|
|
89
|
+
accountScope: string;
|
|
90
|
+
runId: string;
|
|
91
|
+
manifest: ModelManifestV1;
|
|
92
|
+
cloud: CloudSpec;
|
|
93
|
+
cacheKey: string;
|
|
94
|
+
manifestDigest: string;
|
|
95
|
+
}): CacheLedger;
|
|
96
|
+
export interface CleanupOutcome {
|
|
97
|
+
confirmed: boolean;
|
|
98
|
+
pending: Array<{
|
|
99
|
+
role: "cpu" | "gpu" | "volume";
|
|
100
|
+
providerId: string;
|
|
101
|
+
}>;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Terminates and independently verifies every Pod this run owns. Uses the journal (never names
|
|
105
|
+
* alone) for ownership, resolves ambiguous creates first, and never touches volumes. Bounded
|
|
106
|
+
* retries; success means the provider shows the Pod gone, not that a call returned.
|
|
107
|
+
*/
|
|
108
|
+
export declare function confirmRunResourcesGone(runId: string, deps: Pick<CloudRunDeps, "provider" | "sleep" | "cleanupAttempts">, journal: CloudJournal): Promise<CleanupOutcome>;
|
|
109
|
+
/**
|
|
110
|
+
* Runs an approved cloud job end to end. Before allocation it rejects like a local run; from
|
|
111
|
+
* allocation on it always resolves with a finalized `run.json`. Paid machines are terminated and
|
|
112
|
+
* verified before the report is generated, and cleanup failure never discards a valid result.
|
|
113
|
+
*/
|
|
114
|
+
export declare function runCloudModel(modelName: string, inputPath: string, options: CloudRunModelOptions, deps: CloudRunDeps): Promise<RunModelResult>;
|