@audiofool/moondog 0.3.0-beta.1 → 0.3.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -4
- package/package.json +1 -1
- package/scripts/moondog.mjs +148 -14
- package/src/core/apple-projection-domain-services.mjs +4 -0
- package/src/core/capability-catalog.mjs +49 -0
- package/src/core/library-store.mjs +500 -0
- package/src/core/listening-now.mjs +80 -6
- package/src/core/listening-profile-domain-services.mjs +4 -0
- package/src/core/moondog-application.mjs +578 -32
- package/src/i18n/index.mjs +6 -3
- package/src/i18n/messages/de.mjs +1 -0
- package/src/i18n/messages/en.mjs +2 -0
- package/src/i18n/messages/es.mjs +1 -0
- package/src/i18n/messages/fr.mjs +1 -0
- package/src/i18n/messages/it.mjs +1 -0
- package/src/i18n/messages/ja.mjs +1 -0
- package/src/i18n/messages/ko.mjs +1 -0
- package/src/i18n/messages/pt.mjs +1 -0
- package/src/i18n/messages/zh-Hant.mjs +1 -0
- package/src/i18n/messages/zh.mjs +1 -0
- package/src/i18n/preferences.mjs +24 -0
- package/src/i18n/screen/es.mjs +94 -0
- package/src/i18n/screen/ja.mjs +94 -0
- package/src/i18n/screen/pt.mjs +94 -0
- package/src/i18n/screen/zh.mjs +94 -0
- package/src/integrations/apple-music/music-app.mjs +9 -3
- package/src/integrations/apple-music/terminal-preview-player.mjs +31 -1
- package/src/integrations/netease/ncm-cli-player.mjs +12 -6
- package/src/integrations/playlist-links.mjs +102 -0
- package/src/integrations/public-playlists.mjs +39 -8
- package/src/integrations/spotify/service.mjs +22 -1
- package/src/integrations/stable-audio/cover.mjs +324 -0
- package/src/integrations/stable-audio/local-generator.mjs +259 -87
- package/src/integrations/stable-audio/setup.mjs +653 -0
- package/src/integrations/stable-audio/sources.mjs +106 -0
- package/src/integrations/web/deepseek-web.mjs +1 -1
- package/src/integrations/web/web-research.mjs +84 -17
- package/src/integrations/web/web-result.mjs +60 -12
- package/src/profile/listening-history-store.mjs +55 -1
- package/src/profile/listening-profile-projection.mjs +1 -1
- package/src/profile/music-import-bundle.mjs +3 -1
- package/src/profile/music-providers.mjs +3 -0
- package/src/profile/taste-compiler.mjs +14 -5
- package/src/runtime/pi/agent-runtime.mjs +359 -37
- package/src/runtime/pi/configured-runtime.mjs +3 -0
- package/src/runtime/pi/image-input.mjs +148 -0
- package/src/runtime/pi/model-roles.mjs +121 -0
- package/src/runtime/pi/model-settings.mjs +63 -2
- package/src/runtime/pi/model-switch.mjs +26 -0
- package/src/runtime/pi/model-transport.mjs +34 -4
- package/src/runtime/pi/record-name.mjs +80 -0
- package/src/runtime/pi/reply-language.mjs +45 -0
- package/src/runtime/pi/runtime-settings.mjs +4 -2
- package/src/runtime/pi/tool-arguments.mjs +89 -0
- package/src/surfaces/cli/command-completions.mjs +15 -1
- package/src/surfaces/cli/model-roles-command.mjs +60 -0
- package/src/surfaces/cli/tui.mjs +419 -19
|
@@ -0,0 +1,653 @@
|
|
|
1
|
+
// Moondog's own Stable Audio 3 install. Only when the listener asks, it downloads uv, Stability AI's
|
|
2
|
+
// code, a Python environment and the chosen model's weights into one folder Moondog manages, only
|
|
3
|
+
// from the pinned official sources (sources.mjs), and checks every file before anything runs it.
|
|
4
|
+
// The terminal and the window are separate processes on that folder: a lock file names the process
|
|
5
|
+
// that runs the job, and that process writes its progress to a status file the other one reads.
|
|
6
|
+
import { spawn } from "node:child_process";
|
|
7
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
8
|
+
import { createReadStream, existsSync, readFileSync, statSync } from "node:fs";
|
|
9
|
+
import { access, constants, lstat, mkdir, open, readdir, rename, rm, stat, statfs, writeFile } from "node:fs/promises";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
|
|
13
|
+
import { checkStableAudioInstall, readGenerationSettings, readSavedSettings, resolveGenerationSettingsFile, resolveStableAudioManagedRoot, writeGenerationSettings } from "./local-generator.mjs";
|
|
14
|
+
import { DEFAULT_STABLE_AUDIO_MODEL, isStableAudioModel, SHARED_WEIGHTS, STABLE_AUDIO_MODELS, STABLE_AUDIO_SOURCES } from "./sources.mjs";
|
|
15
|
+
|
|
16
|
+
const MB = 1024 ** 2;
|
|
17
|
+
// Room kept free after each step, so a download never fills the disk.
|
|
18
|
+
const SPACE_MARGIN = 512 * MB;
|
|
19
|
+
// What the steps without a known size need on disk: uv unpacked, the code's tarball and tree, Python with its packages.
|
|
20
|
+
const STEP_SPACE = Object.freeze({ uv: 100 * MB, code: 200 * MB, python: 1536 * MB });
|
|
21
|
+
// At most four snapshots a second reach the surfaces while bytes arrive.
|
|
22
|
+
const PUSH_MS = 250;
|
|
23
|
+
// How often a surface that listens reads the folder again, which is how a job in the other process shows.
|
|
24
|
+
const POLL_MS = 1000;
|
|
25
|
+
// A lock file still being written is unreadable for a moment; one unreadable for longer is left over.
|
|
26
|
+
const LOCK_WRITE_MS = 5000;
|
|
27
|
+
|
|
28
|
+
const setupError = (code, message, extra = {}) => Object.assign(new Error(message), { code, ...extra });
|
|
29
|
+
const cancelled = () => setupError("setup_cancelled", "The download was cancelled.");
|
|
30
|
+
const gigabytes = (bytes) => `${(bytes / 1024 ** 3).toFixed(1)} GB`;
|
|
31
|
+
|
|
32
|
+
/** The listener's agreement to the licenses, kept with the settings so removing the install keeps it. */
|
|
33
|
+
export function resolveStableAudioLicenseFile(environment = process.env) {
|
|
34
|
+
return path.join(path.dirname(resolveGenerationSettingsFile(environment)), "stable-audio-3-licenses.json");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const readJson = (file) => {
|
|
38
|
+
try {
|
|
39
|
+
return JSON.parse(readFileSync(file, "utf8"));
|
|
40
|
+
} catch {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
const sizeOf = (file) => {
|
|
45
|
+
try {
|
|
46
|
+
const info = statSync(file);
|
|
47
|
+
return info.isFile() ? info.size : null;
|
|
48
|
+
} catch {
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
const alive = (pid) => {
|
|
53
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
54
|
+
try {
|
|
55
|
+
process.kill(pid, 0);
|
|
56
|
+
return true;
|
|
57
|
+
} catch (error) {
|
|
58
|
+
return error?.code === "EPERM";
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
const executable = async (file) => {
|
|
62
|
+
try {
|
|
63
|
+
await access(file, constants.X_OK);
|
|
64
|
+
return (await stat(file)).isFile();
|
|
65
|
+
} catch {
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
async function writeJson(file, value) {
|
|
70
|
+
const temporary = `${file}.tmp-${process.pid}-${randomBytes(4).toString("hex")}`;
|
|
71
|
+
await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
72
|
+
await rename(temporary, file);
|
|
73
|
+
}
|
|
74
|
+
async function sha256File(file, signal) {
|
|
75
|
+
const hash = createHash("sha256");
|
|
76
|
+
for await (const chunk of createReadStream(file)) {
|
|
77
|
+
if (signal?.aborted) throw cancelled();
|
|
78
|
+
hash.update(chunk);
|
|
79
|
+
}
|
|
80
|
+
return hash.digest("hex");
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The setup both surfaces share. status() reads the folder each time, so it also shows a job the
|
|
85
|
+
* other process runs; install() starts a job in the background; subscribe() hears its progress.
|
|
86
|
+
*/
|
|
87
|
+
export function createStableAudioSetup({
|
|
88
|
+
environment = process.env,
|
|
89
|
+
fetchImpl = globalThis.fetch,
|
|
90
|
+
spawnImpl = spawn,
|
|
91
|
+
platform = process.platform,
|
|
92
|
+
arch = process.arch,
|
|
93
|
+
now = () => new Date(),
|
|
94
|
+
sources = STABLE_AUDIO_SOURCES,
|
|
95
|
+
freeSpace = async (folder) => { const info = await statfs(folder); return info.bavail * info.bsize; },
|
|
96
|
+
} = {}) {
|
|
97
|
+
const supported = platform === "darwin" && arch === "arm64";
|
|
98
|
+
const root = resolveStableAudioManagedRoot(environment);
|
|
99
|
+
const files = {
|
|
100
|
+
lock: path.join(root, "setup.lock"),
|
|
101
|
+
job: path.join(root, "job.json"),
|
|
102
|
+
cancel: path.join(root, "cancel.json"),
|
|
103
|
+
downloads: path.join(root, "downloads"),
|
|
104
|
+
uv: path.join(root, "uv"),
|
|
105
|
+
uvMarker: path.join(root, "uv", ".moondog.json"),
|
|
106
|
+
mlx: path.join(root, "mlx"),
|
|
107
|
+
codeMarker: path.join(root, "mlx", ".moondog-code.json"),
|
|
108
|
+
venv: path.join(root, "mlx", ".venv"),
|
|
109
|
+
envMarker: path.join(root, "mlx", ".venv", ".moondog-env.json"),
|
|
110
|
+
weights: path.join(root, "mlx", "models", "mlx"),
|
|
111
|
+
licenses: resolveStableAudioLicenseFile(environment),
|
|
112
|
+
};
|
|
113
|
+
const weightFiles = (model) => [...SHARED_WEIGHTS, ...STABLE_AUDIO_MODELS[model].weights];
|
|
114
|
+
const weightSize = (name) => sources.weights.files[name].size;
|
|
115
|
+
const part = (name) => path.join(files.downloads, `${name}.part`);
|
|
116
|
+
const pythonPins = { python: sources.python.version, packages: [...sources.python.packages] };
|
|
117
|
+
|
|
118
|
+
// The job this process runs, or null.
|
|
119
|
+
let job = null;
|
|
120
|
+
const listeners = new Set();
|
|
121
|
+
let shown = null;
|
|
122
|
+
let pushTimer = null;
|
|
123
|
+
let poll = null;
|
|
124
|
+
|
|
125
|
+
const base = () => ({
|
|
126
|
+
uv: readJson(files.uvMarker)?.version === sources.uv.version && sizeOf(path.join(files.uv, "uv")) !== null ? "ready" : "absent",
|
|
127
|
+
code: readJson(files.codeMarker)?.commit === sources.code.commit ? "ready" : "absent",
|
|
128
|
+
python: JSON.stringify(readJson(files.envMarker)) === JSON.stringify(pythonPins) ? "ready" : "absent",
|
|
129
|
+
});
|
|
130
|
+
const jobView = () => {
|
|
131
|
+
if (job) return { model: job.model, step: job.step, received: job.received, total: job.total, running: true };
|
|
132
|
+
const saved = readJson(files.job);
|
|
133
|
+
if (!saved || !isStableAudioModel(saved.model)) return null;
|
|
134
|
+
const view = { model: saved.model, step: saved.step, received: saved.received ?? 0, total: saved.total ?? 0 };
|
|
135
|
+
// A failed job stays until the next install, so the surfaces can say why and offer Try again.
|
|
136
|
+
if (saved.error) return { ...view, running: false, error: saved.error };
|
|
137
|
+
// Another process's job shows while that process holds the lock; a crashed one's does not.
|
|
138
|
+
const lock = readJson(files.lock);
|
|
139
|
+
return lock && lock.token === saved.token && alive(lock.pid) ? { ...view, running: true } : null;
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* { supported, root, licenses: { accepted, at, links }, job, models, base }. job is null or
|
|
144
|
+
* { model, step, received, total, running, error? }: received and total count the step's bytes.
|
|
145
|
+
* Each model's bytes and total count all its weights, the shared text encoder included.
|
|
146
|
+
*/
|
|
147
|
+
const status = () => {
|
|
148
|
+
const agreement = readJson(files.licenses);
|
|
149
|
+
const steps = base();
|
|
150
|
+
const current = jobView();
|
|
151
|
+
const baseReady = Object.values(steps).every((state) => state === "ready");
|
|
152
|
+
const ownFiles = (model) => STABLE_AUDIO_MODELS[model].weights.some((name) => sizeOf(path.join(files.weights, name)) !== null || sizeOf(part(name)) !== null);
|
|
153
|
+
// A first setup stopped before any model's own weights arrived still left uv, Python, caches or the
|
|
154
|
+
// shared encoder: the model it was for reads as partial, so its Remove can take them away.
|
|
155
|
+
const leftover = !Object.keys(STABLE_AUDIO_MODELS).some(ownFiles) && existsSync(root) ? (current?.model ?? DEFAULT_STABLE_AUDIO_MODEL) : null;
|
|
156
|
+
const models = Object.fromEntries(Object.keys(STABLE_AUDIO_MODELS).map((model) => {
|
|
157
|
+
let bytes = 0;
|
|
158
|
+
let complete = true;
|
|
159
|
+
for (const name of weightFiles(model)) {
|
|
160
|
+
const size = weightSize(name);
|
|
161
|
+
const present = sizeOf(path.join(files.weights, name)) === size;
|
|
162
|
+
if (!present) complete = false;
|
|
163
|
+
bytes += present ? size : Math.min(size, sizeOf(part(name)) ?? 0);
|
|
164
|
+
}
|
|
165
|
+
const own = ownFiles(model) || leftover === model;
|
|
166
|
+
const total = weightFiles(model).reduce((sum, name) => sum + weightSize(name), 0);
|
|
167
|
+
const state = complete && baseReady ? "ready" : own || (current?.running && current.model === model) ? "partial" : "absent";
|
|
168
|
+
return [model, { state, bytes, total }];
|
|
169
|
+
}));
|
|
170
|
+
return {
|
|
171
|
+
supported,
|
|
172
|
+
root,
|
|
173
|
+
licenses: { accepted: Boolean(agreement?.accepted_at), at: agreement?.accepted_at ?? null, links: sources.licenses.map((license) => ({ ...license })) },
|
|
174
|
+
job: current,
|
|
175
|
+
models,
|
|
176
|
+
base: steps,
|
|
177
|
+
};
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
const push = () => {
|
|
181
|
+
clearTimeout(pushTimer);
|
|
182
|
+
pushTimer = null;
|
|
183
|
+
if (!listeners.size) return;
|
|
184
|
+
const snapshot = status();
|
|
185
|
+
const key = JSON.stringify(snapshot);
|
|
186
|
+
if (key === shown) return;
|
|
187
|
+
shown = key;
|
|
188
|
+
for (const listener of [...listeners]) {
|
|
189
|
+
try { listener(structuredClone(snapshot)); } catch { /* A surface's redraw cannot stop the download. */ }
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
const pushSoon = () => { pushTimer ??= setTimeout(push, PUSH_MS); pushTimer.unref?.(); };
|
|
193
|
+
|
|
194
|
+
// The status file the other process reads, written with each snapshot at most.
|
|
195
|
+
let saving = Promise.resolve();
|
|
196
|
+
const saveJob = (extra = {}) => {
|
|
197
|
+
if (!job) return saving;
|
|
198
|
+
const record = { pid: process.pid, token: job.token, model: job.model, step: job.step, received: job.received, total: job.total, updated_at: now().toISOString(), ...extra };
|
|
199
|
+
saving = saving.then(() => writeJson(files.job, record)).catch(() => {});
|
|
200
|
+
return saving;
|
|
201
|
+
};
|
|
202
|
+
let saveTimer = null;
|
|
203
|
+
const progress = (received, total) => {
|
|
204
|
+
job.received = received;
|
|
205
|
+
job.total = total;
|
|
206
|
+
if (!saveTimer) {
|
|
207
|
+
saveTimer = setTimeout(() => { saveTimer = null; void saveJob(); }, PUSH_MS);
|
|
208
|
+
saveTimer.unref?.();
|
|
209
|
+
}
|
|
210
|
+
pushSoon();
|
|
211
|
+
};
|
|
212
|
+
const step = (name, total = 0) => {
|
|
213
|
+
job.step = name;
|
|
214
|
+
job.received = 0;
|
|
215
|
+
job.total = total;
|
|
216
|
+
void saveJob();
|
|
217
|
+
push();
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
const takeLock = async (model, token) => {
|
|
221
|
+
await mkdir(root, { recursive: true, mode: 0o700 });
|
|
222
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
223
|
+
try {
|
|
224
|
+
await writeFile(files.lock, JSON.stringify({ pid: process.pid, token, model, started_at: now().toISOString() }), { flag: "wx", mode: 0o600 });
|
|
225
|
+
return;
|
|
226
|
+
} catch (error) {
|
|
227
|
+
if (error?.code !== "EEXIST") throw error;
|
|
228
|
+
}
|
|
229
|
+
const held = readJson(files.lock);
|
|
230
|
+
let age = 0;
|
|
231
|
+
try { age = Date.now() - statSync(files.lock).mtimeMs; } catch { continue; }
|
|
232
|
+
if (held ? alive(held.pid) : age < LOCK_WRITE_MS) break;
|
|
233
|
+
// The process that held it has ended, so its job is over.
|
|
234
|
+
await rm(files.lock, { force: true });
|
|
235
|
+
}
|
|
236
|
+
throw setupError("setup_busy", "Stable Audio 3 is already being set up, here or in the other Moondog window or terminal.");
|
|
237
|
+
};
|
|
238
|
+
const releaseLock = async (token) => {
|
|
239
|
+
if (readJson(files.lock)?.token === token) await rm(files.lock, { force: true });
|
|
240
|
+
};
|
|
241
|
+
const runningElsewhere = () => {
|
|
242
|
+
const lock = readJson(files.lock);
|
|
243
|
+
return Boolean(lock && alive(lock.pid) && lock.token !== job?.token);
|
|
244
|
+
};
|
|
245
|
+
|
|
246
|
+
const ensureSpace = async (bytes) => {
|
|
247
|
+
const free = await freeSpace(root);
|
|
248
|
+
const needed = bytes + SPACE_MARGIN;
|
|
249
|
+
if (free < needed) {
|
|
250
|
+
throw setupError("setup_disk_full", `There is not enough space on this Mac: ${gigabytes(needed)} is needed and ${gigabytes(free)} is free.`, { needed, free });
|
|
251
|
+
}
|
|
252
|
+
};
|
|
253
|
+
|
|
254
|
+
// A command the setup runs (tar, uv). Its output can hold paths, so only the exit code is reported.
|
|
255
|
+
const runCommand = (command, args, { env = environment, code, message }) => new Promise((resolve, reject) => {
|
|
256
|
+
const { signal } = job.controller;
|
|
257
|
+
if (signal.aborted) return reject(cancelled());
|
|
258
|
+
const child = spawnImpl(command, args, { stdio: ["ignore", "ignore", "pipe"], env });
|
|
259
|
+
let tail = "";
|
|
260
|
+
child.stderr?.setEncoding("utf8");
|
|
261
|
+
child.stderr?.on("data", (chunk) => { tail = (tail + chunk).slice(-4000); });
|
|
262
|
+
const stop = () => child.kill("SIGTERM");
|
|
263
|
+
signal.addEventListener("abort", stop, { once: true });
|
|
264
|
+
child.once("error", () => { signal.removeEventListener("abort", stop); reject(setupError(code, message)); });
|
|
265
|
+
child.once("exit", (exit) => {
|
|
266
|
+
signal.removeEventListener("abort", stop);
|
|
267
|
+
if (signal.aborted) reject(cancelled());
|
|
268
|
+
else if (exit === 0) resolve();
|
|
269
|
+
else reject(setupError(code, message, { detail: tail.trim().split("\n").slice(-3).join("\n") }));
|
|
270
|
+
});
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Downloads one file into downloads/<name>.part, resuming with an HTTP Range request, checks its
|
|
275
|
+
* size and sha256, and renames it into place. A file that fails the check is deleted; one cut off
|
|
276
|
+
* by a dropped connection or a cancel stays, so the next install resumes it.
|
|
277
|
+
*/
|
|
278
|
+
const download = async ({ url, name, size, sha256, target, onBytes }) => {
|
|
279
|
+
const { signal } = job.controller;
|
|
280
|
+
const partial = part(name);
|
|
281
|
+
await mkdir(files.downloads, { recursive: true, mode: 0o700 });
|
|
282
|
+
let offset = sizeOf(partial) ?? 0;
|
|
283
|
+
if (size !== undefined && offset > size) {
|
|
284
|
+
await rm(partial, { force: true });
|
|
285
|
+
offset = 0;
|
|
286
|
+
}
|
|
287
|
+
let hash = createHash("sha256");
|
|
288
|
+
if (offset > 0) {
|
|
289
|
+
for await (const chunk of createReadStream(partial, { end: offset - 1 })) {
|
|
290
|
+
if (signal.aborted) throw cancelled();
|
|
291
|
+
hash.update(chunk);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
const host = new URL(url).host;
|
|
295
|
+
let response;
|
|
296
|
+
try {
|
|
297
|
+
response = await fetchImpl(url, { headers: offset > 0 ? { Range: `bytes=${offset}-` } : {}, redirect: "follow", signal });
|
|
298
|
+
} catch {
|
|
299
|
+
if (signal.aborted) throw cancelled();
|
|
300
|
+
throw setupError("setup_offline", `Moondog could not reach ${host}. Check the connection, then try again.`);
|
|
301
|
+
}
|
|
302
|
+
const finished = offset > 0 && response.status === 416 && size !== undefined && offset === size;
|
|
303
|
+
if (offset > 0 && response.status === 416 && !finished) {
|
|
304
|
+
// A partial file the server no longer accepts cannot be continued.
|
|
305
|
+
await rm(partial, { force: true });
|
|
306
|
+
throw setupError("setup_download_failed", `The download from ${host} could not continue. Try again.`);
|
|
307
|
+
}
|
|
308
|
+
if (offset > 0 && !finished && response.status !== 206) {
|
|
309
|
+
// The server sent the whole file again, or nothing usable: start over.
|
|
310
|
+
offset = 0;
|
|
311
|
+
hash = createHash("sha256");
|
|
312
|
+
}
|
|
313
|
+
if (!finished && !response.ok) {
|
|
314
|
+
await response.body?.cancel().catch(() => {});
|
|
315
|
+
throw setupError("setup_download_failed", `The download from ${host} failed (HTTP ${response.status}). Try again later.`);
|
|
316
|
+
}
|
|
317
|
+
let received = offset;
|
|
318
|
+
const length = Number(response.headers.get("content-length"));
|
|
319
|
+
const total = size ?? (Number.isFinite(length) && length > 0 ? offset + length : 0);
|
|
320
|
+
onBytes?.(received, total);
|
|
321
|
+
if (finished) await response.body?.cancel().catch(() => {});
|
|
322
|
+
else {
|
|
323
|
+
const handle = await open(partial, offset > 0 ? "a" : "w", 0o600);
|
|
324
|
+
try {
|
|
325
|
+
for await (const chunk of response.body) {
|
|
326
|
+
if (signal.aborted) throw cancelled();
|
|
327
|
+
await handle.write(chunk);
|
|
328
|
+
hash.update(chunk);
|
|
329
|
+
received += chunk.length;
|
|
330
|
+
if (size !== undefined && received > size) break;
|
|
331
|
+
onBytes?.(received, total);
|
|
332
|
+
}
|
|
333
|
+
} catch (error) {
|
|
334
|
+
if (signal.aborted || error?.code === "setup_cancelled") throw cancelled();
|
|
335
|
+
throw setupError("setup_download_failed", `The download from ${host} stopped. Try again to continue it.`);
|
|
336
|
+
} finally {
|
|
337
|
+
await handle.close();
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
if (size !== undefined && received < size) {
|
|
341
|
+
throw setupError("setup_download_failed", `The download from ${host} stopped. Try again to continue it.`);
|
|
342
|
+
}
|
|
343
|
+
if ((size !== undefined && received !== size) || (sha256 && hash.digest("hex") !== sha256)) {
|
|
344
|
+
await rm(partial, { force: true });
|
|
345
|
+
throw setupError("setup_verification_failed", `${name} did not match the file Moondog expects, so it was deleted. Try again.`, { file: name });
|
|
346
|
+
}
|
|
347
|
+
await mkdir(path.dirname(target), { recursive: true, mode: 0o700 });
|
|
348
|
+
await rename(partial, target);
|
|
349
|
+
return target;
|
|
350
|
+
};
|
|
351
|
+
|
|
352
|
+
// Checks an unpacked tree against the code's manifest: every file there, the same bytes, and
|
|
353
|
+
// nothing else, links included, so the run never loads a file that was not checked.
|
|
354
|
+
const verifyTree = async (folder) => {
|
|
355
|
+
const expected = new Map(sources.code.files.map(([file, size, sha256]) => [file, { size, sha256 }]));
|
|
356
|
+
const found = new Set();
|
|
357
|
+
const walk = async (relative) => {
|
|
358
|
+
for (const entry of await readdir(path.join(folder, relative))) {
|
|
359
|
+
const file = relative ? `${relative}/${entry}` : entry;
|
|
360
|
+
const info = await lstat(path.join(folder, file));
|
|
361
|
+
if (info.isDirectory()) { await walk(file); continue; }
|
|
362
|
+
const want = expected.get(file);
|
|
363
|
+
if (!info.isFile() || !want || info.size !== want.size || (await sha256File(path.join(folder, file), job.controller.signal)) !== want.sha256) {
|
|
364
|
+
throw setupError("setup_verification_failed", "Stable Audio 3's code did not match the files Moondog expects, so it was deleted. Try again.", { file });
|
|
365
|
+
}
|
|
366
|
+
found.add(file);
|
|
367
|
+
}
|
|
368
|
+
};
|
|
369
|
+
await walk("");
|
|
370
|
+
if (found.size !== expected.size) throw setupError("setup_verification_failed", "Stable Audio 3's code is missing files Moondog expects, so it was deleted. Try again.");
|
|
371
|
+
};
|
|
372
|
+
|
|
373
|
+
const setUpUv = async () => {
|
|
374
|
+
if ((await executable(path.join(files.uv, "uv"))) && readJson(files.uvMarker)?.version === sources.uv.version) return;
|
|
375
|
+
step("uv", sources.uv.size);
|
|
376
|
+
await ensureSpace(sources.uv.size * 2 + STEP_SPACE.uv);
|
|
377
|
+
const archive = await download({ url: sources.uv.url, name: "uv.tar.gz", size: sources.uv.size, sha256: sources.uv.sha256,
|
|
378
|
+
target: path.join(files.downloads, "uv.tar.gz"), onBytes: progress });
|
|
379
|
+
const temporary = `${files.uv}.tmp-${job.token}`;
|
|
380
|
+
try {
|
|
381
|
+
await rm(temporary, { recursive: true, force: true });
|
|
382
|
+
await mkdir(temporary, { recursive: true, mode: 0o700 });
|
|
383
|
+
await runCommand("/usr/bin/tar", ["-xzf", archive, "-C", temporary, "--strip-components", "1"],
|
|
384
|
+
{ code: "setup_verification_failed", message: "uv could not be unpacked. Try again." });
|
|
385
|
+
if (!(await executable(path.join(temporary, "uv")))) throw setupError("setup_verification_failed", "uv was not in its download. Try again.");
|
|
386
|
+
await writeJson(path.join(temporary, ".moondog.json"), { version: sources.uv.version });
|
|
387
|
+
await rm(files.uv, { recursive: true, force: true });
|
|
388
|
+
await rename(temporary, files.uv);
|
|
389
|
+
} finally {
|
|
390
|
+
await rm(temporary, { recursive: true, force: true });
|
|
391
|
+
await rm(archive, { force: true });
|
|
392
|
+
}
|
|
393
|
+
};
|
|
394
|
+
|
|
395
|
+
const setUpCode = async () => {
|
|
396
|
+
if (readJson(files.codeMarker)?.commit === sources.code.commit) return;
|
|
397
|
+
step("code");
|
|
398
|
+
await ensureSpace(STEP_SPACE.code);
|
|
399
|
+
const archive = await download({ url: sources.code.url, name: "code.tar.gz", target: path.join(files.downloads, "code.tar.gz"), onBytes: progress });
|
|
400
|
+
const temporary = `${files.mlx}.tmp-${job.token}`;
|
|
401
|
+
try {
|
|
402
|
+
await rm(temporary, { recursive: true, force: true });
|
|
403
|
+
await mkdir(temporary, { recursive: true, mode: 0o700 });
|
|
404
|
+
step("verifying");
|
|
405
|
+
const strip = String(sources.code.folder.split("/").length);
|
|
406
|
+
await runCommand("/usr/bin/tar", ["-xzf", archive, "-C", temporary, "--strip-components", strip, sources.code.folder],
|
|
407
|
+
{ code: "setup_verification_failed", message: "Stable Audio 3's code could not be unpacked. Try again." });
|
|
408
|
+
await verifyTree(temporary);
|
|
409
|
+
// Weights and the environment from an earlier install move into the new tree.
|
|
410
|
+
for (const kept of [path.join("models", "mlx"), ".venv"]) {
|
|
411
|
+
if ((await stat(path.join(files.mlx, kept)).catch(() => null))?.isDirectory()) {
|
|
412
|
+
await mkdir(path.dirname(path.join(temporary, kept)), { recursive: true });
|
|
413
|
+
await rename(path.join(files.mlx, kept), path.join(temporary, kept));
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
await writeJson(path.join(temporary, ".moondog-code.json"), { commit: sources.code.commit });
|
|
417
|
+
await rm(files.mlx, { recursive: true, force: true });
|
|
418
|
+
await rename(temporary, files.mlx);
|
|
419
|
+
} finally {
|
|
420
|
+
await rm(temporary, { recursive: true, force: true });
|
|
421
|
+
await rm(archive, { force: true });
|
|
422
|
+
}
|
|
423
|
+
};
|
|
424
|
+
|
|
425
|
+
const setUpPython = async () => {
|
|
426
|
+
const python = path.join(files.venv, "bin", "python");
|
|
427
|
+
if ((await executable(python)) && JSON.stringify(readJson(files.envMarker)) === JSON.stringify(pythonPins)) return;
|
|
428
|
+
step("python");
|
|
429
|
+
await ensureSpace(STEP_SPACE.python);
|
|
430
|
+
// Only the pinned Python and packages from uv's defaults: the listener's own uv or pip settings
|
|
431
|
+
// could point elsewhere, and the caches and Python stay in this folder so remove takes them too.
|
|
432
|
+
const env = Object.fromEntries(Object.entries(environment).filter(([name]) => !/^(UV_|PIP_|VIRTUAL_ENV$|CONDA_|PYTHON)/u.test(name)));
|
|
433
|
+
Object.assign(env, { UV_CACHE_DIR: path.join(root, "uv-cache"), UV_PYTHON_INSTALL_DIR: path.join(root, "python"),
|
|
434
|
+
UV_PYTHON_PREFERENCE: "only-managed", UV_NO_CONFIG: "1", UV_NO_PROGRESS: "1" });
|
|
435
|
+
const uv = path.join(files.uv, "uv");
|
|
436
|
+
const failed = { code: "setup_python_failed", message: "The Python environment could not be set up. Check the connection, then try again." };
|
|
437
|
+
await runCommand(uv, ["venv", "--clear", "--python", sources.python.version, files.venv], { env, ...failed });
|
|
438
|
+
await runCommand(uv, ["pip", "install", "--python", python, ...sources.python.packages], { env, ...failed });
|
|
439
|
+
await writeJson(files.envMarker, pythonPins);
|
|
440
|
+
};
|
|
441
|
+
|
|
442
|
+
const setUpWeights = async (model) => {
|
|
443
|
+
const names = weightFiles(model);
|
|
444
|
+
const total = names.reduce((sum, name) => sum + weightSize(name), 0);
|
|
445
|
+
const missing = names.filter((name) => sizeOf(path.join(files.weights, name)) !== weightSize(name));
|
|
446
|
+
if (!missing.length) return;
|
|
447
|
+
step("weights", total);
|
|
448
|
+
await ensureSpace(missing.reduce((sum, name) => sum + weightSize(name) - Math.min(weightSize(name), sizeOf(part(name)) ?? 0), 0));
|
|
449
|
+
let done = total - missing.reduce((sum, name) => sum + weightSize(name), 0);
|
|
450
|
+
for (const name of missing) {
|
|
451
|
+
const { size, sha256 } = sources.weights.files[name];
|
|
452
|
+
await download({ url: sources.weights.url(name), name, size, sha256, target: path.join(files.weights, name),
|
|
453
|
+
onBytes: (received) => progress(done + received, total) });
|
|
454
|
+
done += size;
|
|
455
|
+
}
|
|
456
|
+
};
|
|
457
|
+
|
|
458
|
+
const finish = async (model) => {
|
|
459
|
+
step("verifying");
|
|
460
|
+
const install = await checkStableAudioInstall(files.mlx, { model, platform, arch });
|
|
461
|
+
if (install.state !== "ready") throw setupError("setup_verification_failed", "Stable Audio 3 was set up but is not ready to run. Try again.");
|
|
462
|
+
// The download was the listener's ask, so this model is turned on, unless a folder they chose
|
|
463
|
+
// themselves runs now (they can still pick this one with Use) or MOONDOG_SA3_DIR decides.
|
|
464
|
+
const settings = await readGenerationSettings(environment);
|
|
465
|
+
if (settings.source === "environment") return;
|
|
466
|
+
// A choice the listener made while this ran, here or in the other window or terminal, stands.
|
|
467
|
+
if (JSON.stringify(await readSavedSettings(environment)) !== job.settingsAtStart) return;
|
|
468
|
+
const runsNow = async () => (await checkStableAudioInstall(settings.path, { model: settings.model, platform, arch })).state === "ready";
|
|
469
|
+
// A folder they chose, or one turned on that runs (~/Developer included), or a model of Moondog's
|
|
470
|
+
// own already in use stays in use.
|
|
471
|
+
if ((settings.source === "settings" || settings.enabled) && (await runsNow())) return;
|
|
472
|
+
await writeGenerationSettings({ source: "managed", model, enabled: true }, environment);
|
|
473
|
+
};
|
|
474
|
+
|
|
475
|
+
const run = async (model, { baseOnly }) => {
|
|
476
|
+
await setUpUv();
|
|
477
|
+
await setUpCode();
|
|
478
|
+
await setUpPython();
|
|
479
|
+
if (baseOnly) return;
|
|
480
|
+
await setUpWeights(model);
|
|
481
|
+
await finish(model);
|
|
482
|
+
};
|
|
483
|
+
|
|
484
|
+
// A cancel asked in the other process reaches the job through a file that names it.
|
|
485
|
+
const watchCancel = (token, controller) => {
|
|
486
|
+
const timer = setInterval(() => { if (readJson(files.cancel)?.token === token) controller.abort(); }, 500);
|
|
487
|
+
timer.unref?.();
|
|
488
|
+
return () => clearInterval(timer);
|
|
489
|
+
};
|
|
490
|
+
|
|
491
|
+
return {
|
|
492
|
+
supported,
|
|
493
|
+
root,
|
|
494
|
+
status,
|
|
495
|
+
/**
|
|
496
|
+
* Starts setting up a model in the background; resolves with status() once the job holds the
|
|
497
|
+
* lock. Refuses without the licenses: pass accepted: true once, and the agreement is kept.
|
|
498
|
+
*/
|
|
499
|
+
async install(model, { accepted = false, baseOnly = false } = {}) {
|
|
500
|
+
if (!isStableAudioModel(model)) throw setupError("setup_model_unknown", "Choose medium or small-music.");
|
|
501
|
+
if (!supported) throw setupError("setup_unsupported", "Stable Audio 3 needs a Mac with Apple silicon.");
|
|
502
|
+
if (job) throw setupError("setup_busy", "Stable Audio 3 is already being set up.");
|
|
503
|
+
if (!readJson(files.licenses)?.accepted_at) {
|
|
504
|
+
if (accepted !== true) {
|
|
505
|
+
throw setupError("setup_license_required", "Agree to the Stability AI Community License and the Gemma Terms of Use first.", { licenses: sources.licenses.map((license) => ({ ...license })) });
|
|
506
|
+
}
|
|
507
|
+
await mkdir(path.dirname(files.licenses), { recursive: true, mode: 0o700 });
|
|
508
|
+
await writeJson(files.licenses, { version: 1, accepted_at: now().toISOString(), licenses: sources.licenses.map(({ id, url }) => ({ id, url })) });
|
|
509
|
+
}
|
|
510
|
+
const token = randomBytes(8).toString("hex");
|
|
511
|
+
await takeLock(model, token);
|
|
512
|
+
await rm(files.cancel, { force: true });
|
|
513
|
+
const controller = new AbortController();
|
|
514
|
+
job = { model, token, controller, step: "uv", received: 0, total: 0,
|
|
515
|
+
settingsAtStart: JSON.stringify(await readSavedSettings(environment)) };
|
|
516
|
+
await saveJob();
|
|
517
|
+
const unwatch = watchCancel(token, controller);
|
|
518
|
+
job.done = (async () => {
|
|
519
|
+
let failure = null;
|
|
520
|
+
try {
|
|
521
|
+
await run(model, { baseOnly });
|
|
522
|
+
} catch (error) {
|
|
523
|
+
if (!controller.signal.aborted && error?.code !== "setup_cancelled") {
|
|
524
|
+
failure = { code: typeof error?.code === "string" && error.code.startsWith("setup_") ? error.code : "setup_failed",
|
|
525
|
+
message: typeof error?.code === "string" && error.code.startsWith("setup_") ? error.message : "Stable Audio 3 could not be set up. Try again." };
|
|
526
|
+
}
|
|
527
|
+
} finally {
|
|
528
|
+
unwatch();
|
|
529
|
+
clearTimeout(saveTimer);
|
|
530
|
+
saveTimer = null;
|
|
531
|
+
if (failure) await saveJob({ error: failure });
|
|
532
|
+
await saving;
|
|
533
|
+
if (!failure) await rm(files.job, { force: true });
|
|
534
|
+
await rm(files.cancel, { force: true });
|
|
535
|
+
await releaseLock(token);
|
|
536
|
+
job = null;
|
|
537
|
+
push();
|
|
538
|
+
}
|
|
539
|
+
return status();
|
|
540
|
+
})();
|
|
541
|
+
push();
|
|
542
|
+
return status();
|
|
543
|
+
},
|
|
544
|
+
/** Waits for this process's job to end; resolves with status(). */
|
|
545
|
+
wait: () => job?.done ?? Promise.resolve(status()),
|
|
546
|
+
/** Stops the job, here or in the other process; verified and .part files stay for a later install. */
|
|
547
|
+
/** Stops this process's own job as it closes, keeping what was verified; another process's job runs on. */
|
|
548
|
+
async close() {
|
|
549
|
+
if (!job) return false;
|
|
550
|
+
job.controller.abort();
|
|
551
|
+
await job.done;
|
|
552
|
+
return true;
|
|
553
|
+
},
|
|
554
|
+
async cancel() {
|
|
555
|
+
if (job) {
|
|
556
|
+
job.controller.abort();
|
|
557
|
+
await job.done;
|
|
558
|
+
return true;
|
|
559
|
+
}
|
|
560
|
+
const lock = readJson(files.lock);
|
|
561
|
+
if (!lock || !alive(lock.pid)) return false;
|
|
562
|
+
await writeJson(files.cancel, { token: lock.token });
|
|
563
|
+
return true;
|
|
564
|
+
},
|
|
565
|
+
/**
|
|
566
|
+
* Deletes a model's own weights. With no model left, the whole folder goes: uv, the code,
|
|
567
|
+
* Python, the caches and the shared weights. The licenses stay agreed. Refuses while a job runs.
|
|
568
|
+
*/
|
|
569
|
+
async remove(model) {
|
|
570
|
+
if (!isStableAudioModel(model)) throw setupError("setup_model_unknown", "Choose medium or small-music.");
|
|
571
|
+
if (job || runningElsewhere()) throw setupError("setup_busy", "Stable Audio 3 is being set up. Cancel it first.");
|
|
572
|
+
if (!(await stat(root).catch(() => null))) return status();
|
|
573
|
+
const token = randomBytes(8).toString("hex");
|
|
574
|
+
await takeLock(model, token);
|
|
575
|
+
const other = Object.keys(STABLE_AUDIO_MODELS).find((name) => name !== model);
|
|
576
|
+
try {
|
|
577
|
+
for (const name of STABLE_AUDIO_MODELS[model].weights) {
|
|
578
|
+
await rm(path.join(files.weights, name), { force: true });
|
|
579
|
+
await rm(part(name), { force: true });
|
|
580
|
+
}
|
|
581
|
+
await rm(files.job, { force: true });
|
|
582
|
+
const otherLeft = STABLE_AUDIO_MODELS[other].weights.some((name) => sizeOf(path.join(files.weights, name)) !== null || sizeOf(part(name)) !== null);
|
|
583
|
+
if (!otherLeft) await rm(root, { recursive: true, force: true });
|
|
584
|
+
} finally {
|
|
585
|
+
await releaseLock(token);
|
|
586
|
+
}
|
|
587
|
+
// Music making that used this model moves to the other one, or back to the listener's folder with a
|
|
588
|
+
// model that folder has, so a working own install is never checked for weights it never had.
|
|
589
|
+
const settings = await readGenerationSettings({ ...environment, MOONDOG_SA3_DIR: "" }, { platform, arch });
|
|
590
|
+
if (settings.source === "managed" && settings.model === model) {
|
|
591
|
+
if (status().models[other].state === "ready") await writeGenerationSettings({ model: other }, environment);
|
|
592
|
+
else {
|
|
593
|
+
const candidates = [DEFAULT_STABLE_AUDIO_MODEL, ...Object.keys(STABLE_AUDIO_MODELS).filter((name) => name !== DEFAULT_STABLE_AUDIO_MODEL)];
|
|
594
|
+
let found = DEFAULT_STABLE_AUDIO_MODEL;
|
|
595
|
+
for (const name of candidates) {
|
|
596
|
+
if ((await checkStableAudioInstall(settings.folder, { model: name, platform, arch })).state === "ready") { found = name; break; }
|
|
597
|
+
}
|
|
598
|
+
await writeGenerationSettings({ source: "settings", model: found }, environment);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
push();
|
|
602
|
+
return status();
|
|
603
|
+
},
|
|
604
|
+
/** Hears status() snapshots while anything changes, from this process's job or the other's. */
|
|
605
|
+
subscribe(listener) {
|
|
606
|
+
listeners.add(listener);
|
|
607
|
+
if (!poll) {
|
|
608
|
+
poll = setInterval(push, POLL_MS);
|
|
609
|
+
poll.unref?.();
|
|
610
|
+
}
|
|
611
|
+
return () => {
|
|
612
|
+
listeners.delete(listener);
|
|
613
|
+
if (!listeners.size && poll) {
|
|
614
|
+
clearInterval(poll);
|
|
615
|
+
poll = null;
|
|
616
|
+
shown = null;
|
|
617
|
+
}
|
|
618
|
+
};
|
|
619
|
+
},
|
|
620
|
+
};
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// A developer's way to try the real downloads from a terminal, into the state folder
|
|
624
|
+
// MOONDOG_STATE_HOME names: node src/integrations/stable-audio/setup.mjs status | install
|
|
625
|
+
// medium|small-music [--agree] [--base-only] | remove medium|small-music. --base-only stops before
|
|
626
|
+
// the weights, so uv, the code and Python can be tried without gigabytes.
|
|
627
|
+
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
628
|
+
const [command = "status", model, ...flags] = process.argv.slice(2);
|
|
629
|
+
const setup = createStableAudioSetup();
|
|
630
|
+
const line = (snapshot) => {
|
|
631
|
+
const { job } = snapshot;
|
|
632
|
+
const state = Object.entries(snapshot.models).map(([name, entry]) => `${name} ${entry.state} ${gigabytes(entry.bytes)}/${gigabytes(entry.total)}`).join(", ");
|
|
633
|
+
return `${job ? `${job.step} ${job.received}/${job.total}${job.error ? ` ${job.error.code}: ${job.error.message}` : ""} | ` : ""}${state} | base ${JSON.stringify(snapshot.base)}`;
|
|
634
|
+
};
|
|
635
|
+
try {
|
|
636
|
+
if (command === "install") {
|
|
637
|
+
const unsubscribe = setup.subscribe((snapshot) => console.log(line(snapshot)));
|
|
638
|
+
process.once("SIGINT", () => { void setup.cancel(); });
|
|
639
|
+
await setup.install(model, { accepted: flags.includes("--agree"), baseOnly: flags.includes("--base-only") });
|
|
640
|
+
const end = await setup.wait();
|
|
641
|
+
unsubscribe();
|
|
642
|
+
console.log(line(end));
|
|
643
|
+
process.exitCode = end.job?.error ? 1 : 0;
|
|
644
|
+
} else if (command === "remove") {
|
|
645
|
+
console.log(line(await setup.remove(model)));
|
|
646
|
+
} else {
|
|
647
|
+
console.log(JSON.stringify(setup.status(), null, 2));
|
|
648
|
+
}
|
|
649
|
+
} catch (error) {
|
|
650
|
+
console.error(`${error.code ?? "error"}: ${error.message}`);
|
|
651
|
+
process.exitCode = 1;
|
|
652
|
+
}
|
|
653
|
+
}
|