@osmo.inc/cli 0.1.24 → 0.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DEPLOY.md +36 -1
- package/README.md +5 -0
- package/dist/osmo.js +3019 -286
- package/package.json +4 -4
- package/release.json +11 -11
package/dist/osmo.js
CHANGED
|
@@ -1,10 +1,3 @@
|
|
|
1
|
-
// src/commands.ts
|
|
2
|
-
import { existsSync as existsSync3, readFileSync as readFileSync2 } from "node:fs";
|
|
3
|
-
import { createRequire } from "node:module";
|
|
4
|
-
import path8 from "node:path";
|
|
5
|
-
import { fileURLToPath as fileURLToPath2, pathToFileURL } from "node:url";
|
|
6
|
-
import { parseArgs } from "node:util";
|
|
7
|
-
|
|
8
1
|
// src/auth.ts
|
|
9
2
|
import { createHash as createHash2, randomUUID as randomUUID2 } from "node:crypto";
|
|
10
3
|
import {
|
|
@@ -3052,8 +3045,8 @@ async function withCredentials(origin, action) {
|
|
|
3052
3045
|
}
|
|
3053
3046
|
try {
|
|
3054
3047
|
try {
|
|
3055
|
-
const
|
|
3056
|
-
if (!
|
|
3048
|
+
const stat9 = await lstat(file);
|
|
3049
|
+
if (!stat9.isFile()) throw new Error("Not a regular file");
|
|
3057
3050
|
await chmod(file, 384);
|
|
3058
3051
|
state = JSON.parse(await readFile2(file, "utf8"));
|
|
3059
3052
|
if (state.version !== 1 || state.origin !== origin || state.credentials && !validCredentials(state.credentials) || state.pending && (!validClient(state.pending.client) || typeof state.pending.sessionId !== "string" || typeof state.pending.deviceCode !== "string" || typeof state.pending.code !== "string" || !Number.isFinite(state.pending.expiresAt) || !Number.isFinite(state.pending.nextPollAt) || !Number.isFinite(state.pending.interval) || state.pending.interval < 5 || !validApprovalUrl(state.pending.url, origin)))
|
|
@@ -3493,6 +3486,71 @@ Next: ${result.next}`;
|
|
|
3493
3486
|
Next: ${result.next}`;
|
|
3494
3487
|
}
|
|
3495
3488
|
|
|
3489
|
+
// src/cloud-request.ts
|
|
3490
|
+
var PERMISSION_DOES = {
|
|
3491
|
+
"projects:write": "change projects",
|
|
3492
|
+
"energy:spend": "run models on your plan"
|
|
3493
|
+
};
|
|
3494
|
+
function scopeRequired(origin, scope) {
|
|
3495
|
+
const login = origin === authOrigin() ? "osmo login" : `OSMO_BASE_URL=${origin} osmo login`;
|
|
3496
|
+
return new OsmoError(
|
|
3497
|
+
"scope_required",
|
|
3498
|
+
`This login can't ${PERMISSION_DOES[scope ?? ""] ?? "do that"}. Sign in again to allow it: ${login}`,
|
|
3499
|
+
login
|
|
3500
|
+
);
|
|
3501
|
+
}
|
|
3502
|
+
async function call(origin, route, { method = "GET", body: body2, timeoutMs = 6e4, allow = [] } = {}) {
|
|
3503
|
+
const response = await authenticatedFetch(
|
|
3504
|
+
route,
|
|
3505
|
+
{ method, body: body2, timeoutMs },
|
|
3506
|
+
origin
|
|
3507
|
+
);
|
|
3508
|
+
const data = await response.json().catch(() => ({}));
|
|
3509
|
+
if (response.status === 403 && data.error === "scope_required" && !allow.includes(403))
|
|
3510
|
+
throw scopeRequired(origin, data.scope);
|
|
3511
|
+
if (!response.ok && !allow.includes(response.status))
|
|
3512
|
+
throw new Error(
|
|
3513
|
+
`Cloud ${response.status}: ${typeof data.error === "string" ? data.error : response.status === 400 ? "the request was invalid" : "request failed"}`
|
|
3514
|
+
);
|
|
3515
|
+
return { status: response.status, ...data };
|
|
3516
|
+
}
|
|
3517
|
+
|
|
3518
|
+
// src/cloud-render.ts
|
|
3519
|
+
var isCloudRenderJob = (job) => /^render_[a-z0-9]{20,32}$/.test(job);
|
|
3520
|
+
async function cloudRenderJob(job, origin = authOrigin(), cancel = false) {
|
|
3521
|
+
if (!isCloudRenderJob(job))
|
|
3522
|
+
throw new OsmoError(
|
|
3523
|
+
"invalid_job",
|
|
3524
|
+
"Use the render job ID returned by cloud deploy."
|
|
3525
|
+
);
|
|
3526
|
+
return call(
|
|
3527
|
+
origin,
|
|
3528
|
+
`/api/cloud-renders/${job}${cancel ? "/cancel" : ""}`,
|
|
3529
|
+
cancel ? { method: "POST" } : {}
|
|
3530
|
+
);
|
|
3531
|
+
}
|
|
3532
|
+
function formatCloudRender(job) {
|
|
3533
|
+
return `Cloud render ${job.job}: ${job.cancelRequested ? "cancellation requested" : job.status}${job.progress ? ` (${job.progress}%)` : ""}.
|
|
3534
|
+
Review: ${job.link}
|
|
3535
|
+
Payer: ${job.payer}${job.error ? `
|
|
3536
|
+
${job.error.message}` : ""}${job.next ? `
|
|
3537
|
+
Next: ${job.next}` : ""}`;
|
|
3538
|
+
}
|
|
3539
|
+
async function listCloudRenderJobs() {
|
|
3540
|
+
const result = await call(
|
|
3541
|
+
authOrigin(),
|
|
3542
|
+
"/api/cloud-renders"
|
|
3543
|
+
);
|
|
3544
|
+
return { status: "ok", jobs: result.jobs };
|
|
3545
|
+
}
|
|
3546
|
+
|
|
3547
|
+
// src/commands.ts
|
|
3548
|
+
import { existsSync as existsSync5, readFileSync as readFileSync2 } from "node:fs";
|
|
3549
|
+
import { createRequire } from "node:module";
|
|
3550
|
+
import path13 from "node:path";
|
|
3551
|
+
import { fileURLToPath as fileURLToPath2, pathToFileURL } from "node:url";
|
|
3552
|
+
import { parseArgs } from "node:util";
|
|
3553
|
+
|
|
3496
3554
|
// src/help.ts
|
|
3497
3555
|
function cliHelp(version2, commands) {
|
|
3498
3556
|
const groups = /* @__PURE__ */ new Map();
|
|
@@ -3546,140 +3604,2731 @@ Node 22 or later. Cloud commands need osmo login; OSMO_BASE_URL picks the cloud
|
|
|
3546
3604
|
`;
|
|
3547
3605
|
}
|
|
3548
3606
|
|
|
3549
|
-
// src/
|
|
3550
|
-
import {
|
|
3551
|
-
import {
|
|
3607
|
+
// src/media-command.ts
|
|
3608
|
+
import { spawn as spawn2 } from "node:child_process";
|
|
3609
|
+
import { createReadStream as createReadStream3, createWriteStream as createWriteStream2, existsSync as existsSync3 } from "node:fs";
|
|
3610
|
+
import { mkdir as mkdir6, readFile as readFile5, rename as rename6, rm as rm6, stat as stat5, writeFile as writeFile5 } from "node:fs/promises";
|
|
3611
|
+
import path9 from "node:path";
|
|
3612
|
+
import { Readable as Readable2 } from "node:stream";
|
|
3613
|
+
import { pipeline as pipeline2 } from "node:stream/promises";
|
|
3614
|
+
|
|
3615
|
+
// src/cloud-pointer.ts
|
|
3616
|
+
import { spawnSync as spawnSync2 } from "node:child_process";
|
|
3617
|
+
|
|
3618
|
+
// src/git.ts
|
|
3619
|
+
import { execFileSync, spawnSync } from "node:child_process";
|
|
3552
3620
|
import {
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
rename as rename3,
|
|
3561
|
-
rm as rm3,
|
|
3562
|
-
writeFile as writeFile3
|
|
3563
|
-
} from "node:fs/promises";
|
|
3621
|
+
existsSync,
|
|
3622
|
+
mkdirSync,
|
|
3623
|
+
readFileSync,
|
|
3624
|
+
renameSync,
|
|
3625
|
+
rmSync,
|
|
3626
|
+
writeFileSync
|
|
3627
|
+
} from "node:fs";
|
|
3564
3628
|
import path4 from "node:path";
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
var
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
var
|
|
3584
|
-
|
|
3629
|
+
var git = (root, ...args) => execFileSync("git", ["-C", root, ...args], {
|
|
3630
|
+
encoding: "utf8",
|
|
3631
|
+
maxBuffer: 1 << 28
|
|
3632
|
+
}).replace(/\n$/, "");
|
|
3633
|
+
var hasRepo = (root) => existsSync(path4.join(root, ".git"));
|
|
3634
|
+
|
|
3635
|
+
// src/cloud-pointer.ts
|
|
3636
|
+
var PROJECT_ID = /^[a-z0-9]{10,40}$/;
|
|
3637
|
+
function pointerFromGit(root) {
|
|
3638
|
+
const read = (key) => {
|
|
3639
|
+
const r = spawnSync2("git", ["-C", root, "config", "--get", key], {
|
|
3640
|
+
encoding: "utf8"
|
|
3641
|
+
});
|
|
3642
|
+
return r.status === 0 ? r.stdout.trim() : "";
|
|
3643
|
+
};
|
|
3644
|
+
const origin = read("osmo.cloud.origin"), id = read("osmo.cloud.id");
|
|
3645
|
+
return origin && id ? { origin, id } : null;
|
|
3646
|
+
}
|
|
3647
|
+
var cloudPointer = (root) => hasRepo(root) ? pointerFromGit(root) : null;
|
|
3648
|
+
|
|
3649
|
+
// src/media-api.ts
|
|
3650
|
+
var MediaApiError = class extends OsmoError {
|
|
3651
|
+
status;
|
|
3652
|
+
body;
|
|
3653
|
+
constructor(status, body2) {
|
|
3654
|
+
super(
|
|
3655
|
+
typeof body2.error === "string" ? body2.error : `http_${status}`,
|
|
3656
|
+
describe(status, body2)
|
|
3657
|
+
);
|
|
3658
|
+
this.status = status;
|
|
3659
|
+
this.body = body2;
|
|
3660
|
+
}
|
|
3585
3661
|
};
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
|
|
3662
|
+
function describe(status, body2) {
|
|
3663
|
+
switch (body2.error) {
|
|
3664
|
+
case "media_upload_not_enabled":
|
|
3665
|
+
return "Footage uploads aren't enabled for this account.";
|
|
3666
|
+
case "quota_exceeded":
|
|
3667
|
+
return `This would go over the account's footage storage (${gb(body2.usedBytes)} of ${gb(body2.capBytes)} used).`;
|
|
3668
|
+
case "not_found":
|
|
3669
|
+
return "Not found among your footage.";
|
|
3670
|
+
case "unsupported_file_type":
|
|
3671
|
+
return "That file type isn't footage this accepts.";
|
|
3672
|
+
default:
|
|
3673
|
+
return `The footage service answered ${status}${typeof body2.error === "string" ? ` (${body2.error})` : ""}.`;
|
|
3674
|
+
}
|
|
3675
|
+
}
|
|
3676
|
+
var gb = (n) => typeof n === "number" ? `${(n / 1e9).toFixed(1)} GB` : "?";
|
|
3677
|
+
var RETRY_DELAYS = [1, 2, 4, 8, 15, 30, 30, 30].map((s) => s * 1e3);
|
|
3678
|
+
var sleep2 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
3679
|
+
var COMPLETING_WAIT_MS = 15e3;
|
|
3680
|
+
async function media(route, {
|
|
3681
|
+
method = "GET",
|
|
3682
|
+
body: body2,
|
|
3683
|
+
timeoutMs = 6e4,
|
|
3684
|
+
retry = true
|
|
3685
|
+
} = {}, origin = authOrigin()) {
|
|
3686
|
+
for (let attempt = 0; ; attempt++) {
|
|
3687
|
+
let response;
|
|
3596
3688
|
try {
|
|
3597
|
-
|
|
3598
|
-
|
|
3689
|
+
response = await authenticatedFetch(
|
|
3690
|
+
`/api/media${route}`,
|
|
3691
|
+
{ method, body: body2, timeoutMs },
|
|
3692
|
+
origin
|
|
3693
|
+
);
|
|
3694
|
+
} catch (error) {
|
|
3695
|
+
if (retry && error.code === "network_error" && attempt < RETRY_DELAYS.length) {
|
|
3696
|
+
await sleep2(RETRY_DELAYS[attempt]);
|
|
3697
|
+
continue;
|
|
3698
|
+
}
|
|
3699
|
+
throw error;
|
|
3599
3700
|
}
|
|
3600
|
-
|
|
3701
|
+
if (response.status === 204) return void 0;
|
|
3702
|
+
const data = await response.json().catch(() => ({}));
|
|
3703
|
+
if (retry && (response.status >= 500 || response.status === 429 && data.error !== "too_many_open_uploads") && attempt < RETRY_DELAYS.length) {
|
|
3704
|
+
await sleep2(RETRY_DELAYS[attempt]);
|
|
3705
|
+
continue;
|
|
3706
|
+
}
|
|
3707
|
+
if (!response.ok) throw new MediaApiError(response.status, data);
|
|
3708
|
+
return data;
|
|
3601
3709
|
}
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
|
|
3608
|
-
"
|
|
3609
|
-
|
|
3610
|
-
|
|
3710
|
+
}
|
|
3711
|
+
var mediaApi = {
|
|
3712
|
+
/** Which account this login is: the scope its uploads are recorded in. */
|
|
3713
|
+
account: async (origin = authOrigin()) => {
|
|
3714
|
+
const account2 = await currentAccount(origin);
|
|
3715
|
+
if (!account2.user?.id)
|
|
3716
|
+
throw new OsmoError("no_account", "Couldn't tell which account this is.");
|
|
3717
|
+
return { origin, userId: account2.user.id };
|
|
3718
|
+
},
|
|
3719
|
+
createUpload: (input) => media("/uploads", {
|
|
3720
|
+
method: "POST",
|
|
3721
|
+
body: input,
|
|
3722
|
+
retry: false
|
|
3723
|
+
}),
|
|
3724
|
+
session: (id) => media(`/uploads/${encodeURIComponent(id)}`),
|
|
3725
|
+
signParts: (id, parts) => media(`/uploads/${encodeURIComponent(id)}/parts`, {
|
|
3726
|
+
method: "POST",
|
|
3727
|
+
body: { parts }
|
|
3728
|
+
}),
|
|
3729
|
+
/** Complete a session. While an earlier complete of it is still at work
|
|
3730
|
+
* (a retry after a timeout), the server says `completing`: wait for it. */
|
|
3731
|
+
complete: async (id, crc64nvme) => {
|
|
3732
|
+
for (let waited = 0; ; waited += COMPLETING_WAIT_MS) {
|
|
3733
|
+
try {
|
|
3734
|
+
return await media(
|
|
3735
|
+
`/uploads/${encodeURIComponent(id)}/complete`,
|
|
3736
|
+
{ method: "POST", body: { crc64nvme }, timeoutMs: 5 * 6e4 }
|
|
3737
|
+
);
|
|
3738
|
+
} catch (error) {
|
|
3739
|
+
if (!(error instanceof MediaApiError && error.code === "completing") || waited >= 12 * 6e4)
|
|
3740
|
+
throw error;
|
|
3741
|
+
await sleep2(COMPLETING_WAIT_MS);
|
|
3742
|
+
}
|
|
3743
|
+
}
|
|
3744
|
+
},
|
|
3745
|
+
abort: (id) => media(`/uploads/${encodeURIComponent(id)}`, { method: "DELETE" }),
|
|
3746
|
+
file: (id) => media(`/${encodeURIComponent(id)}`),
|
|
3747
|
+
/** What pointer ids play: each followed through its clip to its proxy. */
|
|
3748
|
+
resolve: async (ids, origin) => (await media(
|
|
3749
|
+
"/resolve",
|
|
3750
|
+
{ method: "POST", body: { ids } },
|
|
3751
|
+
origin
|
|
3752
|
+
)).files,
|
|
3753
|
+
list: (opts = {}) => {
|
|
3754
|
+
const q = new URLSearchParams();
|
|
3755
|
+
if (opts.role) q.set("role", opts.role);
|
|
3756
|
+
if (opts.cursor) q.set("cursor", opts.cursor);
|
|
3757
|
+
const qs = q.toString();
|
|
3758
|
+
return media(
|
|
3759
|
+
qs ? `?${qs}` : ""
|
|
3611
3760
|
);
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3761
|
+
}
|
|
3762
|
+
};
|
|
3763
|
+
|
|
3764
|
+
// src/media-crc.ts
|
|
3765
|
+
var POLY = 0x9a6c9329ac4bc9b5n;
|
|
3766
|
+
var MASK = 0xffffffffffffffffn;
|
|
3767
|
+
var TLO = new Uint32Array(8 * 256);
|
|
3768
|
+
var THI = new Uint32Array(8 * 256);
|
|
3769
|
+
{
|
|
3770
|
+
const t0 = [];
|
|
3771
|
+
for (let b = 0; b < 256; b++) {
|
|
3772
|
+
let c = BigInt(b);
|
|
3773
|
+
for (let k = 0; k < 8; k++) c = c & 1n ? c >> 1n ^ POLY : c >> 1n;
|
|
3774
|
+
t0[b] = c;
|
|
3775
|
+
}
|
|
3776
|
+
let prev = t0;
|
|
3777
|
+
for (let k = 0; k < 8; k++) {
|
|
3778
|
+
const cur = k === 0 ? t0 : prev.map((v) => v >> 8n ^ t0[Number(v & 0xffn)]);
|
|
3779
|
+
for (let b = 0; b < 256; b++) {
|
|
3780
|
+
TLO[k * 256 + b] = Number(cur[b] & 0xffffffffn);
|
|
3781
|
+
THI[k * 256 + b] = Number(cur[b] >> 32n);
|
|
3782
|
+
}
|
|
3783
|
+
prev = cur;
|
|
3784
|
+
}
|
|
3615
3785
|
}
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3786
|
+
var Crc64Nvme = class {
|
|
3787
|
+
lo = 4294967295;
|
|
3788
|
+
hi = 4294967295;
|
|
3789
|
+
update(u8) {
|
|
3790
|
+
let lo = this.lo;
|
|
3791
|
+
let hi = this.hi;
|
|
3792
|
+
let i = 0;
|
|
3793
|
+
const n = u8.length;
|
|
3794
|
+
while (i < n && (u8.byteOffset + i) % 4 !== 0) {
|
|
3795
|
+
const idx = (lo ^ u8[i++]) & 255;
|
|
3796
|
+
const nlo = TLO[idx] ^ (lo >>> 8 | hi << 24);
|
|
3797
|
+
hi = THI[idx] ^ hi >>> 8;
|
|
3798
|
+
lo = nlo;
|
|
3799
|
+
}
|
|
3800
|
+
const words2 = n - i >>> 3;
|
|
3801
|
+
if (words2 > 0) {
|
|
3802
|
+
const w = new Uint32Array(u8.buffer, u8.byteOffset + i, words2 * 2);
|
|
3803
|
+
for (let j = 0; j < w.length; j += 2) {
|
|
3804
|
+
const a = lo ^ w[j];
|
|
3805
|
+
const b = hi ^ w[j + 1];
|
|
3806
|
+
lo = TLO[1792 + (a & 255)] ^ TLO[1536 + (a >>> 8 & 255)] ^ TLO[1280 + (a >>> 16 & 255)] ^ TLO[1024 + (a >>> 24)] ^ TLO[768 + (b & 255)] ^ TLO[512 + (b >>> 8 & 255)] ^ TLO[256 + (b >>> 16 & 255)] ^ TLO[b >>> 24];
|
|
3807
|
+
hi = THI[1792 + (a & 255)] ^ THI[1536 + (a >>> 8 & 255)] ^ THI[1280 + (a >>> 16 & 255)] ^ THI[1024 + (a >>> 24)] ^ THI[768 + (b & 255)] ^ THI[512 + (b >>> 8 & 255)] ^ THI[256 + (b >>> 16 & 255)] ^ THI[b >>> 24];
|
|
3808
|
+
}
|
|
3809
|
+
i += words2 * 8;
|
|
3810
|
+
}
|
|
3811
|
+
while (i < n) {
|
|
3812
|
+
const idx = (lo ^ u8[i++]) & 255;
|
|
3813
|
+
const nlo = TLO[idx] ^ (lo >>> 8 | hi << 24);
|
|
3814
|
+
hi = THI[idx] ^ hi >>> 8;
|
|
3815
|
+
lo = nlo;
|
|
3816
|
+
}
|
|
3817
|
+
this.lo = lo >>> 0;
|
|
3818
|
+
this.hi = hi >>> 0;
|
|
3819
|
+
return this;
|
|
3620
3820
|
}
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3821
|
+
digest() {
|
|
3822
|
+
return (BigInt(this.hi ^ 4294967295) << 32n | BigInt((this.lo ^ 4294967295) >>> 0)) & MASK;
|
|
3823
|
+
}
|
|
3824
|
+
};
|
|
3825
|
+
var crc64 = (u8) => new Crc64Nvme().update(u8).digest();
|
|
3826
|
+
function multmodp(a, b) {
|
|
3827
|
+
let m = 1n << 63n;
|
|
3828
|
+
let p = 0n;
|
|
3829
|
+
for (; ; ) {
|
|
3830
|
+
if (a & m) {
|
|
3831
|
+
p ^= b;
|
|
3832
|
+
if ((a & m - 1n) === 0n) break;
|
|
3628
3833
|
}
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
fail(
|
|
3632
|
-
"network_error",
|
|
3633
|
-
"Lost the connection to Osmo partway through. Check your connection and retry."
|
|
3634
|
-
);
|
|
3834
|
+
m >>= 1n;
|
|
3835
|
+
b = b & 1n ? b >> 1n ^ POLY : b >> 1n;
|
|
3635
3836
|
}
|
|
3636
|
-
return
|
|
3837
|
+
return p;
|
|
3637
3838
|
}
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
);
|
|
3648
|
-
if (body3.error === "unknown_type" && strings(body3.types))
|
|
3649
|
-
throw new UsageError(
|
|
3650
|
-
`Unknown --type. Types: ${body3.types.join(", ")}.`,
|
|
3651
|
-
"osmo resources list",
|
|
3652
|
-
"unknown_type",
|
|
3653
|
-
{ types: body3.types }
|
|
3654
|
-
);
|
|
3655
|
-
fail(
|
|
3656
|
-
"resource_request_failed",
|
|
3657
|
-
`Osmo resource request failed (${response.status}). Retry later.`
|
|
3658
|
-
);
|
|
3839
|
+
var X2N = [1n << 62n];
|
|
3840
|
+
for (let k = 1; k < 64; k++) X2N[k] = multmodp(X2N[k - 1], X2N[k - 1]);
|
|
3841
|
+
function x2nmodp(n, k) {
|
|
3842
|
+
let p = 1n << 63n;
|
|
3843
|
+
let nn = BigInt(n);
|
|
3844
|
+
while (nn) {
|
|
3845
|
+
if (nn & 1n) p = multmodp(X2N[k & 63], p);
|
|
3846
|
+
nn >>= 1n;
|
|
3847
|
+
k++;
|
|
3659
3848
|
}
|
|
3660
|
-
|
|
3661
|
-
|
|
3849
|
+
return p;
|
|
3850
|
+
}
|
|
3851
|
+
var opCache = /* @__PURE__ */ new Map();
|
|
3852
|
+
function crc64Combine(crcA, crcB, lenB) {
|
|
3853
|
+
let op = opCache.get(lenB);
|
|
3854
|
+
if (op === void 0) {
|
|
3855
|
+
op = x2nmodp(lenB, 3);
|
|
3856
|
+
if (opCache.size < 16) opCache.set(lenB, op);
|
|
3857
|
+
}
|
|
3858
|
+
return multmodp(op, crcA) ^ crcB;
|
|
3859
|
+
}
|
|
3860
|
+
function crcToB64(crc) {
|
|
3861
|
+
const bytes2 = Buffer.alloc(8);
|
|
3862
|
+
bytes2.writeBigUInt64BE(crc & MASK);
|
|
3863
|
+
return bytes2.toString("base64");
|
|
3864
|
+
}
|
|
3865
|
+
function crcFromB64(b64) {
|
|
3866
|
+
return Buffer.from(b64, "base64").readBigUInt64BE();
|
|
3867
|
+
}
|
|
3868
|
+
function combineParts(parts) {
|
|
3869
|
+
let full = null;
|
|
3870
|
+
for (const p of parts) {
|
|
3871
|
+
const crc = crcFromB64(p.crc);
|
|
3872
|
+
full = full === null ? crc : crc64Combine(full, crc, p.size);
|
|
3873
|
+
}
|
|
3874
|
+
return crcToB64(full ?? crc64(new Uint8Array()));
|
|
3875
|
+
}
|
|
3876
|
+
|
|
3877
|
+
// src/media-ledger.ts
|
|
3878
|
+
import { createHash as createHash3, randomUUID as randomUUID3 } from "node:crypto";
|
|
3879
|
+
import {
|
|
3880
|
+
link,
|
|
3881
|
+
mkdir as mkdir3,
|
|
3882
|
+
open,
|
|
3883
|
+
readdir,
|
|
3884
|
+
readFile as readFile3,
|
|
3885
|
+
rename as rename3,
|
|
3886
|
+
rm as rm3,
|
|
3887
|
+
writeFile as writeFile3
|
|
3888
|
+
} from "node:fs/promises";
|
|
3889
|
+
import os3 from "node:os";
|
|
3890
|
+
import path5 from "node:path";
|
|
3891
|
+
function mediaHome() {
|
|
3892
|
+
return path5.join(
|
|
3893
|
+
process.env.OSMO_CONFIG_DIR || path5.join(os3.homedir(), ".osmo"),
|
|
3894
|
+
"media"
|
|
3895
|
+
);
|
|
3896
|
+
}
|
|
3897
|
+
var filesDir = () => path5.join(mediaHome(), "files");
|
|
3898
|
+
function ledgerKey(file, scope) {
|
|
3899
|
+
return hashKey(
|
|
3900
|
+
`${scope}\0${file.name}\0${file.size}\0${Math.floor(file.mtimeMs)}`
|
|
3901
|
+
);
|
|
3902
|
+
}
|
|
3903
|
+
function contentKey(file, crc, scope) {
|
|
3904
|
+
return hashKey(
|
|
3905
|
+
`${scope}\0${file.name}\0${file.size}\0${Math.floor(file.mtimeMs)}\0${crc}`
|
|
3906
|
+
);
|
|
3907
|
+
}
|
|
3908
|
+
async function fingerprintOf(file) {
|
|
3909
|
+
const span = 64 * 1024;
|
|
3910
|
+
const handle = await open(file.path, "r");
|
|
3662
3911
|
try {
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3912
|
+
const hash = new Crc64Nvme();
|
|
3913
|
+
for (const at of /* @__PURE__ */ new Set([0, Math.max(0, file.size - span)])) {
|
|
3914
|
+
const buffer = Buffer.alloc(Math.min(span, file.size - at));
|
|
3915
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.length, at);
|
|
3916
|
+
hash.update(buffer.subarray(0, bytesRead));
|
|
3917
|
+
}
|
|
3918
|
+
return crcToB64(hash.digest());
|
|
3919
|
+
} finally {
|
|
3920
|
+
await handle.close();
|
|
3666
3921
|
}
|
|
3667
3922
|
}
|
|
3668
|
-
|
|
3669
|
-
|
|
3923
|
+
var hashKey = (text2) => createHash3("sha256").update(text2).digest("hex").slice(0, 32);
|
|
3924
|
+
async function readEntry(key) {
|
|
3925
|
+
let text2;
|
|
3670
3926
|
try {
|
|
3671
|
-
|
|
3672
|
-
redirect: "error",
|
|
3673
|
-
signal: AbortSignal.timeout(2e4)
|
|
3674
|
-
});
|
|
3927
|
+
text2 = await readFile3(path5.join(filesDir(), `${key}.json`), "utf8");
|
|
3675
3928
|
} catch (error) {
|
|
3676
|
-
if (error
|
|
3677
|
-
|
|
3678
|
-
|
|
3679
|
-
|
|
3680
|
-
);
|
|
3929
|
+
if (error.code === "ENOENT") return null;
|
|
3930
|
+
throw error;
|
|
3931
|
+
}
|
|
3932
|
+
try {
|
|
3933
|
+
const entry = JSON.parse(text2);
|
|
3934
|
+
return entry.version === 1 && entry.key === key && entry.scope ? entry : null;
|
|
3935
|
+
} catch {
|
|
3936
|
+
return null;
|
|
3681
3937
|
}
|
|
3682
|
-
|
|
3938
|
+
}
|
|
3939
|
+
async function writeEntry(entry) {
|
|
3940
|
+
await mkdir3(filesDir(), { recursive: true, mode: 448 });
|
|
3941
|
+
entry.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3942
|
+
const file = path5.join(filesDir(), `${entry.key}.json`);
|
|
3943
|
+
const temporary = `${file}.${randomUUID3()}.tmp`;
|
|
3944
|
+
await writeFile3(temporary, `${JSON.stringify(entry, null, " ")}
|
|
3945
|
+
`, {
|
|
3946
|
+
mode: 384,
|
|
3947
|
+
flag: "wx"
|
|
3948
|
+
});
|
|
3949
|
+
await rename3(temporary, file);
|
|
3950
|
+
}
|
|
3951
|
+
async function allEntries() {
|
|
3952
|
+
let names;
|
|
3953
|
+
try {
|
|
3954
|
+
names = await readdir(filesDir());
|
|
3955
|
+
} catch (error) {
|
|
3956
|
+
if (error.code === "ENOENT") return [];
|
|
3957
|
+
throw error;
|
|
3958
|
+
}
|
|
3959
|
+
const out2 = [];
|
|
3960
|
+
for (const name of names) {
|
|
3961
|
+
if (!name.endsWith(".json")) continue;
|
|
3962
|
+
const entry = await readEntry(name.slice(0, -5)).catch(() => null);
|
|
3963
|
+
if (entry) out2.push(entry);
|
|
3964
|
+
}
|
|
3965
|
+
return out2;
|
|
3966
|
+
}
|
|
3967
|
+
function bootedAt() {
|
|
3968
|
+
return Math.round((Date.now() / 1e3 - os3.uptime()) / 30);
|
|
3969
|
+
}
|
|
3970
|
+
var alive2 = (pid) => {
|
|
3971
|
+
try {
|
|
3972
|
+
process.kill(pid, 0);
|
|
3973
|
+
return true;
|
|
3974
|
+
} catch (error) {
|
|
3975
|
+
return error.code === "EPERM";
|
|
3976
|
+
}
|
|
3977
|
+
};
|
|
3978
|
+
var LedgerLockedError = class extends Error {
|
|
3979
|
+
file;
|
|
3980
|
+
pid;
|
|
3981
|
+
constructor(file, pid) {
|
|
3982
|
+
super(`Another osmo process (pid ${pid}) is uploading ${file}.`);
|
|
3983
|
+
this.file = file;
|
|
3984
|
+
this.pid = pid;
|
|
3985
|
+
}
|
|
3986
|
+
};
|
|
3987
|
+
async function withEntryLock(key, file, action) {
|
|
3988
|
+
await mkdir3(filesDir(), { recursive: true, mode: 448 });
|
|
3989
|
+
const lock = path5.join(filesDir(), `${key}.lock`);
|
|
3990
|
+
const nonce = randomUUID3();
|
|
3991
|
+
const mine = `${lock}.${process.pid}.${nonce}`;
|
|
3992
|
+
const content = JSON.stringify({ pid: process.pid, boot: bootedAt(), nonce });
|
|
3993
|
+
await writeFile3(mine, content, { mode: 384 });
|
|
3994
|
+
const read = (file2) => readFile3(file2, "utf8").catch(() => null);
|
|
3995
|
+
try {
|
|
3996
|
+
for (let attempt = 0; ; attempt++) {
|
|
3997
|
+
try {
|
|
3998
|
+
await link(mine, lock);
|
|
3999
|
+
break;
|
|
4000
|
+
} catch (error) {
|
|
4001
|
+
if (error.code !== "EEXIST" || attempt > 2)
|
|
4002
|
+
throw error;
|
|
4003
|
+
const text2 = await read(lock);
|
|
4004
|
+
const holder = (() => {
|
|
4005
|
+
try {
|
|
4006
|
+
return JSON.parse(text2 ?? "");
|
|
4007
|
+
} catch {
|
|
4008
|
+
return null;
|
|
4009
|
+
}
|
|
4010
|
+
})();
|
|
4011
|
+
if (holder?.pid && Math.abs((holder.boot ?? 0) - bootedAt()) <= 1 && alive2(holder.pid))
|
|
4012
|
+
throw new LedgerLockedError(file, holder.pid);
|
|
4013
|
+
const aside = `${lock}.${nonce}.stale`;
|
|
4014
|
+
try {
|
|
4015
|
+
await rename3(lock, aside);
|
|
4016
|
+
} catch (e) {
|
|
4017
|
+
if (e.code === "ENOENT") continue;
|
|
4018
|
+
throw e;
|
|
4019
|
+
}
|
|
4020
|
+
const moved = await read(aside);
|
|
4021
|
+
if (moved !== text2) {
|
|
4022
|
+
await link(aside, lock).catch(() => {
|
|
4023
|
+
});
|
|
4024
|
+
await rm3(aside, { force: true });
|
|
4025
|
+
const taker = JSON.parse(moved ?? "{}");
|
|
4026
|
+
throw new LedgerLockedError(file, taker.pid ?? 0);
|
|
4027
|
+
}
|
|
4028
|
+
await rm3(aside, { force: true });
|
|
4029
|
+
}
|
|
4030
|
+
}
|
|
4031
|
+
} finally {
|
|
4032
|
+
await rm3(mine, { force: true });
|
|
4033
|
+
}
|
|
4034
|
+
try {
|
|
4035
|
+
return await action();
|
|
4036
|
+
} finally {
|
|
4037
|
+
if (await read(lock) === content) await rm3(lock, { force: true });
|
|
4038
|
+
}
|
|
4039
|
+
}
|
|
4040
|
+
|
|
4041
|
+
// src/media-project.ts
|
|
4042
|
+
import { createReadStream, createWriteStream, existsSync as existsSync2 } from "node:fs";
|
|
4043
|
+
import {
|
|
4044
|
+
copyFile,
|
|
4045
|
+
lstat as lstat2,
|
|
4046
|
+
mkdir as mkdir4,
|
|
4047
|
+
readdir as readdir2,
|
|
4048
|
+
readFile as readFile4,
|
|
4049
|
+
realpath,
|
|
4050
|
+
rename as rename4,
|
|
4051
|
+
rm as rm4,
|
|
4052
|
+
stat as stat2,
|
|
4053
|
+
writeFile as writeFile4
|
|
4054
|
+
} from "node:fs/promises";
|
|
4055
|
+
import path6 from "node:path";
|
|
4056
|
+
import { Readable } from "node:stream";
|
|
4057
|
+
import { pipeline } from "node:stream/promises";
|
|
4058
|
+
var MEDIA_DIR = "osmo-media";
|
|
4059
|
+
var MEDIA_ID = /^[a-z0-9]{20,40}$/;
|
|
4060
|
+
var POINTER_SUFFIX = ".osmo-media";
|
|
4061
|
+
var BLOCK_HEADER = "# Footage (osmo media): proxies stay out of git";
|
|
4062
|
+
var BLOCK = [
|
|
4063
|
+
BLOCK_HEADER,
|
|
4064
|
+
`/${MEDIA_DIR}/**`,
|
|
4065
|
+
`!/${MEDIA_DIR}/**/`,
|
|
4066
|
+
`!/${MEDIA_DIR}/**/*${POINTER_SUFFIX}`
|
|
4067
|
+
];
|
|
4068
|
+
var excludeFile = (root) => path6.resolve(root, git(root, "rev-parse", "--git-path", "info/exclude"));
|
|
4069
|
+
async function managed(root) {
|
|
4070
|
+
const text2 = await readFile4(excludeFile(root), "utf8").catch(() => "");
|
|
4071
|
+
return text2.split("\n").includes(BLOCK_HEADER);
|
|
4072
|
+
}
|
|
4073
|
+
async function adopt(root) {
|
|
4074
|
+
if (await managed(root)) return;
|
|
4075
|
+
const tracked = git(root, "ls-files", "-z", "--", MEDIA_DIR).split("\0").filter((f) => f && !f.endsWith(POINTER_SUFFIX));
|
|
4076
|
+
if (tracked.length)
|
|
4077
|
+
throw new Error(
|
|
4078
|
+
`${MEDIA_DIR}/ already holds files in this project's history (${tracked[0]}). Move them elsewhere first; ${MEDIA_DIR}/ is where osmo media keeps footage out of git.`
|
|
4079
|
+
);
|
|
4080
|
+
const file = excludeFile(root);
|
|
4081
|
+
const previous = await readFile4(file, "utf8").catch(() => "");
|
|
4082
|
+
await mkdir4(path6.dirname(file), { recursive: true });
|
|
4083
|
+
await writeFile4(
|
|
4084
|
+
file,
|
|
4085
|
+
`${previous}${previous && !previous.endsWith("\n") ? "\n" : ""}${BLOCK.join("\n")}
|
|
4086
|
+
`
|
|
4087
|
+
);
|
|
4088
|
+
}
|
|
4089
|
+
async function realFolder(dir) {
|
|
4090
|
+
return realpath(dir).catch(async () => {
|
|
4091
|
+
const parent = path6.dirname(dir);
|
|
4092
|
+
return parent === dir ? dir : path6.join(await realFolder(parent), path6.basename(dir));
|
|
4093
|
+
});
|
|
4094
|
+
}
|
|
4095
|
+
async function excludeSources(root, sources) {
|
|
4096
|
+
const top = await realFolder(path6.resolve(root));
|
|
4097
|
+
const folders = /* @__PURE__ */ new Map();
|
|
4098
|
+
const inside = [];
|
|
4099
|
+
for (const source of sources) {
|
|
4100
|
+
const absolute = path6.resolve(source);
|
|
4101
|
+
const dir = path6.dirname(absolute);
|
|
4102
|
+
let folder = folders.get(dir);
|
|
4103
|
+
if (!folder) {
|
|
4104
|
+
folder = realFolder(dir);
|
|
4105
|
+
folders.set(dir, folder);
|
|
4106
|
+
}
|
|
4107
|
+
const rel = path6.relative(top, path6.join(await folder, path6.basename(absolute))).split(path6.sep).join("/");
|
|
4108
|
+
if (rel && !rel.startsWith("../") && !path6.isAbsolute(rel))
|
|
4109
|
+
inside.push(rel);
|
|
4110
|
+
}
|
|
4111
|
+
if (!inside.length) return [];
|
|
4112
|
+
const file = excludeFile(root);
|
|
4113
|
+
const previous = await readFile4(file, "utf8").catch(() => "");
|
|
4114
|
+
const lines = previous.split("\n");
|
|
4115
|
+
const escape = (s) => s.replace(/[\\!*?[\]# ]/g, "\\$&");
|
|
4116
|
+
const additions = inside.map((rel) => `/${escape(rel)}`).filter((p) => !lines.includes(p));
|
|
4117
|
+
if (additions.length)
|
|
4118
|
+
await writeFile4(
|
|
4119
|
+
file,
|
|
4120
|
+
`${previous}${previous && !previous.endsWith("\n") ? "\n" : ""}# Footage sources (osmo media)
|
|
4121
|
+
${additions.join("\n")}
|
|
4122
|
+
`
|
|
4123
|
+
);
|
|
4124
|
+
return inside;
|
|
4125
|
+
}
|
|
4126
|
+
function placeFor(root, sourcePath) {
|
|
4127
|
+
const safe = sourcePath.split(/[\\/]+/).filter((part) => part && part !== "." && part !== "..").join("/");
|
|
4128
|
+
const parsed = path6.posix.parse(safe || "clip");
|
|
4129
|
+
const dir = path6.posix.join(MEDIA_DIR, parsed.dir);
|
|
4130
|
+
for (let n = 1; ; n++) {
|
|
4131
|
+
const name = n === 1 ? `${parsed.name}.mp4` : `${parsed.name} (${n}).mp4`;
|
|
4132
|
+
const rel = path6.posix.join(dir, name);
|
|
4133
|
+
if (!existsSync2(path6.join(root, rel)) && !existsSync2(path6.join(root, `${rel}${POINTER_SUFFIX}`)))
|
|
4134
|
+
return rel;
|
|
4135
|
+
}
|
|
4136
|
+
}
|
|
4137
|
+
async function placeClip(root, rel, fileId, proxyFile) {
|
|
4138
|
+
const target = path6.join(root, rel);
|
|
4139
|
+
await mkdir4(path6.dirname(target), { recursive: true });
|
|
4140
|
+
await writeFile4(
|
|
4141
|
+
`${target}${POINTER_SUFFIX}`,
|
|
4142
|
+
`${JSON.stringify({ media: fileId })}
|
|
4143
|
+
`
|
|
4144
|
+
);
|
|
4145
|
+
if (proxyFile && existsSync2(proxyFile) && !existsSync2(target))
|
|
4146
|
+
await copyFile(proxyFile, target);
|
|
4147
|
+
return rel;
|
|
4148
|
+
}
|
|
4149
|
+
async function pointersIn(root) {
|
|
4150
|
+
const out2 = [];
|
|
4151
|
+
const visit = async (dir) => {
|
|
4152
|
+
let names;
|
|
4153
|
+
try {
|
|
4154
|
+
names = await readdir2(path6.join(root, dir));
|
|
4155
|
+
} catch {
|
|
4156
|
+
return;
|
|
4157
|
+
}
|
|
4158
|
+
for (const name of names) {
|
|
4159
|
+
const rel = path6.posix.join(dir, name);
|
|
4160
|
+
const info = await lstat2(path6.join(root, rel)).catch(() => null);
|
|
4161
|
+
if (info?.isDirectory()) await visit(rel);
|
|
4162
|
+
else if (info?.isFile() && info.size <= 1024 && name.endsWith(POINTER_SUFFIX) && !name.slice(0, -POINTER_SUFFIX.length).endsWith(POINTER_SUFFIX)) {
|
|
4163
|
+
try {
|
|
4164
|
+
const { media: media2 } = JSON.parse(
|
|
4165
|
+
await readFile4(path6.join(root, rel), "utf8")
|
|
4166
|
+
);
|
|
4167
|
+
if (typeof media2 === "string" && MEDIA_ID.test(media2))
|
|
4168
|
+
out2.push({ path: rel.slice(0, -POINTER_SUFFIX.length), id: media2 });
|
|
4169
|
+
} catch {
|
|
4170
|
+
}
|
|
4171
|
+
}
|
|
4172
|
+
}
|
|
4173
|
+
};
|
|
4174
|
+
await visit(MEDIA_DIR);
|
|
4175
|
+
return out2.sort((a, b) => a.path.localeCompare(b.path));
|
|
4176
|
+
}
|
|
4177
|
+
async function crcOfFile(file) {
|
|
4178
|
+
const crc = new Crc64Nvme();
|
|
4179
|
+
for await (const chunk of createReadStream(file, { highWaterMark: 8 << 20 }))
|
|
4180
|
+
crc.update(chunk);
|
|
4181
|
+
return crcToB64(crc.digest());
|
|
4182
|
+
}
|
|
4183
|
+
var pulledFile = () => path6.join(mediaHome(), "pulled.json");
|
|
4184
|
+
async function readPulled() {
|
|
4185
|
+
try {
|
|
4186
|
+
return JSON.parse(await readFile4(pulledFile(), "utf8"));
|
|
4187
|
+
} catch {
|
|
4188
|
+
return {};
|
|
4189
|
+
}
|
|
4190
|
+
}
|
|
4191
|
+
async function recordPulled(entries) {
|
|
4192
|
+
if (!entries.length) return;
|
|
4193
|
+
const all = await readPulled();
|
|
4194
|
+
for (const e of entries) all[`${e.size}:${e.crc64nvme}`] = e;
|
|
4195
|
+
await mkdir4(mediaHome(), { recursive: true, mode: 448 });
|
|
4196
|
+
const temporary = `${pulledFile()}.${process.pid}.tmp`;
|
|
4197
|
+
await writeFile4(temporary, JSON.stringify(all), { mode: 384 });
|
|
4198
|
+
await rename4(temporary, pulledFile());
|
|
4199
|
+
}
|
|
4200
|
+
function pullLines(r) {
|
|
4201
|
+
return [
|
|
4202
|
+
`Footage: ${r.pulled.length} fetched, ${r.current.length} already here.`,
|
|
4203
|
+
...r.pending.map((p) => ` ${printable(p)}: its clip has no proxy`),
|
|
4204
|
+
...r.unavailable.map(
|
|
4205
|
+
(p) => ` ${printable(p)}: its clip is missing or was deleted`
|
|
4206
|
+
),
|
|
4207
|
+
...r.failed.map(
|
|
4208
|
+
(f) => ` ${printable(f.path)}: ${f.message}; run osmo media pull to retry`
|
|
4209
|
+
)
|
|
4210
|
+
];
|
|
4211
|
+
}
|
|
4212
|
+
async function pull(root, project, log = () => {
|
|
4213
|
+
}) {
|
|
4214
|
+
const pointers = await pointersIn(root);
|
|
4215
|
+
const result = {
|
|
4216
|
+
pulled: [],
|
|
4217
|
+
current: [],
|
|
4218
|
+
pending: [],
|
|
4219
|
+
unavailable: [],
|
|
4220
|
+
failed: []
|
|
4221
|
+
};
|
|
4222
|
+
if (!pointers.length) return result;
|
|
4223
|
+
await adopt(root);
|
|
4224
|
+
const ids = [...new Set(pointers.map((p) => p.id))];
|
|
4225
|
+
const resolved = /* @__PURE__ */ new Map();
|
|
4226
|
+
for (let i = 0; i < ids.length; i += 100)
|
|
4227
|
+
for (const r of await mediaApi.resolve(
|
|
4228
|
+
ids.slice(i, i + 100),
|
|
4229
|
+
project.origin
|
|
4230
|
+
))
|
|
4231
|
+
resolved.set(r.id, r);
|
|
4232
|
+
const recorded = [];
|
|
4233
|
+
for (const p of pointers) {
|
|
4234
|
+
const served = resolved.get(p.id);
|
|
4235
|
+
if (served?.status !== "ready") {
|
|
4236
|
+
(served?.status === "pending" ? result.pending : result.unavailable).push(
|
|
4237
|
+
p.path
|
|
4238
|
+
);
|
|
4239
|
+
continue;
|
|
4240
|
+
}
|
|
4241
|
+
const target = path6.join(root, p.path);
|
|
4242
|
+
const here2 = await stat2(target).catch(() => null);
|
|
4243
|
+
if (here2 && Number(here2.size) === served.size && await knownCrc(target, here2) === served.crc64nvme) {
|
|
4244
|
+
result.current.push(p.path);
|
|
4245
|
+
recorded.push({
|
|
4246
|
+
fileId: served.servedId,
|
|
4247
|
+
size: served.size,
|
|
4248
|
+
crc64nvme: served.crc64nvme,
|
|
4249
|
+
path: p.path
|
|
4250
|
+
});
|
|
4251
|
+
continue;
|
|
4252
|
+
}
|
|
4253
|
+
log(`Fetching ${p.path}`);
|
|
4254
|
+
await mkdir4(path6.dirname(target), { recursive: true });
|
|
4255
|
+
const partial = `${target}.${process.pid}.partial`;
|
|
4256
|
+
try {
|
|
4257
|
+
const res = await fetch(served.url);
|
|
4258
|
+
if (!res.ok || !res.body)
|
|
4259
|
+
throw new Error(`download failed (${res.status})`);
|
|
4260
|
+
const crc = new Crc64Nvme();
|
|
4261
|
+
const body2 = Readable.fromWeb(
|
|
4262
|
+
res.body
|
|
4263
|
+
);
|
|
4264
|
+
body2.on("data", (chunk) => crc.update(chunk));
|
|
4265
|
+
await pipeline(body2, createWriteStream(partial));
|
|
4266
|
+
const got = crcToB64(crc.digest());
|
|
4267
|
+
if (got !== served.crc64nvme)
|
|
4268
|
+
throw new Error("the download didn't match its checksum");
|
|
4269
|
+
await rename4(partial, target);
|
|
4270
|
+
const after = await stat2(target);
|
|
4271
|
+
rememberCrc(target, after, got);
|
|
4272
|
+
result.pulled.push(p.path);
|
|
4273
|
+
recorded.push({
|
|
4274
|
+
fileId: served.servedId,
|
|
4275
|
+
size: Number(after.size),
|
|
4276
|
+
crc64nvme: got,
|
|
4277
|
+
path: p.path
|
|
4278
|
+
});
|
|
4279
|
+
} catch (error) {
|
|
4280
|
+
result.failed.push({ path: p.path, message: error.message });
|
|
4281
|
+
} finally {
|
|
4282
|
+
await rm4(partial, { force: true });
|
|
4283
|
+
}
|
|
4284
|
+
}
|
|
4285
|
+
await recordPulled(recorded);
|
|
4286
|
+
await saveCrcCache();
|
|
4287
|
+
return result;
|
|
4288
|
+
}
|
|
4289
|
+
var crcCache = null;
|
|
4290
|
+
var crcCacheFile = () => path6.join(mediaHome(), "local-crcs.json");
|
|
4291
|
+
async function knownCrc(file, info) {
|
|
4292
|
+
crcCache ??= await readFile4(crcCacheFile(), "utf8").then((t) => JSON.parse(t)).catch(() => ({}));
|
|
4293
|
+
const hit = crcCache[file];
|
|
4294
|
+
if (hit && hit.size === Number(info.size) && hit.mtimeMs === Number(info.mtimeMs) && hit.ino === Number(info.ino))
|
|
4295
|
+
return hit.crc;
|
|
4296
|
+
const crc = await crcOfFile(file);
|
|
4297
|
+
rememberCrc(file, info, crc);
|
|
4298
|
+
return crc;
|
|
4299
|
+
}
|
|
4300
|
+
function rememberCrc(file, info, crc) {
|
|
4301
|
+
crcCache ??= {};
|
|
4302
|
+
crcCache[file] = {
|
|
4303
|
+
size: Number(info.size),
|
|
4304
|
+
mtimeMs: Number(info.mtimeMs),
|
|
4305
|
+
ino: Number(info.ino),
|
|
4306
|
+
crc
|
|
4307
|
+
};
|
|
4308
|
+
}
|
|
4309
|
+
async function saveCrcCache() {
|
|
4310
|
+
if (!crcCache) return;
|
|
4311
|
+
await mkdir4(mediaHome(), { recursive: true, mode: 448 });
|
|
4312
|
+
const temporary = `${crcCacheFile()}.${process.pid}.tmp`;
|
|
4313
|
+
await writeFile4(temporary, JSON.stringify(crcCache), { mode: 384 });
|
|
4314
|
+
await rename4(temporary, crcCacheFile());
|
|
4315
|
+
}
|
|
4316
|
+
|
|
4317
|
+
// src/media-proxy.ts
|
|
4318
|
+
import { spawn, spawnSync as spawnSync3 } from "node:child_process";
|
|
4319
|
+
import { mkdir as mkdir5, rename as rename5, rm as rm5, stat as stat4, statfs } from "node:fs/promises";
|
|
4320
|
+
import os4 from "node:os";
|
|
4321
|
+
import path8 from "node:path";
|
|
4322
|
+
|
|
4323
|
+
// src/media-walk.ts
|
|
4324
|
+
import { lstat as lstat3, readdir as readdir3, stat as stat3 } from "node:fs/promises";
|
|
4325
|
+
import path7 from "node:path";
|
|
4326
|
+
var CLIP_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
4327
|
+
".mov",
|
|
4328
|
+
".mp4",
|
|
4329
|
+
".m4v",
|
|
4330
|
+
".mxf",
|
|
4331
|
+
".mkv",
|
|
4332
|
+
// RAW containers: uploaded as originals, with no proxy (nothing here decodes them).
|
|
4333
|
+
".braw",
|
|
4334
|
+
".r3d",
|
|
4335
|
+
".crm"
|
|
4336
|
+
]);
|
|
4337
|
+
var RAW_EXTENSIONS = /* @__PURE__ */ new Set([".braw", ".r3d", ".crm"]);
|
|
4338
|
+
var hidden = (name) => name.startsWith(".");
|
|
4339
|
+
async function walk(paths) {
|
|
4340
|
+
const result = { clips: [], skipped: [], unreadable: [] };
|
|
4341
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4342
|
+
const folders = /* @__PURE__ */ new Set();
|
|
4343
|
+
const visit = async (file, base) => {
|
|
4344
|
+
let info;
|
|
4345
|
+
try {
|
|
4346
|
+
info = await lstat3(file);
|
|
4347
|
+
if (info.isSymbolicLink()) info = await stat3(file);
|
|
4348
|
+
} catch (error) {
|
|
4349
|
+
result.unreadable.push({
|
|
4350
|
+
path: file,
|
|
4351
|
+
code: error.code ?? "unknown"
|
|
4352
|
+
});
|
|
4353
|
+
return;
|
|
4354
|
+
}
|
|
4355
|
+
if (info.isDirectory()) {
|
|
4356
|
+
const folder = `${info.dev}:${info.ino}`;
|
|
4357
|
+
if (folders.has(folder)) return;
|
|
4358
|
+
folders.add(folder);
|
|
4359
|
+
let names;
|
|
4360
|
+
try {
|
|
4361
|
+
names = await readdir3(file);
|
|
4362
|
+
} catch (error) {
|
|
4363
|
+
result.unreadable.push({
|
|
4364
|
+
path: file,
|
|
4365
|
+
code: error.code ?? "unknown"
|
|
4366
|
+
});
|
|
4367
|
+
return;
|
|
4368
|
+
}
|
|
4369
|
+
for (const name2 of names.sort()) {
|
|
4370
|
+
if (hidden(name2)) continue;
|
|
4371
|
+
await visit(path7.join(file, name2), base);
|
|
4372
|
+
}
|
|
4373
|
+
return;
|
|
4374
|
+
}
|
|
4375
|
+
if (!info.isFile()) return;
|
|
4376
|
+
const id = `${info.dev}:${info.ino}`;
|
|
4377
|
+
if (seen.has(id)) return;
|
|
4378
|
+
seen.add(id);
|
|
4379
|
+
const name = path7.basename(file);
|
|
4380
|
+
if (hidden(name)) return;
|
|
4381
|
+
const ext = path7.extname(name).toLowerCase();
|
|
4382
|
+
if (!CLIP_EXTENSIONS.has(ext)) {
|
|
4383
|
+
result.skipped.push({ path: file, reason: skipReason(ext) });
|
|
4384
|
+
return;
|
|
4385
|
+
}
|
|
4386
|
+
result.clips.push({
|
|
4387
|
+
path: file,
|
|
4388
|
+
sourcePath: path7.relative(base, file) || name,
|
|
4389
|
+
name,
|
|
4390
|
+
size: Number(info.size),
|
|
4391
|
+
mtimeMs: Number(info.mtimeMs),
|
|
4392
|
+
ino: Number(info.ino),
|
|
4393
|
+
dev: Number(info.dev)
|
|
4394
|
+
});
|
|
4395
|
+
};
|
|
4396
|
+
for (const p of paths) {
|
|
4397
|
+
const absolute = path7.resolve(p);
|
|
4398
|
+
let base = path7.dirname(absolute);
|
|
4399
|
+
try {
|
|
4400
|
+
if ((await stat3(absolute)).isDirectory()) base = path7.dirname(absolute);
|
|
4401
|
+
} catch {
|
|
4402
|
+
}
|
|
4403
|
+
await visit(absolute, base);
|
|
4404
|
+
}
|
|
4405
|
+
return result;
|
|
4406
|
+
}
|
|
4407
|
+
function skipReason(ext) {
|
|
4408
|
+
if ([".wav", ".aif", ".aiff", ".mp3", ".m4a", ".bwf"].includes(ext))
|
|
4409
|
+
return "audio";
|
|
4410
|
+
if ([".xml", ".mhl", ".bim", ".ale", ".edl", ".txt", ".csv"].includes(ext))
|
|
4411
|
+
return "sidecar";
|
|
4412
|
+
if ([".jpg", ".jpeg", ".png", ".tif", ".tiff", ".heic"].includes(ext))
|
|
4413
|
+
return "still";
|
|
4414
|
+
if ([".ari", ".dng", ".exr", ".dpx"].includes(ext)) return "frame sequence";
|
|
4415
|
+
return "not a clip";
|
|
4416
|
+
}
|
|
4417
|
+
|
|
4418
|
+
// src/media-proxy.ts
|
|
4419
|
+
var RECIPE_VERSION = 1;
|
|
4420
|
+
var ffmpegBin = () => process.env.OSMO_FFMPEG ?? "ffmpeg";
|
|
4421
|
+
var ffprobeBin = () => process.env.OSMO_FFPROBE ?? "ffprobe";
|
|
4422
|
+
function preflight() {
|
|
4423
|
+
const run = (bin, args) => spawnSync3(bin, args, { encoding: "utf8", maxBuffer: 8 << 20 });
|
|
4424
|
+
const version2 = run(ffmpegBin(), ["-hide_banner", "-version"]);
|
|
4425
|
+
const probe = run(ffprobeBin(), ["-hide_banner", "-version"]);
|
|
4426
|
+
const ffmpeg = version2.status === 0;
|
|
4427
|
+
const ffprobe = probe.status === 0;
|
|
4428
|
+
const encoders = ffmpeg ? run(ffmpegBin(), ["-hide_banner", "-encoders"]).stdout ?? "" : "";
|
|
4429
|
+
const filters = ffmpeg ? run(ffmpegBin(), ["-hide_banner", "-filters"]).stdout ?? "" : "";
|
|
4430
|
+
let macOk = false;
|
|
4431
|
+
if (process.platform === "darwin") {
|
|
4432
|
+
const sw = run("sw_vers", ["-productVersion"]).stdout?.trim() ?? "";
|
|
4433
|
+
macOk = Number(sw.split(".")[0]) >= 13;
|
|
4434
|
+
}
|
|
4435
|
+
const result = {
|
|
4436
|
+
ffmpeg,
|
|
4437
|
+
ffprobe,
|
|
4438
|
+
ffmpegVersion: /ffmpeg version (\S+)/.exec(version2.stdout ?? "")?.[1],
|
|
4439
|
+
libx264: /\blibx264\b/.test(encoders),
|
|
4440
|
+
videotoolbox: macOk && /\bh264_videotoolbox\b/.test(encoders),
|
|
4441
|
+
scaleVt: macOk && /\bscale_vt\b/.test(filters)
|
|
4442
|
+
};
|
|
4443
|
+
const [major, minor] = (result.ffmpegVersion ?? "").replace(/^n/, "").split(".").map(Number);
|
|
4444
|
+
if (!ffmpeg || !ffprobe)
|
|
4445
|
+
result.problem = "ffmpeg and ffprobe are needed to make proxies here. Install FFmpeg; until then originals upload alone, without proxies.";
|
|
4446
|
+
else if (major < 5 || major === 5 && minor < 1)
|
|
4447
|
+
result.problem = `FFmpeg ${result.ffmpegVersion} is too old to make proxies; install 5.1 or newer.`;
|
|
4448
|
+
else if (!result.libx264 && !result.videotoolbox)
|
|
4449
|
+
result.problem = "This FFmpeg has neither libx264 nor VideoToolbox. Install an FFmpeg build with libx264.";
|
|
4450
|
+
return result;
|
|
4451
|
+
}
|
|
4452
|
+
function ratio(value) {
|
|
4453
|
+
if (!value) return 0;
|
|
4454
|
+
const [n, d] = value.split(/[/:]/).map(Number);
|
|
4455
|
+
return d ? n / d : n;
|
|
4456
|
+
}
|
|
4457
|
+
var tag = (value) => value && value !== "unknown" && value !== "reserved" ? String(value) : void 0;
|
|
4458
|
+
async function probeClip(file) {
|
|
4459
|
+
const out2 = await capture(ffprobeBin(), [
|
|
4460
|
+
"-v",
|
|
4461
|
+
"error",
|
|
4462
|
+
"-show_streams",
|
|
4463
|
+
"-show_format",
|
|
4464
|
+
"-of",
|
|
4465
|
+
"json",
|
|
4466
|
+
file
|
|
4467
|
+
]);
|
|
4468
|
+
if (out2.code !== 0) return null;
|
|
4469
|
+
let data;
|
|
4470
|
+
try {
|
|
4471
|
+
data = JSON.parse(out2.stdout);
|
|
4472
|
+
} catch {
|
|
4473
|
+
return null;
|
|
4474
|
+
}
|
|
4475
|
+
const streams = data.streams ?? [];
|
|
4476
|
+
const v = streams.find(
|
|
4477
|
+
(s) => s.codec_type === "video" && !s.disposition?.attached_pic
|
|
4478
|
+
);
|
|
4479
|
+
const audioTracks = streams.filter((s) => s.codec_type === "audio").length;
|
|
4480
|
+
const durationS = Number(data.format?.duration ?? 0) || 0;
|
|
4481
|
+
if (!v) return { durationS, audioTracks };
|
|
4482
|
+
const sideRotation = (v.side_data_list ?? []).find((d) => typeof d.rotation === "number")?.rotation;
|
|
4483
|
+
const tagRotation = Number(
|
|
4484
|
+
v.tags?.rotate ?? 0
|
|
4485
|
+
);
|
|
4486
|
+
const frames = Number(v.nb_frames);
|
|
4487
|
+
return {
|
|
4488
|
+
durationS,
|
|
4489
|
+
audioTracks,
|
|
4490
|
+
video: {
|
|
4491
|
+
codec: String(v.codec_name ?? ""),
|
|
4492
|
+
profile: v.profile ? String(v.profile) : void 0,
|
|
4493
|
+
pixFmt: String(v.pix_fmt ?? ""),
|
|
4494
|
+
width: Number(v.width ?? 0),
|
|
4495
|
+
height: Number(v.height ?? 0),
|
|
4496
|
+
sar: ratio(v.sample_aspect_ratio) || 1,
|
|
4497
|
+
frameRate: String(v.r_frame_rate ?? ""),
|
|
4498
|
+
avgFrameRate: String(v.avg_frame_rate ?? ""),
|
|
4499
|
+
frames: Number.isFinite(frames) && frames > 0 ? frames : void 0,
|
|
4500
|
+
timescale: Number(String(v.time_base ?? "").split("/")[1]) || 0,
|
|
4501
|
+
transfer: tag(v.color_transfer),
|
|
4502
|
+
colorSpace: tag(v.color_space),
|
|
4503
|
+
colorPrimaries: tag(v.color_primaries),
|
|
4504
|
+
colorRange: tag(v.color_range),
|
|
4505
|
+
rotation: Math.round(Number(sideRotation ?? tagRotation) || 0) % 360
|
|
4506
|
+
}
|
|
4507
|
+
};
|
|
4508
|
+
}
|
|
4509
|
+
function proxySize(probe) {
|
|
4510
|
+
const displayWidth = probe.width * (probe.sar || 1);
|
|
4511
|
+
const long = Math.max(displayWidth, probe.height);
|
|
4512
|
+
const f = long > 1920 ? 1920 / long : 1;
|
|
4513
|
+
const even = (n) => Math.max(2, Math.round(n / 2) * 2);
|
|
4514
|
+
return { width: even(displayWidth * f), height: even(probe.height * f) };
|
|
4515
|
+
}
|
|
4516
|
+
function planProxy(file, probe, tools) {
|
|
4517
|
+
const none = (reason) => ({
|
|
4518
|
+
path: "none",
|
|
4519
|
+
noProxyReason: reason,
|
|
4520
|
+
declared: {}
|
|
4521
|
+
});
|
|
4522
|
+
if (RAW_EXTENSIONS.has(path8.extname(file).toLowerCase())) return none("raw");
|
|
4523
|
+
if (tools.problem) return none("no-ffmpeg");
|
|
4524
|
+
if (!probe?.video || !probe.video.width || !probe.video.height)
|
|
4525
|
+
return none("unreadable");
|
|
4526
|
+
const v = probe.video;
|
|
4527
|
+
if (v.transfer === "smpte2084" || v.transfer === "arib-std-b67")
|
|
4528
|
+
return none("hdr");
|
|
4529
|
+
if (v.rotation !== 0) return none("rotated");
|
|
4530
|
+
const { width, height } = proxySize(v);
|
|
4531
|
+
const vfr = !!v.avgFrameRate && Math.abs(ratio(v.frameRate) - ratio(v.avgFrameRate)) > 0.01;
|
|
4532
|
+
const alpha = /^(yuva|rgba|bgra|argb|abgr|gbrap|ya)/.test(v.pixFmt);
|
|
4533
|
+
const tenBit422 = /422/.test(v.pixFmt);
|
|
4534
|
+
let chosen = "libx264";
|
|
4535
|
+
const rec709 = (!v.colorSpace || v.colorSpace === "bt709") && (!v.colorPrimaries || v.colorPrimaries === "bt709") && (!v.transfer || v.transfer === "bt709") && (!v.colorRange || v.colorRange === "tv");
|
|
4536
|
+
if (!vfr && tools.videotoolbox) {
|
|
4537
|
+
if (v.codec === "prores" && tools.scaleVt)
|
|
4538
|
+
chosen = rec709 ? "videotoolbox-hw" : "videotoolbox-sw";
|
|
4539
|
+
else if ((v.codec === "h264" || v.codec === "hevc") && tenBit422)
|
|
4540
|
+
chosen = "videotoolbox-sw";
|
|
4541
|
+
}
|
|
4542
|
+
if (chosen === "libx264" && !tools.libx264) {
|
|
4543
|
+
if (tools.videotoolbox) chosen = "videotoolbox-sw";
|
|
4544
|
+
else return none("no-encoder");
|
|
4545
|
+
}
|
|
4546
|
+
return {
|
|
4547
|
+
path: chosen,
|
|
4548
|
+
width,
|
|
4549
|
+
height,
|
|
4550
|
+
declared: {
|
|
4551
|
+
recipe: RECIPE_VERSION,
|
|
4552
|
+
encoder: chosen,
|
|
4553
|
+
ffmpeg: tools.ffmpegVersion,
|
|
4554
|
+
source: {
|
|
4555
|
+
codec: v.codec,
|
|
4556
|
+
pixFmt: v.pixFmt,
|
|
4557
|
+
width: v.width,
|
|
4558
|
+
height: v.height
|
|
4559
|
+
},
|
|
4560
|
+
...alpha ? { alphaFlattened: true } : {},
|
|
4561
|
+
...vfr ? { variableFrameRate: true } : {}
|
|
4562
|
+
}
|
|
4563
|
+
};
|
|
4564
|
+
}
|
|
4565
|
+
function encodeArgs(source, out2, plan, probe) {
|
|
4566
|
+
const v = probe.video;
|
|
4567
|
+
if (!v || plan.path === "none" || !plan.width || !plan.height)
|
|
4568
|
+
throw new Error("no proxy to make");
|
|
4569
|
+
const pins = [
|
|
4570
|
+
v.colorSpace ? "" : "colorspace=bt709",
|
|
4571
|
+
v.colorPrimaries ? "" : "color_primaries=bt709",
|
|
4572
|
+
v.transfer ? "" : "color_trc=bt709",
|
|
4573
|
+
v.colorRange ? "" : "range=tv"
|
|
4574
|
+
].filter(Boolean);
|
|
4575
|
+
const pin = pins.length ? `setparams=${pins.join(":")},` : "";
|
|
4576
|
+
const scale = plan.path === "videotoolbox-hw" ? `scale_vt=w=${plan.width}:h=${plan.height}` : `scale=w=${plan.width}:h=${plan.height}:out_color_matrix=bt709:out_range=tv:flags=bicubic,setsar=1,format=yuv420p`;
|
|
4577
|
+
const label = "setparams=colorspace=bt709:color_primaries=bt709:color_trc=bt709:range=tv";
|
|
4578
|
+
const video = plan.path === "libx264" ? [
|
|
4579
|
+
"-c:v",
|
|
4580
|
+
"libx264",
|
|
4581
|
+
"-preset",
|
|
4582
|
+
"veryfast",
|
|
4583
|
+
"-profile:v",
|
|
4584
|
+
"high",
|
|
4585
|
+
"-b:v",
|
|
4586
|
+
"10M",
|
|
4587
|
+
"-maxrate",
|
|
4588
|
+
"12M",
|
|
4589
|
+
"-bufsize",
|
|
4590
|
+
"20M",
|
|
4591
|
+
"-sc_threshold",
|
|
4592
|
+
"0"
|
|
4593
|
+
] : [
|
|
4594
|
+
"-c:v",
|
|
4595
|
+
"h264_videotoolbox",
|
|
4596
|
+
"-profile:v",
|
|
4597
|
+
"high",
|
|
4598
|
+
"-b:v",
|
|
4599
|
+
"10M",
|
|
4600
|
+
"-constant_bit_rate",
|
|
4601
|
+
"1",
|
|
4602
|
+
"-g",
|
|
4603
|
+
"9999"
|
|
4604
|
+
];
|
|
4605
|
+
return [
|
|
4606
|
+
"-hide_banner",
|
|
4607
|
+
"-nostdin",
|
|
4608
|
+
"-y",
|
|
4609
|
+
"-v",
|
|
4610
|
+
"error",
|
|
4611
|
+
"-progress",
|
|
4612
|
+
"pipe:1",
|
|
4613
|
+
...plan.path === "videotoolbox-hw" ? [
|
|
4614
|
+
"-hwaccel",
|
|
4615
|
+
"videotoolbox",
|
|
4616
|
+
"-hwaccel_output_format",
|
|
4617
|
+
"videotoolbox_vld"
|
|
4618
|
+
] : [],
|
|
4619
|
+
"-i",
|
|
4620
|
+
source,
|
|
4621
|
+
"-map",
|
|
4622
|
+
"0:v:0",
|
|
4623
|
+
"-map",
|
|
4624
|
+
"0:a?",
|
|
4625
|
+
"-fps_mode",
|
|
4626
|
+
"passthrough",
|
|
4627
|
+
// Preserve source timestamps. FFmpeg 9 removed the legacy -1 alias;
|
|
4628
|
+
// older releases (including supported 5.1) still need that spelling.
|
|
4629
|
+
"-enc_time_base:v",
|
|
4630
|
+
Number(String(plan.declared.ffmpeg ?? "").replace(/^n/, "").split(".")[0]) < 7 ? "-1" : "demux",
|
|
4631
|
+
"-vf",
|
|
4632
|
+
`${pin}${scale},${label}`,
|
|
4633
|
+
...video,
|
|
4634
|
+
"-bf",
|
|
4635
|
+
"0",
|
|
4636
|
+
"-force_key_frames",
|
|
4637
|
+
"expr:gte(t,n_forced*1)",
|
|
4638
|
+
"-colorspace",
|
|
4639
|
+
"bt709",
|
|
4640
|
+
"-color_primaries",
|
|
4641
|
+
"bt709",
|
|
4642
|
+
"-color_trc",
|
|
4643
|
+
"bt709",
|
|
4644
|
+
"-color_range",
|
|
4645
|
+
"tv",
|
|
4646
|
+
"-c:a",
|
|
4647
|
+
"aac",
|
|
4648
|
+
"-b:a",
|
|
4649
|
+
"192k",
|
|
4650
|
+
"-ar",
|
|
4651
|
+
"48000",
|
|
4652
|
+
...v.timescale > 0 ? ["-video_track_timescale", String(v.timescale)] : [],
|
|
4653
|
+
// Nothing of the source's metadata (locations, serials, names,
|
|
4654
|
+
// timecode): proxies are served to everyone the clip reaches.
|
|
4655
|
+
"-map_metadata",
|
|
4656
|
+
"-1",
|
|
4657
|
+
"-map_metadata:s:v",
|
|
4658
|
+
"-1",
|
|
4659
|
+
"-map_metadata:s:a",
|
|
4660
|
+
"-1",
|
|
4661
|
+
"-map_chapters",
|
|
4662
|
+
"-1",
|
|
4663
|
+
"-movflags",
|
|
4664
|
+
"+faststart",
|
|
4665
|
+
out2
|
|
4666
|
+
];
|
|
4667
|
+
}
|
|
4668
|
+
function workDir() {
|
|
4669
|
+
return path8.join(
|
|
4670
|
+
process.env.OSMO_CONFIG_DIR || path8.join(os4.homedir(), ".osmo"),
|
|
4671
|
+
"media",
|
|
4672
|
+
"work"
|
|
4673
|
+
);
|
|
4674
|
+
}
|
|
4675
|
+
async function encodeProxy(source, out2, plan, probe, onProgress) {
|
|
4676
|
+
await mkdir5(path8.dirname(out2), { recursive: true });
|
|
4677
|
+
const partial = `${out2}.partial.mp4`;
|
|
4678
|
+
await rm5(partial, { force: true });
|
|
4679
|
+
const args = encodeArgs(source, partial, plan, probe);
|
|
4680
|
+
await new Promise((resolve, reject) => {
|
|
4681
|
+
const child = spawn(ffmpegBin(), args, {
|
|
4682
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
4683
|
+
});
|
|
4684
|
+
let err = "";
|
|
4685
|
+
let buffered = "";
|
|
4686
|
+
child.stdout.on("data", (d) => {
|
|
4687
|
+
buffered += d.toString();
|
|
4688
|
+
for (let nl = buffered.indexOf("\n"); nl >= 0; nl = buffered.indexOf("\n")) {
|
|
4689
|
+
const line2 = buffered.slice(0, nl);
|
|
4690
|
+
buffered = buffered.slice(nl + 1);
|
|
4691
|
+
const m = /^out_time_us=(\d+)/.exec(line2);
|
|
4692
|
+
if (m) onProgress?.(Number(m[1]) / 1e6);
|
|
4693
|
+
}
|
|
4694
|
+
});
|
|
4695
|
+
child.stderr.on("data", (d) => {
|
|
4696
|
+
err = (err + d.toString()).slice(-4e3);
|
|
4697
|
+
});
|
|
4698
|
+
child.on("error", reject);
|
|
4699
|
+
child.on(
|
|
4700
|
+
"close",
|
|
4701
|
+
(code) => code === 0 ? resolve() : reject(
|
|
4702
|
+
new Error(
|
|
4703
|
+
err.trim().split("\n").slice(-3).join(" ") || `ffmpeg exited ${code}`
|
|
4704
|
+
)
|
|
4705
|
+
)
|
|
4706
|
+
);
|
|
4707
|
+
});
|
|
4708
|
+
await rename5(partial, out2);
|
|
4709
|
+
return out2;
|
|
4710
|
+
}
|
|
4711
|
+
async function checkProxy(proxy, source, plan) {
|
|
4712
|
+
const failures = [];
|
|
4713
|
+
const streams = await capture(ffprobeBin(), [
|
|
4714
|
+
"-v",
|
|
4715
|
+
"error",
|
|
4716
|
+
"-count_packets",
|
|
4717
|
+
"-show_entries",
|
|
4718
|
+
"stream=codec_type,codec_name,profile,width,height,pix_fmt,color_space,color_primaries,color_transfer,color_range,has_b_frames,nb_read_packets:stream_tags:format_tags",
|
|
4719
|
+
"-of",
|
|
4720
|
+
"json",
|
|
4721
|
+
proxy
|
|
4722
|
+
]);
|
|
4723
|
+
if (streams.code !== 0) return { ok: false, failures: ["unreadable"] };
|
|
4724
|
+
const parsed = JSON.parse(streams.stdout);
|
|
4725
|
+
const video = parsed.streams.find((s) => s.codec_type === "video");
|
|
4726
|
+
const audio = parsed.streams.filter((s) => s.codec_type === "audio");
|
|
4727
|
+
if (!video) return { ok: false, failures: ["no video"] };
|
|
4728
|
+
if (parsed.streams.length !== 1 + audio.length)
|
|
4729
|
+
failures.push("streams other than picture and sound");
|
|
4730
|
+
const kept = [
|
|
4731
|
+
...Object.keys(parsed.format?.tags ?? {}).filter(
|
|
4732
|
+
(k) => ![
|
|
4733
|
+
"major_brand",
|
|
4734
|
+
"minor_version",
|
|
4735
|
+
"compatible_brands",
|
|
4736
|
+
"encoder"
|
|
4737
|
+
].includes(k)
|
|
4738
|
+
),
|
|
4739
|
+
...parsed.streams.flatMap(
|
|
4740
|
+
(st) => Object.entries(st.tags ?? {}).filter(
|
|
4741
|
+
([k, v]) => !(["language", "encoder", "vendor_id"].includes(k) || k === "handler_name" && ["VideoHandler", "SoundHandler"].includes(v))
|
|
4742
|
+
).map(([k]) => k)
|
|
4743
|
+
)
|
|
4744
|
+
];
|
|
4745
|
+
if (kept.length) failures.push(`source metadata kept: ${kept.join(", ")}`);
|
|
4746
|
+
const frames = Number(video.nb_read_packets);
|
|
4747
|
+
const expected = source.video?.frames;
|
|
4748
|
+
if (expected !== void 0 && frames !== expected)
|
|
4749
|
+
failures.push(`frames ${frames} of ${expected}`);
|
|
4750
|
+
else if (expected === void 0 && source.video) {
|
|
4751
|
+
const estimate = source.durationS * ratio(source.video.avgFrameRate);
|
|
4752
|
+
if (estimate > 0 && Math.abs(frames - estimate) > 2)
|
|
4753
|
+
failures.push(`frames ${frames}, expected about ${Math.round(estimate)}`);
|
|
4754
|
+
}
|
|
4755
|
+
if (audio.length !== source.audioTracks)
|
|
4756
|
+
failures.push(`audio tracks ${audio.length} of ${source.audioTracks}`);
|
|
4757
|
+
if (video.codec_name !== "h264") failures.push(`codec ${video.codec_name}`);
|
|
4758
|
+
if (String(video.profile) !== "High")
|
|
4759
|
+
failures.push(`profile ${video.profile}`);
|
|
4760
|
+
if (video.pix_fmt !== "yuv420p")
|
|
4761
|
+
failures.push(`pixel format ${video.pix_fmt}`);
|
|
4762
|
+
if (Number(video.has_b_frames) !== 0) failures.push("B-frames");
|
|
4763
|
+
if (Number(video.width) !== plan.width || Number(video.height) !== plan.height)
|
|
4764
|
+
failures.push(`size ${video.width}x${video.height}`);
|
|
4765
|
+
for (const [key, want] of [
|
|
4766
|
+
["color_space", "bt709"],
|
|
4767
|
+
["color_primaries", "bt709"],
|
|
4768
|
+
["color_transfer", "bt709"],
|
|
4769
|
+
["color_range", "tv"]
|
|
4770
|
+
])
|
|
4771
|
+
if (video[key] !== want) failures.push(`${key} ${video[key]}`);
|
|
4772
|
+
const packets = await capture(ffprobeBin(), [
|
|
4773
|
+
"-v",
|
|
4774
|
+
"error",
|
|
4775
|
+
"-select_streams",
|
|
4776
|
+
"v:0",
|
|
4777
|
+
"-show_entries",
|
|
4778
|
+
"packet=pts_time,flags",
|
|
4779
|
+
"-of",
|
|
4780
|
+
"csv=p=0",
|
|
4781
|
+
proxy
|
|
4782
|
+
]);
|
|
4783
|
+
const rows = [];
|
|
4784
|
+
for (const line2 of packets.stdout.split("\n")) {
|
|
4785
|
+
const [time = "", flags = ""] = line2.split(",");
|
|
4786
|
+
const t = Number.parseFloat(time);
|
|
4787
|
+
if (Number.isFinite(t)) rows.push({ time: t, key: flags.includes("K") });
|
|
4788
|
+
}
|
|
4789
|
+
rows.sort((a, b) => a.time - b.time);
|
|
4790
|
+
let lastKey = Number.NEGATIVE_INFINITY;
|
|
4791
|
+
for (const { time, key } of rows) {
|
|
4792
|
+
if (key) lastKey = time;
|
|
4793
|
+
else if (time - lastKey > 1.1) {
|
|
4794
|
+
failures.push(`keyframe gap ${(time - lastKey).toFixed(2)} s`);
|
|
4795
|
+
break;
|
|
4796
|
+
}
|
|
4797
|
+
}
|
|
4798
|
+
return { ok: failures.length === 0, failures, frames };
|
|
4799
|
+
}
|
|
4800
|
+
async function capture(bin, args) {
|
|
4801
|
+
return new Promise((resolve) => {
|
|
4802
|
+
const child = spawn(bin, args, { stdio: ["ignore", "pipe", "ignore"] });
|
|
4803
|
+
const chunks = [];
|
|
4804
|
+
child.stdout.on("data", (d) => chunks.push(d));
|
|
4805
|
+
child.on("error", () => resolve({ code: -1, stdout: "" }));
|
|
4806
|
+
child.on(
|
|
4807
|
+
"close",
|
|
4808
|
+
(code) => resolve({ code: code ?? -1, stdout: Buffer.concat(chunks).toString() })
|
|
4809
|
+
);
|
|
4810
|
+
});
|
|
4811
|
+
}
|
|
4812
|
+
async function freeBytes(dir = workDir()) {
|
|
4813
|
+
try {
|
|
4814
|
+
await mkdir5(dir, { recursive: true });
|
|
4815
|
+
const s = await statfs(dir);
|
|
4816
|
+
return Number(s.bavail) * Number(s.bsize);
|
|
4817
|
+
} catch {
|
|
4818
|
+
return null;
|
|
4819
|
+
}
|
|
4820
|
+
}
|
|
4821
|
+
async function fileSize(file) {
|
|
4822
|
+
return Number((await stat4(file)).size);
|
|
4823
|
+
}
|
|
4824
|
+
|
|
4825
|
+
// src/media-transfer.ts
|
|
4826
|
+
import { createReadStream as createReadStream2 } from "node:fs";
|
|
4827
|
+
import { open as open2 } from "node:fs/promises";
|
|
4828
|
+
var STALL_BODY_MS = 6e4;
|
|
4829
|
+
var STALL_RESPONSE_MS = 12e4;
|
|
4830
|
+
async function putWatched(put, url, headers, body2) {
|
|
4831
|
+
const controller = new AbortController();
|
|
4832
|
+
let last = Date.now();
|
|
4833
|
+
let sent = false;
|
|
4834
|
+
const watchdog = setInterval(() => {
|
|
4835
|
+
const idle = Date.now() - last;
|
|
4836
|
+
if (idle > (sent ? STALL_RESPONSE_MS : STALL_BODY_MS))
|
|
4837
|
+
controller.abort(new Error("the connection stalled"));
|
|
4838
|
+
}, 5e3);
|
|
4839
|
+
let at = 0;
|
|
4840
|
+
const stream = new ReadableStream({
|
|
4841
|
+
pull(c) {
|
|
4842
|
+
last = Date.now();
|
|
4843
|
+
if (at >= body2.length) {
|
|
4844
|
+
sent = true;
|
|
4845
|
+
c.close();
|
|
4846
|
+
return;
|
|
4847
|
+
}
|
|
4848
|
+
const end = Math.min(body2.length, at + CHUNK);
|
|
4849
|
+
c.enqueue(body2.subarray(at, end));
|
|
4850
|
+
at = end;
|
|
4851
|
+
}
|
|
4852
|
+
});
|
|
4853
|
+
try {
|
|
4854
|
+
return await put(url, {
|
|
4855
|
+
method: "PUT",
|
|
4856
|
+
headers,
|
|
4857
|
+
body: stream,
|
|
4858
|
+
signal: controller.signal,
|
|
4859
|
+
// Node's fetch needs this for a streamed body.
|
|
4860
|
+
duplex: "half"
|
|
4861
|
+
});
|
|
4862
|
+
} finally {
|
|
4863
|
+
clearInterval(watchdog);
|
|
4864
|
+
}
|
|
4865
|
+
}
|
|
4866
|
+
var Semaphore = class {
|
|
4867
|
+
queue = [];
|
|
4868
|
+
available;
|
|
4869
|
+
constructor(units) {
|
|
4870
|
+
this.available = units;
|
|
4871
|
+
}
|
|
4872
|
+
async acquire(weight = 1) {
|
|
4873
|
+
if (this.queue.length === 0 && this.available >= weight) {
|
|
4874
|
+
this.available -= weight;
|
|
4875
|
+
} else {
|
|
4876
|
+
await new Promise(
|
|
4877
|
+
(resolve) => this.queue.push({ weight, resolve })
|
|
4878
|
+
);
|
|
4879
|
+
}
|
|
4880
|
+
let released = false;
|
|
4881
|
+
return () => {
|
|
4882
|
+
if (released) return;
|
|
4883
|
+
released = true;
|
|
4884
|
+
this.available += weight;
|
|
4885
|
+
this.drain();
|
|
4886
|
+
};
|
|
4887
|
+
}
|
|
4888
|
+
drain() {
|
|
4889
|
+
while (this.queue.length > 0 && this.available >= this.queue[0].weight) {
|
|
4890
|
+
const next = this.queue.shift();
|
|
4891
|
+
if (!next) break;
|
|
4892
|
+
this.available -= next.weight;
|
|
4893
|
+
next.resolve();
|
|
4894
|
+
}
|
|
4895
|
+
}
|
|
4896
|
+
};
|
|
4897
|
+
var SourceChangedError = class extends Error {
|
|
4898
|
+
};
|
|
4899
|
+
var UploadGoneError = class extends Error {
|
|
4900
|
+
};
|
|
4901
|
+
var HaltedError = class extends Error {
|
|
4902
|
+
};
|
|
4903
|
+
var MiB = 1024 * 1024;
|
|
4904
|
+
var CHUNK = 64 * 1024;
|
|
4905
|
+
var RAM_BUDGET = 1024 * MiB;
|
|
4906
|
+
var READS_PER_DRIVE = 3;
|
|
4907
|
+
var STALL_MS = 30 * 60 * 1e3;
|
|
4908
|
+
var Transfer = class {
|
|
4909
|
+
slots;
|
|
4910
|
+
ram = new Semaphore(RAM_BUDGET);
|
|
4911
|
+
readers = /* @__PURE__ */ new Map();
|
|
4912
|
+
opts;
|
|
4913
|
+
api;
|
|
4914
|
+
constructor(opts) {
|
|
4915
|
+
this.opts = opts;
|
|
4916
|
+
this.api = opts.api ?? mediaApi;
|
|
4917
|
+
this.slots = new Semaphore(opts.jobs);
|
|
4918
|
+
}
|
|
4919
|
+
reader(dev) {
|
|
4920
|
+
let s = this.readers.get(dev);
|
|
4921
|
+
if (!s) {
|
|
4922
|
+
s = new Semaphore(READS_PER_DRIVE);
|
|
4923
|
+
this.readers.set(dev, s);
|
|
4924
|
+
}
|
|
4925
|
+
return s;
|
|
4926
|
+
}
|
|
4927
|
+
/** Read `length` bytes at `offset`, holding a read slot for the drive. */
|
|
4928
|
+
async read(file, offset, length) {
|
|
4929
|
+
const release = await this.reader(file.dev).acquire();
|
|
4930
|
+
try {
|
|
4931
|
+
const handle = await open2(file.path, "r");
|
|
4932
|
+
try {
|
|
4933
|
+
const buffer = Buffer.allocUnsafe(length);
|
|
4934
|
+
let at = 0;
|
|
4935
|
+
while (at < length) {
|
|
4936
|
+
const { bytesRead } = await handle.read(
|
|
4937
|
+
buffer,
|
|
4938
|
+
at,
|
|
4939
|
+
length - at,
|
|
4940
|
+
offset + at
|
|
4941
|
+
);
|
|
4942
|
+
if (bytesRead === 0)
|
|
4943
|
+
throw new SourceChangedError(`${file.path} is shorter than it was`);
|
|
4944
|
+
at += bytesRead;
|
|
4945
|
+
}
|
|
4946
|
+
return buffer;
|
|
4947
|
+
} finally {
|
|
4948
|
+
await handle.close();
|
|
4949
|
+
}
|
|
4950
|
+
} finally {
|
|
4951
|
+
release();
|
|
4952
|
+
}
|
|
4953
|
+
}
|
|
4954
|
+
/** Each part's CRC, reading the whole file once (the duplicate check). */
|
|
4955
|
+
async hashParts(file, partSize, partCount) {
|
|
4956
|
+
const parts = {};
|
|
4957
|
+
for (let n = 1; n <= partCount; n++) {
|
|
4958
|
+
const offset = (n - 1) * partSize;
|
|
4959
|
+
const length = Math.min(partSize, file.size - offset);
|
|
4960
|
+
const release = await this.ram.acquire(Math.min(length, RAM_BUDGET));
|
|
4961
|
+
try {
|
|
4962
|
+
parts[n] = crcToB64(crc64(await this.read(file, offset, length)));
|
|
4963
|
+
} finally {
|
|
4964
|
+
release();
|
|
4965
|
+
}
|
|
4966
|
+
}
|
|
4967
|
+
return parts;
|
|
4968
|
+
}
|
|
4969
|
+
/**
|
|
4970
|
+
* Upload a file into a session, resuming the one `state` names when it can.
|
|
4971
|
+
* `save` persists `state` (it is called often; it should be cheap).
|
|
4972
|
+
*/
|
|
4973
|
+
async send(file, state, save3) {
|
|
4974
|
+
for (let restarts = 0; ; restarts++) {
|
|
4975
|
+
if (state.fileId)
|
|
4976
|
+
return { fileId: state.fileId, duplicate: !!state.duplicateOf };
|
|
4977
|
+
try {
|
|
4978
|
+
return await this.attempt(file, state, save3);
|
|
4979
|
+
} catch (error) {
|
|
4980
|
+
const reason = error instanceof SourceChangedError || error instanceof UploadGoneError ? error.message : error instanceof MediaApiError && ["upload_lost", "checksum_mismatch"].includes(error.code) ? error.code : error instanceof MediaApiError && error.code === "complete_refused" ? error.code : null;
|
|
4981
|
+
if (!reason || restarts >= 1) throw error;
|
|
4982
|
+
if (state.sessionId)
|
|
4983
|
+
await this.api.abort(state.sessionId).catch(() => {
|
|
4984
|
+
});
|
|
4985
|
+
for (const key of Object.keys(state))
|
|
4986
|
+
delete state[key];
|
|
4987
|
+
await save3();
|
|
4988
|
+
this.opts.onEvent?.({ event: "restart", file: file.path, reason });
|
|
4989
|
+
}
|
|
4990
|
+
}
|
|
4991
|
+
}
|
|
4992
|
+
async attempt(file, state, save3) {
|
|
4993
|
+
let session = state.sessionId ? await this.api.session(state.sessionId).catch((e) => {
|
|
4994
|
+
if (e instanceof MediaApiError && e.status === 404) return null;
|
|
4995
|
+
throw e;
|
|
4996
|
+
}) : null;
|
|
4997
|
+
if (session?.state === "completed") {
|
|
4998
|
+
state.fileId = session.id;
|
|
4999
|
+
await save3();
|
|
5000
|
+
return { fileId: session.id, duplicate: false };
|
|
5001
|
+
}
|
|
5002
|
+
if (!session || session.state === "aborted" || session.state === "expired") {
|
|
5003
|
+
const created = await this.api.createUpload({
|
|
5004
|
+
role: file.role,
|
|
5005
|
+
size: file.size,
|
|
5006
|
+
name: file.name,
|
|
5007
|
+
sourcePath: file.sourcePath,
|
|
5008
|
+
proxyId: file.proxyId,
|
|
5009
|
+
noProxyReason: file.noProxyReason,
|
|
5010
|
+
declared: file.declared
|
|
5011
|
+
});
|
|
5012
|
+
state.sessionId = created.id;
|
|
5013
|
+
state.partSize = created.partSize;
|
|
5014
|
+
state.partCount = created.partCount;
|
|
5015
|
+
state.parts = {};
|
|
5016
|
+
await save3();
|
|
5017
|
+
if (created.existing.length > 0) {
|
|
5018
|
+
state.parts = await this.hashParts(
|
|
5019
|
+
file,
|
|
5020
|
+
created.partSize,
|
|
5021
|
+
created.partCount
|
|
5022
|
+
);
|
|
5023
|
+
await save3();
|
|
5024
|
+
const full = combineParts(this.partList(file, state));
|
|
5025
|
+
const match = created.existing.find((e) => e.crc64nvme === full);
|
|
5026
|
+
if (match) {
|
|
5027
|
+
await this.api.abort(created.id).catch(() => {
|
|
5028
|
+
});
|
|
5029
|
+
state.fileId = match.id;
|
|
5030
|
+
state.duplicateOf = match.id;
|
|
5031
|
+
await save3();
|
|
5032
|
+
this.opts.onEvent?.({
|
|
5033
|
+
event: "duplicate",
|
|
5034
|
+
file: file.path,
|
|
5035
|
+
id: match.id
|
|
5036
|
+
});
|
|
5037
|
+
return { fileId: match.id, duplicate: true };
|
|
5038
|
+
}
|
|
5039
|
+
}
|
|
5040
|
+
session = { ...created, state: "open", size: file.size, parts: [] };
|
|
5041
|
+
}
|
|
5042
|
+
const sessionId = state.sessionId;
|
|
5043
|
+
state.partSize ??= session.partSize;
|
|
5044
|
+
state.partCount ??= session.partCount;
|
|
5045
|
+
const partSize = state.partSize;
|
|
5046
|
+
const partCount = state.partCount;
|
|
5047
|
+
state.parts ??= {};
|
|
5048
|
+
for (let round = 0; ; round++) {
|
|
5049
|
+
if (session.state === "open") {
|
|
5050
|
+
const committed = new Map(
|
|
5051
|
+
(session.parts ?? []).map((p) => [p.partNumber, p])
|
|
5052
|
+
);
|
|
5053
|
+
for (const [n, p] of committed) {
|
|
5054
|
+
const recorded = state.parts[n];
|
|
5055
|
+
if (recorded && p.crc64nvme && recorded !== p.crc64nvme)
|
|
5056
|
+
throw new SourceChangedError(
|
|
5057
|
+
`${file.path} changed since part ${n} was uploaded`
|
|
5058
|
+
);
|
|
5059
|
+
if (!recorded && p.crc64nvme) {
|
|
5060
|
+
const offset = (n - 1) * partSize;
|
|
5061
|
+
const length = Math.min(partSize, file.size - offset);
|
|
5062
|
+
const crc = crcToB64(crc64(await this.read(file, offset, length)));
|
|
5063
|
+
if (crc !== p.crc64nvme)
|
|
5064
|
+
throw new SourceChangedError(
|
|
5065
|
+
`${file.path} differs from the part ${n} already uploaded`
|
|
5066
|
+
);
|
|
5067
|
+
state.parts[n] = crc;
|
|
5068
|
+
}
|
|
5069
|
+
}
|
|
5070
|
+
const missing = [];
|
|
5071
|
+
for (let n = 1; n <= partCount; n++) {
|
|
5072
|
+
const p = committed.get(n);
|
|
5073
|
+
if (!p || p.size !== this.partLength(file, partSize, n))
|
|
5074
|
+
missing.push(n);
|
|
5075
|
+
}
|
|
5076
|
+
const halt = { failed: false };
|
|
5077
|
+
const results = await Promise.allSettled(
|
|
5078
|
+
missing.map(async (n) => {
|
|
5079
|
+
try {
|
|
5080
|
+
await this.sendPart(
|
|
5081
|
+
file,
|
|
5082
|
+
sessionId,
|
|
5083
|
+
n,
|
|
5084
|
+
partSize,
|
|
5085
|
+
state,
|
|
5086
|
+
save3,
|
|
5087
|
+
halt
|
|
5088
|
+
);
|
|
5089
|
+
} catch (error) {
|
|
5090
|
+
halt.failed = true;
|
|
5091
|
+
throw error;
|
|
5092
|
+
}
|
|
5093
|
+
})
|
|
5094
|
+
);
|
|
5095
|
+
await save3();
|
|
5096
|
+
const failed = results.find(
|
|
5097
|
+
(r) => r.status === "rejected" && !(r.reason instanceof HaltedError)
|
|
5098
|
+
);
|
|
5099
|
+
if (failed) throw failed.reason;
|
|
5100
|
+
}
|
|
5101
|
+
if (file.unchanged && !await file.unchanged())
|
|
5102
|
+
throw new SourceChangedError(
|
|
5103
|
+
`${file.path} changed while uploading; make sure nothing is writing to it`
|
|
5104
|
+
);
|
|
5105
|
+
try {
|
|
5106
|
+
const done = await this.api.complete(
|
|
5107
|
+
sessionId,
|
|
5108
|
+
combineParts(this.partList(file, state))
|
|
5109
|
+
);
|
|
5110
|
+
state.fileId = done.id;
|
|
5111
|
+
await save3();
|
|
5112
|
+
return { fileId: done.id, duplicate: false };
|
|
5113
|
+
} catch (error) {
|
|
5114
|
+
if (error instanceof MediaApiError && (error.code === "parts_missing" || error.code === "parts_changed") && round < 2) {
|
|
5115
|
+
session = await this.api.session(sessionId);
|
|
5116
|
+
continue;
|
|
5117
|
+
}
|
|
5118
|
+
throw error;
|
|
5119
|
+
}
|
|
5120
|
+
}
|
|
5121
|
+
}
|
|
5122
|
+
partLength(file, partSize, n) {
|
|
5123
|
+
return Math.min(partSize, file.size - (n - 1) * partSize);
|
|
5124
|
+
}
|
|
5125
|
+
partList(file, state) {
|
|
5126
|
+
const partSize = state.partSize;
|
|
5127
|
+
const out2 = [];
|
|
5128
|
+
for (let n = 1; n <= state.partCount; n++) {
|
|
5129
|
+
const crc = state.parts?.[n];
|
|
5130
|
+
if (!crc) throw new Error(`part ${n} of ${file.path} was never hashed`);
|
|
5131
|
+
out2.push({ crc, size: this.partLength(file, partSize, n) });
|
|
5132
|
+
}
|
|
5133
|
+
return out2;
|
|
5134
|
+
}
|
|
5135
|
+
signer = /* @__PURE__ */ new Map();
|
|
5136
|
+
/** Sign a part, batched with whatever else is waiting for this session. */
|
|
5137
|
+
sign(sessionId, part) {
|
|
5138
|
+
let batch = this.signer.get(sessionId);
|
|
5139
|
+
if (!batch) {
|
|
5140
|
+
batch = { pending: [], timer: null };
|
|
5141
|
+
this.signer.set(sessionId, batch);
|
|
5142
|
+
}
|
|
5143
|
+
const b = batch;
|
|
5144
|
+
return new Promise((resolve, reject) => {
|
|
5145
|
+
b.pending.push({ part, resolve, reject });
|
|
5146
|
+
const flush = async () => {
|
|
5147
|
+
b.timer = null;
|
|
5148
|
+
const taken = b.pending.splice(0, 32);
|
|
5149
|
+
if (b.pending.length > 0) b.timer = setTimeout(flush, 0);
|
|
5150
|
+
try {
|
|
5151
|
+
const { urls } = await this.api.signParts(
|
|
5152
|
+
sessionId,
|
|
5153
|
+
taken.map((t) => t.part)
|
|
5154
|
+
);
|
|
5155
|
+
const byNumber = new Map(urls.map((u) => [u.partNumber, u]));
|
|
5156
|
+
for (const t of taken) {
|
|
5157
|
+
const u = byNumber.get(t.part.partNumber);
|
|
5158
|
+
if (u) t.resolve(u);
|
|
5159
|
+
else t.reject(new Error(`part ${t.part.partNumber} wasn't signed`));
|
|
5160
|
+
}
|
|
5161
|
+
} catch (error) {
|
|
5162
|
+
for (const t of taken) t.reject(error);
|
|
5163
|
+
}
|
|
5164
|
+
};
|
|
5165
|
+
if (b.pending.length >= 32) {
|
|
5166
|
+
if (b.timer) clearTimeout(b.timer);
|
|
5167
|
+
void flush();
|
|
5168
|
+
} else b.timer ??= setTimeout(flush, 25);
|
|
5169
|
+
});
|
|
5170
|
+
}
|
|
5171
|
+
async sendPart(file, sessionId, n, partSize, state, save3, halt) {
|
|
5172
|
+
const length = this.partLength(file, partSize, n);
|
|
5173
|
+
const releaseSlot = await this.slots.acquire();
|
|
5174
|
+
const releaseRam = await this.ram.acquire(Math.min(length, RAM_BUDGET));
|
|
5175
|
+
try {
|
|
5176
|
+
if (halt.failed) throw new HaltedError();
|
|
5177
|
+
let body2 = await this.read(file, (n - 1) * partSize, length);
|
|
5178
|
+
let crc = crcToB64(crc64(body2));
|
|
5179
|
+
const recorded = state.parts?.[n];
|
|
5180
|
+
if (recorded && recorded !== crc)
|
|
5181
|
+
throw new SourceChangedError(`${file.path} changed while uploading`);
|
|
5182
|
+
state.parts ??= {};
|
|
5183
|
+
state.parts[n] = crc;
|
|
5184
|
+
save3().catch(() => {
|
|
5185
|
+
});
|
|
5186
|
+
let signed2 = await this.sign(sessionId, {
|
|
5187
|
+
partNumber: n,
|
|
5188
|
+
size: length,
|
|
5189
|
+
crc64nvme: crc
|
|
5190
|
+
});
|
|
5191
|
+
let resigned = false;
|
|
5192
|
+
let reread = false;
|
|
5193
|
+
let delay2 = 1e3;
|
|
5194
|
+
const started = Date.now();
|
|
5195
|
+
for (; ; ) {
|
|
5196
|
+
let status = 0;
|
|
5197
|
+
let detail = "";
|
|
5198
|
+
try {
|
|
5199
|
+
const res = await putWatched(
|
|
5200
|
+
this.opts.put ?? fetch,
|
|
5201
|
+
signed2.url,
|
|
5202
|
+
signed2.headers,
|
|
5203
|
+
body2
|
|
5204
|
+
);
|
|
5205
|
+
status = res.status;
|
|
5206
|
+
detail = status >= 400 ? await res.text().catch(() => "") : "";
|
|
5207
|
+
if (res.ok) {
|
|
5208
|
+
await res.arrayBuffer().catch(() => {
|
|
5209
|
+
});
|
|
5210
|
+
this.opts.onBytes?.(length, file.role);
|
|
5211
|
+
return;
|
|
5212
|
+
}
|
|
5213
|
+
} catch (error) {
|
|
5214
|
+
detail = error instanceof Error ? error.message : String(error);
|
|
5215
|
+
}
|
|
5216
|
+
if (status === 400 && /BadDigest/.test(detail)) {
|
|
5217
|
+
if (reread)
|
|
5218
|
+
throw new Error(
|
|
5219
|
+
`Unstable reads from the drive holding ${file.path}: part ${n} read differently twice.`
|
|
5220
|
+
);
|
|
5221
|
+
reread = true;
|
|
5222
|
+
body2 = await this.read(file, (n - 1) * partSize, length);
|
|
5223
|
+
const again = crcToB64(crc64(body2));
|
|
5224
|
+
if (again !== crc)
|
|
5225
|
+
throw new SourceChangedError(
|
|
5226
|
+
`${file.path} changed while uploading`
|
|
5227
|
+
);
|
|
5228
|
+
crc = again;
|
|
5229
|
+
continue;
|
|
5230
|
+
}
|
|
5231
|
+
if (status === 404 && /NoSuchUpload/.test(detail))
|
|
5232
|
+
throw new UploadGoneError(`${file.path}: the upload expired`);
|
|
5233
|
+
if (status === 403) {
|
|
5234
|
+
if (resigned)
|
|
5235
|
+
throw new Error(`S3 refused part ${n} of ${file.path} (403).`);
|
|
5236
|
+
resigned = true;
|
|
5237
|
+
signed2 = await this.sign(sessionId, {
|
|
5238
|
+
partNumber: n,
|
|
5239
|
+
size: length,
|
|
5240
|
+
crc64nvme: crc
|
|
5241
|
+
});
|
|
5242
|
+
continue;
|
|
5243
|
+
}
|
|
5244
|
+
const transient = status === 408 || status === 429 || status === 400 && /RequestTimeout/.test(detail);
|
|
5245
|
+
if (status >= 400 && status < 500 && !transient)
|
|
5246
|
+
throw new Error(`S3 refused part ${n} of ${file.path} (${status}).`);
|
|
5247
|
+
if (Date.now() - started > STALL_MS)
|
|
5248
|
+
throw new Error(
|
|
5249
|
+
`No progress on ${file.path} for 30 minutes. Re-run the same command to resume.`
|
|
5250
|
+
);
|
|
5251
|
+
this.opts.onEvent?.({
|
|
5252
|
+
event: "retry",
|
|
5253
|
+
file: file.path,
|
|
5254
|
+
part: n,
|
|
5255
|
+
reason: status ? `HTTP ${status}` : detail
|
|
5256
|
+
});
|
|
5257
|
+
await new Promise((r) => setTimeout(r, delay2));
|
|
5258
|
+
delay2 = Math.min(delay2 * 2, 6e4);
|
|
5259
|
+
}
|
|
5260
|
+
} finally {
|
|
5261
|
+
releaseRam();
|
|
5262
|
+
releaseSlot();
|
|
5263
|
+
}
|
|
5264
|
+
}
|
|
5265
|
+
};
|
|
5266
|
+
async function heldPartsMatch(file, sessionId, api2 = mediaApi) {
|
|
5267
|
+
const session = await api2.session(sessionId).catch((e) => {
|
|
5268
|
+
if (e instanceof MediaApiError && e.status === 404) return null;
|
|
5269
|
+
throw e;
|
|
5270
|
+
});
|
|
5271
|
+
if (session?.state === "completed") return "completed";
|
|
5272
|
+
const held = (session?.parts ?? []).filter((p) => p.crc64nvme);
|
|
5273
|
+
if (!session || held.length === 0) return null;
|
|
5274
|
+
for (const p of held) {
|
|
5275
|
+
const start = (p.partNumber - 1) * session.partSize;
|
|
5276
|
+
if (p.size !== Math.min(session.partSize, file.size - start)) return false;
|
|
5277
|
+
const crc = new Crc64Nvme();
|
|
5278
|
+
for await (const chunk of createReadStream2(file.path, {
|
|
5279
|
+
start,
|
|
5280
|
+
end: start + p.size - 1,
|
|
5281
|
+
highWaterMark: 8 << 20
|
|
5282
|
+
}))
|
|
5283
|
+
crc.update(chunk);
|
|
5284
|
+
if (crcToB64(crc.digest()) !== p.crc64nvme) return false;
|
|
5285
|
+
}
|
|
5286
|
+
return true;
|
|
5287
|
+
}
|
|
5288
|
+
|
|
5289
|
+
// src/media-command.ts
|
|
5290
|
+
var Reporter = class {
|
|
5291
|
+
io;
|
|
5292
|
+
tty;
|
|
5293
|
+
lines = [];
|
|
5294
|
+
drawn = 0;
|
|
5295
|
+
last = 0;
|
|
5296
|
+
constructor(io) {
|
|
5297
|
+
this.io = io;
|
|
5298
|
+
this.tty = !io.json && !!process.stderr.isTTY;
|
|
5299
|
+
}
|
|
5300
|
+
event(event, fields) {
|
|
5301
|
+
if (this.io.json)
|
|
5302
|
+
process.stderr.write(`${JSON.stringify({ event, ...fields })}
|
|
5303
|
+
`);
|
|
5304
|
+
}
|
|
5305
|
+
/** A line that stays. Paths and names in it are printed safely. */
|
|
5306
|
+
note(message) {
|
|
5307
|
+
if (this.io.json) return;
|
|
5308
|
+
this.clear();
|
|
5309
|
+
this.io.log(printable(message));
|
|
5310
|
+
this.redraw(true);
|
|
5311
|
+
}
|
|
5312
|
+
/** The progress block, replaced in place on a terminal. */
|
|
5313
|
+
progress(lines) {
|
|
5314
|
+
this.lines = lines.map((line2) => printable(line2));
|
|
5315
|
+
this.redraw(false);
|
|
5316
|
+
}
|
|
5317
|
+
clear() {
|
|
5318
|
+
if (!this.tty || this.drawn === 0) return;
|
|
5319
|
+
process.stderr.write(`\x1B[${this.drawn}A\x1B[0J`);
|
|
5320
|
+
this.drawn = 0;
|
|
5321
|
+
}
|
|
5322
|
+
redraw(force) {
|
|
5323
|
+
if (!this.tty) return;
|
|
5324
|
+
const now = Date.now();
|
|
5325
|
+
if (!force && now - this.last < 250) return;
|
|
5326
|
+
this.last = now;
|
|
5327
|
+
this.clear();
|
|
5328
|
+
if (this.lines.length === 0) return;
|
|
5329
|
+
process.stderr.write(`${this.lines.join("\n")}
|
|
5330
|
+
`);
|
|
5331
|
+
this.drawn = this.lines.length;
|
|
5332
|
+
}
|
|
5333
|
+
done() {
|
|
5334
|
+
this.clear();
|
|
5335
|
+
this.lines = [];
|
|
5336
|
+
}
|
|
5337
|
+
};
|
|
5338
|
+
var clips = (n) => `${n} ${n === 1 ? "clip" : "clips"}`;
|
|
5339
|
+
var gb2 = (n) => n >= 1e9 ? `${(n / 1e9).toFixed(1)} GB` : `${Math.max(0, Math.round(n / 1e6))} MB`;
|
|
5340
|
+
function saver(entry) {
|
|
5341
|
+
let chain = Promise.resolve();
|
|
5342
|
+
return () => {
|
|
5343
|
+
chain = chain.then(
|
|
5344
|
+
() => writeEntry(entry),
|
|
5345
|
+
() => writeEntry(entry)
|
|
5346
|
+
);
|
|
5347
|
+
return chain;
|
|
5348
|
+
};
|
|
5349
|
+
}
|
|
5350
|
+
function holdLock(key, file) {
|
|
5351
|
+
return new Promise((resolve, reject) => {
|
|
5352
|
+
let release;
|
|
5353
|
+
const held = new Promise((r) => {
|
|
5354
|
+
release = r;
|
|
5355
|
+
});
|
|
5356
|
+
withEntryLock(key, file, async () => {
|
|
5357
|
+
resolve(release);
|
|
5358
|
+
await held;
|
|
5359
|
+
}).catch(reject);
|
|
5360
|
+
});
|
|
5361
|
+
}
|
|
5362
|
+
async function inPool(items, limit, work) {
|
|
5363
|
+
const queue = [...items];
|
|
5364
|
+
await Promise.all(
|
|
5365
|
+
Array.from({ length: Math.min(limit, queue.length) }, async () => {
|
|
5366
|
+
for (let item = queue.shift(); item !== void 0; item = queue.shift())
|
|
5367
|
+
await work(item);
|
|
5368
|
+
})
|
|
5369
|
+
);
|
|
5370
|
+
}
|
|
5371
|
+
function acrossDrives(work) {
|
|
5372
|
+
const byDrive = /* @__PURE__ */ new Map();
|
|
5373
|
+
for (const w of [...work].sort((a, b) => a.clip.size - b.clip.size)) {
|
|
5374
|
+
const list2 = byDrive.get(w.clip.dev) ?? [];
|
|
5375
|
+
list2.push(w);
|
|
5376
|
+
byDrive.set(w.clip.dev, list2);
|
|
5377
|
+
}
|
|
5378
|
+
const out2 = [];
|
|
5379
|
+
const lists = [...byDrive.values()];
|
|
5380
|
+
for (let i = 0; out2.length < work.length; i++)
|
|
5381
|
+
for (const list2 of lists) if (list2[i]) out2.push(list2[i]);
|
|
5382
|
+
return out2;
|
|
5383
|
+
}
|
|
5384
|
+
function keepAwake() {
|
|
5385
|
+
if (process.platform !== "darwin") return;
|
|
5386
|
+
try {
|
|
5387
|
+
const child = spawn2("caffeinate", ["-i", "-w", String(process.pid)], {
|
|
5388
|
+
stdio: "ignore",
|
|
5389
|
+
detached: true
|
|
5390
|
+
});
|
|
5391
|
+
child.on("error", () => {
|
|
5392
|
+
});
|
|
5393
|
+
child.unref();
|
|
5394
|
+
} catch {
|
|
5395
|
+
}
|
|
5396
|
+
}
|
|
5397
|
+
function ownOriginals() {
|
|
5398
|
+
let bySize = null;
|
|
5399
|
+
return () => {
|
|
5400
|
+
bySize ??= (async () => {
|
|
5401
|
+
const map = /* @__PURE__ */ new Map();
|
|
5402
|
+
let cursor;
|
|
5403
|
+
do {
|
|
5404
|
+
const page = await mediaApi.list({ role: "original", cursor });
|
|
5405
|
+
for (const f of page.files) {
|
|
5406
|
+
const list2 = map.get(f.size) ?? [];
|
|
5407
|
+
list2.push(f);
|
|
5408
|
+
map.set(f.size, list2);
|
|
5409
|
+
}
|
|
5410
|
+
cursor = page.cursor ?? void 0;
|
|
5411
|
+
} while (cursor);
|
|
5412
|
+
return map;
|
|
5413
|
+
})();
|
|
5414
|
+
return bySize;
|
|
5415
|
+
};
|
|
5416
|
+
}
|
|
5417
|
+
function wholeCrcs() {
|
|
5418
|
+
const crcs = /* @__PURE__ */ new Map();
|
|
5419
|
+
return (clip) => {
|
|
5420
|
+
let crc = crcs.get(clip.path);
|
|
5421
|
+
if (!crc) {
|
|
5422
|
+
crc = (async () => {
|
|
5423
|
+
const hash = new Crc64Nvme();
|
|
5424
|
+
const stream = createReadStream3(clip.path, {
|
|
5425
|
+
highWaterMark: 8 << 20
|
|
5426
|
+
});
|
|
5427
|
+
for await (const chunk of stream) hash.update(chunk);
|
|
5428
|
+
return crcToB64(hash.digest());
|
|
5429
|
+
})();
|
|
5430
|
+
crcs.set(clip.path, crc);
|
|
5431
|
+
}
|
|
5432
|
+
return crc;
|
|
5433
|
+
};
|
|
5434
|
+
}
|
|
5435
|
+
async function accountScope() {
|
|
5436
|
+
const { origin, userId } = await mediaApi.account();
|
|
5437
|
+
const scope = `${origin} ${userId}`;
|
|
5438
|
+
const known = await readFile5(scopesFile(), "utf8").then((t) => JSON.parse(t)).catch(() => ({}));
|
|
5439
|
+
if (known[origin] !== scope) {
|
|
5440
|
+
known[origin] = scope;
|
|
5441
|
+
await mkdir6(mediaHome(), { recursive: true, mode: 448 });
|
|
5442
|
+
await writeFile5(scopesFile(), JSON.stringify(known), { mode: 384 });
|
|
5443
|
+
}
|
|
5444
|
+
return scope;
|
|
5445
|
+
}
|
|
5446
|
+
var scopesFile = () => path9.join(mediaHome(), "accounts.json");
|
|
5447
|
+
async function knownScope() {
|
|
5448
|
+
const known = await readFile5(scopesFile(), "utf8").then((t) => JSON.parse(t)).catch(() => ({}));
|
|
5449
|
+
return known[authOrigin()] ?? null;
|
|
5450
|
+
}
|
|
5451
|
+
async function recognize(clip, entry, uploaded, crcOf) {
|
|
5452
|
+
const candidates = (await uploaded()).get(clip.size);
|
|
5453
|
+
if (!candidates?.length) return;
|
|
5454
|
+
const full = await crcOf(clip);
|
|
5455
|
+
const match = candidates.find((f) => f.crc64nvme === full);
|
|
5456
|
+
if (!match) return;
|
|
5457
|
+
if (entry.original?.sessionId)
|
|
5458
|
+
await mediaApi.abort(entry.original.sessionId).catch(() => {
|
|
5459
|
+
});
|
|
5460
|
+
entry.original = {
|
|
5461
|
+
state: "done",
|
|
5462
|
+
fileId: match.id,
|
|
5463
|
+
duplicateOf: match.id,
|
|
5464
|
+
crc64nvme: full
|
|
5465
|
+
};
|
|
5466
|
+
if (match.proxyId) entry.proxy = { state: "done", fileId: match.proxyId };
|
|
5467
|
+
}
|
|
5468
|
+
async function isThisFile(entry, clip, fingerprint, uploaded, crcOf) {
|
|
5469
|
+
if (entry.source.fingerprint !== fingerprint) return false;
|
|
5470
|
+
if (entry.source.path === clip.path && entry.source.ino === clip.ino)
|
|
5471
|
+
return true;
|
|
5472
|
+
let fileId = entry.original?.fileId;
|
|
5473
|
+
const sessionId = entry.original?.sessionId;
|
|
5474
|
+
if (!fileId && sessionId) {
|
|
5475
|
+
const held = await heldPartsMatch(clip, sessionId);
|
|
5476
|
+
if (held !== "completed") return held !== false;
|
|
5477
|
+
fileId = sessionId;
|
|
5478
|
+
}
|
|
5479
|
+
if (!fileId) return true;
|
|
5480
|
+
const recorded = (await uploaded()).get(clip.size)?.find((f) => f.id === fileId);
|
|
5481
|
+
return !!recorded && recorded.crc64nvme === await crcOf(clip);
|
|
5482
|
+
}
|
|
5483
|
+
function projectFolder(value, command) {
|
|
5484
|
+
const root = path9.resolve(value);
|
|
5485
|
+
const pointer = cloudPointer(root);
|
|
5486
|
+
if (!pointer)
|
|
5487
|
+
throw new UsageError(
|
|
5488
|
+
`${value} isn't a synced Studio project's folder.`,
|
|
5489
|
+
`osmo media ${command} --help`
|
|
5490
|
+
);
|
|
5491
|
+
if (pointer.origin !== authOrigin())
|
|
5492
|
+
throw new UsageError(
|
|
5493
|
+
`That project is on ${pointer.origin}. Run the command with OSMO_BASE_URL=${pointer.origin}.`,
|
|
5494
|
+
`osmo media ${command} --help`
|
|
5495
|
+
);
|
|
5496
|
+
return root;
|
|
5497
|
+
}
|
|
5498
|
+
async function mediaUpload(paths, flags, io) {
|
|
5499
|
+
const out2 = new Reporter(io);
|
|
5500
|
+
const scope = await accountScope();
|
|
5501
|
+
const projectRoot = typeof flags.project === "string" ? projectFolder(flags.project, "upload") : null;
|
|
5502
|
+
if (!projectRoot && cloudPointer(process.cwd()))
|
|
5503
|
+
out2.note(
|
|
5504
|
+
"This folder is a Studio project; add --project . to place these clips in it."
|
|
5505
|
+
);
|
|
5506
|
+
const proxiesOnly = !!flags["proxies-only"];
|
|
5507
|
+
const noProxy = !!flags["no-proxy"];
|
|
5508
|
+
const rerun = [
|
|
5509
|
+
"osmo media upload",
|
|
5510
|
+
...paths.map(quote),
|
|
5511
|
+
...proxiesOnly ? ["--proxies-only"] : [],
|
|
5512
|
+
...noProxy ? ["--no-proxy"] : [],
|
|
5513
|
+
...projectRoot ? ["--project", quote(projectRoot)] : []
|
|
5514
|
+
].join(" ");
|
|
5515
|
+
const jobs = Number(flags.jobs ?? 12);
|
|
5516
|
+
const found = await walk(paths);
|
|
5517
|
+
for (const u of found.unreadable) {
|
|
5518
|
+
const hint = u.code === "EPERM" && process.platform === "darwin" ? " macOS refused to read it: allow your terminal in System Settings > Privacy & Security > Files and Folders (or Full Disk Access)." : "";
|
|
5519
|
+
out2.note(`Can't read ${u.path} (${u.code}).${hint}`);
|
|
5520
|
+
}
|
|
5521
|
+
for (const s of found.skipped)
|
|
5522
|
+
out2.event("skip", { file: s.path, reason: s.reason });
|
|
5523
|
+
if (found.skipped.length)
|
|
5524
|
+
out2.note(
|
|
5525
|
+
`Skipping ${found.skipped.length} ${found.skipped.length === 1 ? "file that isn't a clip" : "files that aren't clips"} (audio, sidecars, stills, frame sequences).`
|
|
5526
|
+
);
|
|
5527
|
+
if (found.clips.length === 0)
|
|
5528
|
+
return {
|
|
5529
|
+
status: found.unreadable.length ? "incomplete" : "nothing_to_upload",
|
|
5530
|
+
...found.unreadable.length ? { unreadable: found.unreadable } : {},
|
|
5531
|
+
next: "osmo media upload --help"
|
|
5532
|
+
};
|
|
5533
|
+
const tools = noProxy ? {
|
|
5534
|
+
ffmpeg: false,
|
|
5535
|
+
ffprobe: false,
|
|
5536
|
+
libx264: false,
|
|
5537
|
+
videotoolbox: false,
|
|
5538
|
+
scaleVt: false
|
|
5539
|
+
} : preflight();
|
|
5540
|
+
if (!noProxy && tools.problem) out2.note(tools.problem);
|
|
5541
|
+
const work = [];
|
|
5542
|
+
const locked = [];
|
|
5543
|
+
const uploaded = ownOriginals();
|
|
5544
|
+
const crcOf = wholeCrcs();
|
|
5545
|
+
const claimed = /* @__PURE__ */ new Map();
|
|
5546
|
+
for (const clip of found.clips) {
|
|
5547
|
+
const lock = (key2) => holdLock(key2, clip.path).catch((error) => {
|
|
5548
|
+
if (!(error instanceof LedgerLockedError)) throw error;
|
|
5549
|
+
locked.push(error.message);
|
|
5550
|
+
out2.note(error.message);
|
|
5551
|
+
return null;
|
|
5552
|
+
});
|
|
5553
|
+
let key = ledgerKey(clip, scope);
|
|
5554
|
+
let twin = claimed.get(key);
|
|
5555
|
+
if (twin && await crcOf(twin) !== await crcOf(clip)) {
|
|
5556
|
+
key = contentKey(clip, await crcOf(clip), scope);
|
|
5557
|
+
twin = claimed.get(key);
|
|
5558
|
+
}
|
|
5559
|
+
if (twin) {
|
|
5560
|
+
out2.note(
|
|
5561
|
+
`${clip.path} is the same file as ${twin.path}; uploading it once.`
|
|
5562
|
+
);
|
|
5563
|
+
continue;
|
|
5564
|
+
}
|
|
5565
|
+
claimed.set(key, clip);
|
|
5566
|
+
let unlock = await lock(key);
|
|
5567
|
+
if (!unlock) continue;
|
|
5568
|
+
let fingerprint;
|
|
5569
|
+
try {
|
|
5570
|
+
fingerprint = await fingerprintOf(clip);
|
|
5571
|
+
} catch (error) {
|
|
5572
|
+
unlock();
|
|
5573
|
+
found.unreadable.push({
|
|
5574
|
+
path: clip.path,
|
|
5575
|
+
code: error.code ?? "unknown"
|
|
5576
|
+
});
|
|
5577
|
+
out2.note(`Can't read ${clip.path}.`);
|
|
5578
|
+
continue;
|
|
5579
|
+
}
|
|
5580
|
+
let recorded = await readEntry(key);
|
|
5581
|
+
if (recorded && !await isThisFile(recorded, clip, fingerprint, uploaded, crcOf)) {
|
|
5582
|
+
unlock();
|
|
5583
|
+
key = contentKey(clip, await crcOf(clip), scope);
|
|
5584
|
+
claimed.set(key, clip);
|
|
5585
|
+
unlock = await lock(key);
|
|
5586
|
+
if (!unlock) continue;
|
|
5587
|
+
recorded = await readEntry(key);
|
|
5588
|
+
}
|
|
5589
|
+
const entry = recorded ?? {
|
|
5590
|
+
version: 1,
|
|
5591
|
+
key,
|
|
5592
|
+
scope,
|
|
5593
|
+
source: {
|
|
5594
|
+
path: clip.path,
|
|
5595
|
+
sourcePath: clip.sourcePath,
|
|
5596
|
+
name: clip.name,
|
|
5597
|
+
size: clip.size,
|
|
5598
|
+
mtimeMs: clip.mtimeMs,
|
|
5599
|
+
ino: clip.ino,
|
|
5600
|
+
fingerprint
|
|
5601
|
+
},
|
|
5602
|
+
updatedAt: ""
|
|
5603
|
+
};
|
|
5604
|
+
entry.source.path = clip.path;
|
|
5605
|
+
entry.source.ino = clip.ino;
|
|
5606
|
+
if (!entry.original?.fileId) await recognize(clip, entry, uploaded, crcOf);
|
|
5607
|
+
const needsProbe = !noProxy && !entry.original?.fileId;
|
|
5608
|
+
const probe = needsProbe && tools.ffprobe ? await probeClip(clip.path) : null;
|
|
5609
|
+
const plan = noProxy ? { path: "none", noProxyReason: "skipped", declared: {} } : entry.plan && (entry.proxy?.fileId || entry.original?.fileId) ? {
|
|
5610
|
+
path: entry.plan.proxy,
|
|
5611
|
+
noProxyReason: entry.plan.noProxyReason,
|
|
5612
|
+
declared: entry.plan.declared ?? {}
|
|
5613
|
+
} : planProxy(clip.path, probe, tools);
|
|
5614
|
+
entry.plan = {
|
|
5615
|
+
proxy: plan.path,
|
|
5616
|
+
...plan.noProxyReason ? { noProxyReason: plan.noProxyReason } : {},
|
|
5617
|
+
...probe ? { durationS: probe.durationS } : {},
|
|
5618
|
+
declared: plan.declared
|
|
5619
|
+
};
|
|
5620
|
+
await writeEntry(entry);
|
|
5621
|
+
work.push({ clip, entry, probe, plan, unlock });
|
|
5622
|
+
out2.event("plan", {
|
|
5623
|
+
file: clip.path,
|
|
5624
|
+
bytes: clip.size,
|
|
5625
|
+
proxy: plan.path,
|
|
5626
|
+
...plan.noProxyReason ? { noProxyReason: plan.noProxyReason } : {}
|
|
5627
|
+
});
|
|
5628
|
+
}
|
|
5629
|
+
const release = () => {
|
|
5630
|
+
for (const w of work) w.unlock();
|
|
5631
|
+
};
|
|
5632
|
+
try {
|
|
5633
|
+
const encodeList = work.filter(
|
|
5634
|
+
(w) => w.plan.path !== "none" && !w.entry.proxy?.fileId && !w.entry.original?.fileId && w.probe
|
|
5635
|
+
);
|
|
5636
|
+
const totalBytes = work.reduce((n, w) => n + w.clip.size, 0);
|
|
5637
|
+
const proxyHours = encodeList.reduce(
|
|
5638
|
+
(h, w) => h + (w.probe?.durationS ?? 0) / 3600,
|
|
5639
|
+
0
|
|
5640
|
+
);
|
|
5641
|
+
if (flags["dry-run"]) {
|
|
5642
|
+
return {
|
|
5643
|
+
status: "planned",
|
|
5644
|
+
clips: work.map((w) => ({
|
|
5645
|
+
file: w.clip.path,
|
|
5646
|
+
bytes: w.clip.size,
|
|
5647
|
+
proxy: w.plan.path,
|
|
5648
|
+
noProxyReason: w.plan.noProxyReason,
|
|
5649
|
+
done: !!(w.entry.original?.fileId || proxiesOnly && w.entry.proxy?.fileId)
|
|
5650
|
+
})),
|
|
5651
|
+
skipped: found.skipped.length,
|
|
5652
|
+
totalBytes,
|
|
5653
|
+
proxyHours: Math.round(proxyHours * 100) / 100,
|
|
5654
|
+
next: rerun
|
|
5655
|
+
};
|
|
5656
|
+
}
|
|
5657
|
+
const need = proxyHours * 46e8 * 1.2;
|
|
5658
|
+
const free = await freeBytes();
|
|
5659
|
+
if (free !== null && need > free)
|
|
5660
|
+
throw new UsageError(
|
|
5661
|
+
`Making these proxies needs about ${gb2(need)} free in ${workDir()}; there is ${gb2(free)}. Free some space and re-run.`,
|
|
5662
|
+
"osmo media upload --help",
|
|
5663
|
+
"disk_full"
|
|
5664
|
+
);
|
|
5665
|
+
keepAwake();
|
|
5666
|
+
if (projectRoot) {
|
|
5667
|
+
await adopt(projectRoot);
|
|
5668
|
+
const inside = await excludeSources(
|
|
5669
|
+
projectRoot,
|
|
5670
|
+
work.map((w) => w.clip.path)
|
|
5671
|
+
);
|
|
5672
|
+
if (inside.length)
|
|
5673
|
+
out2.note(
|
|
5674
|
+
`${inside.length === 1 ? "A source file is" : `${inside.length} source files are`} inside the project; they are kept out of its history.`
|
|
5675
|
+
);
|
|
5676
|
+
}
|
|
5677
|
+
const progress = {
|
|
5678
|
+
encoded: 0,
|
|
5679
|
+
encodeTotal: encodeList.length,
|
|
5680
|
+
encodeNote: "",
|
|
5681
|
+
proxyBytes: 0,
|
|
5682
|
+
proxyTotal: 0,
|
|
5683
|
+
proxiesDone: 0,
|
|
5684
|
+
originalBytes: 0,
|
|
5685
|
+
originalTotal: 0,
|
|
5686
|
+
originalsDone: 0,
|
|
5687
|
+
originalsStarted: false,
|
|
5688
|
+
started: Date.now()
|
|
5689
|
+
};
|
|
5690
|
+
const draw = () => {
|
|
5691
|
+
const secs = (Date.now() - progress.started) / 1e3 || 1;
|
|
5692
|
+
out2.progress([
|
|
5693
|
+
`Encoding proxies ${progress.encoded}/${progress.encodeTotal} ${progress.encodeNote}`,
|
|
5694
|
+
`Uploading proxies ${progress.proxiesDone}/${progress.encodeTotal} ${gb2(progress.proxyBytes)} of ${gb2(progress.proxyTotal)}`,
|
|
5695
|
+
`Originals ${progress.originalsDone}/${proxiesOnly ? 0 : work.length} ${gb2(progress.originalBytes)} of ${gb2(progress.originalTotal)}${progress.originalsStarted ? `, ${gb2((progress.originalBytes + progress.proxyBytes) / secs)}/s` : ", starts after the proxies"}`
|
|
5696
|
+
]);
|
|
5697
|
+
};
|
|
5698
|
+
const transfer = new Transfer({
|
|
5699
|
+
jobs,
|
|
5700
|
+
onBytes: (bytes2, role) => {
|
|
5701
|
+
if (role === "proxy") progress.proxyBytes += bytes2;
|
|
5702
|
+
else progress.originalBytes += bytes2;
|
|
5703
|
+
draw();
|
|
5704
|
+
},
|
|
5705
|
+
onEvent: (e) => {
|
|
5706
|
+
out2.event(e.event, e);
|
|
5707
|
+
if (e.event === "restart")
|
|
5708
|
+
out2.note(`Restarting ${e.file}: ${e.reason}`);
|
|
5709
|
+
}
|
|
5710
|
+
});
|
|
5711
|
+
const encodeJobs = Number(
|
|
5712
|
+
flags["encode-jobs"] ?? (tools.videotoolbox ? 2 : 1)
|
|
5713
|
+
);
|
|
5714
|
+
await inPool(encodeList, encodeJobs, async (w) => {
|
|
5715
|
+
const save3 = saver(w.entry);
|
|
5716
|
+
const target = path9.join(workDir(), w.entry.key, "proxy.mp4");
|
|
5717
|
+
const existing = w.entry.proxy?.workPath ? await stat5(w.entry.proxy.workPath).catch(() => null) : null;
|
|
5718
|
+
if (!(w.entry.proxy?.state === "encoded" && existing)) {
|
|
5719
|
+
try {
|
|
5720
|
+
const t0 = Date.now();
|
|
5721
|
+
await encodeProxy(
|
|
5722
|
+
w.clip.path,
|
|
5723
|
+
target,
|
|
5724
|
+
w.plan,
|
|
5725
|
+
w.probe,
|
|
5726
|
+
(s) => {
|
|
5727
|
+
const x = s / ((Date.now() - t0) / 1e3 || 1);
|
|
5728
|
+
progress.encodeNote = `${w.clip.name}, ${w.plan.path}, ${x.toFixed(1)}x realtime`;
|
|
5729
|
+
draw();
|
|
5730
|
+
}
|
|
5731
|
+
);
|
|
5732
|
+
} catch (error) {
|
|
5733
|
+
w.plan = {
|
|
5734
|
+
path: "none",
|
|
5735
|
+
noProxyReason: "encode-failed",
|
|
5736
|
+
declared: {}
|
|
5737
|
+
};
|
|
5738
|
+
w.entry.plan = { proxy: "none", noProxyReason: "encode-failed" };
|
|
5739
|
+
w.entry.proxy = { state: "failed", error: error.message };
|
|
5740
|
+
await save3();
|
|
5741
|
+
out2.event("error", {
|
|
5742
|
+
file: w.clip.path,
|
|
5743
|
+
stage: "encode",
|
|
5744
|
+
message: error.message
|
|
5745
|
+
});
|
|
5746
|
+
out2.note(
|
|
5747
|
+
`Couldn't make a proxy of ${w.clip.name}; its original uploads alone (${error.message}).`
|
|
5748
|
+
);
|
|
5749
|
+
return;
|
|
5750
|
+
}
|
|
5751
|
+
const check = await checkProxy(target, w.probe, w.plan);
|
|
5752
|
+
out2.event("check", {
|
|
5753
|
+
file: w.clip.path,
|
|
5754
|
+
ok: check.ok,
|
|
5755
|
+
failures: check.failures
|
|
5756
|
+
});
|
|
5757
|
+
if (!check.ok) {
|
|
5758
|
+
await rm6(target, { force: true });
|
|
5759
|
+
w.plan = {
|
|
5760
|
+
path: "none",
|
|
5761
|
+
noProxyReason: "local-check-failed",
|
|
5762
|
+
declared: {}
|
|
5763
|
+
};
|
|
5764
|
+
w.entry.plan = { proxy: "none", noProxyReason: "local-check-failed" };
|
|
5765
|
+
w.entry.proxy = { state: "failed", error: check.failures.join("; ") };
|
|
5766
|
+
await save3();
|
|
5767
|
+
out2.note(
|
|
5768
|
+
`The proxy of ${w.clip.name} failed its check (${check.failures.join("; ")}); its original uploads alone.`
|
|
5769
|
+
);
|
|
5770
|
+
return;
|
|
5771
|
+
}
|
|
5772
|
+
w.entry.proxy = {
|
|
5773
|
+
state: "encoded",
|
|
5774
|
+
workPath: target,
|
|
5775
|
+
size: await fileSize(target),
|
|
5776
|
+
crc64nvme: await crcOfFile(target)
|
|
5777
|
+
};
|
|
5778
|
+
await save3();
|
|
5779
|
+
}
|
|
5780
|
+
progress.encoded++;
|
|
5781
|
+
out2.event("encode", { file: w.clip.path, bytes: w.entry.proxy?.size });
|
|
5782
|
+
draw();
|
|
5783
|
+
});
|
|
5784
|
+
const proxies = work.filter(
|
|
5785
|
+
(w) => w.entry.proxy?.workPath && w.entry.proxy.state !== "failed"
|
|
5786
|
+
).sort((a, b) => (a.entry.proxy?.size ?? 0) - (b.entry.proxy?.size ?? 0));
|
|
5787
|
+
progress.proxyTotal = proxies.reduce(
|
|
5788
|
+
(n, w) => n + (w.entry.proxy?.fileId ? 0 : w.entry.proxy?.size ?? 0),
|
|
5789
|
+
0
|
|
5790
|
+
);
|
|
5791
|
+
const failures = [];
|
|
5792
|
+
const stopOriginals = { quota: false };
|
|
5793
|
+
await inPool(proxies, 4, async (w) => {
|
|
5794
|
+
const proxy = w.entry.proxy;
|
|
5795
|
+
if (!proxy?.workPath) return;
|
|
5796
|
+
const save3 = saver(w.entry);
|
|
5797
|
+
try {
|
|
5798
|
+
const state = {
|
|
5799
|
+
sessionId: proxy.sessionId,
|
|
5800
|
+
partSize: proxy.partSize,
|
|
5801
|
+
partCount: proxy.partCount,
|
|
5802
|
+
parts: proxy.parts,
|
|
5803
|
+
fileId: proxy.fileId
|
|
5804
|
+
};
|
|
5805
|
+
const file = {
|
|
5806
|
+
path: proxy.workPath,
|
|
5807
|
+
dev: -1,
|
|
5808
|
+
size: proxy.size ?? await fileSize(proxy.workPath),
|
|
5809
|
+
role: "proxy",
|
|
5810
|
+
name: `${path9.parse(w.clip.name).name}.mp4`,
|
|
5811
|
+
sourcePath: w.clip.sourcePath,
|
|
5812
|
+
declared: w.plan.declared
|
|
5813
|
+
};
|
|
5814
|
+
const sent = await transfer.send(file, state, async () => {
|
|
5815
|
+
Object.assign(proxy, state);
|
|
5816
|
+
await save3();
|
|
5817
|
+
});
|
|
5818
|
+
proxy.fileId = sent.fileId;
|
|
5819
|
+
proxy.state = "done";
|
|
5820
|
+
await save3();
|
|
5821
|
+
progress.proxiesDone++;
|
|
5822
|
+
out2.event("complete", {
|
|
5823
|
+
file: w.clip.path,
|
|
5824
|
+
role: "proxy",
|
|
5825
|
+
id: sent.fileId,
|
|
5826
|
+
duplicate: sent.duplicate
|
|
5827
|
+
});
|
|
5828
|
+
draw();
|
|
5829
|
+
} catch (error) {
|
|
5830
|
+
failures.push({ file: w.clip.path, message: error.message });
|
|
5831
|
+
out2.event("error", {
|
|
5832
|
+
file: w.clip.path,
|
|
5833
|
+
stage: "proxy",
|
|
5834
|
+
message: error.message
|
|
5835
|
+
});
|
|
5836
|
+
if (error instanceof MediaApiError && error.code === "media_upload_not_enabled")
|
|
5837
|
+
throw error;
|
|
5838
|
+
}
|
|
5839
|
+
});
|
|
5840
|
+
const existingPointers = projectRoot ? new Map((await pointersIn(projectRoot)).map((p) => [p.id, p.path])) : /* @__PURE__ */ new Map();
|
|
5841
|
+
const placed = [];
|
|
5842
|
+
let needsPull = false;
|
|
5843
|
+
const place = async (w, fileId, proxyFile) => {
|
|
5844
|
+
if (!projectRoot || w.entry.placed?.[projectRoot]) return;
|
|
5845
|
+
const already = [w.entry.proxy?.fileId, w.entry.original?.fileId].map((id) => id ? existingPointers.get(id) : void 0).find(Boolean);
|
|
5846
|
+
const rel = already ?? placeFor(projectRoot, w.clip.sourcePath);
|
|
5847
|
+
if (!already) {
|
|
5848
|
+
await placeClip(projectRoot, rel, fileId, proxyFile);
|
|
5849
|
+
placed.push(rel);
|
|
5850
|
+
}
|
|
5851
|
+
w.entry.placed = { ...w.entry.placed, [projectRoot]: rel };
|
|
5852
|
+
await writeEntry(w.entry);
|
|
5853
|
+
if (w.entry.proxy?.fileId && !existsSync3(path9.join(projectRoot, rel)))
|
|
5854
|
+
needsPull = true;
|
|
5855
|
+
out2.event("place", { file: w.clip.path, path: rel });
|
|
5856
|
+
};
|
|
5857
|
+
const unplaced = [];
|
|
5858
|
+
for (const w of work) {
|
|
5859
|
+
if (w.entry.proxy?.fileId)
|
|
5860
|
+
await place(w, w.entry.proxy.fileId, w.entry.proxy.workPath ?? null);
|
|
5861
|
+
else if (projectRoot) unplaced.push(w.clip.path);
|
|
5862
|
+
}
|
|
5863
|
+
const pointer = projectRoot ? cloudPointer(projectRoot) : null;
|
|
5864
|
+
if (needsPull && projectRoot && pointer)
|
|
5865
|
+
await pull(projectRoot, pointer).catch(
|
|
5866
|
+
(error) => out2.note(`Couldn't fetch some proxies: ${error.message}`)
|
|
5867
|
+
);
|
|
5868
|
+
const originals = proxiesOnly ? [] : acrossDrives(
|
|
5869
|
+
work.filter(
|
|
5870
|
+
(w) => !w.entry.original?.fileId && (w.plan.path === "none" || !!w.entry.proxy?.fileId)
|
|
5871
|
+
)
|
|
5872
|
+
);
|
|
5873
|
+
progress.originalTotal = originals.reduce((n, w) => n + w.clip.size, 0);
|
|
5874
|
+
progress.originalsStarted = true;
|
|
5875
|
+
await inPool(originals, 4, async (w) => {
|
|
5876
|
+
if (stopOriginals.quota) return;
|
|
5877
|
+
const save3 = saver(w.entry);
|
|
5878
|
+
w.entry.original ??= { state: "uploading" };
|
|
5879
|
+
const original = w.entry.original;
|
|
5880
|
+
const unchanged = async () => {
|
|
5881
|
+
const now = await stat5(w.clip.path).catch(() => null);
|
|
5882
|
+
return !!now && Number(now.size) === w.entry.source.size && Math.floor(Number(now.mtimeMs)) === Math.floor(w.entry.source.mtimeMs);
|
|
5883
|
+
};
|
|
5884
|
+
if (!await unchanged())
|
|
5885
|
+
return void failures.push({
|
|
5886
|
+
file: w.clip.path,
|
|
5887
|
+
message: "the file changed or went away; re-run to upload it"
|
|
5888
|
+
});
|
|
5889
|
+
const state = {
|
|
5890
|
+
sessionId: original.sessionId,
|
|
5891
|
+
partSize: original.partSize,
|
|
5892
|
+
partCount: original.partCount,
|
|
5893
|
+
parts: original.parts,
|
|
5894
|
+
fileId: original.fileId,
|
|
5895
|
+
duplicateOf: original.duplicateOf
|
|
5896
|
+
};
|
|
5897
|
+
try {
|
|
5898
|
+
const proxyId = w.entry.proxy?.fileId;
|
|
5899
|
+
const sent = await transfer.send(
|
|
5900
|
+
{
|
|
5901
|
+
path: w.clip.path,
|
|
5902
|
+
dev: w.clip.dev,
|
|
5903
|
+
size: w.clip.size,
|
|
5904
|
+
role: "original",
|
|
5905
|
+
name: w.clip.name,
|
|
5906
|
+
sourcePath: w.clip.sourcePath,
|
|
5907
|
+
...proxyId ? { proxyId } : { noProxyReason: w.plan.noProxyReason ?? "none" },
|
|
5908
|
+
...w.probe ? {
|
|
5909
|
+
declared: {
|
|
5910
|
+
durationS: w.probe.durationS,
|
|
5911
|
+
video: w.probe.video,
|
|
5912
|
+
audioTracks: w.probe.audioTracks
|
|
5913
|
+
}
|
|
5914
|
+
} : {},
|
|
5915
|
+
// Checked again before complete: written to while it was
|
|
5916
|
+
// read, what S3 holds would be a mix.
|
|
5917
|
+
unchanged
|
|
5918
|
+
},
|
|
5919
|
+
state,
|
|
5920
|
+
async () => {
|
|
5921
|
+
Object.assign(original, state);
|
|
5922
|
+
await save3();
|
|
5923
|
+
}
|
|
5924
|
+
);
|
|
5925
|
+
original.fileId = sent.fileId;
|
|
5926
|
+
original.state = "done";
|
|
5927
|
+
if (state.parts && Object.keys(state.parts).length === state.partCount)
|
|
5928
|
+
original.crc64nvme = combineParts(
|
|
5929
|
+
Array.from({ length: state.partCount }, (_, i) => ({
|
|
5930
|
+
crc: state.parts[i + 1],
|
|
5931
|
+
size: Math.min(
|
|
5932
|
+
state.partSize,
|
|
5933
|
+
w.clip.size - i * state.partSize
|
|
5934
|
+
)
|
|
5935
|
+
}))
|
|
5936
|
+
);
|
|
5937
|
+
if (sent.duplicate) {
|
|
5938
|
+
const known = await mediaApi.file(sent.fileId).catch(() => null);
|
|
5939
|
+
w.entry.proxy = known?.proxyId ? { state: "done", fileId: known.proxyId } : void 0;
|
|
5940
|
+
}
|
|
5941
|
+
await save3();
|
|
5942
|
+
progress.originalsDone++;
|
|
5943
|
+
out2.event(sent.duplicate ? "duplicate" : "complete", {
|
|
5944
|
+
file: w.clip.path,
|
|
5945
|
+
role: "original",
|
|
5946
|
+
id: sent.fileId
|
|
5947
|
+
});
|
|
5948
|
+
draw();
|
|
5949
|
+
} catch (error) {
|
|
5950
|
+
if (error instanceof MediaApiError && error.code === "quota_exceeded")
|
|
5951
|
+
stopOriginals.quota = true;
|
|
5952
|
+
if (error instanceof MediaApiError && error.code === "proxy_not_found") {
|
|
5953
|
+
w.entry.proxy = void 0;
|
|
5954
|
+
delete w.entry.plan;
|
|
5955
|
+
}
|
|
5956
|
+
original.state = "failed";
|
|
5957
|
+
original.error = error.message;
|
|
5958
|
+
await save3();
|
|
5959
|
+
failures.push({ file: w.clip.path, message: error.message });
|
|
5960
|
+
out2.event("error", {
|
|
5961
|
+
file: w.clip.path,
|
|
5962
|
+
stage: "original",
|
|
5963
|
+
message: error.message
|
|
5964
|
+
});
|
|
5965
|
+
}
|
|
5966
|
+
});
|
|
5967
|
+
out2.done();
|
|
5968
|
+
const clips2 = work.map((w) => ({
|
|
5969
|
+
file: w.clip.path,
|
|
5970
|
+
proxyId: w.entry.proxy?.fileId,
|
|
5971
|
+
originalId: w.entry.original?.fileId,
|
|
5972
|
+
...w.plan.noProxyReason ? { noProxyReason: w.plan.noProxyReason } : {},
|
|
5973
|
+
...w.entry.original?.duplicateOf ? { duplicate: true } : {}
|
|
5974
|
+
}));
|
|
5975
|
+
const unfinished = work.filter(
|
|
5976
|
+
(w) => !(proxiesOnly ? w.entry.proxy?.fileId || w.plan.path === "none" : w.entry.original?.fileId)
|
|
5977
|
+
);
|
|
5978
|
+
for (const w of work)
|
|
5979
|
+
if (w.entry.proxy?.state === "done" && w.entry.proxy.workPath) {
|
|
5980
|
+
if (w.entry.original?.fileId)
|
|
5981
|
+
await rm6(path9.dirname(w.entry.proxy.workPath), {
|
|
5982
|
+
recursive: true,
|
|
5983
|
+
force: true
|
|
5984
|
+
});
|
|
5985
|
+
}
|
|
5986
|
+
const result = {
|
|
5987
|
+
status: unfinished.length || locked.length || failures.length || found.unreadable.length ? "incomplete" : "done",
|
|
5988
|
+
clips: clips2,
|
|
5989
|
+
skipped: found.skipped.length,
|
|
5990
|
+
...found.unreadable.length ? { unreadable: found.unreadable } : {},
|
|
5991
|
+
...projectRoot ? { project: projectRoot, placed, unplaced } : {},
|
|
5992
|
+
...failures.length ? { failures } : {},
|
|
5993
|
+
...locked.length ? { locked } : {},
|
|
5994
|
+
...stopOriginals.quota ? { quota: "exceeded" } : {},
|
|
5995
|
+
next: unfinished.length ? rerun : "osmo media ls"
|
|
5996
|
+
};
|
|
5997
|
+
out2.event("done", { status: result.status });
|
|
5998
|
+
return result;
|
|
5999
|
+
} finally {
|
|
6000
|
+
out2.done();
|
|
6001
|
+
release();
|
|
6002
|
+
}
|
|
6003
|
+
}
|
|
6004
|
+
function formatUpload(result) {
|
|
6005
|
+
const r = result;
|
|
6006
|
+
if (r.status === "nothing_to_upload") return "No clips found.";
|
|
6007
|
+
if (r.status === "planned")
|
|
6008
|
+
return [
|
|
6009
|
+
`${clips(r.clips?.length ?? 0)}, ${gb2(r.totalBytes ?? 0)}; ${Math.round((r.proxyHours ?? 0) * 60)} min of footage to proxy here.`,
|
|
6010
|
+
...(r.clips ?? []).map(
|
|
6011
|
+
(c) => ` ${c.file} ${c.proxy}${c.noProxyReason ? ` (${c.noProxyReason})` : ""}${c.done ? " done" : ""}`
|
|
6012
|
+
)
|
|
6013
|
+
].join("\n");
|
|
6014
|
+
const done = r.clips?.filter((c) => c.originalId).length ?? 0;
|
|
6015
|
+
const duplicates = r.clips?.filter((c) => c.duplicate).length ?? 0;
|
|
6016
|
+
const lines = [
|
|
6017
|
+
`${done} of ${clips(r.clips?.length ?? 0)} uploaded${duplicates ? ` (${duplicates} ${duplicates === 1 ? "was" : "were"} already there)` : ""}.`,
|
|
6018
|
+
...(r.clips ?? []).map(
|
|
6019
|
+
(c) => ` ${path9.basename(c.file)} ${c.originalId ?? "-"}${c.proxyId ? ` proxy ${c.proxyId}` : c.noProxyReason ? ` no proxy here (${c.noProxyReason})` : ""}`
|
|
6020
|
+
)
|
|
6021
|
+
];
|
|
6022
|
+
if (r.project && r.placed?.length)
|
|
6023
|
+
lines.push(
|
|
6024
|
+
`Placed ${clips(r.placed.length)} in ${MEDIA_DIR}/; scenes use their paths (sync sends the pointers, never the proxies).`
|
|
6025
|
+
);
|
|
6026
|
+
if (r.unplaced?.length)
|
|
6027
|
+
lines.push(
|
|
6028
|
+
`Not placed: ${clips(r.unplaced.length)} with no proxy up (HDR, rotated or RAW, or a proxy that failed or didn't finish uploading: re-run to retry it).`
|
|
6029
|
+
);
|
|
6030
|
+
if (r.quota)
|
|
6031
|
+
lines.push(
|
|
6032
|
+
"The account's footage storage is full; the remaining originals were skipped."
|
|
6033
|
+
);
|
|
6034
|
+
for (const f of r.failures ?? [])
|
|
6035
|
+
lines.push(`Failed: ${f.file}: ${f.message}`);
|
|
6036
|
+
for (const u of r.unreadable ?? [])
|
|
6037
|
+
lines.push(`Couldn't read: ${u.path} (${u.code})`);
|
|
6038
|
+
if (r.status !== "done") lines.push(`Re-run to resume: ${r.next}`);
|
|
6039
|
+
return lines.join("\n");
|
|
6040
|
+
}
|
|
6041
|
+
async function mediaStatus(paths) {
|
|
6042
|
+
const scope = await knownScope() ?? await accountScope();
|
|
6043
|
+
let entries = (await allEntries()).filter((e) => e.scope === scope);
|
|
6044
|
+
if (paths.length) {
|
|
6045
|
+
const found = await walk(paths);
|
|
6046
|
+
const keys = new Set(found.clips.map((c) => ledgerKey(c, scope)));
|
|
6047
|
+
const seen = new Set(found.clips.map((c) => c.path));
|
|
6048
|
+
entries = entries.filter((e) => keys.has(e.key) || seen.has(e.source.path));
|
|
6049
|
+
}
|
|
6050
|
+
const files = entries.sort((a, b) => a.source.path.localeCompare(b.source.path)).map((e) => ({
|
|
6051
|
+
file: e.source.path,
|
|
6052
|
+
proxy: e.proxy?.fileId ? "uploaded" : e.proxy?.state ?? e.plan?.noProxyReason ?? "none",
|
|
6053
|
+
original: e.original?.fileId ? e.original.duplicateOf ? "already uploaded" : "uploaded" : e.original?.state ?? "waiting",
|
|
6054
|
+
proxyId: e.proxy?.fileId,
|
|
6055
|
+
originalId: e.original?.fileId,
|
|
6056
|
+
partsSent: e.original?.parts ? Object.keys(e.original.parts).length : 0,
|
|
6057
|
+
partCount: e.original?.partCount
|
|
6058
|
+
}));
|
|
6059
|
+
return { status: "ok", files, next: "osmo media upload PATH" };
|
|
6060
|
+
}
|
|
6061
|
+
function formatStatus(result) {
|
|
6062
|
+
const r = result;
|
|
6063
|
+
if (!r.files.length) return "No footage uploads recorded on this machine.";
|
|
6064
|
+
return r.files.map(
|
|
6065
|
+
(f) => `${f.file}
|
|
6066
|
+
proxy: ${f.proxy}; original: ${f.original}${f.original === "uploading" && f.partCount ? ` (${f.partsSent}/${f.partCount} parts read)` : ""}`
|
|
6067
|
+
).join("\n");
|
|
6068
|
+
}
|
|
6069
|
+
async function mediaList() {
|
|
6070
|
+
const files = [];
|
|
6071
|
+
let cursor;
|
|
6072
|
+
do {
|
|
6073
|
+
const page = await mediaApi.list({ cursor });
|
|
6074
|
+
files.push(...page.files);
|
|
6075
|
+
cursor = page.cursor ?? void 0;
|
|
6076
|
+
} while (cursor && files.length < 5e3);
|
|
6077
|
+
return { status: "ok", files, next: "osmo media get ID" };
|
|
6078
|
+
}
|
|
6079
|
+
function formatList(result) {
|
|
6080
|
+
const r = result;
|
|
6081
|
+
if (!r.files.length) return "No footage yet.";
|
|
6082
|
+
return r.files.map(
|
|
6083
|
+
(f) => `${f.id} ${f.role.padEnd(8)} ${gb2(f.size).padStart(8)} ${f.displayName ?? ""}${f.status ? ` [${f.status}]` : ""}`
|
|
6084
|
+
).join("\n");
|
|
6085
|
+
}
|
|
6086
|
+
async function mediaAdd(ids, flags) {
|
|
6087
|
+
if (typeof flags.project !== "string")
|
|
6088
|
+
throw new UsageError("--project is required.", "osmo media add --help");
|
|
6089
|
+
const root = projectFolder(flags.project, "add");
|
|
6090
|
+
const files = await Promise.all(ids.map((id) => mediaApi.file(id)));
|
|
6091
|
+
await adopt(root);
|
|
6092
|
+
const existing = new Set((await pointersIn(root)).map((p) => p.id));
|
|
6093
|
+
const placed = [];
|
|
6094
|
+
const unplaced = [];
|
|
6095
|
+
for (const f of files) {
|
|
6096
|
+
if ([f.id, f.proxyId, f.originalId].some((id) => id && existing.has(id)))
|
|
6097
|
+
continue;
|
|
6098
|
+
if (f.role === "original" && !f.proxyId) {
|
|
6099
|
+
unplaced.push(f.displayName ?? f.id);
|
|
6100
|
+
continue;
|
|
6101
|
+
}
|
|
6102
|
+
const rel = placeFor(root, f.sourcePath ?? f.displayName ?? `${f.id}.mp4`);
|
|
6103
|
+
await placeClip(root, rel, f.id, null);
|
|
6104
|
+
existing.add(f.id);
|
|
6105
|
+
placed.push(rel);
|
|
6106
|
+
}
|
|
6107
|
+
const pointer = cloudPointer(root);
|
|
6108
|
+
const pulled = pointer ? await pull(root, pointer) : null;
|
|
6109
|
+
return {
|
|
6110
|
+
status: "added",
|
|
6111
|
+
placed,
|
|
6112
|
+
...unplaced.length ? { unplaced } : {},
|
|
6113
|
+
...pulled ? { pull: pulled } : {},
|
|
6114
|
+
next: "osmo studio sync"
|
|
6115
|
+
};
|
|
6116
|
+
}
|
|
6117
|
+
function formatAdd(result) {
|
|
6118
|
+
const r = result;
|
|
6119
|
+
return [
|
|
6120
|
+
r.placed.length ? `Placed ${clips(r.placed.length)}:` : r.unplaced?.length ? "Nothing placed." : "Those clips are already in the project.",
|
|
6121
|
+
...r.placed.map((p) => ` ${printable(p)}`),
|
|
6122
|
+
...r.unplaced?.length ? [
|
|
6123
|
+
`Not placed, having no proxy: ${r.unplaced.map((n) => printable(n)).join(", ")}`
|
|
6124
|
+
] : [],
|
|
6125
|
+
...r.pull ? pullLines(r.pull) : []
|
|
6126
|
+
].join("\n");
|
|
6127
|
+
}
|
|
6128
|
+
async function mediaPull(flags) {
|
|
6129
|
+
const root = path9.resolve(
|
|
6130
|
+
typeof flags.project === "string" ? flags.project : "."
|
|
6131
|
+
);
|
|
6132
|
+
const pointer = cloudPointer(root);
|
|
6133
|
+
if (!pointer)
|
|
6134
|
+
throw new UsageError(
|
|
6135
|
+
`${root} isn't a synced Studio project's folder.`,
|
|
6136
|
+
"osmo media pull --help"
|
|
6137
|
+
);
|
|
6138
|
+
const result = await pull(root, pointer, (m) => console.error(m));
|
|
6139
|
+
return {
|
|
6140
|
+
status: result.unavailable.length || result.pending.length || result.failed.length ? "incomplete" : "pulled",
|
|
6141
|
+
...result,
|
|
6142
|
+
next: "osmo studio open"
|
|
6143
|
+
};
|
|
6144
|
+
}
|
|
6145
|
+
function formatPull(result) {
|
|
6146
|
+
return pullLines(result).join("\n");
|
|
6147
|
+
}
|
|
6148
|
+
async function mediaGet(id, flags) {
|
|
6149
|
+
let file = await mediaApi.file(id);
|
|
6150
|
+
if (file.role === "original" && !flags.original && file.proxyId)
|
|
6151
|
+
file = await mediaApi.file(file.proxyId);
|
|
6152
|
+
const fallback = `${file.id}.${file.role === "proxy" ? "mp4" : "bin"}`;
|
|
6153
|
+
const name = path9.basename(file.displayName ?? "") || fallback;
|
|
6154
|
+
const target = typeof flags.out === "string" ? path9.resolve(flags.out) : path9.resolve([".", ".."].includes(name) ? fallback : name);
|
|
6155
|
+
if (existsSync3(target) && !flags.force)
|
|
6156
|
+
throw new UsageError(
|
|
6157
|
+
`${target} already exists. Give another path with -o, or add --force to replace it.`,
|
|
6158
|
+
"osmo media get --help"
|
|
6159
|
+
);
|
|
6160
|
+
const partial = `${target}.${process.pid}.partial`;
|
|
6161
|
+
const res = await fetch(file.url);
|
|
6162
|
+
if (!res.ok || !res.body) throw new Error(`Download failed (${res.status}).`);
|
|
6163
|
+
const crc = new Crc64Nvme();
|
|
6164
|
+
const body2 = Readable2.fromWeb(
|
|
6165
|
+
res.body
|
|
6166
|
+
);
|
|
6167
|
+
body2.on("data", (chunk) => crc.update(chunk));
|
|
6168
|
+
await pipeline2(body2, createWriteStream2(partial, { flags: "wx" }));
|
|
6169
|
+
if (crcToB64(crc.digest()) !== file.crc64nvme) {
|
|
6170
|
+
await rm6(partial, { force: true });
|
|
6171
|
+
throw new Error(
|
|
6172
|
+
"The download didn't match the file's checksum. Try again."
|
|
6173
|
+
);
|
|
6174
|
+
}
|
|
6175
|
+
await rename6(partial, target);
|
|
6176
|
+
await recordPulled([
|
|
6177
|
+
{
|
|
6178
|
+
fileId: file.id,
|
|
6179
|
+
size: file.size,
|
|
6180
|
+
crc64nvme: file.crc64nvme,
|
|
6181
|
+
path: target
|
|
6182
|
+
}
|
|
6183
|
+
]);
|
|
6184
|
+
return {
|
|
6185
|
+
status: "downloaded",
|
|
6186
|
+
id: file.id,
|
|
6187
|
+
role: file.role,
|
|
6188
|
+
path: target,
|
|
6189
|
+
bytes: file.size,
|
|
6190
|
+
next: "osmo media ls"
|
|
6191
|
+
};
|
|
6192
|
+
}
|
|
6193
|
+
function formatGet(result) {
|
|
6194
|
+
const r = result;
|
|
6195
|
+
return `Saved the ${r.role} to ${r.path} (${gb2(r.bytes)}), checksum verified.`;
|
|
6196
|
+
}
|
|
6197
|
+
|
|
6198
|
+
// src/resources.ts
|
|
6199
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
6200
|
+
import { constants } from "node:fs";
|
|
6201
|
+
import {
|
|
6202
|
+
copyFile as copyFile2,
|
|
6203
|
+
lstat as lstat4,
|
|
6204
|
+
mkdir as mkdir7,
|
|
6205
|
+
mkdtemp,
|
|
6206
|
+
readdir as readdir4,
|
|
6207
|
+
readFile as readFile6,
|
|
6208
|
+
realpath as realpath2,
|
|
6209
|
+
rename as rename7,
|
|
6210
|
+
rm as rm7,
|
|
6211
|
+
writeFile as writeFile6
|
|
6212
|
+
} from "node:fs/promises";
|
|
6213
|
+
import path10 from "node:path";
|
|
6214
|
+
import { unzipSync } from "fflate";
|
|
6215
|
+
var CATALOG_OWNER = "osmo";
|
|
6216
|
+
var COST = {
|
|
6217
|
+
second: "grows with seconds of output",
|
|
6218
|
+
image: "is the same for each image",
|
|
6219
|
+
character: "grows with characters of text"
|
|
6220
|
+
};
|
|
6221
|
+
var packageFiles = (primitive) => primitive === "Model" ? ["model.glb", "resource.json"] : ["resource.json", "shader.d.ts", "shader.wgsl"];
|
|
6222
|
+
var LOCK = ".osmo-get.lock";
|
|
6223
|
+
var maxFile = 1024 * 1024;
|
|
6224
|
+
var maxResponse = 4 * maxFile;
|
|
6225
|
+
var maxModel = 64 * maxFile;
|
|
6226
|
+
var fileLimit = (name) => name.endsWith(".glb") ? maxModel : maxFile;
|
|
6227
|
+
var DOWNLOAD_MS = 30 * 6e4;
|
|
6228
|
+
var ownerPattern = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
6229
|
+
var namePattern = /^[a-z0-9]+(?:[.-][a-z0-9]+)*$/;
|
|
6230
|
+
var versionPattern = /^\d+\.\d+\.\d+$/;
|
|
6231
|
+
var digest2 = (bytes2) => createHash4("sha256").update(bytes2).digest("hex");
|
|
6232
|
+
var fail = (code, message, next) => {
|
|
6233
|
+
throw new AuthError(code, message, next);
|
|
6234
|
+
};
|
|
6235
|
+
var invalid = () => fail(
|
|
6236
|
+
"invalid_resource",
|
|
6237
|
+
"Osmo returned an invalid resource package or metadata."
|
|
6238
|
+
);
|
|
6239
|
+
var text = (value) => typeof value === "string" && value.length > 0;
|
|
6240
|
+
var strings = (value) => Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
6241
|
+
function catalogRef(input) {
|
|
6242
|
+
let value = input.trim();
|
|
6243
|
+
if (/^https?:\/\//i.test(value)) {
|
|
6244
|
+
let page = null;
|
|
6245
|
+
try {
|
|
6246
|
+
page = /^\/resources\/([^/]+)\/?$/.exec(new URL(value).pathname);
|
|
6247
|
+
} catch {
|
|
6248
|
+
}
|
|
6249
|
+
value = page ? decodeURIComponent(page[1]) : "";
|
|
6250
|
+
}
|
|
6251
|
+
const at = value.indexOf("@");
|
|
6252
|
+
const version2 = at === -1 ? void 0 : value.slice(at + 1);
|
|
6253
|
+
const parts = (at === -1 ? value : value.slice(0, at)).split("/");
|
|
6254
|
+
const [owner, name] = parts.length === 1 ? [CATALOG_OWNER, ...parts] : parts;
|
|
6255
|
+
if (parts.length > 2 || !ownerPattern.test(owner) || !namePattern.test(name))
|
|
6256
|
+
fail(
|
|
6257
|
+
"invalid_resource_id",
|
|
6258
|
+
"Use an ID from osmo resources list, such as osmo/silk or silk.",
|
|
6259
|
+
"osmo resources list"
|
|
6260
|
+
);
|
|
6261
|
+
if (version2 !== void 0 && !versionPattern.test(version2))
|
|
6262
|
+
fail("invalid_version", "Use a version such as osmo/silk@1.0.0.");
|
|
6263
|
+
return { id: `${owner}/${name}`, owner, name, version: version2 };
|
|
6264
|
+
}
|
|
6265
|
+
async function bytes(response, limit = maxResponse) {
|
|
6266
|
+
if (Number(response.headers.get("content-length")) > limit) {
|
|
6267
|
+
await response.body?.cancel();
|
|
6268
|
+
invalid();
|
|
6269
|
+
}
|
|
6270
|
+
const chunks = [];
|
|
6271
|
+
let size = 0;
|
|
6272
|
+
try {
|
|
6273
|
+
for await (const chunk of response.body ?? []) {
|
|
6274
|
+
size += chunk.length;
|
|
6275
|
+
if (size > limit) invalid();
|
|
6276
|
+
chunks.push(chunk);
|
|
6277
|
+
}
|
|
6278
|
+
} catch (error) {
|
|
6279
|
+
if (error instanceof OsmoError) throw error;
|
|
6280
|
+
fail(
|
|
6281
|
+
"network_error",
|
|
6282
|
+
"Lost the connection to Osmo partway through. Check your connection and retry."
|
|
6283
|
+
);
|
|
6284
|
+
}
|
|
6285
|
+
return Buffer.concat(chunks);
|
|
6286
|
+
}
|
|
6287
|
+
async function responseBody(response, json = true, limit) {
|
|
6288
|
+
if (!response.ok) {
|
|
6289
|
+
const body3 = await response.json().catch(() => ({}));
|
|
6290
|
+
if (response.status === 401) throw loginRequired(authOrigin());
|
|
6291
|
+
if (response.status === 404)
|
|
6292
|
+
fail(
|
|
6293
|
+
"resource_not_found",
|
|
6294
|
+
"No such resource or version. Run osmo resources list.",
|
|
6295
|
+
"osmo resources list"
|
|
6296
|
+
);
|
|
6297
|
+
if (body3.error === "unknown_type" && strings(body3.types))
|
|
6298
|
+
throw new UsageError(
|
|
6299
|
+
`Unknown --type. Types: ${body3.types.join(", ")}.`,
|
|
6300
|
+
"osmo resources list",
|
|
6301
|
+
"unknown_type",
|
|
6302
|
+
{ types: body3.types }
|
|
6303
|
+
);
|
|
6304
|
+
fail(
|
|
6305
|
+
"resource_request_failed",
|
|
6306
|
+
`Osmo resource request failed (${response.status}). Retry later.`
|
|
6307
|
+
);
|
|
6308
|
+
}
|
|
6309
|
+
const body2 = await bytes(response, limit);
|
|
6310
|
+
if (!json) return body2;
|
|
6311
|
+
try {
|
|
6312
|
+
return JSON.parse(String(body2));
|
|
6313
|
+
} catch {
|
|
6314
|
+
invalid();
|
|
6315
|
+
}
|
|
6316
|
+
}
|
|
6317
|
+
async function publicGet(route) {
|
|
6318
|
+
let response;
|
|
6319
|
+
try {
|
|
6320
|
+
response = await osmoFetch(authOrigin() + route, {
|
|
6321
|
+
redirect: "error",
|
|
6322
|
+
signal: AbortSignal.timeout(2e4)
|
|
6323
|
+
});
|
|
6324
|
+
} catch (error) {
|
|
6325
|
+
if (error instanceof OsmoError) throw error;
|
|
6326
|
+
fail(
|
|
6327
|
+
"network_error",
|
|
6328
|
+
"Cannot reach the Osmo resource catalog. Retry or continue using local capabilities."
|
|
6329
|
+
);
|
|
6330
|
+
}
|
|
6331
|
+
return responseBody(response);
|
|
3683
6332
|
}
|
|
3684
6333
|
function validItem(item) {
|
|
3685
6334
|
return !!item && text(item.id) && text(item.title) && typeof item.summary === "string" && (item.use === "run" || item.use === "get") && text(item.kind) && strings(item.tags) && (item.tasks === void 0 || strings(item.tasks));
|
|
@@ -3833,21 +6482,21 @@ function unpack(zip, pkg) {
|
|
|
3833
6482
|
invalid();
|
|
3834
6483
|
return { files, manifest: manifest2, names };
|
|
3835
6484
|
}
|
|
3836
|
-
async function
|
|
6485
|
+
async function stat6(file) {
|
|
3837
6486
|
try {
|
|
3838
|
-
return await
|
|
6487
|
+
return await lstat4(file);
|
|
3839
6488
|
} catch (error) {
|
|
3840
6489
|
if (error.code === "ENOENT") return null;
|
|
3841
6490
|
throw error;
|
|
3842
6491
|
}
|
|
3843
6492
|
}
|
|
3844
6493
|
async function save(destination, shown, files, names) {
|
|
3845
|
-
const parent =
|
|
3846
|
-
const inside = (await
|
|
3847
|
-
if (!inside) await
|
|
3848
|
-
const lock = inside ?
|
|
6494
|
+
const parent = path10.dirname(destination), base = path10.basename(destination);
|
|
6495
|
+
const inside = (await stat6(destination))?.isDirectory() ?? false;
|
|
6496
|
+
if (!inside) await mkdir7(parent, { recursive: true });
|
|
6497
|
+
const lock = inside ? path10.join(destination, LOCK) : path10.join(parent, `.${base}.osmo-get.lock`);
|
|
3849
6498
|
try {
|
|
3850
|
-
await
|
|
6499
|
+
await mkdir7(lock);
|
|
3851
6500
|
} catch (error) {
|
|
3852
6501
|
if (error.code === "EEXIST")
|
|
3853
6502
|
fail(
|
|
@@ -3858,16 +6507,16 @@ async function save(destination, shown, files, names) {
|
|
|
3858
6507
|
}
|
|
3859
6508
|
let stage;
|
|
3860
6509
|
try {
|
|
3861
|
-
const existing = await
|
|
6510
|
+
const existing = await stat6(destination);
|
|
3862
6511
|
if (existing) {
|
|
3863
6512
|
if (!existing.isDirectory())
|
|
3864
6513
|
fail("resource_conflict", `${shown} must be a real folder.`);
|
|
3865
|
-
const entries = (await
|
|
6514
|
+
const entries = (await readdir4(destination)).filter((entry) => entry !== LOCK).sort();
|
|
3866
6515
|
if (entries.length) {
|
|
3867
6516
|
const same = entries.join(",") === names.join(",") && (await Promise.all(
|
|
3868
6517
|
names.map(async (name) => {
|
|
3869
|
-
const file =
|
|
3870
|
-
return (await
|
|
6518
|
+
const file = path10.join(destination, name);
|
|
6519
|
+
return (await stat6(file))?.isFile() && (await readFile6(file)).equals(Buffer.from(files[name]));
|
|
3871
6520
|
})
|
|
3872
6521
|
)).every(Boolean);
|
|
3873
6522
|
if (same) return "already_downloaded";
|
|
@@ -3876,21 +6525,21 @@ async function save(destination, shown, files, names) {
|
|
|
3876
6525
|
`${shown} already holds other files, which were left alone. Download somewhere else with --out DIR.`
|
|
3877
6526
|
);
|
|
3878
6527
|
}
|
|
3879
|
-
stage = await mkdtemp(
|
|
6528
|
+
stage = await mkdtemp(path10.join(destination, ".osmo-get-"));
|
|
3880
6529
|
for (const name of names)
|
|
3881
|
-
await
|
|
6530
|
+
await writeFile6(path10.join(stage, name), files[name], { flag: "wx" });
|
|
3882
6531
|
const copied = [];
|
|
3883
6532
|
try {
|
|
3884
6533
|
for (const name of names) {
|
|
3885
|
-
await
|
|
3886
|
-
|
|
3887
|
-
|
|
6534
|
+
await copyFile2(
|
|
6535
|
+
path10.join(stage, name),
|
|
6536
|
+
path10.join(destination, name),
|
|
3888
6537
|
constants.COPYFILE_EXCL
|
|
3889
6538
|
);
|
|
3890
|
-
copied.push(
|
|
6539
|
+
copied.push(path10.join(destination, name));
|
|
3891
6540
|
}
|
|
3892
6541
|
} catch (error) {
|
|
3893
|
-
for (const file of copied) await
|
|
6542
|
+
for (const file of copied) await rm7(file, { force: true });
|
|
3894
6543
|
if (error.code !== "EEXIST") throw error;
|
|
3895
6544
|
fail(
|
|
3896
6545
|
"resource_conflict",
|
|
@@ -3899,20 +6548,20 @@ async function save(destination, shown, files, names) {
|
|
|
3899
6548
|
}
|
|
3900
6549
|
return "downloaded";
|
|
3901
6550
|
}
|
|
3902
|
-
stage = await mkdtemp(
|
|
6551
|
+
stage = await mkdtemp(path10.join(parent, `.${base}-`));
|
|
3903
6552
|
for (const name of names)
|
|
3904
|
-
await
|
|
3905
|
-
if (await
|
|
6553
|
+
await writeFile6(path10.join(stage, name), files[name], { flag: "wx" });
|
|
6554
|
+
if (await stat6(destination))
|
|
3906
6555
|
fail(
|
|
3907
6556
|
"resource_conflict",
|
|
3908
6557
|
`${shown} appeared during the download. Existing files were left alone.`
|
|
3909
6558
|
);
|
|
3910
|
-
await
|
|
6559
|
+
await rename7(stage, destination);
|
|
3911
6560
|
stage = void 0;
|
|
3912
6561
|
return "downloaded";
|
|
3913
6562
|
} finally {
|
|
3914
|
-
if (stage) await
|
|
3915
|
-
await
|
|
6563
|
+
if (stage) await rm7(stage, { recursive: true, force: true });
|
|
6564
|
+
await rm7(lock, { recursive: true, force: true });
|
|
3916
6565
|
}
|
|
3917
6566
|
}
|
|
3918
6567
|
async function get(ref, flags, io) {
|
|
@@ -3948,7 +6597,7 @@ async function get(ref, flags, io) {
|
|
|
3948
6597
|
version: pkg.version,
|
|
3949
6598
|
scope: "global",
|
|
3950
6599
|
folder: store.directory,
|
|
3951
|
-
files: pkg.files.map((f) =>
|
|
6600
|
+
files: pkg.files.map((f) => path10.join(store.directory, f.path)),
|
|
3952
6601
|
presets: pkg.presets,
|
|
3953
6602
|
defaultPreset: pkg.defaultPreset,
|
|
3954
6603
|
usage: {},
|
|
@@ -3957,7 +6606,7 @@ async function get(ref, flags, io) {
|
|
|
3957
6606
|
guide: "Ready for offline use in every local project. Apply a film effect in Studio or use film({stock: ...}) in a scene."
|
|
3958
6607
|
};
|
|
3959
6608
|
}
|
|
3960
|
-
const destination =
|
|
6609
|
+
const destination = path10.resolve(flags.out ?? pkg.folder);
|
|
3961
6610
|
checkFiles(pkg);
|
|
3962
6611
|
const declared = pkg.files.reduce((sum, file) => sum + file.bytes, 0);
|
|
3963
6612
|
const zip = await responseBody(
|
|
@@ -3970,9 +6619,9 @@ async function get(ref, flags, io) {
|
|
|
3970
6619
|
);
|
|
3971
6620
|
const { files, manifest: manifest2, names } = unpack(zip, pkg);
|
|
3972
6621
|
const status = await save(destination, flags.out ?? pkg.folder, files, names);
|
|
3973
|
-
const relative =
|
|
6622
|
+
const relative = path10.relative(await realpath2(process.cwd()), await realpath2(destination)).split(path10.sep).join("/");
|
|
3974
6623
|
const folder = relative === "" ? "." : relative;
|
|
3975
|
-
const src =
|
|
6624
|
+
const src = path10.isAbsolute(folder) || folder.startsWith("../") || folder === "." ? folder : `./${folder}`;
|
|
3976
6625
|
return {
|
|
3977
6626
|
status,
|
|
3978
6627
|
id: detail.id,
|
|
@@ -4141,73 +6790,13 @@ ${r.guide}`;
|
|
|
4141
6790
|
}
|
|
4142
6791
|
|
|
4143
6792
|
// src/run.ts
|
|
4144
|
-
import { spawnSync as
|
|
4145
|
-
import { randomUUID as
|
|
4146
|
-
import { createWriteStream, existsSync as
|
|
4147
|
-
import { mkdir as
|
|
4148
|
-
import
|
|
4149
|
-
import { Readable } from "node:stream";
|
|
4150
|
-
import { pipeline } from "node:stream/promises";
|
|
4151
|
-
|
|
4152
|
-
// src/cloud-pointer.ts
|
|
4153
|
-
import { spawnSync } from "node:child_process";
|
|
4154
|
-
|
|
4155
|
-
// src/git.ts
|
|
4156
|
-
import {
|
|
4157
|
-
existsSync,
|
|
4158
|
-
mkdirSync,
|
|
4159
|
-
readFileSync,
|
|
4160
|
-
renameSync,
|
|
4161
|
-
rmSync,
|
|
4162
|
-
writeFileSync
|
|
4163
|
-
} from "node:fs";
|
|
4164
|
-
import path5 from "node:path";
|
|
4165
|
-
var hasRepo = (root) => existsSync(path5.join(root, ".git"));
|
|
4166
|
-
|
|
4167
|
-
// src/cloud-pointer.ts
|
|
4168
|
-
var PROJECT_ID = /^[a-z0-9]{10,40}$/;
|
|
4169
|
-
function pointerFromGit(root) {
|
|
4170
|
-
const read = (key) => {
|
|
4171
|
-
const r = spawnSync("git", ["-C", root, "config", "--get", key], {
|
|
4172
|
-
encoding: "utf8"
|
|
4173
|
-
});
|
|
4174
|
-
return r.status === 0 ? r.stdout.trim() : "";
|
|
4175
|
-
};
|
|
4176
|
-
const origin = read("osmo.cloud.origin"), id = read("osmo.cloud.id");
|
|
4177
|
-
return origin && id ? { origin, id } : null;
|
|
4178
|
-
}
|
|
4179
|
-
var cloudPointer = (root) => hasRepo(root) ? pointerFromGit(root) : null;
|
|
4180
|
-
|
|
4181
|
-
// src/cloud-request.ts
|
|
4182
|
-
var PERMISSION_DOES = {
|
|
4183
|
-
"projects:write": "change projects",
|
|
4184
|
-
"energy:spend": "run models on your plan"
|
|
4185
|
-
};
|
|
4186
|
-
function scopeRequired(origin, scope) {
|
|
4187
|
-
const login = origin === authOrigin() ? "osmo login" : `OSMO_BASE_URL=${origin} osmo login`;
|
|
4188
|
-
return new OsmoError(
|
|
4189
|
-
"scope_required",
|
|
4190
|
-
`This login can't ${PERMISSION_DOES[scope ?? ""] ?? "do that"}. Sign in again to allow it: ${login}`,
|
|
4191
|
-
login
|
|
4192
|
-
);
|
|
4193
|
-
}
|
|
4194
|
-
async function call(origin, route, { method = "GET", body: body2, timeoutMs = 6e4, allow = [] } = {}) {
|
|
4195
|
-
const response = await authenticatedFetch(
|
|
4196
|
-
route,
|
|
4197
|
-
{ method, body: body2, timeoutMs },
|
|
4198
|
-
origin
|
|
4199
|
-
);
|
|
4200
|
-
const data = await response.json().catch(() => ({}));
|
|
4201
|
-
if (response.status === 403 && data.error === "scope_required" && !allow.includes(403))
|
|
4202
|
-
throw scopeRequired(origin, data.scope);
|
|
4203
|
-
if (!response.ok && !allow.includes(response.status))
|
|
4204
|
-
throw new Error(
|
|
4205
|
-
`Cloud ${response.status}: ${typeof data.error === "string" ? data.error : response.status === 400 ? "the request was invalid" : "request failed"}`
|
|
4206
|
-
);
|
|
4207
|
-
return { status: response.status, ...data };
|
|
4208
|
-
}
|
|
4209
|
-
|
|
4210
|
-
// src/run.ts
|
|
6793
|
+
import { spawnSync as spawnSync4 } from "node:child_process";
|
|
6794
|
+
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
6795
|
+
import { createWriteStream as createWriteStream3, existsSync as existsSync4, openAsBlob, rmSync as rmSync2 } from "node:fs";
|
|
6796
|
+
import { mkdir as mkdir8, readFile as readFile7, rename as rename8, rm as rm8, stat as stat7, writeFile as writeFile7 } from "node:fs/promises";
|
|
6797
|
+
import path11 from "node:path";
|
|
6798
|
+
import { Readable as Readable3 } from "node:stream";
|
|
6799
|
+
import { pipeline as pipeline3 } from "node:stream/promises";
|
|
4211
6800
|
var WAIT_SECONDS = 60;
|
|
4212
6801
|
var REQUEST_MS = (WAIT_SECONDS + 30) * 1e3;
|
|
4213
6802
|
var READ_RETRIES = 4;
|
|
@@ -4486,7 +7075,7 @@ function fileSource(value) {
|
|
|
4486
7075
|
return { kind: "local", file: value };
|
|
4487
7076
|
}
|
|
4488
7077
|
function uploadType(name) {
|
|
4489
|
-
return UPLOAD_TYPES[
|
|
7078
|
+
return UPLOAD_TYPES[path11.extname(name).slice(1).toLowerCase()];
|
|
4490
7079
|
}
|
|
4491
7080
|
var megabytes = (n) => n < 1048576 ? `${Math.max(1, Math.round(n / 1024))} KB` : `${(n / 1048576).toFixed(1)} MB`;
|
|
4492
7081
|
async function resolveFiles(schema, input, issues, io) {
|
|
@@ -4516,18 +7105,18 @@ async function resolveFiles(schema, input, issues, io) {
|
|
|
4516
7105
|
for (const job of jobs) {
|
|
4517
7106
|
if (job.source.kind === "ours") job.set(job.source.url);
|
|
4518
7107
|
if (job.source.kind !== "local") continue;
|
|
4519
|
-
const info = await
|
|
7108
|
+
const info = await stat7(job.source.file).catch(() => null);
|
|
4520
7109
|
if (!info?.isFile())
|
|
4521
7110
|
issues.push({ path: job.path, message: `no file at ${job.source.file}` });
|
|
4522
7111
|
else if (!uploadType(job.source.file))
|
|
4523
7112
|
issues.push({
|
|
4524
7113
|
path: job.path,
|
|
4525
|
-
message: `${
|
|
7114
|
+
message: `${path11.basename(job.source.file)} isn't a type runs take (${Object.keys(UPLOAD_TYPES).join(", ")})`
|
|
4526
7115
|
});
|
|
4527
7116
|
else if (info.size > UPLOAD_MAX_BYTES || info.size === 0)
|
|
4528
7117
|
issues.push({
|
|
4529
7118
|
path: job.path,
|
|
4530
|
-
message: `${
|
|
7119
|
+
message: `${path11.basename(job.source.file)} must be 1 byte to ${megabytes(UPLOAD_MAX_BYTES)}`
|
|
4531
7120
|
});
|
|
4532
7121
|
}
|
|
4533
7122
|
return async () => {
|
|
@@ -4536,7 +7125,7 @@ async function resolveFiles(schema, input, issues, io) {
|
|
|
4536
7125
|
const { file } = job.source;
|
|
4537
7126
|
const type = uploadType(file);
|
|
4538
7127
|
const blob = await openAsBlob(file, { type });
|
|
4539
|
-
io.log(`uploading ${
|
|
7128
|
+
io.log(`uploading ${path11.basename(file)} (${megabytes(blob.size)})`);
|
|
4540
7129
|
job.set(await upload(blob, type));
|
|
4541
7130
|
} else if (job.source.kind === "web") {
|
|
4542
7131
|
io.log(`fetching ${job.source.url}`);
|
|
@@ -4618,8 +7207,8 @@ async function readInputsFile(file) {
|
|
|
4618
7207
|
let text2;
|
|
4619
7208
|
try {
|
|
4620
7209
|
text2 = file === "-" ? await new Response(
|
|
4621
|
-
|
|
4622
|
-
).text() : await
|
|
7210
|
+
Readable3.toWeb(process.stdin)
|
|
7211
|
+
).text() : await readFile7(file, "utf8");
|
|
4623
7212
|
} catch {
|
|
4624
7213
|
throw new OsmoError("invalid_input", `Can't read ${file}.`);
|
|
4625
7214
|
}
|
|
@@ -4724,14 +7313,14 @@ Next: ${next}`
|
|
|
4724
7313
|
stop();
|
|
4725
7314
|
}
|
|
4726
7315
|
}
|
|
4727
|
-
function
|
|
4728
|
-
for (let dir =
|
|
4729
|
-
if (
|
|
4730
|
-
if (
|
|
7316
|
+
function projectFolder2(start = process.cwd()) {
|
|
7317
|
+
for (let dir = path11.resolve(start); ; dir = path11.dirname(dir)) {
|
|
7318
|
+
if (existsSync4(path11.join(dir, ".osmo/project.json"))) return dir;
|
|
7319
|
+
if (path11.dirname(dir) === dir) return null;
|
|
4731
7320
|
}
|
|
4732
7321
|
}
|
|
4733
7322
|
function here() {
|
|
4734
|
-
const root =
|
|
7323
|
+
const root = projectFolder2();
|
|
4735
7324
|
if (!root) return null;
|
|
4736
7325
|
const pointer = cloudPointer(root);
|
|
4737
7326
|
let same = false;
|
|
@@ -4742,23 +7331,23 @@ function here() {
|
|
|
4742
7331
|
return same && pointer ? { root, project: pointer.id } : { root };
|
|
4743
7332
|
}
|
|
4744
7333
|
async function generationsFolder(root) {
|
|
4745
|
-
const ours =
|
|
4746
|
-
const listed = hasRepo(root) ?
|
|
7334
|
+
const ours = path11.join(root, "generations");
|
|
7335
|
+
const listed = hasRepo(root) ? spawnSync4("git", ["-C", root, "ls-files", "--", "generations"], {
|
|
4747
7336
|
encoding: "utf8"
|
|
4748
7337
|
}) : null;
|
|
4749
7338
|
const tracked = listed?.status === 0 && listed.stdout.trim() !== "";
|
|
4750
|
-
const theirs = tracked ||
|
|
4751
|
-
const folder = theirs ?
|
|
4752
|
-
await
|
|
4753
|
-
const ignore =
|
|
4754
|
-
if (!
|
|
7339
|
+
const theirs = tracked || existsSync4(ours) && !existsSync4(path11.join(ours, ".gitignore"));
|
|
7340
|
+
const folder = theirs ? path11.join(root, "osmo-generations") : ours;
|
|
7341
|
+
await mkdir8(folder, { recursive: true });
|
|
7342
|
+
const ignore = path11.join(folder, ".gitignore");
|
|
7343
|
+
if (!existsSync4(ignore)) await writeFile7(ignore, "*\n");
|
|
4755
7344
|
return folder;
|
|
4756
7345
|
}
|
|
4757
7346
|
async function targets(run, out2) {
|
|
4758
7347
|
const model = (run.model.split("/").pop() ?? "").replace(/[^A-Za-z0-9.-]/g, "-").replace(/^[.-]+/, "") || "output";
|
|
4759
7348
|
const name = `${model}-${run.job.slice(4, 12)}`;
|
|
4760
7349
|
const extension = (o) => {
|
|
4761
|
-
const fromUrl =
|
|
7350
|
+
const fromUrl = path11.extname(new URL(o.url).pathname).slice(1);
|
|
4762
7351
|
return EXTENSIONS[o.mime] ?? (/^[A-Za-z0-9]{1,5}$/.test(fromUrl) ? fromUrl : "bin");
|
|
4763
7352
|
};
|
|
4764
7353
|
const several = run.outputs.length > 1;
|
|
@@ -4770,20 +7359,20 @@ async function targets(run, out2) {
|
|
|
4770
7359
|
if (!project) return named;
|
|
4771
7360
|
const folder2 = await generationsFolder(project.root);
|
|
4772
7361
|
return named.map(
|
|
4773
|
-
(n) =>
|
|
7362
|
+
(n) => path11.relative(process.cwd(), path11.join(folder2, n)) || n
|
|
4774
7363
|
);
|
|
4775
7364
|
}
|
|
4776
|
-
const folder = /[/\\]$/.test(out2) || (await
|
|
7365
|
+
const folder = /[/\\]$/.test(out2) || (await stat7(out2).catch(() => null))?.isDirectory() || several && !path11.extname(out2);
|
|
4777
7366
|
if (!folder) {
|
|
4778
7367
|
if (!several)
|
|
4779
7368
|
return run.outputs.map(
|
|
4780
|
-
(o) =>
|
|
7369
|
+
(o) => path11.extname(out2) ? out2 : `${out2}.${extension(o)}`
|
|
4781
7370
|
);
|
|
4782
|
-
const stem = out2.slice(0, -
|
|
7371
|
+
const stem = out2.slice(0, -path11.extname(out2).length);
|
|
4783
7372
|
return run.outputs.map((o, i) => `${stem}-${i + 1}.${extension(o)}`);
|
|
4784
7373
|
}
|
|
4785
|
-
await
|
|
4786
|
-
return named.map((file) =>
|
|
7374
|
+
await mkdir8(out2, { recursive: true });
|
|
7375
|
+
return named.map((file) => path11.join(out2, file));
|
|
4787
7376
|
}
|
|
4788
7377
|
async function save2(run, out2, io) {
|
|
4789
7378
|
let partial;
|
|
@@ -4817,24 +7406,24 @@ Next: ${pickUp(run.job, out2)}`
|
|
|
4817
7406
|
gone ? `${run.job}'s outputs are no longer kept. Outputs last 30 days.` : `Couldn't download ${run.job}'s output${response ? ` (${response.status})` : ""}.`,
|
|
4818
7407
|
gone ? void 0 : pickUp(run.job, out2)
|
|
4819
7408
|
);
|
|
4820
|
-
await
|
|
7409
|
+
await mkdir8(path11.dirname(path11.resolve(file)), { recursive: true });
|
|
4821
7410
|
partial = `${file}.part-${process.pid}`;
|
|
4822
7411
|
try {
|
|
4823
|
-
await
|
|
4824
|
-
|
|
7412
|
+
await pipeline3(
|
|
7413
|
+
Readable3.fromWeb(
|
|
4825
7414
|
response.body
|
|
4826
7415
|
),
|
|
4827
|
-
|
|
7416
|
+
createWriteStream3(partial)
|
|
4828
7417
|
);
|
|
4829
|
-
await
|
|
7418
|
+
await rename8(partial, file);
|
|
4830
7419
|
} catch (error) {
|
|
4831
|
-
await
|
|
7420
|
+
await rm8(partial, { force: true });
|
|
4832
7421
|
throw error;
|
|
4833
7422
|
} finally {
|
|
4834
7423
|
partial = void 0;
|
|
4835
7424
|
}
|
|
4836
|
-
const bytes2 = (await
|
|
4837
|
-
const said =
|
|
7425
|
+
const bytes2 = (await stat7(file)).size;
|
|
7426
|
+
const said = path11.extname(file).slice(1).toLowerCase();
|
|
4838
7427
|
const made = EXTENSIONS[output.mime];
|
|
4839
7428
|
if (out2 !== void 0 && said && made && (said === "jpeg" ? "jpg" : said) !== made)
|
|
4840
7429
|
io.log(`${file} holds ${output.mime}, which its name doesn't say.`);
|
|
@@ -4923,7 +7512,7 @@ async function runCommand(id, flags, io) {
|
|
|
4923
7512
|
const schema = detail.run.inputSchema;
|
|
4924
7513
|
const base = flags.inputs ? await readInputsFile(flags.inputs) : {};
|
|
4925
7514
|
const { input, issues } = buildInputs(schema, flags.input ?? [], base);
|
|
4926
|
-
const wanted =
|
|
7515
|
+
const wanted = path11.extname(flags.out ?? "").slice(1).toLowerCase();
|
|
4927
7516
|
const formats = schema.properties?.format?.enum;
|
|
4928
7517
|
if (wanted && formats && input.format === void 0 && !issues.length) {
|
|
4929
7518
|
const format = wanted === "jpeg" ? "jpg" : wanted;
|
|
@@ -4956,7 +7545,7 @@ Osmo: ${error.message}
|
|
|
4956
7545
|
Next: ${error.next}`
|
|
4957
7546
|
);
|
|
4958
7547
|
});
|
|
4959
|
-
const key =
|
|
7548
|
+
const key = randomUUID4();
|
|
4960
7549
|
const linked = here()?.project;
|
|
4961
7550
|
const refusedFirst = (code) => ["runs_disabled", "model_unavailable"].includes(code ?? "");
|
|
4962
7551
|
let started;
|
|
@@ -5335,7 +7924,7 @@ async function unshareProject(projectId, origin) {
|
|
|
5335
7924
|
);
|
|
5336
7925
|
}
|
|
5337
7926
|
async function shareComments(value, {
|
|
5338
|
-
open = false,
|
|
7927
|
+
open: open3 = false,
|
|
5339
7928
|
origin = authOrigin()
|
|
5340
7929
|
} = {}) {
|
|
5341
7930
|
const share = await resolveShare(value, origin);
|
|
@@ -5347,7 +7936,7 @@ async function shareComments(value, {
|
|
|
5347
7936
|
`/api/projects/${share.id}/review-comments${cursor ? `?before=${encodeURIComponent(cursor)}` : ""}`
|
|
5348
7937
|
);
|
|
5349
7938
|
for (const { render, ...comment } of page.comments)
|
|
5350
|
-
if (!
|
|
7939
|
+
if (!open3 || comment.status === "open")
|
|
5351
7940
|
comments.push({
|
|
5352
7941
|
...comment,
|
|
5353
7942
|
version: render,
|
|
@@ -5565,8 +8154,8 @@ function formatUsage(result, now = Date.now()) {
|
|
|
5565
8154
|
|
|
5566
8155
|
// src/video-deploy.ts
|
|
5567
8156
|
import { execFile } from "node:child_process";
|
|
5568
|
-
import { stat as
|
|
5569
|
-
import
|
|
8157
|
+
import { stat as stat8 } from "node:fs/promises";
|
|
8158
|
+
import path12 from "node:path";
|
|
5570
8159
|
import { promisify } from "node:util";
|
|
5571
8160
|
|
|
5572
8161
|
// src/upload-render.ts
|
|
@@ -5671,7 +8260,7 @@ async function uploadRender(origin, project, file, metadata, log = () => {
|
|
|
5671
8260
|
// src/video-deploy.ts
|
|
5672
8261
|
var exec = promisify(execFile);
|
|
5673
8262
|
async function probeVideo(file) {
|
|
5674
|
-
const info = await
|
|
8263
|
+
const info = await stat8(file).catch(() => null);
|
|
5675
8264
|
if (!info?.isFile() || !info.size || info.size > 2 * 1024 ** 3)
|
|
5676
8265
|
throw new Error(
|
|
5677
8266
|
"Video must be an existing, non-empty file no larger than 2 GiB."
|
|
@@ -5735,9 +8324,9 @@ async function shareVideo(file, {
|
|
|
5735
8324
|
if (to !== void 0) parseShare(to, origin);
|
|
5736
8325
|
if (name !== void 0 && (!name.trim() || name.trim().length > 200))
|
|
5737
8326
|
throw new Error("--name takes 1 to 200 characters.");
|
|
5738
|
-
const source =
|
|
8327
|
+
const source = path12.resolve(file), metadata = await probeVideo(source);
|
|
5739
8328
|
await currentAccount(origin);
|
|
5740
|
-
const title = name?.trim() ??
|
|
8329
|
+
const title = name?.trim() ?? path12.basename(source).slice(0, 200);
|
|
5741
8330
|
const target = to === void 0 ? void 0 : await resolveShare(to);
|
|
5742
8331
|
if (target?.kind === "repo")
|
|
5743
8332
|
throw new OsmoError(
|
|
@@ -5780,14 +8369,14 @@ Retry with --to ${share} to add it to the same share.`,
|
|
|
5780
8369
|
}
|
|
5781
8370
|
|
|
5782
8371
|
// src/commands.ts
|
|
5783
|
-
var auth = (
|
|
5784
|
-
path:
|
|
8372
|
+
var auth = (path14, usage, about, args, flags) => ({
|
|
8373
|
+
path: path14,
|
|
5785
8374
|
group: "Account",
|
|
5786
8375
|
usage,
|
|
5787
8376
|
about,
|
|
5788
8377
|
args,
|
|
5789
8378
|
flags,
|
|
5790
|
-
run: (rest, flags2) => authCommand([...
|
|
8379
|
+
run: (rest, flags2) => authCommand([...path14, ...rest], flags2),
|
|
5791
8380
|
text: (result) => formatAuth(result),
|
|
5792
8381
|
failed: (result) => ["denied", "expired"].includes(result.status)
|
|
5793
8382
|
});
|
|
@@ -6030,15 +8619,16 @@ var COMMANDS = [
|
|
|
6030
8619
|
example: "osmo run osmo/seedance-2.5 --input prompt='slow push in' --input durationSeconds=5 --out shot.mp4",
|
|
6031
8620
|
run: (args, flags, io) => runCommand(args[0], flags, io),
|
|
6032
8621
|
text: formatRun,
|
|
6033
|
-
failed: (result) => result.status === "failed"
|
|
8622
|
+
failed: (result) => result.status === "failed" || "job" in result && result.status === "expired"
|
|
6034
8623
|
},
|
|
6035
8624
|
{
|
|
6036
8625
|
path: ["jobs"],
|
|
6037
8626
|
group: "Resources and models",
|
|
6038
|
-
usage: "jobs [JOB] [--out PATH] [--limit N] [--all]",
|
|
6039
|
-
about: "
|
|
8627
|
+
usage: "jobs [JOB] [--out PATH] [--limit N] [--all] [--cloud]",
|
|
8628
|
+
about: "With a cloud render ID, shows its current status and review link. --cloud lists recent cloud renders. For model runs, lists your recent runs, newest first: in a project linked to the cloud, that project's. With a JOB, waits for it and saves its outputs, which are kept for 30 days.",
|
|
6040
8629
|
args: [0, 1],
|
|
6041
8630
|
flags: {
|
|
8631
|
+
cloud: { type: "boolean", help: "List your 50 most recent cloud renders across projects." },
|
|
6042
8632
|
out: out(
|
|
6043
8633
|
"PATH",
|
|
6044
8634
|
"A file for one output, replacing one of that name, or a folder. A file name with no extension gets the output's. Default: MODEL-JOB.EXT in the project's generations/ (kept out of its versions), or here outside a project."
|
|
@@ -6060,6 +8650,8 @@ var COMMANDS = [
|
|
|
6060
8650
|
},
|
|
6061
8651
|
example: "osmo jobs run_abc12345",
|
|
6062
8652
|
check(args, flags) {
|
|
8653
|
+
if (flags.cloud && (args[0] || flags.out || flags.limit || flags.cursor || flags.all)) throw new UsageError("Use osmo jobs --cloud on its own to list cloud renders.", "osmo jobs --help");
|
|
8654
|
+
if (args[0] && isCloudRenderJob(args[0]) && flags.out) throw new UsageError("Cloud renders publish to the review link; --out applies to model runs.", "osmo jobs --help");
|
|
6063
8655
|
if (args[0] === "list")
|
|
6064
8656
|
throw new UsageError(
|
|
6065
8657
|
"There is no osmo jobs list. osmo jobs lists your runs.",
|
|
@@ -6091,19 +8683,19 @@ var COMMANDS = [
|
|
|
6091
8683
|
"osmo jobs --help"
|
|
6092
8684
|
);
|
|
6093
8685
|
},
|
|
6094
|
-
run: (args, flags, io) => jobsCommand(args[0], flags, io),
|
|
6095
|
-
text: (result) => "runs" in result ? formatJobs(result) : formatRun(result),
|
|
6096
|
-
failed: (result) => result.status === "failed"
|
|
8686
|
+
run: (args, flags, io) => flags.cloud ? listCloudRenderJobs() : args[0] && isCloudRenderJob(args[0]) ? cloudRenderJob(args[0]) : jobsCommand(args[0], flags, io),
|
|
8687
|
+
text: (result) => "jobs" in result ? result.jobs.map(formatCloudRender).join("\n\n") || "No cloud renders yet." : "job" in result && isCloudRenderJob(String(result.job)) ? formatCloudRender(result) : "runs" in result ? formatJobs(result) : formatRun(result),
|
|
8688
|
+
failed: (result) => result.status === "failed" || "job" in result && result.status === "expired"
|
|
6097
8689
|
},
|
|
6098
8690
|
{
|
|
6099
8691
|
path: ["jobs", "cancel"],
|
|
6100
8692
|
group: "Resources and models",
|
|
6101
8693
|
usage: "jobs cancel JOB",
|
|
6102
|
-
about: "Cancels a run
|
|
8694
|
+
about: "Cancels a queued model run or a cloud render. Queued cancellation is free. A cloud render already processing is charged for elapsed processing. Running model generations cannot be stopped.",
|
|
6103
8695
|
args: [1, 1],
|
|
6104
8696
|
example: "osmo jobs cancel run_abc12345",
|
|
6105
|
-
run: (args) => cancelCommand(args[0]),
|
|
6106
|
-
text: formatCancel
|
|
8697
|
+
run: (args) => isCloudRenderJob(args[0]) ? cloudRenderJob(args[0], void 0, true) : cancelCommand(args[0]),
|
|
8698
|
+
text: (result) => "job" in result && isCloudRenderJob(String(result.job)) ? formatCloudRender(result) : formatCancel(result)
|
|
6107
8699
|
},
|
|
6108
8700
|
{
|
|
6109
8701
|
path: ["share"],
|
|
@@ -6267,10 +8859,18 @@ Version: ${r.versionLink}`;
|
|
|
6267
8859
|
),
|
|
6268
8860
|
studio(
|
|
6269
8861
|
"clone",
|
|
6270
|
-
"PROJECT [FOLDER]",
|
|
6271
|
-
"Copies a cloud project and its history into a new folder. Legacy web-editor projects download as text exports with asset URLs and require conversion before local editing. PROJECT is an ID or project/review link. FOLDER defaults to the project name, or legacy-<id> for legacy exports. Needs osmo login and source access.",
|
|
8862
|
+
"PROJECT [FOLDER] [--no-media]",
|
|
8863
|
+
"Copies a cloud project and its history into a new folder. Legacy web-editor projects download as text exports with asset URLs and require conversion before local editing. PROJECT is an ID or project/review link. FOLDER defaults to the project name, or legacy-<id> for legacy exports. A project that points to footage also gets those clips' proxies (osmo media pull). Needs osmo login and source access.",
|
|
6272
8864
|
[1, 2],
|
|
6273
|
-
{
|
|
8865
|
+
{
|
|
8866
|
+
example: "osmo studio clone https://studio.osmo.inc/app/projects/ID",
|
|
8867
|
+
flags: {
|
|
8868
|
+
"no-media": {
|
|
8869
|
+
type: "boolean",
|
|
8870
|
+
help: "Don't fetch the footage the project points to."
|
|
8871
|
+
}
|
|
8872
|
+
}
|
|
8873
|
+
}
|
|
6274
8874
|
),
|
|
6275
8875
|
studio(
|
|
6276
8876
|
"sync",
|
|
@@ -6355,11 +8955,13 @@ Version: ${r.versionLink}`;
|
|
|
6355
8955
|
),
|
|
6356
8956
|
studio(
|
|
6357
8957
|
"deploy",
|
|
6358
|
-
"[SCENE] [--overwrite]",
|
|
8958
|
+
"[SCENE] [--overwrite] [--cloud] [--wait]",
|
|
6359
8959
|
"Deploys the project for review: sends this folder's version, renders the scene, and adds the video as a new version on the project's review link, where people comment on moments, areas and layers. The first run makes the cloud project, named after the folder (osmo studio sync --name NAME first, to name it yourself) and private: sourceAccess in the result is who can open the source. The review link is separate, and opens for anyone who has it. A deploy posts only what this folder holds: when the cloud has edits the folder has not seen (someone else's, from the web or another folder), it stops before rendering, lists them under arrived, and leaves the folder as it was. Then run osmo studio sync to bring them in and look at them, and deploy again. Needs osmo login.",
|
|
6360
8960
|
[0, 1],
|
|
6361
8961
|
{
|
|
6362
8962
|
flags: {
|
|
8963
|
+
cloud: { type: "boolean", help: "Render on Osmo GPU workers using paid allowance or credits. Returns a job ID; no local GPU is needed." },
|
|
8964
|
+
wait: { type: "boolean", help: "With --cloud, wait for completion. Disconnecting leaves the job running." },
|
|
6363
8965
|
overwrite: {
|
|
6364
8966
|
type: "boolean",
|
|
6365
8967
|
help: "Put this folder's version over cloud edits it has not seen, as one new revision on top of them; theirs stay in history. Only when that is what the user wants. It refuses while someone is editing in the web."
|
|
@@ -6368,7 +8970,7 @@ Version: ${r.versionLink}`;
|
|
|
6368
8970
|
removed: {
|
|
6369
8971
|
"--access": "a project starts private, and osmo studio permissions set --access LEVEL changes it."
|
|
6370
8972
|
},
|
|
6371
|
-
failed: (result) =>
|
|
8973
|
+
failed: (result) => !["deployed", "queued", "starting", "running", "uploading", "succeeded"].includes(String(result.status))
|
|
6372
8974
|
}
|
|
6373
8975
|
),
|
|
6374
8976
|
studio(
|
|
@@ -6455,7 +9057,138 @@ Version: ${r.versionLink}`;
|
|
|
6455
9057
|
"EMAIL",
|
|
6456
9058
|
"Stops sharing the project with someone, whether it was shared here or on the web. Access they have through a workspace, or through the access level, stays.",
|
|
6457
9059
|
[1, 1]
|
|
6458
|
-
)
|
|
9060
|
+
),
|
|
9061
|
+
{
|
|
9062
|
+
path: ["media", "upload"],
|
|
9063
|
+
group: "Media",
|
|
9064
|
+
usage: "media upload PATH... [--project DIR] [--proxies-only] [--no-proxy] [--jobs N] [--encode-jobs N] [--dry-run]",
|
|
9065
|
+
about: "Uploads camera footage: files or whole cards and folders. Each clip's proxy is made and checked on this machine and uploaded first, so cloud work can start, then the original, linked to it. Every part is checksummed end to end. Re-run the same command (or any command over the same files) to resume after a crash, a network drop or a remount. With --project, each clip is also placed in the project's osmo-media/ folder: a pointer, which sync sends, and the proxy beside it, which stays out of git. Like other uploaded media, anyone with a file's link can download it.",
|
|
9066
|
+
args: [1, Number.POSITIVE_INFINITY],
|
|
9067
|
+
flags: {
|
|
9068
|
+
project: {
|
|
9069
|
+
type: "string",
|
|
9070
|
+
value: "DIR",
|
|
9071
|
+
help: "Also place the clips in this synced Studio project's folder."
|
|
9072
|
+
},
|
|
9073
|
+
"proxies-only": {
|
|
9074
|
+
type: "boolean",
|
|
9075
|
+
help: "Upload the proxies only; the originals can follow later with the same command."
|
|
9076
|
+
},
|
|
9077
|
+
"no-proxy": {
|
|
9078
|
+
type: "boolean",
|
|
9079
|
+
help: "Upload originals only, with no proxies."
|
|
9080
|
+
},
|
|
9081
|
+
jobs: {
|
|
9082
|
+
type: "string",
|
|
9083
|
+
value: "N",
|
|
9084
|
+
help: "Parts uploaded at once, 8 to 16 (default 12)."
|
|
9085
|
+
},
|
|
9086
|
+
"encode-jobs": {
|
|
9087
|
+
type: "string",
|
|
9088
|
+
value: "N",
|
|
9089
|
+
help: "Proxies made at once (default 2 with VideoToolbox, else 1)."
|
|
9090
|
+
},
|
|
9091
|
+
"dry-run": {
|
|
9092
|
+
type: "boolean",
|
|
9093
|
+
help: "Plan only: what would be proxied and uploaded."
|
|
9094
|
+
}
|
|
9095
|
+
},
|
|
9096
|
+
example: "osmo media upload /Volumes/CARD_A --project .",
|
|
9097
|
+
check(_args, flags) {
|
|
9098
|
+
if (flags["proxies-only"] && flags["no-proxy"])
|
|
9099
|
+
throw new UsageError(
|
|
9100
|
+
"--proxies-only and --no-proxy can't both be given.",
|
|
9101
|
+
"osmo media upload --help"
|
|
9102
|
+
);
|
|
9103
|
+
const jobs = Number(flags.jobs ?? 12);
|
|
9104
|
+
if (!Number.isInteger(jobs) || jobs < 8 || jobs > 16)
|
|
9105
|
+
throw new UsageError(
|
|
9106
|
+
"--jobs takes a whole number from 8 to 16.",
|
|
9107
|
+
"osmo media upload --help"
|
|
9108
|
+
);
|
|
9109
|
+
const encode = Number(flags["encode-jobs"] ?? 1);
|
|
9110
|
+
if (!Number.isInteger(encode) || encode < 1 || encode > 8)
|
|
9111
|
+
throw new UsageError(
|
|
9112
|
+
"--encode-jobs takes a whole number from 1 to 8.",
|
|
9113
|
+
"osmo media upload --help"
|
|
9114
|
+
);
|
|
9115
|
+
},
|
|
9116
|
+
run: (args, flags, io) => mediaUpload(args, flags, io),
|
|
9117
|
+
text: formatUpload,
|
|
9118
|
+
failed: (result) => result.status === "incomplete"
|
|
9119
|
+
},
|
|
9120
|
+
{
|
|
9121
|
+
path: ["media", "status"],
|
|
9122
|
+
group: "Media",
|
|
9123
|
+
usage: "media status [PATH...]",
|
|
9124
|
+
about: "Shows what this machine has uploaded, or is part way through uploading: every file, or those under the paths given.",
|
|
9125
|
+
args: [0, Number.POSITIVE_INFINITY],
|
|
9126
|
+
run: (args) => mediaStatus(args),
|
|
9127
|
+
text: formatStatus
|
|
9128
|
+
},
|
|
9129
|
+
{
|
|
9130
|
+
path: ["media", "ls"],
|
|
9131
|
+
group: "Media",
|
|
9132
|
+
usage: "media ls",
|
|
9133
|
+
about: "Lists your uploaded footage.",
|
|
9134
|
+
args: [0, 0],
|
|
9135
|
+
run: () => mediaList(),
|
|
9136
|
+
text: formatList
|
|
9137
|
+
},
|
|
9138
|
+
{
|
|
9139
|
+
path: ["media", "add"],
|
|
9140
|
+
group: "Media",
|
|
9141
|
+
usage: "media add ID... --project DIR",
|
|
9142
|
+
about: "Places your uploaded clips in a project folder's osmo-media/: a pointer each, which sync sends, and its proxy, fetched here. To stop using a clip, delete its pointer.",
|
|
9143
|
+
args: [1, 100],
|
|
9144
|
+
flags: {
|
|
9145
|
+
project: {
|
|
9146
|
+
type: "string",
|
|
9147
|
+
value: "DIR",
|
|
9148
|
+
help: "The synced Studio project's folder."
|
|
9149
|
+
}
|
|
9150
|
+
},
|
|
9151
|
+
example: "osmo media add ID --project .",
|
|
9152
|
+
run: (args, flags) => mediaAdd(args, flags),
|
|
9153
|
+
text: formatAdd
|
|
9154
|
+
},
|
|
9155
|
+
{
|
|
9156
|
+
path: ["media", "pull"],
|
|
9157
|
+
group: "Media",
|
|
9158
|
+
usage: "media pull [--project DIR]",
|
|
9159
|
+
about: "Fetches the proxy of every clip the project points to (its osmo-media/*.osmo-media files) that is missing or differs here.",
|
|
9160
|
+
args: [0, 0],
|
|
9161
|
+
flags: {
|
|
9162
|
+
project: {
|
|
9163
|
+
type: "string",
|
|
9164
|
+
value: "DIR",
|
|
9165
|
+
help: "The project's folder (default: this one)."
|
|
9166
|
+
}
|
|
9167
|
+
},
|
|
9168
|
+
run: (_args, flags) => mediaPull(flags),
|
|
9169
|
+
text: formatPull,
|
|
9170
|
+
failed: (result) => result.status === "incomplete"
|
|
9171
|
+
},
|
|
9172
|
+
{
|
|
9173
|
+
path: ["media", "get"],
|
|
9174
|
+
group: "Media",
|
|
9175
|
+
usage: "media get ID [-o PATH] [--original] [--force]",
|
|
9176
|
+
about: "Downloads one of your clips, checking it against its checksum. For an original, downloads its proxy unless --original is given. Never replaces a file already at the path unless --force is given.",
|
|
9177
|
+
args: [1, 1],
|
|
9178
|
+
flags: {
|
|
9179
|
+
out: out("PATH", "Where to save it (default: its own name, here)."),
|
|
9180
|
+
original: {
|
|
9181
|
+
type: "boolean",
|
|
9182
|
+
help: "The camera original, not its proxy."
|
|
9183
|
+
},
|
|
9184
|
+
force: {
|
|
9185
|
+
type: "boolean",
|
|
9186
|
+
help: "Replace a file already at the path."
|
|
9187
|
+
}
|
|
9188
|
+
},
|
|
9189
|
+
run: ([id], flags) => mediaGet(id, flags),
|
|
9190
|
+
text: formatGet
|
|
9191
|
+
}
|
|
6459
9192
|
];
|
|
6460
9193
|
var STUDIO_NOW = {
|
|
6461
9194
|
"studio publish": "studio sync",
|
|
@@ -6473,9 +9206,9 @@ var GLOBAL_FLAGS = {
|
|
|
6473
9206
|
function findCommand(words2) {
|
|
6474
9207
|
let found;
|
|
6475
9208
|
for (const command of COMMANDS)
|
|
6476
|
-
for (const
|
|
6477
|
-
if (
|
|
6478
|
-
found = { command, length:
|
|
9209
|
+
for (const path14 of [command.path, ...command.aliases ?? []])
|
|
9210
|
+
if (path14.length <= words2.length && path14.every((word, i) => words2[i] === word) && path14.length > (found?.length ?? 0))
|
|
9211
|
+
found = { command, length: path14.length };
|
|
6479
9212
|
return found && { command: found.command, rest: words2.slice(found.length) };
|
|
6480
9213
|
}
|
|
6481
9214
|
function commandHelp(command) {
|
|
@@ -6487,7 +9220,7 @@ function commandHelp(command) {
|
|
|
6487
9220
|
);
|
|
6488
9221
|
const width = Math.max(...names.map((name) => name.length));
|
|
6489
9222
|
const aliases = (command.aliases ?? []).map(
|
|
6490
|
-
(
|
|
9223
|
+
(path14) => `osmo ${path14.join(" ")}`
|
|
6491
9224
|
);
|
|
6492
9225
|
return [
|
|
6493
9226
|
`osmo ${command.usage}`,
|
|
@@ -6667,10 +9400,10 @@ async function loadStudio() {
|
|
|
6667
9400
|
manifest2 = createRequire(import.meta.url).resolve(
|
|
6668
9401
|
`@osmo.inc/cli-${platform}/package.json`
|
|
6669
9402
|
);
|
|
6670
|
-
entry =
|
|
9403
|
+
entry = path13.join(path13.dirname(manifest2), "dist/studio.js");
|
|
6671
9404
|
} catch {
|
|
6672
9405
|
}
|
|
6673
|
-
if (!
|
|
9406
|
+
if (!existsSync5(entry))
|
|
6674
9407
|
throw new OsmoError(
|
|
6675
9408
|
"studio_unavailable",
|
|
6676
9409
|
PLATFORMS.includes(platform) ? `Osmo Studio and its renderer aren't installed: @osmo.inc/cli-${platform} is missing. Reinstall @osmo.inc/cli with optional dependencies; if that doesn't add it, this release has no package for ${platform}.` : `Osmo Studio's renderer isn't available for ${platform} yet. It runs on ${PLATFORMS.join(", ")}.`,
|