@mieweb/pulsevault 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +216 -1
- package/OPERATIONS.md +135 -44
- package/PROTOCOL.md +63 -4
- package/README.md +186 -34
- package/dist/app.d.ts +30 -5
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +24 -5
- package/dist/app.js.map +1 -1
- package/dist/augment.d.ts +3 -0
- package/dist/augment.d.ts.map +1 -1
- package/dist/augment.js.map +1 -1
- package/dist/core.d.ts +107 -8
- package/dist/core.d.ts.map +1 -1
- package/dist/core.js +381 -47
- package/dist/core.js.map +1 -1
- package/dist/lib/authorize.d.ts +28 -6
- package/dist/lib/authorize.d.ts.map +1 -1
- package/dist/lib/capability-token.d.ts +26 -0
- package/dist/lib/capability-token.d.ts.map +1 -1
- package/dist/lib/capability-token.js +73 -5
- package/dist/lib/capability-token.js.map +1 -1
- package/dist/lib/completion.d.ts +138 -0
- package/dist/lib/completion.d.ts.map +1 -0
- package/dist/lib/completion.js +375 -0
- package/dist/lib/completion.js.map +1 -0
- package/dist/lib/deeplinks.d.ts +18 -2
- package/dist/lib/deeplinks.d.ts.map +1 -1
- package/dist/lib/deeplinks.js +13 -2
- package/dist/lib/deeplinks.js.map +1 -1
- package/dist/lib/magic.d.ts +41 -1
- package/dist/lib/magic.d.ts.map +1 -1
- package/dist/lib/magic.js +104 -0
- package/dist/lib/magic.js.map +1 -1
- package/dist/lib/options.d.ts +2 -0
- package/dist/lib/options.d.ts.map +1 -1
- package/dist/lib/options.js +8 -2
- package/dist/lib/options.js.map +1 -1
- package/dist/lib/protocol-version.gen.js +1 -1
- package/dist/lib/pulsevaultTus.d.ts +28 -26
- package/dist/lib/pulsevaultTus.d.ts.map +1 -1
- package/dist/lib/pulsevaultTus.js +149 -32
- package/dist/lib/pulsevaultTus.js.map +1 -1
- package/dist/lib/web-ready.d.ts +138 -18
- package/dist/lib/web-ready.d.ts.map +1 -1
- package/dist/lib/web-ready.js +453 -91
- package/dist/lib/web-ready.js.map +1 -1
- package/dist/routes/pulsevault.d.ts +12 -1
- package/dist/routes/pulsevault.d.ts.map +1 -1
- package/dist/routes/pulsevault.js +92 -1
- package/dist/routes/pulsevault.js.map +1 -1
- package/dist/storage/local.d.ts +7 -1
- package/dist/storage/local.d.ts.map +1 -1
- package/dist/storage/local.js +408 -52
- package/dist/storage/local.js.map +1 -1
- package/dist/storage/s3.d.ts +7 -1
- package/dist/storage/s3.d.ts.map +1 -1
- package/dist/storage/s3.js +256 -14
- package/dist/storage/s3.js.map +1 -1
- package/dist/storage/types.d.ts +126 -0
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/storage/types.js +7 -0
- package/dist/storage/types.js.map +1 -1
- package/package.json +2 -2
- package/protocol/openapi.json +270 -1
- package/protocol/schemas/artifact-status.schema.json +61 -0
- package/protocol/schemas/capability-token.schema.json +3 -0
- package/protocol/schemas/deep-link.schema.json +2 -2
- package/scripts/web-ready-migrate.mjs +35 -11
package/dist/storage/local.js
CHANGED
|
@@ -1,11 +1,21 @@
|
|
|
1
1
|
import { FileStore } from '@tus/file-store';
|
|
2
|
+
import { ERRORS, Upload } from '@tus/server';
|
|
2
3
|
import fs from 'node:fs/promises';
|
|
3
4
|
import path from 'node:path';
|
|
5
|
+
import { randomUUID } from 'node:crypto';
|
|
4
6
|
import { isUuid } from '../lib/uuid.js';
|
|
7
|
+
import { EXTENSION_REGEX } from '../lib/options.js';
|
|
8
|
+
import { CONFORM_TARGET } from '../lib/web-ready.js';
|
|
5
9
|
import { parseUploadKind } from './types.js';
|
|
6
10
|
const SIDECAR_VERSION = 1;
|
|
7
11
|
/** Hidden directory inside workspaceRoot that holds per-upload sidecar files. */
|
|
8
12
|
const PULSEVAULT_META_DIR = '.pulsevault';
|
|
13
|
+
/**
|
|
14
|
+
* Relation index under the metadata directory: `related/<anchorId>/<artifactId>` is an empty
|
|
15
|
+
* marker for every artifact that declared `relatedTo` the anchor, so a pulse's files are one
|
|
16
|
+
* `readdir` away instead of a scan of every sidecar.
|
|
17
|
+
*/
|
|
18
|
+
const RELATED_DIR = 'related';
|
|
9
19
|
/** Default cap on the in-memory metadata cache before evicting the oldest entry. */
|
|
10
20
|
const DEFAULT_META_CACHE_LIMIT = 10_000;
|
|
11
21
|
/** Map a file extension to the `Content-Type` the GET route should return. */
|
|
@@ -17,6 +27,23 @@ function extToContentType(ext) {
|
|
|
17
27
|
return 'application/zip';
|
|
18
28
|
case '.vtt':
|
|
19
29
|
return 'text/vtt';
|
|
30
|
+
case '.mov':
|
|
31
|
+
return 'video/quicktime';
|
|
32
|
+
case '.m4v':
|
|
33
|
+
return 'video/x-m4v';
|
|
34
|
+
case '.webm':
|
|
35
|
+
return 'video/webm';
|
|
36
|
+
case '.mkv':
|
|
37
|
+
return 'video/x-matroska';
|
|
38
|
+
case '.3gp':
|
|
39
|
+
return 'video/3gpp';
|
|
40
|
+
case '.avi':
|
|
41
|
+
return 'video/x-msvideo';
|
|
42
|
+
case '.srt':
|
|
43
|
+
return 'application/x-subrip';
|
|
44
|
+
case '.pulse':
|
|
45
|
+
// The beat manifest (PROTOCOL.md §8) is JSON.
|
|
46
|
+
return 'application/json';
|
|
20
47
|
case '.jpg':
|
|
21
48
|
case '.jpeg':
|
|
22
49
|
return 'image/jpeg';
|
|
@@ -30,6 +57,7 @@ function extToContentType(ext) {
|
|
|
30
57
|
function sidecarToCachedMeta(sidecar, ready) {
|
|
31
58
|
return {
|
|
32
59
|
ext: sidecar.ext,
|
|
60
|
+
...(sidecar.sourceExt ? { sourceExt: sidecar.sourceExt } : {}),
|
|
33
61
|
ready,
|
|
34
62
|
kind: sidecar.kind ?? 'video',
|
|
35
63
|
relatedTo: sidecar.relatedTo,
|
|
@@ -37,9 +65,71 @@ function sidecarToCachedMeta(sidecar, ready) {
|
|
|
37
65
|
name: sidecar.name,
|
|
38
66
|
};
|
|
39
67
|
}
|
|
68
|
+
/** A recorded web-ready result read back from a sidecar. */
|
|
69
|
+
function isWebReadyResult(value) {
|
|
70
|
+
const result = value;
|
|
71
|
+
return (typeof result === 'object' &&
|
|
72
|
+
result !== null &&
|
|
73
|
+
typeof result.action === 'string' &&
|
|
74
|
+
typeof result.reason === 'string');
|
|
75
|
+
}
|
|
40
76
|
export function createLocalStorage(opts) {
|
|
41
77
|
const workspaceRoot = path.resolve(opts.workspaceDir);
|
|
42
|
-
|
|
78
|
+
/**
|
|
79
|
+
* Whether the tus upload `uploadId` (`<kind>/<artifactId><ext>`) finished and a conversion then
|
|
80
|
+
* replaced its file: the sidecar is ready, names this upload's extension as `sourceExt`, and the
|
|
81
|
+
* file is gone. tus still has its record of the upload (`<uploadId>.json`).
|
|
82
|
+
*/
|
|
83
|
+
const replacedByConversion = async (uploadId) => {
|
|
84
|
+
const slash = uploadId.indexOf('/');
|
|
85
|
+
const kind = uploadId.slice(0, slash);
|
|
86
|
+
const artifactId = uploadId.slice(slash + 1, slash + 37);
|
|
87
|
+
const ext = uploadId.slice(slash + 37);
|
|
88
|
+
if (slash < 0 || !isUuid(artifactId) || !EXTENSION_REGEX.test(ext))
|
|
89
|
+
return false;
|
|
90
|
+
const sidecar = await readSidecar(artifactId);
|
|
91
|
+
if (!sidecar || sidecar.status !== 'ready' || (sidecar.kind ?? 'video') !== kind)
|
|
92
|
+
return false;
|
|
93
|
+
if (sidecar.sourceExt !== ext || sidecar.ext === ext)
|
|
94
|
+
return false;
|
|
95
|
+
return (await storedExt(artifactId, sidecar.kind ?? 'video', ext)) !== ext;
|
|
96
|
+
};
|
|
97
|
+
/**
|
|
98
|
+
* tus's store for the workspace, which also answers for an upload a conversion replaced (a
|
|
99
|
+
* finished `.webm` conformed to `<id>.mp4`): tus would answer a client re-sending the final
|
|
100
|
+
* `PATCH` whose response it lost with `410`, the file being gone. Such an upload reads as
|
|
101
|
+
* complete, and its (empty) tail is taken without a write, so the retry reaches
|
|
102
|
+
* `onUploadFinish`'s already-finished branch and gets `204`, as one without a conversion does.
|
|
103
|
+
*/
|
|
104
|
+
class LocalFileStore extends FileStore {
|
|
105
|
+
async getUpload(id) {
|
|
106
|
+
try {
|
|
107
|
+
return await super.getUpload(id);
|
|
108
|
+
}
|
|
109
|
+
catch (err) {
|
|
110
|
+
if (err !== ERRORS.FILE_NO_LONGER_EXISTS || !(await replacedByConversion(id)))
|
|
111
|
+
throw err;
|
|
112
|
+
const record = await this.configstore.get(id);
|
|
113
|
+
if (!record || typeof record.size !== 'number')
|
|
114
|
+
throw err;
|
|
115
|
+
return new Upload({
|
|
116
|
+
id,
|
|
117
|
+
size: record.size,
|
|
118
|
+
offset: record.size,
|
|
119
|
+
metadata: record.metadata,
|
|
120
|
+
creation_date: record.creation_date,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
async write(readable, id, offset) {
|
|
125
|
+
if (!(await replacedByConversion(id)))
|
|
126
|
+
return super.write(readable, id, offset);
|
|
127
|
+
for await (const _ of readable)
|
|
128
|
+
; // a finished upload: nothing more to store
|
|
129
|
+
return offset;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
const datastore = new LocalFileStore({ directory: workspaceRoot });
|
|
43
133
|
const metaCacheLimit = opts.metaCacheLimit ?? DEFAULT_META_CACHE_LIMIT;
|
|
44
134
|
// Metadata cache keyed by artifactId. Populated eagerly on reserve and
|
|
45
135
|
// lazily from the sidecar on cache-miss — so we never do a workspace-wide
|
|
@@ -80,14 +170,78 @@ export function createLocalStorage(opts) {
|
|
|
80
170
|
};
|
|
81
171
|
/** Relative path (from workspaceRoot) to the artifact bytes. */
|
|
82
172
|
const artifactRelPath = (artifactId, kind, ext) => `${kind}/${artifactId}${ext}`;
|
|
173
|
+
/** The artifact's bytes file, strictly inside its kind directory, as `sidecarPath` checks. */
|
|
174
|
+
const storedPath = (artifactId, kind, ext) => {
|
|
175
|
+
const base = path.resolve(workspaceRoot, kind);
|
|
176
|
+
const resolved = path.resolve(base, `${artifactId}${ext}`);
|
|
177
|
+
if (!resolved.startsWith(`${base}${path.sep}`)) {
|
|
178
|
+
throw new Error('artifactId escapes the kind directory');
|
|
179
|
+
}
|
|
180
|
+
return resolved;
|
|
181
|
+
};
|
|
182
|
+
/**
|
|
183
|
+
* The extension of the file that holds an artifact's bytes, given the `ext` its sidecar names:
|
|
184
|
+
* that file, or for a video the conform target (`<id>.mp4`) when only that exists. A sidecar
|
|
185
|
+
* another instance wrote from a copy older than a conversion still names the uploaded file,
|
|
186
|
+
* which the conversion deleted; its bytes are in the converted file. `null` when neither exists.
|
|
187
|
+
*/
|
|
188
|
+
const storedExt = async (artifactId, kind, ext) => {
|
|
189
|
+
// Only a real id becomes a path, as everywhere else in this adapter.
|
|
190
|
+
if (!isUuid(artifactId) || !EXTENSION_REGEX.test(ext))
|
|
191
|
+
return null;
|
|
192
|
+
const candidates = kind === 'video' && ext !== CONFORM_TARGET.extension ? [ext, CONFORM_TARGET.extension] : [ext];
|
|
193
|
+
for (const candidate of candidates) {
|
|
194
|
+
try {
|
|
195
|
+
const stat = await fs.stat(storedPath(artifactId, kind, candidate));
|
|
196
|
+
if (stat.isFile())
|
|
197
|
+
return candidate;
|
|
198
|
+
}
|
|
199
|
+
catch (err) {
|
|
200
|
+
if (err?.code !== 'ENOENT')
|
|
201
|
+
throw err;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return null;
|
|
205
|
+
};
|
|
83
206
|
const writeSidecar = async (artifactId, sidecar) => {
|
|
84
207
|
// Atomic tmp + rename so a crash mid-write can never leave a truncated
|
|
85
|
-
// JSON blob that `loadMeta` would then treat as corrupt.
|
|
208
|
+
// JSON blob that `loadMeta` would then treat as corrupt. The tmp name is
|
|
209
|
+
// unique, so two writers (two instances on a shared filesystem) never
|
|
210
|
+
// write into, or rename away, each other's tmp file.
|
|
86
211
|
const finalPath = sidecarPath(artifactId);
|
|
87
|
-
const tmpPath = `${finalPath}.tmp`;
|
|
212
|
+
const tmpPath = `${finalPath}.${randomUUID()}.tmp`;
|
|
88
213
|
await fs.mkdir(sidecarDir(), { recursive: true, mode: 0o750 });
|
|
89
|
-
|
|
90
|
-
|
|
214
|
+
try {
|
|
215
|
+
await fs.writeFile(tmpPath, JSON.stringify(sidecar), 'utf8');
|
|
216
|
+
await fs.rename(tmpPath, finalPath);
|
|
217
|
+
}
|
|
218
|
+
catch (err) {
|
|
219
|
+
await fs.rm(tmpPath, { force: true });
|
|
220
|
+
throw err;
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
/**
|
|
224
|
+
* Read-modify-write of a sidecar runs one at a time per artifact in this process, so an
|
|
225
|
+
* acknowledgement, a conversion switching the stored file and a removal each act on the sidecar
|
|
226
|
+
* as the last of them left it. Across instances sharing the workspace (OPERATIONS.md,
|
|
227
|
+
* "Horizontal scaling") there is no lock: the last rename wins. The completion bookkeeping
|
|
228
|
+
* recovers: the replay redoes a lost acknowledgement or conversion record, idempotently, and a
|
|
229
|
+
* sidecar written from a copy older than a conversion still finds the bytes: a conversion's file
|
|
230
|
+
* is always `<id>.mp4`, which `resolve` and `getLocalPath` fall back to (`storedExt`). A lost
|
|
231
|
+
* `outcome` isn't redone; nothing records it a second time.
|
|
232
|
+
*/
|
|
233
|
+
const sidecarWrites = new Map();
|
|
234
|
+
const withSidecarLock = async (artifactId, work) => {
|
|
235
|
+
const previous = sidecarWrites.get(artifactId) ?? Promise.resolve();
|
|
236
|
+
const run = previous.catch(() => { }).then(work);
|
|
237
|
+
sidecarWrites.set(artifactId, run);
|
|
238
|
+
try {
|
|
239
|
+
return await run;
|
|
240
|
+
}
|
|
241
|
+
finally {
|
|
242
|
+
if (sidecarWrites.get(artifactId) === run)
|
|
243
|
+
sidecarWrites.delete(artifactId);
|
|
244
|
+
}
|
|
91
245
|
};
|
|
92
246
|
const readSidecar = async (artifactId) => {
|
|
93
247
|
// Every sidecar/artifact path in this module is built by joining
|
|
@@ -126,6 +280,9 @@ export function createLocalStorage(opts) {
|
|
|
126
280
|
return {
|
|
127
281
|
version: SIDECAR_VERSION,
|
|
128
282
|
ext: parsed.ext,
|
|
283
|
+
...(typeof parsed.sourceExt === 'string' && EXTENSION_REGEX.test(parsed.sourceExt)
|
|
284
|
+
? { sourceExt: parsed.sourceExt }
|
|
285
|
+
: {}),
|
|
129
286
|
filename: parsed.filename,
|
|
130
287
|
status,
|
|
131
288
|
kind,
|
|
@@ -133,6 +290,15 @@ export function createLocalStorage(opts) {
|
|
|
133
290
|
checksum: typeof parsed.checksum === 'string' ? parsed.checksum : undefined,
|
|
134
291
|
name: typeof parsed.name === 'string' ? parsed.name : undefined,
|
|
135
292
|
appVersion: typeof parsed.appVersion === 'string' ? parsed.appVersion : undefined,
|
|
293
|
+
...(parsed.context !== undefined ? { context: parsed.context } : {}),
|
|
294
|
+
// A sidecar from before the flag existed: finished means nothing to replay; still
|
|
295
|
+
// uploading means its completion hasn't happened yet.
|
|
296
|
+
acknowledged: parsed.acknowledged ?? status === 'ready',
|
|
297
|
+
converted: parsed.converted ?? status === 'ready',
|
|
298
|
+
...(isWebReadyResult(parsed.webReady) ? { webReady: parsed.webReady } : {}),
|
|
299
|
+
...(typeof parsed.generation === 'string' ? { generation: parsed.generation } : {}),
|
|
300
|
+
...(parsed.outcome !== undefined ? { outcome: parsed.outcome } : {}),
|
|
301
|
+
...(typeof parsed.readyAt === 'number' ? { readyAt: parsed.readyAt } : {}),
|
|
136
302
|
};
|
|
137
303
|
}
|
|
138
304
|
catch {
|
|
@@ -141,6 +307,52 @@ export function createLocalStorage(opts) {
|
|
|
141
307
|
return null;
|
|
142
308
|
}
|
|
143
309
|
};
|
|
310
|
+
/** Directory of relation markers for one anchor artifact. */
|
|
311
|
+
const relatedDir = (anchorId) => {
|
|
312
|
+
const base = path.resolve(sidecarDir(), RELATED_DIR);
|
|
313
|
+
const resolved = path.resolve(base, anchorId);
|
|
314
|
+
if (!resolved.startsWith(`${base}${path.sep}`)) {
|
|
315
|
+
throw new Error('artifactId escapes the metadata directory');
|
|
316
|
+
}
|
|
317
|
+
return resolved;
|
|
318
|
+
};
|
|
319
|
+
/** When the upload last moved: the sidecar's mtime, or the bytes file's while still uploading. */
|
|
320
|
+
const lastActivity = async (artifactId, sidecar) => {
|
|
321
|
+
let updatedAt;
|
|
322
|
+
try {
|
|
323
|
+
updatedAt = (await fs.stat(sidecarPath(artifactId))).mtimeMs;
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
return 0;
|
|
327
|
+
}
|
|
328
|
+
if (sidecar.status === 'uploading') {
|
|
329
|
+
const kind = sidecar.kind ?? 'video';
|
|
330
|
+
const bytes = path.join(workspaceRoot, kind, `${artifactId}${sidecar.ext}`);
|
|
331
|
+
const written = await fs.stat(bytes).then((stats) => stats.mtimeMs, () => 0);
|
|
332
|
+
updatedAt = Math.max(updatedAt, written);
|
|
333
|
+
}
|
|
334
|
+
return updatedAt;
|
|
335
|
+
};
|
|
336
|
+
const sidecarToMeta = (artifactId, sidecar, updatedAt) => ({
|
|
337
|
+
artifactId,
|
|
338
|
+
kind: sidecar.kind ?? 'video',
|
|
339
|
+
ext: sidecar.ext,
|
|
340
|
+
...(sidecar.sourceExt ? { sourceExt: sidecar.sourceExt } : {}),
|
|
341
|
+
filename: sidecar.filename,
|
|
342
|
+
...(sidecar.relatedTo ? { relatedTo: sidecar.relatedTo } : {}),
|
|
343
|
+
...(sidecar.checksum ? { checksum: sidecar.checksum } : {}),
|
|
344
|
+
...(sidecar.name ? { name: sidecar.name } : {}),
|
|
345
|
+
...(sidecar.appVersion ? { appVersion: sidecar.appVersion } : {}),
|
|
346
|
+
...(sidecar.context !== undefined ? { context: sidecar.context } : {}),
|
|
347
|
+
ready: sidecar.status === 'ready',
|
|
348
|
+
acknowledged: sidecar.acknowledged !== false,
|
|
349
|
+
converted: sidecar.converted !== false,
|
|
350
|
+
...(sidecar.webReady ? { webReady: sidecar.webReady } : {}),
|
|
351
|
+
...(sidecar.generation ? { generation: sidecar.generation } : {}),
|
|
352
|
+
...(sidecar.outcome !== undefined ? { outcome: sidecar.outcome } : {}),
|
|
353
|
+
updatedAt,
|
|
354
|
+
...(sidecar.readyAt !== undefined ? { readyAt: sidecar.readyAt } : {}),
|
|
355
|
+
});
|
|
144
356
|
const loadMeta = async (artifactId) => {
|
|
145
357
|
const cached = metaCache.get(artifactId);
|
|
146
358
|
if (cached)
|
|
@@ -161,7 +373,7 @@ export function createLocalStorage(opts) {
|
|
|
161
373
|
// tighten it so the upload tree isn't world-writable under a permissive umask.
|
|
162
374
|
await fs.chmod(workspaceRoot, 0o750).catch(() => { });
|
|
163
375
|
};
|
|
164
|
-
const reserveUpload = async ({ artifactId, filename, ext, kind, relatedTo, checksum, name, appVersion, }) => {
|
|
376
|
+
const reserveUpload = async ({ artifactId, filename, ext, kind, relatedTo, checksum, name, appVersion, context, }) => {
|
|
165
377
|
await fs.mkdir(path.join(workspaceRoot, kind), { recursive: true, mode: 0o750 });
|
|
166
378
|
await fs.mkdir(sidecarDir(), { recursive: true, mode: 0o750 });
|
|
167
379
|
const sidecar = {
|
|
@@ -174,61 +386,75 @@ export function createLocalStorage(opts) {
|
|
|
174
386
|
checksum,
|
|
175
387
|
name,
|
|
176
388
|
appVersion,
|
|
389
|
+
...(context !== undefined ? { context } : {}),
|
|
390
|
+
acknowledged: false,
|
|
391
|
+
converted: false,
|
|
392
|
+
generation: randomUUID(),
|
|
177
393
|
};
|
|
178
|
-
//
|
|
179
|
-
//
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
throw err;
|
|
190
|
-
// A file already exists at this path, but `readSidecar` treats a malformed/corrupt
|
|
191
|
-
// one as absent (e.g. debris from a crash mid-write) — re-check before deciding this
|
|
192
|
-
// is a genuine collision rather than debris that's safe to overwrite.
|
|
193
|
-
const existing = await readSidecar(artifactId);
|
|
194
|
-
if (existing) {
|
|
195
|
-
throw Object.assign(new Error(`artifactId ${artifactId} already has an upload`), {
|
|
196
|
-
statusCode: 409,
|
|
197
|
-
status_code: 409,
|
|
198
|
-
});
|
|
394
|
+
// Under the sidecar lock, so a removal of the same id in this process either finished
|
|
395
|
+
// (its sidecar went last) or hasn't started.
|
|
396
|
+
await withSidecarLock(artifactId, async () => {
|
|
397
|
+
// Collision guard: `wx` fails atomically with EEXIST if a sidecar already exists for
|
|
398
|
+
// this artifactId, rather than the previous read-then-write (`loadMeta` then
|
|
399
|
+
// `writeSidecar`) which left a window for two concurrent/retried requests to both pass
|
|
400
|
+
// the check before either had written — letting the second silently clobber the first's
|
|
401
|
+
// sidecar and race @tus/file-store's own offset tracking. Translates to HTTP 409 via
|
|
402
|
+
// @tus/server's error path, same as before.
|
|
403
|
+
try {
|
|
404
|
+
await fs.writeFile(sidecarPath(artifactId), JSON.stringify(sidecar), { flag: 'wx' });
|
|
199
405
|
}
|
|
200
|
-
|
|
406
|
+
catch (err) {
|
|
407
|
+
if (err?.code !== 'EEXIST')
|
|
408
|
+
throw err;
|
|
409
|
+
// A file already exists at this path, but `readSidecar` treats a malformed/corrupt
|
|
410
|
+
// one as absent (e.g. debris from a crash mid-write) — re-check before deciding this
|
|
411
|
+
// is a genuine collision rather than debris that's safe to overwrite.
|
|
412
|
+
const existing = await readSidecar(artifactId);
|
|
413
|
+
if (existing) {
|
|
414
|
+
throw Object.assign(new Error(`artifactId ${artifactId} already has an upload`), {
|
|
415
|
+
statusCode: 409,
|
|
416
|
+
status_code: 409,
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
await writeSidecar(artifactId, sidecar);
|
|
420
|
+
}
|
|
421
|
+
});
|
|
422
|
+
if (relatedTo) {
|
|
423
|
+
// The relation index entry. Written after the sidecar so a crash between the two leaves
|
|
424
|
+
// an artifact without an index entry (it still works, it's just not listed under its
|
|
425
|
+
// anchor), never an index entry without an artifact.
|
|
426
|
+
const dir = relatedDir(relatedTo);
|
|
427
|
+
await fs.mkdir(dir, { recursive: true, mode: 0o750 });
|
|
428
|
+
await fs.writeFile(path.join(dir, artifactId), '', { flag: 'w' });
|
|
201
429
|
}
|
|
202
430
|
cacheSet(artifactId, { ext, ready: false, kind, relatedTo, checksum, name });
|
|
203
431
|
// @tus/file-store joins this onto its configured `directory`, so the
|
|
204
432
|
// actual file lands at `<workspaceRoot>/<kind>/<artifactId><ext>`.
|
|
205
433
|
return artifactRelPath(artifactId, kind, ext);
|
|
206
434
|
};
|
|
207
|
-
const resolve = async (artifactId) => {
|
|
435
|
+
const resolve = async (artifactId, reread = false) => {
|
|
208
436
|
const meta = await loadMeta(artifactId);
|
|
209
437
|
// Only serve ready uploads. In-progress uploads stay hidden — a client
|
|
210
438
|
// GETting mid-upload would otherwise receive a truncated file.
|
|
211
439
|
if (!meta || !meta.ready)
|
|
212
440
|
return null;
|
|
213
|
-
const
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
}
|
|
219
|
-
catch (err) {
|
|
220
|
-
if (err?.code === 'ENOENT')
|
|
441
|
+
const ext = await storedExt(artifactId, meta.kind, meta.ext);
|
|
442
|
+
if (ext === null) {
|
|
443
|
+
// The cache may be older than another instance's removal and new upload of the id: read
|
|
444
|
+
// the sidecar once more.
|
|
445
|
+
if (reread)
|
|
221
446
|
return null;
|
|
222
|
-
|
|
447
|
+
metaCache.delete(artifactId);
|
|
448
|
+
return resolve(artifactId, true);
|
|
223
449
|
}
|
|
224
450
|
return {
|
|
225
451
|
kind: 'stream',
|
|
226
452
|
root: workspaceRoot,
|
|
227
|
-
filename:
|
|
228
|
-
contentType: extToContentType(
|
|
453
|
+
filename: artifactRelPath(artifactId, meta.kind, ext),
|
|
454
|
+
contentType: extToContentType(ext),
|
|
229
455
|
};
|
|
230
456
|
};
|
|
231
|
-
const markReady = async (
|
|
457
|
+
const markReady = (artifactId) => withSidecarLock(artifactId, async () => {
|
|
232
458
|
const sidecar = await readSidecar(artifactId);
|
|
233
459
|
if (!sidecar) {
|
|
234
460
|
// No sidecar means no `reserveUpload` happened for this artifactId —
|
|
@@ -240,29 +466,72 @@ export function createLocalStorage(opts) {
|
|
|
240
466
|
cacheSet(artifactId, sidecarToCachedMeta(sidecar, true));
|
|
241
467
|
return;
|
|
242
468
|
}
|
|
243
|
-
|
|
469
|
+
// An upload reserved before generations were kept gets one now, so its conversion runs.
|
|
470
|
+
await writeSidecar(artifactId, {
|
|
471
|
+
...sidecar,
|
|
472
|
+
status: 'ready',
|
|
473
|
+
readyAt: Date.now(),
|
|
474
|
+
generation: sidecar.generation ?? randomUUID(),
|
|
475
|
+
});
|
|
244
476
|
cacheSet(artifactId, sidecarToCachedMeta(sidecar, true));
|
|
245
|
-
};
|
|
246
|
-
|
|
247
|
-
|
|
477
|
+
});
|
|
478
|
+
// Under the sidecar lock, from the sidecar itself: in this process, a conversion switching the
|
|
479
|
+
// artifact to a new file (`patchArtifact` with `ext`) either finishes first — and this removes
|
|
480
|
+
// the new file — or finds the sidecar gone and drops what it wrote.
|
|
481
|
+
const remove = (artifactId) => withSidecarLock(artifactId, async () => {
|
|
482
|
+
// The sidecar decides, never the cache: a read racing this removal (or an earlier one) can
|
|
483
|
+
// have cached the artifact again, and a removal that trusted it would report the same
|
|
484
|
+
// artifact removed twice.
|
|
485
|
+
const sidecar = await readSidecar(artifactId);
|
|
248
486
|
// Drop from cache before rm so a racing `resolve` arriving after the
|
|
249
487
|
// rm but before cache eviction can't hand back a stale path.
|
|
250
488
|
metaCache.delete(artifactId);
|
|
251
|
-
if (!
|
|
489
|
+
if (!sidecar)
|
|
252
490
|
return false;
|
|
491
|
+
const meta = sidecarToCachedMeta(sidecar, sidecar.status === 'ready');
|
|
253
492
|
const artifactPath = path.join(workspaceRoot, meta.kind, `${artifactId}${meta.ext}`);
|
|
493
|
+
// The uploaded file and tus's record of it, when a conversion stored the bytes under a new
|
|
494
|
+
// extension (normally already gone; left behind by a crash during the switch).
|
|
495
|
+
const sourcePath = meta.sourceExt
|
|
496
|
+
? path.join(workspaceRoot, meta.kind, `${artifactId}${meta.sourceExt}`)
|
|
497
|
+
: null;
|
|
498
|
+
// A video's conversion output, whatever this sidecar's copy says (`storedExt`).
|
|
499
|
+
const conformedPath = meta.kind === 'video' && meta.ext !== CONFORM_TARGET.extension
|
|
500
|
+
? path.join(workspaceRoot, artifactRelPath(artifactId, meta.kind, CONFORM_TARGET.extension))
|
|
501
|
+
: null;
|
|
254
502
|
await Promise.all([
|
|
255
503
|
fs.rm(artifactPath, { force: true }),
|
|
256
504
|
fs.rm(`${artifactPath}.json`, { force: true }),
|
|
257
|
-
|
|
505
|
+
...(sourcePath
|
|
506
|
+
? [fs.rm(sourcePath, { force: true }), fs.rm(`${sourcePath}.json`, { force: true })]
|
|
507
|
+
: []),
|
|
508
|
+
...(conformedPath ? [fs.rm(conformedPath, { force: true })] : []),
|
|
509
|
+
...(meta.relatedTo
|
|
510
|
+
? [fs.rm(path.join(relatedDir(meta.relatedTo), artifactId), { force: true })]
|
|
511
|
+
: []),
|
|
258
512
|
]);
|
|
513
|
+
// Last: until the sidecar is gone the id can't be reserved again, so a new upload of it
|
|
514
|
+
// never has its bytes removed by this one's removal.
|
|
515
|
+
await fs.rm(sidecarPath(artifactId), { force: true });
|
|
516
|
+
// Again: a read during the removal may have cached it from the sidecar it still saw.
|
|
517
|
+
metaCache.delete(artifactId);
|
|
259
518
|
return true;
|
|
260
|
-
};
|
|
519
|
+
});
|
|
520
|
+
/**
|
|
521
|
+
* From the sidecar, not the cache: another instance on the same workspace may have removed
|
|
522
|
+
* the artifact and reserved its id again (a new extension) since this one cached it, and a
|
|
523
|
+
* caller about to read or rewrite the bytes must get today's file.
|
|
524
|
+
*/
|
|
261
525
|
const getLocalPath = async (artifactId) => {
|
|
262
|
-
const
|
|
263
|
-
if (!
|
|
526
|
+
const sidecar = await readSidecar(artifactId);
|
|
527
|
+
if (!sidecar) {
|
|
528
|
+
metaCache.delete(artifactId);
|
|
264
529
|
return null;
|
|
265
|
-
|
|
530
|
+
}
|
|
531
|
+
cacheSet(artifactId, sidecarToCachedMeta(sidecar, sidecar.status === 'ready'));
|
|
532
|
+
const kind = sidecar.kind ?? 'video';
|
|
533
|
+
const ext = (await storedExt(artifactId, kind, sidecar.ext)) ?? sidecar.ext;
|
|
534
|
+
return path.join(workspaceRoot, artifactRelPath(artifactId, kind, ext));
|
|
266
535
|
};
|
|
267
536
|
const getKind = async (artifactId) => {
|
|
268
537
|
const meta = await loadMeta(artifactId);
|
|
@@ -280,6 +549,90 @@ export function createLocalStorage(opts) {
|
|
|
280
549
|
const meta = await loadMeta(artifactId);
|
|
281
550
|
return meta?.name ?? null;
|
|
282
551
|
};
|
|
552
|
+
/** Straight from the sidecar, never the cache: its flags change after the cache was filled. */
|
|
553
|
+
const describeArtifact = async (artifactId) => {
|
|
554
|
+
const sidecar = await readSidecar(artifactId);
|
|
555
|
+
if (!sidecar)
|
|
556
|
+
return null;
|
|
557
|
+
return sidecarToMeta(artifactId, sidecar, await lastActivity(artifactId, sidecar));
|
|
558
|
+
};
|
|
559
|
+
const patchArtifact = (artifactId, patch) => withSidecarLock(artifactId, async () => {
|
|
560
|
+
if (patch.ext !== undefined && !EXTENSION_REGEX.test(patch.ext)) {
|
|
561
|
+
throw new TypeError(`patchArtifact: invalid ext ${JSON.stringify(patch.ext)}`);
|
|
562
|
+
}
|
|
563
|
+
const sidecar = await readSidecar(artifactId);
|
|
564
|
+
if (!sidecar)
|
|
565
|
+
return false;
|
|
566
|
+
// Another pass already recorded its conversion, or the id was removed and reserved again:
|
|
567
|
+
// this patch belongs to neither.
|
|
568
|
+
if (patch.unlessConverted && sidecar.converted !== false)
|
|
569
|
+
return false;
|
|
570
|
+
if (patch.generation !== undefined && (sidecar.generation ?? null) !== patch.generation)
|
|
571
|
+
return false;
|
|
572
|
+
const kind = sidecar.kind ?? 'video';
|
|
573
|
+
const stored = (ext) => path.join(workspaceRoot, kind, `${artifactId}${ext}`);
|
|
574
|
+
const ext = patch.ext ?? sidecar.ext;
|
|
575
|
+
if (patch.file !== undefined) {
|
|
576
|
+
if (path.dirname(path.resolve(patch.file)) !== path.join(workspaceRoot, kind)) {
|
|
577
|
+
throw new TypeError('patchArtifact: `file` must be in the artifact\'s kind directory');
|
|
578
|
+
}
|
|
579
|
+
await fs.rename(patch.file, stored(ext));
|
|
580
|
+
}
|
|
581
|
+
const next = { ...sidecar };
|
|
582
|
+
if (patch.acknowledged !== undefined)
|
|
583
|
+
next.acknowledged = patch.acknowledged;
|
|
584
|
+
if (patch.converted !== undefined)
|
|
585
|
+
next.converted = patch.converted;
|
|
586
|
+
if (patch.webReady !== undefined)
|
|
587
|
+
next.webReady = patch.webReady;
|
|
588
|
+
if (patch.ext !== undefined && patch.ext !== sidecar.ext) {
|
|
589
|
+
next.sourceExt = sidecar.sourceExt ?? sidecar.ext;
|
|
590
|
+
next.ext = patch.ext;
|
|
591
|
+
}
|
|
592
|
+
if (patch.outcome !== undefined) {
|
|
593
|
+
if (patch.outcome === null)
|
|
594
|
+
delete next.outcome;
|
|
595
|
+
else
|
|
596
|
+
next.outcome = patch.outcome;
|
|
597
|
+
}
|
|
598
|
+
await writeSidecar(artifactId, next);
|
|
599
|
+
// The cache holds nothing a patch changes, but a `ready` flip elsewhere must not be undone
|
|
600
|
+
// by a stale entry either — keep it consistent with what was just written.
|
|
601
|
+
cacheSet(artifactId, sidecarToCachedMeta(next, next.status === 'ready'));
|
|
602
|
+
// The sidecar names the new file: the one under the old extension goes. (A crash before
|
|
603
|
+
// this leaves it for `remove`, which deletes `sourceExt` too. A sidecar copy that still
|
|
604
|
+
// names it finds the new file instead: `storedExt`.)
|
|
605
|
+
if (ext !== sidecar.ext)
|
|
606
|
+
await fs.rm(stored(sidecar.ext), { force: true });
|
|
607
|
+
return true;
|
|
608
|
+
});
|
|
609
|
+
async function* listRelated(artifactId) {
|
|
610
|
+
if (!isUuid(artifactId))
|
|
611
|
+
return;
|
|
612
|
+
let names;
|
|
613
|
+
try {
|
|
614
|
+
names = await fs.readdir(relatedDir(artifactId));
|
|
615
|
+
}
|
|
616
|
+
catch (err) {
|
|
617
|
+
if (err?.code === 'ENOENT')
|
|
618
|
+
return;
|
|
619
|
+
throw err;
|
|
620
|
+
}
|
|
621
|
+
for (const relatedId of names) {
|
|
622
|
+
if (!isUuid(relatedId))
|
|
623
|
+
continue;
|
|
624
|
+
const sidecar = await readSidecar(relatedId);
|
|
625
|
+
if (!sidecar || sidecar.relatedTo !== artifactId)
|
|
626
|
+
continue; // A marker whose artifact is gone.
|
|
627
|
+
yield {
|
|
628
|
+
artifactId: relatedId,
|
|
629
|
+
kind: sidecar.kind ?? 'video',
|
|
630
|
+
relatedTo: artifactId,
|
|
631
|
+
ready: sidecar.status === 'ready',
|
|
632
|
+
updatedAt: await lastActivity(relatedId, sidecar),
|
|
633
|
+
};
|
|
634
|
+
}
|
|
635
|
+
}
|
|
283
636
|
/**
|
|
284
637
|
* Walk the sidecars. A sidecar is rewritten only when its upload finishes, so its mtime is
|
|
285
638
|
* when the upload started (still uploading) or when it finished (ready). An upload in flight
|
|
@@ -336,7 +689,7 @@ export function createLocalStorage(opts) {
|
|
|
336
689
|
workspaceRoot,
|
|
337
690
|
initialize,
|
|
338
691
|
reserveUpload,
|
|
339
|
-
resolve,
|
|
692
|
+
resolve: (artifactId) => resolve(artifactId),
|
|
340
693
|
markReady,
|
|
341
694
|
remove,
|
|
342
695
|
getLocalPath,
|
|
@@ -345,6 +698,9 @@ export function createLocalStorage(opts) {
|
|
|
345
698
|
getChecksum,
|
|
346
699
|
getName,
|
|
347
700
|
listArtifacts,
|
|
701
|
+
describeArtifact,
|
|
702
|
+
patchArtifact,
|
|
703
|
+
listRelated,
|
|
348
704
|
};
|
|
349
705
|
}
|
|
350
706
|
//# sourceMappingURL=local.js.map
|