@mieweb/pulsevault 0.4.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +216 -1
  2. package/OPERATIONS.md +135 -44
  3. package/PROTOCOL.md +63 -4
  4. package/README.md +186 -34
  5. package/dist/app.d.ts +30 -5
  6. package/dist/app.d.ts.map +1 -1
  7. package/dist/app.js +24 -5
  8. package/dist/app.js.map +1 -1
  9. package/dist/augment.d.ts +3 -0
  10. package/dist/augment.d.ts.map +1 -1
  11. package/dist/augment.js.map +1 -1
  12. package/dist/core.d.ts +107 -8
  13. package/dist/core.d.ts.map +1 -1
  14. package/dist/core.js +381 -47
  15. package/dist/core.js.map +1 -1
  16. package/dist/lib/authorize.d.ts +28 -6
  17. package/dist/lib/authorize.d.ts.map +1 -1
  18. package/dist/lib/capability-token.d.ts +26 -0
  19. package/dist/lib/capability-token.d.ts.map +1 -1
  20. package/dist/lib/capability-token.js +73 -5
  21. package/dist/lib/capability-token.js.map +1 -1
  22. package/dist/lib/completion.d.ts +138 -0
  23. package/dist/lib/completion.d.ts.map +1 -0
  24. package/dist/lib/completion.js +375 -0
  25. package/dist/lib/completion.js.map +1 -0
  26. package/dist/lib/deeplinks.d.ts +18 -2
  27. package/dist/lib/deeplinks.d.ts.map +1 -1
  28. package/dist/lib/deeplinks.js +13 -2
  29. package/dist/lib/deeplinks.js.map +1 -1
  30. package/dist/lib/magic.d.ts +41 -1
  31. package/dist/lib/magic.d.ts.map +1 -1
  32. package/dist/lib/magic.js +104 -0
  33. package/dist/lib/magic.js.map +1 -1
  34. package/dist/lib/options.d.ts +2 -0
  35. package/dist/lib/options.d.ts.map +1 -1
  36. package/dist/lib/options.js +8 -2
  37. package/dist/lib/options.js.map +1 -1
  38. package/dist/lib/protocol-version.gen.js +1 -1
  39. package/dist/lib/pulsevaultTus.d.ts +28 -26
  40. package/dist/lib/pulsevaultTus.d.ts.map +1 -1
  41. package/dist/lib/pulsevaultTus.js +149 -32
  42. package/dist/lib/pulsevaultTus.js.map +1 -1
  43. package/dist/lib/web-ready.d.ts +138 -18
  44. package/dist/lib/web-ready.d.ts.map +1 -1
  45. package/dist/lib/web-ready.js +453 -91
  46. package/dist/lib/web-ready.js.map +1 -1
  47. package/dist/routes/pulsevault.d.ts +12 -1
  48. package/dist/routes/pulsevault.d.ts.map +1 -1
  49. package/dist/routes/pulsevault.js +92 -1
  50. package/dist/routes/pulsevault.js.map +1 -1
  51. package/dist/storage/local.d.ts +7 -1
  52. package/dist/storage/local.d.ts.map +1 -1
  53. package/dist/storage/local.js +408 -52
  54. package/dist/storage/local.js.map +1 -1
  55. package/dist/storage/s3.d.ts +7 -1
  56. package/dist/storage/s3.d.ts.map +1 -1
  57. package/dist/storage/s3.js +256 -14
  58. package/dist/storage/s3.js.map +1 -1
  59. package/dist/storage/types.d.ts +126 -0
  60. package/dist/storage/types.d.ts.map +1 -1
  61. package/dist/storage/types.js +7 -0
  62. package/dist/storage/types.js.map +1 -1
  63. package/package.json +2 -2
  64. package/protocol/openapi.json +270 -1
  65. package/protocol/schemas/artifact-status.schema.json +61 -0
  66. package/protocol/schemas/capability-token.schema.json +3 -0
  67. package/protocol/schemas/deep-link.schema.json +2 -2
  68. package/scripts/web-ready-migrate.mjs +35 -11
@@ -1,11 +1,21 @@
1
1
  import { FileStore } from '@tus/file-store';
2
+ import { ERRORS, Upload } from '@tus/server';
2
3
  import fs from 'node:fs/promises';
3
4
  import path from 'node:path';
5
+ import { randomUUID } from 'node:crypto';
4
6
  import { isUuid } from '../lib/uuid.js';
7
+ import { EXTENSION_REGEX } from '../lib/options.js';
8
+ import { CONFORM_TARGET } from '../lib/web-ready.js';
5
9
  import { parseUploadKind } from './types.js';
6
10
  const SIDECAR_VERSION = 1;
7
11
  /** Hidden directory inside workspaceRoot that holds per-upload sidecar files. */
8
12
  const PULSEVAULT_META_DIR = '.pulsevault';
13
+ /**
14
+ * Relation index under the metadata directory: `related/<anchorId>/<artifactId>` is an empty
15
+ * marker for every artifact that declared `relatedTo` the anchor, so a pulse's files are one
16
+ * `readdir` away instead of a scan of every sidecar.
17
+ */
18
+ const RELATED_DIR = 'related';
9
19
  /** Default cap on the in-memory metadata cache before evicting the oldest entry. */
10
20
  const DEFAULT_META_CACHE_LIMIT = 10_000;
11
21
  /** Map a file extension to the `Content-Type` the GET route should return. */
@@ -17,6 +27,23 @@ function extToContentType(ext) {
17
27
  return 'application/zip';
18
28
  case '.vtt':
19
29
  return 'text/vtt';
30
+ case '.mov':
31
+ return 'video/quicktime';
32
+ case '.m4v':
33
+ return 'video/x-m4v';
34
+ case '.webm':
35
+ return 'video/webm';
36
+ case '.mkv':
37
+ return 'video/x-matroska';
38
+ case '.3gp':
39
+ return 'video/3gpp';
40
+ case '.avi':
41
+ return 'video/x-msvideo';
42
+ case '.srt':
43
+ return 'application/x-subrip';
44
+ case '.pulse':
45
+ // The beat manifest (PROTOCOL.md §8) is JSON.
46
+ return 'application/json';
20
47
  case '.jpg':
21
48
  case '.jpeg':
22
49
  return 'image/jpeg';
@@ -30,6 +57,7 @@ function extToContentType(ext) {
30
57
  function sidecarToCachedMeta(sidecar, ready) {
31
58
  return {
32
59
  ext: sidecar.ext,
60
+ ...(sidecar.sourceExt ? { sourceExt: sidecar.sourceExt } : {}),
33
61
  ready,
34
62
  kind: sidecar.kind ?? 'video',
35
63
  relatedTo: sidecar.relatedTo,
@@ -37,9 +65,71 @@ function sidecarToCachedMeta(sidecar, ready) {
37
65
  name: sidecar.name,
38
66
  };
39
67
  }
68
+ /** A recorded web-ready result read back from a sidecar. */
69
+ function isWebReadyResult(value) {
70
+ const result = value;
71
+ return (typeof result === 'object' &&
72
+ result !== null &&
73
+ typeof result.action === 'string' &&
74
+ typeof result.reason === 'string');
75
+ }
40
76
  export function createLocalStorage(opts) {
41
77
  const workspaceRoot = path.resolve(opts.workspaceDir);
42
- const datastore = new FileStore({ directory: workspaceRoot });
78
+ /**
79
+ * Whether the tus upload `uploadId` (`<kind>/<artifactId><ext>`) finished and a conversion then
80
+ * replaced its file: the sidecar is ready, names this upload's extension as `sourceExt`, and the
81
+ * file is gone. tus still has its record of the upload (`<uploadId>.json`).
82
+ */
83
+ const replacedByConversion = async (uploadId) => {
84
+ const slash = uploadId.indexOf('/');
85
+ const kind = uploadId.slice(0, slash);
86
+ const artifactId = uploadId.slice(slash + 1, slash + 37);
87
+ const ext = uploadId.slice(slash + 37);
88
+ if (slash < 0 || !isUuid(artifactId) || !EXTENSION_REGEX.test(ext))
89
+ return false;
90
+ const sidecar = await readSidecar(artifactId);
91
+ if (!sidecar || sidecar.status !== 'ready' || (sidecar.kind ?? 'video') !== kind)
92
+ return false;
93
+ if (sidecar.sourceExt !== ext || sidecar.ext === ext)
94
+ return false;
95
+ return (await storedExt(artifactId, sidecar.kind ?? 'video', ext)) !== ext;
96
+ };
97
+ /**
98
+ * tus's store for the workspace, which also answers for an upload a conversion replaced (a
99
+ * finished `.webm` conformed to `<id>.mp4`): tus would answer a client re-sending the final
100
+ * `PATCH` whose response it lost with `410`, the file being gone. Such an upload reads as
101
+ * complete, and its (empty) tail is taken without a write, so the retry reaches
102
+ * `onUploadFinish`'s already-finished branch and gets `204`, as one without a conversion does.
103
+ */
104
+ class LocalFileStore extends FileStore {
105
+ async getUpload(id) {
106
+ try {
107
+ return await super.getUpload(id);
108
+ }
109
+ catch (err) {
110
+ if (err !== ERRORS.FILE_NO_LONGER_EXISTS || !(await replacedByConversion(id)))
111
+ throw err;
112
+ const record = await this.configstore.get(id);
113
+ if (!record || typeof record.size !== 'number')
114
+ throw err;
115
+ return new Upload({
116
+ id,
117
+ size: record.size,
118
+ offset: record.size,
119
+ metadata: record.metadata,
120
+ creation_date: record.creation_date,
121
+ });
122
+ }
123
+ }
124
+ async write(readable, id, offset) {
125
+ if (!(await replacedByConversion(id)))
126
+ return super.write(readable, id, offset);
127
+ for await (const _ of readable)
128
+ ; // a finished upload: nothing more to store
129
+ return offset;
130
+ }
131
+ }
132
+ const datastore = new LocalFileStore({ directory: workspaceRoot });
43
133
  const metaCacheLimit = opts.metaCacheLimit ?? DEFAULT_META_CACHE_LIMIT;
44
134
  // Metadata cache keyed by artifactId. Populated eagerly on reserve and
45
135
  // lazily from the sidecar on cache-miss — so we never do a workspace-wide
@@ -80,14 +170,78 @@ export function createLocalStorage(opts) {
80
170
  };
81
171
  /** Relative path (from workspaceRoot) to the artifact bytes. */
82
172
  const artifactRelPath = (artifactId, kind, ext) => `${kind}/${artifactId}${ext}`;
173
+ /** The artifact's bytes file, strictly inside its kind directory, as `sidecarPath` checks. */
174
+ const storedPath = (artifactId, kind, ext) => {
175
+ const base = path.resolve(workspaceRoot, kind);
176
+ const resolved = path.resolve(base, `${artifactId}${ext}`);
177
+ if (!resolved.startsWith(`${base}${path.sep}`)) {
178
+ throw new Error('artifactId escapes the kind directory');
179
+ }
180
+ return resolved;
181
+ };
182
+ /**
183
+ * The extension of the file that holds an artifact's bytes, given the `ext` its sidecar names:
184
+ * that file, or for a video the conform target (`<id>.mp4`) when only that exists. A sidecar
185
+ * another instance wrote from a copy older than a conversion still names the uploaded file,
186
+ * which the conversion deleted; its bytes are in the converted file. `null` when neither exists.
187
+ */
188
+ const storedExt = async (artifactId, kind, ext) => {
189
+ // Only a real id becomes a path, as everywhere else in this adapter.
190
+ if (!isUuid(artifactId) || !EXTENSION_REGEX.test(ext))
191
+ return null;
192
+ const candidates = kind === 'video' && ext !== CONFORM_TARGET.extension ? [ext, CONFORM_TARGET.extension] : [ext];
193
+ for (const candidate of candidates) {
194
+ try {
195
+ const stat = await fs.stat(storedPath(artifactId, kind, candidate));
196
+ if (stat.isFile())
197
+ return candidate;
198
+ }
199
+ catch (err) {
200
+ if (err?.code !== 'ENOENT')
201
+ throw err;
202
+ }
203
+ }
204
+ return null;
205
+ };
83
206
  const writeSidecar = async (artifactId, sidecar) => {
84
207
  // Atomic tmp + rename so a crash mid-write can never leave a truncated
85
- // JSON blob that `loadMeta` would then treat as corrupt.
208
+ // JSON blob that `loadMeta` would then treat as corrupt. The tmp name is
209
+ // unique, so two writers (two instances on a shared filesystem) never
210
+ // write into, or rename away, each other's tmp file.
86
211
  const finalPath = sidecarPath(artifactId);
87
- const tmpPath = `${finalPath}.tmp`;
212
+ const tmpPath = `${finalPath}.${randomUUID()}.tmp`;
88
213
  await fs.mkdir(sidecarDir(), { recursive: true, mode: 0o750 });
89
- await fs.writeFile(tmpPath, JSON.stringify(sidecar), 'utf8');
90
- await fs.rename(tmpPath, finalPath);
214
+ try {
215
+ await fs.writeFile(tmpPath, JSON.stringify(sidecar), 'utf8');
216
+ await fs.rename(tmpPath, finalPath);
217
+ }
218
+ catch (err) {
219
+ await fs.rm(tmpPath, { force: true });
220
+ throw err;
221
+ }
222
+ };
223
+ /**
224
+ * Read-modify-write of a sidecar runs one at a time per artifact in this process, so an
225
+ * acknowledgement, a conversion switching the stored file and a removal each act on the sidecar
226
+ * as the last of them left it. Across instances sharing the workspace (OPERATIONS.md,
227
+ * "Horizontal scaling") there is no lock: the last rename wins. The completion bookkeeping
228
+ * recovers: the replay redoes a lost acknowledgement or conversion record, idempotently, and a
229
+ * sidecar written from a copy older than a conversion still finds the bytes: a conversion's file
230
+ * is always `<id>.mp4`, which `resolve` and `getLocalPath` fall back to (`storedExt`). A lost
231
+ * `outcome` isn't redone; nothing records it a second time.
232
+ */
233
+ const sidecarWrites = new Map();
234
+ const withSidecarLock = async (artifactId, work) => {
235
+ const previous = sidecarWrites.get(artifactId) ?? Promise.resolve();
236
+ const run = previous.catch(() => { }).then(work);
237
+ sidecarWrites.set(artifactId, run);
238
+ try {
239
+ return await run;
240
+ }
241
+ finally {
242
+ if (sidecarWrites.get(artifactId) === run)
243
+ sidecarWrites.delete(artifactId);
244
+ }
91
245
  };
92
246
  const readSidecar = async (artifactId) => {
93
247
  // Every sidecar/artifact path in this module is built by joining
@@ -126,6 +280,9 @@ export function createLocalStorage(opts) {
126
280
  return {
127
281
  version: SIDECAR_VERSION,
128
282
  ext: parsed.ext,
283
+ ...(typeof parsed.sourceExt === 'string' && EXTENSION_REGEX.test(parsed.sourceExt)
284
+ ? { sourceExt: parsed.sourceExt }
285
+ : {}),
129
286
  filename: parsed.filename,
130
287
  status,
131
288
  kind,
@@ -133,6 +290,15 @@ export function createLocalStorage(opts) {
133
290
  checksum: typeof parsed.checksum === 'string' ? parsed.checksum : undefined,
134
291
  name: typeof parsed.name === 'string' ? parsed.name : undefined,
135
292
  appVersion: typeof parsed.appVersion === 'string' ? parsed.appVersion : undefined,
293
+ ...(parsed.context !== undefined ? { context: parsed.context } : {}),
294
+ // A sidecar from before the flag existed: finished means nothing to replay; still
295
+ // uploading means its completion hasn't happened yet.
296
+ acknowledged: parsed.acknowledged ?? status === 'ready',
297
+ converted: parsed.converted ?? status === 'ready',
298
+ ...(isWebReadyResult(parsed.webReady) ? { webReady: parsed.webReady } : {}),
299
+ ...(typeof parsed.generation === 'string' ? { generation: parsed.generation } : {}),
300
+ ...(parsed.outcome !== undefined ? { outcome: parsed.outcome } : {}),
301
+ ...(typeof parsed.readyAt === 'number' ? { readyAt: parsed.readyAt } : {}),
136
302
  };
137
303
  }
138
304
  catch {
@@ -141,6 +307,52 @@ export function createLocalStorage(opts) {
141
307
  return null;
142
308
  }
143
309
  };
310
+ /** Directory of relation markers for one anchor artifact. */
311
+ const relatedDir = (anchorId) => {
312
+ const base = path.resolve(sidecarDir(), RELATED_DIR);
313
+ const resolved = path.resolve(base, anchorId);
314
+ if (!resolved.startsWith(`${base}${path.sep}`)) {
315
+ throw new Error('artifactId escapes the metadata directory');
316
+ }
317
+ return resolved;
318
+ };
319
+ /** When the upload last moved: the sidecar's mtime, or the bytes file's while still uploading. */
320
+ const lastActivity = async (artifactId, sidecar) => {
321
+ let updatedAt;
322
+ try {
323
+ updatedAt = (await fs.stat(sidecarPath(artifactId))).mtimeMs;
324
+ }
325
+ catch {
326
+ return 0;
327
+ }
328
+ if (sidecar.status === 'uploading') {
329
+ const kind = sidecar.kind ?? 'video';
330
+ const bytes = path.join(workspaceRoot, kind, `${artifactId}${sidecar.ext}`);
331
+ const written = await fs.stat(bytes).then((stats) => stats.mtimeMs, () => 0);
332
+ updatedAt = Math.max(updatedAt, written);
333
+ }
334
+ return updatedAt;
335
+ };
336
+ const sidecarToMeta = (artifactId, sidecar, updatedAt) => ({
337
+ artifactId,
338
+ kind: sidecar.kind ?? 'video',
339
+ ext: sidecar.ext,
340
+ ...(sidecar.sourceExt ? { sourceExt: sidecar.sourceExt } : {}),
341
+ filename: sidecar.filename,
342
+ ...(sidecar.relatedTo ? { relatedTo: sidecar.relatedTo } : {}),
343
+ ...(sidecar.checksum ? { checksum: sidecar.checksum } : {}),
344
+ ...(sidecar.name ? { name: sidecar.name } : {}),
345
+ ...(sidecar.appVersion ? { appVersion: sidecar.appVersion } : {}),
346
+ ...(sidecar.context !== undefined ? { context: sidecar.context } : {}),
347
+ ready: sidecar.status === 'ready',
348
+ acknowledged: sidecar.acknowledged !== false,
349
+ converted: sidecar.converted !== false,
350
+ ...(sidecar.webReady ? { webReady: sidecar.webReady } : {}),
351
+ ...(sidecar.generation ? { generation: sidecar.generation } : {}),
352
+ ...(sidecar.outcome !== undefined ? { outcome: sidecar.outcome } : {}),
353
+ updatedAt,
354
+ ...(sidecar.readyAt !== undefined ? { readyAt: sidecar.readyAt } : {}),
355
+ });
144
356
  const loadMeta = async (artifactId) => {
145
357
  const cached = metaCache.get(artifactId);
146
358
  if (cached)
@@ -161,7 +373,7 @@ export function createLocalStorage(opts) {
161
373
  // tighten it so the upload tree isn't world-writable under a permissive umask.
162
374
  await fs.chmod(workspaceRoot, 0o750).catch(() => { });
163
375
  };
164
- const reserveUpload = async ({ artifactId, filename, ext, kind, relatedTo, checksum, name, appVersion, }) => {
376
+ const reserveUpload = async ({ artifactId, filename, ext, kind, relatedTo, checksum, name, appVersion, context, }) => {
165
377
  await fs.mkdir(path.join(workspaceRoot, kind), { recursive: true, mode: 0o750 });
166
378
  await fs.mkdir(sidecarDir(), { recursive: true, mode: 0o750 });
167
379
  const sidecar = {
@@ -174,61 +386,75 @@ export function createLocalStorage(opts) {
174
386
  checksum,
175
387
  name,
176
388
  appVersion,
389
+ ...(context !== undefined ? { context } : {}),
390
+ acknowledged: false,
391
+ converted: false,
392
+ generation: randomUUID(),
177
393
  };
178
- // Collision guard: `wx` fails atomically with EEXIST if a sidecar already exists for
179
- // this artifactId, rather than the previous read-then-write (`loadMeta` then
180
- // `writeSidecar`) which left a window for two concurrent/retried requests to both pass
181
- // the check before either had written — letting the second silently clobber the first's
182
- // sidecar and race @tus/file-store's own offset tracking. Translates to HTTP 409 via
183
- // @tus/server's error path, same as before.
184
- try {
185
- await fs.writeFile(sidecarPath(artifactId), JSON.stringify(sidecar), { flag: 'wx' });
186
- }
187
- catch (err) {
188
- if (err?.code !== 'EEXIST')
189
- throw err;
190
- // A file already exists at this path, but `readSidecar` treats a malformed/corrupt
191
- // one as absent (e.g. debris from a crash mid-write) — re-check before deciding this
192
- // is a genuine collision rather than debris that's safe to overwrite.
193
- const existing = await readSidecar(artifactId);
194
- if (existing) {
195
- throw Object.assign(new Error(`artifactId ${artifactId} already has an upload`), {
196
- statusCode: 409,
197
- status_code: 409,
198
- });
394
+ // Under the sidecar lock, so a removal of the same id in this process either finished
395
+ // (its sidecar went last) or hasn't started.
396
+ await withSidecarLock(artifactId, async () => {
397
+ // Collision guard: `wx` fails atomically with EEXIST if a sidecar already exists for
398
+ // this artifactId, rather than the previous read-then-write (`loadMeta` then
399
+ // `writeSidecar`) which left a window for two concurrent/retried requests to both pass
400
+ // the check before either had written — letting the second silently clobber the first's
401
+ // sidecar and race @tus/file-store's own offset tracking. Translates to HTTP 409 via
402
+ // @tus/server's error path, same as before.
403
+ try {
404
+ await fs.writeFile(sidecarPath(artifactId), JSON.stringify(sidecar), { flag: 'wx' });
199
405
  }
200
- await writeSidecar(artifactId, sidecar);
406
+ catch (err) {
407
+ if (err?.code !== 'EEXIST')
408
+ throw err;
409
+ // A file already exists at this path, but `readSidecar` treats a malformed/corrupt
410
+ // one as absent (e.g. debris from a crash mid-write) — re-check before deciding this
411
+ // is a genuine collision rather than debris that's safe to overwrite.
412
+ const existing = await readSidecar(artifactId);
413
+ if (existing) {
414
+ throw Object.assign(new Error(`artifactId ${artifactId} already has an upload`), {
415
+ statusCode: 409,
416
+ status_code: 409,
417
+ });
418
+ }
419
+ await writeSidecar(artifactId, sidecar);
420
+ }
421
+ });
422
+ if (relatedTo) {
423
+ // The relation index entry. Written after the sidecar so a crash between the two leaves
424
+ // an artifact without an index entry (it still works, it's just not listed under its
425
+ // anchor), never an index entry without an artifact.
426
+ const dir = relatedDir(relatedTo);
427
+ await fs.mkdir(dir, { recursive: true, mode: 0o750 });
428
+ await fs.writeFile(path.join(dir, artifactId), '', { flag: 'w' });
201
429
  }
202
430
  cacheSet(artifactId, { ext, ready: false, kind, relatedTo, checksum, name });
203
431
  // @tus/file-store joins this onto its configured `directory`, so the
204
432
  // actual file lands at `<workspaceRoot>/<kind>/<artifactId><ext>`.
205
433
  return artifactRelPath(artifactId, kind, ext);
206
434
  };
207
- const resolve = async (artifactId) => {
435
+ const resolve = async (artifactId, reread = false) => {
208
436
  const meta = await loadMeta(artifactId);
209
437
  // Only serve ready uploads. In-progress uploads stay hidden — a client
210
438
  // GETting mid-upload would otherwise receive a truncated file.
211
439
  if (!meta || !meta.ready)
212
440
  return null;
213
- const relFile = artifactRelPath(artifactId, meta.kind, meta.ext);
214
- try {
215
- const stat = await fs.stat(path.join(workspaceRoot, relFile));
216
- if (!stat.isFile())
217
- return null;
218
- }
219
- catch (err) {
220
- if (err?.code === 'ENOENT')
441
+ const ext = await storedExt(artifactId, meta.kind, meta.ext);
442
+ if (ext === null) {
443
+ // The cache may be older than another instance's removal and new upload of the id: read
444
+ // the sidecar once more.
445
+ if (reread)
221
446
  return null;
222
- throw err;
447
+ metaCache.delete(artifactId);
448
+ return resolve(artifactId, true);
223
449
  }
224
450
  return {
225
451
  kind: 'stream',
226
452
  root: workspaceRoot,
227
- filename: relFile,
228
- contentType: extToContentType(meta.ext),
453
+ filename: artifactRelPath(artifactId, meta.kind, ext),
454
+ contentType: extToContentType(ext),
229
455
  };
230
456
  };
231
- const markReady = async (artifactId) => {
457
+ const markReady = (artifactId) => withSidecarLock(artifactId, async () => {
232
458
  const sidecar = await readSidecar(artifactId);
233
459
  if (!sidecar) {
234
460
  // No sidecar means no `reserveUpload` happened for this artifactId —
@@ -240,29 +466,72 @@ export function createLocalStorage(opts) {
240
466
  cacheSet(artifactId, sidecarToCachedMeta(sidecar, true));
241
467
  return;
242
468
  }
243
- await writeSidecar(artifactId, { ...sidecar, status: 'ready' });
469
+ // An upload reserved before generations were kept gets one now, so its conversion runs.
470
+ await writeSidecar(artifactId, {
471
+ ...sidecar,
472
+ status: 'ready',
473
+ readyAt: Date.now(),
474
+ generation: sidecar.generation ?? randomUUID(),
475
+ });
244
476
  cacheSet(artifactId, sidecarToCachedMeta(sidecar, true));
245
- };
246
- const remove = async (artifactId) => {
247
- const meta = await loadMeta(artifactId);
477
+ });
478
+ // Under the sidecar lock, from the sidecar itself: in this process, a conversion switching the
479
+ // artifact to a new file (`patchArtifact` with `ext`) either finishes first — and this removes
480
+ // the new file — or finds the sidecar gone and drops what it wrote.
481
+ const remove = (artifactId) => withSidecarLock(artifactId, async () => {
482
+ // The sidecar decides, never the cache: a read racing this removal (or an earlier one) can
483
+ // have cached the artifact again, and a removal that trusted it would report the same
484
+ // artifact removed twice.
485
+ const sidecar = await readSidecar(artifactId);
248
486
  // Drop from cache before rm so a racing `resolve` arriving after the
249
487
  // rm but before cache eviction can't hand back a stale path.
250
488
  metaCache.delete(artifactId);
251
- if (!meta)
489
+ if (!sidecar)
252
490
  return false;
491
+ const meta = sidecarToCachedMeta(sidecar, sidecar.status === 'ready');
253
492
  const artifactPath = path.join(workspaceRoot, meta.kind, `${artifactId}${meta.ext}`);
493
+ // The uploaded file and tus's record of it, when a conversion stored the bytes under a new
494
+ // extension (normally already gone; left behind by a crash during the switch).
495
+ const sourcePath = meta.sourceExt
496
+ ? path.join(workspaceRoot, meta.kind, `${artifactId}${meta.sourceExt}`)
497
+ : null;
498
+ // A video's conversion output, whatever this sidecar's copy says (`storedExt`).
499
+ const conformedPath = meta.kind === 'video' && meta.ext !== CONFORM_TARGET.extension
500
+ ? path.join(workspaceRoot, artifactRelPath(artifactId, meta.kind, CONFORM_TARGET.extension))
501
+ : null;
254
502
  await Promise.all([
255
503
  fs.rm(artifactPath, { force: true }),
256
504
  fs.rm(`${artifactPath}.json`, { force: true }),
257
- fs.rm(sidecarPath(artifactId), { force: true }),
505
+ ...(sourcePath
506
+ ? [fs.rm(sourcePath, { force: true }), fs.rm(`${sourcePath}.json`, { force: true })]
507
+ : []),
508
+ ...(conformedPath ? [fs.rm(conformedPath, { force: true })] : []),
509
+ ...(meta.relatedTo
510
+ ? [fs.rm(path.join(relatedDir(meta.relatedTo), artifactId), { force: true })]
511
+ : []),
258
512
  ]);
513
+ // Last: until the sidecar is gone the id can't be reserved again, so a new upload of it
514
+ // never has its bytes removed by this one's removal.
515
+ await fs.rm(sidecarPath(artifactId), { force: true });
516
+ // Again: a read during the removal may have cached it from the sidecar it still saw.
517
+ metaCache.delete(artifactId);
259
518
  return true;
260
- };
519
+ });
520
+ /**
521
+ * From the sidecar, not the cache: another instance on the same workspace may have removed
522
+ * the artifact and reserved its id again (a new extension) since this one cached it, and a
523
+ * caller about to read or rewrite the bytes must get today's file.
524
+ */
261
525
  const getLocalPath = async (artifactId) => {
262
- const meta = await loadMeta(artifactId);
263
- if (!meta)
526
+ const sidecar = await readSidecar(artifactId);
527
+ if (!sidecar) {
528
+ metaCache.delete(artifactId);
264
529
  return null;
265
- return path.join(workspaceRoot, meta.kind, `${artifactId}${meta.ext}`);
530
+ }
531
+ cacheSet(artifactId, sidecarToCachedMeta(sidecar, sidecar.status === 'ready'));
532
+ const kind = sidecar.kind ?? 'video';
533
+ const ext = (await storedExt(artifactId, kind, sidecar.ext)) ?? sidecar.ext;
534
+ return path.join(workspaceRoot, artifactRelPath(artifactId, kind, ext));
266
535
  };
267
536
  const getKind = async (artifactId) => {
268
537
  const meta = await loadMeta(artifactId);
@@ -280,6 +549,90 @@ export function createLocalStorage(opts) {
280
549
  const meta = await loadMeta(artifactId);
281
550
  return meta?.name ?? null;
282
551
  };
552
+ /** Straight from the sidecar, never the cache: its flags change after the cache was filled. */
553
+ const describeArtifact = async (artifactId) => {
554
+ const sidecar = await readSidecar(artifactId);
555
+ if (!sidecar)
556
+ return null;
557
+ return sidecarToMeta(artifactId, sidecar, await lastActivity(artifactId, sidecar));
558
+ };
559
+ const patchArtifact = (artifactId, patch) => withSidecarLock(artifactId, async () => {
560
+ if (patch.ext !== undefined && !EXTENSION_REGEX.test(patch.ext)) {
561
+ throw new TypeError(`patchArtifact: invalid ext ${JSON.stringify(patch.ext)}`);
562
+ }
563
+ const sidecar = await readSidecar(artifactId);
564
+ if (!sidecar)
565
+ return false;
566
+ // Another pass already recorded its conversion, or the id was removed and reserved again:
567
+ // this patch belongs to neither.
568
+ if (patch.unlessConverted && sidecar.converted !== false)
569
+ return false;
570
+ if (patch.generation !== undefined && (sidecar.generation ?? null) !== patch.generation)
571
+ return false;
572
+ const kind = sidecar.kind ?? 'video';
573
+ const stored = (ext) => path.join(workspaceRoot, kind, `${artifactId}${ext}`);
574
+ const ext = patch.ext ?? sidecar.ext;
575
+ if (patch.file !== undefined) {
576
+ if (path.dirname(path.resolve(patch.file)) !== path.join(workspaceRoot, kind)) {
577
+ throw new TypeError('patchArtifact: `file` must be in the artifact\'s kind directory');
578
+ }
579
+ await fs.rename(patch.file, stored(ext));
580
+ }
581
+ const next = { ...sidecar };
582
+ if (patch.acknowledged !== undefined)
583
+ next.acknowledged = patch.acknowledged;
584
+ if (patch.converted !== undefined)
585
+ next.converted = patch.converted;
586
+ if (patch.webReady !== undefined)
587
+ next.webReady = patch.webReady;
588
+ if (patch.ext !== undefined && patch.ext !== sidecar.ext) {
589
+ next.sourceExt = sidecar.sourceExt ?? sidecar.ext;
590
+ next.ext = patch.ext;
591
+ }
592
+ if (patch.outcome !== undefined) {
593
+ if (patch.outcome === null)
594
+ delete next.outcome;
595
+ else
596
+ next.outcome = patch.outcome;
597
+ }
598
+ await writeSidecar(artifactId, next);
599
+ // The cache holds nothing a patch changes, but a `ready` flip elsewhere must not be undone
600
+ // by a stale entry either — keep it consistent with what was just written.
601
+ cacheSet(artifactId, sidecarToCachedMeta(next, next.status === 'ready'));
602
+ // The sidecar names the new file: the one under the old extension goes. (A crash before
603
+ // this leaves it for `remove`, which deletes `sourceExt` too. A sidecar copy that still
604
+ // names it finds the new file instead: `storedExt`.)
605
+ if (ext !== sidecar.ext)
606
+ await fs.rm(stored(sidecar.ext), { force: true });
607
+ return true;
608
+ });
609
+ async function* listRelated(artifactId) {
610
+ if (!isUuid(artifactId))
611
+ return;
612
+ let names;
613
+ try {
614
+ names = await fs.readdir(relatedDir(artifactId));
615
+ }
616
+ catch (err) {
617
+ if (err?.code === 'ENOENT')
618
+ return;
619
+ throw err;
620
+ }
621
+ for (const relatedId of names) {
622
+ if (!isUuid(relatedId))
623
+ continue;
624
+ const sidecar = await readSidecar(relatedId);
625
+ if (!sidecar || sidecar.relatedTo !== artifactId)
626
+ continue; // A marker whose artifact is gone.
627
+ yield {
628
+ artifactId: relatedId,
629
+ kind: sidecar.kind ?? 'video',
630
+ relatedTo: artifactId,
631
+ ready: sidecar.status === 'ready',
632
+ updatedAt: await lastActivity(relatedId, sidecar),
633
+ };
634
+ }
635
+ }
283
636
  /**
284
637
  * Walk the sidecars. A sidecar is rewritten only when its upload finishes, so its mtime is
285
638
  * when the upload started (still uploading) or when it finished (ready). An upload in flight
@@ -336,7 +689,7 @@ export function createLocalStorage(opts) {
336
689
  workspaceRoot,
337
690
  initialize,
338
691
  reserveUpload,
339
- resolve,
692
+ resolve: (artifactId) => resolve(artifactId),
340
693
  markReady,
341
694
  remove,
342
695
  getLocalPath,
@@ -345,6 +698,9 @@ export function createLocalStorage(opts) {
345
698
  getChecksum,
346
699
  getName,
347
700
  listArtifacts,
701
+ describeArtifact,
702
+ patchArtifact,
703
+ listRelated,
348
704
  };
349
705
  }
350
706
  //# sourceMappingURL=local.js.map