@ttt-productions/media-processing-core 0.8.16 → 0.8.18

package/dist/server/storage-ops.js

@@ -1,107 +1,66 @@
-// Final-media object-store chokepoint — every final-media write/copy/delete in
-// the system goes through a MediaObjectStore. Inline storage-provider calls for
-// FINAL media are forbidden outside this module. (Upload STAGING stays plain
-// Firebase Storage and is not this module's concern.)
-//
-// Implementations (the consuming app picks one per environment):
-//  - R2 (S3-compatible) for deployed environments whose final media lives in
-//    R2 and is served by an edge gateway.
-//  - Firebase Storage for deployed environments whose final media lives in
-//    Firebase Storage. Token-free: it writes and copies no download token, so
-//    its objects are readable only through the app's own access path (Storage
-//    rules). Firebase can still mint a token later (a client getDownloadURL, the
-//    console); the app's read path must never do so.
-//  - Firebase Storage emulator for local dev/tests — the Firebase Storage store
-//    plus a FIXED caller-supplied download token on every object, so
-//    deterministic emulator URLs work. Production NEVER mints download tokens.
-//
-// Writes return `{ key }` — never URLs. Display URLs are built at render time
-// from asset refs (see the consuming app's media-asset-url helper).
-import { createReadStream, createWriteStream } from "node:fs";
-import { stat } from "node:fs/promises";
-import { Readable } from "node:stream";
-import { pipeline } from "node:stream/promises";
-import { AwsClient } from "aws4fetch";
-import { NEUTRAL_CONTENT_TYPE } from "@ttt-productions/media-schemas";
-import { isObjectNotFoundError } from "./gcs-errors.js";
-import { defaultR2RetrySeams, fetchWithR2Retry, R2StorageError, } from "./r2-retry.js";
-import { requireStorageObjectGeneration, UnreadableStorageObjectMetadataError } from "./staged-object.js";
-// Re-export the exhaustion error so callers can recognize an exhausted R2
-// operation. The rest of the retry helper stays private to this module.
-export { R2StorageError };
-/**
- * A copy found an object at its destination key that is not this copy's result, and
- * wrote nothing; that object is untouched. The key is a PROPERTY, never message text, so
- * an escaping error never carries an object key to monitoring.
- */
+import { createReadStream, createWriteStream } from 'node:fs';
+import { stat } from 'node:fs/promises';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { AwsClient } from 'aws4fetch';
+import { NEUTRAL_CONTENT_TYPE } from '@ttt-productions/media-schemas';
+import { isObjectNotFoundError } from './gcs-errors.js';
+import { defaultR2RetrySeams, fetchWithR2Retry, R2StorageError, } from './r2-retry.js';
+import { requireStorageObjectGeneration, UnreadableStorageObjectMetadataError, } from './staged-object.js';
 export class ObjectAlreadyExistsError extends Error {
     key;
     constructor(key) {
-        super("copy destination already holds another object");
+        super('copy destination already holds another object');
         this.key = key;
-        this.name = "ObjectAlreadyExistsError";
+        this.name = 'ObjectAlreadyExistsError';
     }
 }
-// Every copy records on its destination which source version it holds. A copy whose
-// destination already exists compares this record with its own, so it recognizes its
-// own earlier result and never mistakes another writer's object for it. The values are
-// URI-encoded because R2 carries custom metadata in HTTP headers, which must be ASCII.
-const COPY_SOURCE_METADATA_KEY = "copy-source";
-const COPY_SOURCE_VERSION_METADATA_KEY = "copy-source-version";
+// URI-encoded: R2 carries custom metadata in HTTP headers, which must be ASCII.
+const COPY_SOURCE_METADATA_KEY = 'copy-source';
+const COPY_SOURCE_VERSION_METADATA_KEY = 'copy-source-version';
 function copyProvenance(fromKey, sourceVersion) {
     return {
         [COPY_SOURCE_METADATA_KEY]: encodeURIComponent(fromKey),
         [COPY_SOURCE_VERSION_METADATA_KEY]: encodeURIComponent(sourceVersion),
     };
 }
-/** Whether an object's custom metadata, read through `readMetadata`, names exactly this provenance. */
 function holdsProvenance(readMetadata, provenance) {
     return Object.entries(provenance).every(([name, value]) => readMetadata(name) === value);
 }
-/**
- * Whether an object's custom metadata records a copy of `fromKey`, of whichever source
- * version: `deleteCopy` reads only the destination, so it cannot know the source's version.
- * A record naming the source without a version is not provenance this package wrote.
- */
 function recordsCopyOf(readMetadata, fromKey) {
     const version = readMetadata(COPY_SOURCE_VERSION_METADATA_KEY);
     return (readMetadata(COPY_SOURCE_METADATA_KEY) === encodeURIComponent(fromKey) &&
-        typeof version === "string" &&
-        version !== "");
+        typeof version === 'string' &&
+        version !== '');
 }
-/** The content headers a copy carries from its source: HTTP header name → Cloud Storage field. */
 const CARRIED_CONTENT_HEADERS = {
-    "content-type": "contentType",
-    "cache-control": "cacheControl",
-    "content-disposition": "contentDisposition",
-    "content-encoding": "contentEncoding",
+    'content-type': 'contentType',
+    'cache-control': 'cacheControl',
+    'content-disposition': 'contentDisposition',
+    'content-encoding': 'contentEncoding',
 };
-const DEFAULT_CACHE_CONTROL = "public, max-age=31536000, immutable";
+const DEFAULT_CACHE_CONTROL = 'public, max-age=31536000, immutable';
 export function createR2ObjectStore(args) {
-    const endpoint = (args.endpoint ?? `https://${args.accountId}.r2.cloudflarestorage.com`).replace(/\/$/, "");
+    const endpoint = (args.endpoint ?? `https://${args.accountId}.r2.cloudflarestorage.com`).replace(/\/$/, '');
     const client = new AwsClient({
         accessKeyId: args.accessKeyId,
         secretAccessKey: args.secretAccessKey,
-        service: "s3",
-        region: "auto",
+        service: 's3',
+        region: 'auto',
     });
     const doFetch = args.fetchImpl ?? fetch;
-    // Retry seams: production defaults are internal; tests inject deterministic
-    // sleep/random/now. putFile/copy/readToFile ride the bounded retry helper;
-    // delete stays one-shot (never routes through it — see below).
     const defaults = defaultR2RetrySeams();
     const seams = {
         sleep: args.sleepImpl ?? defaults.sleep,
         random: args.randomImpl ?? defaults.random,
         now: args.nowImpl ?? defaults.now,
     };
-    const objectUrl = (key) => `${endpoint}/${args.bucket}/${key.split("/").map(encodeURIComponent).join("/")}`;
+    const objectUrl = (key) => `${endpoint}/${args.bucket}/${key.split('/').map(encodeURIComponent).join('/')}`;
     async function signedFetch(url, init) {
         const signed = await client.sign(url, { ...init, aws: { allHeaders: true } });
-        return doFetch(signed, { ...(init.body ? { duplex: "half" } : {}) });
+        return doFetch(signed, { ...(init.body ? { duplex: 'half' } : {}) });
     }
-    // A GET whose body is discarded unread, not a HEAD: a HEAD's error response carries no
-    // body, so the S3 error code that tells a missing key from a missing bucket would be lost.
+    // A GET, not a HEAD: a HEAD's error response has no body, so the S3 error code would be lost.
     async function readObjectHeaders(key, operation) {
         return fetchWithR2Retry({
             operation,
@@ -109,74 +68,64 @@ export function createR2ObjectStore(args) {
             key,
             seams,
             runAttempt: async () => {
-                const res = await signedFetch(objectUrl(key), { method: "GET" });
+                const res = await signedFetch(objectUrl(key), { method: 'GET' });
                 if (!res.ok)
-                    return { kind: "response", response: res };
+                    return { kind: 'response', response: res };
                 await res.body?.cancel();
-                return { kind: "success", value: res.headers };
+                return { kind: 'success', value: res.headers };
             },
         });
     }
-    // R2 exposes no object version id, so a source version is its ETag, last-modified time,
-    // and size together: replacing the source changes at least one of them unless the new
-    // content has the same MD5-derived ETag and lands within the same second. A multipart
-    // upload's ETag is not the content's MD5, but it is the same kind of version token.
+    // R2 exposes no object version id, so a source version is its ETag, last-modified time, and size together.
     function readSourceVersion(headers, key) {
-        const etag = headers.get("etag");
+        const etag = headers.get('etag');
         if (!etag)
-            throw new UnreadableStorageObjectMetadataError(args.bucket, key, "etag");
-        const lastModified = headers.get("last-modified");
+            throw new UnreadableStorageObjectMetadataError(args.bucket, key, 'etag');
+        const lastModified = headers.get('last-modified');
         if (!lastModified)
-            throw new UnreadableStorageObjectMetadataError(args.bucket, key, "lastModified");
-        const size = headers.get("content-length");
+            throw new UnreadableStorageObjectMetadataError(args.bucket, key, 'lastModified');
+        const size = headers.get('content-length');
         if (!size)
-            throw new UnreadableStorageObjectMetadataError(args.bucket, key, "size");
+            throw new UnreadableStorageObjectMetadataError(args.bucket, key, 'size');
         return { etag, version: `${etag} ${lastModified} ${size}` };
     }
     return {
         async putFile({ localPath, key, contentType }) {
-            // stat() once, outside the retry loop: a missing input file is a
-            // deterministic local error and must fail immediately, never retry.
             const { size } = await stat(localPath);
             const ct = contentType ?? getMimeFromExt(extractExt(localPath));
             return fetchWithR2Retry({
-                operation: "putFile",
+                operation: 'putFile',
                 bucket: args.bucket,
                 key,
                 seams,
-                // Fresh signed request AND a fresh read stream on every attempt — a
-                // consumed stream is never reused.
+                // A fresh signed request and read stream on every attempt: a consumed stream cannot be re-read.
                 runAttempt: async () => {
                     const res = await signedFetch(objectUrl(key), {
-                        method: "PUT",
+                        method: 'PUT',
                         headers: {
-                            "content-type": ct,
-                            "content-length": String(size),
-                            "cache-control": DEFAULT_CACHE_CONTROL,
+                            'content-type': ct,
+                            'content-length': String(size),
+                            'cache-control': DEFAULT_CACHE_CONTROL,
                         },
                         body: createReadStream(localPath),
                     });
                     if (res.ok) {
-                        return { kind: "success", value: { key, sizeBytes: size, contentType: ct } };
+                        return { kind: 'success', value: { key, sizeBytes: size, contentType: ct } };
                     }
-                    return { kind: "response", response: res };
+                    return { kind: 'response', response: res };
                 },
             });
         },
         async copy({ fromKey, toKey }) {
-            const source = await readObjectHeaders(fromKey, "copy");
+            const source = await readObjectHeaders(fromKey, 'copy');
             const { etag, version } = readSourceVersion(source, fromKey);
             const provenance = copyProvenance(fromKey, version);
             const headers = {
-                "x-amz-copy-source": `/${args.bucket}/${fromKey.split("/").map(encodeURIComponent).join("/")}`,
-                // Pins the copy to the source version the provenance names.
-                "x-amz-copy-source-if-match": etag,
-                // R2's destination-conditional CopyObject extension, checked when the copy commits;
-                // S3's x-amz-copy-source-if-* headers condition only the source.
-                "cf-copy-destination-if-none-match": "*",
-                // The destination takes exactly the headers below rather than the source's, so it
-                // commits fully formed in this one call.
-                "x-amz-metadata-directive": "REPLACE",
+                'x-amz-copy-source': `/${args.bucket}/${fromKey.split('/').map(encodeURIComponent).join('/')}`,
+                'x-amz-copy-source-if-match': etag,
+                // R2's destination-conditional CopyObject extension; S3's x-amz-copy-source-if-* headers condition only the source.
+                'cf-copy-destination-if-none-match': '*',
+                'x-amz-metadata-directive': 'REPLACE',
             };
             for (const name of Object.keys(CARRIED_CONTENT_HEADERS)) {
                 const value = source.get(name);
@@ -187,15 +136,15 @@ export function createR2ObjectStore(args) {
                 headers[`x-amz-meta-${name}`] = value;
             try {
                 await fetchWithR2Retry({
-                    operation: "copy",
+                    operation: 'copy',
                     bucket: args.bucket,
                     key: toKey,
                     seams,
                     runAttempt: async () => {
-                        const res = await signedFetch(objectUrl(toKey), { method: "PUT", headers });
+                        const res = await signedFetch(objectUrl(toKey), { method: 'PUT', headers });
                         if (res.ok)
-                            return { kind: "success", value: undefined };
-                        return { kind: "response", response: res };
+                            return { kind: 'success', value: undefined };
+                        return { kind: 'response', response: res };
                     },
                 });
                 return { key: toKey, alreadyPresent: false };
@@ -203,11 +152,10 @@ export function createR2ObjectStore(args) {
             catch (e) {
                 if (!(e instanceof R2StorageError && e.status === 412))
                     throw e;
-                // R2 answers 412 both for an existing destination and for a source that changed
-                // after it was read; only an existing destination can be this copy's result.
+                // R2 answers 412 both for an existing destination and for a source that changed after it was read.
                 let destination;
                 try {
-                    destination = await readObjectHeaders(toKey, "copy");
+                    destination = await readObjectHeaders(toKey, 'copy');
                 }
                 catch (readError) {
                     if (isObjectNotFoundError(readError))
@@ -222,35 +170,29 @@ export function createR2ObjectStore(args) {
         },
         async readToFile({ key, localPath }) {
             return fetchWithR2Retry({
-                operation: "readToFile",
+                operation: 'readToFile',
                 bucket: args.bucket,
                 key,
                 seams,
                 runAttempt: async () => {
-                    const res = await signedFetch(objectUrl(key), { method: "GET" });
+                    const res = await signedFetch(objectUrl(key), { method: 'GET' });
                     if (!res.ok || !res.body) {
-                        return { kind: "response", response: res };
+                        return { kind: 'response', response: res };
                     }
-                    const contentType = res.headers.get("content-type") ?? NEUTRAL_CONTENT_TYPE;
-                    // Pipe ONLY after a successful response, and open the destination in
-                    // truncating mode ('w') on every attempt so a partial download from a
-                    // prior attempt can never prefix/corrupt this one. res.body is a web
-                    // ReadableStream; stream it to disk so large media never sits fully in
-                    // memory. A transient body/network stream failure here throws and is
-                    // retried by the helper; a deterministic local fs error is not.
-                    await pipeline(Readable.fromWeb(res.body), createWriteStream(localPath, { flags: "w" }));
+                    const contentType = res.headers.get('content-type') ?? NEUTRAL_CONTENT_TYPE;
+                    // Truncating mode ('w') on every attempt, so a partial earlier download never prefixes this one.
+                    await pipeline(Readable.fromWeb(res.body), createWriteStream(localPath, { flags: 'w' }));
                     const { size } = await stat(localPath);
-                    return { kind: "success", value: { sizeBytes: size, contentType } };
+                    return { kind: 'success', value: { sizeBytes: size, contentType } };
                 },
             });
         },
         async delete(key) {
-            // Intentionally one-shot: delete is never retried (see r2-retry.ts header).
-            const res = await signedFetch(objectUrl(key), { method: "DELETE" });
+            const res = await signedFetch(objectUrl(key), { method: 'DELETE' });
             // S3 DELETE returns 204 even for missing keys; anything else non-ok is real.
             if (!res.ok && res.status !== 404) {
                 throw new R2StorageError({
-                    operation: "delete",
+                    operation: 'delete',
                     bucket: args.bucket,
                     key,
                     attempts: 1,
@@ -262,23 +204,21 @@ export function createR2ObjectStore(args) {
         async deleteCopy({ fromKey, toKey }) {
             let destination;
             try {
-                destination = await readObjectHeaders(toKey, "deleteCopy");
+                destination = await readObjectHeaders(toKey, 'deleteCopy');
             }
             catch (e) {
                 if (isObjectNotFoundError(e))
-                    return "absent";
+                    return 'absent';
                 throw e;
             }
             if (!recordsCopyOf((name) => destination.get(`x-amz-meta-${name}`), fromKey))
-                return "notThisCopy";
-            // Unconditioned: R2's DeleteObject honors no precondition header, so none is sent — one
-            // it silently ignored would only look like protection. `MediaObjectStore.deleteCopy`
-            // states why the gap after the check is harmless. One-shot, like `delete`.
-            const res = await signedFetch(objectUrl(toKey), { method: "DELETE" });
+                return 'notThisCopy';
+            // R2's DeleteObject honors no precondition header, so none is sent.
+            const res = await signedFetch(objectUrl(toKey), { method: 'DELETE' });
             if (res.ok)
-                return "deleted";
+                return 'deleted';
             const error = new R2StorageError({
-                operation: "deleteCopy",
+                operation: 'deleteCopy',
                 bucket: args.bucket,
                 key: toKey,
                 attempts: 1,
@@ -286,7 +226,7 @@ export function createR2ObjectStore(args) {
                 responseText: await safeText(res),
             });
             if (isObjectNotFoundError(error))
-                return "absent";
+                return 'absent';
             throw error;
         },
     };
@@ -296,29 +236,17 @@ async function safeText(res) {
         return (await res.text()).slice(0, 500);
     }
     catch {
-        return "<no body>";
+        return '<no body>';
     }
 }
-// ---------------------------------------------------------------------------
-// Firebase Storage stores — deployed (token-free) and emulator (fixed token)
-// ---------------------------------------------------------------------------
-/** Custom-metadata key Firebase Storage reads download tokens from. */
-const DOWNLOAD_TOKENS_METADATA_KEY = "firebaseStorageDownloadTokens";
-/**
- * How many times `deleteCopy` checks a Firebase Storage destination before giving up. Keys
- * are write-once, so one change between the check and the delete is already rare; an object
- * still changing after that is left alone and the precondition failure is rethrown.
- */
+const DOWNLOAD_TOKENS_METADATA_KEY = 'firebaseStorageDownloadTokens';
 const DELETE_COPY_MAX_READS = 2;
-/** Firebase Storage store for deployed environments. Token-free (see header). */
 export function createFirebaseStorageObjectStore(args) {
     return createFirebaseStore(args.bucket, null);
 }
-/** Local dev / tests only: the Firebase Storage store plus a fixed download token. */
 export function createFirebaseEmulatorObjectStore(args) {
     return createFirebaseStore(args.bucket, args.downloadToken);
 }
-/** `downloadToken` is the store's token policy: `null` = token-free, a string = that token on every object. */
 function createFirebaseStore(bucket, downloadToken) {
     async function deleteObject(key) {
         try {
@@ -334,37 +262,38 @@ function createFirebaseStore(bucket, downloadToken) {
         async putFile({ localPath, key, contentType }) {
             const { size } = await stat(localPath);
             const ct = contentType ?? getMimeFromExt(extractExt(localPath));
-            // An upload replaces the object and all of its metadata, so a token-free
-            // put carries no custom metadata at all.
             await bucket.upload(localPath, {
                 destination: key,
                 metadata: {
                     contentType: ct,
                     cacheControl: DEFAULT_CACHE_CONTROL,
-                    ...(downloadToken === null ? {} : { metadata: { [DOWNLOAD_TOKENS_METADATA_KEY]: downloadToken } }),
+                    ...(downloadToken === null
+                        ? {}
+                        : { metadata: { [DOWNLOAD_TOKENS_METADATA_KEY]: downloadToken } }),
                 },
             });
             return { key, sizeBytes: size, contentType: ct };
         },
         async copy({ fromKey, toKey }) {
             const [source] = await bucket.file(fromKey).getMetadata();
-            const generation = requireStorageObjectGeneration(source, { bucket: bucket.name, key: fromKey });
+            const generation = requireStorageObjectGeneration(source, {
+                bucket: bucket.name,
+                key: fromKey,
+            });
             const provenance = copyProvenance(fromKey, generation);
             const contentHeaders = {};
             for (const field of Object.values(CARRIED_CONTENT_HEADERS)) {
                 const value = source[field];
-                if (typeof value === "string" && value !== "")
+                if (typeof value === 'string' && value !== '')
                     contentHeaders[field] = value;
             }
             try {
-                // A rewrite's resource REPLACES the destination's metadata rather than merging with
-                // the source's, so this one call commits the destination fully formed: the source's
-                // content headers, the provenance, and this store's token policy — never a token the
-                // source carried. The source is pinned to the generation the provenance names, so a
-                // 412 can only mean the destination exists; a replaced source fails as not-found.
+                // A rewrite's resource replaces the destination's metadata rather than merging with the source's.
                 await bucket.file(fromKey, { generation }).copy(bucket.file(toKey), {
                     ...contentHeaders,
-                    metadata: downloadToken === null ? provenance : { ...provenance, [DOWNLOAD_TOKENS_METADATA_KEY]: downloadToken },
+                    metadata: downloadToken === null
+                        ? provenance
+                        : { ...provenance, [DOWNLOAD_TOKENS_METADATA_KEY]: downloadToken },
                     preconditionOpts: { ifGenerationMatch: 0 },
                 });
                 return { key: toKey, alreadyPresent: false };
@@ -409,21 +338,19 @@ function createFirebaseStore(bucket, downloadToken) {
                 }
                 catch (e) {
                     if (isObjectNotFoundError(e))
-                        return "absent";
+                        return 'absent';
                     throw e;
                 }
                 if (!recordsCopyOf((name) => custom?.[name], fromKey))
-                    return "notThisCopy";
-                // The delete names the generation just checked, so it can only remove that object.
+                    return 'notThisCopy';
                 const checked = requireStorageObjectGeneration({ generation }, { bucket: bucket.name, key: toKey });
                 try {
                     await file.delete({ ifGenerationMatch: checked });
-                    return "deleted";
+                    return 'deleted';
                 }
                 catch (e) {
                     if (isObjectNotFoundError(e))
-                        return "absent";
-                    // A 412 means the object changed after it was checked: check what is there now.
+                        return 'absent';
                     if (!isPreconditionFailedError(e) || read >= DELETE_COPY_MAX_READS)
                         throw e;
                 }
@@ -431,33 +358,31 @@ function createFirebaseStore(bucket, downloadToken) {
         },
     };
 }
-/** The Cloud Storage Admin SDK's numeric `412`: a generation precondition did not hold. */
 function isPreconditionFailedError(e) {
     return e?.code === 412;
 }
-// ---------------------------------------------------------------------------
 function extractExt(localPath) {
-    return localPath.split(".").pop() || "";
+    return localPath.split('.').pop() || '';
 }
 export function getMimeFromExt(ext) {
     switch (ext.toLowerCase()) {
-        case "jpg":
-        case "jpeg":
-            return "image/jpeg";
-        case "png":
-            return "image/png";
-        case "webp":
-            return "image/webp";
-        case "mp4":
-            return "video/mp4";
-        case "webm":
-            return "video/webm";
-        case "m4a":
-            return "audio/mp4";
-        case "aac":
-            return "audio/aac";
-        case "mp3":
-            return "audio/mpeg";
+        case 'jpg':
+        case 'jpeg':
+            return 'image/jpeg';
+        case 'png':
+            return 'image/png';
+        case 'webp':
+            return 'image/webp';
+        case 'mp4':
+            return 'video/mp4';
+        case 'webm':
+            return 'video/webm';
+        case 'm4a':
+            return 'audio/mp4';
+        case 'aac':
+            return 'audio/aac';
+        case 'mp3':
+            return 'audio/mpeg';
         default:
             return NEUTRAL_CONTENT_TYPE;
     }