@devultur/self-host 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +127 -4
- package/dist/index.js +783 -84
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as hono_types from 'hono/types';
|
|
2
2
|
import { Context, MiddlewareHandler, Hono } from 'hono';
|
|
3
|
-
import { Visibility, ObjectLock, SecurityPolicy, CacheRule, TranscodeSettings, TranscodeOutputKeys, TranscodeOutputUrls } from '@devultur/core';
|
|
3
|
+
import { Visibility, ObjectLock, SecurityPolicy, CacheRule, TranscodeSettings, RetentionMode, TranscodeOutputKeys, TranscodeOutputUrls } from '@devultur/core';
|
|
4
4
|
import { T as TranscodeContainerNamespace } from './cloudflare-kZ7J3LQ2.js';
|
|
5
5
|
|
|
6
6
|
/** What an upload token allows (claims `res`, `visibility`, `max_bytes`, `max_files`, `file_ttl`, `lock`). */
|
|
@@ -167,7 +167,22 @@ interface WorkerConfig {
|
|
|
167
167
|
*/
|
|
168
168
|
lock?: ObjectLock;
|
|
169
169
|
transcode?: TranscodeConfig;
|
|
170
|
+
/**
|
|
171
|
+
* What an upload whose first bytes contradict its declared Content-Type gets (see `ContentSniffingMode`). Every
|
|
172
|
+
* mismatch is logged (`[content-sniff] {declared, detected, key, action}`). A transcode's outputs are never checked.
|
|
173
|
+
* Unset: `scriptable`.
|
|
174
|
+
*/
|
|
175
|
+
contentSniffing?: ContentSniffingMode;
|
|
170
176
|
}
|
|
177
|
+
/**
|
|
178
|
+
* - `scriptable` (default): `415 content_type_mismatch` only when the bytes are markup (HTML, SVG, XML) under an inert
|
|
179
|
+
* type; any other mismatch is stored with its declared type.
|
|
180
|
+
* - `reject`: `415` for every mismatch.
|
|
181
|
+
* - `octet-stream`: a single PUT that mismatches is stored as `application/octet-stream`; a multipart upload is
|
|
182
|
+
* rejected (R2 fixes its type when it starts).
|
|
183
|
+
* - `off`: no check.
|
|
184
|
+
*/
|
|
185
|
+
type ContentSniffingMode = "scriptable" | "reject" | "octet-stream" | "off";
|
|
171
186
|
/** What an auth plugin established about the request's credential. */
|
|
172
187
|
interface RequestAccess {
|
|
173
188
|
/**
|
|
@@ -444,6 +459,9 @@ declare const LOCK_UNTIL_HEADER = "x-devultur-lock-until";
|
|
|
444
459
|
/** `true` while a hold is set. */
|
|
445
460
|
declare const LOCK_HOLD_HEADER = "x-devultur-lock-hold";
|
|
446
461
|
|
|
462
|
+
/** `customMetadata` key holding the lock set at upload (JSON `{ retention?, hold }`). Owned by the Worker. */
|
|
463
|
+
declare const LOCK_METADATA_KEY = "lock";
|
|
464
|
+
|
|
447
465
|
/** Prefixes the Worker keeps for lock sidecars and audit records: no route reads or writes a key under them. */
|
|
448
466
|
declare const RESERVED_LOCK_PREFIXES: readonly ["__lock/", "__lock-audit/"];
|
|
449
467
|
|
|
@@ -458,8 +476,113 @@ interface HoldSkip {
|
|
|
458
476
|
reason: "not_found" | "unchanged" | "lock_conflict" | "lock_record_unreadable";
|
|
459
477
|
}
|
|
460
478
|
|
|
461
|
-
/**
|
|
462
|
-
|
|
479
|
+
/** The lock in force on one stored object. */
|
|
480
|
+
interface StoredObjectLock {
|
|
481
|
+
/** True while the object can't be deleted or overwritten. */
|
|
482
|
+
locked: boolean;
|
|
483
|
+
/** A hold is set (an unreadable lock record reads as one). */
|
|
484
|
+
hold: boolean;
|
|
485
|
+
/** ISO time the retention ends; left out with no retention, or while a hold keeps a `period` from starting. */
|
|
486
|
+
until?: string;
|
|
487
|
+
mode?: RetentionMode;
|
|
488
|
+
/** The lock record can't be read, so it's held until an operator repairs it. */
|
|
489
|
+
unreadable?: true;
|
|
490
|
+
/**
|
|
491
|
+
* The key isn't laid out like the Worker's (`{storage prefix}{public|private}/…`), so its lock record can't be
|
|
492
|
+
* found: it's kept, never deleted.
|
|
493
|
+
*/
|
|
494
|
+
unrecognized?: true;
|
|
495
|
+
/** Storage key of the original the lock belongs to, when it isn't the object itself (a video's raw, say). */
|
|
496
|
+
lockedBy?: string;
|
|
497
|
+
}
|
|
498
|
+
interface LockLookupOptions {
|
|
499
|
+
mediaBucket: R2Bucket;
|
|
500
|
+
/**
|
|
501
|
+
* The storage prefix the keys live under (the Worker's `storagePrefix`, `""` for the bucket root). Omitted: read
|
|
502
|
+
* from each key, as everything before its first `public` or `private` segment. Pass it when a prefix of yours
|
|
503
|
+
* has a segment named `public` or `private`.
|
|
504
|
+
*/
|
|
505
|
+
root?: string;
|
|
506
|
+
/** Default: now. */
|
|
507
|
+
now?: Date;
|
|
508
|
+
}
|
|
509
|
+
/**
|
|
510
|
+
* The lock in force on the object at `storageKey` (the full R2 key), or `null` when there's no such object (or it's
|
|
511
|
+
* a lock record). A file under a video's folder (an HLS segment, a caption) is locked by the video's raws unless
|
|
512
|
+
* they're `original-only`; an image variant by its original, likewise.
|
|
513
|
+
*/
|
|
514
|
+
declare function readObjectLock(options: LockLookupOptions, storageKey: string): Promise<StoredObjectLock | null>;
|
|
515
|
+
|
|
516
|
+
interface PrefixWalkOptions extends LockLookupOptions {
|
|
517
|
+
/** Storage prefix to walk (full R2 keys, the storage prefix of the keys included). */
|
|
518
|
+
prefix: string;
|
|
519
|
+
/** From a previous call's result, to go on where it stopped. */
|
|
520
|
+
cursor?: string;
|
|
521
|
+
/** Objects listed per page. Default 100, at most 1000. */
|
|
522
|
+
batchSize?: number;
|
|
523
|
+
/**
|
|
524
|
+
* Bucket calls this call may make. Default 40: under the free plan's 50 subrequests per request, with room for the
|
|
525
|
+
* caller's own. Raise it on a paid plan.
|
|
526
|
+
*/
|
|
527
|
+
maxSubrequests?: number;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/**
|
|
531
|
+
* No `now`: locks are judged by the real clock, so no caller can make a retention read as expired. (The read-only
|
|
532
|
+
* `readObjectLock` and `retainedSummary` take one, to ask "what about then?".)
|
|
533
|
+
*/
|
|
534
|
+
interface PrefixDeleteOptions extends Omit<PrefixWalkOptions, "now"> {
|
|
535
|
+
/** The Worker's `KEYS_BUCKET`, when it has one: the encryption keys of what's deleted go too. */
|
|
536
|
+
keysBucket?: R2Bucket;
|
|
537
|
+
}
|
|
538
|
+
/** An object kept because it's locked (or its original is, or its key isn't laid out like the Worker's). */
|
|
539
|
+
interface RetainedObject extends StoredObjectLock {
|
|
540
|
+
/** Its storage key. */
|
|
541
|
+
key: string;
|
|
542
|
+
size: number;
|
|
543
|
+
}
|
|
544
|
+
interface PrefixDeleteResult {
|
|
545
|
+
deletedCount: number;
|
|
546
|
+
/** The sizes of the versions deleted, read right before each delete. */
|
|
547
|
+
deletedBytes: number;
|
|
548
|
+
/** Encryption keys deleted from the keys bucket. */
|
|
549
|
+
deletedKeyCount: number;
|
|
550
|
+
retainedCount: number;
|
|
551
|
+
retainedBytes: number;
|
|
552
|
+
/** What this call kept, in key order. */
|
|
553
|
+
retained: RetainedObject[];
|
|
554
|
+
/** Lock records and audit objects passed over (never deleted). */
|
|
555
|
+
skippedCount: number;
|
|
556
|
+
/** Set when the prefix isn't done: pass it back to go on. */
|
|
557
|
+
cursor?: string;
|
|
558
|
+
/** Bucket calls this call made. */
|
|
559
|
+
subrequests: number;
|
|
560
|
+
}
|
|
561
|
+
/**
|
|
562
|
+
* Deletes what's free under `options.prefix` (full storage keys) and keeps what's locked. Bounded: it never makes more
|
|
563
|
+
* than `maxSubrequests` bucket calls, and returns a `cursor` to call it again with. Nothing is audited: log `retained`
|
|
564
|
+
* if you need a trail. R2 deletes aren't conditional: a lock set between an object's check and its delete isn't seen
|
|
565
|
+
* (milliseconds for its own lock; for a lock it inherits, up to `LOCK_CACHE_MAX_AGE_MS`, as that lock is cached).
|
|
566
|
+
*/
|
|
567
|
+
declare function deletePrefixRespectingLocks(options: PrefixDeleteOptions): Promise<PrefixDeleteResult>;
|
|
568
|
+
|
|
569
|
+
interface RetainedSummary {
|
|
570
|
+
/** Objects looked at (lock records and audit not included). */
|
|
571
|
+
scannedCount: number;
|
|
572
|
+
/** What a delete would keep, keys not laid out like the Worker's included. */
|
|
573
|
+
retainedCount: number;
|
|
574
|
+
retainedBytes: number;
|
|
575
|
+
/** The latest retention end among what's retained (ISO). Holds have none: see `hasHold`. */
|
|
576
|
+
latestUntil?: string;
|
|
577
|
+
/** Something retained is held (or has an unreadable lock record): it stays until released, not until a date. */
|
|
578
|
+
hasHold: boolean;
|
|
579
|
+
/** Set when the prefix isn't done: pass it back to go on. Combine the counts of every call. */
|
|
580
|
+
cursor?: string;
|
|
581
|
+
/** Bucket calls this call made. */
|
|
582
|
+
subrequests: number;
|
|
583
|
+
}
|
|
584
|
+
/** What's retained under `options.prefix` (full storage keys). Bounded and resumable like the delete. */
|
|
585
|
+
declare function retainedSummary(options: PrefixWalkOptions): Promise<RetainedSummary>;
|
|
463
586
|
|
|
464
587
|
declare const LOCK_SIDECAR_PREFIX = "__lock/";
|
|
465
588
|
interface LockSidecar {
|
|
@@ -644,4 +767,4 @@ interface CreateWorkerOptions {
|
|
|
644
767
|
}
|
|
645
768
|
declare function createWorker(options?: CreateWorkerOptions): Hono<WorkerApp, hono_types.BlankSchema, "/">;
|
|
646
769
|
|
|
647
|
-
export { type AllowedCredential, type AssetListEntry, type AuthorizeLockChangeHook, type BeforeLinkRedeemHook, type ComposedPlugins, type CreateWorkerOptions, type CredentialDecision, type CredentialKind, type CredentialPolicyOptions, type CredentialRequest, DownloadLink, type DownloadLinkInfo, type DownloadLinkRecord, type DownloadLinkStorage, ENCRYPTION_KEY_MISSING_CODE, type EncodeMetrics, HOLD_PAGE_SIZE, type HoldSkip, type ImageTransformParams, type IssueCredentialHook, type Json, KEYS_BUCKET_MISSING_CODE, LOCK_AUDIT_PREFIX, LOCK_HOLD_HEADER, LOCK_METADATA_KEY, LOCK_SIDECAR_PREFIX, LOCK_UNTIL_HEADER, type LockAuditAction, type LockAuditRecord, type LockChangeAction, type LockChangeDecision, type LockChangeRequest, type LockSidecar, type LockSource, MAX_ASSET_LIST_LIMIT, MAX_EXPIRED_LOOKUPS, MAX_UPLOAD_TOKEN_TTL_SECONDS, MaxResolution, ObjectMeta, type OnLockAuditHook, type PendingUploadInfo, type PluginContext, type PluginMiddleware, type PluginRoute, RESERVED_LOCK_PREFIXES, type RawStorageClass, type RequestAccess, type RequestState, type RequestStatePatch, type RouteResult, TOKEN_REJECTED_CODE, type TranscodeConfig, type TranscodeJob, UPLOAD_TOKEN_TYPE, type UploadGrant, type UploadRules, type UploadTypeRule, type WorkerApp, WorkerAuthErrorCode, type WorkerConfig, type WorkerEnv, type WorkerPlugin, type WorkerVariables, cancelPendingUpload, createWorker, credentialPolicy, defaultPlugins, deletePrefix, downloadLinks, edgeCacheTag, getPendingUpload, getRequestState, getRouteResult, hashKey, imageOriginAuthorization, isAuthenticated, isImageOriginRequest, isObjectExpired, isOpenRoute, isServiceToken, isUploadUrl, isVariantKey, logEncodeMetrics, parseEncodeMetrics, parseTransformParams, readImageOriginScope, readStoryboardKey, readTokenScopes, readTranscodeOutputKeys, readTranscodeOutputs, servedCacheControl, staticConfig, storageKey, updateRequestState, uploadGrantFromClaims, variantPrefix, vcpuSecondsPerVideoMinute, verifyViewerToken, withOutputUrls, workerSecretAuth };
|
|
770
|
+
export { type AllowedCredential, type AssetListEntry, type AuthorizeLockChangeHook, type BeforeLinkRedeemHook, type ComposedPlugins, type ContentSniffingMode, type CreateWorkerOptions, type CredentialDecision, type CredentialKind, type CredentialPolicyOptions, type CredentialRequest, DownloadLink, type DownloadLinkInfo, type DownloadLinkRecord, type DownloadLinkStorage, ENCRYPTION_KEY_MISSING_CODE, type EncodeMetrics, HOLD_PAGE_SIZE, type HoldSkip, type ImageTransformParams, type IssueCredentialHook, type Json, KEYS_BUCKET_MISSING_CODE, LOCK_AUDIT_PREFIX, LOCK_HOLD_HEADER, LOCK_METADATA_KEY, LOCK_SIDECAR_PREFIX, LOCK_UNTIL_HEADER, type LockAuditAction, type LockAuditRecord, type LockChangeAction, type LockChangeDecision, type LockChangeRequest, type LockLookupOptions, type LockSidecar, type LockSource, MAX_ASSET_LIST_LIMIT, MAX_EXPIRED_LOOKUPS, MAX_UPLOAD_TOKEN_TTL_SECONDS, MaxResolution, ObjectMeta, type OnLockAuditHook, type PendingUploadInfo, type PluginContext, type PluginMiddleware, type PluginRoute, type PrefixDeleteOptions, type PrefixDeleteResult, type PrefixWalkOptions, RESERVED_LOCK_PREFIXES, type RawStorageClass, type RequestAccess, type RequestState, type RequestStatePatch, type RetainedObject, type RetainedSummary, type RouteResult, type StoredObjectLock, TOKEN_REJECTED_CODE, type TranscodeConfig, type TranscodeJob, UPLOAD_TOKEN_TYPE, type UploadGrant, type UploadRules, type UploadTypeRule, type WorkerApp, WorkerAuthErrorCode, type WorkerConfig, type WorkerEnv, type WorkerPlugin, type WorkerVariables, cancelPendingUpload, createWorker, credentialPolicy, defaultPlugins, deletePrefix, deletePrefixRespectingLocks, downloadLinks, edgeCacheTag, getPendingUpload, getRequestState, getRouteResult, hashKey, imageOriginAuthorization, isAuthenticated, isImageOriginRequest, isObjectExpired, isOpenRoute, isServiceToken, isUploadUrl, isVariantKey, logEncodeMetrics, parseEncodeMetrics, parseTransformParams, readImageOriginScope, readObjectLock, readStoryboardKey, readTokenScopes, readTranscodeOutputKeys, readTranscodeOutputs, retainedSummary, servedCacheControl, staticConfig, storageKey, updateRequestState, uploadGrantFromClaims, variantPrefix, vcpuSecondsPerVideoMinute, verifyViewerToken, withOutputUrls, workerSecretAuth };
|
package/dist/index.js
CHANGED
|
@@ -73,7 +73,7 @@ import { cors } from "hono/cors";
|
|
|
73
73
|
import { logger } from "hono/logger";
|
|
74
74
|
|
|
75
75
|
// src/object-lock/guard.ts
|
|
76
|
-
import {
|
|
76
|
+
import { isVideoRawKey } from "@devultur/core";
|
|
77
77
|
|
|
78
78
|
// src/plugin-hooks.ts
|
|
79
79
|
var PLUGIN_HOOKS_VARIABLE = "pluginHooks";
|
|
@@ -152,11 +152,47 @@ async function recordLockAudit(context, input) {
|
|
|
152
152
|
await inBackground(context, deliver(context, root, record));
|
|
153
153
|
}
|
|
154
154
|
|
|
155
|
+
// src/object-lock/object-keys.ts
|
|
156
|
+
import { decodeEncryptionMeta } from "@devultur/core";
|
|
157
|
+
var DELETE_BATCH_SIZE = 1e3;
|
|
158
|
+
async function listAll(bucket, options) {
|
|
159
|
+
const objects = [];
|
|
160
|
+
let cursor;
|
|
161
|
+
do {
|
|
162
|
+
const listing = await bucket.list({ ...options, ...cursor ? { cursor } : {} });
|
|
163
|
+
objects.push(...listing.objects);
|
|
164
|
+
cursor = listing.truncated ? listing.cursor : void 0;
|
|
165
|
+
} while (cursor);
|
|
166
|
+
return objects;
|
|
167
|
+
}
|
|
168
|
+
var isWrittenBy = (key, writtenBy) => key.uploaded.getTime() <= writtenBy.getTime();
|
|
169
|
+
async function keysWrittenBy(keysBucket, prefix, writtenBy) {
|
|
170
|
+
return (await listAll(keysBucket, { prefix })).filter((key) => isWrittenBy(key, writtenBy)).map((key) => key.key);
|
|
171
|
+
}
|
|
172
|
+
async function objectKeysToDelete(keysBucket, fullKey, deleted) {
|
|
173
|
+
const own = await listAll(keysBucket, { prefix: `${fullKey}/`, delimiter: "/" });
|
|
174
|
+
const keys = new Set(own.filter((key) => isWrittenBy(key, deleted.uploaded)).map((key) => key.key));
|
|
175
|
+
const meta = decodeEncryptionMeta(deleted.customMetadata?.enc);
|
|
176
|
+
if (meta) keys.add(`${fullKey}/${meta.keyId}`);
|
|
177
|
+
for (const variantKey of await listAll(keysBucket, { prefix: variantPrefix(fullKey) })) keys.add(variantKey.key);
|
|
178
|
+
return [...keys];
|
|
179
|
+
}
|
|
180
|
+
var deleteCallsFor = (keys) => Math.ceil(keys.length / DELETE_BATCH_SIZE);
|
|
181
|
+
async function deleteKeys(keysBucket, keys) {
|
|
182
|
+
for (let start = 0; start < keys.length; start += DELETE_BATCH_SIZE) {
|
|
183
|
+
await keysBucket.delete(keys.slice(start, start + DELETE_BATCH_SIZE));
|
|
184
|
+
}
|
|
185
|
+
return keys.length;
|
|
186
|
+
}
|
|
187
|
+
async function sweepObjectKeys(keysBucket, fullKey, deleted) {
|
|
188
|
+
return deleteKeys(keysBucket, await objectKeysToDelete(keysBucket, fullKey, deleted));
|
|
189
|
+
}
|
|
190
|
+
|
|
155
191
|
// src/object-lock/sidecar.ts
|
|
156
192
|
import { isIsoInstant, validateLock as validateLock2 } from "@devultur/core";
|
|
157
193
|
|
|
158
194
|
// src/object-lock/state.ts
|
|
159
|
-
import { isLocked, validateLock } from "@devultur/core";
|
|
195
|
+
import { effectiveUntil, isLocked, validateLock } from "@devultur/core";
|
|
160
196
|
var LOCK_METADATA_KEY = "lock";
|
|
161
197
|
function encodeLockMetadata(lock) {
|
|
162
198
|
if (!lock || !lock.retention && !lock.hold) return void 0;
|
|
@@ -187,6 +223,13 @@ function isStateLocked(state, now) {
|
|
|
187
223
|
return true;
|
|
188
224
|
}
|
|
189
225
|
}
|
|
226
|
+
function untilOf(state) {
|
|
227
|
+
try {
|
|
228
|
+
return effectiveUntil(state.lock, { uploadedAt: state.uploadedAt, holdReleasedAt: state.holdReleasedAt });
|
|
229
|
+
} catch {
|
|
230
|
+
return void 0;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
190
233
|
|
|
191
234
|
// src/object-lock/sidecar.ts
|
|
192
235
|
var LOCK_SIDECAR_PREFIX = "__lock/";
|
|
@@ -289,32 +332,34 @@ function protectsDerived(state) {
|
|
|
289
332
|
const derivatives = state.lock.retention?.derivatives ?? "inherit";
|
|
290
333
|
return derivatives === "inherit";
|
|
291
334
|
}
|
|
292
|
-
async function
|
|
293
|
-
const
|
|
294
|
-
const state = object ? (await effectiveLock(context.env.MEDIA_BUCKET, root, key, object)).state : null;
|
|
335
|
+
async function lockedObject(bucket, root, key, object, now) {
|
|
336
|
+
const { state } = await effectiveLock(bucket, root, key, object);
|
|
295
337
|
return state && isStateLocked(state, now) ? { key, state } : null;
|
|
296
338
|
}
|
|
297
|
-
async function
|
|
339
|
+
async function lockedOriginal(bucket, root, key, now) {
|
|
340
|
+
const object = await bucket.head(`${root}${key}`);
|
|
341
|
+
return object ? lockedObject(bucket, root, key, object, now) : null;
|
|
342
|
+
}
|
|
343
|
+
async function lockedRaw(bucket, root, rawPrefix, now) {
|
|
298
344
|
let cursor;
|
|
299
345
|
do {
|
|
300
|
-
const listing = await
|
|
346
|
+
const listing = await bucket.list({ prefix: `${root}${rawPrefix}`, cursor });
|
|
301
347
|
for (const listed of listing.objects) {
|
|
302
348
|
const key = listed.key.slice(root.length);
|
|
303
|
-
const object = await
|
|
304
|
-
const state = object ? (await effectiveLock(
|
|
349
|
+
const object = await bucket.head(listed.key);
|
|
350
|
+
const state = object ? (await effectiveLock(bucket, root, key, object)).state : null;
|
|
305
351
|
if (state && protectsDerived(state) && isStateLocked(state, now)) return { key, state };
|
|
306
352
|
}
|
|
307
353
|
cursor = listing.truncated ? listing.cursor : void 0;
|
|
308
354
|
} while (cursor);
|
|
309
355
|
return null;
|
|
310
356
|
}
|
|
357
|
+
function findLockedOriginal(bucket, root, subject, now) {
|
|
358
|
+
return subject.kind === "original" ? lockedOriginal(bucket, root, subject.key, now) : lockedRaw(bucket, root, subject.rawPrefix, now);
|
|
359
|
+
}
|
|
311
360
|
function retentionActive(context, locked, op) {
|
|
312
|
-
const { lock
|
|
313
|
-
|
|
314
|
-
try {
|
|
315
|
-
until = effectiveUntil(lock, { uploadedAt, holdReleasedAt });
|
|
316
|
-
} catch {
|
|
317
|
-
}
|
|
361
|
+
const { lock } = locked.state;
|
|
362
|
+
const until = untilOf(locked.state);
|
|
318
363
|
const verb = op === "delete" ? "deleted" : "overwritten";
|
|
319
364
|
return context.json(
|
|
320
365
|
{
|
|
@@ -339,7 +384,7 @@ async function assertWritable(context, keys, op, root = getRequestState(context)
|
|
|
339
384
|
const subjectId = subject.kind === "original" ? subject.key : subject.rawPrefix;
|
|
340
385
|
if (checked.has(subjectId)) continue;
|
|
341
386
|
checked.add(subjectId);
|
|
342
|
-
const locked =
|
|
387
|
+
const locked = await findLockedOriginal(context.env.MEDIA_BUCKET, root, subject, now);
|
|
343
388
|
if (!locked) continue;
|
|
344
389
|
const requested = key === locked.key ? [key] : [key, locked.key];
|
|
345
390
|
await recordLockAudit(context, {
|
|
@@ -355,7 +400,7 @@ async function assertWritable(context, keys, op, root = getRequestState(context)
|
|
|
355
400
|
}
|
|
356
401
|
return null;
|
|
357
402
|
}
|
|
358
|
-
var
|
|
403
|
+
var DELETE_BATCH_SIZE2 = 1e3;
|
|
359
404
|
async function guardedDeletePrefix(context, prefix) {
|
|
360
405
|
const root = getRequestState(context).storagePrefix;
|
|
361
406
|
const storageKeys = [];
|
|
@@ -368,38 +413,17 @@ async function guardedDeletePrefix(context, prefix) {
|
|
|
368
413
|
const keys = storageKeys.map((key) => key.slice(root.length));
|
|
369
414
|
const blocked = await assertWritable(context, keys, "delete");
|
|
370
415
|
if (blocked) return { blocked };
|
|
371
|
-
for (let start = 0; start < storageKeys.length; start +=
|
|
372
|
-
await context.env.MEDIA_BUCKET.delete(storageKeys.slice(start, start +
|
|
416
|
+
for (let start = 0; start < storageKeys.length; start += DELETE_BATCH_SIZE2) {
|
|
417
|
+
await context.env.MEDIA_BUCKET.delete(storageKeys.slice(start, start + DELETE_BATCH_SIZE2));
|
|
373
418
|
}
|
|
374
419
|
return { deletedCount: storageKeys.length };
|
|
375
420
|
}
|
|
376
|
-
async function ownKeysOf(keysBucket, fullKey) {
|
|
377
|
-
const keys = [];
|
|
378
|
-
let cursor;
|
|
379
|
-
do {
|
|
380
|
-
const listing = await keysBucket.list({ prefix: `${fullKey}/`, delimiter: "/", cursor });
|
|
381
|
-
keys.push(...listing.objects);
|
|
382
|
-
cursor = listing.truncated ? listing.cursor : void 0;
|
|
383
|
-
} while (cursor);
|
|
384
|
-
return keys;
|
|
385
|
-
}
|
|
386
|
-
function isKeyOfDeletedVersion(key, deleted) {
|
|
387
|
-
return key.uploaded.getTime() <= deleted.uploaded.getTime();
|
|
388
|
-
}
|
|
389
421
|
async function deleteKeysFor(context, objectKey, deleted) {
|
|
390
422
|
const blocked = await assertWritable(context, [objectKey], "delete");
|
|
391
423
|
if (blocked) return { blocked };
|
|
392
424
|
const keysBucket = context.env.KEYS_BUCKET;
|
|
393
425
|
if (!keysBucket) return { deletedCount: 0 };
|
|
394
|
-
|
|
395
|
-
const ownKeys = new Set(
|
|
396
|
-
(await ownKeysOf(keysBucket, fullKey)).filter((key) => isKeyOfDeletedVersion(key, deleted)).map((key) => key.key)
|
|
397
|
-
);
|
|
398
|
-
const meta = decodeEncryptionMeta(deleted.customMetadata?.enc);
|
|
399
|
-
if (meta) ownKeys.add(`${fullKey}/${meta.keyId}`);
|
|
400
|
-
if (ownKeys.size > 0) await keysBucket.delete([...ownKeys]);
|
|
401
|
-
const variantKeyCount = await deletePrefix(keysBucket, variantPrefix(fullKey));
|
|
402
|
-
return { deletedCount: variantKeyCount + ownKeys.size };
|
|
426
|
+
return { deletedCount: await sweepObjectKeys(keysBucket, storageKey(context, objectKey), deleted) };
|
|
403
427
|
}
|
|
404
428
|
async function deleteFolderKeys(context, videoFolder) {
|
|
405
429
|
const blocked = await assertWritable(context, [videoFolder], "delete");
|
|
@@ -777,6 +801,62 @@ function addRouteResult(context, result) {
|
|
|
777
801
|
context.set(ROUTE_RESULT_VARIABLE, { ...getRouteResult(context), ...result });
|
|
778
802
|
}
|
|
779
803
|
|
|
804
|
+
// src/routes/content-safety.ts
|
|
805
|
+
var SANDBOX_CSP = "sandbox; default-src 'none'";
|
|
806
|
+
var SAFE_INLINE_TYPES = /* @__PURE__ */ new Set([
|
|
807
|
+
"image/png",
|
|
808
|
+
"image/jpeg",
|
|
809
|
+
"image/webp",
|
|
810
|
+
"image/gif",
|
|
811
|
+
"image/avif",
|
|
812
|
+
"application/pdf",
|
|
813
|
+
"text/plain",
|
|
814
|
+
"text/csv",
|
|
815
|
+
"application/json",
|
|
816
|
+
"text/vtt",
|
|
817
|
+
"application/vnd.apple.mpegurl",
|
|
818
|
+
"application/x-mpegurl"
|
|
819
|
+
]);
|
|
820
|
+
var SAFE_INLINE_FAMILIES = ["video/", "audio/"];
|
|
821
|
+
function contentTypeEssence(contentType) {
|
|
822
|
+
return contentType?.split(";")[0]?.trim().toLowerCase() ?? "";
|
|
823
|
+
}
|
|
824
|
+
function isSafeInlineType(contentType) {
|
|
825
|
+
const essence = contentTypeEssence(contentType);
|
|
826
|
+
if (SAFE_INLINE_TYPES.has(essence)) return true;
|
|
827
|
+
return SAFE_INLINE_FAMILIES.some((family) => essence.startsWith(family));
|
|
828
|
+
}
|
|
829
|
+
function attachmentOf(disposition, filename) {
|
|
830
|
+
if (disposition?.startsWith("attachment")) return disposition;
|
|
831
|
+
if (disposition?.startsWith("inline")) return `attachment${disposition.slice("inline".length)}`;
|
|
832
|
+
return contentDispositionAttachment(filename || "download");
|
|
833
|
+
}
|
|
834
|
+
function applyContentSafety(headers, filename, isPublic) {
|
|
835
|
+
headers.set("X-Content-Type-Options", "nosniff");
|
|
836
|
+
if (isPublic) headers.set("Cross-Origin-Resource-Policy", "cross-origin");
|
|
837
|
+
if (isSafeInlineType(headers.get("Content-Type"))) return;
|
|
838
|
+
headers.set("Content-Security-Policy", SANDBOX_CSP);
|
|
839
|
+
headers.set("Content-Disposition", attachmentOf(headers.get("Content-Disposition"), filename));
|
|
840
|
+
}
|
|
841
|
+
function basenameOf(url) {
|
|
842
|
+
const last = new URL(url).pathname.split("/").pop() ?? "";
|
|
843
|
+
try {
|
|
844
|
+
return decodeURIComponent(last);
|
|
845
|
+
} catch {
|
|
846
|
+
return last;
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
function isPublicMediaUrl(url) {
|
|
850
|
+
return new URL(url).pathname.startsWith("/v1/media/public/");
|
|
851
|
+
}
|
|
852
|
+
var contentSafety = async (context, next) => {
|
|
853
|
+
await next();
|
|
854
|
+
const headers = new Headers(context.res.headers);
|
|
855
|
+
applyContentSafety(headers, basenameOf(context.req.url), isPublicMediaUrl(context.req.url));
|
|
856
|
+
const { body, status, statusText } = context.res;
|
|
857
|
+
context.res = new Response(body, { status, statusText, headers });
|
|
858
|
+
};
|
|
859
|
+
|
|
780
860
|
// src/routes/object-response.ts
|
|
781
861
|
import {
|
|
782
862
|
AES_BLOCK_BYTES,
|
|
@@ -841,6 +921,25 @@ async function digestBody(body, declared, header) {
|
|
|
841
921
|
if (expected && expected !== actual) return { mismatch: { expected, actual } };
|
|
842
922
|
return { sha256: actual };
|
|
843
923
|
}
|
|
924
|
+
async function checkPartSha256(context, multipart, body) {
|
|
925
|
+
const partSha256 = headerSha256(context);
|
|
926
|
+
if (partSha256 === null) {
|
|
927
|
+
return context.json({ error: "x-devultur-sha256 must be a 64-character hex string", code: "invalid_sha256" }, 400);
|
|
928
|
+
}
|
|
929
|
+
if (multipart.sha256 && !partSha256) {
|
|
930
|
+
return context.json(
|
|
931
|
+
{
|
|
932
|
+
error: "Each part must carry its x-devultur-sha256 when the upload declares one",
|
|
933
|
+
code: "part_sha256_required"
|
|
934
|
+
},
|
|
935
|
+
400
|
|
936
|
+
);
|
|
937
|
+
}
|
|
938
|
+
if (!partSha256) return null;
|
|
939
|
+
const digest = await digestBody(body, void 0, partSha256);
|
|
940
|
+
if ("mismatch" in digest) return sha256Mismatch(context, digest.mismatch.expected, digest.mismatch.actual);
|
|
941
|
+
return null;
|
|
942
|
+
}
|
|
844
943
|
function lockMetadata(lock) {
|
|
845
944
|
const encoded = encodeLockMetadata(lock);
|
|
846
945
|
return encoded ? { [LOCK_METADATA_KEY]: encoded } : {};
|
|
@@ -1029,6 +1128,7 @@ async function gateRedemption(context, stub) {
|
|
|
1029
1128
|
return gateLinkRedemption(context, linkInfoOf(await peeked.json()));
|
|
1030
1129
|
}
|
|
1031
1130
|
var download = new Hono2();
|
|
1131
|
+
download.use("*", contentSafety);
|
|
1032
1132
|
download.get("/:linkId", async (context) => {
|
|
1033
1133
|
if (context.req.method === "HEAD") return new Response(null, { status: 405, headers: { Allow: "GET" } });
|
|
1034
1134
|
const linkId = context.req.param("linkId");
|
|
@@ -1232,6 +1332,14 @@ function inferMediaType(contentType) {
|
|
|
1232
1332
|
if (contentType.startsWith("application/")) return MediaType.FILE;
|
|
1233
1333
|
return null;
|
|
1234
1334
|
}
|
|
1335
|
+
function resolveUploadMediaType(declared, contentType, isAssetRequest) {
|
|
1336
|
+
const inferredType = inferMediaType(contentType);
|
|
1337
|
+
if (declared && inferredType && declared !== inferredType) {
|
|
1338
|
+
return failure(400, "media_type_mismatch", `mediaType "${declared}" does not match content type ${contentType}`);
|
|
1339
|
+
}
|
|
1340
|
+
const mediaType = declared ?? inferredType ?? (isAssetRequest ? MediaType.FILE : null);
|
|
1341
|
+
return mediaType ?? failure(400, "unsupported_content_type", `Unsupported content type: ${contentType}`);
|
|
1342
|
+
}
|
|
1235
1343
|
|
|
1236
1344
|
// src/routes/token-upload.ts
|
|
1237
1345
|
var UPLOAD_TOKEN_TYPE = "upload";
|
|
@@ -2484,8 +2592,8 @@ function readJpeg(bytes) {
|
|
|
2484
2592
|
return null;
|
|
2485
2593
|
}
|
|
2486
2594
|
function readPng(bytes) {
|
|
2487
|
-
const
|
|
2488
|
-
if (!
|
|
2595
|
+
const isPng2 = bytes[0] === 137 && bytes[1] === 80 && bytes[2] === 78 && bytes[3] === 71;
|
|
2596
|
+
if (!isPng2 || bytes.length < 24) return null;
|
|
2489
2597
|
return { width: uint32BE(bytes, 16), height: uint32BE(bytes, 20) };
|
|
2490
2598
|
}
|
|
2491
2599
|
function readWebp(bytes) {
|
|
@@ -2896,6 +3004,7 @@ function downloadDisposition(storedDisposition, key, filename) {
|
|
|
2896
3004
|
return contentDispositionAttachment(key.split("/").pop() || "download");
|
|
2897
3005
|
}
|
|
2898
3006
|
var media = new Hono7();
|
|
3007
|
+
media.use("*", contentSafety);
|
|
2899
3008
|
media.get("/:key{.+}", async (context) => {
|
|
2900
3009
|
const key = context.req.param("key");
|
|
2901
3010
|
const fullKey = storageKey(context, key);
|
|
@@ -3018,6 +3127,7 @@ media.get("/:key{.+}", async (context) => {
|
|
|
3018
3127
|
headers.set("Content-Disposition", object.httpMetadata.contentDisposition);
|
|
3019
3128
|
}
|
|
3020
3129
|
if (isOwnerAccess) for (const [name, value] of await lockHeaders(context, key, object)) headers.set(name, value);
|
|
3130
|
+
applyContentSafety(headers, key.split("/").pop() ?? "", !isPrivate);
|
|
3021
3131
|
const { response, kind } = await streamObject({
|
|
3022
3132
|
bucket: context.env.MEDIA_BUCKET,
|
|
3023
3133
|
keysBucket: context.env.KEYS_BUCKET,
|
|
@@ -4345,6 +4455,17 @@ var DERIVED_CONTENT_TYPES = /* @__PURE__ */ new Set([
|
|
|
4345
4455
|
function isDerivedAssetKey(key) {
|
|
4346
4456
|
return DERIVED_ASSET_KEY.test(key);
|
|
4347
4457
|
}
|
|
4458
|
+
var DERIVED_TYPES_BY_FOLDER = {
|
|
4459
|
+
hls: /* @__PURE__ */ new Set(["application/vnd.apple.mpegurl", "video/mp2t"]),
|
|
4460
|
+
mp4: /* @__PURE__ */ new Set(["video/mp4"]),
|
|
4461
|
+
thumbnails: /* @__PURE__ */ new Set(["image/jpeg"]),
|
|
4462
|
+
storyboard: /* @__PURE__ */ new Set(["image/jpeg", "text/vtt"])
|
|
4463
|
+
};
|
|
4464
|
+
var DERIVED_OUTPUT_KEY = /^(?:public|private)\/video\/[A-Za-z0-9-]+\/(hls|mp4|thumbnails|storyboard)\/.+$/;
|
|
4465
|
+
function isTrustedDerivedOutput(key, contentType) {
|
|
4466
|
+
const folder = DERIVED_OUTPUT_KEY.exec(key)?.[1];
|
|
4467
|
+
return folder !== void 0 && (DERIVED_TYPES_BY_FOLDER[folder]?.has(contentType) ?? false);
|
|
4468
|
+
}
|
|
4348
4469
|
var RAW_LIST_LIMIT = 100;
|
|
4349
4470
|
async function inheritedExpiresAt(context, key) {
|
|
4350
4471
|
if (!isDerivedAssetKey(key)) return void 0;
|
|
@@ -4367,6 +4488,7 @@ async function derivedAssetPending(context, key) {
|
|
|
4367
4488
|
return {
|
|
4368
4489
|
contentType: DERIVED_CONTENT_TYPES.has(declared) ? declared : "application/octet-stream",
|
|
4369
4490
|
storageKey: storageKey(context, key),
|
|
4491
|
+
isDerived: true,
|
|
4370
4492
|
...security ? { security } : {},
|
|
4371
4493
|
...expiresAt ? { expiresAt } : {}
|
|
4372
4494
|
};
|
|
@@ -4433,12 +4555,298 @@ async function afterUploadWrite(context, key, pending, write, writtenEnc) {
|
|
|
4433
4555
|
});
|
|
4434
4556
|
}
|
|
4435
4557
|
|
|
4558
|
+
// src/routes/upload-content-check.ts
|
|
4559
|
+
import { decodeEncryptionMeta as decodeEncryptionMeta9 } from "@devultur/core";
|
|
4560
|
+
|
|
4561
|
+
// src/markup-sniff.ts
|
|
4562
|
+
var MARKUP_WINDOW_BYTES = 1024;
|
|
4563
|
+
function utf16Encoding(head) {
|
|
4564
|
+
if (head[0] === 255 && head[1] === 254) return "utf-16le";
|
|
4565
|
+
if (head[0] === 254 && head[1] === 255) return "utf-16be";
|
|
4566
|
+
return null;
|
|
4567
|
+
}
|
|
4568
|
+
function isBinaryDataByte(byte) {
|
|
4569
|
+
return byte <= 8 || byte === 11 || byte >= 14 && byte <= 26 || byte >= 28 && byte <= 31;
|
|
4570
|
+
}
|
|
4571
|
+
function decodesStrictly(head, encoding) {
|
|
4572
|
+
try {
|
|
4573
|
+
new TextDecoder(encoding, { fatal: true }).decode(head, { stream: true });
|
|
4574
|
+
return true;
|
|
4575
|
+
} catch {
|
|
4576
|
+
return false;
|
|
4577
|
+
}
|
|
4578
|
+
}
|
|
4579
|
+
function leadingText(head) {
|
|
4580
|
+
const window = head.subarray(0, MARKUP_WINDOW_BYTES);
|
|
4581
|
+
const decoded = new TextDecoder(utf16Encoding(head) ?? "utf-8").decode(window, { stream: true });
|
|
4582
|
+
return decoded.replace(/^/, "").trimStart();
|
|
4583
|
+
}
|
|
4584
|
+
var HTML_TAGS = ["html", "head", "script", "iframe", "h1", "div", "font", "table", "a", "style", "title", "b"];
|
|
4585
|
+
var MORE_HTML_TAGS = ["body", "br", "p"];
|
|
4586
|
+
var DISTINCT_TAGS = ["html", "head", "script", "iframe", "div", "font", "table", "style", "title", "body"];
|
|
4587
|
+
var tagSource = (tags) => `<(?:${tags.join("|")})[\\s>]`;
|
|
4588
|
+
var ANY_HTML_TAG = new RegExp(tagSource([...HTML_TAGS, ...MORE_HTML_TAGS]));
|
|
4589
|
+
var DISTINCT_HTML_TAG = new RegExp(tagSource(DISTINCT_TAGS));
|
|
4590
|
+
var SVG_TAG = /<svg[\s>/]/;
|
|
4591
|
+
var LEADING_HTML = new RegExp(`^(?:<!doctype html|<!--|${tagSource([...HTML_TAGS, ...MORE_HTML_TAGS])})`);
|
|
4592
|
+
function detectScriptable(head) {
|
|
4593
|
+
const start = leadingText(head).slice(0, 512).toLowerCase();
|
|
4594
|
+
if (start.startsWith("<?xml")) return SVG_TAG.test(start) ? "svg" : "xml";
|
|
4595
|
+
if (/^<svg[\s>/]/.test(start)) return "svg";
|
|
4596
|
+
if (LEADING_HTML.test(start)) return "html";
|
|
4597
|
+
return null;
|
|
4598
|
+
}
|
|
4599
|
+
function windowText(head, end) {
|
|
4600
|
+
const window = head.subarray(0, Math.min(end, MARKUP_WINDOW_BYTES));
|
|
4601
|
+
const utf16 = utf16Encoding(head);
|
|
4602
|
+
const text = utf16 ? new TextDecoder(utf16).decode(window, { stream: true }) : String.fromCharCode(...window);
|
|
4603
|
+
return text.toLowerCase();
|
|
4604
|
+
}
|
|
4605
|
+
function markupIn(text, htmlTag) {
|
|
4606
|
+
if (SVG_TAG.test(text)) return "svg";
|
|
4607
|
+
if (text.includes("<!doctype html") || htmlTag.test(text)) return "html";
|
|
4608
|
+
return null;
|
|
4609
|
+
}
|
|
4610
|
+
function detectEmbeddedMarkup(head) {
|
|
4611
|
+
return markupIn(windowText(head, MARKUP_WINDOW_BYTES), DISTINCT_HTML_TAG);
|
|
4612
|
+
}
|
|
4613
|
+
function detectMarkupBefore(head, end) {
|
|
4614
|
+
const text = windowText(head, end);
|
|
4615
|
+
if (text.includes("<!--")) return "html";
|
|
4616
|
+
return markupIn(text, ANY_HTML_TAG);
|
|
4617
|
+
}
|
|
4618
|
+
|
|
4619
|
+
// src/content-sniff.ts
|
|
4620
|
+
var SNIFF_HEAD_BYTES = 4096;
|
|
4621
|
+
var ascii = (text) => Array.from(text, (char) => char.charCodeAt(0));
|
|
4622
|
+
function hasBytesAt(head, offset, bytes) {
|
|
4623
|
+
if (head.length < offset + bytes.length) return false;
|
|
4624
|
+
return bytes.every((byte, index) => head[offset + index] === byte);
|
|
4625
|
+
}
|
|
4626
|
+
function readAscii(head, start, end) {
|
|
4627
|
+
return String.fromCharCode(...head.subarray(start, Math.min(end, head.length)));
|
|
4628
|
+
}
|
|
4629
|
+
var startsWith = (...bytes) => (head) => hasBytesAt(head, 0, bytes);
|
|
4630
|
+
var riff = (form) => (head) => hasBytesAt(head, 0, ascii("RIFF")) && hasBytesAt(head, 8, ascii(form));
|
|
4631
|
+
var PDF_HEADER = ascii("%PDF-");
|
|
4632
|
+
function pdfHeaderOffset(head) {
|
|
4633
|
+
const searchEnd = Math.min(head.length, MARKUP_WINDOW_BYTES) - PDF_HEADER.length;
|
|
4634
|
+
for (let offset = 0; offset <= searchEnd; offset++) {
|
|
4635
|
+
if (hasBytesAt(head, offset, PDF_HEADER)) return offset;
|
|
4636
|
+
}
|
|
4637
|
+
return -1;
|
|
4638
|
+
}
|
|
4639
|
+
var isPdf = (head) => pdfHeaderOffset(head) >= 0;
|
|
4640
|
+
function ftypBrands(head) {
|
|
4641
|
+
if (!hasBytesAt(head, 4, ascii("ftyp"))) return null;
|
|
4642
|
+
const boxSize = (head[0] ?? 0) << 24 | (head[1] ?? 0) << 16 | (head[2] ?? 0) << 8 | (head[3] ?? 0);
|
|
4643
|
+
const boxEnd = Math.min(boxSize >= 16 ? boxSize : 16, head.length);
|
|
4644
|
+
const brands = [readAscii(head, 8, 12)];
|
|
4645
|
+
for (let offset = 16; offset + 4 <= boxEnd; offset += 4) brands.push(readAscii(head, offset, offset + 4));
|
|
4646
|
+
return brands;
|
|
4647
|
+
}
|
|
4648
|
+
var AVIF_BRANDS = /* @__PURE__ */ new Set(["avif", "avis"]);
|
|
4649
|
+
var HEIF_BRANDS = /* @__PURE__ */ new Set(["heic", "heix", "heim", "heis", "hevc", "hevx", "mif1", "msf1"]);
|
|
4650
|
+
var LEGACY_MOV_ATOMS = ["moov", "mdat", "wide", "free", "skip", "pnot"];
|
|
4651
|
+
var hasBrand = (brands) => (head) => ftypBrands(head)?.some((brand) => brands.has(brand)) ?? false;
|
|
4652
|
+
var isIsoMedia = (head) => {
|
|
4653
|
+
const brands = ftypBrands(head);
|
|
4654
|
+
if (!brands) return LEGACY_MOV_ATOMS.some((atom) => hasBytesAt(head, 4, ascii(atom)));
|
|
4655
|
+
const majorBrand = brands[0] ?? "";
|
|
4656
|
+
return !AVIF_BRANDS.has(majorBrand) && !HEIF_BRANDS.has(majorBrand);
|
|
4657
|
+
};
|
|
4658
|
+
var EBML_MAGIC = [26, 69, 223, 163];
|
|
4659
|
+
var ebmlDocType = (...docTypes) => (head) => {
|
|
4660
|
+
if (!hasBytesAt(head, 0, EBML_MAGIC)) return false;
|
|
4661
|
+
const header = readAscii(head, 4, 64);
|
|
4662
|
+
return docTypes.some((docType) => header.includes(docType));
|
|
4663
|
+
};
|
|
4664
|
+
var isPng = startsWith(137, 80, 78, 71, 13, 10, 26, 10);
|
|
4665
|
+
var isJpeg = startsWith(255, 216, 255);
|
|
4666
|
+
var isGif = (head) => hasBytesAt(head, 0, ascii("GIF87a")) || hasBytesAt(head, 0, ascii("GIF89a"));
|
|
4667
|
+
var isWebp = riff("WEBP");
|
|
4668
|
+
var isWav = riff("WAVE");
|
|
4669
|
+
var isAvif = hasBrand(AVIF_BRANDS);
|
|
4670
|
+
var isHeif = hasBrand(HEIF_BRANDS);
|
|
4671
|
+
var isWebm = ebmlDocType("webm");
|
|
4672
|
+
var isMatroskaOrWebm = ebmlDocType("matroska", "webm");
|
|
4673
|
+
var isOgg = startsWith(...ascii("OggS"), 0);
|
|
4674
|
+
var isFlac = startsWith(...ascii("fLaC"));
|
|
4675
|
+
var isMp3 = (head) => hasBytesAt(head, 0, ascii("ID3")) || head[0] === 255 && ((head[1] ?? 0) & 224) === 224;
|
|
4676
|
+
var isZip = (head) => hasBytesAt(head, 0, [80, 75, 3, 4]) || hasBytesAt(head, 0, [80, 75, 5, 6]);
|
|
4677
|
+
var isText = (head) => {
|
|
4678
|
+
const utf16 = utf16Encoding(head);
|
|
4679
|
+
if (utf16) return decodesStrictly(head.subarray(2), utf16);
|
|
4680
|
+
return !head.some(isBinaryDataByte);
|
|
4681
|
+
};
|
|
4682
|
+
var isJson = (head) => !head.some(isBinaryDataByte) && decodesStrictly(head, "utf-8") && /^[{[]/.test(leadingText(head));
|
|
4683
|
+
var FAMILIES = [
|
|
4684
|
+
["pdf", startsWith(...PDF_HEADER)],
|
|
4685
|
+
["png", isPng],
|
|
4686
|
+
["jpeg", isJpeg],
|
|
4687
|
+
["gif", isGif],
|
|
4688
|
+
["webp", isWebp],
|
|
4689
|
+
["wav", isWav],
|
|
4690
|
+
["avif", isAvif],
|
|
4691
|
+
["heif", isHeif],
|
|
4692
|
+
["mp4", isIsoMedia],
|
|
4693
|
+
["webm", isWebm],
|
|
4694
|
+
["matroska", isMatroskaOrWebm],
|
|
4695
|
+
["ogg", isOgg],
|
|
4696
|
+
["flac", isFlac],
|
|
4697
|
+
["zip", isZip],
|
|
4698
|
+
["mp3", isMp3],
|
|
4699
|
+
["json", isJson],
|
|
4700
|
+
["text", isText],
|
|
4701
|
+
["pdf", isPdf]
|
|
4702
|
+
];
|
|
4703
|
+
function detectFamily(head) {
|
|
4704
|
+
if (head.length === 0) return "empty";
|
|
4705
|
+
const scriptable = detectScriptable(head);
|
|
4706
|
+
if (scriptable) return scriptable;
|
|
4707
|
+
return FAMILIES.find(([, signature]) => signature(head))?.[0] ?? "binary";
|
|
4708
|
+
}
|
|
4709
|
+
var OFFICE_ZIP_TYPES = ["wordprocessingml.document", "spreadsheetml.sheet", "presentationml.presentation"].map(
|
|
4710
|
+
(subtype) => `application/vnd.openxmlformats-officedocument.${subtype}`
|
|
4711
|
+
);
|
|
4712
|
+
var typesWith = (types, signature) => types.map((type) => [type, signature]);
|
|
4713
|
+
var SIGNATURES = new Map([
|
|
4714
|
+
["application/pdf", isPdf],
|
|
4715
|
+
["image/png", isPng],
|
|
4716
|
+
["image/jpeg", isJpeg],
|
|
4717
|
+
["image/gif", isGif],
|
|
4718
|
+
["image/webp", isWebp],
|
|
4719
|
+
["image/avif", isAvif],
|
|
4720
|
+
...typesWith(["image/heic", "image/heif"], isHeif),
|
|
4721
|
+
...typesWith(["video/mp4", "video/quicktime", "video/x-m4v", "video/3gpp", "audio/mp4", "audio/x-m4a"], isIsoMedia),
|
|
4722
|
+
...typesWith(["video/webm", "audio/webm"], isWebm),
|
|
4723
|
+
...typesWith(["video/x-matroska", "audio/x-matroska"], isMatroskaOrWebm),
|
|
4724
|
+
...typesWith(["audio/mpeg", "audio/mp3"], isMp3),
|
|
4725
|
+
...typesWith(["audio/ogg", "video/ogg", "application/ogg", "audio/opus"], isOgg),
|
|
4726
|
+
...typesWith(["audio/wav", "audio/x-wav", "audio/wave", "audio/vnd.wave"], isWav),
|
|
4727
|
+
...typesWith(["audio/flac", "audio/x-flac"], isFlac),
|
|
4728
|
+
...typesWith(["application/zip", "application/x-zip-compressed", ...OFFICE_ZIP_TYPES], isZip),
|
|
4729
|
+
...typesWith(["text/plain", "text/csv", "text/markdown", "text/tab-separated-values"], isText),
|
|
4730
|
+
["application/json", isJson]
|
|
4731
|
+
]);
|
|
4732
|
+
function essenceOf(contentType) {
|
|
4733
|
+
return contentType.split(";")[0]?.trim().toLowerCase() ?? "";
|
|
4734
|
+
}
|
|
4735
|
+
var TEXT_TYPES = /* @__PURE__ */ new Set([
|
|
4736
|
+
"text/plain",
|
|
4737
|
+
"text/csv",
|
|
4738
|
+
"text/markdown",
|
|
4739
|
+
"text/tab-separated-values",
|
|
4740
|
+
"application/json"
|
|
4741
|
+
]);
|
|
4742
|
+
function embeddedMarkup(essence, head) {
|
|
4743
|
+
if (essence === "application/pdf") {
|
|
4744
|
+
const headerOffset = pdfHeaderOffset(head);
|
|
4745
|
+
const before = detectMarkupBefore(head, headerOffset < 0 ? MARKUP_WINDOW_BYTES : headerOffset);
|
|
4746
|
+
return before ?? detectEmbeddedMarkup(head);
|
|
4747
|
+
}
|
|
4748
|
+
return TEXT_TYPES.has(essence) ? detectEmbeddedMarkup(head) : null;
|
|
4749
|
+
}
|
|
4750
|
+
var SCRIPTABLE_FAMILIES = /* @__PURE__ */ new Set(["html", "svg", "xml"]);
|
|
4751
|
+
var mismatch = (detected) => ({
|
|
4752
|
+
verdict: "mismatch",
|
|
4753
|
+
detected,
|
|
4754
|
+
isScriptable: SCRIPTABLE_FAMILIES.has(detected)
|
|
4755
|
+
});
|
|
4756
|
+
function sniffContent(declaredType, head) {
|
|
4757
|
+
const essence = essenceOf(declaredType);
|
|
4758
|
+
const signature = SIGNATURES.get(essence);
|
|
4759
|
+
if (!signature) return { verdict: "unverifiable" };
|
|
4760
|
+
const sample = head.subarray(0, SNIFF_HEAD_BYTES);
|
|
4761
|
+
const scriptable = detectScriptable(sample) ?? embeddedMarkup(essence, sample);
|
|
4762
|
+
if (scriptable) return mismatch(scriptable);
|
|
4763
|
+
if (signature(sample)) return { verdict: "match" };
|
|
4764
|
+
return mismatch(detectFamily(sample));
|
|
4765
|
+
}
|
|
4766
|
+
|
|
4767
|
+
// src/routes/upload-content-check.ts
|
|
4768
|
+
var OCTET_STREAM = "application/octet-stream";
|
|
4769
|
+
var MULTIPART_PENDING_TTL_SECONDS = 24 * 60 * 60;
|
|
4770
|
+
function sniffingModeOf(context, pending) {
|
|
4771
|
+
return pending?.contentSniffing ?? getRequestState(context).config?.contentSniffing ?? "scriptable";
|
|
4772
|
+
}
|
|
4773
|
+
function shouldSniffUpload(context, key, pending, contentType) {
|
|
4774
|
+
const isRunnerWrite = pending ? Boolean(pending.isDerived) : true;
|
|
4775
|
+
if (isRunnerWrite && isTrustedDerivedOutput(key, contentType)) return false;
|
|
4776
|
+
return sniffingModeOf(context, pending) !== "off";
|
|
4777
|
+
}
|
|
4778
|
+
function mismatchAction(mode, mismatch2, isMultipart) {
|
|
4779
|
+
if (mode === "scriptable") return mismatch2.isScriptable ? "rejected" : "accepted";
|
|
4780
|
+
if (mode === "octet-stream" && !isMultipart) return "stored_as_octet_stream";
|
|
4781
|
+
return "rejected";
|
|
4782
|
+
}
|
|
4783
|
+
function logMismatch(declared, mismatch2, key, action) {
|
|
4784
|
+
console.warn(`[content-sniff] ${JSON.stringify({ declared, detected: mismatch2.detected, key, action })}`);
|
|
4785
|
+
}
|
|
4786
|
+
function contentTypeMismatch(context, declared, detected) {
|
|
4787
|
+
return context.json(
|
|
4788
|
+
{
|
|
4789
|
+
error: `The file's content (${detected}) does not match its declared Content-Type ${declared}`,
|
|
4790
|
+
code: "content_type_mismatch",
|
|
4791
|
+
declared,
|
|
4792
|
+
detected
|
|
4793
|
+
},
|
|
4794
|
+
415
|
|
4795
|
+
);
|
|
4796
|
+
}
|
|
4797
|
+
function sniffHead(declared, body) {
|
|
4798
|
+
return sniffContent(declared, new Uint8Array(body, 0, Math.min(body.byteLength, SNIFF_HEAD_BYTES)));
|
|
4799
|
+
}
|
|
4800
|
+
function checkSinglePutContent(context, upload2) {
|
|
4801
|
+
const { contentType, key, pending } = upload2;
|
|
4802
|
+
if (!shouldSniffUpload(context, key, pending, contentType)) return { contentType };
|
|
4803
|
+
const result = sniffHead(contentType, upload2.body);
|
|
4804
|
+
if (result.verdict !== "mismatch") return { contentType };
|
|
4805
|
+
const action = mismatchAction(sniffingModeOf(context, pending), result, false);
|
|
4806
|
+
logMismatch(contentType, result, key, action);
|
|
4807
|
+
if (action === "accepted") return { contentType };
|
|
4808
|
+
if (action === "stored_as_octet_stream") return { contentType: OCTET_STREAM };
|
|
4809
|
+
return { rejected: contentTypeMismatch(context, contentType, result.detected) };
|
|
4810
|
+
}
|
|
4811
|
+
async function discardMultipart(context, active) {
|
|
4812
|
+
const { pending, multipart, kvKey } = active;
|
|
4813
|
+
await context.env.MEDIA_BUCKET.resumeMultipartUpload(pending.storageKey, multipart.uploadId).abort();
|
|
4814
|
+
await context.env.MEDIA_KV.delete(kvKey);
|
|
4815
|
+
const meta = decodeEncryptionMeta9(multipart.enc);
|
|
4816
|
+
if (meta && context.env.KEYS_BUCKET) await context.env.KEYS_BUCKET.delete(`${pending.storageKey}/${meta.keyId}`);
|
|
4817
|
+
}
|
|
4818
|
+
async function checkFirstPart(context, active, key, partNumber, body) {
|
|
4819
|
+
const { pending, multipart } = active;
|
|
4820
|
+
if (partNumber !== 1 || !multipart.checksFirstPart) return null;
|
|
4821
|
+
const result = sniffHead(pending.contentType, body);
|
|
4822
|
+
if (result.verdict !== "mismatch") return null;
|
|
4823
|
+
const action = mismatchAction(sniffingModeOf(context, pending), result, true);
|
|
4824
|
+
logMismatch(pending.contentType, result, key, action);
|
|
4825
|
+
if (action === "accepted") return null;
|
|
4826
|
+
await discardMultipart(context, active);
|
|
4827
|
+
return contentTypeMismatch(context, pending.contentType, result.detected);
|
|
4828
|
+
}
|
|
4829
|
+
async function recordFirstPart(context, active, partNumber, etag) {
|
|
4830
|
+
const { pending, multipart, kvKey } = active;
|
|
4831
|
+
if (partNumber !== 1 || !multipart.checksFirstPart) return;
|
|
4832
|
+
const recorded = { ...pending, multipart: { ...multipart, firstPartEtag: etag } };
|
|
4833
|
+
await context.env.MEDIA_KV.put(kvKey, JSON.stringify(recorded), { expirationTtl: MULTIPART_PENDING_TTL_SECONDS });
|
|
4834
|
+
}
|
|
4835
|
+
function unverifiedFirstPart(context, multipart, parts) {
|
|
4836
|
+
if (!multipart.checksFirstPart) return null;
|
|
4837
|
+
if (multipart.firstPartEtag && parts[0]?.etag === multipart.firstPartEtag) return null;
|
|
4838
|
+
return context.json(
|
|
4839
|
+
{ error: "Part 1 of this upload has not been checked yet, or was replaced", code: "content_unverified" },
|
|
4840
|
+
409
|
|
4841
|
+
);
|
|
4842
|
+
}
|
|
4843
|
+
|
|
4436
4844
|
// src/routes/upload-multipart.ts
|
|
4437
4845
|
import {
|
|
4438
4846
|
AES_BLOCK_BYTES as AES_BLOCK_BYTES2,
|
|
4439
4847
|
classifyUploadRequest,
|
|
4440
4848
|
counterForBlock,
|
|
4441
|
-
decodeEncryptionMeta as
|
|
4849
|
+
decodeEncryptionMeta as decodeEncryptionMeta10,
|
|
4442
4850
|
encryptBytes,
|
|
4443
4851
|
MULTIPART_PART_SIZE_BYTES,
|
|
4444
4852
|
MULTIPART_QUERY,
|
|
@@ -4544,7 +4952,6 @@ async function cancelPendingUpload(context, uploadUrl) {
|
|
|
4544
4952
|
}
|
|
4545
4953
|
|
|
4546
4954
|
// src/routes/upload-multipart.ts
|
|
4547
|
-
var MULTIPART_PENDING_TTL_SECONDS = 24 * 60 * 60;
|
|
4548
4955
|
var MAX_PART_COUNT = 1e4;
|
|
4549
4956
|
async function readMultipartAt(context, kvKey) {
|
|
4550
4957
|
const raw = await context.env.MEDIA_KV.get(kvKey);
|
|
@@ -4636,7 +5043,9 @@ async function handleMultipartCreate(context, key, target) {
|
|
|
4636
5043
|
size,
|
|
4637
5044
|
partSize,
|
|
4638
5045
|
...enc ? { enc } : {},
|
|
4639
|
-
...declaredSha256 ? { sha256: declaredSha256 } : {}
|
|
5046
|
+
...declaredSha256 ? { sha256: declaredSha256 } : {},
|
|
5047
|
+
// The type is fixed now, before any byte: part 1 is checked against it when it arrives.
|
|
5048
|
+
...shouldSniffUpload(context, key, pending, pending.contentType) ? { checksFirstPart: true } : {}
|
|
4640
5049
|
};
|
|
4641
5050
|
await context.env.MEDIA_KV.put(kvKey, JSON.stringify({ ...pending, multipart }), {
|
|
4642
5051
|
expirationTtl: MULTIPART_PENDING_TTL_SECONDS
|
|
@@ -4650,7 +5059,7 @@ async function handleMultipartCreate(context, key, target) {
|
|
|
4650
5059
|
}
|
|
4651
5060
|
async function encryptPart(context, pending, multipart, partNumber, body) {
|
|
4652
5061
|
if (!multipart.enc) return body;
|
|
4653
|
-
const meta =
|
|
5062
|
+
const meta = decodeEncryptionMeta10(multipart.enc);
|
|
4654
5063
|
const material = meta ? await loadStorageKey(context.env.KEYS_BUCKET, pending.storageKey, meta.keyId) : null;
|
|
4655
5064
|
if (!meta || !material) return null;
|
|
4656
5065
|
const startBlock = (partNumber - 1) * multipart.partSize / AES_BLOCK_BYTES2;
|
|
@@ -4674,10 +5083,13 @@ async function handleMultipartPart(context) {
|
|
|
4674
5083
|
}
|
|
4675
5084
|
const partChecksumFailure = await checkPartSha256(context, multipart, body);
|
|
4676
5085
|
if (partChecksumFailure) return partChecksumFailure;
|
|
5086
|
+
const contentFailure = await checkFirstPart(context, active, key, partNumber, body);
|
|
5087
|
+
if (contentFailure) return contentFailure;
|
|
4677
5088
|
const storedPart = await encryptPart(context, pending, multipart, partNumber, body);
|
|
4678
5089
|
if (!storedPart) return context.json({ error: "Encryption key not found for this upload" }, 500);
|
|
4679
5090
|
const multipartUpload = context.env.MEDIA_BUCKET.resumeMultipartUpload(pending.storageKey, multipart.uploadId);
|
|
4680
5091
|
const uploaded = await multipartUpload.uploadPart(partNumber, storedPart);
|
|
5092
|
+
await recordFirstPart(context, active, partNumber, uploaded.etag);
|
|
4681
5093
|
return context.json({ partNumber: uploaded.partNumber, etag: uploaded.etag });
|
|
4682
5094
|
}
|
|
4683
5095
|
async function handleMultipartComplete(context, key) {
|
|
@@ -4687,6 +5099,8 @@ async function handleMultipartComplete(context, key) {
|
|
|
4687
5099
|
const { pending, multipart, kvKey } = active;
|
|
4688
5100
|
const parts = normalizeCompletedParts(multipart, body?.parts);
|
|
4689
5101
|
if (!parts) return context.json({ error: "parts must list every part exactly once", code: "invalid_parts" }, 400);
|
|
5102
|
+
const unverified = unverifiedFirstPart(context, multipart, parts);
|
|
5103
|
+
if (unverified) return unverified;
|
|
4690
5104
|
const write = await prepareUploadWrite(context, pending.key ?? key, pending, pending.storageKey);
|
|
4691
5105
|
if ("blocked" in write) return write.blocked;
|
|
4692
5106
|
const multipartUpload = context.env.MEDIA_BUCKET.resumeMultipartUpload(pending.storageKey, multipart.uploadId);
|
|
@@ -4701,25 +5115,6 @@ async function handleMultipartComplete(context, key) {
|
|
|
4701
5115
|
const integrity = multipart.sha256 ? { sha256: multipart.sha256, sha256Verified: false } : {};
|
|
4702
5116
|
return context.json({ key, size: object.size, ...integrity });
|
|
4703
5117
|
}
|
|
4704
|
-
async function checkPartSha256(context, multipart, body) {
|
|
4705
|
-
const partSha256 = headerSha256(context);
|
|
4706
|
-
if (partSha256 === null) {
|
|
4707
|
-
return context.json({ error: "x-devultur-sha256 must be a 64-character hex string", code: "invalid_sha256" }, 400);
|
|
4708
|
-
}
|
|
4709
|
-
if (multipart.sha256 && !partSha256) {
|
|
4710
|
-
return context.json(
|
|
4711
|
-
{
|
|
4712
|
-
error: "Each part must carry its x-devultur-sha256 when the upload declares one",
|
|
4713
|
-
code: "part_sha256_required"
|
|
4714
|
-
},
|
|
4715
|
-
400
|
|
4716
|
-
);
|
|
4717
|
-
}
|
|
4718
|
-
if (!partSha256) return null;
|
|
4719
|
-
const digest = await digestBody(body, void 0, partSha256);
|
|
4720
|
-
if ("mismatch" in digest) return sha256Mismatch(context, digest.mismatch.expected, digest.mismatch.actual);
|
|
4721
|
-
return null;
|
|
4722
|
-
}
|
|
4723
5118
|
async function handleMultipartPost(context) {
|
|
4724
5119
|
const key = context.req.param("key") ?? "";
|
|
4725
5120
|
const kind = uploadRequestKindOf(context);
|
|
@@ -4753,20 +5148,8 @@ upload.post("/", async (context) => {
|
|
|
4753
5148
|
if (!filename || !body.contentType) {
|
|
4754
5149
|
return context.json({ error: "filename and contentType are required" }, 400);
|
|
4755
5150
|
}
|
|
4756
|
-
const
|
|
4757
|
-
if (
|
|
4758
|
-
return context.json(
|
|
4759
|
-
{
|
|
4760
|
-
error: `mediaType "${body.mediaType}" does not match content type ${body.contentType}`,
|
|
4761
|
-
code: "media_type_mismatch"
|
|
4762
|
-
},
|
|
4763
|
-
400
|
|
4764
|
-
);
|
|
4765
|
-
}
|
|
4766
|
-
const mediaType = body.mediaType ?? inferredType ?? (isAssetRequest ? MediaType2.FILE : null);
|
|
4767
|
-
if (!mediaType) {
|
|
4768
|
-
return context.json({ error: `Unsupported content type: ${body.contentType}` }, 400);
|
|
4769
|
-
}
|
|
5151
|
+
const mediaType = resolveUploadMediaType(body.mediaType, body.contentType, isAssetRequest);
|
|
5152
|
+
if (isUploadTargetError(mediaType)) return targetFailure(context, mediaType);
|
|
4770
5153
|
const scope = getRequestState(context);
|
|
4771
5154
|
const uploadRule = scope.config?.uploadRules?.[mediaType];
|
|
4772
5155
|
if (!isContentTypeAllowed(body.contentType, uploadRule?.accept)) {
|
|
@@ -4830,7 +5213,8 @@ upload.post("/", async (context) => {
|
|
|
4830
5213
|
...target.isAsset ? { isAsset: true, overwrite: shouldOverwrite } : {},
|
|
4831
5214
|
...cacheControl ? { cacheControl } : {},
|
|
4832
5215
|
...scope.config?.uploadMetadata ? { uploadMetadata: scope.config.uploadMetadata } : {},
|
|
4833
|
-
...lock ? { lock } : {}
|
|
5216
|
+
...lock ? { lock } : {},
|
|
5217
|
+
...scope.config?.contentSniffing ? { contentSniffing: scope.config.contentSniffing } : {}
|
|
4834
5218
|
};
|
|
4835
5219
|
const pendingId = newUploadPendingId();
|
|
4836
5220
|
await context.env.MEDIA_KV.put(uploadPendingKvKey(pendingId), JSON.stringify(pending), {
|
|
@@ -4861,9 +5245,9 @@ upload.put("/:key{.+}", async (context) => {
|
|
|
4861
5245
|
const maxSizeBytes = pending?.maxSizeBytes ?? null;
|
|
4862
5246
|
const storageClass = pending?.storageClass ?? "Standard";
|
|
4863
5247
|
const resourceSecurity = pending?.security ?? parseSecurityHeader(context.req.header(RESOURCE_SECURITY_HEADER));
|
|
4864
|
-
const
|
|
4865
|
-
if (expectedContentType &&
|
|
4866
|
-
return context.json({ error: `Expected Content-Type ${expectedContentType}, got ${
|
|
5248
|
+
const declaredContentType = context.req.header("Content-Type") ?? OCTET_STREAM;
|
|
5249
|
+
if (expectedContentType && declaredContentType !== expectedContentType) {
|
|
5250
|
+
return context.json({ error: `Expected Content-Type ${expectedContentType}, got ${declaredContentType}` }, 400);
|
|
4867
5251
|
}
|
|
4868
5252
|
const declaredSize = Number(context.req.header("content-length") ?? "0");
|
|
4869
5253
|
if (maxSizeBytes != null && Number.isFinite(declaredSize) && declaredSize > maxSizeBytes) {
|
|
@@ -4884,6 +5268,9 @@ upload.put("/:key{.+}", async (context) => {
|
|
|
4884
5268
|
return context.json({ error: "x-devultur-sha256 must be a 64-character hex string", code: "invalid_sha256" }, 400);
|
|
4885
5269
|
}
|
|
4886
5270
|
if ("mismatch" in digest) return sha256Mismatch(context, digest.mismatch.expected, digest.mismatch.actual);
|
|
5271
|
+
const content = checkSinglePutContent(context, { key, pending, contentType: declaredContentType, body });
|
|
5272
|
+
if ("rejected" in content) return content.rejected;
|
|
5273
|
+
const { contentType } = content;
|
|
4887
5274
|
const policy = resolveSecurity5(resourceSecurity, getRequestState(context).config?.security);
|
|
4888
5275
|
const shouldEncrypt = shouldEncryptAtRest(policy, key);
|
|
4889
5276
|
if (isKeysBucketMissing(shouldEncrypt, context.env)) return keysBucketMissing(context);
|
|
@@ -4923,6 +5310,315 @@ upload.put("/:key{.+}", async (context) => {
|
|
|
4923
5310
|
return context.json({ key, size: body.byteLength, sha256: digest.sha256, sha256Verified: true });
|
|
4924
5311
|
});
|
|
4925
5312
|
|
|
5313
|
+
// src/object-lock/lock-status.ts
|
|
5314
|
+
var VISIBILITY_SEGMENTS = /* @__PURE__ */ new Set(["public", "private"]);
|
|
5315
|
+
var RESERVED_SEGMENT = /__lock(?:-audit)?$/;
|
|
5316
|
+
function placeOfStorageKey(storageKey2, root) {
|
|
5317
|
+
const segments = storageKey2.split("/");
|
|
5318
|
+
if (segments.some((segment) => RESERVED_SEGMENT.test(segment))) return { kind: "reserved" };
|
|
5319
|
+
if (root !== void 0) {
|
|
5320
|
+
const key = storageKey2.slice(root.length);
|
|
5321
|
+
const isWorkerKey = storageKey2.startsWith(root) && VISIBILITY_SEGMENTS.has(key.split("/")[0] ?? "");
|
|
5322
|
+
return isWorkerKey ? { kind: "object", root, key } : { kind: "unrecognized" };
|
|
5323
|
+
}
|
|
5324
|
+
const first = segments.findIndex((segment) => VISIBILITY_SEGMENTS.has(segment));
|
|
5325
|
+
if (first < 0) return { kind: "unrecognized" };
|
|
5326
|
+
const inferredRoot = first === 0 ? "" : `${segments.slice(0, first).join("/")}/`;
|
|
5327
|
+
return { kind: "object", root: inferredRoot, key: storageKey2.slice(inferredRoot.length) };
|
|
5328
|
+
}
|
|
5329
|
+
var UNRECOGNIZED_LOCK = { locked: true, hold: false, unrecognized: true };
|
|
5330
|
+
var VARIANTS_SEGMENT = "/__variants/";
|
|
5331
|
+
function retentionSubjectOf(key) {
|
|
5332
|
+
const isVariant = isVariantKey(key);
|
|
5333
|
+
const original = isVariant ? key.slice(0, key.indexOf(VARIANTS_SEGMENT)) : key;
|
|
5334
|
+
return { subject: lockSubjectOf(original) ?? { kind: "original", key: original }, isVariant };
|
|
5335
|
+
}
|
|
5336
|
+
function toStoredObjectLock(locked, place) {
|
|
5337
|
+
if (!locked) return { locked: false, hold: false };
|
|
5338
|
+
const { lock, isUnreadable } = locked.state;
|
|
5339
|
+
const until = untilOf(locked.state);
|
|
5340
|
+
return {
|
|
5341
|
+
locked: true,
|
|
5342
|
+
hold: lock.hold === true,
|
|
5343
|
+
...until ? { until } : {},
|
|
5344
|
+
...lock.retention ? { mode: lock.retention.mode } : {},
|
|
5345
|
+
...isUnreadable ? { unreadable: true } : {},
|
|
5346
|
+
...locked.key === place.key ? {} : { lockedBy: `${place.root}${locked.key}` }
|
|
5347
|
+
};
|
|
5348
|
+
}
|
|
5349
|
+
var LOCK_CACHE_MAX_USES = 50;
|
|
5350
|
+
var LOCK_CACHE_MAX_AGE_MS = 5e3;
|
|
5351
|
+
function lockResolver(mediaBucket, now) {
|
|
5352
|
+
const cache = /* @__PURE__ */ new Map();
|
|
5353
|
+
const resolveSubject = (root, subject) => {
|
|
5354
|
+
const id = `${root}\0${subject.kind === "original" ? subject.key : subject.rawPrefix}`;
|
|
5355
|
+
const cached = cache.get(id);
|
|
5356
|
+
const isFresh = cached && cached.uses < LOCK_CACHE_MAX_USES && Date.now() - cached.readAt < LOCK_CACHE_MAX_AGE_MS;
|
|
5357
|
+
if (cached && isFresh) {
|
|
5358
|
+
cached.uses += 1;
|
|
5359
|
+
return cached.pending;
|
|
5360
|
+
}
|
|
5361
|
+
const pending = findLockedOriginal(mediaBucket, root, subject, now);
|
|
5362
|
+
pending.catch(() => cache.delete(id));
|
|
5363
|
+
cache.set(id, { pending, readAt: Date.now(), uses: 1 });
|
|
5364
|
+
return pending;
|
|
5365
|
+
};
|
|
5366
|
+
return async (place, object) => {
|
|
5367
|
+
const { subject, isVariant } = retentionSubjectOf(place.key);
|
|
5368
|
+
if (subject.kind === "original" && subject.key === place.key) {
|
|
5369
|
+
return lockedObject(mediaBucket, place.root, place.key, object, now);
|
|
5370
|
+
}
|
|
5371
|
+
const locked = await resolveSubject(place.root, subject);
|
|
5372
|
+
if (isVariant && subject.kind === "original" && locked && !protectsDerived(locked.state)) return null;
|
|
5373
|
+
return locked;
|
|
5374
|
+
};
|
|
5375
|
+
}
|
|
5376
|
+
async function readObjectLock(options, storageKey2) {
|
|
5377
|
+
const place = placeOfStorageKey(storageKey2, options.root);
|
|
5378
|
+
if (place.kind === "reserved") return null;
|
|
5379
|
+
const object = await options.mediaBucket.head(storageKey2);
|
|
5380
|
+
if (!object) return null;
|
|
5381
|
+
if (place.kind === "unrecognized") return UNRECOGNIZED_LOCK;
|
|
5382
|
+
const locked = await lockResolver(options.mediaBucket, options.now ?? /* @__PURE__ */ new Date())(place, object);
|
|
5383
|
+
return toStoredObjectLock(locked, place);
|
|
5384
|
+
}
|
|
5385
|
+
|
|
5386
|
+
// src/object-lock/prefix-walk.ts
|
|
5387
|
+
var DEFAULT_MAX_SUBREQUESTS = 40;
|
|
5388
|
+
var DEFAULT_BATCH_SIZE = 100;
|
|
5389
|
+
var MAX_BATCH_SIZE = 1e3;
|
|
5390
|
+
var BudgetExhausted = class extends Error {
|
|
5391
|
+
constructor() {
|
|
5392
|
+
super("subrequest budget exhausted");
|
|
5393
|
+
}
|
|
5394
|
+
};
|
|
5395
|
+
function subrequestBudget(max) {
|
|
5396
|
+
if (!Number.isInteger(max) || max < 1) throw new RangeError("maxSubrequests must be a positive integer");
|
|
5397
|
+
let used = 0;
|
|
5398
|
+
return {
|
|
5399
|
+
get used() {
|
|
5400
|
+
return used;
|
|
5401
|
+
},
|
|
5402
|
+
canAfford: (count) => used + count <= max,
|
|
5403
|
+
track: (bucket) => new Proxy(bucket, {
|
|
5404
|
+
get(target, property) {
|
|
5405
|
+
const value = Reflect.get(target, property);
|
|
5406
|
+
if (typeof value !== "function") return value;
|
|
5407
|
+
return (...args) => {
|
|
5408
|
+
if (used >= max) throw new BudgetExhausted();
|
|
5409
|
+
used += 1;
|
|
5410
|
+
return value.apply(target, args);
|
|
5411
|
+
};
|
|
5412
|
+
}
|
|
5413
|
+
})
|
|
5414
|
+
};
|
|
5415
|
+
}
|
|
5416
|
+
function encodeCursor(prefix, cursor) {
|
|
5417
|
+
return btoa(JSON.stringify({ prefix, ...cursor }));
|
|
5418
|
+
}
|
|
5419
|
+
var VIDEO_FOLDER_SHAPE = /^(?:public|private)\/video\/[^/]+$/;
|
|
5420
|
+
function invalidCursor(reason) {
|
|
5421
|
+
throw new RangeError(`cursor isn't one this function returned for this prefix: ${reason}`);
|
|
5422
|
+
}
|
|
5423
|
+
function parseCursor(cursor) {
|
|
5424
|
+
try {
|
|
5425
|
+
const parsed = JSON.parse(atob(cursor));
|
|
5426
|
+
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed))
|
|
5427
|
+
return parsed;
|
|
5428
|
+
} catch {
|
|
5429
|
+
}
|
|
5430
|
+
return invalidCursor("not a cursor");
|
|
5431
|
+
}
|
|
5432
|
+
var isOptionalString = (value) => value === void 0 || typeof value === "string";
|
|
5433
|
+
function validFolder(value, options) {
|
|
5434
|
+
if (value === void 0) return void 0;
|
|
5435
|
+
const { root, folder } = typeof value === "object" && value !== null ? value : {};
|
|
5436
|
+
if (typeof root !== "string" || typeof folder !== "string" || !VIDEO_FOLDER_SHAPE.test(folder)) {
|
|
5437
|
+
return invalidCursor("folder");
|
|
5438
|
+
}
|
|
5439
|
+
const folderPrefix = `${root}${folder}/`;
|
|
5440
|
+
const place = placeOfStorageKey(`${folderPrefix}x`, options.root);
|
|
5441
|
+
const isPlaced = place.kind === "object" && place.root === root;
|
|
5442
|
+
if (!isPlaced || !folderPrefix.startsWith(options.prefix)) return invalidCursor("folder outside the prefix");
|
|
5443
|
+
return { root, folder };
|
|
5444
|
+
}
|
|
5445
|
+
function decodeCursor(cursor, options) {
|
|
5446
|
+
if (cursor === void 0) return {};
|
|
5447
|
+
const { prefix, after, listCursor, folder } = parseCursor(cursor);
|
|
5448
|
+
if (prefix !== options.prefix) return invalidCursor("another prefix");
|
|
5449
|
+
if (!isOptionalString(after) || !isOptionalString(listCursor)) return invalidCursor("position");
|
|
5450
|
+
if (typeof after === "string" && !after.startsWith(options.prefix))
|
|
5451
|
+
return invalidCursor("position outside the prefix");
|
|
5452
|
+
const pendingFolder = validFolder(folder, options);
|
|
5453
|
+
return {
|
|
5454
|
+
...typeof after === "string" ? { after } : {},
|
|
5455
|
+
...typeof listCursor === "string" ? { listCursor } : {},
|
|
5456
|
+
...pendingFolder ? { folder: pendingFolder } : {}
|
|
5457
|
+
};
|
|
5458
|
+
}
|
|
5459
|
+
async function walkPrefix(options, mediaBucket, start, visit) {
|
|
5460
|
+
const limit = Math.min(Math.max(1, Math.floor(options.batchSize ?? DEFAULT_BATCH_SIZE)), MAX_BATCH_SIZE);
|
|
5461
|
+
let position = { ...start };
|
|
5462
|
+
let hasProgressed = false;
|
|
5463
|
+
const stopped = () => ({ position, isDone: false, hasProgressed });
|
|
5464
|
+
for (; ; ) {
|
|
5465
|
+
let listing;
|
|
5466
|
+
try {
|
|
5467
|
+
listing = await mediaBucket.list({ prefix: options.prefix, limit, ...listFrom(position) });
|
|
5468
|
+
} catch (error) {
|
|
5469
|
+
if (error instanceof BudgetExhausted) return stopped();
|
|
5470
|
+
throw error;
|
|
5471
|
+
}
|
|
5472
|
+
const stray = listing.objects.find((listed) => !listed.key.startsWith(options.prefix));
|
|
5473
|
+
if (stray) throw new RangeError(`listing returned ${stray.key}, outside the prefix ${options.prefix}`);
|
|
5474
|
+
for (const listed of listing.objects) {
|
|
5475
|
+
try {
|
|
5476
|
+
await visit({ listed, place: placeOfStorageKey(listed.key, options.root) });
|
|
5477
|
+
} catch (error) {
|
|
5478
|
+
if (error instanceof BudgetExhausted) return stopped();
|
|
5479
|
+
throw error;
|
|
5480
|
+
}
|
|
5481
|
+
position = { after: listed.key };
|
|
5482
|
+
hasProgressed = true;
|
|
5483
|
+
}
|
|
5484
|
+
if (!listing.truncated) return { position, isDone: true, hasProgressed };
|
|
5485
|
+
if (listing.objects.length > 0) continue;
|
|
5486
|
+
position = { ...position, listCursor: listing.cursor };
|
|
5487
|
+
hasProgressed = true;
|
|
5488
|
+
}
|
|
5489
|
+
}
|
|
5490
|
+
function listFrom(position) {
|
|
5491
|
+
if (position.listCursor) return { cursor: position.listCursor };
|
|
5492
|
+
return position.after ? { startAfter: position.after } : {};
|
|
5493
|
+
}
|
|
5494
|
+
function assertProgress(hasProgressed) {
|
|
5495
|
+
if (hasProgressed) return;
|
|
5496
|
+
throw new RangeError("maxSubrequests is too small to finish the next object: raise it");
|
|
5497
|
+
}
|
|
5498
|
+
|
|
5499
|
+
// src/object-lock/prefix-delete.ts
|
|
5500
|
+
var VIDEO_FOLDER = /^((?:public|private)\/video\/[^/]+)\//;
|
|
5501
|
+
var isPending = (folder) => folder?.hasDeleted === true && !folder.hasRetained;
|
|
5502
|
+
async function deletePrefixRespectingLocks(options) {
|
|
5503
|
+
const startedAt = /* @__PURE__ */ new Date();
|
|
5504
|
+
const budget = subrequestBudget(options.maxSubrequests ?? DEFAULT_MAX_SUBREQUESTS);
|
|
5505
|
+
const mediaBucket = budget.track(options.mediaBucket);
|
|
5506
|
+
const keysBucket = options.keysBucket ? budget.track(options.keysBucket) : void 0;
|
|
5507
|
+
const resolveLock3 = lockResolver(mediaBucket, startedAt);
|
|
5508
|
+
const start = decodeCursor(options.cursor, options);
|
|
5509
|
+
const result = {
|
|
5510
|
+
deletedCount: 0,
|
|
5511
|
+
deletedBytes: 0,
|
|
5512
|
+
deletedKeyCount: 0,
|
|
5513
|
+
retainedCount: 0,
|
|
5514
|
+
retainedBytes: 0,
|
|
5515
|
+
retained: [],
|
|
5516
|
+
skippedCount: 0,
|
|
5517
|
+
subrequests: 0
|
|
5518
|
+
};
|
|
5519
|
+
let openFolder = start.folder ? { ...start.folder, hasDeleted: true, hasRetained: false } : null;
|
|
5520
|
+
let hasClosed = false;
|
|
5521
|
+
const closeFolder = async () => {
|
|
5522
|
+
if (isPending(openFolder)) {
|
|
5523
|
+
const { root, folder } = openFolder;
|
|
5524
|
+
const folderPrefix = `${root}${folder}/`;
|
|
5525
|
+
const remaining = await mediaBucket.list({ prefix: folderPrefix, limit: 1 });
|
|
5526
|
+
if (remaining.objects.length === 0) {
|
|
5527
|
+
if (keysBucket) {
|
|
5528
|
+
const keys = await keysWrittenBy(keysBucket, folderPrefix, startedAt);
|
|
5529
|
+
result.deletedKeyCount += await deleteKeys(keysBucket, keys);
|
|
5530
|
+
}
|
|
5531
|
+
await forgetTranscodeCompleted(mediaBucket, root, folder);
|
|
5532
|
+
}
|
|
5533
|
+
hasClosed = true;
|
|
5534
|
+
}
|
|
5535
|
+
openFolder = null;
|
|
5536
|
+
};
|
|
5537
|
+
const enterFolder = async (place) => {
|
|
5538
|
+
const videoFolder = VIDEO_FOLDER.exec(place.key)?.[1];
|
|
5539
|
+
const folder = videoFolder && `${place.root}${videoFolder}/`.startsWith(options.prefix) ? videoFolder : void 0;
|
|
5540
|
+
if (openFolder && openFolder.root === place.root && openFolder.folder === folder) return openFolder;
|
|
5541
|
+
await closeFolder();
|
|
5542
|
+
openFolder = folder ? { root: place.root, folder, hasDeleted: false, hasRetained: false } : null;
|
|
5543
|
+
return openFolder;
|
|
5544
|
+
};
|
|
5545
|
+
const retain = (key, size, lock) => {
|
|
5546
|
+
result.retained.push({ key, size, ...lock });
|
|
5547
|
+
result.retainedCount += 1;
|
|
5548
|
+
result.retainedBytes += size;
|
|
5549
|
+
};
|
|
5550
|
+
const visit = async ({ listed, place }) => {
|
|
5551
|
+
if (place.kind === "reserved") {
|
|
5552
|
+
result.skippedCount += 1;
|
|
5553
|
+
return;
|
|
5554
|
+
}
|
|
5555
|
+
if (place.kind === "unrecognized") return retain(listed.key, listed.size, UNRECOGNIZED_LOCK);
|
|
5556
|
+
const folder = await enterFolder(place);
|
|
5557
|
+
const object = await mediaBucket.head(listed.key);
|
|
5558
|
+
if (!object) return;
|
|
5559
|
+
const locked = await resolveLock3(place, object);
|
|
5560
|
+
if (locked) {
|
|
5561
|
+
if (folder) folder.hasRetained = true;
|
|
5562
|
+
return retain(listed.key, object.size, toStoredObjectLock(locked, place));
|
|
5563
|
+
}
|
|
5564
|
+
const keys = keysBucket ? await objectKeysToDelete(keysBucket, listed.key, object) : [];
|
|
5565
|
+
if (!budget.canAfford(1 + deleteCallsFor(keys))) throw new BudgetExhausted();
|
|
5566
|
+
await mediaBucket.delete(listed.key);
|
|
5567
|
+
result.deletedCount += 1;
|
|
5568
|
+
result.deletedBytes += object.size;
|
|
5569
|
+
if (folder) folder.hasDeleted = true;
|
|
5570
|
+
if (keysBucket) result.deletedKeyCount += await deleteKeys(keysBucket, keys);
|
|
5571
|
+
};
|
|
5572
|
+
const walk = await walkPrefix(options, mediaBucket, start, visit);
|
|
5573
|
+
if (walk.isDone) {
|
|
5574
|
+
try {
|
|
5575
|
+
await closeFolder();
|
|
5576
|
+
} catch (error) {
|
|
5577
|
+
if (!(error instanceof BudgetExhausted)) throw error;
|
|
5578
|
+
}
|
|
5579
|
+
}
|
|
5580
|
+
const pendingFolder = isPending(openFolder) ? { root: openFolder.root, folder: openFolder.folder } : void 0;
|
|
5581
|
+
const isFinished = walk.isDone && !pendingFolder;
|
|
5582
|
+
assertProgress(walk.hasProgressed || hasClosed || isFinished);
|
|
5583
|
+
const cursor = isFinished ? void 0 : encodeCursor(options.prefix, { ...walk.position, ...pendingFolder ? { folder: pendingFolder } : {} });
|
|
5584
|
+
return { ...result, ...cursor ? { cursor } : {}, subrequests: budget.used };
|
|
5585
|
+
}
|
|
5586
|
+
|
|
5587
|
+
// src/object-lock/retained-summary.ts
|
|
5588
|
+
async function retainedSummary(options) {
|
|
5589
|
+
const budget = subrequestBudget(options.maxSubrequests ?? DEFAULT_MAX_SUBREQUESTS);
|
|
5590
|
+
const mediaBucket = budget.track(options.mediaBucket);
|
|
5591
|
+
const resolveLock3 = lockResolver(mediaBucket, options.now ?? /* @__PURE__ */ new Date());
|
|
5592
|
+
const summary = {
|
|
5593
|
+
scannedCount: 0,
|
|
5594
|
+
retainedCount: 0,
|
|
5595
|
+
retainedBytes: 0,
|
|
5596
|
+
hasHold: false,
|
|
5597
|
+
subrequests: 0
|
|
5598
|
+
};
|
|
5599
|
+
const visit = async ({ listed, place }) => {
|
|
5600
|
+
if (place.kind === "reserved") return;
|
|
5601
|
+
let lock = UNRECOGNIZED_LOCK;
|
|
5602
|
+
let size = listed.size;
|
|
5603
|
+
if (place.kind === "object") {
|
|
5604
|
+
const object = await mediaBucket.head(listed.key);
|
|
5605
|
+
if (!object) return;
|
|
5606
|
+
lock = toStoredObjectLock(await resolveLock3(place, object), place);
|
|
5607
|
+
size = object.size;
|
|
5608
|
+
}
|
|
5609
|
+
summary.scannedCount += 1;
|
|
5610
|
+
if (!lock.locked) return;
|
|
5611
|
+
summary.retainedCount += 1;
|
|
5612
|
+
summary.retainedBytes += size;
|
|
5613
|
+
if (lock.hold) summary.hasHold = true;
|
|
5614
|
+
if (lock.until && (!summary.latestUntil || lock.until > summary.latestUntil)) summary.latestUntil = lock.until;
|
|
5615
|
+
};
|
|
5616
|
+
const walk = await walkPrefix(options, mediaBucket, decodeCursor(options.cursor, options), visit);
|
|
5617
|
+
assertProgress(walk.hasProgressed || walk.isDone);
|
|
5618
|
+
const cursor = walk.isDone ? void 0 : encodeCursor(options.prefix, walk.position);
|
|
5619
|
+
return { ...summary, ...cursor ? { cursor } : {}, subrequests: budget.used };
|
|
5620
|
+
}
|
|
5621
|
+
|
|
4926
5622
|
// src/index.ts
|
|
4927
5623
|
function createWorker(options) {
|
|
4928
5624
|
const { middleware, routes, hooks } = composePlugins(options?.plugins ?? defaultPlugins());
|
|
@@ -4985,6 +5681,7 @@ export {
|
|
|
4985
5681
|
credentialPolicy,
|
|
4986
5682
|
defaultPlugins,
|
|
4987
5683
|
deletePrefix,
|
|
5684
|
+
deletePrefixRespectingLocks,
|
|
4988
5685
|
downloadLinks,
|
|
4989
5686
|
edgeCacheTag,
|
|
4990
5687
|
getPendingUpload,
|
|
@@ -5003,10 +5700,12 @@ export {
|
|
|
5003
5700
|
parseEncodeMetrics,
|
|
5004
5701
|
parseTransformParams,
|
|
5005
5702
|
readImageOriginScope,
|
|
5703
|
+
readObjectLock,
|
|
5006
5704
|
readStoryboardKey,
|
|
5007
5705
|
readTokenScopes,
|
|
5008
5706
|
readTranscodeOutputKeys,
|
|
5009
5707
|
readTranscodeOutputs,
|
|
5708
|
+
retainedSummary,
|
|
5010
5709
|
servedCacheControl,
|
|
5011
5710
|
staticConfig,
|
|
5012
5711
|
storageKey,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devultur/self-host",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "Self-hosted media Worker for Cloudflare: R2 uploads, edge-cached delivery, image transforms, HLS transcode and captions",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cloudflare-workers",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"@jsquash/resize": "2.1.1",
|
|
47
47
|
"@jsquash/webp": "1.5.0",
|
|
48
48
|
"hono": "^4.7.0",
|
|
49
|
-
"@devultur/core": "1.
|
|
49
|
+
"@devultur/core": "1.3.0"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
52
|
"@cf-wasm/photon": "0.4.0",
|