@budarin/psw-plugin-opfs-serve-range 1.0.0

package/dist/opfsRangeFromNetworkAndCache.d.ts

@@ -0,0 +1,8 @@
+import type { Plugin } from '@budarin/pluggable-serviceworker';
+export interface OpfsRangeFromNetworkAndCacheOptions {
+    order?: number;
+    include?: string[];
+    exclude?: string[];
+    enableLogging?: boolean;
+}
+export declare function opfsRangeFromNetworkAndCache(options?: OpfsRangeFromNetworkAndCacheOptions): Plugin | undefined;

package/dist/opfsRangeFromNetworkAndCache.js

@@ -0,0 +1,191 @@
+import { notifyClients } from '@budarin/pluggable-serviceworker/utils';
+import { HEADER_RANGE } from '@budarin/http-constants/headers';
+import { getOpfsDir, urlToOpfsKey } from './index.js';
+import { parseRangeHeader, build206Response, build206ResponseFromStream, createRangeExtractTransform, } from './opfsRangeUtil.js';
+import { writeToOpfs, metadataFromResponse } from './opfsWrite.js';
+import { isOpfsAvailable, shouldProcessFile } from './opfsUtil.js';
+import { isBlacklisted } from './opfsLru.js';
+import { OPFS_MSG_SKIP_QUOTA_EXCEEDED } from './opfsMessages.js';
+const loadingUrls = new Set();
+async function backgroundFullFetchToOpfs(url, logger, enableLogging) {
+    try {
+        if (isBlacklisted(url)) {
+            if (enableLogging) {
+                logger.debug(`opfsRangeFromNetworkAndCache: skip ${url} (blacklisted, quota exceeded)`);
+            }
+            return;
+        }
+        const fullRequest = new Request(url, { method: 'GET' });
+        const response = await fetch(fullRequest);
+        if (!response.ok || !response.body) {
+            if (enableLogging) {
+                logger.debug(`opfsRangeFromNetworkAndCache: background full GET ${url} -> ${response.status}, skip cache`);
+            }
+            return;
+        }
+        if (response.status !== 200) {
+            if (enableLogging) {
+                logger.debug(`opfsRangeFromNetworkAndCache: background full GET ${url} -> ${response.status}, skip cache`);
+            }
+            return;
+        }
+        const metadata = metadataFromResponse(response, url);
+        const key = await urlToOpfsKey(url);
+        const root = await navigator.storage.getDirectory();
+        const dir = await getOpfsDir(root, true);
+        await writeToOpfs(dir, key, response.body, metadata, {
+            url,
+            ...(metadata.size > 0 && { knownSize: metadata.size }),
+        });
+        if (enableLogging) {
+            logger.debug(`opfsRangeFromNetworkAndCache: background cached ${url} -> ${key} (${metadata.size} bytes)`);
+        }
+    }
+    catch (err) {
+        if (enableLogging) {
+            logger.error(`opfsRangeFromNetworkAndCache: background full GET failed ${url}`, err);
+        }
+    }
+    finally {
+        loadingUrls.delete(url);
+    }
+}
+export function opfsRangeFromNetworkAndCache(options = {}) {
+    if (!isOpfsAvailable()) {
+        return undefined;
+    }
+    const { order = -10, include, exclude, enableLogging = false, } = options;
+    return {
+        name: 'opfs-range-from-network-and-cache',
+        order,
+        async fetch(event, logger) {
+            const request = event.request;
+            if (request.method !== 'GET') {
+                return;
+            }
+            if (!shouldProcessFile(request.url, include, exclude)) {
+                return;
+            }
+            const url = request.url;
+            const rangeHeader = request.headers.get(HEADER_RANGE);
+            if (!rangeHeader) {
+                try {
+                    const response = await fetch(request);
+                    if (!response.ok || !response.body) {
+                        return response;
+                    }
+                    if (response.status !== 200) {
+                        return response;
+                    }
+                    if (isBlacklisted(url)) {
+                        notifyClients(OPFS_MSG_SKIP_QUOTA_EXCEEDED, { url });
+                        return new Response(response.body, {
+                            status: response.status,
+                            statusText: response.statusText,
+                            headers: response.headers,
+                        });
+                    }
+                    const metadata = metadataFromResponse(response, url);
+                    const key = await urlToOpfsKey(url);
+                    const root = await navigator.storage.getDirectory();
+                    const dir = await getOpfsDir(root, true);
+                    const [branch1, branch2] = response.body.tee();
+                    writeToOpfs(dir, key, branch2, metadata, {
+                        url,
+                        ...(metadata.size > 0 && { knownSize: metadata.size }),
+                    }).catch((err) => {
+                        if (enableLogging) {
+                            logger.error(`opfsRangeFromNetworkAndCache: write failed ${url}`, err);
+                        }
+                    });
+                    if (enableLogging) {
+                        logger.debug(`opfsRangeFromNetworkAndCache: caching full GET ${url} (${metadata.size} bytes)`);
+                    }
+                    return new Response(branch1, {
+                        status: response.status,
+                        statusText: response.statusText,
+                        headers: response.headers,
+                    });
+                }
+                catch {
+                    return;
+                }
+            }
+            try {
+                const key = await urlToOpfsKey(url);
+                const root = await navigator.storage.getDirectory();
+                try {
+                    const dir = await getOpfsDir(root, false);
+                    await dir.getFileHandle(key);
+                    logger.warn(`opfsRangeFromNetworkAndCache: file exists in OPFS for ${url} but request was not served from cache; fetching from network (possible: If-Range mismatch, invalid range, or opfsServeRange order)`);
+                }
+                catch {
+                }
+                const response = await fetch(request);
+                if (!response.body) {
+                    return response;
+                }
+                if (response.status === 206) {
+                    if (!loadingUrls.has(url)) {
+                        loadingUrls.add(url);
+                        backgroundFullFetchToOpfs(url, logger, enableLogging);
+                    }
+                    return new Response(response.body, {
+                        status: response.status,
+                        statusText: response.statusText,
+                        headers: response.headers,
+                    });
+                }
+                if (response.status === 416) {
+                    return response;
+                }
+                if (response.status === 200) {
+                    const contentLength = response.headers.get('Content-Length');
+                    const fullSize = contentLength
+                        ? parseInt(contentLength, 10)
+                        : 0;
+                    const type = response.headers.get('Content-Type') ??
+                        'application/octet-stream';
+                    const etag = response.headers.get('ETag') ?? undefined;
+                    const lastModified = response.headers.get('Last-Modified') ?? undefined;
+                    if (fullSize > 0 &&
+                        Number.isInteger(fullSize)) {
+                        const range = parseRangeHeader(rangeHeader, fullSize);
+                        const metadata = metadataFromResponse(response, url);
+                        const key = await urlToOpfsKey(url);
+                        const root = await navigator.storage.getDirectory();
+                        const dir = await getOpfsDir(root, true);
+                        const [branch1, branch2] = response.body.tee();
+                        writeToOpfs(dir, key, branch2, metadata, {
+                            url,
+                            knownSize: fullSize,
+                        }).catch((err) => {
+                            if (enableLogging) {
+                                logger.error(`opfsRangeFromNetworkAndCache: write failed ${url}`, err);
+                            }
+                        });
+                        const rangeStream = branch1.pipeThrough(createRangeExtractTransform(range));
+                        return build206ResponseFromStream(rangeStream, range, fullSize, {
+                            type,
+                            ...(etag && { etag }),
+                            ...(lastModified && { lastModified }),
+                        });
+                    }
+                    const blob = await response.blob();
+                    const size = blob.size;
+                    const range = parseRangeHeader(rangeHeader, size);
+                    const rangeBlob = blob.slice(range.start, range.end + 1);
+                    return build206Response(rangeBlob, range, size, {
+                        type,
+                        ...(etag && { etag }),
+                        ...(lastModified && { lastModified }),
+                    });
+                }
+                return response;
+            }
+            catch {
+                return;
+            }
+        },
+    };
+}

package/dist/opfsRangeUtil.d.ts

@@ -0,0 +1,14 @@
+export interface RangeSpec {
+    start: number;
+    end: number;
+}
+export declare function parseRangeHeader(rangeHeader: string, fullSize: number): RangeSpec;
+export interface Build206Options {
+    type?: string;
+    etag?: string;
+    lastModified?: string;
+    cacheControl?: string;
+}
+export declare function createRangeExtractTransform(range: RangeSpec): TransformStream<Uint8Array, Uint8Array>;
+export declare function build206Response(rangeBlob: Blob, range: RangeSpec, fullSize: number, options?: Build206Options): Response;
+export declare function build206ResponseFromStream(rangeStream: ReadableStream<Uint8Array>, range: RangeSpec, fullSize: number, options?: Build206Options): Response;

package/dist/opfsRangeUtil.js

@@ -0,0 +1,99 @@
+import { HEADER_CONTENT_TYPE, HEADER_CONTENT_RANGE, HEADER_CONTENT_LENGTH, HEADER_ETAG, HEADER_LAST_MODIFIED, } from '@budarin/http-constants/headers';
+import { HTTP_STATUS_PARTIAL_CONTENT } from '@budarin/http-constants/statuses';
+import { MIME_APPLICATION_OCTET_STREAM } from '@budarin/http-constants/mime-types';
+const HEADER_ACCEPT_RANGES = 'Accept-Ranges';
+export function parseRangeHeader(rangeHeader, fullSize) {
+    const trimmed = rangeHeader.trim();
+    const suffixMatch = /^bytes=-(\d+)$/.exec(trimmed);
+    if (suffixMatch) {
+        const suffixLength = parseInt(suffixMatch[1], 10);
+        if (isNaN(suffixLength) || suffixLength <= 0) {
+            throw new Error('Invalid suffix range value');
+        }
+        const start = Math.max(0, fullSize - suffixLength);
+        const end = fullSize - 1;
+        return { start, end };
+    }
+    const rangeMatch = /^bytes=(\d+)-(\d*)$/.exec(trimmed);
+    if (!rangeMatch) {
+        throw new Error('Invalid or unsupported range header format');
+    }
+    const start = parseInt(rangeMatch[1], 10);
+    const end = rangeMatch[2] ? parseInt(rangeMatch[2], 10) : fullSize - 1;
+    if (isNaN(start) || isNaN(end)) {
+        throw new Error('Invalid range values');
+    }
+    if (start < 0 || start >= fullSize) {
+        throw new Error('Range start is out of bounds');
+    }
+    if (end < start || end >= fullSize) {
+        throw new Error('Range end is out of bounds');
+    }
+    return { start, end };
+}
+export function createRangeExtractTransform(range) {
+    let offset = 0;
+    const { start, end } = range;
+    return new TransformStream({
+        transform(chunk, controller) {
+            const chunkStart = offset;
+            const chunkEnd = offset + chunk.byteLength;
+            offset = chunkEnd;
+            if (chunkEnd <= start || chunkStart > end) {
+                return;
+            }
+            const sliceStart = Math.max(0, start - chunkStart);
+            const sliceEnd = Math.min(chunk.byteLength, end - chunkStart + 1);
+            if (sliceStart < sliceEnd) {
+                controller.enqueue(chunk.slice(sliceStart, sliceEnd));
+            }
+        },
+    });
+}
+export function build206Response(rangeBlob, range, fullSize, options = {}) {
+    const { type = MIME_APPLICATION_OCTET_STREAM, etag, lastModified, cacheControl, } = options;
+    const contentRange = `bytes ${String(range.start)}-${String(range.end)}/${String(fullSize)}`;
+    const headers = new Headers({
+        [HEADER_CONTENT_RANGE]: contentRange,
+        [HEADER_CONTENT_LENGTH]: String(rangeBlob.size),
+        [HEADER_CONTENT_TYPE]: type,
+        [HEADER_ACCEPT_RANGES]: 'bytes',
+    });
+    if (cacheControl) {
+        headers.set('Cache-Control', cacheControl);
+    }
+    if (etag) {
+        headers.set(HEADER_ETAG, etag);
+    }
+    if (lastModified) {
+        headers.set(HEADER_LAST_MODIFIED, lastModified);
+    }
+    return new Response(rangeBlob, {
+        status: HTTP_STATUS_PARTIAL_CONTENT,
+        headers,
+    });
+}
+export function build206ResponseFromStream(rangeStream, range, fullSize, options = {}) {
+    const { type = MIME_APPLICATION_OCTET_STREAM, etag, lastModified, cacheControl, } = options;
+    const rangeLength = range.end - range.start + 1;
+    const contentRange = `bytes ${String(range.start)}-${String(range.end)}/${String(fullSize)}`;
+    const headers = new Headers({
+        [HEADER_CONTENT_RANGE]: contentRange,
+        [HEADER_CONTENT_LENGTH]: String(rangeLength),
+        [HEADER_CONTENT_TYPE]: type,
+        [HEADER_ACCEPT_RANGES]: 'bytes',
+    });
+    if (cacheControl) {
+        headers.set('Cache-Control', cacheControl);
+    }
+    if (etag) {
+        headers.set(HEADER_ETAG, etag);
+    }
+    if (lastModified) {
+        headers.set(HEADER_LAST_MODIFIED, lastModified);
+    }
+    return new Response(rangeStream, {
+        status: HTTP_STATUS_PARTIAL_CONTENT,
+        headers,
+    });
+}

package/dist/opfsUtil.d.ts

@@ -0,0 +1,11 @@
+export interface OpfsConfigOptions {
+    folderName?: string;
+    maxCacheFraction?: number;
+}
+export declare function isOpfsAvailable(): boolean;
+export declare function configureOpfs(options?: OpfsConfigOptions): void;
+export declare function getMaxCacheFraction(): number;
+export declare function matchesGlob(url: string, pattern: string): boolean;
+export declare function shouldProcessFile(url: string, include?: string[], exclude?: string[]): boolean;
+export declare function getOpfsDir(root: FileSystemDirectoryHandle, create: boolean): Promise<FileSystemDirectoryHandle>;
+export declare function clearOpfsCache(): Promise<void>;

package/dist/opfsUtil.js

@@ -0,0 +1,61 @@
+import { OPFS_FOLDER_NAME } from './opfsFormat.js';
+const DEFAULT_MAX_CACHE_FRACTION = 0.5;
+let opfsConfig = {};
+export function isOpfsAvailable() {
+    return (typeof navigator !== 'undefined' &&
+        navigator?.storage != null &&
+        typeof navigator.storage.getDirectory === 'function');
+}
+export function configureOpfs(options = {}) {
+    opfsConfig = { ...options };
+}
+function getResolvedFolderName() {
+    return opfsConfig.folderName ?? OPFS_FOLDER_NAME;
+}
+export function getMaxCacheFraction() {
+    const v = opfsConfig.maxCacheFraction;
+    if (v === undefined || v < 0 || v > 1) {
+        return DEFAULT_MAX_CACHE_FRACTION;
+    }
+    return v;
+}
+export function matchesGlob(url, pattern) {
+    const pathname = new URL(url, 'https://example.com').pathname;
+    const regexPattern = pattern
+        .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+        .replace(/\*/g, '.*')
+        .replace(/\?/g, '.');
+    const regex = new RegExp(`^${regexPattern}$`);
+    return regex.test(pathname);
+}
+export function shouldProcessFile(url, include, exclude) {
+    if (exclude?.length) {
+        for (const pattern of exclude) {
+            if (matchesGlob(url, pattern)) {
+                return false;
+            }
+        }
+    }
+    if (include?.length) {
+        for (const pattern of include) {
+            if (matchesGlob(url, pattern)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    return true;
+}
+export async function getOpfsDir(root, create) {
+    const name = getResolvedFolderName();
+    return root.getDirectoryHandle(name, { create });
+}
+export async function clearOpfsCache() {
+    const root = await navigator.storage.getDirectory();
+    const name = getResolvedFolderName();
+    try {
+        await root.removeEntry(name, { recursive: true });
+    }
+    catch {
+    }
+}

package/dist/opfsWrite.d.ts

@@ -0,0 +1,7 @@
+import { type OpfsMetadata } from './opfsFormat.js';
+export interface WriteToOpfsOptions {
+    url?: string;
+    knownSize?: number;
+}
+export declare function writeToOpfs(dir: FileSystemDirectoryHandle, key: string, bodyStream: ReadableStream<Uint8Array>, metadata: OpfsMetadata, options?: WriteToOpfsOptions): Promise<void>;
+export declare function metadataFromResponse(response: Response, url: string): OpfsMetadata;

package/dist/opfsWrite.js

@@ -0,0 +1,94 @@
+import { notifyClients } from '@budarin/pluggable-serviceworker/utils';
+import { OPFS_META_FOOTER_LENGTH, MEGABYTE, } from './opfsFormat.js';
+import { ensureSpaceForWrite, listCacheFilesWithMeta, getTotalCacheSize, computeEvictionSet, evictFiles, getCacheLimit, getStorageEstimate, addToBlacklist, } from './opfsLru.js';
+import { OPFS_MSG_WRITE_SKIPPED_SIZE, OPFS_MSG_QUOTA_EXCEEDED, OPFS_MSG_EVICTION_COMPLETED, OPFS_MSG_WRITE_FAILED, } from './opfsMessages.js';
+export async function writeToOpfs(dir, key, bodyStream, metadata, options = {}) {
+    const { url, knownSize } = options;
+    if (knownSize !== undefined && knownSize > 0) {
+        const result = await ensureSpaceForWrite(dir, knownSize, {
+            onEvicted(keys) {
+                if (keys.length > 0) {
+                    notifyClients(OPFS_MSG_EVICTION_COMPLETED, { count: keys.length });
+                }
+            },
+        });
+        if (!result.ok) {
+            notifyClients(OPFS_MSG_WRITE_SKIPPED_SIZE, {
+                url,
+                size: knownSize,
+                reason: result.reason,
+            });
+            throw new Error(result.reason);
+        }
+    }
+    const handle = await dir.getFileHandle(key, { create: true });
+    const writable = await handle.createWritable();
+    let bodySize = 0;
+    const wrapper = new WritableStream({
+        write(chunk) {
+            bodySize += chunk.byteLength;
+            return writable.write(chunk);
+        },
+        async close() {
+            const meta = {
+                ...metadata,
+                size: bodySize,
+                lastAccessed: Date.now(),
+            };
+            const metaJson = JSON.stringify(meta);
+            const metaBytes = new TextEncoder().encode(metaJson);
+            const lengthAb = new ArrayBuffer(OPFS_META_FOOTER_LENGTH);
+            new DataView(lengthAb).setUint32(0, metaBytes.length, true);
+            await writable.seek(bodySize);
+            await writable.write(metaBytes);
+            await writable.write(lengthAb);
+            await writable.close();
+        },
+    });
+    try {
+        await bodyStream.pipeTo(wrapper);
+    }
+    catch (err) {
+        const isQuotaExceeded = err instanceof Error &&
+            (err.name === 'QuotaExceededError' || err.name === 'QuotaExceeded');
+        try {
+            await dir.removeEntry(key);
+        }
+        catch {
+        }
+        if (isQuotaExceeded && url !== undefined) {
+            const entries = await listCacheFilesWithMeta(dir);
+            const totalCacheSize = getTotalCacheSize(entries);
+            const bytesWritten = bodySize;
+            if (bytesWritten >= totalCacheSize) {
+                addToBlacklist(url);
+                notifyClients(OPFS_MSG_QUOTA_EXCEEDED, { url });
+            }
+            else {
+                const estimate = await getStorageEstimate();
+                const limit = getCacheLimit(estimate);
+                const headroom = Math.min(MEGABYTE, Math.max(0, Math.floor(limit * 0.1)));
+                const needToFree = bytesWritten + headroom;
+                const keysToDelete = computeEvictionSet(entries, needToFree);
+                await evictFiles(dir, keysToDelete);
+            }
+        }
+        notifyClients(OPFS_MSG_WRITE_FAILED, { url, reason: err instanceof Error ? err.message : String(err) });
+        throw err;
+    }
+}
+export function metadataFromResponse(response, url) {
+    const contentLength = response.headers.get('Content-Length');
+    const parsed = contentLength ? parseInt(contentLength, 10) : 0;
+    const size = parsed > 0 && Number.isInteger(parsed) ? parsed : 0;
+    const type = response.headers.get('Content-Type') ?? 'application/octet-stream';
+    const etag = response.headers.get('ETag') ?? undefined;
+    const lastModified = response.headers.get('Last-Modified') ?? undefined;
+    return {
+        url,
+        size,
+        type,
+        ...(etag && { etag }),
+        ...(lastModified && { lastModified }),
+    };
+}