@xiboplayer/cache 0.5.19 → 0.6.0

package/docs/CACHE_PROXY_ARCHITECTURE.md

@@ -130,13 +130,17 @@ The proxy server (`packages/proxy/src/proxy.js`) exposes these endpoints backed
 
 | Method | Route | Purpose |
 |--------|-------|---------|
-| `GET` | `/store/:type/:id` | Serve file (Range support) |
-| `HEAD` | `/store/:type/:id` | Existence + size check |
-| `PUT` | `/store/:type/:id` | Store file |
+| `HEAD` | `/store/:type/*` | Existence + size check (returns 404 for incomplete chunked files) |
+| `GET` | `/store/:type/*` | Serve file (Range support) |
+| `PUT` | `/store/:type/*` | Store file |
 | `POST` | `/store/delete` | Delete files |
 | `POST` | `/store/mark-complete` | Mark chunked download complete |
+| `POST` | `/store/unmark-complete` | Unmark chunked file (keeps chunks, allows partial re-download) |
+| `GET` | `/store/missing-chunks/:type/*` | Return missing chunk indices for a chunked file |
 | `GET` | `/store/list` | List all cached files |
 
+**Note:** HEAD must be registered before GET in Express 5 because `app.get()` also matches HEAD requests.
+
 ### ContentStore (Filesystem)
 
 **Location**: `packages/proxy/src/content-store.js`
@@ -208,6 +212,41 @@ Widget HTML is processed on the main thread by `cacheWidgetHtml()`:
 
 Static resources are stored before widget HTML to prevent race conditions when the iframe loads.
 
+## Chunked Download Resume
+
+Large media files are downloaded in chunks (configurable size, default ~100MB per chunk). If a download is interrupted (crash, network failure, process restart), the player resumes by only downloading missing chunks:
+
+### How it works
+
+1. **On startup/collection**: `enqueueFile()` calls `GET /store/missing-chunks/{storeKey}` before enqueueing
+2. If chunks exist on disk, it populates `file.skipChunks` with the indices of cached chunks
+3. The download pipeline skips those chunks, downloading only the missing ones
+4. After all chunks arrive, the file is marked complete
+
+### On video playback error
+
+1. The renderer emits a `videoError` event when a `<video>` element fails
+2. The PWA calls `GET /store/missing-chunks/{storeKey}` to check for missing chunks
+3. If chunks are missing, it calls `POST /store/unmark-complete` (keeps all existing chunks on disk)
+4. Triggers `collectNow()` — the normal enqueue path populates `skipChunks` and re-downloads only the missing chunks
+
+### Incomplete file detection
+
+- `HEAD /store/:type/*` returns **404** for chunked files with missing chunks (so the download pipeline picks them up)
+- `cacheThrough()` falls through to the CMS for incomplete chunked files instead of serving broken data from the store
+
+### Example
+
+A 2GB video (21 chunks) where chunks 0 and 9 are cached:
+```
+GET /store/missing-chunks/api/v2/player/media/15
+→ { "missing": [1,2,3,4,5,6,7,8,10,11,12,13,14,15,16,17,18,19,20], "numChunks": 21 }
+
+# enqueueFile sets skipChunks = {0, 9}
+# Download pipeline fetches only 19 chunks instead of 21
+# Saves ~200MB of bandwidth
+```
+
 ## Error Handling
 
 | Scenario | StoreClient | DownloadClient |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xiboplayer/cache",
-  "version": "0.5.19",
+  "version": "0.6.0",
   "description": "Offline caching and download management with parallel chunk downloads",
   "type": "module",
   "main": "./src/index.js",
@@ -16,7 +16,7 @@
   },
   "dependencies": {
     "spark-md5": "^3.0.2",
-    "@xiboplayer/utils": "0.5.19"
+    "@xiboplayer/utils": "0.6.0"
   },
   "devDependencies": {
     "vitest": "^2.0.0",

package/src/cache-proxy.test.js

@@ -1,13 +1,11 @@
 /**
- * StoreClient & DownloadClient Tests
+ * StoreClient Tests
  *
  * StoreClient: pure REST client for ContentStore — no SW dependency
- * DownloadClient: SW postMessage client for download orchestration
  */
 
 import { describe, it, expect, beforeEach, vi } from 'vitest';
 import { StoreClient } from './store-client.js';
-import { DownloadClient } from './download-client.js';
 import { createTestBlob } from './test-utils.js';
 
 /**
@@ -15,7 +13,6 @@ import { createTestBlob } from './test-utils.js';
  */
 function resetMocks() {
   global.fetch = vi.fn();
-  delete global.MessageChannel;
 }
 
 // ===========================================================================
@@ -189,205 +186,3 @@ describe('StoreClient', () => {
   });
 });
 
-// ===========================================================================
-// DownloadClient Tests
-// ===========================================================================
-
-/**
- * Helper: set up navigator.serviceWorker mock.
- */
-function setupServiceWorker(opts = {}) {
-  const {
-    supported = true,
-    controller = null,
-    active = undefined,
-    installing = null,
-    waiting = null,
-    swReadyResolves = true,
-  } = opts;
-
-  if (!supported) {
-    Object.defineProperty(navigator, 'serviceWorker', {
-      value: undefined,
-      configurable: true,
-      writable: true,
-    });
-    delete navigator.serviceWorker;
-    return;
-  }
-
-  const activeSW = active !== undefined
-    ? active
-    : controller
-      ? { state: 'activated', postMessage: controller.postMessage }
-      : null;
-
-  const registration = {
-    active: activeSW,
-    installing,
-    waiting,
-  };
-
-  const messageListeners = [];
-
-  const swContainer = {
-    controller,
-    ready: swReadyResolves
-      ? Promise.resolve(registration)
-      : new Promise(() => {}),
-    getRegistration: vi.fn().mockResolvedValue(registration),
-    addEventListener: vi.fn((event, handler) => {
-      if (event === 'message') {
-        messageListeners.push(handler);
-      }
-    }),
-    removeEventListener: vi.fn(),
-  };
-
-  swContainer._messageListeners = messageListeners;
-  swContainer._registration = registration;
-
-  Object.defineProperty(navigator, 'serviceWorker', {
-    value: swContainer,
-    configurable: true,
-    writable: true,
-  });
-
-  return swContainer;
-}
-
-function dispatchSWMessage(swContainer, data) {
-  for (const listener of swContainer._messageListeners || []) {
-    listener({ data });
-  }
-}
-
-function setupMessageChannel() {
-  const channels = [];
-
-  global.MessageChannel = class {
-    constructor() {
-      const self = { port1: { onmessage: null }, port2: {} };
-      channels.push(self);
-      this.port1 = self.port1;
-      this.port2 = self.port2;
-    }
-  };
-
-  return {
-    get lastChannel() {
-      return channels[channels.length - 1];
-    },
-    respondOnLastChannel(data) {
-      const ch = channels[channels.length - 1];
-      if (ch && ch.port1.onmessage) {
-        ch.port1.onmessage({ data });
-      }
-    },
-    channels,
-  };
-}
-
-async function createInitialisedDownloadClient() {
-  const controller = { postMessage: vi.fn() };
-  const sw = setupServiceWorker({ controller });
-
-  const client = new DownloadClient();
-  const initPromise = client.init();
-
-  await Promise.resolve();
-  dispatchSWMessage(sw, { type: 'SW_READY' });
-
-  await initPromise;
-  return { client, sw, controller };
-}
-
-describe('DownloadClient', () => {
-  beforeEach(() => {
-    resetMocks();
-    Object.defineProperty(navigator, 'serviceWorker', {
-      value: undefined,
-      configurable: true,
-      writable: true,
-    });
-  });
-
-  describe('init()', () => {
-    it('should initialize with SW controller', async () => {
-      setupMessageChannel();
-      const { client } = await createInitialisedDownloadClient();
-
-      expect(client.controller).toBeTruthy();
-    });
-
-    it('should throw if SW not supported', async () => {
-      setupServiceWorker({ supported: false });
-
-      const client = new DownloadClient();
-
-      await expect(client.init()).rejects.toThrow('Service Worker not supported');
-    });
-  });
-
-  describe('download()', () => {
-    it('should post DOWNLOAD_FILES message to SW', async () => {
-      setupMessageChannel();
-      const { client } = await createInitialisedDownloadClient();
-
-      client.controller.postMessage = vi.fn();
-
-      const files = [
-        { id: '1', type: 'media', path: 'http://test.com/file1.mp4' },
-        { id: '2', type: 'media', path: 'http://test.com/file2.mp4' },
-      ];
-
-      const mc = setupMessageChannel();
-      const downloadPromise = client.download(files);
-
-      // Simulate SW acknowledging the download
-      mc.respondOnLastChannel({
-        success: true,
-        enqueuedCount: 2,
-        activeCount: 2,
-        queuedCount: 0,
-      });
-
-      await expect(downloadPromise).resolves.toBeUndefined();
-    });
-
-    it('should reject when SW returns error', async () => {
-      setupMessageChannel();
-      const { client } = await createInitialisedDownloadClient();
-
-      client.controller.postMessage = vi.fn();
-
-      const mc = setupMessageChannel();
-      const downloadPromise = client.download([]);
-
-      mc.respondOnLastChannel({ success: false, error: 'Download failed' });
-
-      await expect(downloadPromise).rejects.toThrow('Download failed');
-    });
-
-    it('should throw if SW controller not available', async () => {
-      setupMessageChannel();
-      const { client } = await createInitialisedDownloadClient();
-
-      client.controller = null;
-
-      await expect(client.download([])).rejects.toThrow('Service Worker not available');
-    });
-  });
-
-  describe('getProgress()', () => {
-    it('should return empty object when controller is null', async () => {
-      setupMessageChannel();
-      const { client } = await createInitialisedDownloadClient();
-      client.controller = null;
-
-      const progress = await client.getProgress();
-
-      expect(progress).toEqual({});
-    });
-  });
-});

package/src/download-manager.js

@@ -26,37 +26,17 @@
  */
 
 import { createLogger } from '@xiboplayer/utils';
+import { getFileTypeConfig } from './file-types.js';
 
 const log = createLogger('Download');
 const DEFAULT_CONCURRENCY = 6; // Max concurrent HTTP connections (matches Chromium per-host limit)
 const DEFAULT_CHUNK_SIZE = 50 * 1024 * 1024; // 50MB chunks
 const DEFAULT_MAX_CHUNKS_PER_FILE = 3; // Max parallel chunk downloads per file
 const CHUNK_THRESHOLD = 100 * 1024 * 1024; // Files > 100MB get chunked
-const MAX_RETRIES = 3;
-const RETRY_DELAY_MS = 500; // Fast: 500ms, 1s, 1.5s → total ~3s
-
-// getData (widget data) retry config — CMS "cache not ready" (HTTP 500) resolves
-// when the XTR task runs (30-120s). Use longer backoff to ride it out.
-const GETDATA_MAX_RETRIES = 4;
-const GETDATA_RETRY_DELAYS = [15_000, 30_000, 60_000, 120_000]; // 15s, 30s, 60s, 120s
-const GETDATA_REENQUEUE_DELAY_MS = 60_000; // Re-add to queue after 60s if all retries fail
-const GETDATA_MAX_REENQUEUES = 5; // Max times a getData can be re-enqueued before permanent failure
 const URGENT_CONCURRENCY = 2; // Slots when urgent chunk is active (bandwidth focus)
 const FETCH_TIMEOUT_MS = 600_000; // 10 minutes — 100MB chunk at ~2 Mbps
 const HEAD_TIMEOUT_MS = 15_000; // 15 seconds for HEAD requests
 
-// CMS origin for proxy filtering — set via setCmsOrigin() at init
-let _cmsOrigin = null;
-
-/**
- * Set the CMS origin so toProxyUrl() only proxies CMS URLs.
- * External URLs (CDNs, Google Fonts, geolocation APIs) pass through unchanged.
- * @param {string} origin - e.g. 'https://cms.example.com'
- */
-export function setCmsOrigin(origin) {
-  _cmsOrigin = origin;
-}
-
 /**
  * Infer Content-Type from file path extension.
  * Used when we skip HEAD (size already known from RequiredFiles).
@@ -116,20 +96,6 @@ export function isUrlExpired(url, graceSeconds = 30) {
   return (Date.now() / 1000) >= (expiry - graceSeconds);
 }
 
-/**
- * Rewrite an absolute CMS URL through the local proxy when running behind
- * the proxy server (Chromium kiosk or Electron).
- * Detection: SW/window on localhost (any port) = proxy mode.
- */
-export function toProxyUrl(url) {
-  if (!url.startsWith('http')) return url;
-  const loc = typeof self !== 'undefined' ? self.location : undefined;
-  if (!loc || loc.hostname !== 'localhost') return url;
-  const parsed = new URL(url);
-  // Only proxy URLs belonging to the CMS server; external URLs pass through
-  if (_cmsOrigin && parsed.origin !== _cmsOrigin) return url;
-  return `/file-proxy?cms=${encodeURIComponent(parsed.origin)}&url=${encodeURIComponent(parsed.pathname + parsed.search)}`;
-}
 
 /**
  * DownloadTask - Single HTTP fetch unit
@@ -147,8 +113,7 @@ export class DownloadTask {
     this.blob = null;
     this._parentFile = null;
     this._priority = PRIORITY.normal;
-    // Widget data (getData) uses longer retry backoff — CMS "cache not ready" is transient
-    this.isGetData = fileInfo.isGetData || false;
+    this._typeConfig = getFileTypeConfig(fileInfo.type);
   }
 
   getUrl() {
@@ -156,24 +121,7 @@ export class DownloadTask {
     if (isUrlExpired(url)) {
       throw new Error(`URL expired for ${this.fileInfo.type}/${this.fileInfo.id} — waiting for fresh URL from next collection cycle`);
     }
-    let proxyUrl = toProxyUrl(url);
-
-    // Append store key params so the proxy can save to ContentStore
-    if (proxyUrl.startsWith('/file-proxy')) {
-      const storeKey = `${this.fileInfo.type || 'media'}/${this.fileInfo.id}`;
-      proxyUrl += `&storeKey=${encodeURIComponent(storeKey)}`;
-      if (this.chunkIndex != null) {
-        proxyUrl += `&chunkIndex=${this.chunkIndex}`;
-        if (this._parentFile) {
-          proxyUrl += `&numChunks=${this._parentFile.totalChunks}`;
-          proxyUrl += `&chunkSize=${this._parentFile.options.chunkSize || 104857600}`;
-        }
-      }
-      if (this.fileInfo.md5) {
-        proxyUrl += `&md5=${encodeURIComponent(this.fileInfo.md5)}`;
-      }
-    }
-    return proxyUrl;
+    return url;
   }
 
   async start() {
@@ -182,8 +130,22 @@ export class DownloadTask {
     if (this.rangeStart != null) {
       headers['Range'] = `bytes=${this.rangeStart}-${this.rangeEnd}`;
     }
+    // Pass chunk metadata and MD5 via custom headers for cache-through proxy
+    if (this.chunkIndex != null) {
+      headers['X-Store-Chunk-Index'] = String(this.chunkIndex);
+      if (this._parentFile) {
+        headers['X-Store-Num-Chunks'] = String(this._parentFile.totalChunks);
+        headers['X-Store-Chunk-Size'] = String(this._parentFile.options.chunkSize || 104857600);
+      }
+    }
+    if (this.fileInfo.md5) {
+      headers['X-Store-MD5'] = this.fileInfo.md5;
+    }
+    if (this.fileInfo.updateInterval) {
+      headers['X-Cache-TTL'] = String(this.fileInfo.updateInterval);
+    }
 
-    const maxRetries = this.isGetData ? GETDATA_MAX_RETRIES : MAX_RETRIES;
+    const maxRetries = this._typeConfig.maxRetries;
 
     for (let attempt = 1; attempt <= maxRetries; attempt++) {
       const ac = new AbortController();
@@ -205,9 +167,8 @@ export class DownloadTask {
       } catch (error) {
         const msg = ac.signal.aborted ? `Timeout after ${FETCH_TIMEOUT_MS / 1000}s` : error.message;
         if (attempt < maxRetries) {
-          const delay = this.isGetData
-            ? GETDATA_RETRY_DELAYS[attempt - 1]
-            : RETRY_DELAY_MS * attempt;
+          const delay = this._typeConfig.retryDelays?.[attempt - 1]
+            ?? this._typeConfig.retryDelayMs * attempt;
           const chunkLabel = this.chunkIndex != null ? ` chunk ${this.chunkIndex}` : '';
           log.warn(`[DownloadTask] ${this.fileInfo.type}/${this.fileInfo.id}${chunkLabel} attempt ${attempt}/${maxRetries} failed: ${msg}. Retrying in ${delay / 1000}s...`);
           await new Promise(resolve => setTimeout(resolve, delay));
@@ -264,17 +225,7 @@ export class FileDownload {
     if (isUrlExpired(url)) {
       throw new Error(`URL expired for ${this.fileInfo.type}/${this.fileInfo.id} — waiting for fresh URL from next collection cycle`);
     }
-    let proxyUrl = toProxyUrl(url);
-
-    // Append store key for ContentStore (same as DownloadTask)
-    if (proxyUrl.startsWith('/file-proxy')) {
-      const storeKey = `${this.fileInfo.type || 'media'}/${this.fileInfo.id}`;
-      proxyUrl += `&storeKey=${encodeURIComponent(storeKey)}`;
-      if (this.fileInfo.md5) {
-        proxyUrl += `&md5=${encodeURIComponent(this.fileInfo.md5)}`;
-      }
-    }
-    return proxyUrl;
+    return url;
   }
 
   wait() {
@@ -296,7 +247,12 @@ export class FileDownload {
       this.totalBytes = (size && size > 0) ? parseInt(size) : 0;
       this._contentType = inferContentType(this.fileInfo);
 
-      if (this.totalBytes === 0) {
+      // Skip HEAD for types that declare skipHead (e.g. datasets — dynamic API endpoints).
+      // These generate responses server-side; HEAD triggers the full handler for nothing
+      // and may fail if the CMS cache isn't warm yet. They're always small, never chunked.
+      const skipHead = getFileTypeConfig(this.fileInfo.type).skipHead;
+
+      if (this.totalBytes === 0 && !skipHead) {
         // No size declared — HEAD fallback (rare: only for files without CMS size)
         const url = this.getUrl();
         const ac = new AbortController();
@@ -916,25 +872,26 @@ export class DownloadQueue {
         task._parentFile._runningCount--;
         this._activeTasks = this._activeTasks.filter(t => t !== task);
 
-        // getData (widget data): defer re-enqueue instead of permanent failure.
+        // Re-enqueueable types (e.g. datasets): defer re-enqueue instead of permanent failure.
         // CMS "cache not ready" resolves when the XTR task runs (30-120s).
-        if (task.isGetData) {
+        const { maxReenqueues, reenqueueDelayMs } = task._typeConfig;
+        if (maxReenqueues > 0) {
           task._reenqueueCount = (task._reenqueueCount || 0) + 1;
-          if (task._reenqueueCount > GETDATA_MAX_REENQUEUES) {
-            log.error(`[DownloadQueue] getData ${key} exceeded ${GETDATA_MAX_REENQUEUES} re-enqueues, failing permanently`);
+          if (task._reenqueueCount > maxReenqueues) {
+            log.error(`[DownloadQueue] ${key} exceeded ${maxReenqueues} re-enqueues, failing permanently`);
             this.processQueue();
             task._parentFile.onTaskFailed(task, err);
             return;
           }
-          log.warn(`[DownloadQueue] getData ${key} failed all retries (attempt ${task._reenqueueCount}/${GETDATA_MAX_REENQUEUES}), scheduling re-enqueue in ${GETDATA_REENQUEUE_DELAY_MS / 1000}s`);
+          log.warn(`[DownloadQueue] ${key} failed all retries (attempt ${task._reenqueueCount}/${maxReenqueues}), scheduling re-enqueue in ${reenqueueDelayMs / 1000}s`);
           const timerId = setTimeout(() => {
             this._reenqueueTimers.delete(timerId);
             task.state = 'pending';
             task._parentFile.state = 'downloading';
             this.queue.push(task);
-            log.info(`[DownloadQueue] getData ${key} re-enqueued for retry`);
+            log.info(`[DownloadQueue] ${key} re-enqueued for retry`);
             this.processQueue();
-          }, GETDATA_REENQUEUE_DELAY_MS);
+          }, reenqueueDelayMs);
           this._reenqueueTimers.add(timerId);
           this.processQueue();
           return;
@@ -978,6 +935,8 @@ export class DownloadQueue {
   getProgress() {
     const progress = {};
     for (const [key, file] of this.active.entries()) {
+      // Skip completed/failed — they stay in active until removeCompleted() runs
+      if (file.state === 'complete' || file.state === 'failed') continue;
       progress[key] = {
         downloaded: file.downloadedBytes,
         total: file.totalBytes,

package/src/file-types.js

@@ -0,0 +1,26 @@
+/**
+ * FILE_TYPES — centralized download behavior per file type.
+ *
+ * Each type declares retry strategy, HEAD skip, and cache TTL.
+ * Used by DownloadTask/FileDownload instead of ad-hoc isGetData checks.
+ */
+
+export const FILE_TYPES = {
+  media:   { maxRetries: 3, retryDelayMs: 500, retryDelays: null,
+             maxReenqueues: 0, reenqueueDelayMs: 0,
+             skipHead: false, cacheTtl: Infinity },
+  layout:  { maxRetries: 3, retryDelayMs: 500, retryDelays: null,
+             maxReenqueues: 0, reenqueueDelayMs: 0,
+             skipHead: false, cacheTtl: Infinity },
+  dataset: { maxRetries: 4, retryDelayMs: 0,
+             retryDelays: [15_000, 30_000, 60_000, 120_000],
+             maxReenqueues: 5, reenqueueDelayMs: 60_000,
+             skipHead: true, cacheTtl: 300 },
+  static:  { maxRetries: 3, retryDelayMs: 500, retryDelays: null,
+             maxReenqueues: 0, reenqueueDelayMs: 0,
+             skipHead: false, cacheTtl: Infinity },
+};
+
+export function getFileTypeConfig(type) {
+  return FILE_TYPES[type] || FILE_TYPES.media;
+}

package/src/index.d.ts

@@ -8,25 +8,26 @@ export class StoreClient {
   list(): Promise<Array<{ id: string; type: string; size: number }>>;
 }
 
-export class DownloadClient {
-  controller: ServiceWorker | null;
-  fetchReady: boolean;
-  init(): Promise<void>;
-  download(payload: object | any[]): Promise<void>;
-  prioritize(fileType: string, fileId: string): Promise<void>;
-  prioritizeLayout(mediaIds: string[]): Promise<void>;
-  getProgress(): Promise<Record<string, any>>;
-  cleanup(): void;
-}
-
 export class DownloadManager {
-  constructor(options?: { concurrency?: number; chunkSize?: number; maxChunksPerFile?: number });
+  constructor(options?: { concurrency?: number; chunkSize?: number; chunksPerFile?: number });
   enqueue(fileInfo: any): any;
+  getTask(key: string): any;
+  getProgress(): Record<string, any>;
   prioritizeLayoutFiles(mediaIds: string[]): void;
+  clear(): void;
+  queue: any;
 }
 
-export class FileDownload {}
-export class LayoutTaskBuilder {}
+export class FileDownload {
+  state: string;
+  wait(): Promise<Blob>;
+}
+export class LayoutTaskBuilder {
+  constructor(queue: any);
+  addFile(fileInfo: any): FileDownload;
+  build(): Promise<any[]>;
+}
+export const BARRIER: symbol;
 export class CacheManager {}
 export class CacheAnalyzer {
   constructor(store: StoreClient);
@@ -35,6 +36,4 @@ export class CacheAnalyzer {
 export const cacheManager: CacheManager;
 
 export function isUrlExpired(url: string): boolean;
-export function toProxyUrl(url: string): string;
-export function setCmsOrigin(origin: string): void;
 export function cacheWidgetHtml(...args: any[]): any;

package/src/index.js

@@ -3,7 +3,7 @@ import pkg from '../package.json' with { type: 'json' };
 export const VERSION = pkg.version;
 export { CacheManager, cacheManager } from './cache.js';
 export { StoreClient } from './store-client.js';
-export { DownloadClient } from './download-client.js';
-export { DownloadManager, FileDownload, LayoutTaskBuilder, isUrlExpired, toProxyUrl, setCmsOrigin } from './download-manager.js';
+export { DownloadManager, FileDownload, LayoutTaskBuilder, BARRIER, isUrlExpired } from './download-manager.js';
 export { CacheAnalyzer } from './cache-analyzer.js';
 export { cacheWidgetHtml } from './widget-html.js';
+export { FILE_TYPES, getFileTypeConfig } from './file-types.js';