@xiboplayer/cache 0.3.7 → 0.4.1

package/README.md

@@ -11,6 +11,8 @@ Manages media downloads and offline storage for Xibo players:
 - **MD5 verification** — integrity checking with CRC32-based skip optimization
 - **Download queue** — flat queue with barriers for layout-ordered downloading
 - **CacheProxy** — browser-side proxy that communicates with a Service Worker backend
+- **Widget data via enriched RequiredFiles** — RSS/dataset widget data is fetched through server-side enriched RequiredFiles paths (CMS adds download URLs), not via client-side pre-fetching
+- **Dynamic BASE path** — widget HTML `<base>` tag uses a dynamic path within the Service Worker scope for correct relative URL resolution
 
 ## Installation
 

package/docs/CACHE_PROXY_ARCHITECTURE.md

@@ -363,6 +363,18 @@ it('should download and cache files', async () => {
    const blob = await cacheProxy.getFile('media', id);
    ```
 
+## Widget Data Download Flow
+
+Widget data for RSS feeds and dataset widgets is handled server-side. The CMS enriches
+the RequiredFiles response with absolute download URLs for widget data files. These are
+downloaded through the normal CacheProxy/Service Worker pipeline alongside regular media,
+rather than being fetched client-side by the player. This ensures widget data is available
+offline and benefits from the same parallel chunk download and caching infrastructure.
+
+Widget HTML served from cache uses a dynamic `<base>` tag pointing to the Service Worker
+scope path, ensuring relative URLs within widget HTML resolve correctly regardless of the
+player's deployment path.
+
 ### For New Platforms
 
 Simply use CacheProxy from the start:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xiboplayer/cache",
-  "version": "0.3.7",
+  "version": "0.4.1",
   "description": "Offline caching and download management with parallel chunk downloads",
   "type": "module",
   "main": "./src/index.js",
@@ -12,7 +12,7 @@
   },
   "dependencies": {
     "spark-md5": "^3.0.2",
-    "@xiboplayer/utils": "0.3.7"
+    "@xiboplayer/utils": "0.4.1"
   },
   "devDependencies": {
     "vitest": "^2.0.0",

package/src/cache-analyzer.js

@@ -56,6 +56,14 @@ export class CacheAnalyzer {
     for (const file of cachedFiles) {
       if (requiredIds.has(String(file.id))) {
         required.push(file);
+      } else if (file.type === 'widget') {
+        // Widget HTML IDs are "layoutId/regionId/widgetId" — check parent layout
+        const parentLayoutId = String(file.id).split('/')[0];
+        if (requiredIds.has(parentLayoutId)) {
+          required.push(file);
+        } else {
+          orphaned.push(file);
+        }
       } else {
         orphaned.push(file);
       }

package/src/cache-analyzer.test.js

@@ -170,6 +170,20 @@ describe('CacheAnalyzer', () => {
       vi.unstubAllGlobals();
     });
 
+    it('should treat widget HTML as required when parent layout is required', async () => {
+      mockCache._addFile({ id: '470', type: 'layout', size: 500, cachedAt: 100 });
+      mockCache._addFile({ id: '470/213/182', type: 'widget', size: 0, cachedAt: 0 });
+      mockCache._addFile({ id: '470/215/184', type: 'widget', size: 0, cachedAt: 0 });
+      mockCache._addFile({ id: '99/10/5', type: 'widget', size: 0, cachedAt: 0 });
+
+      const report = await analyzer.analyze([{ id: '470', type: 'layout' }]);
+
+      // Layout 470 + its 2 widgets = 3 required; widget for layout 99 = orphaned
+      expect(report.files.required).toBe(3);
+      expect(report.files.orphaned).toBe(1);
+      expect(report.orphaned[0].id).toBe('99/10/5');
+    });
+
     it('should handle files with missing size or cachedAt', async () => {
       mockCache._addFile({ id: '1', type: 'media' }); // no size, no cachedAt
 

package/src/cache.js

@@ -8,6 +8,9 @@
  * - Full cache clearing
  */
 
+import { createLogger } from '@xiboplayer/utils';
+
+const log = createLogger('Cache');
 const CACHE_NAME = 'xibo-media-v1';
 
 // Dynamic base path for multi-variant deployment (pwa, pwa-xmds, pwa-xlr)
@@ -61,7 +64,7 @@ export class CacheManager {
     }
 
     if (orphaned.length > 0) {
-      console.log(`[Cache] ${orphaned.length} media files orphaned after layout ${layoutId} removed:`, orphaned);
+      log.info(`${orphaned.length} media files orphaned after layout ${layoutId} removed:`, orphaned);
     }
     return orphaned;
   }

package/src/download-manager.js

@@ -25,6 +25,9 @@
  *   const blob = await file.wait();
  */
 
+import { createLogger } from '@xiboplayer/utils';
+
+const log = createLogger('Download');
 const DEFAULT_CONCURRENCY = 6; // Max concurrent HTTP connections (matches Chromium per-host limit)
 const DEFAULT_CHUNK_SIZE = 50 * 1024 * 1024; // 50MB chunks
 const DEFAULT_MAX_CHUNKS_PER_FILE = 3; // Max parallel chunk downloads per file
@@ -94,6 +97,20 @@ export function isUrlExpired(url, graceSeconds = 30) {
   return (Date.now() / 1000) >= (expiry - graceSeconds);
 }
 
+/**
+ * Rewrite an absolute CMS URL through the local proxy when running behind
+ * the proxy server (Chromium kiosk or Electron).
+ * Detection: SW/window on localhost:8765 = proxy mode.
+ */
+export function rewriteUrlForProxy(url) {
+  if (!url.startsWith('http')) return url;
+  const loc = typeof self !== 'undefined' ? self.location : undefined;
+  if (!loc || loc.hostname !== 'localhost' || loc.port !== '8765') return url;
+  const parsed = new URL(url);
+  const cmsOrigin = parsed.origin;
+  return `/file-proxy?cms=${encodeURIComponent(cmsOrigin)}&url=${encodeURIComponent(parsed.pathname + parsed.search)}`;
+}
+
 /**
  * DownloadTask - Single HTTP fetch unit
  *
@@ -117,7 +134,7 @@ export class DownloadTask {
     if (isUrlExpired(url)) {
       throw new Error(`URL expired for ${this.fileInfo.type}/${this.fileInfo.id} — waiting for fresh URL from next collection cycle`);
     }
-    return url;
+    return rewriteUrlForProxy(url);
   }
 
   async start() {
@@ -149,7 +166,7 @@ export class DownloadTask {
         if (attempt < MAX_RETRIES) {
           const delay = RETRY_DELAY_MS * attempt;
           const chunkLabel = this.chunkIndex != null ? ` chunk ${this.chunkIndex}` : '';
-          console.warn(`[DownloadTask] ${this.fileInfo.type}/${this.fileInfo.id}${chunkLabel} attempt ${attempt}/${MAX_RETRIES} failed: ${msg}. Retrying in ${delay / 1000}s...`);
+          log.warn(`[DownloadTask] ${this.fileInfo.type}/${this.fileInfo.id}${chunkLabel} attempt ${attempt}/${MAX_RETRIES} failed: ${msg}. Retrying in ${delay / 1000}s...`);
           await new Promise(resolve => setTimeout(resolve, delay));
         } else {
           this.state = 'failed';
@@ -204,7 +221,7 @@ export class FileDownload {
     if (isUrlExpired(url)) {
       throw new Error(`URL expired for ${this.fileInfo.type}/${this.fileInfo.id} — waiting for fresh URL from next collection cycle`);
     }
-    return url;
+    return rewriteUrlForProxy(url);
   }
 
   wait() {
@@ -220,7 +237,7 @@ export class FileDownload {
     try {
       this.state = 'preparing';
       const { id, type, size } = this.fileInfo;
-      console.log('[FileDownload] Starting:', `${type}/${id}`);
+      log.info('[FileDownload] Starting:', `${type}/${id}`);
 
       // Use declared size from RequiredFiles — no HEAD needed for queue building
       this.totalBytes = (size && size > 0) ? parseInt(size) : 0;
@@ -242,7 +259,7 @@ export class FileDownload {
         }
       }
 
-      console.log('[FileDownload] File size:', (this.totalBytes / 1024 / 1024).toFixed(1), 'MB');
+      log.info('[FileDownload] File size:', (this.totalBytes / 1024 / 1024).toFixed(1), 'MB');
 
       const chunkSize = this.options.chunkSize || DEFAULT_CHUNK_SIZE;
 
@@ -267,14 +284,14 @@ export class FileDownload {
         }
 
         if (needed.length === 0) {
-          console.log('[FileDownload] All chunks already cached, nothing to download');
+          log.info('[FileDownload] All chunks already cached, nothing to download');
           this.state = 'complete';
           this._resolve(new Blob([], { type: this._contentType }));
           return;
         }
 
         if (skippedCount > 0) {
-          console.log(`[FileDownload] Resuming: ${skippedCount} chunks cached, ${needed.length} to download`);
+          log.info(`[FileDownload] Resuming: ${skippedCount} chunks cached, ${needed.length} to download`);
         }
 
         const isResume = skippedCount > 0;
@@ -301,7 +318,7 @@ export class FileDownload {
         }
 
         const highCount = this.tasks.filter(t => t._priority >= PRIORITY.high).length;
-        console.log(`[FileDownload] ${type}/${id}: ${this.tasks.length} chunks` +
+        log.info(`[FileDownload] ${type}/${id}: ${this.tasks.length} chunks` +
           (highCount > 0 ? ` (${highCount} priority)` : '') +
           (isResume ? ' (resume)' : ''));
 
@@ -316,7 +333,7 @@ export class FileDownload {
       this.state = 'downloading';
 
     } catch (error) {
-      console.error('[FileDownload] Prepare failed:', `${this.fileInfo.type}/${this.fileInfo.id}`, error);
+      log.error('[FileDownload] Prepare failed:', `${this.fileInfo.type}/${this.fileInfo.id}`, error);
       this.state = 'failed';
       this._reject(error);
     }
@@ -339,7 +356,7 @@ export class FileDownload {
       try {
         await this.onChunkDownloaded(task.chunkIndex, task.blob, this.totalChunks);
       } catch (e) {
-        console.warn('[FileDownload] onChunkDownloaded callback error:', e);
+        log.warn('[FileDownload] onChunkDownloaded callback error:', e);
       }
     }
 
@@ -348,10 +365,10 @@ export class FileDownload {
       const { type, id } = this.fileInfo;
 
       if (task.chunkIndex == null) {
-        console.log('[FileDownload] Complete:', `${type}/${id}`, `(${task.blob.size} bytes)`);
+        log.info('[FileDownload] Complete:', `${type}/${id}`, `(${task.blob.size} bytes)`);
         this._resolve(task.blob);
       } else if (this.onChunkDownloaded) {
-        console.log('[FileDownload] Complete:', `${type}/${id}`, `(progressive, ${this.totalChunks} chunks)`);
+        log.info('[FileDownload] Complete:', `${type}/${id}`, `(progressive, ${this.totalChunks} chunks)`);
         this._resolve(new Blob([], { type: this._contentType }));
       } else {
         const ordered = [];
@@ -360,7 +377,7 @@ export class FileDownload {
           if (blob) ordered.push(blob);
         }
         const assembled = new Blob(ordered, { type: this._contentType });
-        console.log('[FileDownload] Complete:', `${type}/${id}`, `(${assembled.size} bytes, reassembled)`);
+        log.info('[FileDownload] Complete:', `${type}/${id}`, `(${assembled.size} bytes, reassembled)`);
         this._resolve(assembled);
       }
 
@@ -376,7 +393,7 @@ export class FileDownload {
     // provides fresh URLs and the resume logic (skipChunks) fills the gaps.
     if (error.message?.includes('URL expired')) {
       const chunkLabel = task.chunkIndex != null ? ` chunk ${task.chunkIndex}` : '';
-      console.warn(`[FileDownload] URL expired, dropping${chunkLabel}:`, `${this.fileInfo.type}/${this.fileInfo.id}`);
+      log.warn(`[FileDownload] URL expired, dropping${chunkLabel}:`, `${this.fileInfo.type}/${this.fileInfo.id}`);
       this.tasks = this.tasks.filter(t => t !== task);
       // If all remaining tasks completed, resolve as partial
       if (this.tasks.length === 0 || this.completedChunks >= this.tasks.length) {
@@ -387,7 +404,7 @@ export class FileDownload {
       return;
     }
 
-    console.error('[FileDownload] Failed:', `${this.fileInfo.type}/${this.fileInfo.id}`, error);
+    log.error('[FileDownload] Failed:', `${this.fileInfo.type}/${this.fileInfo.id}`, error);
     this.state = 'failed';
     this._reject(error);
   }
@@ -564,7 +581,7 @@ export class DownloadQueue {
         const oldExpiry = getUrlExpiry(existing.fileInfo.path);
         const newExpiry = getUrlExpiry(fileInfo.path);
         if (newExpiry > oldExpiry) {
-          console.log('[DownloadQueue] Refreshing URL for', key);
+          log.info('[DownloadQueue] Refreshing URL for', key);
           existing.fileInfo.path = fileInfo.path;
         }
       }
@@ -578,7 +595,7 @@ export class DownloadQueue {
     });
 
     this.active.set(key, file);
-    console.log('[DownloadQueue] Enqueued:', key);
+    log.info('[DownloadQueue] Enqueued:', key);
 
     // Throttled prepare: HEAD requests are limited to avoid flooding connections
     this._schedulePrepare(file);
@@ -612,7 +629,7 @@ export class DownloadQueue {
     }
     this._sortQueue();
 
-    console.log(`[DownloadQueue] ${tasks.length} tasks added (${this.queue.length} pending, ${this.running} active)`);
+    log.info(`[DownloadQueue] ${tasks.length} tasks added (${this.queue.length} pending, ${this.running} active)`);
     this.processQueue();
   }
 
@@ -638,7 +655,7 @@ export class DownloadQueue {
       }
     }
 
-    console.log(`[DownloadQueue] Ordered queue: ${taskCount} tasks, ${barrierCount} barriers (${this.queue.length} pending, ${this.running} active)`);
+    log.info(`[DownloadQueue] Ordered queue: ${taskCount} tasks, ${barrierCount} barriers (${this.queue.length} pending, ${this.running} active)`);
     this.processQueue();
   }
 
@@ -652,7 +669,7 @@ export class DownloadQueue {
     const file = this.active.get(key);
 
     if (!file) {
-      console.log('[DownloadQueue] Not found:', key);
+      log.info('[DownloadQueue] Not found:', key);
       return false;
     }
 
@@ -665,7 +682,7 @@ export class DownloadQueue {
     }
     this._sortQueue();
 
-    console.log('[DownloadQueue] Prioritized:', key, `(${boosted} tasks boosted)`);
+    log.info('[DownloadQueue] Prioritized:', key, `(${boosted} tasks boosted)`);
     return true;
   }
 
@@ -691,7 +708,7 @@ export class DownloadQueue {
     }
     this._sortQueue();
 
-    console.log('[DownloadQueue] Layout files prioritized:', idSet.size, 'files,', boosted, 'tasks boosted to', priority);
+    log.info('[DownloadQueue] Layout files prioritized:', idSet.size, 'files,', boosted, 'tasks boosted to', priority);
   }
 
   /**
@@ -704,7 +721,7 @@ export class DownloadQueue {
     const file = this.active.get(key);
 
     if (!file) {
-      console.log('[DownloadQueue] urgentChunk: file not active:', key, 'chunk', chunkIndex);
+      log.info('[DownloadQueue] urgentChunk: file not active:', key, 'chunk', chunkIndex);
       return false;
     }
 
@@ -719,13 +736,13 @@ export class DownloadQueue {
       );
       if (activeTask && activeTask._priority < PRIORITY.urgent) {
         activeTask._priority = PRIORITY.urgent;
-        console.log(`[DownloadQueue] URGENT: ${key} chunk ${chunkIndex} (already in-flight, limiting slots)`);
+        log.info(`[DownloadQueue] URGENT: ${key} chunk ${chunkIndex} (already in-flight, limiting slots)`);
         // Don't call processQueue() — can't stop in-flight tasks, but next
         // processQueue() call (when any task completes) will see hasUrgent
         // and limit new starts to URGENT_CONCURRENCY.
         return true;
       }
-      console.log('[DownloadQueue] urgentChunk: already urgent:', key, 'chunk', chunkIndex);
+      log.info('[DownloadQueue] urgentChunk: already urgent:', key, 'chunk', chunkIndex);
       return false;
     }
 
@@ -735,7 +752,7 @@ export class DownloadQueue {
     );
 
     if (idx === -1) {
-      console.log('[DownloadQueue] urgentChunk: chunk not in queue:', key, 'chunk', chunkIndex);
+      log.info('[DownloadQueue] urgentChunk: chunk not in queue:', key, 'chunk', chunkIndex);
       return false;
     }
 
@@ -744,7 +761,7 @@ export class DownloadQueue {
     // Move to front of queue (past any barriers)
     this.queue.unshift(task);
 
-    console.log(`[DownloadQueue] URGENT: ${key} chunk ${chunkIndex} (moved to front)`);
+    log.info(`[DownloadQueue] URGENT: ${key} chunk ${chunkIndex} (moved to front)`);
     this.processQueue();
     return true;
   }
@@ -809,7 +826,7 @@ export class DownloadQueue {
     }
 
     if (this.queue.length === 0 && this.running === 0) {
-      console.log('[DownloadQueue] All downloads complete');
+      log.info('[DownloadQueue] All downloads complete');
     }
   }
 
@@ -827,14 +844,14 @@ export class DownloadQueue {
     this._activeTasks.push(task);
     const key = `${task.fileInfo.type}/${task.fileInfo.id}`;
     const chunkLabel = task.chunkIndex != null ? ` chunk ${task.chunkIndex}` : '';
-    console.log(`[DownloadQueue] Starting: ${key}${chunkLabel} (${this.running}/${this.concurrency} active)`);
+    log.info(`[DownloadQueue] Starting: ${key}${chunkLabel} (${this.running}/${this.concurrency} active)`);
 
     task.start()
       .then(() => {
         this.running--;
         task._parentFile._runningCount--;
         this._activeTasks = this._activeTasks.filter(t => t !== task);
-        console.log(`[DownloadQueue] Fetched: ${key}${chunkLabel} (${this.running} active, ${this.queue.length} pending)`);
+        log.info(`[DownloadQueue] Fetched: ${key}${chunkLabel} (${this.running} active, ${this.queue.length} pending)`);
         this.processQueue();
         return task._parentFile.onTaskComplete(task);
       })

package/src/widget-html.js

@@ -12,6 +12,9 @@
  * Uses Cache API directly — the SW also serves from the same cache.
  */
 
+import { createLogger } from '@xiboplayer/utils';
+
+const log = createLogger('Cache');
 const CACHE_NAME = 'xibo-media-v1';
 
 // Dynamic base path for multi-variant deployment (pwa, pwa-xmds, pwa-xlr)
@@ -32,8 +35,8 @@ export async function cacheWidgetHtml(layoutId, regionId, mediaId, html) {
   const cache = await caches.open(CACHE_NAME);
 
   // Inject <base> tag to fix relative paths for widget dependencies
-  // Widget HTML has relative paths like "bundle.min.js" that should resolve to /player/cache/media/
-  const baseTag = '<base href="/player/cache/media/">';
+  // Widget HTML has relative paths like "bundle.min.js" that should resolve to cache/media/
+  const baseTag = `<base href="${BASE}/cache/media/">`;
   let modifiedHtml = html;
 
   // Insert base tag after <head> opening tag
@@ -54,7 +57,7 @@ export async function cacheWidgetHtml(layoutId, regionId, mediaId, html) {
   modifiedHtml = modifiedHtml.replace(cmsUrlRegex, (match, filename) => {
     const localPath = `${BASE}/cache/static/${filename}`;
     staticResources.push({ filename, originalUrl: match });
-    console.log(`[Cache] Rewrote widget URL: ${filename} → ${localPath}`);
+    log.info(`Rewrote widget URL: ${filename} → ${localPath}`);
     return localPath;
   });
 
@@ -77,7 +80,7 @@ export async function cacheWidgetHtml(layoutId, regionId, mediaId, html) {
     `hostAddress: "${BASE}/ic"`
   );
 
-  console.log(`[Cache] Injected base tag and rewrote CMS URLs in widget HTML`);
+  log.info('Injected base tag and rewrote CMS/data URLs in widget HTML');
 
   // Construct full URL for cache storage
   const cacheUrl = new URL(cacheKey, window.location.origin);
@@ -90,7 +93,7 @@ export async function cacheWidgetHtml(layoutId, regionId, mediaId, html) {
   });
 
   await cache.put(cacheUrl, response);
-  console.log(`[Cache] Stored widget HTML at ${cacheKey} (${modifiedHtml.length} bytes)`);
+  log.info(`Stored widget HTML at ${cacheKey} (${modifiedHtml.length} bytes)`);
 
   // Fetch and cache static resources (shared Cache API - accessible from main thread and SW)
   if (staticResources.length > 0) {
@@ -105,7 +108,7 @@ export async function cacheWidgetHtml(layoutId, regionId, mediaId, html) {
       try {
         const resp = await fetch(originalUrl);
         if (!resp.ok) {
-          console.warn(`[Cache] Failed to fetch static resource: ${filename} (HTTP ${resp.status})`);
+          log.warn(`Failed to fetch static resource: ${filename} (HTTP ${resp.status})`);
           return;
         }
 
@@ -126,14 +129,14 @@ export async function cacheWidgetHtml(layoutId, regionId, mediaId, html) {
           const fontUrlRegex = /url\((['"]?)(https?:\/\/[^'")\s]+\?[^'")\s]*file=([^&'")\s]+\.(?:woff2?|ttf|otf|eot|svg))[^'")\s]*)\1\)/gi;
           cssText = cssText.replace(fontUrlRegex, (_match, quote, fullUrl, fontFilename) => {
             fontResources.push({ filename: fontFilename, originalUrl: fullUrl });
-            console.log(`[Cache] Rewrote font URL in CSS: ${fontFilename}`);
+            log.info(`Rewrote font URL in CSS: ${fontFilename}`);
             return `url(${quote}${BASE}/cache/static/${encodeURIComponent(fontFilename)}${quote})`;
           });
 
           await staticCache.put(staticKey, new Response(cssText, {
             headers: { 'Content-Type': 'text/css' }
           }));
-          console.log(`[Cache] Cached CSS with ${fontResources.length} rewritten font URLs: ${filename}`);
+          log.info(`Cached CSS with ${fontResources.length} rewritten font URLs: ${filename}`);
 
           // Fetch and cache referenced font files
           await Promise.all(fontResources.map(async ({ filename: fontFile, originalUrl: fontUrl }) => {
@@ -144,7 +147,7 @@ export async function cacheWidgetHtml(layoutId, regionId, mediaId, html) {
             try {
               const fontResp = await fetch(fontUrl);
               if (!fontResp.ok) {
-                console.warn(`[Cache] Failed to fetch font: ${fontFile} (HTTP ${fontResp.status})`);
+                log.warn(`Failed to fetch font: ${fontFile} (HTTP ${fontResp.status})`);
                 return;
               }
               const fontBlob = await fontResp.blob();
@@ -159,9 +162,9 @@ export async function cacheWidgetHtml(layoutId, regionId, mediaId, html) {
               await staticCache.put(fontKey, new Response(fontBlob, {
                 headers: { 'Content-Type': fontContentType }
               }));
-              console.log(`[Cache] Cached font: ${fontFile} (${fontContentType}, ${fontBlob.size} bytes)`);
+              log.info(`Cached font: ${fontFile} (${fontContentType}, ${fontBlob.size} bytes)`);
             } catch (fontErr) {
-              console.warn(`[Cache] Failed to cache font: ${fontFile}`, fontErr);
+              log.warn(`Failed to cache font: ${fontFile}`, fontErr);
             }
           }));
         } else {
@@ -169,10 +172,10 @@ export async function cacheWidgetHtml(layoutId, regionId, mediaId, html) {
           await staticCache.put(staticKey, new Response(blob, {
             headers: { 'Content-Type': contentType }
           }));
-          console.log(`[Cache] Cached static resource: ${filename} (${contentType}, ${blob.size} bytes)`);
+          log.info(`Cached static resource: ${filename} (${contentType}, ${blob.size} bytes)`);
         }
       } catch (error) {
-        console.warn(`[Cache] Failed to cache static resource: ${filename}`, error);
+        log.warn(`Failed to cache static resource: ${filename}`, error);
       }
     }));
   }