@autonomys/file-server 1.5.18 → 1.5.19

package/dist/caching/fileCache.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fileCache.d.ts","sourceRoot":"","sources":["../../src/caching/fileCache.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAS9E,eAAO,MAAM,eAAe,GAAI,QAAQ,eAAe;eA8B7B,MAAM,YAAY,gBAAgB,KAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;eAyBjE,MAAM,gBAAgB,YAAY;eARlC,MAAM,KAAG,OAAO,CAAC,OAAO,CAAC;kBAoBtB,MAAM;CA0BlC,CAAA"}
+{"version":3,"file":"fileCache.d.ts","sourceRoot":"","sources":["../../src/caching/fileCache.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAU9E,eAAO,MAAM,eAAe,GAAI,QAAQ,eAAe;eA8B7B,MAAM,YAAY,gBAAgB,KAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;eAiCjE,MAAM,gBAAgB,YAAY;eARlC,MAAM,KAAG,OAAO,CAAC,OAAO,CAAC;kBAoBtB,MAAM;CA0BlC,CAAA"}

package/dist/caching/fileCache.js

@@ -22,6 +22,7 @@ import { createCache } from 'cache-manager';
 import fs from 'fs';
 import fsPromises from 'fs/promises';
 import path from 'path';
+import { createErrorResilientStream } from './streamUtils.js';
 import { writeFile } from './utils.js';
 const CHARS_PER_PARTITION = 2;
 export const createFileCache = (config) => {
@@ -54,10 +55,16 @@ export const createFileCache = (config) => {
             return null;
         }
         const path = cidToFilePath(cid);
-        return Object.assign(Object.assign({}, data), { data: fs.createReadStream(path, {
-                start: (_a = options === null || options === void 0 ? void 0 : options.byteRange) === null || _a === void 0 ? void 0 : _a[0],
-                end: (_b = options === null || options === void 0 ? void 0 : options.byteRange) === null || _b === void 0 ? void 0 : _b[1],
-            }) });
+        const sourceStream = fs.createReadStream(path, {
+            start: (_a = options === null || options === void 0 ? void 0 : options.byteRange) === null || _a === void 0 ? void 0 : _a[0],
+            end: (_b = options === null || options === void 0 ? void 0 : options.byteRange) === null || _b === void 0 ? void 0 : _b[1],
+        });
+        // Wrap with error-resilient stream for stalled stream detection
+        const resilientStream = createErrorResilientStream(sourceStream, {
+            stallTimeout: 30000, // 30 seconds
+            healthCheckInterval: 5000, // 5 seconds
+        });
+        return Object.assign(Object.assign({}, data), { data: resilientStream });
     });
     const has = (cid) => __awaiter(void 0, void 0, void 0, function* () {
         const path = cidToFilePath(cid);

package/dist/caching/index.d.ts

@@ -1,4 +1,5 @@
 export * from './defaultConfigs.js';
 export * from './fileCache.js';
+export * from './streamUtils.js';
 export * from './utils.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/caching/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/caching/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAA;AACnC,cAAc,gBAAgB,CAAA;AAC9B,cAAc,YAAY,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/caching/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAA;AACnC,cAAc,gBAAgB,CAAA;AAC9B,cAAc,kBAAkB,CAAA;AAChC,cAAc,YAAY,CAAA"}

package/dist/caching/index.js

@@ -1,3 +1,4 @@
 export * from './defaultConfigs.js';
 export * from './fileCache.js';
+export * from './streamUtils.js';
 export * from './utils.js';

package/dist/caching/streamUtils.d.ts

@@ -0,0 +1,20 @@
+import { Readable } from 'stream';
+export interface StreamHealthMetrics {
+    isStalled: boolean;
+    lastActivity: number;
+    isDestroyed: boolean;
+}
+export interface ErrorResilientStreamOptions {
+    stallTimeout?: number;
+    healthCheckInterval?: number;
+}
+interface ErrorResilientStream extends Readable {
+    healthCheck?: () => StreamHealthMetrics;
+}
+/**
+ * Creates an error-resilient stream wrapper that prevents consumer errors from stalling the source stream
+ * Focuses on detecting and handling stalled streams
+ */
+export declare const createErrorResilientStream: (sourceStream: Readable, options?: ErrorResilientStreamOptions) => ErrorResilientStream;
+export {};
+//# sourceMappingURL=streamUtils.d.ts.map

package/dist/caching/streamUtils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streamUtils.d.ts","sourceRoot":"","sources":["../../src/caching/streamUtils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAE9C,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,OAAO,CAAA;IAClB,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,OAAO,CAAA;CACrB;AAED,MAAM,WAAW,2BAA2B;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAA;CAC7B;AAED,UAAU,oBAAqB,SAAQ,QAAQ;IAC7C,WAAW,CAAC,EAAE,MAAM,mBAAmB,CAAA;CACxC;AAED;;;GAGG;AACH,eAAO,MAAM,0BAA0B,GACrC,cAAc,QAAQ,EACtB,UAAS,2BAAgC,KACxC,oBAyHF,CAAA"}

package/dist/caching/streamUtils.js

@@ -0,0 +1,107 @@
+import { PassThrough } from 'stream';
+/**
+ * Creates an error-resilient stream wrapper that prevents consumer errors from stalling the source stream
+ * Focuses on detecting and handling stalled streams
+ */
+export const createErrorResilientStream = (sourceStream, options = {}) => {
+    const { stallTimeout = 30000, // 30 seconds
+    healthCheckInterval = 5000, // 5 seconds
+     } = options;
+    const passThrough = new PassThrough({
+        highWaterMark: 64 * 1024, // 64KB buffer to handle backpressure
+        objectMode: false,
+    });
+    // Simple metrics for stalled stream detection
+    const metrics = {
+        isStalled: false,
+        lastActivity: Date.now(),
+        isDestroyed: false,
+    };
+    let healthCheckTimer = null;
+    // Health check function - only checks for stalled streams
+    const healthCheck = () => {
+        const now = Date.now();
+        const timeSinceLastActivity = now - metrics.lastActivity;
+        metrics.isStalled = timeSinceLastActivity > stallTimeout && !metrics.isDestroyed;
+        return Object.assign({}, metrics);
+    };
+    // Start periodic health checks for stalled streams
+    const startHealthChecks = () => {
+        if (healthCheckTimer) {
+            clearInterval(healthCheckTimer);
+        }
+        healthCheckTimer = setInterval(() => {
+            const health = healthCheck();
+            if (health.isStalled) {
+                console.warn('Stream detected as stalled:', {
+                    timeSinceLastActivity: Date.now() - health.lastActivity,
+                    stallTimeout,
+                });
+                // Destroy the stalled stream
+                passThrough.destroy(new Error('Stream stalled - no activity detected'));
+            }
+        }, healthCheckInterval);
+    };
+    // Set up backpressure-aware data flow
+    sourceStream.on('data', (chunk) => {
+        try {
+            metrics.lastActivity = Date.now();
+            metrics.isStalled = false;
+            if (!passThrough.destroyed) {
+                const canContinue = passThrough.write(chunk);
+                if (!canContinue) {
+                    // Backpressure: pause source stream until drain event
+                    sourceStream.pause();
+                }
+            }
+        }
+        catch (error) {
+            console.error('Error processing stream data:', error);
+            // Don't destroy the stream on consumer errors, just log
+        }
+    });
+    // Resume source stream when pass-through is ready for more data
+    passThrough.on('drain', () => {
+        if (!sourceStream.destroyed && !passThrough.destroyed) {
+            sourceStream.resume();
+        }
+    });
+    // Handle source stream end
+    sourceStream.on('end', () => {
+        if (!passThrough.destroyed) {
+            passThrough.end();
+        }
+        metrics.lastActivity = Date.now();
+    });
+    // Handle source stream errors
+    sourceStream.on('error', (error) => {
+        console.error('Source stream error:', error);
+        metrics.lastActivity = Date.now();
+        passThrough.destroy(error);
+    });
+    // Handle pass-through stream errors (consumer errors)
+    passThrough.on('error', (error) => {
+        console.error('Consumer stream error:', error);
+        metrics.lastActivity = Date.now();
+        // Don't propagate consumer errors to source stream
+    });
+    // Handle pass-through stream on end
+    passThrough.on('end', () => {
+        metrics.lastActivity = Date.now();
+        if (healthCheckTimer)
+            clearInterval(healthCheckTimer);
+    });
+    // Handle pass-through stream close
+    passThrough.on('close', () => {
+        metrics.isDestroyed = true;
+        metrics.lastActivity = Date.now();
+        if (healthCheckTimer) {
+            clearInterval(healthCheckTimer);
+        }
+    });
+    // Start health monitoring
+    startHealthChecks();
+    const errorResilientStream = passThrough;
+    errorResilientStream.healthCheck = healthCheck;
+    return errorResilientStream;
+};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@autonomys/file-server",
   "packageManager": "yarn@4.7.0",
-  "version": "1.5.18",
+  "version": "1.5.19",
   "license": "MIT",
   "type": "module",
   "repository": {
@@ -47,9 +47,9 @@
     "typescript": "^5.8.3"
   },
   "dependencies": {
-    "@autonomys/asynchronous": "^1.5.18",
-    "@autonomys/auto-dag-data": "^1.5.18",
-    "@autonomys/auto-utils": "^1.5.18",
+    "@autonomys/asynchronous": "^1.5.19",
+    "@autonomys/auto-dag-data": "^1.5.19",
+    "@autonomys/auto-utils": "^1.5.19",
     "@keyvhq/sqlite": "^2.1.7",
     "cache-manager": "^6.4.2",
     "express": "^4.19.2",
@@ -61,5 +61,5 @@
     "uuid": "^11.1.0",
     "zod": "^3.24.2"
   },
-  "gitHead": "b339864e15c46187ddcc75ebeed3d8170fbe7c9e"
+  "gitHead": "ada024eb3cb14bf67b3d9f1cb8545e7b7d9d14ed"
 }