@constructive-io/s3-streamer 2.11.0 → 2.14.0

package/esm/streamer.js

@@ -1,5 +1,6 @@
+import { streamContentType } from '@constructive-io/content-type-stream';
 import getS3 from './s3';
-import { upload as streamUpload } from './utils';
+import { upload as streamUpload, uploadWithContentType as streamUploadWithContentType, } from './utils';
 export class Streamer {
     s3;
     defaultBucket;
@@ -25,6 +26,22 @@ export class Streamer {
             bucket
         });
     }
+    async uploadWithContentType({ readStream, contentType, key, bucket = this.defaultBucket, magic, }) {
+        if (!bucket) {
+            throw new Error('Bucket is required');
+        }
+        return await streamUploadWithContentType({
+            client: this.s3,
+            readStream,
+            contentType,
+            key,
+            bucket,
+            magic,
+        });
+    }
+    async detectContentType({ readStream, filename, peekBytes, }) {
+        return streamContentType({ readStream, filename, peekBytes });
+    }
     destroy() {
         this.s3.destroy();
     }

package/esm/utils.js

@@ -28,8 +28,25 @@ export const uploadFromStream = ({ client, key, contentType, bucket }) => {
     });
     return pass;
 };
+const UPLOAD_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
 export const asyncUpload = ({ client, key, contentType, readStream, magic, bucket }) => {
     return new Promise((resolve, reject) => {
+        let settled = false;
+        const settle = (fn) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            fn();
+        };
+        const timer = setTimeout(() => {
+            settle(() => {
+                readStream.destroy();
+                uploadStream.destroy();
+                contentStream.destroy();
+                reject(new Error(`Upload timed out after ${UPLOAD_TIMEOUT_MS / 1000}s for key=${key}`));
+            });
+        }, UPLOAD_TIMEOUT_MS);
         // upload stream
         let upload;
         const uploadStream = uploadFromStream({
@@ -43,21 +60,34 @@ export const asyncUpload = ({ client, key, contentType, readStream, magic, bucke
         const contentStream = new ContentStream();
         const tryResolve = () => {
             if (contents && upload) {
-                resolve({
-                    upload,
-                    magic,
-                    contentType,
-                    contents
+                settle(() => {
+                    resolve({
+                        upload: upload,
+                        magic,
+                        contentType,
+                        contents
+                    });
                 });
             }
         };
+        readStream.on('error', (error) => {
+            settle(() => {
+                uploadStream.destroy();
+                contentStream.destroy();
+                reject(error);
+            });
+        });
         contentStream
             .on('contents', function (results) {
             contents = results;
             tryResolve();
         })
             .on('error', (error) => {
-            reject(error);
+            settle(() => {
+                readStream.destroy();
+                uploadStream.destroy();
+                reject(error);
+            });
         });
         uploadStream
             .on('upload', (results) => {
@@ -65,7 +95,11 @@ export const asyncUpload = ({ client, key, contentType, readStream, magic, bucke
             tryResolve();
         })
             .on('error', (error) => {
-            reject(error);
+            settle(() => {
+                readStream.destroy();
+                contentStream.destroy();
+                reject(error);
+            });
         });
         // Ensure proper cleanup on stream end
         uploadStream.on('finish', () => {
@@ -80,12 +114,22 @@ export const upload = async ({ client, readStream, filename, key, bucket }) => {
         readStream,
         filename
     });
-    return await asyncUpload({
+    return await uploadWithContentType({
         client,
-        key,
-        contentType,
         readStream: newStream,
+        contentType,
         magic,
+        key,
         bucket
     });
 };
+export const uploadWithContentType = async ({ client, readStream, contentType, key, bucket, magic }) => {
+    return await asyncUpload({
+        client,
+        readStream,
+        contentType,
+        magic: magic ?? { charset: 'binary' },
+        key,
+        bucket,
+    });
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@constructive-io/s3-streamer",
-  "version": "2.11.0",
+  "version": "2.14.0",
   "author": "Constructive <developers@constructive.io>",
   "description": "stream files to s3",
   "main": "index.js",
@@ -25,20 +25,20 @@
     "build": "makage build",
     "build:dev": "makage build --dev",
     "lint": "eslint . --fix",
-    "test": "jest --passWithNoTests",
+    "test": "jest",
     "test:watch": "jest --watch"
   },
   "devDependencies": {
-    "@constructive-io/s3-utils": "^2.6.0",
-    "@pgpmjs/env": "^2.11.0",
+    "@constructive-io/s3-utils": "^2.7.0",
+    "@pgpmjs/env": "^2.13.0",
     "glob": "^13.0.0",
     "makage": "^0.1.12"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.971.0",
     "@aws-sdk/lib-storage": "^3.958.0",
-    "@constructive-io/content-type-stream": "^2.6.0",
-    "@pgpmjs/types": "^2.16.0"
+    "@constructive-io/content-type-stream": "^2.8.0",
+    "@pgpmjs/types": "^2.17.0"
   },
   "keywords": [
     "s3",
@@ -48,5 +48,5 @@
     "minio",
     "constructive"
   ],
-  "gitHead": "048188f6b43ffaa6146e7694b2b0d35d34cb2945"
+  "gitHead": "b758178b808ce0bf451e86c0bd7e92079155db7c"
 }

package/streamer.d.ts

@@ -1,5 +1,6 @@
-import { ReadStream } from 'fs';
+import { streamContentType } from '@constructive-io/content-type-stream';
 import type { BucketProvider } from '@pgpmjs/types';
+import type { Readable } from 'stream';
 import { type AsyncUploadResult } from './utils';
 interface StreamerOptions {
     awsRegion: string;
@@ -10,16 +11,32 @@ interface StreamerOptions {
     defaultBucket: string;
 }
 interface UploadParams {
-    readStream: ReadStream;
+    readStream: Readable;
     filename: string;
     key: string;
     bucket?: string;
 }
+interface UploadWithContentTypeParams {
+    readStream: Readable;
+    contentType: string;
+    key: string;
+    bucket?: string;
+    magic?: {
+        charset: string;
+        type?: string;
+    };
+}
 export declare class Streamer {
     private s3;
     private defaultBucket?;
     constructor({ awsRegion, awsSecretKey, awsAccessKey, minioEndpoint, provider, defaultBucket }: StreamerOptions);
     upload({ readStream, filename, key, bucket }: UploadParams): Promise<AsyncUploadResult>;
+    uploadWithContentType({ readStream, contentType, key, bucket, magic, }: UploadWithContentTypeParams): Promise<AsyncUploadResult>;
+    detectContentType({ readStream, filename, peekBytes, }: {
+        readStream: Readable;
+        filename: string;
+        peekBytes?: number;
+    }): ReturnType<typeof streamContentType>;
     destroy(): void;
 }
 export default Streamer;

package/streamer.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Streamer = void 0;
+const content_type_stream_1 = require("@constructive-io/content-type-stream");
 const s3_1 = __importDefault(require("./s3"));
 const utils_1 = require("./utils");
 class Streamer {
@@ -31,6 +32,22 @@ class Streamer {
             bucket
         });
     }
+    async uploadWithContentType({ readStream, contentType, key, bucket = this.defaultBucket, magic, }) {
+        if (!bucket) {
+            throw new Error('Bucket is required');
+        }
+        return await (0, utils_1.uploadWithContentType)({
+            client: this.s3,
+            readStream,
+            contentType,
+            key,
+            bucket,
+            magic,
+        });
+    }
+    async detectContentType({ readStream, filename, peekBytes, }) {
+        return (0, content_type_stream_1.streamContentType)({ readStream, filename, peekBytes });
+    }
     destroy() {
         this.s3.destroy();
     }

package/utils.d.ts

@@ -10,6 +10,7 @@ export interface AsyncUploadParams extends UploadParams {
     readStream: Readable;
     magic: {
         charset: string;
+        type?: string;
     };
 }
 export interface UploadWithFilenameParams {
@@ -19,6 +20,17 @@ export interface UploadWithFilenameParams {
     key: string;
     bucket: string;
 }
+export interface UploadWithContentTypeParams {
+    client: S3Client;
+    readStream: Readable;
+    contentType: string;
+    key: string;
+    bucket: string;
+    magic?: {
+        charset: string;
+        type?: string;
+    };
+}
 export interface UploadResult {
     Location: string;
     ETag?: string;
@@ -29,6 +41,7 @@ export interface AsyncUploadResult {
     upload: UploadResult;
     magic: {
         charset: string;
+        type?: string;
     };
     contentType: string;
     contents: unknown;
@@ -36,3 +49,4 @@ export interface AsyncUploadResult {
 export declare const uploadFromStream: ({ client, key, contentType, bucket }: UploadParams) => PassThrough;
 export declare const asyncUpload: ({ client, key, contentType, readStream, magic, bucket }: AsyncUploadParams) => Promise<AsyncUploadResult>;
 export declare const upload: ({ client, readStream, filename, key, bucket }: UploadWithFilenameParams) => Promise<AsyncUploadResult>;
+export declare const uploadWithContentType: ({ client, readStream, contentType, key, bucket, magic }: UploadWithContentTypeParams) => Promise<AsyncUploadResult>;

package/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.upload = exports.asyncUpload = exports.uploadFromStream = void 0;
+exports.uploadWithContentType = exports.upload = exports.asyncUpload = exports.uploadFromStream = void 0;
 const lib_storage_1 = require("@aws-sdk/lib-storage");
 const content_type_stream_1 = require("@constructive-io/content-type-stream");
 const stream_1 = __importDefault(require("stream"));
@@ -35,8 +35,25 @@ const uploadFromStream = ({ client, key, contentType, bucket }) => {
     return pass;
 };
 exports.uploadFromStream = uploadFromStream;
+const UPLOAD_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
 const asyncUpload = ({ client, key, contentType, readStream, magic, bucket }) => {
     return new Promise((resolve, reject) => {
+        let settled = false;
+        const settle = (fn) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            fn();
+        };
+        const timer = setTimeout(() => {
+            settle(() => {
+                readStream.destroy();
+                uploadStream.destroy();
+                contentStream.destroy();
+                reject(new Error(`Upload timed out after ${UPLOAD_TIMEOUT_MS / 1000}s for key=${key}`));
+            });
+        }, UPLOAD_TIMEOUT_MS);
         // upload stream
         let upload;
         const uploadStream = (0, exports.uploadFromStream)({
@@ -50,21 +67,34 @@ const asyncUpload = ({ client, key, contentType, readStream, magic, bucket }) =>
         const contentStream = new content_type_stream_1.ContentStream();
         const tryResolve = () => {
             if (contents && upload) {
-                resolve({
-                    upload,
-                    magic,
-                    contentType,
-                    contents
+                settle(() => {
+                    resolve({
+                        upload: upload,
+                        magic,
+                        contentType,
+                        contents
+                    });
                 });
             }
         };
+        readStream.on('error', (error) => {
+            settle(() => {
+                uploadStream.destroy();
+                contentStream.destroy();
+                reject(error);
+            });
+        });
         contentStream
             .on('contents', function (results) {
             contents = results;
             tryResolve();
         })
             .on('error', (error) => {
-            reject(error);
+            settle(() => {
+                readStream.destroy();
+                uploadStream.destroy();
+                reject(error);
+            });
         });
         uploadStream
             .on('upload', (results) => {
@@ -72,7 +102,11 @@ const asyncUpload = ({ client, key, contentType, readStream, magic, bucket }) =>
             tryResolve();
         })
             .on('error', (error) => {
-            reject(error);
+            settle(() => {
+                readStream.destroy();
+                contentStream.destroy();
+                reject(error);
+            });
         });
         // Ensure proper cleanup on stream end
         uploadStream.on('finish', () => {
@@ -88,13 +122,24 @@ const upload = async ({ client, readStream, filename, key, bucket }) => {
         readStream,
         filename
     });
-    return await (0, exports.asyncUpload)({
+    return await (0, exports.uploadWithContentType)({
         client,
-        key,
-        contentType,
         readStream: newStream,
+        contentType,
         magic,
+        key,
         bucket
     });
 };
 exports.upload = upload;
+const uploadWithContentType = async ({ client, readStream, contentType, key, bucket, magic }) => {
+    return await (0, exports.asyncUpload)({
+        client,
+        readStream,
+        contentType,
+        magic: magic ?? { charset: 'binary' },
+        key,
+        bucket,
+    });
+};
+exports.uploadWithContentType = uploadWithContentType;