@8ms/helpers 1.5.10 → 1.6.0

package/aws/s3/writeGzip.d.ts

@@ -1,4 +1,4 @@
-type WriteBrotli = {
+type WriteGzip = {
     bucket: string;
     data: any;
     key: string;
@@ -8,5 +8,5 @@ type WriteBrotli = {
 /**
  * Write a gzip file to S3.
  */
-declare const writeBrotli: ({ bucket, data, key, isJson, ...options }: WriteBrotli) => Promise<any>;
-export default writeBrotli;
+declare const writeGzip: ({ bucket, data, key, isJson, ...options }: WriteGzip) => Promise<any>;
+export default writeGzip;

package/aws/s3/writeGzip.js

@@ -8,7 +8,7 @@ const writeFile_1 = __importDefault(require("./writeFile"));
 /**
  * Write a gzip file to S3.
  */
-const writeBrotli = async ({ bucket, data, key, isJson, ...options }) => {
+const writeGzip = async ({ bucket, data, key, isJson, ...options }) => {
     let finalData = data;
     if (false !== isJson) {
         finalData = JSON.stringify(data);
@@ -25,4 +25,4 @@ const writeBrotli = async ({ bucket, data, key, isJson, ...options }) => {
     });
     return apiResponse;
 };
-exports.default = writeBrotli;
+exports.default = writeGzip;

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name":            "@8ms/helpers",
 	"license":         "UNLICENSED",
-	"version":         "1.5.10",
+	"version":         "1.6.0",
 	"repository":      {
 		"type": "git",
 		"url":  "git+https://github.com/8millionstories-organisation/8ms-helpers-ts.git"
@@ -39,6 +39,8 @@
 		"babel-jest":                    "29.7.0",
 		"jest":                          "29.7.0",
 		"node-fetch":                    "3.3.2",
+		"stream-chain":                  "2.2.5",
+		"stream-json":                   "1.8.0",
 		"timezone-mock":                 "1.3.6",
 		"ts-node":                       "10.9.1",
 		"tslib":                         "2.6.2",

package/stream/sort.d.ts

@@ -0,0 +1,13 @@
+declare const Transform: any;
+/**
+ * Batch sort stream without overloading the memory limit.
+ * https://www.npmjs.com/package/fast-stream-sort
+ */
+declare class SortStreamer extends Transform {
+    constructor(options: any, comp: any);
+    _transform(data: any, _: any, callback: any): void;
+    _flush(): void;
+    fullflush(): void;
+}
+declare const sort: (comp: any, options: any) => SortStreamer;
+export default sort;

package/stream/sort.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const Transform = require('stream').Transform;
+const batch = 15000;
+/**
+ * Batch sort stream without overloading the memory limit.
+ * https://www.npmjs.com/package/fast-stream-sort
+ */
+class SortStreamer extends Transform {
+    constructor(options, comp) {
+        super(options);
+        this.a = new Array(batch);
+        this.i = 0;
+        this.k = 0;
+        this.currentSize = batch;
+        this.isDone = false;
+        this.sortedArray = undefined;
+        this.comp = comp;
+    }
+    _transform(data, _, callback) {
+        if (data) {
+            this.a[this.i++] = data;
+            if (this.i == this.currentSize) {
+                this.currentSize = this.currentSize * 2;
+                var tmp = new Array(this.currentSize);
+                for (var j = 0; j < this.i; j++) {
+                    tmp[j] = this.a[j];
+                }
+                this.a = tmp;
+            }
+        }
+        callback();
+    }
+    _flush() {
+        this.sortedArray = this.a.slice(0, this.i)
+            .sort(this.comp);
+        this.fullflush();
+    }
+    fullflush() {
+        while (this.k < this.i) {
+            this.push(this.sortedArray[this.k++]);
+        }
+        this.push(null);
+    }
+}
+const sort = (comp, options) => {
+    options = options || { objectMode: true };
+    return new SortStreamer(options, comp);
+};
+exports.default = sort;