@miso.ai/server-commons 0.6.5-beta.6 → 0.6.5-beta.8

package/package.json

@@ -21,5 +21,5 @@
     "uuid": "^9.0.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.5-beta.6"
+  "version": "0.6.5-beta.8"
 }

package/src/file.js

@@ -1,5 +1,6 @@
 import { access } from 'fs/promises';
-import { accessSync, constants } from 'fs';
+import { accessSync, constants, createReadStream } from 'fs';
+import { createInterface } from 'readline';
 
 export async function fileExists(file, mode = constants.F_OK) {
   try {
@@ -18,3 +19,23 @@ export function fileExistsSync(file, mode = constants.F_OK) {
     return false;
   }
 }
+
+export async function readFileAsLines(file) {
+  const fileStream = createReadStream(file, { encoding: 'utf8' });
+  const rl = createInterface({
+    input: fileStream,
+    crlfDelay: Infinity
+  });
+  
+  const lines = [];
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (trimmed) {
+      lines.push(trimmed);
+    }
+  }
+
+  rl.close();
+
+  return lines;
+}

package/src/sink/bps.js

@@ -12,7 +12,7 @@ export default class BpsSink {
 
   _normalizeOptions({
     recordsPerSecord = 100000,
-    bytesPerSecond = 10 * 1024 * 1024,
+    bytesPerSecond = 100 * 1024 * 1024,
     ...options
   } = {}) {
     return {

package/src/store.js

@@ -1,6 +1,7 @@
 import { resolve } from 'path';
 import fs from 'fs/promises';
 import { Transform } from 'stream';
+import { readFileAsLines } from './file.js';
 
 const DEFAULT_FLUSH_THRESHOLD = 100;
 
@@ -88,8 +89,7 @@ export default class HashStore {
 
   async _read() {
     try {
-      const content = await fs.readFile(this._file, 'utf-8');
-      return content.split('\n').map(v => v.trim()).filter(v => v);
+      return await readFileAsLines(this._file);
     } catch (err) {
       if (err.code !== 'ENOENT') {
         throw err;

package/src/stream/buffered-write-state.js

@@ -120,6 +120,14 @@ export default class State {
     category.records += request.records;
     category.bytes += request.bytes;
 
+    if (response.errors && response.recovered && response.recovered.records > 0) {
+      this._failed.records -= response.recovered.records;
+      this._failed.bytes -= response.recovered.bytes; // not so accurate, but close enough
+      this._successful.requests++;
+      this._successful.records += response.recovered.records;
+      this._successful.bytes += response.recovered.bytes;
+    }
+
     this._time.addWrite(response.timestamp - request.timestamp);
 
     this._resolutions.get(request).resolve();