@miso.ai/server-commons 0.6.5-beta.1 → 0.6.5-beta.11

package/package.json

@@ -21,5 +21,5 @@
     "uuid": "^9.0.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.5-beta.1"
+  "version": "0.6.5-beta.11"
 }

package/src/file.js

@@ -1,5 +1,6 @@
 import { access } from 'fs/promises';
-import { accessSync, constants } from 'fs';
+import { accessSync, constants, createReadStream } from 'fs';
+import { createInterface } from 'readline';
 
 export async function fileExists(file, mode = constants.F_OK) {
   try {
@@ -18,3 +19,23 @@ export function fileExistsSync(file, mode = constants.F_OK) {
     return false;
   }
 }
+
+export async function readFileAsLines(file) {
+  const fileStream = createReadStream(file, { encoding: 'utf8' });
+  const rl = createInterface({
+    input: fileStream,
+    crlfDelay: Infinity
+  });
+  
+  const lines = [];
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (trimmed) {
+      lines.push(trimmed);
+    }
+  }
+
+  rl.close();
+
+  return lines;
+}

package/src/sink/bps.js

@@ -11,11 +11,13 @@ export default class BpsSink {
   }
 
   _normalizeOptions({
+    writesPerSecond = 5,
     recordsPerSecord = 100000,
-    bytesPerSecond = 4 * 1024 * 1024,
+    bytesPerSecond = 100 * 1024 * 1024,
     ...options
   } = {}) {
     return {
+      writesPerSecond,
       recordsPerSecord,
       bytesPerSecond,
       ...options,
@@ -62,9 +64,10 @@ export default class BpsSink {
     const elapsed = now - this._firstWriteAt;
     const targetBps = this._targetBps(now);
     const targetRps = this._targetRps(now);
+    const targetWps = this._targetWps(now);
 
     const { started } = this._stats;
-    const shallElapsed = Math.max(started.records / targetRps, started.bytes / targetBps) * 1000;
+    const shallElapsed = Math.max(started.records / targetRps, started.bytes / targetBps, started.count / targetWps) * 1000;
 
     const blockedTime = shallElapsed - elapsed;
     if (blockedTime <= 1000) {
@@ -86,4 +89,8 @@ export default class BpsSink {
     return this._options.recordsPerSecord;
   }
 
+  _targetWps(timestamp) {
+    return this._options.writesPerSecord;
+  }
+
 }

package/src/store.js

@@ -1,5 +1,7 @@
 import { resolve } from 'path';
 import fs from 'fs/promises';
+import { Transform } from 'stream';
+import { readFileAsLines } from './file.js';
 
 const DEFAULT_FLUSH_THRESHOLD = 100;
 
@@ -15,7 +17,7 @@ export default class HashStore {
     this._file = file;
     this._hashFn = hashFn;
     this._flushThreshold = flushThreshold;
-    this._hashes = undefined;
+    this._hashes = new Set();
     this._pending = [];
   }
 
@@ -38,6 +40,8 @@ export default class HashStore {
 
   async load() {
     this._hashes = new Set(await this._read());
+    this._pending = [];
+    return this;
   }
 
   get() {
@@ -65,9 +69,10 @@ export default class HashStore {
     if (this._pending.length === 0) {
       return;
     }
-    await this._mkdir();
-    await fs.appendFile(this._file, this._pending.join('\n') + '\n');
+    const pending = this._pending;
     this._pending = [];
+    await this._mkdir();
+    await fs.appendFile(this._file, pending.join('\n') + '\n');
   }
 
   exclusionStream() {
@@ -85,8 +90,7 @@ export default class HashStore {
 
   async _read() {
     try {
-      const content = await fs.readFile(this._file, 'utf-8');
-      return content.split('\n').map(v => v.trim()).filter(v => v);
+      return await readFileAsLines(this._file);
     } catch (err) {
       if (err.code !== 'ENOENT') {
         throw err;

package/src/stream/buffered-read.js

@@ -85,6 +85,7 @@ export default class BufferedReadStream extends Readable {
     const request = this._state.request(this._source.request());
 
     this._debug(`[BufferedReadStream] Load request: ${request}`);
+    // TODO: racing here! we need to somehow keep the order of loads
     const { data, ...info } = await this._source.get(request);
     const response = new Response(request, info);
     this._debug(`[BufferedReadStream] Load response: ${JSON.stringify(response)} => data = ${data && data.length}`);
@@ -192,12 +193,13 @@ class Strategy {
 
   constructor({
     highWatermark = 1000,
+    maxPendingLoads = 100,
     eagerLoad = false,
     initialize,
     shallLoad,
     terminate,
   } = {}) {
-    this.options = Object.freeze({ highWatermark, eagerLoad });
+    this.options = Object.freeze({ highWatermark, maxPendingLoads, eagerLoad });
     // overwrite methods
     Object.assign(this, trimObj({ initialize, shallLoad, terminate }));
   }
@@ -210,7 +212,7 @@ class Strategy {
 
   shallLoad(state) {
     // TODO: we can have a slower start
-    return state.watermark < this.options.highWatermark;
+    return state.pendingLoads < this.options.maxPendingLoads && state.watermark < this.options.highWatermark;
   }
 
   terminate(record, state) {

package/src/stream/buffered-write-state.js

@@ -120,6 +120,14 @@ export default class State {
     category.records += request.records;
     category.bytes += request.bytes;
 
+    if (response.errors && response.recovered && response.recovered.records > 0) {
+      this._failed.records -= response.recovered.records;
+      this._failed.bytes -= response.recovered.bytes; // not so accurate, but close enough
+      this._successful.requests++;
+      this._successful.records += response.recovered.records;
+      this._successful.bytes += response.recovered.bytes;
+    }
+
     this._time.addWrite(response.timestamp - request.timestamp);
 
     this._resolutions.get(request).resolve();

package/src/stream/buffered-write.js

@@ -183,6 +183,7 @@ export default class BufferedWriteStream extends Transform {
       result: failed ? 'failed' : 'successful',
       index: request.index,
       records: request.records,
+      recovered: response.recovered || { records: 0, bytes: 0 },
       bytes: request.bytes,
       time: response.timestamp - request.timestamp,
     });

package/src/yargs.js

@@ -73,6 +73,6 @@ export function handleFail(msg, err) {
 
 export function coerceToArray(arg) {
   return Array.isArray(arg) ? arg :
-    typeof arg === 'string' ? arg.split(',') :
+    typeof arg === 'string' ? arg.split(',').map(s => s.trim()) :
     arg === undefined || arg === null ? [] : [arg];
 }