@miso.ai/server-commons 0.6.5-beta.0 → 0.6.5-beta.10

package/package.json

@@ -21,5 +21,5 @@
     "uuid": "^9.0.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.5-beta.0"
+  "version": "0.6.5-beta.10"
 }

package/src/file.js

@@ -1,5 +1,6 @@
 import { access } from 'fs/promises';
-import { accessSync, constants } from 'fs';
+import { accessSync, constants, createReadStream } from 'fs';
+import { createInterface } from 'readline';
 
 export async function fileExists(file, mode = constants.F_OK) {
   try {
@@ -18,3 +19,23 @@ export function fileExistsSync(file, mode = constants.F_OK) {
     return false;
   }
 }
+
+export async function readFileAsLines(file) {
+  const fileStream = createReadStream(file, { encoding: 'utf8' });
+  const rl = createInterface({
+    input: fileStream,
+    crlfDelay: Infinity
+  });
+  
+  const lines = [];
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (trimmed) {
+      lines.push(trimmed);
+    }
+  }
+
+  rl.close();
+
+  return lines;
+}

package/src/index.js

@@ -14,3 +14,4 @@ export * as yargs from './yargs.js';
 export { default as Resolution } from './resolution.js';
 export { default as TaskQueue } from './task-queue.js';
 export { default as RateLimitingQueue } from './rate-limiting-queue.js';
+export { default as HashStore } from './store.js';

package/src/sink/bps.js

@@ -12,7 +12,7 @@ export default class BpsSink {
 
   _normalizeOptions({
     recordsPerSecord = 100000,
-    bytesPerSecond = 4 * 1024 * 1024,
+    bytesPerSecond = 100 * 1024 * 1024,
     ...options
   } = {}) {
     return {

package/src/store.js

@@ -0,0 +1,129 @@
+import { resolve } from 'path';
+import fs from 'fs/promises';
+import { Transform } from 'stream';
+import { readFileAsLines } from './file.js';
+
+const DEFAULT_FLUSH_THRESHOLD = 100;
+
+export default class HashStore {
+
+  constructor({ file, hashFn, flushThreshold = DEFAULT_FLUSH_THRESHOLD } = {}) {
+    if (!file) {
+      throw new Error('File path is required');
+    }
+    if (!hashFn) {
+      throw new Error('Hash function is required');
+    }
+    this._file = file;
+    this._hashFn = hashFn;
+    this._flushThreshold = flushThreshold;
+    this._hashes = new Set();
+    this._pending = [];
+  }
+
+  async purge() {
+    // peek data length
+    const length = (await this._read()).length;
+
+    this._hashes = new Set();
+    // delete file
+    try {
+      await fs.unlink(this._file);
+    } catch (err) {
+      if (err.code !== 'ENOENT') {
+        throw err;
+      }
+    }
+
+    return length;
+  }
+
+  async load() {
+    this._hashes = new Set(await this._read());
+    this._pending = [];
+    return this;
+  }
+
+  get() {
+    return this._hashes;
+  }
+
+  contains(item) {
+    return this._hashes.has(this._hashFn(item));
+  }
+
+  async add(...items) {
+    for (const item of items) {
+      const hash = this._hashFn(item);
+      if (!this._hashes.has(hash)) {
+        this._hashes.add(hash);
+        this._pending.push(hash);
+      }
+    }
+    if (this._pending.length >= this._flushThreshold) {
+      await this.flush();
+    }
+  }
+
+  async flush() {
+    if (this._pending.length === 0) {
+      return;
+    }
+    const pending = this._pending;
+    this._pending = [];
+    await this._mkdir();
+    await fs.appendFile(this._file, pending.join('\n') + '\n');
+  }
+
+  exclusionStream() {
+    return new HashStoreFilterTransform(this, { mode: 'exclude' });
+  }
+
+  dedupeStream() {
+    return new HashStoreFilterTransform(this, { mode: 'dedupe' });
+  }
+
+  async _mkdir() {
+    const dir = resolve(this._file, '..');
+    await fs.mkdir(dir, { recursive: true });
+  }
+
+  async _read() {
+    try {
+      return await readFileAsLines(this._file);
+    } catch (err) {
+      if (err.code !== 'ENOENT') {
+        throw err;
+      }
+      return [];
+    }
+  }
+
+}
+
+class HashStoreFilterTransform extends Transform {
+
+  constructor(store, { mode } = {}) {
+    super({ objectMode: true });
+    this._store = store;
+    switch (mode) {
+      case 'exclude':
+      case 'dedupe':
+        break;
+      default:
+        throw new Error(`Unrecognized mode: ${mode}`);
+    }
+    this._mode = mode;
+  }
+
+  _transform(item, _, next) {
+    if (!this._store.contains(item)) {
+      this.push(item);
+      if (this._mode === 'dedupe') {
+        this._store.add(item);
+      }
+    }
+    next();
+  }
+
+}

package/src/stream/buffered-read.js

@@ -85,6 +85,7 @@ export default class BufferedReadStream extends Readable {
     const request = this._state.request(this._source.request());
 
     this._debug(`[BufferedReadStream] Load request: ${request}`);
+    // TODO: racing here! we need to somehow keep the order of loads
     const { data, ...info } = await this._source.get(request);
     const response = new Response(request, info);
     this._debug(`[BufferedReadStream] Load response: ${JSON.stringify(response)} => data = ${data && data.length}`);
@@ -192,12 +193,13 @@ class Strategy {
 
   constructor({
     highWatermark = 1000,
+    maxPendingLoads = 100,
     eagerLoad = false,
     initialize,
     shallLoad,
     terminate,
   } = {}) {
-    this.options = Object.freeze({ highWatermark, eagerLoad });
+    this.options = Object.freeze({ highWatermark, maxPendingLoads, eagerLoad });
     // overwrite methods
     Object.assign(this, trimObj({ initialize, shallLoad, terminate }));
   }
@@ -210,7 +212,7 @@ class Strategy {
 
   shallLoad(state) {
     // TODO: we can have a slower start
-    return state.watermark < this.options.highWatermark;
+    return state.pendingLoads < this.options.maxPendingLoads && state.watermark < this.options.highWatermark;
   }
 
   terminate(record, state) {

package/src/stream/buffered-write-state.js

@@ -120,6 +120,14 @@ export default class State {
     category.records += request.records;
     category.bytes += request.bytes;
 
+    if (response.errors && response.recovered && response.recovered.records > 0) {
+      this._failed.records -= response.recovered.records;
+      this._failed.bytes -= response.recovered.bytes; // not so accurate, but close enough
+      this._successful.requests++;
+      this._successful.records += response.recovered.records;
+      this._successful.bytes += response.recovered.bytes;
+    }
+
     this._time.addWrite(response.timestamp - request.timestamp);
 
     this._resolutions.get(request).resolve();

package/src/stream/buffered-write.js

@@ -183,6 +183,7 @@ export default class BufferedWriteStream extends Transform {
       result: failed ? 'failed' : 'successful',
       index: request.index,
       records: request.records,
+      recovered: response.recovered || { records: 0, bytes: 0 },
       bytes: request.bytes,
       time: response.timestamp - request.timestamp,
     });

package/src/yargs.js

@@ -73,6 +73,6 @@ export function handleFail(msg, err) {
 
 export function coerceToArray(arg) {
   return Array.isArray(arg) ? arg :
-    typeof arg === 'string' ? arg.split(',') :
+    typeof arg === 'string' ? arg.split(',').map(s => s.trim()) :
     arg === undefined || arg === null ? [] : [arg];
 }