@miso.ai/server-commons 0.6.5-beta.9 → 0.6.6-beta.1

package/package.json

@@ -6,7 +6,9 @@
   "publishConfig": {
     "access": "public"
   },
-  "scripts": {},
+  "scripts": {
+    "test": "npx uvu test -i 'ignored'"
+  },
   "repository": "MisoAI/miso-server-js-sdk",
   "license": "MIT",
   "contributors": [
@@ -21,5 +23,5 @@
     "uuid": "^9.0.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.5-beta.9"
+  "version": "0.6.6-beta.1"
 }

package/src/date.js

@@ -7,8 +7,6 @@ TS_PER_UNIT.m = TS_PER_UNIT.s * 60;
 TS_PER_UNIT.h = TS_PER_UNIT.m * 60;
 TS_PER_UNIT.d = TS_PER_UNIT.h * 24;
 TS_PER_UNIT.w = TS_PER_UNIT.d * 7;
-//TS_PER_UNIT.M = TS_PER_UNIT.d * 31;
-//TS_PER_UNIT.y = TS_PER_UNIT.d * 366;
 
 export function parseDuration(expr, unit) {
   if (expr === undefined || typeof expr === 'number') {
@@ -33,69 +31,202 @@ export function getYear(dateStr) {
   return new Date(dateStr).getFullYear();
 }
 
+const RE_DATE_EXPR = /^(?:\d{4})|(?:\d{4}-[Qq\d]\d)|(?:\d{4}-\d{2}-[Ww\d]\d)$/;
+
+// TODO: support hour, minute
+
 export function startOfDate(expr) {
+  return floorDate(expr);
+}
+
+export function endOfDate(expr) {
   if (expr === undefined) {
     return undefined;
   }
-  if (typeof expr === 'number') {
-    if (expr < 10000) {
-      expr = startOfYearByYearNum(expr);
-    }
-    return expr;
+  return nextDate(expr) - 1000; // 1 sec
+}
+
+export function floorDate(expr, unit) {
+  validateUnit(unit);
+  if (expr === undefined) {
+    return undefined;
   }
-  const ts = Date.parse(expr);
-  if (isNaN(ts)) {
-    throw new Error(`Unrecognized date: ${expr}`);
+  const [_unit, ts] = parseDateExpr(expr);
+  //console.error('floorDate', expr, unit, _unit, ts, unit === undefined || !isGranular(unit, _unit));
+  if (unit === undefined || isGranular(unit, _unit)) {
+    return ts;
   }
-  return ts;
+  return floorTimestamp(ts, unit);
 }
 
-const RE_YEAR = /^\d{4}$/g;
-const RE_MONTH = /^\d{4}-\d{2}$/g;
-const RE_DATE = /^\d{4}-\d{2}-\d{2}$/g;
+function floorTimestamp(ts, unit) {
+  switch (unit) {
+    case 'millisecond':
+      return ts;
+    case 'second':
+      return ts - ts % TS_PER_UNIT.s;
+    case 'minute':
+      return ts - ts % TS_PER_UNIT.m;
+    case 'hour':
+      return ts - ts % TS_PER_UNIT.h;
+    case 'day':
+      return ts - ts % TS_PER_UNIT.d;
+  }
+  const d = new Date(ts);
+  const year = d.getUTCFullYear();
+  const month = d.getUTCMonth() + 1;
+  const day = d.getUTCDate();
+  switch (unit) {
+    case 'week':
+      return Date.UTC(year, month - 1, day > 21 ? 22 : day - (day - 1) % 7);
+    case 'month':
+      return Date.UTC(year, month - 1);
+    case 'quarter':
+      return Date.UTC(year, month - 1 - (month - 1) % 3);
+    case 'year':
+      return Date.UTC(year);
+    default:
+      throw new Error(`Unrecognized unit: ${unit}`);
+  }
+}
 
-// TODO: support end of hour, minute
+export function ceilDate(expr, unit) {
+  validateUnit(unit);
+  if (expr === undefined) {
+    return undefined;
+  }
+  const [_unit, ts] = parseDateExpr(expr);
+  unit = unit || _unit;
+  const floored = floorTimestamp(ts, unit);
+  return floored === ts ? floored : nextTimestamp(floored, unit);
+}
 
-export function endOfDate(expr) {
+export function nextDate(expr, unit) {
+  validateUnit(unit);
   if (expr === undefined) {
     return undefined;
   }
-  if (typeof expr === 'number') {
-    if (expr < 10000) {
-      expr = endOfYearByYearNum(expr);
-    }
-    return expr;
+  const [_unit, ts] = parseDateExpr(expr);
+  return nextTimestamp(ts + 1, unit || _unit);
+}
+
+function nextTimestamp(ts, unit) {
+  switch (unit) {
+    case 'millisecond':
+      return ts;
+    case 'second':
+      return ts + TS_PER_UNIT.s - ts % TS_PER_UNIT.s;
+    case 'minute':
+      return ts + TS_PER_UNIT.m - ts % TS_PER_UNIT.m;
+    case 'hour':
+      return ts + TS_PER_UNIT.h - ts % TS_PER_UNIT.h;
+    case 'day':
+      return ts + TS_PER_UNIT.d - ts % TS_PER_UNIT.d;
   }
-  let ts = Date.parse(expr);
-  if (isNaN(ts)) {
-    throw new Error(`Unrecognized date: ${expr}`);
+  const d = new Date(ts);
+  const year = d.getUTCFullYear();
+  const month = d.getUTCMonth() + 1;
+  const day = d.getUTCDate();
+  switch (unit) {
+    case 'week':
+      return day > 21 ? Date.UTC(year, month, 1) : Date.UTC(year, month - 1, day - (day - 1) % 7 + 7);
+    case 'month':
+      return Date.UTC(year, month, 1);
+    case 'quarter':
+      return Date.UTC(year, month - month % 3 + 3);
+    case 'year':
+      return Date.UTC(year + 1);
+    default:
+      throw new Error(`Unrecognized unit: ${unit}`);
   }
-  return expr.match(RE_YEAR) ? endOfYear(ts) :
-    expr.match(RE_MONTH) ? endOfMonth(ts) :
-    expr.match(RE_DATE) ? endOfDay(ts) : ts;
 }
 
-function endOfYear(ts) {
-  return endOfYearByYearNum(new Date(ts).getFullYear());
+export function prevDate(expr, unit) {
+  validateUnit(unit);
+  if (expr === undefined) {
+    return undefined;
+  }
+  const [_unit, ts] = parseDateExpr(expr);
+  return floorTimestamp(ts - 1, unit || _unit);
 }
 
-function startOfYearByYearNum(fullYear) {
-  return new Date(fullYear, 0, 1).getTime();
-}
 
-function endOfYearByYearNum(fullYear) {
-  return new Date(fullYear + 1, 0, 1).getTime() - 1000;
-}
 
-function endOfMonth(ts) {
-  const date = new Date(ts);
-  const year = date.getFullYear();
-  const month = date.getMonth();
-  return (month == 11 ? new Date(year + 1, 0, 1) : new Date(year, month + 1, 1)).getTime() - 1000;
+const UNIT_GRANULARITY = {
+  'millisecond': 10,
+  'second': 9,
+  'minute': 8,
+  'hour': 7,
+  'day': 6,
+  'week': 5,
+  'month': 4,
+  'quarter': 3,
+  'year': 2,
+};
+
+function validateUnit(unit) {
+  if (unit === undefined) {
+    return;
+  }
+  if (typeof unit !== 'string' || !UNIT_GRANULARITY[unit]) {
+    throw new Error(`Unrecognized unit: ${unit}`);
+  }
 }
 
-const MS_PER_DAY = 1000 * 60 * 60 * 24;
+function isGranular(unit0, unit1) {
+  return UNIT_GRANULARITY[unit0] > UNIT_GRANULARITY[unit1];
+}
 
-function endOfDay(ts) {
-  return (Math.floor(ts / MS_PER_DAY) + 1) * MS_PER_DAY - 1000;
+function parseDateExpr(expr) {
+  switch (typeof expr) {
+    case 'number':
+      if (expr < 100) {
+        throw new Error(`Unrecognized date: ${expr}`);
+      }
+      return expr < 10000 ? ['year', Date.UTC(expr)] : ['millisecond', expr];
+    case 'string':
+      expr = expr.trim();
+      const len = expr.length;
+      if (!RE_DATE_EXPR.test(expr)) {
+        throw new Error(`Unrecognized date: ${expr}`);
+      }
+      const year = parseInt(expr.slice(0, 4), 10);
+      if (isNaN(year)) {
+        throw new Error(`Unrecognized date: ${expr}`);
+      }
+      if (len === 4) {
+        return ['year', Date.UTC(year)];
+      }
+      if (len === 7 && (expr.charAt(5) === 'Q' || expr.charAt(5) === 'q')) {
+        const quarter = parseInt(expr.charAt(6), 10);
+        if (isNaN(quarter) || quarter < 1 || quarter > 4) {
+          throw new Error(`Unrecognized date: ${expr}`);
+        }
+        return ['quarter', Date.UTC(year, (quarter - 1) * 3)];
+      }
+      const month = parseInt(expr.slice(5, 7), 10);
+      if (isNaN(month) || month < 1 || month > 12) {
+        throw new Error(`Unrecognized date: ${expr}`);
+      }
+      if (len === 7) {
+        return ['month', Date.UTC(year, month - 1)];
+      }
+      if (expr.charAt(8) === 'W' || expr.charAt(8) === 'w') {
+        const week = parseInt(expr.charAt(9), 10);
+        if (isNaN(week) || week < 1 || week > 4) {
+          throw new Error(`Unrecognized date: ${expr}`);
+        }
+        return ['week', Date.UTC(year, month - 1, week * 7 - 6)];
+      }
+      let date;
+      try {
+        // parse so we can validate the day part
+        date = new Date(Date.parse(expr));
+      } catch (_) {
+        throw new Error(`Unrecognized date: ${expr}`);
+      }
+      return ['day', Date.UTC(year, month - 1, date.getUTCDate())];
+    default:
+      throw new Error(`Unrecognized date: ${expr}`);
+  }
 }

package/src/sink/bps.js

@@ -11,11 +11,13 @@ export default class BpsSink {
   }
 
   _normalizeOptions({
+    writesPerSecond = 10,
     recordsPerSecord = 100000,
     bytesPerSecond = 100 * 1024 * 1024,
     ...options
   } = {}) {
     return {
+      writesPerSecond,
       recordsPerSecord,
       bytesPerSecond,
       ...options,
@@ -62,9 +64,10 @@ export default class BpsSink {
     const elapsed = now - this._firstWriteAt;
     const targetBps = this._targetBps(now);
     const targetRps = this._targetRps(now);
+    const targetWps = this._targetWps(now);
 
     const { started } = this._stats;
-    const shallElapsed = Math.max(started.records / targetRps, started.bytes / targetBps) * 1000;
+    const shallElapsed = Math.max(started.records / targetRps, started.bytes / targetBps, started.count / targetWps) * 1000;
 
     const blockedTime = shallElapsed - elapsed;
     if (blockedTime <= 1000) {
@@ -86,4 +89,8 @@ export default class BpsSink {
     return this._options.recordsPerSecord;
   }
 
+  _targetWps(timestamp) {
+    return this._options.writesPerSecond;
+  }
+
 }

package/src/stream/buffered-read.js

@@ -193,12 +193,13 @@ class Strategy {
 
   constructor({
     highWatermark = 1000,
+    maxPendingLoads = 100,
     eagerLoad = false,
     initialize,
     shallLoad,
     terminate,
   } = {}) {
-    this.options = Object.freeze({ highWatermark, eagerLoad });
+    this.options = Object.freeze({ highWatermark, maxPendingLoads, eagerLoad });
     // overwrite methods
     Object.assign(this, trimObj({ initialize, shallLoad, terminate }));
   }
@@ -211,7 +212,7 @@ class Strategy {
 
   shallLoad(state) {
     // TODO: we can have a slower start
-    return state.watermark < this.options.highWatermark;
+    return state.pendingLoads < this.options.maxPendingLoads && state.watermark < this.options.highWatermark;
   }
 
   terminate(record, state) {

package/src/stream/buffered-write.js

@@ -183,6 +183,7 @@ export default class BufferedWriteStream extends Transform {
       result: failed ? 'failed' : 'successful',
       index: request.index,
       records: request.records,
+      recovered: response.recovered || { records: 0, bytes: 0 },
       bytes: request.bytes,
       time: response.timestamp - request.timestamp,
     });

package/src/yargs.js

@@ -73,6 +73,6 @@ export function handleFail(msg, err) {
 
 export function coerceToArray(arg) {
   return Array.isArray(arg) ? arg :
-    typeof arg === 'string' ? arg.split(',') :
+    typeof arg === 'string' ? arg.split(',').map(s => s.trim()) :
     arg === undefined || arg === null ? [] : [arg];
 }

package/test/date.test.js

@@ -0,0 +1,111 @@
+import { test } from 'uvu';
+import * as assert from 'uvu/assert';
+import { floorDate, ceilDate, nextDate, prevDate, startOfDate, endOfDate } from '../src/index.js';
+
+function testDateFn(fn, input, output) {
+  assert.equal(new Date(fn(input)).toISOString(), output, `${fn.name}(${input})`);
+}
+
+function testDateFnWithUnit(fn, input, unit, output) {
+  assert.equal(new Date(fn(input, unit)).toISOString(), output, `${fn.name}(${input}, ${unit})`);
+}
+
+test('startOfDate', () => {
+  testDateFn(startOfDate, '2025', '2025-01-01T00:00:00.000Z');
+  testDateFn(startOfDate, '2025-Q1', '2025-01-01T00:00:00.000Z');
+  testDateFn(startOfDate, '2025-q2', '2025-04-01T00:00:00.000Z');
+  testDateFn(startOfDate, '2025-Q3', '2025-07-01T00:00:00.000Z');
+  testDateFn(startOfDate, '2025-q4', '2025-10-01T00:00:00.000Z');
+  testDateFn(startOfDate, '2025-02', '2025-02-01T00:00:00.000Z');
+  testDateFn(startOfDate, '2025-02-W1', '2025-02-01T00:00:00.000Z');
+  testDateFn(startOfDate, '2025-02-w2', '2025-02-08T00:00:00.000Z');
+  testDateFn(startOfDate, '2025-02-03', '2025-02-03T00:00:00.000Z');
+});
+
+test('endOfDate', () => {
+  testDateFn(endOfDate, '2025', '2025-12-31T23:59:59.000Z');
+  testDateFn(endOfDate, '2025-Q1', '2025-03-31T23:59:59.000Z');
+  testDateFn(endOfDate, '2025-q2', '2025-06-30T23:59:59.000Z');
+  testDateFn(endOfDate, '2025-Q3', '2025-09-30T23:59:59.000Z');
+  testDateFn(endOfDate, '2025-q4', '2025-12-31T23:59:59.000Z');
+  testDateFn(endOfDate, '2025-02', '2025-02-28T23:59:59.000Z');
+  testDateFn(endOfDate, '2025-03-W1', '2025-03-07T23:59:59.000Z');
+  testDateFn(endOfDate, '2025-03-w2', '2025-03-14T23:59:59.000Z');
+  testDateFn(endOfDate, '2025-03-W3', '2025-03-21T23:59:59.000Z');
+  testDateFn(endOfDate, '2025-03-w4', '2025-03-31T23:59:59.000Z');
+  testDateFn(endOfDate, '2025-03-03', '2025-03-03T23:59:59.000Z');
+});
+
+test('floorDate', () => {
+  testDateFn(floorDate, '2025', '2025-01-01T00:00:00.000Z');
+  testDateFn(floorDate, '2025-Q1', '2025-01-01T00:00:00.000Z');
+  testDateFn(floorDate, '2025-q2', '2025-04-01T00:00:00.000Z');
+  testDateFn(floorDate, '2025-Q3', '2025-07-01T00:00:00.000Z');
+  testDateFn(floorDate, '2025-q4', '2025-10-01T00:00:00.000Z');
+  testDateFn(floorDate, '2025-02', '2025-02-01T00:00:00.000Z');
+  testDateFn(floorDate, '2025-02-W1', '2025-02-01T00:00:00.000Z');
+  testDateFn(floorDate, '2025-02-w2', '2025-02-08T00:00:00.000Z');
+  testDateFn(floorDate, '2025-02-03', '2025-02-03T00:00:00.000Z');
+
+  const ts = Date.parse('2025-05-10T04:05:06Z');
+  testDateFnWithUnit(floorDate, ts, 'day', '2025-05-10T00:00:00.000Z');
+  testDateFnWithUnit(floorDate, ts, 'week', '2025-05-08T00:00:00.000Z');
+  testDateFnWithUnit(floorDate, ts, 'month', '2025-05-01T00:00:00.000Z');
+  testDateFnWithUnit(floorDate, ts, 'quarter', '2025-04-01T00:00:00.000Z');
+  testDateFnWithUnit(floorDate, ts, 'year', '2025-01-01T00:00:00.000Z');
+});
+
+test('ceilDate', () => {
+  testDateFn(ceilDate, '2025', '2025-01-01T00:00:00.000Z');
+  testDateFn(ceilDate, '2025-Q1', '2025-01-01T00:00:00.000Z');
+  testDateFn(ceilDate, '2025-q2', '2025-04-01T00:00:00.000Z');
+  testDateFn(ceilDate, '2025-Q3', '2025-07-01T00:00:00.000Z');
+  testDateFn(ceilDate, '2025-q4', '2025-10-01T00:00:00.000Z');
+  testDateFn(ceilDate, '2025-02', '2025-02-01T00:00:00.000Z');
+  testDateFn(ceilDate, '2025-02-W1', '2025-02-01T00:00:00.000Z');
+  testDateFn(ceilDate, '2025-02-w2', '2025-02-08T00:00:00.000Z');
+  testDateFn(ceilDate, '2025-02-03', '2025-02-03T00:00:00.000Z');
+});
+
+test('nextDate', () => {
+  testDateFn(nextDate, '2025', '2026-01-01T00:00:00.000Z');
+  testDateFn(nextDate, '2025-Q1', '2025-04-01T00:00:00.000Z');
+  testDateFn(nextDate, '2025-q2', '2025-07-01T00:00:00.000Z');
+  testDateFn(nextDate, '2025-Q3', '2025-10-01T00:00:00.000Z');
+  testDateFn(nextDate, '2025-q4', '2026-01-01T00:00:00.000Z');
+  testDateFn(nextDate, '2025-02', '2025-03-01T00:00:00.000Z');
+  testDateFn(nextDate, '2025-02-W1', '2025-02-08T00:00:00.000Z');
+  testDateFn(nextDate, '2025-02-w2', '2025-02-15T00:00:00.000Z');
+  testDateFn(nextDate, '2025-02-W3', '2025-02-22T00:00:00.000Z');
+  testDateFn(nextDate, '2025-02-w4', '2025-03-01T00:00:00.000Z');
+  testDateFn(nextDate, '2025-02-03', '2025-02-04T00:00:00.000Z');
+});
+
+test('prevDate', () => {
+  testDateFn(prevDate, '2025', '2024-01-01T00:00:00.000Z');
+  testDateFn(prevDate, '2025-Q1', '2024-10-01T00:00:00.000Z');
+  testDateFn(prevDate, '2025-q2', '2025-01-01T00:00:00.000Z');
+  testDateFn(prevDate, '2025-Q3', '2025-04-01T00:00:00.000Z');
+  testDateFn(prevDate, '2025-q4', '2025-07-01T00:00:00.000Z');
+  testDateFn(prevDate, '2025-02', '2025-01-01T00:00:00.000Z');
+  testDateFn(prevDate, '2025-02-W1', '2025-01-22T00:00:00.000Z');
+  testDateFn(prevDate, '2025-02-w2', '2025-02-01T00:00:00.000Z');
+  testDateFn(prevDate, '2025-02-W3', '2025-02-08T00:00:00.000Z');
+  testDateFn(prevDate, '2025-02-w4', '2025-02-15T00:00:00.000Z');
+  testDateFn(prevDate, '2025-02-03', '2025-02-02T00:00:00.000Z');
+
+  const ts = Date.parse('2025-05-10T04:05:06Z');
+  testDateFnWithUnit(prevDate, ts, 'day', '2025-05-10T00:00:00.000Z');
+  testDateFnWithUnit(prevDate, ts, 'week', '2025-05-08T00:00:00.000Z');
+  testDateFnWithUnit(prevDate, ts, 'month', '2025-05-01T00:00:00.000Z');
+  testDateFnWithUnit(prevDate, ts, 'quarter', '2025-04-01T00:00:00.000Z');
+  testDateFnWithUnit(prevDate, ts, 'year', '2025-01-01T00:00:00.000Z');
+
+  testDateFnWithUnit(prevDate, '2025', 'day', '2024-12-31T00:00:00.000Z');
+  testDateFnWithUnit(prevDate, '2025', 'week', '2024-12-22T00:00:00.000Z');
+  testDateFnWithUnit(prevDate, '2025', 'month', '2024-12-01T00:00:00.000Z');
+  testDateFnWithUnit(prevDate, '2025', 'quarter', '2024-10-01T00:00:00.000Z');
+  
+});
+
+test.run();