@dra2020/dra-types 1.4.8 → 1.5.10

package/dist/all.d.ts

@@ -3,3 +3,6 @@ export * from './schemas';
 export * from './bucketmap';
 export * from './colordata';
 export * from './gencolor';
+export * from './vfeature';
+export * from './csv';
+export * from './stats';

package/dist/csv.d.ts

@@ -0,0 +1,40 @@
+import * as VF from './vfeature';
+export declare type BlockMap = {
+    [id: string]: number;
+};
+export declare type BlockMapping = {
+    [id: string]: string;
+};
+export declare function canonicalDistrictID(districtID: string): string;
+export declare function canonicalSortingDistrictID(districtID: string): string;
+export declare function canonicalNumericFromDistrictID(districtID: string): number;
+export declare function canonicalDistrictIDFromNumber(districtID: string, n: number): string;
+export declare type DistrictOrder = {
+    [districtID: string]: number;
+};
+export declare function canonicalDistrictIDGapFill(keys: string[]): string[];
+export declare function canonicalDistrictIDOrdering(order: DistrictOrder): DistrictOrder;
+export interface OneCSVLine {
+    geoid: string;
+    districtID: string;
+}
+export declare function parseCSVLine(line: string): OneCSVLine;
+export interface ConvertResult {
+    inBlockMap: BlockMapping;
+    inStateMap: BlockMapping;
+    outValid: boolean;
+    outState: string;
+    outMap: BlockMapping;
+    outOrder: DistrictOrder;
+    outDistrictToSplit: VF.DistrictToSplitBlock;
+}
+export declare function blockmapToState(blockMap: BlockMapping): string;
+export declare function blockmapToVTDmap(blockMap: BlockMapping, stateMap: BlockMapping): ConvertResult;
+export declare const GEOIDToState: any;
+export declare const StateToGEOID: any;
+export declare function geoidToState(geoid: string): string;
+export declare type StateUrls = ('alabama' | 'alaska' | 'arizona' | 'arkansas' | 'california' | 'colorado' | 'connecticut' | 'delaware' | 'florida' | 'georgia' | 'hawaii' | 'idaho' | 'illinois' | 'indiana' | 'iowa' | 'kansas' | 'kentucky' | 'louisiana' | 'maine' | 'maryland' | 'massachusetts' | 'michigan' | 'minnesota' | 'mississippi' | 'missouri' | 'montana' | 'nebraska' | 'nevada' | 'new-hampshire' | 'new-jersey' | 'new-mexico' | 'new-york' | 'north-carolina' | 'north-dakota' | 'ohio' | 'oklahoma' | 'oregon' | 'pennsylvania' | 'rhode-island' | 'south-carolina' | 'south-dakota' | 'tennessee' | 'texas' | 'utah' | 'vermont' | 'virginia' | 'washington' | 'west-virginia' | 'wisconsin' | 'wyoming');
+export declare type ValidStateUrlsType = {
+    readonly [stateUrl in StateUrls]: boolean;
+};
+export declare function isStateUrl(s: any): s is StateUrls;

package/dist/dra-types.d.ts

@@ -1,9 +1,3 @@
-export declare type BlockMap = {
-    [id: string]: number;
-};
-export declare type BlockMapping = {
-    [id: string]: string;
-};
 export interface Comment {
     userid: string;
     text: string;
@@ -27,55 +21,3 @@ export interface UserLikes {
     id?: string;
     [aid: string]: Like | string;
 }
-export interface SplitBlock {
-    id?: string;
-    chunkKey?: string;
-    chunk?: string;
-    state: string;
-    datasource: string;
-    geoid: string;
-    blocks: string[];
-}
-export declare type DistrictToSplitBlock = {
-    [districtID: string]: SplitBlock[];
-};
-export declare function vgeoidToGeoid(vgeoid: string): string;
-export declare function vgeoidToChunk(vgeoid: string): string;
-export declare function vgeoidToHash(vgeoid: string): string;
-export declare function isVfeature(geoid: string): boolean;
-export declare function splitToCacheKey(s: SplitBlock): string;
-export declare function splitToChunkKey(s: SplitBlock): string;
-export declare function splitToPrefix(s: SplitBlock): string;
-export declare function cacheKeysToChunkHash(keys: string[]): string;
-export declare function canonicalDistrictID(districtID: string): string;
-export declare function canonicalSortingDistrictID(districtID: string): string;
-export declare function canonicalNumericFromDistrictID(districtID: string): number;
-export declare function canonicalDistrictIDFromNumber(districtID: string, n: number): string;
-export declare type DistrictOrder = {
-    [districtID: string]: number;
-};
-export declare function canonicalDistrictIDOrdering(order: DistrictOrder): DistrictOrder;
-export interface OneCSVLine {
-    geoid: string;
-    districtID: string;
-}
-export declare function parseCSVLine(line: string): OneCSVLine;
-export interface ConvertResult {
-    inBlockMap: BlockMapping;
-    inStateMap: BlockMapping;
-    outValid: boolean;
-    outState: string;
-    outMap: BlockMapping;
-    outOrder: DistrictOrder;
-    outDistrictToSplit: DistrictToSplitBlock;
-}
-export declare function blockmapToState(blockMap: BlockMapping): string;
-export declare function blockmapToVTDmap(blockMap: BlockMapping, stateMap: BlockMapping): ConvertResult;
-export declare const GEOIDToState: any;
-export declare const StateToGEOID: any;
-export declare function geoidToState(geoid: string): string;
-export declare type StateUrls = ('alabama' | 'alaska' | 'arizona' | 'arkansas' | 'california' | 'colorado' | 'connecticut' | 'delaware' | 'florida' | 'georgia' | 'hawaii' | 'idaho' | 'illinois' | 'indiana' | 'iowa' | 'kansas' | 'kentucky' | 'louisiana' | 'maine' | 'maryland' | 'massachusetts' | 'michigan' | 'minnesota' | 'mississippi' | 'missouri' | 'montana' | 'nebraska' | 'nevada' | 'new-hampshire' | 'new-jersey' | 'new-mexico' | 'new-york' | 'north-carolina' | 'north-dakota' | 'ohio' | 'oklahoma' | 'oregon' | 'pennsylvania' | 'rhode-island' | 'south-carolina' | 'south-dakota' | 'tennessee' | 'texas' | 'utah' | 'vermont' | 'virginia' | 'washington' | 'west-virginia' | 'wisconsin' | 'wyoming');
-export declare type ValidStateUrlsType = {
-    readonly [stateUrl in StateUrls]: boolean;
-};
-export declare function isStateUrl(s: any): s is StateUrls;

package/dist/dra-types.js

@@ -109,11 +109,13 @@ function __export(m) {
     for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p];
 }
 Object.defineProperty(exports, "__esModule", { value: true });
-__export(__webpack_require__(/*! ./dra-types */ "./lib/dra-types.ts"));
 __export(__webpack_require__(/*! ./schemas */ "./lib/schemas.ts"));
 __export(__webpack_require__(/*! ./bucketmap */ "./lib/bucketmap.ts"));
 __export(__webpack_require__(/*! ./colordata */ "./lib/colordata.ts"));
 __export(__webpack_require__(/*! ./gencolor */ "./lib/gencolor.ts"));
+__export(__webpack_require__(/*! ./vfeature */ "./lib/vfeature.ts"));
+__export(__webpack_require__(/*! ./csv */ "./lib/csv.ts"));
+__export(__webpack_require__(/*! ./stats */ "./lib/stats.ts"));
 
 
 /***/ }),
@@ -135,8 +137,11 @@ exports.BucketMap = {
     'state': 'dra-us-west-2-723398989493',
     'state-dev': 'dev-dra-us-west-2-723398989493',
     'logs': 'dra-uswest-logs',
+    'logs-dev': 'dra-uswest-logs',
     'memsqs': 'dra-uswest-memsqs',
+    'memsqs-dev': 'dra-uswest-memsqs',
     'images': 'dra-uswest-images',
+    'images-dev': 'dra-uswest-images',
     'data': 'dra-us-west-datafiles',
     'data-dev': 'dra-us-west-datafiles-dev',
     'splits': 'dra-block-cache',
@@ -147,6 +152,8 @@ exports.BucketMap = {
     'geojson-dev': 'dra-uswest-geojson',
     'transfers': 'dra-transfers',
     'transfers-dev': 'dra-transfers-dev',
+    'downloads': 'dra-uswest-downloads',
+    'downloads-dev': 'dra-uswest-downloads',
 };
 
 
@@ -473,10 +480,10 @@ lum.forEach((n) => { exports.IntensityBackgroundColor.push(`rgba(${n}, ${n}, ${n
 
 /***/ }),
 
-/***/ "./lib/dra-types.ts":
-/*!**************************!*\
-  !*** ./lib/dra-types.ts ***!
-  \**************************/
+/***/ "./lib/csv.ts":
+/*!********************!*\
+  !*** ./lib/csv.ts ***!
+  \********************/
 /*! no static exports found */
 /***/ (function(module, exports, __webpack_require__) {
 
@@ -484,82 +491,7 @@ lum.forEach((n) => { exports.IntensityBackgroundColor.push(`rgba(${n}, ${n}, ${n
 
 Object.defineProperty(exports, "__esModule", { value: true });
 // Public libraries
-const Hash = __webpack_require__(/*! object-hash */ "object-hash");
 const Util = __webpack_require__(/*! @dra2020/util */ "@dra2020/util");
-// Canonical hashing of splitblock data
-function hash(o) {
-    return Hash(o, { respectType: false,
-        unorderedArrays: true,
-        unorderedObjects: true,
-        excludeKeys: (k) => (k === 'id' || k === 'chunk')
-    });
-}
-function vgeoidToGeoid(vgeoid) {
-    let re = /vfeature_([^_]*)_.*/;
-    let a = re.exec(vgeoid);
-    if (a == null || a.length != 2)
-        return '';
-    else
-        return a[1];
-}
-exports.vgeoidToGeoid = vgeoidToGeoid;
-function vgeoidToChunk(vgeoid) {
-    // vgeoid is string of form: "vfeature_[geoid]_[chunkid]_[hash]"
-    // the contents are chunked into a file of form "vfeature_chunk_[chunkid]"
-    // So extract the chunk ID and download that.
-    let re = /vfeature_([^_]*)_([^_*])_(.*)/;
-    let a = re.exec(vgeoid);
-    if (a && a.length == 4)
-        vgeoid = `vfeature_chunk_${a[2]}`;
-    else
-        vgeoid = null;
-    return vgeoid;
-}
-exports.vgeoidToChunk = vgeoidToChunk;
-function vgeoidToHash(vgeoid) {
-    // vgeoid is string of form: "vfeature_[geoid]_[chunkid]_[hash]"
-    let re = /vfeature_([^_]*)_([^_*])_(.*)/;
-    let a = re.exec(vgeoid);
-    if (a && a.length == 4)
-        vgeoid = a[3];
-    else
-        vgeoid = null;
-    return vgeoid;
-}
-exports.vgeoidToHash = vgeoidToHash;
-function isVfeature(geoid) {
-    return geoid.indexOf('vfeature') === 0;
-}
-exports.isVfeature = isVfeature;
-function splitToCacheKey(s) {
-    if (s.id === undefined)
-        s.id = hash(s);
-    if (s.chunk === undefined)
-        s.chunk = "0";
-    return `_${s.state}_${s.datasource}_vfeature_${s.geoid}_${s.chunk}_${s.id}.geojson`;
-}
-exports.splitToCacheKey = splitToCacheKey;
-function splitToChunkKey(s) {
-    if (s.chunk === undefined)
-        s.chunk = "0";
-    return `_${s.state}_${s.datasource}_vfeature_chunk_${s.chunk}.geojson`;
-}
-exports.splitToChunkKey = splitToChunkKey;
-function splitToPrefix(s) {
-    if (s.blocks === undefined) {
-        let re = /_([^_]*)_(.*)_vfeature.*\.geojson$/;
-        let a = re.exec(s.id);
-        if (a && a.length == 3)
-            return `_${a[1]}_${a[2]}`;
-        return s.id;
-    }
-    return `_${s.state}_${s.datasource}`;
-}
-exports.splitToPrefix = splitToPrefix;
-function cacheKeysToChunkHash(keys) {
-    return hash(keys);
-}
-exports.cacheKeysToChunkHash = cacheKeysToChunkHash;
 let reNumeric = /^(\D*)(\d*)(\D*)$/;
 let reDistrictNumber = /^\d+$/;
 let reDistrictNumeric = /^\d/;
@@ -575,6 +507,8 @@ function canonicalDistrictID(districtID) {
 }
 exports.canonicalDistrictID = canonicalDistrictID;
 // Normalize any numeric part to have four digits with padded leading zeros
+// so alphabetic sorting will result in correct numeric sort for mixed alphanumber
+// district labels.
 function canonicalSortingDistrictID(districtID) {
     let a = reNumeric.exec(districtID);
     if (a && a.length == 4) {
@@ -620,12 +554,34 @@ function canonicalDistrictIDFromNumber(districtID, n) {
     return districtID;
 }
 exports.canonicalDistrictIDFromNumber = canonicalDistrictIDFromNumber;
+// If purely numeric districtIDs and we are missing some number of IDs less than
+function canonicalDistrictIDGapFill(keys) {
+    if (keys == null || keys.length == 0)
+        return keys;
+    let nonNumeric = keys.find((s) => !reDistrictNumber.test(s)) !== undefined;
+    if (nonNumeric)
+        return keys;
+    let max = Number(keys[keys.length - 1]);
+    if (max == keys.length || (max - keys.length) > keys.length)
+        return keys; // no gaps or too many gaps
+    // OK, finally going to fill some gaps
+    for (let i = 0; i < keys.length; i++) {
+        let here = Number(keys[i]);
+        while (here > i + 1) {
+            keys.splice(i, 0, canonicalSortingDistrictID(String(i + 1)));
+            i++;
+        }
+    }
+    return keys;
+}
+exports.canonicalDistrictIDGapFill = canonicalDistrictIDGapFill;
 function canonicalDistrictIDOrdering(order) {
     let keys = Object.keys(order);
     let i;
     let a = [];
     let template = undefined;
     keys = keys.map((s) => canonicalSortingDistrictID(s));
+    keys = canonicalDistrictIDGapFill(keys);
     keys.sort();
     order = {};
     for (i = 0; i < keys.length; i++)
@@ -1070,9 +1026,263 @@ exports.Schemas = {
     'likes': {},
     'comments': {},
     'stats': {},
+    'livestats': {},
 };
 
 
+/***/ }),
+
+/***/ "./lib/stats.ts":
+/*!**********************!*\
+  !*** ./lib/stats.ts ***!
+  \**********************/
+/*! no static exports found */
+/***/ (function(module, exports, __webpack_require__) {
+
+"use strict";
+
+// In constrast to logging, the stats manager is responsible for tracking live server status.
+// The results are stored in a compact database record which can be easily queried for current
+// status. Each server instance maintains its own field in the overall "values" record in the
+// stats database. The reporting tool aggregates all instances to report overall server activity.
+//
+// There are basically three kinds of values:
+//  Sum - we want to sum the value across all server instances (e.g. number of connnected clients)
+//  Avg - we just take average across all server instances (e.g. memory heap size)
+//  Rate - we want to track the rate of some activity over some time period
+//
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ValTypeSum = 0;
+exports.ValTypeAvg = 1;
+exports.ValTypeRate = 2;
+const ExpiryAge = 1000 * 60 * 60 * 24; // throw away instance record after this time period
+function statExpiryTime() {
+    let time = new Date();
+    time.setTime(time.getTime() + ExpiryAge);
+    return time.toJSON();
+}
+exports.statExpiryTime = statExpiryTime;
+// Aggregate running value. This applies both in memory (logging multiple times before
+// saving to database) as well as aggregating into an instance structure in the DB.
+function statValueRecord(sv, cur, valType) {
+    if (sv === undefined)
+        sv = {};
+    if (sv.min === undefined || cur < sv.min)
+        sv.min = cur;
+    if (sv.max === undefined || cur > sv.max)
+        sv.max = cur;
+    sv.cur = cur;
+    sv.valType = valType;
+    return sv;
+}
+exports.statValueRecord = statValueRecord;
+function statRateIncr(sr, incr = 1) {
+    if (sr == null)
+        sr = {};
+    sr.valType = exports.ValTypeRate;
+    sr.thisMin = sr.thisMin === undefined ? incr : sr.thisMin + incr;
+    sr.thisHour = sr.thisHour === undefined ? incr : sr.thisHour + incr;
+    sr.thisDay = sr.thisDay === undefined ? incr : sr.thisDay + incr;
+    return sr;
+}
+exports.statRateIncr = statRateIncr;
+exports.OneMinute = 1000 * 60;
+exports.OneHour = exports.OneMinute * 60;
+exports.OneDay = exports.OneHour * 24;
+function statRateRollover(sr, period) {
+    if ((period % exports.OneMinute) == 0) {
+        sr.lastMin = sr.thisMin;
+        sr.thisMin = 0;
+    }
+    if ((period % exports.OneHour) == 0) {
+        sr.lastHour = sr.thisHour;
+        sr.thisHour = 0;
+    }
+    if ((period % exports.OneDay) == 0) {
+        sr.lastDay = sr.thisDay;
+        sr.thisDay = 0;
+    }
+}
+exports.statRateRollover = statRateRollover;
+function statEntryIndexMerge(accum, si) {
+    if (accum == null)
+        accum = {};
+    // We just discard and replace old rate information
+    Object.keys(accum).forEach((p) => { if (accum[p].valType == exports.ValTypeRate)
+        delete accum[p]; });
+    // But merge in all the other values
+    Object.keys(si).forEach((p) => {
+        let se = si[p];
+        if (se.valType == exports.ValTypeRate)
+            accum[p] = se;
+        else {
+            let sv = se;
+            accum[p] = statValueRecord(se, sv.cur, sv.valType);
+        }
+    });
+    return accum;
+}
+exports.statEntryIndexMerge = statEntryIndexMerge;
+function statEntryIndexRollover(si, period) {
+    Object.keys(si).forEach((p) => {
+        let se = si[p];
+        if (se.valType == exports.ValTypeRate)
+            statRateRollover(se, period);
+    });
+}
+exports.statEntryIndexRollover = statEntryIndexRollover;
+function statRecordMerge(accum, si) {
+    if (accum == null)
+        accum = { id: si.id, production: si.production, time: si.time, expires: si.expires, index: {} };
+    accum.expires = si.expires;
+    accum.time = si.time;
+    statEntryIndexMerge(accum.index, si.index);
+    return accum;
+}
+exports.statRecordMerge = statRecordMerge;
+// Accumulate a set of values together (e.g. from multiple running instances)
+// for reporting current state.
+//
+function statValueAccum(accum, sv) {
+    if (accum == null) {
+        accum = {};
+        accum.valType = sv.valType;
+    }
+    if (accum.min === undefined || sv.min < accum.min)
+        accum.min = sv.min;
+    if (accum.max === undefined || sv.max > accum.max)
+        accum.max = sv.max;
+    if (accum.valType === undefined)
+        accum.valType = sv.valType;
+    if (accum.valType == exports.ValTypeSum)
+        accum.cur = (accum.cur === undefined ? sv.cur : accum.cur + sv.cur);
+    else if (accum.valType == exports.ValTypeAvg) {
+        accum.tot = (accum.tot === undefined ? 0 : accum.tot) + sv.cur;
+        accum.cnt = (accum.cnt === undefined ? 0 : accum.cnt) + 1;
+        accum.cur = accum.tot / accum.cnt;
+    }
+    return accum;
+}
+exports.statValueAccum = statValueAccum;
+function statRateAccum(accum, sr) {
+    if (accum == null) {
+        accum = {};
+        accum.valType = sr.valType;
+    }
+    accum.thisMin = accum.thisMin === undefined ? sr.thisMin : (accum.thisMin + (sr.thisMin === undefined ? 0 : sr.thisMin));
+    accum.lastMin = accum.lastMin === undefined ? sr.lastMin : (accum.lastMin + (sr.lastMin === undefined ? 0 : sr.lastMin));
+    accum.thisHour = accum.thisHour === undefined ? sr.thisHour : (accum.thisHour + (sr.thisHour === undefined ? 0 : sr.thisHour));
+    accum.lastHour = accum.lastHour === undefined ? sr.lastHour : (accum.lastHour + (sr.lastHour === undefined ? 0 : sr.lastHour));
+    accum.thisDay = accum.thisDay === undefined ? sr.thisDay : (accum.thisDay + (sr.thisDay === undefined ? 0 : sr.thisDay));
+    accum.lastDay = accum.lastDay === undefined ? sr.lastDay : (accum.lastDay + (sr.lastDay === undefined ? 0 : sr.lastDay));
+    return accum;
+}
+exports.statRateAccum = statRateAccum;
+function statIndexAccum(accum, si) {
+    if (accum == null)
+        accum = {};
+    Object.keys(si).forEach((p) => {
+        let se = si[p];
+        if (se.valType == exports.ValTypeRate)
+            accum[p] = statRateAccum(accum[p], se);
+        else
+            accum[p] = statValueAccum(accum[p], se);
+    });
+    return accum;
+}
+exports.statIndexAccum = statIndexAccum;
+
+
+/***/ }),
+
+/***/ "./lib/vfeature.ts":
+/*!*************************!*\
+  !*** ./lib/vfeature.ts ***!
+  \*************************/
+/*! no static exports found */
+/***/ (function(module, exports, __webpack_require__) {
+
+"use strict";
+
+Object.defineProperty(exports, "__esModule", { value: true });
+// Public libraries
+const Hash = __webpack_require__(/*! object-hash */ "object-hash");
+// Canonical hashing of splitblock data
+function hash(o) {
+    return Hash(o, { respectType: false,
+        unorderedArrays: true,
+        unorderedObjects: true,
+        excludeKeys: (k) => (k === 'id' || k === 'chunk')
+    });
+}
+function vgeoidToGeoid(vgeoid) {
+    let re = /vfeature_([^_]*)_.*/;
+    let a = re.exec(vgeoid);
+    if (a == null || a.length != 2)
+        return '';
+    else
+        return a[1];
+}
+exports.vgeoidToGeoid = vgeoidToGeoid;
+function vgeoidToChunk(vgeoid) {
+    // vgeoid is string of form: "vfeature_[geoid]_[chunkid]_[hash]"
+    // the contents are chunked into a file of form "vfeature_chunk_[chunkid]"
+    // So extract the chunk ID and download that.
+    let re = /vfeature_([^_]*)_([^_*])_(.*)/;
+    let a = re.exec(vgeoid);
+    if (a && a.length == 4)
+        vgeoid = `vfeature_chunk_${a[2]}`;
+    else
+        vgeoid = null;
+    return vgeoid;
+}
+exports.vgeoidToChunk = vgeoidToChunk;
+function vgeoidToHash(vgeoid) {
+    // vgeoid is string of form: "vfeature_[geoid]_[chunkid]_[hash]"
+    let re = /vfeature_([^_]*)_([^_*])_(.*)/;
+    let a = re.exec(vgeoid);
+    if (a && a.length == 4)
+        vgeoid = a[3];
+    else
+        vgeoid = null;
+    return vgeoid;
+}
+exports.vgeoidToHash = vgeoidToHash;
+function isVfeature(geoid) {
+    return geoid.indexOf('vfeature') === 0;
+}
+exports.isVfeature = isVfeature;
+function splitToCacheKey(s) {
+    if (s.id === undefined)
+        s.id = hash(s);
+    if (s.chunk === undefined)
+        s.chunk = "0";
+    return `_${s.state}_${s.datasource}_vfeature_${s.geoid}_${s.chunk}_${s.id}.geojson`;
+}
+exports.splitToCacheKey = splitToCacheKey;
+function splitToChunkKey(s) {
+    if (s.chunk === undefined)
+        s.chunk = "0";
+    return `_${s.state}_${s.datasource}_vfeature_chunk_${s.chunk}.geojson`;
+}
+exports.splitToChunkKey = splitToChunkKey;
+function splitToPrefix(s) {
+    if (s.blocks === undefined) {
+        let re = /_([^_]*)_(.*)_vfeature.*\.geojson$/;
+        let a = re.exec(s.id);
+        if (a && a.length == 3)
+            return `_${a[1]}_${a[2]}`;
+        return s.id;
+    }
+    return `_${s.state}_${s.datasource}`;
+}
+exports.splitToPrefix = splitToPrefix;
+function cacheKeysToChunkHash(keys) {
+    return hash(keys);
+}
+exports.cacheKeysToChunkHash = cacheKeysToChunkHash;
+
+
 /***/ }),
 
 /***/ "@dra2020/util":