@dra2020/baseclient 1.0.13 → 1.0.14

package/lib/logserver/logblob.ts

@@ -0,0 +1,84 @@
+import * as zlib from 'zlib';
+
+// Shared libraries
+import * as Util from '../util/all';
+import * as Context from '../context/all';
+import * as LogAbstract from '../logabstract/all';
+import * as FSM from '../fsm/all';
+import * as Storage from '../storage/all';
+
+import { LogKey } from './logkey';
+
+interface Environment
+{
+  context: Context.IContext;
+  log: LogAbstract.ILog;
+  fsmManager: FSM.FsmManager;
+  storageManager: Storage.StorageManager;
+}
+
+export class LogBlob extends Storage.StorageBlob
+{
+  key: LogKey;
+
+  constructor(env: Environment, params: Storage.BlobParams)
+  {
+    if (params.bucket === undefined) params.bucket = 'logs';
+    super(env, params);
+    this.key = LogKey.create(this.params.id);
+    this.filter = this.filter.bind(this);
+  }
+
+  get env(): Environment { return this._env as Environment; }
+
+  get result(): string { return this.params.loadTo as string }
+
+  static createForDownload(env: Environment, id: string): LogBlob
+  {
+    let blob = new LogBlob(env, { id: id, loadToType: 'string' });
+    blob.startLoad(env.storageManager);
+    return blob;
+  }
+
+  static createForUpload(env: Environment, id: string, data: string): LogBlob
+  {
+    let params: Storage.BlobParams = {
+      id: id,
+      ContentEncoding: 'gzip',
+      ContentType: 'application/json',
+      saveFromType: 'string',
+      saveFrom: data
+      };
+    let blob = new LogBlob(env, params);
+    blob.setDirty();
+    blob.checkSave(env.storageManager);
+    return blob;
+  }
+
+  static createForDelete(env: Environment, id: string): LogBlob
+  {
+    let blob = new LogBlob(env, { id: id });
+    blob.startDelete(env.storageManager);
+    return blob;
+  }
+
+  static createForLs(env: Environment): LogBlob
+  {
+    let blob = new LogBlob(env, { id: '' });
+    blob.startList(env.storageManager);
+    return blob;
+  }
+
+  filter(s: string): boolean
+  {
+    let key = LogKey.create(s);
+    if (key == null) return false;
+    let mode = this.env.context.xflag('production') ? 'Prod' : 'Dev';
+    return key.mode === mode;
+  }
+
+  asString(): string
+  {
+    return this.params.loadTo as string;
+  }
+}

package/lib/logserver/logconcat.ts

@@ -0,0 +1,313 @@
+import * as fs from 'fs';
+import * as stream from 'stream';
+import * as zlib from 'zlib';
+
+// Shared libraries
+import * as Util from '../util/all';
+import * as Context from '../context/all';
+import * as LogAbstract from '../logabstract/all';
+import * as FSM from '../fsm/all';
+import * as Storage from '../storage/all';
+
+import { LogBlob } from './logblob';
+
+interface Environment
+{
+  context: Context.IContext;
+  log: LogAbstract.ILog;
+  fsmManager: FSM.FsmManager;
+  storageManager: Storage.StorageManager;
+}
+
+let UniqueState = FSM.FSM_CUSTOM1;
+const FSM_BLOB = UniqueState++;
+const FSM_QUERYING = UniqueState++;
+const FSM_LISTING = UniqueState++;
+const FSM_AGGREGATING = UniqueState++;
+const FSM_SAVING = UniqueState++;
+const FSM_LOADING = UniqueState++;
+const FSM_DELETING = UniqueState++;
+const FSM_FILTERING = UniqueState++;
+
+const reLog = /^Log_([^_]*)_(\d{4}-\d{2}-\d{2}T\d{2}:)\d{2}:\d{2}.\d{3}Z_(\d+)_[^\.]+.csv$/;
+const reLine = /^(.),(\d+),(.*)$/;
+
+// The compaction process involves:
+//    1. Listing the log files.
+//    2. Grouping into sets to compact/concat.
+//    3. Downloading the files and concatenating them.
+//    4. Uploading the concatenated file.
+//    5. Deleting the files used to construct the aggregate.
+//
+// The motivation for this is that further processing on the logs and log bucket will be
+// much more efficient with a smaller number of larger files. In particular, the lambda
+// instances result in lots of small log files.
+//
+
+function logToKey(env: Environment, log: string): string
+{
+  // Log is of form:
+  //  {Log,Agg}_{Prod,Dev}_YYYY-MM-DDTHH:MM:SS.UUUZ_{count}_{guid}.{csv,json}
+  // We are only concerned with Log_{Prod|Dev}_{datetime}_0_*.csv
+  // That is, we just want the small single instance log files.
+  // The key includes Log_Prod_YYYY-MM-DDTHH:
+  
+  let prod = env.context.xflag('production') ? 'Prod' : 'Dev';
+  let a = reLog.exec(log);
+  if (a && a.length == 4 && a[3] == '0' && a[1] === prod)
+    return `Log_${a[1]}_${a[2]}`;
+  return null;
+}
+
+export class FsmCompact extends FSM.Fsm
+{
+  blobLs: LogBlob;
+  clusters: { [key: string]: string[] };
+
+  constructor(env: Environment)
+  {
+    super(env);
+    this.clusters = {};
+  }
+
+  get env(): Environment { return this._env as Environment }
+
+  gatherLogList(logs: string[]): void
+  {
+    logs.forEach((log: string) => {
+        let key = logToKey(this.env, log);
+        if (key)
+        {
+          let cluster = this.clusters[key];
+          if (cluster === undefined)
+          {
+            cluster = [];
+            this.clusters[key] = cluster;
+          }
+          cluster.push(log);
+        }
+      });
+  }
+
+  tick(): void
+  {
+    if (this.ready && this.isDependentError)
+      this.setState(FSM.FSM_ERROR);
+    else if (this.ready)
+    {
+      switch (this.state)
+      {
+        case FSM.FSM_STARTING:
+          this.blobLs = LogBlob.createForLs(this.env);
+          this.waitOn(this.blobLs.fsmArray);
+          this.setState(FSM_LISTING);
+          break;
+
+        case FSM_LISTING:
+          this.gatherLogList(this.blobLs.keys);
+          this.blobLs.resetList();
+          if (this.blobLs.fsmList.done)
+            this.setState(FSM_FILTERING);
+          else
+            this.waitOn(this.blobLs.fsmList);
+          break;
+
+        case FSM_FILTERING:
+          Object.keys(this.clusters).forEach((key: string) => {
+              let cluster = this.clusters[key];
+              if (cluster.length < 10)
+                delete this.clusters[key];
+            });
+          this.env.log.chatter(`logconcat: compacting ${Util.countKeys(this.clusters)} log clusters`);
+          this.setState(FSM_AGGREGATING);
+          break;
+
+        case FSM_AGGREGATING:
+          for (let key in this.clusters) if (this.clusters.hasOwnProperty(key))
+          {
+            let cluster = this.clusters[key];
+            this.waitOn(new FsmCompactOne(this.env, key, cluster));
+            delete this.clusters[key];
+            break;  // one at a time
+          }
+          if (this.ready)
+            this.setState(FSM.FSM_DONE);
+          break;
+      }
+    }
+  }
+}
+
+// This is a little tricky because file format is wacky. Numeric value after 'S' in schema line is just
+// the schema number (redundant to implicit ordering). The numeric value after 'T' in template line is
+// a reference to the schema used on that line. The template number is implicit in the ordering. And then
+// the number after 'L' in a log line is a reference to this implicit template numbering order.
+
+class Permuter
+{
+  tag: string;  // 'T' or 'S'
+  entries: { [key: string]: { key: string, to: string, from: string } };
+  mapper: { [from: string]: string };
+  length: number;
+
+  constructor(tag: string)
+  {
+    this.tag = tag;
+    this.entries = {};
+    this.mapper = {};
+    this.length = 0;
+  }
+
+  reset(): void
+  {
+    Object.keys(this.entries).forEach((key: string) => {
+        let entry = this.entries[key];
+        entry.from = null;
+      });
+    this.mapper = {};
+  }
+
+  add(fields: string[]): void
+  {
+    if (fields.length != 4)
+      throw 'Permuter:add: unexpected line format';
+    let key = this.tag === 'S' ? fields[3] : `${fields[2]},${fields[3]}`;
+    let entry = this.entries[key];
+    if (entry === undefined)
+    {
+      entry = { key: key, to: String(this.length++), from: this.tag === 'S' ? fields[2] : String(Util.countKeys(this.mapper)) };
+      this.entries[key] = entry;
+    }
+    else
+    {
+      if (entry.from != null)
+        throw 'Permuter:add: unexpected duplicate field entry';
+      else
+        entry.from = this.tag === 'S' ? fields[2] : String(Util.countKeys(this.mapper));
+    }
+    this.mapper[entry.from] = entry.to;
+  }
+
+  map(from: string): string
+  {
+    return this.mapper[from];
+  }
+
+  asCSV(): string[]
+  {
+    let entryarray = Object.keys(this.entries).map((key: string) => { return this.entries[key] });
+    entryarray.sort((e1: any, e2: any) => { return Number(e1.to) < Number(e2.to) ? -1 : 1 });
+    return entryarray.map((e: any) => {
+        if (this.tag === 'S')
+          return new Array(this.tag, e.to, e.key).join(',');
+        else
+          return new Array(this.tag, e.key).join(',');
+      });
+  }
+}
+
+export class FsmCompactOne extends FSM.Fsm
+{
+  key: string;
+  cluster: string[];
+  blobs: LogBlob[];
+  csv: string[];
+  schemas: Permuter;
+  templates: Permuter;
+
+  constructor(env: Environment, key: string, cluster: string[])
+  {
+    super(env);
+    this.key = key;
+    this.cluster = cluster;
+    this.csv =  [];
+    this.schemas = new Permuter('S');
+    this.templates = new Permuter('T');
+    this.env.log.chatter(`logconcat: compacting ${this.key} with ${this.cluster.length} log files`);
+  }
+
+  get env(): Environment { return this._env as Environment }
+
+  loadBlobs(): void
+  {
+    this.blobs.forEach((blob: LogBlob) => {
+        let lines = blob.result.split('\n');
+        lines.forEach((line: string) => {
+            let a = reLine.exec(line);
+            if (a && a.length == 4)
+            {
+              switch (a[1])
+              {
+                case 'S': this.schemas.add(a); break;
+                case 'T': a[2] = this.schemas.map(a[2]); this.templates.add(a); break;
+                case 'L': a[2] = this.templates.map(a[2]); this.csv.push(a.slice(1).join(',')); break;
+                default:
+                  throw 'FsmCompactOne: unexpected line in csv file';
+              }
+            }
+          });
+        this.schemas.reset();
+        this.templates.reset();
+      });
+    this.blobs = [];
+  }
+
+  saveBlobs(): void
+  {
+    let csv = [...this.schemas.asCSV(), ...this.templates.asCSV(), ...this.csv];
+    let key = `${this.key}00:00.000Z_0_${Util.createGuid()}.csv`;
+    this.env.log.chatter(`logconcat: uploading ${key}`);
+    let blob = LogBlob.createForUpload(this.env, key, csv.join('\n'));
+    this.waitOn(blob.fsmLoad);
+  }
+
+  deleteBlobs(): void
+  {
+    this.cluster.forEach((log: string) => {
+        let blob = LogBlob.createForDelete(this.env, log);
+        this.waitOn(blob.fsmDel);
+      });
+  }
+
+  tick(): void
+  {
+    if (this.ready && this.isDependentError)
+    {
+      this.env.log.error('logconcat: failure building log cluster');
+      this.env.log.chatter('logconcat: failure building log cluster');
+      this.setState(FSM.FSM_ERROR);
+    }
+    else if (this.ready)
+    {
+      switch (this.state)
+      {
+        case FSM.FSM_STARTING:
+          this.blobs = [];
+          this.cluster.forEach((log: string) => {
+              let blob = LogBlob.createForDownload(this.env, log);
+              this.blobs.push(blob);
+              this.waitOn(blob.fsmLoad);
+            });
+          this.setState(FSM_LOADING);
+          break;
+
+        case FSM_LOADING:
+          this.loadBlobs();
+          this.saveBlobs();
+          this.setState(FSM_SAVING);
+          break;
+
+        case FSM_SAVING:
+          this.deleteBlobs();
+          this.setState(FSM_DELETING);
+          break;
+
+        case FSM_DELETING:
+          this.env.log.chatter(`logconcat: compacted ${this.key}`);
+          this.setState(FSM.FSM_DONE);
+          break;
+      }
+    }
+  }
+}
+

package/lib/logserver/logkey.ts

@@ -0,0 +1,125 @@
+export function dateToKey(d: Date): string
+{
+  let m = d.getUTCMonth() + 1;
+  let day = d.getUTCDate();
+  let yyyy = d.getFullYear();
+  let sm = m < 10 ? `0${m}` : String(m);
+  let sd = day < 10 ? `0${day}` : String(day);
+
+  return `${yyyy}.${sm}.${sd}`;
+}
+
+export function dateToMonthKey(d: Date): string
+{
+  let m = d.getUTCMonth() + 1;
+  let yyyy = d.getFullYear();
+  let sm = m < 10 ? `0${m}` : String(m);
+
+  return `${yyyy}.${sm}`;
+}
+
+export function dateToYearKey(d: Date): string
+{
+  return String(d.getFullYear());
+}
+
+export class LogKey
+{
+  kind: string;
+  mode: string;
+  date: string;
+  dateKey: string;
+  monthKey: string;
+  yearKey: string;
+  count: string;
+  instance: string;
+  ext: string;
+
+  constructor()
+  {
+  }
+
+  static root(): LogKey
+  {
+    let key = new LogKey();
+    key.kind = 'Agg';
+    return key;
+  }
+
+  static create(s: string): LogKey
+  {
+    let key = new LogKey();
+    if (s == null || s == '') return null;
+    let iExt = s.lastIndexOf('.');
+    let ext = s.substring(iExt+1);
+    s = s.substring(0, iExt);
+    let a = s.split('_');
+    if (a.length == 5)
+    {
+      key.kind = a[0]; key.mode = a[1]; key.date = a[2]; key.count = a[3]; key.instance = a[4]; key.ext = ext;
+    }
+    else if (a.length == 3)
+    {
+      key.kind = a[0]; key.mode = a[1]; key.date = a[2]; key.ext = ext;
+    }
+    else
+    {
+      key.kind = 'unknown'; key.dateKey = 'unknown'; key.instance = s;
+    }
+
+    if (key.kind === 'Log')
+    {
+      let d = new Date(key.date);
+      key.dateKey = dateToKey(d);
+      key.monthKey = dateToMonthKey(d);
+      key.yearKey = dateToYearKey(d);
+    }
+    else if (key.date)
+    {
+      let keys = key.date.split('.');
+      if (keys.length > 0)
+        key.yearKey = keys[0];
+      if (keys.length > 1)
+        key.monthKey = `${keys[0]}.${keys[1]}`;
+      if (keys.length > 2)
+        key.dateKey = key.date;
+    }
+    return key;
+  }
+
+  yearClosed(): boolean { return (Number(this.yearKey) < Number(nowKey.yearKey)) }
+  monthClosed(): boolean { return (Number(this.monthKey) < Number(nowKey.monthKey)) } 
+  dateClosed(): boolean { return this.dateKey !== nowKey.dateKey }
+  get closed(): boolean
+  {
+    if (this.kind === 'Log')
+      return true;
+    else if (this.kind === 'Agg')
+    {
+      if (this.dateKey)
+        return this.dateClosed();
+      else if (this.monthKey)
+        return this.monthClosed();
+      else
+        return this.yearClosed();
+    }
+    else
+      return true;
+  }
+
+  get yearID(): string { return `Agg_${this.mode}_${this.yearKey}.json` }
+  get monthID(): string { return `Agg_${this.mode}_${this.monthKey}.json` }
+  get dateID(): string { return `Agg_${this.mode}_${this.dateKey}.json` }
+
+  get isYear(): boolean { return this.monthKey === undefined }
+  get isMonth(): boolean { return this.monthKey !== undefined && this.dateKey === undefined }
+  get isDate(): boolean { return this.kind === 'Agg' && this.dateKey !== undefined }
+  get isLog(): boolean { return this.kind === 'Log' }
+}
+
+let nowKey: LogKey = new LogKey();
+let nowDate = new Date();
+nowKey.dateKey = dateToKey(nowDate);
+nowKey.monthKey = dateToMonthKey(nowDate);
+nowKey.yearKey = dateToYearKey(nowDate);
+

package/lib/memsqs/all.ts

@@ -0,0 +1,4 @@
+export * from './queue';
+export * from './client';
+export * from './server';
+export * from './loopback';