@jambonz/time-series 0.1.1 → 0.1.6

package/index.js

@@ -1,9 +1,9 @@
-const debug = require('debug')('jambonz:time-series');
 const assert = require('assert');
 const Influx = require('influx');
 const AlertType = {
   WEBHOOK_STATUS_FAILURE: 'webhook-failure',
   WEBHOOK_CONNECTION_FAILURE: 'webhook-connection-failure',
+  WEBHOOK_URL_NOTFOUND: 'webhook-url-notfound',
   WEBHOOK_AUTH_FAILURE: 'webhook-auth-failure',
   TTS_NOT_PROVISIONED: 'no-tts',
   STT_NOT_PROVISIONED: 'no-stt',
@@ -12,7 +12,8 @@ const AlertType = {
   CARRIER_NOT_PROVISIONED: 'no-carrier',
   CALL_LIMIT: 'call-limit',
   DEVICE_LIMIT: 'device-limit',
-  API_LIMIT: 'api-limit'
+  API_LIMIT: 'api-limit',
+  ACCOUNT_INACTIVE: 'account is inactive or suspended'
 };
 
 const schemas = {
@@ -50,6 +51,25 @@ const schemas = {
   }
 };
 
+const writeData = async(client) => {
+  const {data, initialized} = client.locals;
+  if (0 === data.length || client.locals.writing) return;
+  if (!initialized) await initDatabase(client, client.locals.db);
+  try {
+    //console.log(`writing data ${JSON.stringify(data)}`);
+    client.locals.writing = true;
+    await client.writePoints(data);
+    client.locals.writing = false;
+    client.locals.data = [];
+  } catch (err) {
+    console.log(err);
+    client.locals.writing = false;
+    client.locals.data = [];
+    throw err;
+  }
+};
+
+
 const createCdrQuery = ({account_sid, page, page_size, trunk, direction, answered, days, start, end}) => {
   let sql = `SELECT * from cdrs WHERE account_sid = '${account_sid}' `;
   if (trunk) sql += `AND trunk = '${trunk}' `;
@@ -67,7 +87,7 @@ const createCdrQuery = ({account_sid, page, page_size, trunk, direction, answere
   return sql;
 };
 const createCdrCountQuery = ({account_sid, trunk, direction, answered, days, start, end}) => {
-  let sql = `SELECT COUNT(call_sid) from cdrs WHERE account_sid = '${account_sid}' `;
+  let sql = `SELECT COUNT(sip_callid) from cdrs WHERE account_sid = '${account_sid}' `;
   if (trunk) sql += `AND trunk = '${trunk}' `;
   if (direction) sql += `AND direction = '${direction}' `;
   if (['true', 'false'].includes(answered)) sql += `AND answered = '${answered}' `;
@@ -96,7 +116,7 @@ const createAlertsQuery = ({account_sid, alert_type, page, page_size, days, star
 };
 
 const createAlertsCountQuery = ({account_sid, alert_type, days, start, end}) => {
-  let sql = `SELECT COUNT(*) FROM alerts WHERE account_sid = '${account_sid}' `;
+  let sql = `SELECT COUNT(message) FROM alerts WHERE account_sid = '${account_sid}' `;
   if (alert_type) sql += `AND alert_type = '${alert_type}' `;
   if (days) sql += `AND time > now() - ${days}d `;
   else {
@@ -112,11 +132,11 @@ const initDatabase = async(client, dbName) => {
   if (!names.includes(dbName)) {
     await client.createDatabase(dbName);
   }
-  client._initialized = true;
+  client.locals.initialized = true;
 };
 
 const writeCdrs = async(client, cdrs) => {
-  if (!client._initialized) await initDatabase(client, 'cdrs');
+  if (!client.locals.initialized) await initDatabase(client, 'cdrs');
   cdrs = (Array.isArray(cdrs) ? cdrs : [cdrs])
     .map((cdr) => {
       const {direction, host, trunk, account_sid, answered, attempted_at, ...fields} = cdr;
@@ -133,13 +153,16 @@ const writeCdrs = async(client, cdrs) => {
         }
       };
     });
-  debug(`writing cdrs: ${JSON.stringify(cdrs)}`);
-  return await client.writePoints(cdrs);
+  //console.log(`writing cdrs: ${JSON.stringify(cdrs)}`);
+  client.locals.data = [...client.locals.data, ...cdrs];
+  if (client.locals.data.length >= client.locals.commitSize) {
+    await writeData(client);
+  }
+  return;
 };
 
 const queryCdrs = async(client, opts) => {
-  if (!client._initialized) await initDatabase(client, 'alerts');
-  //console.log(JSON.stringify(opts));
+  if (!client.locals.initialized) await initDatabase(client, 'cdrs');
   const response = {
     total: 0,
     page_size: opts.page_size,
@@ -148,7 +171,7 @@ const queryCdrs = async(client, opts) => {
   };
   const sqlTotal = createCdrCountQuery(opts);
   const obj = await client.queryRaw(sqlTotal);
-  //console.log(JSON.stringify(obj));
+  //console.log(`sql: ${sqlTotal}, results: ${JSON.stringify(obj)}`);
   if (!obj.results || !obj.results[0].series) return response;
   response.total = obj.results[0].series[0].values[0][1];
 
@@ -173,7 +196,7 @@ const queryCdrs = async(client, opts) => {
 };
 
 const writeAlerts = async(client, alerts) => {
-  if (!client._initialized) await initDatabase(client, 'alerts');
+  if (!client.locals.initialized) await initDatabase(client, 'alerts');
   alerts = (Array.isArray(alerts) ? alerts : [alerts])
     .map((alert) => {
       const {alert_type, account_sid, url, status, vendor, count, detail, timestamp} = alert;
@@ -189,6 +212,9 @@ const writeAlerts = async(client, alerts) => {
           case AlertType.WEBHOOK_AUTH_FAILURE:
             message = `authentication failure: ${url}`;
             break;
+          case AlertType.WEBHOOK_URL_NOTFOUND:
+            message = `webhook url not found: ${url}`;
+            break;
           case AlertType.TTS_NOT_PROVISIONED:
             message = `text to speech credentials for ${vendor} have not been provisioned`;
             break;
@@ -219,16 +245,20 @@ const writeAlerts = async(client, alerts) => {
         }
       }
       const obj = {measurement: 'alerts', fields: { message }, tags: { alert_type, account_sid }};
-      if (timestamp) obj.fields.timestamp = timestamp;
+      if (timestamp) obj.timestamp = timestamp;
       if (detail) obj.fields.detail = detail;
       return obj;
     });
   //console.log(`writing alerts: ${JSON.stringify(alerts)}`);
-  return await client.writePoints(alerts);
+  client.locals.data = [...client.locals.data, ...alerts];
+  if (client.locals.data.length >= client.locals.commitSize) {
+    await writeData(client);
+  }
+  return;
 };
 
 const queryAlerts = async(client, opts) => {
-  if (!client._initialized) await initDatabase(client, 'alerts');
+  if (!client.locals.initialized) await initDatabase(client, 'alerts');
   const response = {
     total: 0,
     page_size: opts.page_size,
@@ -237,7 +267,7 @@ const queryAlerts = async(client, opts) => {
   };
   const sqlTotal = createAlertsCountQuery(opts);
   const obj = await client.queryRaw(sqlTotal);
-  //console.log(JSON.stringify(obj));
+  //console.log(`query total alerts: ${sqlTotal}: ${JSON.stringify(obj)}`);
   if (!obj.results || !obj.results[0].series) return response;
   response.total = obj.results[0].series[0].values[0][1];
 
@@ -264,8 +294,28 @@ module.exports = (logger, opts) => {
 
   const cdrClient = new Influx.InfluxDB({database: 'cdrs', schemas: schemas.cdr, ...opts});
   const alertClient = new Influx.InfluxDB({database: 'alerts', schemas: schemas.alerts, ...opts});
-  cdrClient._initialized = false;
-  alertClient._initialized = false;
+
+  cdrClient.locals = {
+    db: 'cdrs',
+    initialized: false,
+    writing: false,
+    commitSize: opts.commitSize || 1,
+    commitInterval: opts.commitInterval || 10,
+    data: []
+  };
+  alertClient.locals = {
+    db: 'alerts',
+    initialized: false,
+    writing: false,
+    commitSize: opts.commitSize || 1,
+    commitInterval: opts.commitInterval || 10,
+    data: []
+  };
+
+  if (opts.commitSize > 1 && opts.commitInterval && opts.commitInterval > 2) {
+    setInterval(writeData.bind(null, cdrClient), opts.commitInterval * 1000);
+    setInterval(writeData.bind(null, alertClient), opts.commitInterval * 1000);
+  }
 
   return {
     writeCdrs: writeCdrs.bind(null, cdrClient),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/time-series",
-  "version": "0.1.1",
+  "version": "0.1.6",
   "description": "write and query data to time series daetabase",
   "main": "index.js",
   "scripts": {

package/test/unit-tests.js

@@ -8,7 +8,7 @@ const {
   writeAlerts,
   queryAlerts,
   AlertType
-} = require('..')(consoleLogger, '127.0.0.1');
+} = require('..')(consoleLogger, '127.0.0.1', {commitSize: 1});
 
 test('write cdr data', async(t) => {
   let result = await writeCdrs([{