ezekiel-cli 5.8.2

package/lib/swagger-client.js

@@ -0,0 +1,287 @@
+const fetchAuthData = require('../lib/settings').fetchSettings,
+      path = require('path'),
+      Gateway = require('../lib/pos-proxy'),
+      logger = require('../lib/logger'),
+      { apiRequest } = require('./apiRequest'),
+      api = require('./logsv2/http'),
+      fs = require('fs');
+
+// const colors = require('colors');
+
+
+class APIs {
+  constructor(ctx) {
+    this.Search = new Search(ctx);
+    this.Alerts = new Alerts(ctx);
+  }
+}
+
+class Alerts {
+  constructor(ctx) {
+    this.ctx = ctx;
+  }
+
+  // /api/{org_id}/alerts
+  async ListAlerts({org_id}, {securities}) {
+    return await api.get(this.ctx.url + `/api/${org_id}/alerts`, {
+      securities: securities
+    });
+  }
+
+  // /api/{org_id}/{stream_name}/alerts/{alert_name}/trigger
+  async TriggerAlert({org_id, stream_name, alert_name}, {requestBody, securities}) {
+    return await api.put(this.ctx.url + `/api/${org_id}/${stream_name}/alerts/${alert_name}/trigger`, {
+      securities: securities,
+      body: requestBody
+    });
+  }
+  // /api/{org_id}/{stream_name}/alerts/{alert_name}
+  async SaveAlert({org_id, stream_name, alert_name}, {requestBody, securities}) {
+    console.log(requestBody);
+    return await api.post(this.ctx.url + `/api/${org_id}/${stream_name}/alerts/${alert_name}`, {
+      securities: securities,
+      body: requestBody
+    });
+  }
+
+  // /api/{org_id}/alerts/templates/{template_name}
+  async CreateTemplate({org_id, template_name}, {requestBody, securities}) {
+    return await api.post(this.ctx.url + `/api/${org_id}/alerts/templates/${template_name}`, {
+      securities: securities,
+      body: requestBody
+    });
+  }
+
+  // /api/{org_id}/alerts/destinations/{destination_name}
+  async CreateDestination({org_id, destination_name}, {requestBody, securities}) {
+    return await api.post(this.ctx.url + `/api/${org_id}/alerts/destinations/${destination_name}`, {
+      securities: securities,
+      body: requestBody
+    });
+  }
+}
+
+class Search {
+  constructor(ctx) {
+    this.ctx = ctx;
+  }
+
+  async SearchSQL({org_id}, {requestBody, securities}) {
+    return await api.post(this.ctx.url + `/api/${org_id}/_search`, {
+      securities: securities,
+      body: requestBody
+    });
+  }
+
+  async SearchAround({org_id, stream_name, key, size, sql}, {requestBody, securities}) {
+    if (sql == undefined) sql = "";
+    return await api.get(this.ctx.url + `/api/${org_id}/${stream_name}/_around`, {
+      securities: securities,
+      params: { key, size, sql }
+    });
+  }
+
+}
+
+class LogClient {
+  constructor({ url }) {
+    this.apis = new APIs(this);
+    this.url = url;
+  }
+}
+
+class SwaggerProxy {
+  static async client(environment, program) {
+    try {
+      const LOGS_PROXY_URL = process.env.LOGS_PROXY_URL || 'https://openobserve-proxy.platformos.dev'
+
+      const client = new LogClient({url: LOGS_PROXY_URL});
+
+      const authData = fetchAuthData(environment, program);
+      
+      const securities = { authorized: { Authorization: authData.key }}
+      const instance = await new Gateway(authData).getInstance()
+
+      instance.url = authData.url
+
+      logger.Debug({instance, authData, securities})
+
+      return new SwaggerProxy(client, instance, securities);
+    }
+    catch(e) {
+      if (e.statusCode == 401)
+        logger.Error(`[Unauthorized] Please ensure your token is correct.`);
+      else
+        logger.Error(e)
+    }
+  }
+
+  constructor(client, instance, securities) {
+    this.client = client
+    this.instance = instance
+    this.securities = securities
+  }
+
+  async searchAround(program){
+    return this.client.apis.Search
+      .SearchAround(
+        {org_id: this.instance.uuid, stream_name: program.stream_name, key: program.key, size: program.size, sql: program.sql},
+        {securities: this.securities})
+  }
+
+  async searchSQLByQuery(query){
+    return this.client.apis.Search
+      .SearchSQL(
+        {org_id: this.instance.uuid},
+        {requestBody: query, securities: this.securities})
+  }
+
+  async searchSQL(program){
+    return this.searchSQLByQuery(search.buildQuery(program))
+  }
+
+  async alerts(program){
+    return this.client.apis.Alerts
+      .ListAlerts({org_id: this.instance.uuid}, {securities: this.securities})
+  }
+
+  async triggerAlert(program){
+    return this.client.apis.Alerts
+      .TriggerAlert(
+        {org_id: this.instance.uuid, stream_name: 'logs', alert_name: program.name},
+        {securities: this.securities})
+  }
+
+  async createAlert(program){
+    const alert = alerts.buildAlert(program)
+
+    // console.log(this.client.apis.Alerts)
+
+    const template = await this.createTemplate(program)
+    logger.Debug(template.body)
+    const destination = await this.createDestination(program)
+    logger.Debug(destination.body)
+
+    return this.client.apis.Alerts
+      .SaveAlert(
+        {org_id: this.instance.uuid, stream_name: 'logs', alert_name: program.name},
+        {requestBody: alert, securities: this.securities})
+  }
+
+  async createDestination(program){
+    const destination = alerts.buildDestination(program)
+    logger.Debug(destination)
+    return this.client.apis.Alerts
+      .CreateDestination(
+        {org_id: this.instance.uuid, destination_name: program.name},
+        {requestBody: destination, securities: this.securities})
+  }
+
+  async createTemplate(program){
+    program.instance = this.instance
+    const template = { name: program.name, body: alerts.templates.slack(program), isDefault: true }
+    logger.Debug(template)
+    return this.client.apis.Alerts
+      .CreateTemplate(
+        {org_id: this.instance.uuid, template_name: program.name},
+        {requestBody: template, securities: this.securities})
+  }
+}
+
+alerts = {
+  templates: {
+    slack({name, operator, keyword, column, channel, instance}) {
+      const template = JSON.parse(fs.readFileSync(path.join(__dirname, './openobserve/alerts/slack.json')));
+      template.channel = channel
+
+      const description = `*Alert details:*\n *${column}* column ${operator} *${keyword}* keyword`
+      template.text = description
+      template.blocks.push({type: 'section', text: {text: description, type: "mrkdwn"}})
+
+      const instanceInfo = `*Instance*:\n ${instance.url}`
+      template.blocks.push({type: 'section', text: {text: instanceInfo, type: "mrkdwn"}})
+
+      return template
+    }
+  },
+  buildDestination({url, name}){
+    return {
+      "headers": {
+      },
+      "method": "post",
+      "name": name,
+      "skip_tls_verify": false,
+      "template": name,
+      "url": url
+    }
+  },
+  buildAlert({from, name, keyword, sql, column, operator}){
+    return {
+      "condition": {
+        "column": "message",
+        "ignoreCase": true,
+        "isNumeric": true,
+        "operator": operator,
+        "value": keyword
+      },
+      "destination": name,
+      "duration": 0,
+      "frequency": 0,
+      "is_real_time": true,
+      "name": name,
+      "time_between_alerts": 0
+    }
+  }
+}
+
+search = {
+  buildQuery({from, size, start_time, end_time, sql}) {
+    query = {
+      query: {
+        from: parseInt(from),
+        size: parseInt(size),
+        sql: (sql || 'select * from logs')
+      }
+    }
+    if (start_time) query.query['start_time'] = parseInt(start_time)
+    if (end_time) query.query['end_time'] = parseInt(end_time)
+
+    return query
+  },
+  printJSON: logger.Info,
+
+  printLogs(response, key) {
+    Object
+      .entries(response.hits)
+      .map((item) => {
+        item[1].highlight = item[1]._timestamp.toString() == key;
+        return item;
+      })
+      .forEach(search.printLog)
+  },
+  printLog(hit) { console.log(search.formatHit(hit)) },
+  formatHit([id, hit]) {
+    const row = `[${search.toDate(hit._timestamp)}] ${hit._timestamp} ${hit.type} | ${hit.message}`
+    if (hit.highlight) {
+      return row.blue
+    } else {
+      return row
+    }
+  },
+  printReport(response, report) {
+    console.log(report.meta.title)
+    if (response.aggs)
+      console.table(
+        Object.entries(response.aggs.histogram).map((r) => { return r[1] }),
+        report.meta.columns
+      )
+  },
+  toDate(timestamp) {
+    if (timestamp)
+      return new Date(timestamp / 1000).toLocaleString()
+    else
+      "---"
+  },
+}
+
+module.exports = { SwaggerProxy, search };

package/lib/templates.js

@@ -0,0 +1,52 @@
+const fs = require('fs'),
+  path = require('path');
+
+const mustache = require('mustache');
+
+const logger = require('./logger');
+
+const tags = ['<%=', '=%>'];
+const allowedTemplateExtentions = ['.css', '.js', '.liquid', '.yml', '.xml', '.json', '.svg', '.graphql', '.html'];
+
+/**
+ * This function will replace tags with values from template data.
+ * Example: if tags are <%= and =%> and we have code like this:
+ *
+ * ---
+ * slug: <%= page_url =%>
+ * ---
+ *
+ * This function will replace `page_url` with value from templateData
+ * hash or it will **not** raise an error if the value is missing.
+ * @param {string} path - path to the file which will be processed.
+ **/
+const fillInTemplateValues = (filePath, templateData) => {
+  if (fs.existsSync(filePath) && qualifedForTemplateProcessing(filePath) && hasTemplateValues(templateData)) {
+    const fileBody = fs.readFileSync(filePath, 'utf8');
+    try {
+      return mustache.render(fileBody, templateData, {}, tags);
+    } catch (e) {
+      logger.Debug(e.message);
+      return fileBody;
+    }
+  } else {
+    return fs.createReadStream(filePath);
+  }
+};
+
+const qualifedForTemplateProcessing = filePath => {
+  return allowedTemplateExtentions.includes(path.extname(filePath));
+};
+
+const hasTemplateValues = templateData => {
+  try {
+    return Object.keys(templateData).length !== 0;
+  } catch (e) {
+    logger.Debug(e.message);
+    return false;
+  }
+};
+
+module.exports = {
+  fillInTemplateValues: fillInTemplateValues
+};

package/lib/updateNotifier.js

@@ -0,0 +1,8 @@
+// const updateNotifier = require("update-notifier");
+// Inform users of your package of updates in a non-intrusive way.
+// updateNotifier({
+//     pkg: pkg,
+// }).notify({
+//     defer: true,
+//     isGlobal: true,
+// });

package/lib/utils/valid-file-path.js

@@ -0,0 +1,21 @@
+const validCharacters = new RegExp([
+  '[^',   // Not the following characters:
+  '\\w',  // - Word characters
+  '\\s',  // - Whitespace
+  '\\-',  // - Dash
+  '_',
+  '~',
+  '@',
+  '%',
+  '\+',   // - Plus
+  '\\.',  // - Dot
+  '\\/',  // - Forward slash
+  '\\\\', // - Backward slash
+  '\\(',  // - Open parenthesis
+  '\\)',  // - Close parenthesis
+  "'",
+  '&',
+  ']'
+].join(''));
+
+module.exports = str => !validCharacters.test(str);

package/lib/validators/directoryEmpty.js

@@ -0,0 +1,12 @@
+const logger = require('../logger');
+const shell = require('shelljs');
+const noop = () => {};
+
+module.exports = ({ path, message = 'Directory is empty.', fail = noop }) => {
+  const directoryEmpty = shell.ls(path).length == 0;
+
+  if (directoryEmpty) {
+    fail();
+    logger.Error(message, { hideTimestamp: true });
+  }
+};

package/lib/validators/directoryExists.js

@@ -0,0 +1,12 @@
+const logger = require('../logger');
+const fs = require('fs');
+const noop = () => {};
+
+module.exports = ({ path, message = "Directory doesn't exist.", fail = noop }) => {
+  const directoryExists = fs.existsSync(path);
+
+  if (!directoryExists) {
+    fail();
+    logger.Error(message, { hideTimestamp: true });
+  }
+};

package/lib/validators/email.js

@@ -0,0 +1,8 @@
+const logger = require('../logger');
+const emailValidator = require('email-validator');
+
+module.exports = email => {
+  if (!emailValidator.validate(email)) {
+    logger.Error('Please provide valid email', { hideTimestamp: true });
+  }
+};

package/lib/validators/existence.js

@@ -0,0 +1,9 @@
+const logger = require('../logger');
+const noop = () => {};
+
+module.exports = ({ argumentName = 'all required arguments', argumentValue, fail = noop }) => {
+  if (typeof argumentValue === 'undefined') {
+    fail();
+    logger.Error(`Please provide ${argumentName}`, { hideTimestamp: true });
+  }
+};

package/lib/validators/index.js

@@ -0,0 +1,13 @@
+const existence = require('./existence');
+const url = require('./url');
+const email = require('./email');
+const directoryExists = require('./directoryExists');
+const directoryEmpty = require('./directoryEmpty');
+
+module.exports = {
+  existence,
+  url,
+  email,
+  directoryExists,
+  directoryEmpty
+};

package/lib/validators/url.js

@@ -0,0 +1,8 @@
+const logger = require('../logger');
+const isWebUri = require('valid-url').isWebUri;
+
+module.exports = url => {
+  if (!isWebUri(url)) {
+    logger.Error('Please provide valid URL', { hideTimestamp: true });
+  }
+};

package/lib/watch-files-extensions.js

@@ -0,0 +1,39 @@
+module.exports = [
+  'avif',
+  'br',
+  'css',
+  'csv',
+  'doc',
+  'docx',
+  'eot',
+  'gif',
+  'graphql',
+  'gz',
+  'html',
+  'ico',
+  'jpeg',
+  'jpg',
+  'js',
+  'json',
+  'liquid',
+  'map',
+  'mp3',
+  'mp4',
+  'ogg',
+  'otf',
+  'pdf',
+  'png',
+  'ppt',
+  'pptx',
+  'svg',
+  'ttf',
+  'txt',
+  'webm',
+  'webp',
+  'woff',
+  'woff2',
+  'xls',
+  'xlsx',
+  'yml',
+  'zip'
+];

package/lib/watch.js

@@ -0,0 +1,192 @@
+const fs = require('fs');
+const path = require('path');
+const chokidar = require('chokidar');
+const Queue = require('async/queue');
+const cloneDeep = require('lodash.clonedeep');
+const debounce = require('lodash.debounce');
+
+const Gateway = require('../lib/pos-proxy'),
+  logger = require('../lib/logger'),
+  templates = require('../lib/templates'),
+  settings = require('../lib/settings'),
+  dir = require('../lib/directories'),
+  files = require('../lib/files'),
+  livereload = require('livereload'),
+  watchFileExtensions = require('../lib/watch-files-extensions'),
+  manifestGenerateForAssets = require('./assets/manifest').manifestGenerateForAssets,
+  uploadFileFormData = require('./s3UploadFile').uploadFileFormData,
+  presignDirectory = require('./presignUrl').presignDirectory,
+  shouldBeSynced = require('../lib/shouldBeSynced');
+
+const filePathUnixified = (filePath) =>
+  filePath
+    .replace(/\\/g, '/')
+    .replace(new RegExp(`^${dir.APP}/`), '')
+    .replace(new RegExp(`^${dir.LEGACY_APP}/`), '');
+const moduleAssetRegex = new RegExp('^modules/\\w+/public/assets');
+let queue;
+let directUploadData;
+let manifestFilesToAdd = [];
+
+const isAssetsPath = (path) => path.startsWith('app/assets') || moduleAssetRegex.test(path);
+const enqueuePush = (filePath) => queue.push({ path: filePath, op: 'push' }, () => {});
+const enqueueDelete = (filePath) => queue.push({ path: filePath, op: 'delete' }, () => {});
+
+const getBody = (filePath, processTemplate) => {
+  if (processTemplate) {
+    const moduleTemplateData = templateData(filePath.split(path.sep)[1]);
+    return templates.fillInTemplateValues(filePath, moduleTemplateData);
+  } else {
+    return fs.createReadStream(filePath);
+  }
+};
+
+const templateData = (module) => settings.loadSettingsFileForModule(module);
+
+const pushFile = (gateway, syncedFilePath) => {
+  let filePath = filePathUnixified(syncedFilePath); // need path with / separators
+  const formData = {
+    path: filePath,
+    marketplace_builder_file_body: getBody(syncedFilePath, filePath.startsWith('modules')),
+  };
+
+  return gateway.sync(formData).then((body) => {
+    if (body && body.refresh_index) {
+      logger.Warn('[Sync] WARNING: Data schema was updated. It will take a while for the change to be applied.');
+    }
+
+    if (body) {
+      logger.Success(`[Sync] Synced: ${filePath}`);
+    }
+  });
+};
+
+const deleteFile = (gateway, syncedFilePath) => {
+  let filePath = filePathUnixified(syncedFilePath); // need path with / separators
+  const formData = {
+    path: filePath,
+    primary_key: filePath
+  };
+
+  return gateway.delete(formData).then(body => {
+    if (body) {
+      logger.Info(`[Sync] Deleted: ${filePath}`);
+    }
+  });
+};
+
+const pushFileDirectAssets = (gateway, syncedFilePath) => {
+  if (isAssetsPath(syncedFilePath)) {
+    sendAsset(gateway, syncedFilePath);
+    return Promise.resolve(true);
+  }
+  return pushFile(gateway, syncedFilePath);
+};
+
+const manifestSend = debounce(
+  (gateway) => {
+    const manifest = manifestGenerateForAssets(manifestFilesToAdd.slice());
+    logger.Debug(manifest);
+    gateway.sendManifest(manifest);
+    manifestFilesToAdd = [];
+  },
+  1000,
+  { maxWait: 1000 * 10 }
+);
+
+const manifestAddAsset = (path) => manifestFilesToAdd.push(path);
+
+const sendAsset = async (gateway, filePath) => {
+  try {
+    const data = cloneDeep(directUploadData);
+    const fileSubdir = filePath.startsWith('app/assets')
+      ? path.dirname(filePath).replace('app/assets', '')
+      : '/' + path.dirname(filePath).replace('/public/assets', '');
+    const key = data.fields.key.replace('assets/${filename}', `assets${fileSubdir}/\${filename}`);
+    data.fields.key = key;
+    logger.Debug(data);
+    await uploadFileFormData(filePath, data);
+    manifestAddAsset(filePath);
+    manifestSend(gateway);
+    logger.Success(`[Sync] Synced asset: ${filePath}`);
+  } catch (e) {
+    logger.Debug(e.message);
+    logger.Debug(e.stack);
+    logger.Error(`[Sync] Failed to sync: ${filePath}`);
+  }
+};
+
+const fetchDirectUploadData = async (instanceUrl, instanceKey, gateway) => {
+  const instanceId = (await gateway.getInstance()).id;
+  const remoteAssetsDir = `instances/${instanceId}/assets`;
+  const data = await presignDirectory(instanceUrl, instanceKey, remoteAssetsDir);
+  directUploadData = data;
+};
+
+const start = async (env, liveReload) => {
+  const program = {
+    email: env.CONSOLE_USER_EMAIL,
+    token: env.CONSOLE_USER_TOKEN,
+
+    instanceUuid: env.CONSOLE_USER_INSTANCE_UUID,
+    url: env.CONSOLE_USER_INSTANCE_URL,
+
+    key: env.CONSOLE_USER_KEY,
+    concurrency: env.CONCURRENCY,
+  };
+  const gateway = new Gateway(program);
+  const ignoreList = files.getIgnoreList();
+  const push = pushFileDirectAssets;
+  await fetchDirectUploadData(program.url, program.key, gateway);
+
+  let liveReloadServer;
+  if (liveReload) {
+    liveReloadServer = livereload.createServer({
+      exts: watchFileExtensions,
+      delay: 250,
+    });
+
+    liveReloadServer.watch(path.join(process.cwd(), '{app,modules}'));
+
+    logger.Info(`[LiveReload] Server started`);
+  }
+
+  const reload = () => liveReload && liveReloadServer.refresh(program.url);
+
+  queue = Queue((task, callback) => {
+    switch (task.op) {
+      case 'push':
+        push(gateway, task.path).then(reload).then(callback);
+        break;
+      case 'delete':
+        deleteFile(gateway, task.path).then(reload).then(callback);
+        break;
+    }
+  }, program.concurrency);
+
+  return gateway.ping().then(() => {
+    const directories = dir.toWatch();
+
+    if (directories.length === 0) {
+      logger.Error(`${dir.APP} or ${dir.MODULES} directory has to exist!`);
+    }
+
+    chokidar
+      .watch(directories, {
+        ignoreInitial: true,
+        awaitWriteFinish: {
+          stabilityThreshold: 500,
+          pollInterval: 100,
+        },
+      })
+      .on('change', (fp) => shouldBeSynced(fp, ignoreList) && enqueuePush(fp))
+      .on('add', (fp) => shouldBeSynced(fp, ignoreList) && enqueuePush(fp))
+      .on('unlink', (fp) => shouldBeSynced(fp, ignoreList) && enqueueDelete(fp));
+
+    logger.Info(`[Sync] Synchronizing changes to: ${program.url}`);
+  });
+};
+
+module.exports = {
+  start: start,
+};