ezekiel-cli 5.8.2

package/lib/audit/filters.js

@@ -0,0 +1,99 @@
+const fs = require('fs');
+const glob = require('fast-glob');
+
+// TODO: Cleanup after old stack is dead
+const filters = [
+  'active_class',
+  'already_favorite',
+  'attachments_visible_for',
+  'connections_for',
+  'connections_tooltip',
+  'custom_sanitize',
+  'custom_sanitizer',
+  'filter_text',
+  'find_collaborator',
+  'find_collaborators_for_user_transactables',
+  'find_collaborators_for_user',
+  'generate_url_with_user_token',
+  'generate_url',
+  'get_ckeditor_assets',
+  'get_data_contents',
+  'get_enquirer_confirmed_orders',
+  'get_enquirer_draft_orders',
+  'get_enquirer_orders',
+  'get_lister_orders',
+  'get_lowest_price_with_options',
+  'get_payment_gateway_id',
+  'group_rules_by_day',
+  'image_url',
+  'is_approved_collaborator',
+  'is_user_following',
+  'is_visible',
+  'location_path',
+  'lowest_full_price_with_cents_with_currency',
+  'lowest_full_price_without_cents_with_currency',
+  'lowest_price_with_cents_with_currency',
+  'lowest_price_without_cents_with_currency',
+  'meta_attr',
+  'number_of_minutes_to_time',
+  'number_of_minutes_until',
+  'pagination_links',
+  'parse_time_with_format',
+  'parse_time',
+  'parse_to_minute',
+  'price_with_cents_with_currency_as_cost',
+  'price_with_cents_with_currency',
+  'price_without_cents_with_currency',
+  'pricing_units_translation',
+  'query',
+  'render_price',
+  'request_parameter',
+  'search_box_for',
+  'search_button_for',
+  'shorten_url',
+  'soft_concat',
+  'space_listing_placeholder_path',
+  'strip_tags',
+  'tag_filter_link',
+  'timeago',
+  'to_money',
+  'to_time_from_str',
+  'total_entries',
+  'translate_property',
+  'user_message_create_path',
+  'widget_links'
+];
+
+const test = new RegExp(`\\| (${filters.join('|')})`);
+const _message = match => `[DEPRECATED FILTER] ${match}`;
+
+module.exports = {
+  audit: async () => {
+    let results = {};
+    const files = await glob('{app,modules,marketplace_builder}/**/*.liquid');
+
+    for (let file of files) {
+      const fileContents = fs.readFileSync(file, { encoding: 'utf8' });
+      const matches = fileContents.match(test);
+      const match = (matches && matches[1]) || false;
+
+      if (!match) {
+        continue;
+      }
+
+      const currentFiles = results[match] && results[match].files;
+      const updatedFiles = (currentFiles || []).concat(file);
+
+      results = {
+        ...results,
+        [match]: {
+          files: updatedFiles,
+          message: _message(match)
+        }
+      };
+    }
+
+    return results;
+  },
+  message: _message
+};

package/lib/audit/orphanedIncludes.js

@@ -0,0 +1,85 @@
+const fs = require('fs');
+const glob = require('fast-glob');
+const path = require('path');
+
+const logger = require('../logger');
+const { exec } = require('child_process');
+
+// const includes = /include (['"])([\w/-]*[\.]*[\w/-]*)\1/g;
+const includes = /(include|function) (['"])([\w/-]*[.]*[\w/-]*)/g;
+const variableInclude = /(include|function) ([\w]+)/;
+
+const normalizePath = (partialName) => {
+  if (partialName.startsWith('app/')) {
+    return findExistingFile([`app/views/partials/${partialName}.liquid`, `app/views/partials/_${partialName}.liquid`]);
+  }
+
+  if (partialName.startsWith('modules/')) {
+    // Module paths are included as modules/[modulename]/[includepath], rewrite it to private/public directories.
+    const includedFromPath = path.join(...partialName.split(path.sep).slice(0, 2));
+    const partialFilename = partialName.substr(includedFromPath.length + 1) + '.liquid';
+    return findExistingFile([
+      `${includedFromPath}/private/views/partials/${partialFilename}`,
+      `${includedFromPath}/private/views/partials/_${partialFilename}`,
+      `${includedFromPath}/public/views/partials/${partialFilename}`,
+      `${includedFromPath}/public/views/partials/_${partialFilename}`
+    ]);
+  }
+  return findExistingFile([`app/views/partials/${partialName}.liquid`, `app/views/partials/_${partialName}.liquid`]);
+};
+
+const findExistingFile = files => files.find(file => fs.existsSync(file));
+
+const findPartials = async () => {
+  let foundVariableInclude = false;
+  let foundVariableIncludeFile = '';
+
+  const files = await glob('{app,modules}/**/*.liquid');
+  const foundPartials = files.reduce((partials, file) => {
+    const fileContents = fs.readFileSync(file, { encoding: 'utf8' });
+
+    // Drop out if variable include found.
+    // if (fileContents.match(variableInclude)) {
+    const includeAsVar = variableInclude.exec(fileContents)
+    if(includeAsVar) {
+      foundVariableInclude = true;
+      foundVariableIncludeFile = `${file} (${includeAsVar[0]})`;
+      return partials;
+    }
+    while (includesMatches = includes.exec(fileContents)) {
+      partials.add(normalizePath(includesMatches[3]));
+    }
+
+    return partials;
+  }, new Set());
+
+  return { foundPartials, foundVariableInclude, foundVariableIncludeFile };
+};
+
+module.exports = {
+  audit: async () => {
+    let results = {};
+
+    const { foundPartials, foundVariableInclude, foundVariableIncludeFile } = await findPartials();
+    if (foundVariableInclude) {
+      logger.Info(`Found partial included using a variable in: ${foundVariableIncludeFile}`);
+      logger.Info('Orphaned partials check is disabled.');
+      return {};
+    }
+
+    // Look up all partials.
+    let partials = await glob('{app,modules}/**/partials/**/*.liquid');
+    // Find never included partials.
+    const notIncludedPartials = partials.filter(partial => !foundPartials.has(partial));
+    if (notIncludedPartials.length > 0) {
+      results = {
+        'Not included partials': {
+          files: notIncludedPartials,
+          message: 'Found partial that are never included'
+        }
+      };
+    }
+
+    return results;
+  }
+};

package/lib/audit/tags.js

@@ -0,0 +1,69 @@
+const fs = require('fs');
+const glob = require('fast-glob');
+
+// TODO: Cleanup after old stack is dead
+const tags = [
+  'cache_for',
+  'content_holder_tag_for_path',
+  'content_holder',
+  'dropdown_menu_block',
+  'error',
+  'execute_query',
+  'featured_items',
+  'fields_for',
+  'form_tag',
+  'hint',
+  'input_field_error',
+  'input_field',
+  'input',
+  'label',
+  'languages_select',
+  'link_to_add_association',
+  'link_to_association',
+  'link_to_remove_association',
+  'liquid_select',
+  'meta_description',
+  'placeholder',
+  'query_graph',
+  'render_featured_items',
+  'render_form',
+  'select',
+  'submit',
+  'title',
+  'transactable_type_select',
+  'will_paginate'
+];
+
+const test = new RegExp(`{%-?\\s*(${tags.join('|')})\\s`);
+const _message = match => `[DEPRECATED TAG] ${match}`;
+
+module.exports = {
+  audit: async () => {
+    let results = {};
+    const files = await glob('{app,modules,marketplace_builder}/**/*.liquid');
+
+    for (let file of files) {
+      const fileContents = fs.readFileSync(file, { encoding: 'utf8' });
+      const matches = fileContents.match(test);
+      const match = (matches && matches[1]) || false;
+
+      if (!match) {
+        continue;
+      }
+
+      const currentFiles = results[match] && results[match].files;
+      const updatedFiles = (currentFiles || []).concat(file);
+
+      results = {
+        ...results,
+        [match]: {
+          files: updatedFiles,
+          message: _message(match)
+        }
+      };
+    }
+
+    return results;
+  },
+  message: _message
+};

package/lib/audit.js

@@ -0,0 +1,47 @@
+const chalk = require('chalk');
+
+const logger = require('../lib/logger');
+
+const tags = require('../lib/audit/tags'),
+  filters = require('../lib/audit/filters'),
+  detailed = require('../lib/audit/detailed'),
+  extensions = require('../lib/audit/extensions'),
+  duplicateFile = require('../lib/audit/duplicateFile'),
+  fileName = require('./audit/fileName'),
+  orphanedIncludes = require('../lib/audit/orphanedIncludes');
+
+const auditors = [tags, filters, detailed, extensions, duplicateFile, fileName, orphanedIncludes];
+
+const printReport = results => {
+  if (!results) {
+    return;
+  }
+
+  for (let ruleName in results) {
+    const result = results[ruleName];
+    const filesFormatted = result.files.join('\n\t');
+
+    logger.Warn(`${chalk.yellow(result.message)} \nAffected files:\n\t${filesFormatted}`, { hideTimestamp: true });
+  }
+};
+
+const numberOfOffences = reports => {
+  reports = reports.filter(report => Object.keys(report).length > 0);
+  if (reports.length == 0) {
+    return 0;
+  }
+
+  return reports.map(report => Object.keys(report).length).reduce((total, num) => total + num);
+};
+
+const run = async () => {
+  return Promise.all(auditors.map(auditor => auditor.audit())).then(reports => {
+    const offences = numberOfOffences(reports);
+    logger.Warn(`[Audit] ${offences} rule${offences === 1 ? '' : 's'} detected issues.`, { hideTimestamp: true });
+    reports.forEach(printReport);
+  });
+};
+
+module.exports = {
+  run: run
+};

package/lib/data/fetchFiles.js

@@ -0,0 +1,93 @@
+const fs = require('fs'),
+  request = require('request'),
+  url = require('url'),
+  path = require('path'),
+  shell = require('shelljs'),
+  flatten = require('lodash.flatten'),
+  Queue = require('async/queue'),
+  logger = require('./../logger');
+const CONCURRENCY = 12;
+
+const download = (uri, filename) => {
+  return new Promise((resolve, reject) => {
+    request.head(uri, (err, res, body) => {
+      if (err) {
+        logger.Warn(err);
+        reject(err);
+      } else {
+        request(uri)
+          .pipe(fs.createWriteStream(filename))
+          .on('close', () => resolve(filename));
+      }
+    });
+  });
+};
+
+const filenameForUrl = uri => {
+  return path.basename(url.parse(uri).pathname);
+};
+
+const updateItem = (item, newUrl) => {
+  return { url: newUrl, name: item.name };
+};
+
+const filterItemsFromModel = (model, field) => {
+  return (model[field] || []).filter(item => !!item.url);
+};
+
+const tmpFileMeta = (item, field) => {
+  const filename = filenameForUrl(item.url);
+  const dir = `tmp/${field}/${item.id}`;
+  const tmpFilename = `${dir}/${filename}`;
+
+  return { filename: tmpFilename, dir: dir };
+};
+
+const queue = Queue((task, callback) => {
+  download(task.url, task.tmpFilename).then(callback);
+}, CONCURRENCY);
+
+const enqueue = item => {
+  queue.push(item, () => {});
+};
+
+const addTmpFilename = (item) => {
+  const tmpFile = tmpFileMeta(item, item.field);
+  shell.mkdir('-p', tmpFile.dir);
+
+  item.tmpFilename = tmpFile.filename;
+  return item;
+};
+
+const downloadFiles = (collection) => {
+  collection
+    .map(addTmpFilename)
+    .map(enqueue);
+};
+
+async function fetchFiles(model) {
+  const fields = ['attachments', 'images'];
+  const filesForDownload = flatten(
+    fields.map(field => {
+    return filterItemsFromModel(model, field)
+      .map(item => {
+        return { url: item.url, id: item.id, field: field };
+      });
+    })
+  );
+  downloadFiles(filesForDownload);
+
+  for (let field of fields) {
+    const items = filterItemsFromModel(model,field)
+      .map(item => {
+        return { url: item.url, id: item.id, field: field, name: item.name };
+      })
+      .map(item => {
+        return updateItem(item, addTmpFilename(item).tmpFilename);
+      });
+    model[field] = items;
+  }
+  return model;
+}
+
+module.exports = fetchFiles;

package/lib/data/isValidJSON.js

@@ -0,0 +1,12 @@
+// TODO: Move to validators...
+const isValidJSON = data => {
+  try {
+    JSON.parse(data);
+    // TODO check required keyes
+    return true;
+  } catch (e) {
+    return false;
+  }
+};
+
+module.exports = isValidJSON;

package/lib/data/uploadFiles.js

@@ -0,0 +1,92 @@
+const { v4: uuidv4 } = require('uuid'),
+  path = require('path'),
+  mapLimit = require('async/mapLimit'),
+  logger = require('./../logger'),
+  presignUrl = require('./../presignUrl').presignUrl,
+  uploadFile = require('./../s3UploadFile').uploadFile;
+
+const filenameForUrl = uri => {
+  return path.basename(uri);
+};
+
+const askForSlot = (tmpFileName, filename) => {
+  logger.Debug('askForSlot');
+  const s3FileName = `uploads/${uuidv4()}/${filename}`;
+
+  return presignUrl(s3FileName, tmpFileName);
+};
+
+const updateItem = (item, newUrl) => {
+  return Object.assign(item, { url: newUrl });
+};
+
+const uploadFiles = async(items) => {
+  try {
+    const result = [];
+    for(const item of items) {
+      const filename = filenameForUrl(item.url);
+      const data = await askForSlot(item.url, filename);
+      await uploadFile(item.url, data.uploadUrl);
+      result.push(updateItem(item, data.accessUrl));
+    }
+    return result;
+  } catch(e) {
+    logger.Warn(e);
+  }
+};
+
+const processModel = async(model) => {
+  const fields = ['attachments', 'images'];
+  for (let field of fields) {
+    const items = (model[field] || [])
+      .filter(item => item != null && !!item.url);
+
+    model[field] = await uploadFiles(items);
+  }
+
+  logger.Debug('processModel');
+  return model;
+};
+
+const processUser = async(user) => {
+  user['profiles'] = await processFilesFor(user.profiles);
+  return user;
+};
+
+const CONCURRENCY = 4;
+const processFilesFor = (collection) => {
+  let i = 0;
+  return new Promise((resolve, reject) => {
+    mapLimit(collection, CONCURRENCY, async (item) => {
+      i++;
+      logger.Debug(i);
+      return await processModel(item);
+    }, (err, results) => {
+      if (err) throw err;
+      return resolve(results);
+    });
+  });
+};
+
+const processFilesForUsers = async(collection) => {
+  const result = [];
+  for (const item of collection) {
+    result.push(await processUser(item));
+  }
+  return result;
+};
+
+async function processData(data) {
+  if (data.transactables) data.transactables = await processFilesFor(data.transactables);
+  if (data.models) data.models = await processFilesFor(data.models);
+  if (data.users) data.users = await processFilesForUsers(data.users);
+
+  logger.Debug('processData');
+  return data;
+}
+
+const transform = data => {
+  return processData(data);
+};
+
+module.exports = transform;

package/lib/data/waitForStatus.js

@@ -0,0 +1,31 @@
+const logger = require('../logger');
+
+const waitForStatus = (statusCheck, pendingStatus, successStatus) => {
+  return new Promise((resolve, reject) => {
+    const getStatus = () => {
+      statusCheck().then(response => {
+        const status = response.status;
+        if (status) {
+          // if (status === 'pending' || status === 'ready_for_export') {
+          if (status === pendingStatus) {
+            setTimeout(getStatus, 1500);
+          // } else if (status === 'done' || status === 'success') {
+          } else if (status === successStatus) {
+            resolve(response);
+          } else {
+            reject(response);
+          }
+        } else {
+          reject(response);
+        }
+      }).catch((error) => {
+        logger.Debug('[ERR] waitForStatus did not receive `status` in response object', error);
+        reject(error);
+      });
+    };
+    getStatus();
+  });
+};
+
+
+module.exports = waitForStatus;

package/lib/deploy/directAssetsUploadStrategy.js

@@ -0,0 +1,57 @@
+const ora = require('ora');
+const { performance } = require('perf_hooks');
+
+const archive = require('../archive');
+const push = require('../push');
+
+const files = require('../files');
+const Gateway = require('../pos-proxy');
+const assets = require("../assets");
+
+const duration = require('../duration');
+const logger = require('../logger');
+
+const createArchive = async (env) => {
+  return await archive.makeArchive(env, { withoutAssets: true });
+};
+
+const uploadArchive = async (env) => {
+  const res = await push.push(env);
+  if (!res) throw 'Server did not accept release file.';
+};
+
+const deployAssets = async (authData) => {
+  const assetsToDeploy = await files.getAssets();
+  if (assetsToDeploy.length === 0) {
+    logger.Warn('There are no assets to deploy, skipping.');
+    return;
+  }
+  await assets.deployAssets(new Gateway(authData), authData);
+};
+
+const strategy = async ({ env, authData, params }) => {
+  try {
+    process.env.FORCE_COLOR = true;
+    const instanceUuid = env.CONSOLE_USER_INSTANCE_UUID;
+    const msg = (instanceUuid) => `Deploying to: ${instanceUuid}`;
+    const numberOfFiles = await createArchive(env);
+    const spinner = ora({
+      text: msg(instanceUuid),
+      stream: process.stdout,
+      spinner: 'bouncingBar',
+    }).start();
+    const t0 = performance.now();
+    if (numberOfFiles > 0) {
+      await uploadArchive(env);
+    } else {
+      logger.Warn('There are no files in release file, skipping.');
+    }
+    await deployAssets(authData);
+
+    spinner.succeed(`Deploy succeeded after ${duration(t0, performance.now())}`);
+  } catch (e) {
+    logger.Error(`Deploy failed. ${e}`);
+  }
+};
+
+module.exports = strategy;

package/lib/deploy/strategy.js

@@ -0,0 +1,8 @@
+const strategies = {
+  // default: require('./defaultStrategy'),
+  directAssetsUpload: require('./directAssetsUploadStrategy'),
+};
+
+module.exports = {
+  run: ({ strategy, opts }) => strategies[strategy](opts),
+};

package/lib/directories.js

@@ -0,0 +1,23 @@
+const fs = require('fs');
+
+const app = {
+  APP: 'app',
+  LEGACY_APP: 'marketplace_builder',
+  MODULES: 'modules'
+};
+
+const internal = {
+  TMP: '.tmp'
+};
+
+const computed = {
+  ALLOWED: [app.APP, app.LEGACY_APP, app.MODULES]
+};
+
+const methods = {
+  toWatch: () => computed.ALLOWED.filter(fs.existsSync),
+  currentApp: () => [app.APP, app.LEGACY_APP].filter(fs.existsSync)[0],
+  available: () => computed.ALLOWED.filter(fs.existsSync)
+};
+
+module.exports = Object.assign({}, app, internal, computed, methods);

package/lib/downloadFile.js

@@ -0,0 +1,16 @@
+const fs = require('fs'),
+  https = require('https');
+
+module.exports = (url, fileName) => {
+  return new Promise((resolve, reject) => {
+    let file = fs.createWriteStream(fileName).on('close', () => resolve());
+    https.get(url, response => {
+      response.pipe(file);
+      file
+        .on('finish', () => {
+          file.close(resolve);
+        })
+        .on('error', reject);
+    });
+  });
+};

package/lib/duration.js

@@ -0,0 +1,7 @@
+  const formatMMSS = s => (s - (s %= 60)) / 60 + (9 < s ? ':' : ':0') + s;
+  const duration = (t0, t1) => {
+    const duration = Math.round((t1 - t0) / 1000);
+    return formatMMSS(duration);
+  };
+
+module.exports = duration;