@automattic/vip 2.11.0 → 2.11.2

package/package/dist/bin/vip-import-sql.js

@@ -0,0 +1,580 @@
+#!/usr/bin/env node
+
+/**
+ * 
+ * @format
+ */
+
+/**
+ * External dependencies
+ */
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.gates = gates;
+exports.validateAndGetTableNames = validateAndGetTableNames;
+
+var _graphqlTag = _interopRequireDefault(require("graphql-tag"));
+
+var _cliColumns = _interopRequireDefault(require("cli-columns"));
+
+var _chalk = _interopRequireDefault(require("chalk"));
+
+var _debug = _interopRequireDefault(require("debug"));
+
+var _enquirer = require("enquirer");
+
+var _command = _interopRequireDefault(require("../lib/cli/command"));
+
+var _dbFileImport = require("../lib/site-import/db-file-import");
+
+var _status = require("../lib/site-import/status");
+
+var _clientFileUploader = require("../lib/client-file-uploader");
+
+var _tracker = require("../lib/tracker");
+
+var _sql = require("../lib/validations/sql");
+
+var _siteType = require("../lib/validations/site-type");
+
+var _searchAndReplace = require("../lib/search-and-replace");
+
+var _api = _interopRequireDefault(require("../lib/api"));
+
+var exit = _interopRequireWildcard(require("../lib/cli/exit"));
+
+var _lineByLine = require("../lib/validations/line-by-line");
+
+var _format = require("../lib/cli/format");
+
+var _progress = require("../lib/cli/progress");
+
+var _isMultiSite = require("../lib/validations/is-multi-site");
+
+var _rollbar = require("../lib/rollbar");
+
+function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
+
+function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+/**
+ * Internal dependencies
+ */
+const appQuery = `
+	id,
+	name,
+	type,
+	organization { id, name },
+	environments{
+		id
+		appId
+		type
+		name
+		launched
+		isK8sResident
+		syncProgress { status }
+		primaryDomain { name }
+		importStatus {
+			dbOperationInProgress
+			importInProgress
+		}
+		wpSites {
+			nodes {
+				homeUrl
+				id
+			}
+		}
+	}
+`;
+const START_IMPORT_MUTATION = (0, _graphqlTag.default)`
+	mutation StartImport($input: AppEnvironmentImportInput) {
+		startImport(input: $input) {
+			app {
+				id
+				name
+			}
+			message
+			success
+		}
+	}
+`;
+const debug = (0, _debug.default)('@automattic/vip:bin:vip-import-sql');
+const SQL_IMPORT_PREFLIGHT_PROGRESS_STEPS = [{
+  id: 'replace',
+  name: 'Performing Search and Replace'
+}, {
+  id: 'upload',
+  name: 'Uploading file'
+}, {
+  id: 'queue_import',
+  name: 'Queueing Import'
+}];
+
+async function gates(app, env, fileName) {
+  const {
+    id: envId,
+    appId
+  } = env;
+
+  const track = _tracker.trackEventWithEnv.bind(null, appId, envId);
+
+  if (!(0, _dbFileImport.currentUserCanImportForApp)(app)) {
+    await track('import_sql_command_error', {
+      error_type: 'unauthorized'
+    });
+    exit.withError('The currently authenticated account does not have permission to perform a SQL import.');
+  }
+
+  if (!(0, _dbFileImport.isSupportedApp)(app)) {
+    await track('import_sql_command_error', {
+      error_type: 'unsupported-app'
+    });
+    exit.withError('The type of application you specified does not currently support SQL imports.');
+  }
+
+  try {
+    await (0, _clientFileUploader.checkFileAccess)(fileName);
+  } catch (err) {
+    await track('import_sql_command_error', {
+      error_type: 'sqlfile-unreadable'
+    });
+    exit.withError(`File '${fileName}' does not exist or is not readable.`);
+  }
+
+  if (!(await (0, _clientFileUploader.isFile)(fileName))) {
+    await track('import_sql_command_error', {
+      error_type: 'sqlfile-notfile'
+    });
+    exit.withError(`Path '${fileName}' is not a file.`);
+  }
+
+  const fileSize = await (0, _clientFileUploader.getFileSize)(fileName);
+
+  if (!fileSize) {
+    await track('import_sql_command_error', {
+      error_type: 'sqlfile-empty'
+    });
+    exit.withError(`File '${fileName}' is empty.`);
+  }
+
+  const maxFileSize = env !== null && env !== void 0 && env.launched ? _dbFileImport.SQL_IMPORT_FILE_SIZE_LIMIT_LAUNCHED : _dbFileImport.SQL_IMPORT_FILE_SIZE_LIMIT;
+
+  if (fileSize > maxFileSize) {
+    await track('import_sql_command_error', {
+      error_type: 'sqlfile-toobig',
+      file_size: fileSize,
+      launched: !!(env !== null && env !== void 0 && env.launched)
+    });
+    exit.withError(`The sql import file size (${fileSize} bytes) exceeds the limit (${maxFileSize} bytes).` + (env.launched ? ' Note: This limit is lower for launched environments to maintain site stability.' : '') + '\n\nPlease split it into multiple files or contact support for assistance.');
+  }
+
+  if (!(env !== null && env !== void 0 && env.importStatus)) {
+    await track('import_sql_command_error', {
+      error_type: 'empty-import-status'
+    });
+    exit.withError('Could not determine the import status for this environment. Check the app/environment and if the problem persists, contact support for assistance');
+  }
+
+  const {
+    importStatus: {
+      dbOperationInProgress,
+      importInProgress
+    }
+  } = env;
+
+  if (importInProgress) {
+    await track('import_sql_command_error', {
+      error_type: 'existing-import'
+    });
+    exit.withError('There is already an import in progress.\n\nYou can view the status with command:\n    vip import sql status');
+  }
+
+  if (dbOperationInProgress) {
+    await track('import_sql_command_error', {
+      error_type: 'existing-dbop'
+    });
+    exit.withError('There is already a database operation in progress. Please try again later.');
+  }
+} // Command examples for the `vip import sql` help prompt
+
+
+const examples = [// `sql` subcommand
+{
+  usage: 'vip import sql @mysite.develop <file.sql>',
+  description: 'Import the given SQL file to your site'
+}, // `search-replace` flag
+{
+  usage: 'vip import sql @mysite.develop <file.sql> --search-replace="from,to"',
+  description: 'Perform a Search and Replace, then import the replaced file to your site.\n' + '       * Ensure there are no spaces between your search-replace parameters'
+}, // `in-place` flag
+{
+  usage: 'vip import sql @mysite.develop <file.sql> --search-replace="from,to" --in-place',
+  description: 'Search and Replace on the input <file.sql>, then import the replaced file to your site'
+}, // `output` flag
+{
+  usage: 'vip import sql @mysite.develop <file.sql> --search-replace="from,to" --output="<output.sql>"',
+  description: 'Output the performed Search and Replace to the specified output file, then import the replaced file to your site\n' + '       * Has no effect when the `in-place` flag is used'
+}, // `sql status` subcommand
+{
+  usage: 'vip import sql status @mysite.develop',
+  description: 'Check the status of the most recent import. If an import is running, this will poll until it is complete.'
+}];
+
+const promptToContinue = async ({
+  launched,
+  formattedEnvironment,
+  track,
+  domain
+}) => {
+  console.log();
+  const promptToMatch = domain.toUpperCase();
+  const promptResponse = await (0, _enquirer.prompt)({
+    type: 'input',
+    name: 'confirmedDomain',
+    message: `You are about to import the above tables into a ${launched ? 'launched' : 'un-launched'} ${formattedEnvironment} site ${_chalk.default.yellow(domain)}.\nType '${_chalk.default.yellow(promptToMatch)}' (without the quotes) to continue:\n`
+  });
+
+  if (promptResponse.confirmedDomain !== promptToMatch) {
+    await track('import_sql_unexpected_tables');
+    exit.withError('The input did not match the expected environment label. Import aborted.');
+  }
+};
+
+async function validateAndGetTableNames({
+  skipValidate,
+  appId,
+  envId,
+  fileNameToUpload
+}) {
+  const validations = [_sql.staticSqlValidations, _siteType.siteTypeValidations];
+
+  if (skipValidate) {
+    console.log('Skipping SQL file validation.');
+    return [];
+  }
+
+  try {
+    await (0, _lineByLine.fileLineValidations)(appId, envId, fileNameToUpload, validations);
+  } catch (validateErr) {
+    console.log('');
+    exit.withError(`${validateErr.message}
+If you are confident the file does not contain unsupported statements, you can retry the command with the ${_chalk.default.yellow('--skip-validate')} option.
+`);
+  } // this can only be called after static validation of the SQL file
+
+
+  return (0, _sql.getTableNames)();
+}
+
+const displayPlaybook = ({
+  launched,
+  tableNames,
+  searchReplace,
+  fileName,
+  domain,
+  formattedEnvironment,
+  isMultiSite,
+  app
+}) => {
+  console.log();
+  console.log(`  importing: ${_chalk.default.blueBright(fileName)}`);
+  console.log(`         to: ${_chalk.default.cyan(domain)}`);
+  console.log(`       site: ${app.name} (${formattedEnvironment})`);
+
+  if (searchReplace !== null && searchReplace !== void 0 && searchReplace.length) {
+    const output = (from, to) => {
+      const message = `        s-r: ${_chalk.default.blue(from)} -> ${_chalk.default.blue(to)}`;
+      console.log(message);
+    };
+
+    (0, _format.formatSearchReplaceValues)(searchReplace, output);
+  }
+
+  let siteArray = [];
+
+  if (isMultiSite) {
+    var _app$environments$, _app$environments$$wp;
+
+    // eslint-disable-next-line no-multi-spaces
+    console.log(`  multisite: ${isMultiSite.toString()}`);
+    siteArray = app === null || app === void 0 ? void 0 : (_app$environments$ = app.environments[0]) === null || _app$environments$ === void 0 ? void 0 : (_app$environments$$wp = _app$environments$.wpSites) === null || _app$environments$$wp === void 0 ? void 0 : _app$environments$$wp.nodes;
+  }
+
+  if (!tableNames.length) {
+    debug('Validation was skipped, no playbook information will be displayed');
+  } else {
+    // output the table names
+    console.log();
+
+    if (!isMultiSite) {
+      console.log('Below are a list of Tables that will be imported by this process:');
+      console.log((0, _cliColumns.default)(tableNames));
+    } else {
+      var _siteArray;
+
+      // we have siteArray from the API, use it and the table names together
+      if (siteArray === 'undefined' || !siteArray) {
+        console.log(_chalk.default.yellowBright('Unable to determine the subsites affected by this import, please proceed only if you are confident on the contents in the import file.'));
+        return;
+      } else if (!((_siteArray = siteArray) !== null && _siteArray !== void 0 && _siteArray.length)) {
+        throw new Error('There were no sites in your multisite installation');
+      }
+
+      const multiSiteBreakdown = siteArray.map(wpSite => {
+        let siteRegex;
+
+        if (wpSite.id === 1) {
+          siteRegex = /wp_[a-z]+/i;
+        } else {
+          siteRegex = new RegExp(`wp_${wpSite.id}_[a-z]+`, 'i');
+        }
+
+        const tableNamesInGroup = tableNames.filter(name => siteRegex.test(name));
+        return {
+          id: wpSite.id,
+          url: wpSite.homeUrl,
+          tables: tableNamesInGroup
+        };
+      });
+
+      if (launched) {
+        console.log(_chalk.default.yellowBright('You are updating tables in a launched multi site installation. Sites in the same network may have their performance impacted by this operation.'));
+      }
+
+      console.log(_chalk.default.yellow('The following sites will be affected by the import:'));
+      multiSiteBreakdown.map(siteObject => {
+        console.log();
+        console.log(_chalk.default.blueBright(`Blog with ID ${siteObject.id} and URL ${siteObject.url} will import the following tables:`));
+        console.log((0, _cliColumns.default)(siteObject.tables));
+      });
+    }
+  }
+};
+
+(0, _command.default)({
+  appContext: true,
+  appQuery,
+  envContext: true,
+  requiredArgs: 1,
+  module: 'import-sql',
+  requireConfirm: 'Are you sure you want to import the contents of the provided SQL file?',
+  skipConfirmPrompt: true
+}).command('status', 'Check the status of the current running import').option('skip-validate', 'Do not perform pre-upload file validation. If unsupported entries are present, the import is likely to fail').option('search-replace', 'Perform Search and Replace on the specified SQL file').option('in-place', 'Search and Replace explicitly on the given input file').option('output', 'Specify the replacement output file for Search and Replace', 'process.stdout').examples(examples).argv(process.argv, async (arg, opts) => {
+  var _env$primaryDomain;
+
+  const {
+    app,
+    env
+  } = opts;
+  let {
+    skipValidate,
+    searchReplace
+  } = opts;
+  const {
+    id: envId,
+    appId
+  } = env;
+  const [fileName] = arg;
+  const isMultiSite = await (0, _isMultiSite.isMultiSiteInSiteMeta)(appId, envId);
+  let fileMeta = await (0, _clientFileUploader.getFileMeta)(fileName);
+
+  if (fileMeta.isCompressed) {
+    console.log(_chalk.default.yellowBright('You are importing a compressed file. Validation and search-replace operation will be skipped.'));
+    skipValidate = true;
+    searchReplace = undefined;
+  }
+
+  debug('Options: ', opts);
+  debug('Args: ', arg);
+
+  const track = _tracker.trackEventWithEnv.bind(null, appId, envId);
+
+  await track('import_sql_command_execute'); // // halt operation of the import based on some rules
+
+  await gates(app, env, fileName); // Log summary of import details
+
+  const domain = env !== null && env !== void 0 && (_env$primaryDomain = env.primaryDomain) !== null && _env$primaryDomain !== void 0 && _env$primaryDomain.name ? env.primaryDomain.name : `#${env.id}`;
+  const formattedEnvironment = (0, _format.formatEnvironment)(opts.env.type);
+  const launched = opts.env.launched;
+  let fileNameToUpload = fileName; // SQL file validations
+
+  const tableNames = await validateAndGetTableNames({
+    skipValidate,
+    appId,
+    envId,
+    fileNameToUpload
+  }); // display playbook of what will happen during execution
+
+  displayPlaybook({
+    launched,
+    tableNames,
+    searchReplace,
+    fileName,
+    domain,
+    formattedEnvironment,
+    isMultiSite,
+    app
+  }); // PROMPT TO PROCEED WITH THE IMPORT
+
+  await promptToContinue({
+    launched,
+    formattedEnvironment,
+    track,
+    domain
+  });
+  /**
+   * =========== WARNING =============
+   *
+   * NO `console.log` after this point!
+   * Yes, even inside called functions.
+   * It will break the progress printing.
+   *
+   * =========== WARNING =============
+   */
+
+  const progressTracker = new _progress.ProgressTracker(SQL_IMPORT_PREFLIGHT_PROGRESS_STEPS);
+  let status = 'running';
+
+  const setProgressTrackerPrefixAndSuffix = () => {
+    progressTracker.prefix = `
+=============================================================
+Processing the SQL import for your environment...
+`;
+    progressTracker.suffix = `\n${(0, _format.getGlyphForStatus)(status, progressTracker.runningSprite)} ${status === 'running' ? 'Loading remaining steps' : ''}`; // TODO: maybe use progress tracker status
+  };
+
+  const failWithError = failureError => {
+    status = 'failed';
+    setProgressTrackerPrefixAndSuffix();
+    progressTracker.stopPrinting();
+    progressTracker.print({
+      clearAfter: true
+    });
+
+    _rollbar.rollbar.error(failureError);
+
+    exit.withError(failureError);
+  };
+
+  progressTracker.startPrinting(setProgressTrackerPrefixAndSuffix); // Run Search and Replace if the --search-replace flag was provided
+
+  if (searchReplace && searchReplace.length) {
+    progressTracker.stepRunning('replace');
+    const {
+      outputFileName
+    } = await (0, _searchAndReplace.searchAndReplace)(fileName, searchReplace, {
+      isImport: true,
+      inPlace: opts.inPlace,
+      output: true
+    });
+
+    if (typeof outputFileName !== 'string') {
+      progressTracker.stepFailed('replace');
+      return failWithError('Unable to determine location of the intermediate search & replace file.');
+    }
+
+    fileNameToUpload = outputFileName;
+    fileMeta = await (0, _clientFileUploader.getFileMeta)(fileNameToUpload);
+    progressTracker.stepSuccess('replace');
+  } else {
+    progressTracker.stepSkipped('replace');
+  }
+
+  progressTracker.stepRunning('upload'); // Call the Public API
+
+  const api = await (0, _api.default)();
+  const startImportVariables = {};
+
+  const progressCallback = percentage => {
+    progressTracker.setUploadPercentage(percentage);
+  };
+
+  fileMeta.fileName = fileNameToUpload;
+
+  try {
+    const {
+      fileMeta: {
+        basename
+      },
+      md5,
+      result
+    } = await (0, _clientFileUploader.uploadImportSqlFileToS3)({
+      app,
+      env,
+      fileMeta,
+      progressCallback
+    });
+    startImportVariables.input = {
+      id: app.id,
+      environmentId: env.id,
+      basename: basename,
+      md5: md5,
+      searchReplace: []
+    };
+
+    if (searchReplace) {
+      let pairs = searchReplace;
+
+      if (!Array.isArray(pairs)) {
+        pairs = [searchReplace];
+      } // determine all the replacements required
+
+
+      const replacementsArr = pairs.map(pair => pair.split(',').map(str => str.trim()));
+      startImportVariables.input.searchReplace = replacementsArr.map(arr => {
+        return {
+          from: arr[0],
+          to: arr[1]
+        };
+      });
+    }
+
+    debug({
+      basename,
+      md5,
+      result,
+      startImportVariables
+    });
+    debug('Upload complete. Initiating the import.');
+    progressTracker.stepSuccess('upload');
+    await track('import_sql_upload_complete');
+  } catch (uploadError) {
+    await track('import_sql_command_error', {
+      error_type: 'upload_failed',
+      upload_error: uploadError
+    });
+    progressTracker.stepFailed('upload');
+    return failWithError(uploadError);
+  } // Start the import
+
+
+  try {
+    const startImportResults = await api.mutate({
+      mutation: START_IMPORT_MUTATION,
+      variables: startImportVariables
+    });
+    debug({
+      startImportResults
+    });
+  } catch (gqlErr) {
+    progressTracker.stepFailed('queue_import');
+    await track('import_sql_command_error', {
+      error_type: 'StartImport-failed',
+      gql_err: gqlErr
+    });
+    progressTracker.stepFailed('queue_import');
+    return failWithError(`StartImport call failed: ${gqlErr}`);
+  }
+
+  progressTracker.stepSuccess('queue_import');
+  await (0, _status.importSqlCheckStatus)({
+    app,
+    env,
+    progressTracker
+  });
+});