@corva/create-app 0.39.0-rc.0 → 0.40.0-0

package/README.md

@@ -26,6 +26,7 @@ Commands:
   create [options] [project-directory]                 Create a new app
   zip [options] <project-directory> [patterns...]      Bundle app
   release [options] <project-directory> [patterns...]  Release app
+  rerun [options] <project-directory>                  Rerun app
   help [command]                                       display help for command
 ```
 
@@ -199,4 +200,4 @@ create-corva-app release test-app --bump-version=patch
 
 ```sh
 create-corva-app release test-app --bump-version=4.2.0
-```
+```

package/lib/flows/lib/api.js

@@ -0,0 +1,225 @@
+const axios = require('axios');
+
+ /**
+   * Connect to Corva API
+   *
+   */
+class Api {
+  constructor(envName, apiKey, authToken = null) {
+    this.baseURL = `https://api${
+      envName === 'production' ? '' : `.${envName}`
+    }.corva.ai`;
+
+    this.config = {
+      headers: {
+        'Authorization': authToken ? `Bearer ${authToken}` :`API ${apiKey}`,
+        'Content-Type': 'application/json',
+        'Accept': 'application/json',
+      },
+    };
+  }
+
+  /**
+   * Get api url
+   *
+   * @private
+   *
+   * @param {string} path
+   *
+   * @returns  {string}
+   */
+  getUrl(path) {
+    return `${this.baseURL}${path}`;
+  }
+
+  /**
+   * Send api request
+   *
+   * @private
+   *
+   * @param {string} path
+   * @param {string} method
+   * @param {object} data
+   *
+   * @returns  {object}
+   */
+  sendRequest(method, url, data = null) {
+    return data ? axios[method](url, data, this.config) : axios[method](url, this.config);
+  }
+
+  /**
+   * Get app by appKey
+   *
+   * @public
+   *
+   * @param {string} appKey
+   *
+   * @returns  {object}
+   */
+  async getAppByKey(appKey) {
+    const url = this.getUrl(`/v2/apps?per_page=2&search=${appKey}`);
+    const response = await this.sendRequest('get', url);
+
+    const app = response.data.data[0];
+    if (!app || app.attributes.app_key != appKey) {
+      throw new Error(`App with key - ${appKey}, not exist`);
+    }
+
+    return app;
+  }
+
+  /**
+   * Get app datasets
+   *
+   * @public
+   *
+   * @param {string} appId
+   *
+   * @returns  {object}
+   */
+  async getAppDatasetsOperationWrite(appId) {
+    const url = this.getUrl(
+      `/v2/apps/${appId}/app_datasets?dataset_operation=write&fields[]=app_dataset.dataset_name`
+    );
+    const response = await this.sendRequest('get', url);
+
+    if (!response.data.data.length) {
+      throw new Error(`App with ID - ${appId}, has not any datasets`);
+    }
+
+    return response.data.data;
+  }
+
+  /**
+   * Get asset details
+   *
+   * @public
+   *
+   * @param {string} assetId
+   *
+   * @returns  {object[]}
+   */
+  async getAssetById(assetId) {
+    const url = this.getUrl(`/v2/assets/${assetId}`);
+    const response = await this.sendRequest('get', url);
+
+    return response.data.data;
+  }
+
+
+  /**
+   * Get asset streams
+   *
+   * @public
+   *
+   * @param {string} assetId
+   *
+   * @returns  {object[]}
+   */
+   async getStreamByAssetId(assetId) {
+    const url = this.getUrl(`/v1/app_streams?asset_id=${assetId}&segment=drilling&status=complete`);
+    const response = await this.sendRequest('get', url);
+
+    if (!response.data[0]) {
+      throw new Error(`Could not found streams for asset ID - ${assetId}`);
+    }
+
+    return response.data;
+  }
+
+  /**
+   * Get wells data by asset id
+   *
+   * @public
+   *
+   * @param {string} assetId
+   *
+   * @returns  {object[]}
+   */
+  async getWellByAssetId(assetId) {
+    const data = {
+      assets: [assetId],
+    };
+
+    const url = this.getUrl(`/v2/assets/resolve`);
+    const response = await this.sendRequest('post', url, data);
+
+    if (!response.data.wells.length) {
+      throw new Error(`Could not found wells by asset ID - ${assetId}`);
+    }
+
+    return response.data.wells;
+  }
+
+  /**
+   * Queue app run for create new task
+   *
+   * @public
+   *
+   * @param {string} appId
+   * @param {string} version
+   * @param {string} wellId
+   * @param {string[]} appDatasetsNames
+   * @param {string} streamId
+   *
+   * @returns  {object}
+   */
+  async queueAppRun(appId, version, wellId, appDatasetsNames, streamId) {
+    const data = {
+      app_run: {
+        app_stream_id: streamId,
+        well_id: wellId,
+        app_version: version,
+        datasets: appDatasetsNames,
+        settings: {
+          end: 0,
+          interval: null,
+          invokes: null,
+          records_per_event: 300,
+          start: 0,
+        },
+      },
+    };
+
+    const url = this.getUrl(`/v2/apps/${appId}/app_runs`);
+    const response = await this.sendRequest('post', url, data);
+    return response.data.data;
+  }
+
+  /**
+   * Get all active(status - pending or in_progress or running) runs by app ID
+   *
+   * @public
+   *
+   * @param {string} appId
+   *
+   * @returns  {object[]}
+   */
+   async getAppRuns(appId) {
+    const url = this.getUrl(`/v2/apps/${appId}/app_runs?page=1&per_page=500&status[]=pending&status[]=in_progress&status[]=running`);
+    const response = await this.sendRequest('get', url);
+
+    return response.data.data;
+  }
+
+  /**
+   * Upload packages
+   *
+   * @public
+   *
+   * @param {string} appKey
+   * @param {object} formData
+   *
+   * @returns
+   */
+    async uploadPackages(appKey, formData) {
+      const url = this.getUrl(`/v2/apps/${appKey}/packages/upload`);
+      this.config.headers = {
+        ...this.config.headers,
+        ...formData.getHeaders(),
+      };
+      return this.sendRequest('post', url, formData)
+    }
+}
+
+module.exports = { Api };

package/lib/flows/rerun.js

@@ -0,0 +1,10 @@
+const { PREPARE_FLOW } = require('./prepare');
+const { RERUN_STEPS } = require('./steps/rerun');
+const { GET_RELEASE_CONFIG_STEP } = require('./steps/release-get-config');
+
+const RERUN_FLOW = {
+  name: 'rerun',
+  steps: [PREPARE_FLOW, GET_RELEASE_CONFIG_STEP, ...RERUN_STEPS],
+};
+
+module.exports = { RERUN_FLOW };

package/lib/flows/steps/release-upload-zip-to-corva.js

@@ -1,31 +1,19 @@
 const { RELEASE } = require('../../constants/messages');
-const axios = require('axios');
 const FormData = require('form-data');
 const { resolve } = require('path');
 const { createReadStream } = require('fs');
 const { StepError } = require('../lib/step-error');
+const { Api } = require('../lib/api');
 const { get } = require('lodash/fp');
 
 const UPLOAD_ZIP_TO_CORVA_STEP = {
   message: RELEASE.uploadApp,
   fn: async ({ zipFileName, appKey, CORVA_API_ENV, AUTH_TOKEN, API_KEY, dirName }) => {
     const form = new FormData();
-
     form.append('package', createReadStream(resolve(dirName, zipFileName)), 'archive.zip');
 
-    const baseURL = `https://api${
-      CORVA_API_ENV === 'production' ? '' : `.${CORVA_API_ENV}`
-    }.corva.ai`;
-    const uploadURL = `${baseURL}/v2/apps/${appKey}/packages/upload`;
-
-    await axios
-      .post(uploadURL, form, {
-        headers: {
-          ...form.getHeaders(),
-          Authorization: AUTH_TOKEN ? `Bearer ${AUTH_TOKEN}` : `API ${API_KEY}`,
-        },
-      })
-      .catch((e) => {
+    const api = new Api(CORVA_API_ENV, API_KEY, AUTH_TOKEN);
+    api.uploadPackages(appKey, form).catch((e) => {
         throw new StepError(
           `${get('response.data.message', e) || ''} \nPOST: ${uploadURL} failed.`,
           { cause: e }

package/lib/flows/steps/rerun-create-task.js

@@ -0,0 +1,51 @@
+const _ = require('lodash');
+const chalk = require('chalk');
+
+const CREATE_TASK_STEP = {
+  message: 'Creating tasks...',
+  fn: async ({
+    appId,
+    assets,
+    appDatasetsNames,
+    mappedAssetsToStreams,
+    mappedAssetsToWells,
+    api,
+    version,
+    CORVA_API_ENV,
+  }) => {
+    if (!assets.length) {
+      process.stdout.write( 
+        `\n\n${chalk.yellow.bold('There is not a asset ID to create new task')}`
+      );
+    }
+
+    for (const assetId of assets) {
+
+      try {
+        const result = await api.queueAppRun(
+          appId,
+          version,
+          mappedAssetsToWells.get(parseInt(assetId)),
+          appDatasetsNames,
+          mappedAssetsToStreams.get(parseInt(assetId))
+        );
+  
+        process.stdout.write(
+          `\n    Created new task with ID ${chalk.yellow(result.id)} for asset ID - ${chalk.green(
+            assetId
+          )}`
+        );
+
+        process.stdout.write(
+          `\n    Task link - https://app${CORVA_API_ENV === 'production' ? '.' : `.${CORVA_API_ENV}.`}corva.ai/dev-center/apps/${appId}/runner`
+        );
+      } catch (e) {
+        process.stdout.write( 
+          `\n\n${chalk.red.underline.bold(`Could not rerun app for asset ID - ${assetId}, an error occured: ${e.message}`)}\n\n`
+        );
+      }
+    }
+  },
+};
+
+module.exports = { CREATE_TASK_STEP };

package/lib/flows/steps/rerun-prepare-data.js

@@ -0,0 +1,243 @@
+const _ = require('lodash');
+const chalk = require('chalk');
+const inquirer = require('inquirer');
+const { Api } = require('../lib/api');
+const { StepError } = require('../lib/step-error');
+
+const MAX_ASSET_IDS_COUNT = 10;
+
+const PREPARE_DATA_STEP = {
+  message: 'Preparing data for tasks...',
+  fn: async (context) => {
+    const { manifest, patterns, API_KEY, CORVA_API_ENV, pkg } = context;
+    let { assets } = patterns;
+
+    assets = _.uniq(assets);
+
+    if (assets.length > MAX_ASSET_IDS_COUNT) {
+      throw new StepError(`Please no more than ${MAX_ASSET_IDS_COUNT} asset ids!`);
+    }
+
+    if (!['scheduler', 'stream'].includes(manifest.manifest.application.type)) {
+      throw new StepError('Rerun command supports only "scheduler" or "stream" apps');
+    }
+
+    if (!await promptAreYouSure()) {
+      throw new StepError('Command stopped');
+    }
+
+    const api = new Api(CORVA_API_ENV, API_KEY);
+
+    const app = await api.getAppByKey(manifest.manifest.application.key.toLowerCase());
+    const appDatasets = await api.getAppDatasetsOperationWrite(app.id);
+    const appDatasetsNames = appDatasets.map((dataset) => dataset.attributes.dataset_name);
+
+    const existingAppRuns = await getExistAppRuns(app.id, assets, appDatasetsNames, api);
+    if (existingAppRuns.size) {
+      for (let appRunAssetId of existingAppRuns.keys()) {
+        // remove asset ID if similar rerun already exists
+        assets = assets.filter((assetId) => appRunAssetId != assetId);
+
+        const run = existingAppRuns.get(appRunAssetId);
+
+        process.stdout.write( 
+          `\n\n${chalk.yellow.bold(`Similar rerun with ID ${run.id}, for asset ID ${appRunAssetId} - already exist. Will be skipped!`)}`
+        );
+      }
+    }
+
+    const {mappedAssetsToWells, mappedAssetsToStreams, assetsToDelete} = await prepareWellAndStreamData(assets, api);
+
+    if (assetsToDelete.length) {
+      // remove asset ID if could not found stream or well
+      assets = assets.filter((assetId) => !assetsToDelete.includes(assetId));
+    }
+
+    return {
+      appId: app.id,
+      assets,
+      appDatasetsNames,
+      mappedAssetsToStreams,
+      mappedAssetsToWells,
+      api,
+      CORVA_API_ENV,
+      version: pkg.version,
+    };
+  },
+};
+
+/**
+ *  CLI prompt question - Are you sure you want to do this?
+ *
+ * @returns
+ */
+ const promptAreYouSure = async () => {
+    const answers = await inquirer
+      .prompt([
+        {
+          message:
+            '\n This command will create additional load on the server, which may take a long time. \n Are you sure you want to do this?',
+          name: 'option',
+          type: 'list',
+          choices: [
+            { value: true, name: 'Yes' },
+            { value: false, name: 'No' },
+          ],
+          default: false,
+        },
+      ]);
+    return answers.option;
+ }
+
+ /**
+ *  CLI prompt question - Please choose the stream
+ * 
+   * @param {object[]} streams
+ *
+ * @returns
+ */
+  const getStreamWithPrompt = async (streams) => {
+    let stream = {};
+
+    const choices = streams.map((stream) => {
+      return {
+        value: stream,
+        name: stream.name
+      };
+    });
+
+    await inquirer
+      .prompt([
+        {
+          message:
+            '\n Please choose stream?',
+          name: 'option',
+          type: 'list',
+          choices,
+        },
+      ])
+      .then((answers) => {
+        stream = answers.option;
+      });
+
+      return stream;
+ }
+
+ /**
+ *  CLI prompt question - Please choose the well
+ * 
+ * @param {object[]} wells
+ * @param {object} api
+ *
+ * @returns
+ */
+  const getWellWithPrompt = async (wells, api) => {
+    let well = {};
+
+    const choices = await Promise.all(wells.map( async (well) => {
+      const assetDetails = await api.getAssetById(well.data.attributes.asset_id);
+      well.data.name = assetDetails.attributes.name;
+      return {
+        value: well.data,
+        name: well.data.name,
+      };
+    }));
+
+    await inquirer
+      .prompt([
+        {
+          message:
+            '\n Please choose the well?',
+          name: 'option',
+          type: 'list',
+          choices,
+        },
+      ])
+      .then((answers) => {
+        well = answers.option;
+      });
+
+      return well;
+ }
+
+
+ /**
+ *  Get stream and well data for assets
+ * 
+ * @param {string[]} assets
+ * @param {object} api
+ *
+ * @returns {object}
+ */
+  const prepareWellAndStreamData = async (assets, api) => {
+    const mappedAssetsToWells = new Map();
+    const mappedAssetsToStreams = new Map();
+    const assetsToDelete = [];
+
+    for (const assetId of assets) {
+      try {
+        process.stdout.write(
+          `\n\n${chalk.black.underline.bold(`Process asset ID - ${chalk.green(assetId)}`)}`
+        );
+
+        process.stdout.write(
+          '\n    Loading wells...'
+        );
+
+        const wells = await api.getWellByAssetId(assetId);
+        const well = await getWellWithPrompt(wells, api);
+        mappedAssetsToWells.set(well.attributes.asset_id, well.id);
+  
+        process.stdout.write(
+          '\n    Loading streams...'
+        );
+
+        const streams = await api.getStreamByAssetId(assetId);
+        const stream = await getStreamWithPrompt(streams);
+        mappedAssetsToStreams.set(stream.asset_id, stream.id);
+      } catch (e) {
+        process.stdout.write( 
+          `\n\n${chalk.red.underline.bold(`Skipped the asset ID - ${assetId}, an error occured: ${e.message}`)}`
+        );
+
+        assetsToDelete.push(assetId);
+
+        if (mappedAssetsToWells.has(assetId)) mappedAssetsToWells.delete(assetId);
+        if (mappedAssetsToStreams.has(assetId)) mappedAssetsToStreams.delete(assetId);
+      }
+    }
+
+    return {mappedAssetsToWells, mappedAssetsToStreams, assetsToDelete};
+  }
+
+/**
+ *  Check if current runs already exist
+ *
+ * @param {string} appId
+ * @param {string[]} assets
+ * @param {string[]} appDatasetsNames
+ * @param {object} api
+ *
+ * @returns {map}
+ */
+const getExistAppRuns = async (appId, assets, appDatasetsNames, api) => {
+  const appRuns = new Map();
+  const existingAppRuns = await api.getAppRuns(appId);
+
+  // go through all assets ids
+  for (const assetId of assets) {
+    const runsForCurrentAsset = existingAppRuns.filter((run) => run.attributes.well_asset_id == assetId);
+    // go through all existing runs for current asset ID
+    for (const run of runsForCurrentAsset) {
+      const currentAppDatasetsNames = run.attributes.app_run_datasets.map((dataset) => dataset.name);
+      // if datasets names matched for the same asset ID - then run already exist
+      if (appDatasetsNames.sort().toString() === currentAppDatasetsNames.sort().toString()) {
+        appRuns.set(assetId, run);
+      }
+    }
+  }
+
+  return appRuns;
+};
+
+module.exports = { PREPARE_DATA_STEP };

package/lib/flows/steps/rerun.js

@@ -0,0 +1,8 @@
+const { PREPARE_DATA_STEP } = require('./rerun-prepare-data');
+const { CREATE_TASK_STEP } = require('./rerun-create-task');
+
+const RERUN_STEPS = [PREPARE_DATA_STEP, CREATE_TASK_STEP];
+
+module.exports = {
+  RERUN_STEPS,
+};

package/lib/index.js

@@ -30,6 +30,7 @@ const { runFlow } = require('./flow');
 const { ERROR_ICON } = require('./constants/messages');
 const { StepError } = require('./flows/lib/step-error');
 const { RELEASE_FLOW } = require('./flows/release');
+const { RERUN_FLOW } = require('./flows/rerun');
 const { ZIP_FLOW } = require('./flows/zip');
 const { bumpVersionOptionDeprecated, bumpVersionOption } = require('./bump-version.option');
 const { Manifest } = require('./flows/lib/manifest');
@@ -198,6 +199,15 @@ async function initialChecks() {
       return runFlow(RELEASE_FLOW, { dirName, patterns, options });
     });
 
+  program
+    .command('rerun')
+    .description('Rerun app')
+    .argument('<project-directory>', 'Project directory to work with')
+    .addOption(new Option('--assets [assets...]', 'Assets ids list', []))
+    .action(async (dirName, patterns, options) => {
+      return runFlow(RERUN_FLOW, { dirName, patterns, options });
+    });
+
   try {
     await program.parseAsync(process.argv);
   } catch (e) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@corva/create-app",
-    "version": "0.39.0-rc.0",
+    "version": "0.40.0-0",
     "private": false,
     "description": "Create app to use it in CORVA.AI",
     "keywords": [