@jupiterone/integration-sdk-cli 8.3.0 → 8.4.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/integration-sdk-cli",
-  "version": "8.3.0",
+  "version": "8.4.1",
   "description": "The SDK for developing JupiterOne integrations",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -22,8 +22,10 @@
     "prepack": "yarn build:dist"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-runtime": "^8.3.0",
+    "@jupiterone/integration-sdk-runtime": "^8.4.1",
+    "chalk": "^4",
     "commander": "^5.0.0",
+    "fs-extra": "^10.0.0",
     "globby": "^11.0.0",
     "js-yaml": "^4.1.0",
     "json-diff": "^0.5.4",
@@ -35,10 +37,11 @@
     "vis": "^4.21.0-EOL"
   },
   "devDependencies": {
-    "@jupiterone/integration-sdk-private-test-utils": "^8.3.0",
+    "@jupiterone/integration-sdk-private-test-utils": "^8.4.1",
     "@pollyjs/adapter-node-http": "^5.1.1",
     "@pollyjs/core": "^5.1.1",
     "@pollyjs/persister-fs": "^5.1.1",
+    "@types/fs-extra": "^9.0.13",
     "@types/js-yaml": "^4.0.3",
     "@types/json-diff": "^0.5.1",
     "@types/lodash": "^4.14.158",
@@ -50,5 +53,5 @@
     "neo-forgery": "^2.0.0",
     "uuid": "^8.2.0"
   },
-  "gitHead": "c0aadb45b32b5266fd4710a632f6b6a5144c548d"
+  "gitHead": "07778e4a5f0e87137f6036ee8e3ba4e44ff2f4bb"
 }

package/src/__tests__/cli.test.ts

@@ -48,6 +48,84 @@ describe('collect', () => {
     loadProjectStructure('instanceWithDependentSteps');
   });
 
+  test('option --no-cache requires option --step', async () => {
+    await expect(
+      createCli().parseAsync([
+        'node',
+        'j1-integration',
+        'collect',
+        '--no-cache',
+      ]),
+    ).rejects.toThrowError(
+      'Invalid option: Option --no-cache requires option --step to also be specified.',
+    );
+  });
+  test('option --cache-path requires option --step', async () => {
+    await expect(
+      createCli().parseAsync([
+        'node',
+        'j1-integration',
+        'collect',
+        '--cache-path',
+        './',
+      ]),
+    ).rejects.toThrowError(
+      'Invalid option: Option --cache-path requires option --step to also be specified.',
+    );
+  });
+
+  test('Uses cache when --step flag is provided w/out --no-cache', async () => {
+    loadProjectStructure('instanceWithDependentCachedSteps');
+
+    await createCli().parseAsync([
+      'node',
+      'j1-integration',
+      'collect',
+      '--step',
+      'fetch-groups',
+      '--cache-path',
+      process.cwd(),
+    ]);
+
+    expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
+    expect(log.displayExecutionResults).toHaveBeenCalledWith({
+      integrationStepResults: [
+        {
+          id: 'fetch-accounts',
+          name: 'Fetch Accounts',
+          declaredTypes: ['my_account'],
+          dependsOn: undefined,
+          partialTypes: [],
+          encounteredTypes: ['test_account'],
+          status: StepResultStatus.CACHED,
+        },
+        {
+          id: 'fetch-groups',
+          dependsOn: ['fetch-accounts'],
+          name: 'Fetch Groups',
+          declaredTypes: ['my_groups'],
+          partialTypes: [],
+          encounteredTypes: [],
+          status: StepResultStatus.SUCCESS,
+        },
+        {
+          id: 'fetch-users',
+          name: 'Fetch Users',
+          declaredTypes: ['my_user'],
+          dependsOn: undefined,
+          partialTypes: [],
+          encounteredTypes: [],
+          status: StepResultStatus.DISABLED,
+        },
+      ],
+      metadata: {
+        partialDatasets: {
+          types: [],
+        },
+      },
+    });
+  });
+
   test('loads the integration, executes it, and logs the result', async () => {
     await createCli().parseAsync(['node', 'j1-integration', 'collect']);
 

package/src/commands/collect.ts

@@ -1,9 +1,11 @@
 import { createCommand } from 'commander';
 import path from 'path';
+import fs from 'fs-extra';
 
 import {
   executeIntegrationLocally,
   FileSystemGraphObjectStore,
+  getRootStorageDirectory,
   prepareLocalStepCollection,
 } from '@jupiterone/integration-sdk-runtime';
 
@@ -26,12 +28,31 @@ export function collect() {
     )
     .option(
       '-s, --step <steps>',
-      'step(s) to run, comma separated',
+      'step(s) to run, comma separated. Utilizes available caches to speed up dependent steps.',
       collector,
       [],
     )
+    .option(
+      '--no-cache',
+      'Can be used with the `--step` flag to disable the use of the cache.',
+    )
+    .option(
+      '--cache-path <filepath>',
+      'Can be used with the `--step` to specify a path to a non-default cache location.',
+    )
     .option('-V, --disable-schema-validation', 'disable schema validation')
     .action(async (options) => {
+      if (!options.cache && options.step.length === 0) {
+        throw new Error(
+          'Invalid option: Option --no-cache requires option --step to also be specified.',
+        );
+      }
+      if (options.cachePath && options.step.length === 0) {
+        throw new Error(
+          'Invalid option: Option --cache-path requires option --step to also be specified.',
+        );
+      }
+
       // Point `fileSystem.ts` functions to expected location relative to
       // integration project path.
       process.env.JUPITERONE_INTEGRATION_STORAGE_DIRECTORY = path.resolve(
@@ -39,9 +60,21 @@ export function collect() {
         '.j1-integration',
       );
 
+      if (options.step.length > 0 && options.cache && !options.cachePath) {
+        // Step option was used, cache is wanted, and no cache path was provided
+        // therefore, copy .j1-integration into .j1-integration-cache
+        await buildCacheFromJ1Integration();
+      }
+
       const config = prepareLocalStepCollection(
         await loadConfig(path.join(options.projectPath, 'src')),
-        options,
+        {
+          ...options,
+          dependenciesCache: {
+            enabled: options.cache,
+            filepath: getRootCacheDirectory(options.cachePath),
+          },
+        },
       );
       log.info('\nConfiguration loaded! Running integration...\n');
 
@@ -67,3 +100,34 @@ export function collect() {
       log.displayExecutionResults(results);
     });
 }
+
+export const DEFAULT_CACHE_DIRECTORY_NAME = '.j1-integration-cache';
+
+export function getRootCacheDirectory(filepath?: string) {
+  return path.resolve(
+    typeof filepath === 'string' ? filepath : process.cwd(),
+    DEFAULT_CACHE_DIRECTORY_NAME,
+  );
+}
+
+/**
+ * Builds the step cache from the .j1-integration/graph directory
+ * by moving the files to .j1-integration-cache.
+ */
+async function buildCacheFromJ1Integration() {
+  const sourceGraphDirectory = path.join(getRootStorageDirectory(), 'graph');
+  const destinationGraphDirectory = path.join(getRootCacheDirectory(), 'graph');
+
+  const sourceExists = await fs.pathExists(sourceGraphDirectory);
+  if (sourceExists) {
+    await fs
+      .move(sourceGraphDirectory, destinationGraphDirectory, {
+        overwrite: true,
+      })
+      .catch((error) => {
+        log.error(`Failed to seed .j1-integration-cache from .j1-integration`);
+        log.error(error);
+      });
+    log.info(`Populated the .j1-integration-cache from .j1-integration.`);
+  }
+}

package/src/log.ts

@@ -100,6 +100,7 @@ function logStepStatus(stepResult: IntegrationStepResult) {
 function getStepStatusText(status: StepResultStatus) {
   switch (status) {
     case StepResultStatus.SUCCESS:
+    case StepResultStatus.CACHED:
       return chalk.green(status);
     case StepResultStatus.FAILURE:
       return chalk.red(status);