@jupiterone/integration-sdk-cli 8.3.1 → 8.4.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/integration-sdk-cli",
-  "version": "8.3.1",
+  "version": "8.4.2",
   "description": "The SDK for developing JupiterOne integrations",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -22,9 +22,10 @@
     "prepack": "yarn build:dist"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-runtime": "^8.3.1",
+    "@jupiterone/integration-sdk-runtime": "^8.4.2",
     "chalk": "^4",
     "commander": "^5.0.0",
+    "fs-extra": "^10.0.0",
     "globby": "^11.0.0",
     "js-yaml": "^4.1.0",
     "json-diff": "^0.5.4",
@@ -36,10 +37,11 @@
     "vis": "^4.21.0-EOL"
   },
   "devDependencies": {
-    "@jupiterone/integration-sdk-private-test-utils": "^8.3.1",
+    "@jupiterone/integration-sdk-private-test-utils": "^8.4.2",
     "@pollyjs/adapter-node-http": "^5.1.1",
     "@pollyjs/core": "^5.1.1",
     "@pollyjs/persister-fs": "^5.1.1",
+    "@types/fs-extra": "^9.0.13",
     "@types/js-yaml": "^4.0.3",
     "@types/json-diff": "^0.5.1",
     "@types/lodash": "^4.14.158",
@@ -51,5 +53,5 @@
     "neo-forgery": "^2.0.0",
     "uuid": "^8.2.0"
   },
-  "gitHead": "b730a382d3a7cd7b2b2c219af2b8bc13c059521f"
+  "gitHead": "1504c547765bd8636a8adcaf19098bc1b9f802e1"
 }

package/src/__tests__/cli.test.ts

@@ -48,6 +48,84 @@ describe('collect', () => {
     loadProjectStructure('instanceWithDependentSteps');
   });
 
+  test('option --no-cache requires option --step', async () => {
+    await expect(
+      createCli().parseAsync([
+        'node',
+        'j1-integration',
+        'collect',
+        '--no-cache',
+      ]),
+    ).rejects.toThrowError(
+      'Invalid option: Option --no-cache requires option --step to also be specified.',
+    );
+  });
+  test('option --cache-path requires option --step', async () => {
+    await expect(
+      createCli().parseAsync([
+        'node',
+        'j1-integration',
+        'collect',
+        '--cache-path',
+        './',
+      ]),
+    ).rejects.toThrowError(
+      'Invalid option: Option --cache-path requires option --step to also be specified.',
+    );
+  });
+
+  test('Uses cache when --step flag is provided w/out --no-cache', async () => {
+    loadProjectStructure('instanceWithDependentCachedSteps');
+
+    await createCli().parseAsync([
+      'node',
+      'j1-integration',
+      'collect',
+      '--step',
+      'fetch-groups',
+      '--cache-path',
+      process.cwd(),
+    ]);
+
+    expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
+    expect(log.displayExecutionResults).toHaveBeenCalledWith({
+      integrationStepResults: [
+        {
+          id: 'fetch-accounts',
+          name: 'Fetch Accounts',
+          declaredTypes: ['my_account'],
+          dependsOn: undefined,
+          partialTypes: [],
+          encounteredTypes: ['test_account'],
+          status: StepResultStatus.CACHED,
+        },
+        {
+          id: 'fetch-groups',
+          dependsOn: ['fetch-accounts'],
+          name: 'Fetch Groups',
+          declaredTypes: ['my_groups'],
+          partialTypes: [],
+          encounteredTypes: [],
+          status: StepResultStatus.SUCCESS,
+        },
+        {
+          id: 'fetch-users',
+          name: 'Fetch Users',
+          declaredTypes: ['my_user'],
+          dependsOn: undefined,
+          partialTypes: [],
+          encounteredTypes: [],
+          status: StepResultStatus.DISABLED,
+        },
+      ],
+      metadata: {
+        partialDatasets: {
+          types: [],
+        },
+      },
+    });
+  });
+
   test('loads the integration, executes it, and logs the result', async () => {
     await createCli().parseAsync(['node', 'j1-integration', 'collect']);
 

package/src/commands/collect.ts

@@ -1,9 +1,11 @@
 import { createCommand } from 'commander';
 import path from 'path';
+import fs from 'fs-extra';
 
 import {
   executeIntegrationLocally,
   FileSystemGraphObjectStore,
+  getRootStorageDirectory,
   prepareLocalStepCollection,
 } from '@jupiterone/integration-sdk-runtime';
 
@@ -26,12 +28,31 @@ export function collect() {
     )
     .option(
       '-s, --step <steps>',
-      'step(s) to run, comma separated',
+      'step(s) to run, comma separated. Utilizes available caches to speed up dependent steps.',
       collector,
       [],
     )
+    .option(
+      '--no-cache',
+      'Can be used with the `--step` flag to disable the use of the cache.',
+    )
+    .option(
+      '--cache-path <filepath>',
+      'Can be used with the `--step` to specify a path to a non-default cache location.',
+    )
     .option('-V, --disable-schema-validation', 'disable schema validation')
     .action(async (options) => {
+      if (!options.cache && options.step.length === 0) {
+        throw new Error(
+          'Invalid option: Option --no-cache requires option --step to also be specified.',
+        );
+      }
+      if (options.cachePath && options.step.length === 0) {
+        throw new Error(
+          'Invalid option: Option --cache-path requires option --step to also be specified.',
+        );
+      }
+
       // Point `fileSystem.ts` functions to expected location relative to
       // integration project path.
       process.env.JUPITERONE_INTEGRATION_STORAGE_DIRECTORY = path.resolve(
@@ -39,9 +60,21 @@ export function collect() {
         '.j1-integration',
       );
 
+      if (options.step.length > 0 && options.cache && !options.cachePath) {
+        // Step option was used, cache is wanted, and no cache path was provided
+        // therefore, copy .j1-integration into .j1-integration-cache
+        await buildCacheFromJ1Integration();
+      }
+
       const config = prepareLocalStepCollection(
         await loadConfig(path.join(options.projectPath, 'src')),
-        options,
+        {
+          ...options,
+          dependenciesCache: {
+            enabled: options.cache,
+            filepath: getRootCacheDirectory(options.cachePath),
+          },
+        },
       );
       log.info('\nConfiguration loaded! Running integration...\n');
 
@@ -67,3 +100,34 @@ export function collect() {
       log.displayExecutionResults(results);
     });
 }
+
+export const DEFAULT_CACHE_DIRECTORY_NAME = '.j1-integration-cache';
+
+export function getRootCacheDirectory(filepath?: string) {
+  return path.resolve(
+    typeof filepath === 'string' ? filepath : process.cwd(),
+    DEFAULT_CACHE_DIRECTORY_NAME,
+  );
+}
+
+/**
+ * Builds the step cache from the .j1-integration/graph directory
+ * by moving the files to .j1-integration-cache.
+ */
+async function buildCacheFromJ1Integration() {
+  const sourceGraphDirectory = path.join(getRootStorageDirectory(), 'graph');
+  const destinationGraphDirectory = path.join(getRootCacheDirectory(), 'graph');
+
+  const sourceExists = await fs.pathExists(sourceGraphDirectory);
+  if (sourceExists) {
+    await fs
+      .move(sourceGraphDirectory, destinationGraphDirectory, {
+        overwrite: true,
+      })
+      .catch((error) => {
+        log.error(`Failed to seed .j1-integration-cache from .j1-integration`);
+        log.error(error);
+      });
+    log.info(`Populated the .j1-integration-cache from .j1-integration.`);
+  }
+}

package/src/commands/document.ts

@@ -68,9 +68,8 @@ async function executeDocumentAction(
   );
   log.info('\nExisting documentation file successfully loaded!\n');
 
-  const newGeneratedDocumentationSection = generateGraphObjectDocumentationFromStepsMetadata(
-    metadata,
-  );
+  const newGeneratedDocumentationSection =
+    generateGraphObjectDocumentationFromStepsMetadata(metadata);
 
   log.info('\nGenerated integration documentation section:');
   log.info('---------------------------------------------\n');
@@ -187,9 +186,10 @@ ${generatedEntityTable}`;
   }
 
   if (metadata.relationships.length) {
-    const generatedRelationshipTable = generateRelationshipTableFromAllStepEntityMetadata(
-      metadata.relationships,
-    );
+    const generatedRelationshipTable =
+      generateRelationshipTableFromAllStepEntityMetadata(
+        metadata.relationships,
+      );
 
     relationshipSection += `
 
@@ -201,9 +201,10 @@ ${generatedRelationshipTable}`;
   }
 
   if (metadata.mappedRelationships?.length) {
-    const generatedMappedRelationshipTable = generateMappedRelationshipTableFromAllStepEntityMetadata(
-      metadata.mappedRelationships,
-    );
+    const generatedMappedRelationshipTable =
+      generateMappedRelationshipTableFromAllStepEntityMetadata(
+        metadata.mappedRelationships,
+      );
 
     mappedRelationshipSection += `
 

package/src/commands/neo4j.ts

@@ -11,7 +11,9 @@ export function neo4j() {
   dotenvExpand(dotenv.config());
 
   const program = new commander.Command();
-  program.description(`Suite of neo4j commands.  Options are currently 'neo4j push', 'neo4j wipe', and 'neo4j wipe-all'`);
+  program.description(
+    `Suite of neo4j commands.  Options are currently 'neo4j push', 'neo4j wipe', and 'neo4j wipe-all'`,
+  );
   const neo4jCommand = program.command('neo4j');
   neo4jCommand
     .command('push')
@@ -24,7 +26,7 @@ export function neo4j() {
     .option(
       '-i, --integration-instance-id <id>',
       '_integrationInstanceId assigned to uploaded entities',
-      'defaultLocalInstanceID'
+      'defaultLocalInstanceID',
     )
     .action(async (options) => {
       log.info(`Beginning data upload to local neo4j`);
@@ -34,30 +36,34 @@ export function neo4j() {
       process.env.JUPITERONE_INTEGRATION_STORAGE_DIRECTORY = finalDir;
 
       await uploadToNeo4j({
-        pathToData: finalDir, 
-        integrationInstanceID: options.integrationInstanceId
+        pathToData: finalDir,
+        integrationInstanceID: options.integrationInstanceId,
       });
       log.info(`Data uploaded to local neo4j`);
     });
 
-    neo4jCommand
+  neo4jCommand
     .command('wipe')
-    .description('wipe entities and relationships for a given integrationInstanceID in the Neo4j database')
+    .description(
+      'wipe entities and relationships for a given integrationInstanceID in the Neo4j database',
+    )
     .option(
       '-i, --integration-instance-id <id>',
       '_integrationInstanceId assigned to uploaded entities',
-      'defaultLocalInstanceID'
+      'defaultLocalInstanceID',
     )
     .action(async (options) => {
-      await wipeNeo4jByID({integrationInstanceID: options.integrationInstanceId});
+      await wipeNeo4jByID({
+        integrationInstanceID: options.integrationInstanceId,
+      });
     });
 
-    neo4jCommand
+  neo4jCommand
     .command('wipe-all')
     .description('wipe all entities and relationships in the Neo4j database')
     .action(async (options) => {
       await wipeAllNeo4j({});
     });
 
-    return neo4jCommand;
+  return neo4jCommand;
 }

package/src/commands/visualize-types.ts

@@ -138,15 +138,13 @@ export function getNodesAndEdgesFromStepMetadata(
       types: options?.types,
     },
   );
-  const {
-    placeholderEntityNodes,
-    mappedRelationshipEdges,
-  } = getNodesAndEdgesFromStepMappedRelationshipMetadata(
-    metadata.mappedRelationships || [],
-    {
-      types: options?.types,
-    },
-  );
+  const { placeholderEntityNodes, mappedRelationshipEdges } =
+    getNodesAndEdgesFromStepMappedRelationshipMetadata(
+      metadata.mappedRelationships || [],
+      {
+        types: options?.types,
+      },
+    );
   const entityNodes = getNodesFromStepEntityMetadata(metadata.entities, {
     types: options?.types,
     edges: [...relationshipEdges, ...mappedRelationshipEdges],

package/src/config.ts

@@ -43,7 +43,8 @@ export function loadInvocationConfig(
     integrationModule = require(path.resolve(projectSourceDirectory, 'index'));
   } catch (err) {
     throw new IntegrationInvocationConfigLoadError(
-      'Error loading integration invocation configuration. Ensure "invocationConfig" is exported from "src/index". Additional details: ' + err,
+      'Error loading integration invocation configuration. Ensure "invocationConfig" is exported from "src/index". Additional details: ' +
+        err,
     );
   }
 

package/src/log.ts

@@ -100,6 +100,7 @@ function logStepStatus(stepResult: IntegrationStepResult) {
 function getStepStatusText(status: StepResultStatus) {
   switch (status) {
     case StepResultStatus.SUCCESS:
+    case StepResultStatus.CACHED:
       return chalk.green(status);
     case StepResultStatus.FAILURE:
       return chalk.red(status);

package/src/neo4j/README.md

@@ -2,16 +2,13 @@
 
 ## Installation
 
-This command assumes you have three additional values stored in your
-local .env file:
-  NEO4J_URI
-  NEO4J_USER
-  NEO4J_PASSWORD
+This command assumes you have three additional values stored in your local .env
+file: NEO4J_URI NEO4J_USER NEO4J_PASSWORD
 
-This can be used for uploading to local or remote Neo4j databases.  If
-SSL is needed for a remote connection, specify `bolt+s` or `bolt+ssc` 
-in the URI.  For easy access to a local Neo4j instance, you can launch 
-one via a Neo4j provided Docker image with the command:
+This can be used for uploading to local or remote Neo4j databases. If SSL is
+needed for a remote connection, specify `bolt+s` or `bolt+ssc` in the URI. For
+easy access to a local Neo4j instance, you can launch one via a Neo4j provided
+Docker image with the command:
 
 ```
 docker run \
@@ -25,18 +22,18 @@ docker run \
     neo4j:latest
 ```
 
-If you would like to use a different username and password, the NEO4J_AUTH
-value can be modified to whatever username/password you prefer.
+If you would like to use a different username and password, the NEO4J_AUTH value
+can be modified to whatever username/password you prefer.
 
-NOTE:  Future updates are planned to streamline this without removing 
-the option of pushing to an external Neo4j database.
+NOTE: Future updates are planned to streamline this without removing the option
+of pushing to an external Neo4j database.
 
 ## Usage
 
 Data is still collected in the same way as before with a call to `yarn start`.
 
-Once data has been collected, you can run `j1-integration neo4j push`.  This will
-push data to the Neo4j server listed in the NEO4J_URI .env parameter.  If
-running locally, you can then access data in the Neo4j database by visiting
-http://localhost:7474.  Alternatively, you can download the full Neo4j 
-Desktop application at https://neo4j.com/download/.
+Once data has been collected, you can run `j1-integration neo4j push`. This will
+push data to the Neo4j server listed in the NEO4J_URI .env parameter. If running
+locally, you can then access data in the Neo4j database by visiting
+http://localhost:7474. Alternatively, you can download the full Neo4j Desktop
+application at https://neo4j.com/download/.

package/src/neo4j/__tests__/neo4jGraphStore.test.ts

@@ -1,63 +1,66 @@
-import {
-  mockDriver,
-  mockSessionFromQuerySet,
-  QuerySpec
-} from 'neo-forgery';
+import { mockDriver, mockSessionFromQuerySet, QuerySpec } from 'neo-forgery';
 import * as neo4j from 'neo4j-driver';
 import { Neo4jGraphStore } from '../neo4jGraphStore';
 import { Entity, Relationship } from '@jupiterone/integration-sdk-core';
 
 const testInstanceID = 'testInstanceID';
-const testEntityData: Entity[] = [{
-  _type: "testType",
-  _class: "testClass",
-  _key: "testKey",
-}];
-const testRelationshipData: Relationship[] = [{
-  _fromEntityKey: "testKey1",
-  _toEntityKey: "testKey2",
-  _type: "testRelType",
-  _key: "relKey",
-  _class: "testRelationshipClass",
-}];
-const constraintCall = 'CREATE CONSTRAINT unique_testType IF NOT EXISTS ON (n:testType) ASSERT n._key IS UNIQUE;'
+const testEntityData: Entity[] = [
+  {
+    _type: 'testType',
+    _class: 'testClass',
+    _key: 'testKey',
+  },
+];
+const testRelationshipData: Relationship[] = [
+  {
+    _fromEntityKey: 'testKey1',
+    _toEntityKey: 'testKey2',
+    _type: 'testRelType',
+    _key: 'relKey',
+    _class: 'testRelationshipClass',
+  },
+];
+const constraintCall =
+  'CREATE CONSTRAINT unique_testType IF NOT EXISTS ON (n:testType) ASSERT n._key IS UNIQUE;';
 const addEntityCall = `MERGE (n:testType {_key: 'testKey', _integrationInstanceID: '${testInstanceID}'}) SET n._type = 'testType', n._class = 'testClass';`;
 const addRelationshipCall = `
 MATCH (start {_key: 'testKey1', _integrationInstanceID: '${testInstanceID}'})
 MATCH (end {_key: 'testKey2', _integrationInstanceID: '${testInstanceID}'})
-MERGE (start)-[relationship:testRelType]->(end);`
+MERGE (start)-[relationship:testRelType]->(end);`;
 const wipeByIDCall = `MATCH (n {_integrationInstanceID: '${testInstanceID}'}) DETACH DELETE n`;
 const wipeAllCall = 'MATCH (n) DETACH DELETE n';
-const querySet: QuerySpec[] = [{
-  name: 'addConstraint',
-  query: constraintCall,
-  params: undefined,
-  output: {records:[]}
-},
-{
-  name: 'addEntity',
-  query: addEntityCall,
-  params: undefined,
-  output: {records:[]}
-},
-{
-  name: 'addRelationship',
-  query: addRelationshipCall,
-  params: undefined,
-  output: {records:[]}
-},
-{
-  name: 'wipeByID',
-  query: wipeByIDCall,
-  params: undefined,
-  output: {records:[]}
-},
-{
-  name: 'wipeAll',
-  query: wipeAllCall,
-  params: undefined,
-  output: {records:[]}
-}];
+const querySet: QuerySpec[] = [
+  {
+    name: 'addConstraint',
+    query: constraintCall,
+    params: undefined,
+    output: { records: [] },
+  },
+  {
+    name: 'addEntity',
+    query: addEntityCall,
+    params: undefined,
+    output: { records: [] },
+  },
+  {
+    name: 'addRelationship',
+    query: addRelationshipCall,
+    params: undefined,
+    output: { records: [] },
+  },
+  {
+    name: 'wipeByID',
+    query: wipeByIDCall,
+    params: undefined,
+    output: { records: [] },
+  },
+  {
+    name: 'wipeAll',
+    query: wipeAllCall,
+    params: undefined,
+    output: { records: [] },
+  },
+];
 
 describe('#neo4jGraphStore', () => {
   const mockDriverResp = mockDriver();
@@ -88,12 +91,11 @@ describe('#neo4jGraphStore', () => {
   });
 
   test('should generate call to create a Relationship', () => {
-
-    expect(async () => await store.addRelationships(testRelationshipData)).toReturn;
+    expect(async () => await store.addRelationships(testRelationshipData))
+      .toReturn;
   });
 
   test('should generate call to wipe by ID', () => {
     expect(async () => await store.wipeInstanceIdData()).toReturn;
   });
-
-});
+});

package/src/neo4j/__tests__/neo4jUtilities.test.ts

@@ -1,5 +1,9 @@
-import { startsWithNumeric, sanitizePropertyName, sanitizeValue, buildPropertyParameters } from '../neo4jUtilities';
-
+import {
+  startsWithNumeric,
+  sanitizePropertyName,
+  sanitizeValue,
+  buildPropertyParameters,
+} from '../neo4jUtilities';
 
 describe('#neo4jUtilities', () => {
   test('should return true for string starting with a numeric', () => {
@@ -11,15 +15,29 @@ describe('#neo4jUtilities', () => {
     expect(testTrailingNumeric).toEqual(false);
   });
   test('should sanitize property name properly', () => {
-    const testSanitize: string = sanitizePropertyName(`1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p'q"r;s:t/u?v.w,x>y<z\`1~2\t3\n4[5]6{7}8 90`);
-    expect(testSanitize).toEqual('n1a_b_c_d_e_f_g_h_i_j_k_l_m_n_o_p_q_r_s_t_u_v_w_x_y_z_1_2_3_4_5_6_7_8_90');
+    const testSanitize: string = sanitizePropertyName(
+      `1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p'q"r;s:t/u?v.w,x>y<z\`1~2\t3\n4[5]6{7}8 90`,
+    );
+    expect(testSanitize).toEqual(
+      'n1a_b_c_d_e_f_g_h_i_j_k_l_m_n_o_p_q_r_s_t_u_v_w_x_y_z_1_2_3_4_5_6_7_8_90',
+    );
   });
   test('should sanitize value properly', () => {
-    const testSanitize: string = sanitizeValue('1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p\'q"r;s:t/u?v.w,x>y<z`1~2\t3\n4[5]6{7}8 90');
-    expect(testSanitize).toEqual('1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p\'q\\"r;s:t/u?v.w,x>y<z`1~2\t3\n4[5]6{7}8 90');
+    const testSanitize: string = sanitizeValue(
+      '1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p\'q"r;s:t/u?v.w,x>y<z`1~2\t3\n4[5]6{7}8 90',
+    );
+    expect(testSanitize).toEqual(
+      '1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p\'q\\"r;s:t/u?v.w,x>y<z`1~2\t3\n4[5]6{7}8 90',
+    );
   });
   test('should build property string correctly including sanitization', () => {
-    const testPropResults: Object = buildPropertyParameters({test: '123', '1sanitize1hi&$abc d': '1h"i&$abc d'});
-    expect(testPropResults).toEqual({test:'123', n1sanitize1hi__abc_d:'1h\\"i&$abc d'});
+    const testPropResults: Object = buildPropertyParameters({
+      test: '123',
+      '1sanitize1hi&$abc d': '1h"i&$abc d',
+    });
+    expect(testPropResults).toEqual({
+      test: '123',
+      n1sanitize1hi__abc_d: '1h\\"i&$abc d',
+    });
   });
-});
+});

package/src/neo4j/index.ts

@@ -1,2 +1,2 @@
 export * from './uploadToNeo4j';
-export * from './wipeNeo4j';
+export * from './wipeNeo4j';