@jupiterone/integration-sdk-cli 7.3.1 → 8.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/integration-sdk-cli",
-  "version": "7.3.1",
+  "version": "8.1.0",
   "description": "The SDK for developing JupiterOne integrations",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -22,31 +22,33 @@
     "prepack": "yarn build:dist"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-runtime": "^7.3.1",
+    "@jupiterone/integration-sdk-runtime": "^8.1.0",
     "commander": "^5.0.0",
     "globby": "^11.0.0",
     "js-yaml": "^4.1.0",
     "json-diff": "^0.5.4",
     "lodash": "^4.17.19",
     "markdown-table": "^2.0.0",
+    "neo4j-driver": "^4.3.3",
     "runtypes": "5.1.0",
     "upath": "^1.2.0",
     "vis": "^4.21.0-EOL"
   },
   "devDependencies": {
-    "@jupiterone/integration-sdk-private-test-utils": "^7.3.1",
-    "@pollyjs/adapter-node-http": "^4.0.4",
-    "@pollyjs/core": "^4.0.4",
-    "@pollyjs/persister-fs": "^4.0.4",
+    "@jupiterone/integration-sdk-private-test-utils": "^8.1.0",
+    "@pollyjs/adapter-node-http": "^5.1.1",
+    "@pollyjs/core": "^5.1.1",
+    "@pollyjs/persister-fs": "^5.1.1",
     "@types/js-yaml": "^4.0.3",
     "@types/json-diff": "^0.5.1",
     "@types/lodash": "^4.14.158",
-    "@types/pollyjs__adapter-node-http": "^2.0.0",
-    "@types/pollyjs__core": "^4.0.0",
-    "@types/pollyjs__persister": "^2.0.1",
+    "@types/pollyjs__adapter-node-http": "^2.0.1",
+    "@types/pollyjs__core": "^4.3.3",
+    "@types/pollyjs__persister": "^4.3.1",
     "@types/vis": "^4.21.20",
     "memfs": "^3.2.0",
+    "neo-forgery": "^2.0.0",
     "uuid": "^8.2.0"
   },
-  "gitHead": "879afd690d8a369030a29b464a86d47daf03f55b"
+  "gitHead": "0ac133e50ed20a51949810d27d6ed873c977fc46"
 }

package/src/commands/index.ts

@@ -6,3 +6,4 @@ export * from './run';
 export * from './document';
 export * from './visualize-types';
 export * from './validate-question-file';
+export * from './neo4j';

package/src/commands/neo4j.ts

@@ -0,0 +1,63 @@
+import * as commander from 'commander';
+import path from 'path';
+
+import dotenv from 'dotenv';
+import dotenvExpand from 'dotenv-expand';
+
+import * as log from '../log';
+import { uploadToNeo4j, wipeNeo4jByID, wipeAllNeo4j } from '../neo4j';
+
+export function neo4j() {
+  dotenvExpand(dotenv.config());
+
+  const program = new commander.Command();
+  program.description(`Suite of neo4j commands.  Options are currently 'neo4j push', 'neo4j wipe', and 'neo4j wipe-all'`);
+  const neo4jCommand = program.command('neo4j');
+  neo4jCommand
+    .command('push')
+    .description('upload collected entities and relationships to local Neo4j')
+    .option(
+      '-d, --data-dir <directory>',
+      'path to collected entities and relationships',
+      path.resolve(process.cwd(), '.j1-integration'),
+    )
+    .option(
+      '-i, --integration-instance-id <id>',
+      '_integrationInstanceId assigned to uploaded entities',
+      'defaultLocalInstanceID'
+    )
+    .action(async (options) => {
+      log.info(`Beginning data upload to local neo4j`);
+      // Point `fileSystem.ts` functions to expected location relative to
+      // integration project path.
+      const finalDir = path.resolve(process.cwd(), options.dataDir);
+      process.env.JUPITERONE_INTEGRATION_STORAGE_DIRECTORY = finalDir;
+
+      await uploadToNeo4j({
+        pathToData: finalDir, 
+        integrationInstanceID: options.integrationInstanceId
+      });
+      log.info(`Data uploaded to local neo4j`);
+    });
+
+    neo4jCommand
+    .command('wipe')
+    .description('wipe entities and relationships for a given integrationInstanceID in the Neo4j database')
+    .option(
+      '-i, --integration-instance-id <id>',
+      '_integrationInstanceId assigned to uploaded entities',
+      'defaultLocalInstanceID'
+    )
+    .action(async (options) => {
+      await wipeNeo4jByID({integrationInstanceID: options.integrationInstanceId});
+    });
+
+    neo4jCommand
+    .command('wipe-all')
+    .description('wipe all entities and relationships in the Neo4j database')
+    .action(async (options) => {
+      await wipeAllNeo4j({});
+    });
+
+    return neo4jCommand;
+}

package/src/index.ts

@@ -9,6 +9,7 @@ import {
   visualize,
   visualizeTypes,
   validateQuestionFile,
+  neo4j,
 } from './commands';
 
 export function createCli() {
@@ -20,5 +21,6 @@ export function createCli() {
     .addCommand(run())
     .addCommand(visualizeTypes())
     .addCommand(document())
-    .addCommand(validateQuestionFile());
+    .addCommand(validateQuestionFile())
+    .addCommand(neo4j());
 }

package/src/neo4j/README.md

@@ -0,0 +1,42 @@
+# Neo4j JupiterOne CLI Command
+
+## Installation
+
+This command assumes you have three additional values stored in your
+local .env file:
+  NEO4J_URI
+  NEO4J_USER
+  NEO4J_PASSWORD
+
+This can be used for uploading to local or remote Neo4j databases.  If
+SSL is needed for a remote connection, specify `bolt+s` or `bolt+ssc` 
+in the URI.  For easy access to a local Neo4j instance, you can launch 
+one via a Neo4j provided Docker image with the command:
+
+```
+docker run \
+    -p 7474:7474 -p 7687:7687 \
+    -d \
+    -v $PWD/.neo4j/data:/data \
+    -v $PWD/.neo4j/logs:/logs \
+    -v $PWD/.neo4j/import:/var/lib/neo4j/import \
+    -v $PWD/.neo4j/plugins:/plugins \
+    --env NEO4J_AUTH=neo4j/devpass \
+    neo4j:latest
+```
+
+If you would like to use a different username and password, the NEO4J_AUTH
+value can be modified to whatever username/password you prefer.
+
+NOTE:  Future updates are planned to streamline this without removing 
+the option of pushing to an external Neo4j database.
+
+## Usage
+
+Data is still collected in the same way as before with a call to `yarn start`.
+
+Once data has been collected, you can run `j1-integration neo4j push`.  This will
+push data to the Neo4j server listed in the NEO4J_URI .env parameter.  If
+running locally, you can then access data in the Neo4j database by visiting
+http://localhost:7474.  Alternatively, you can download the full Neo4j 
+Desktop application at https://neo4j.com/download/.

package/src/neo4j/__tests__/neo4jGraphStore.test.ts

@@ -0,0 +1,99 @@
+import {
+  mockDriver,
+  mockSessionFromQuerySet,
+  QuerySpec
+} from 'neo-forgery';
+import * as neo4j from 'neo4j-driver';
+import { Neo4jGraphStore } from '../neo4jGraphStore';
+import { Entity, Relationship } from '@jupiterone/integration-sdk-core';
+
+const testInstanceID = 'testInstanceID';
+const testEntityData: Entity[] = [{
+  _type: "testType",
+  _class: "testClass",
+  _key: "testKey",
+}];
+const testRelationshipData: Relationship[] = [{
+  _fromEntityKey: "testKey1",
+  _toEntityKey: "testKey2",
+  _type: "testRelType",
+  _key: "relKey",
+  _class: "testRelationshipClass",
+}];
+const constraintCall = 'CREATE CONSTRAINT unique_testType IF NOT EXISTS ON (n:testType) ASSERT n._key IS UNIQUE;'
+const addEntityCall = `MERGE (n:testType {_key: 'testKey', _integrationInstanceID: '${testInstanceID}'}) SET n._type = 'testType', n._class = 'testClass';`;
+const addRelationshipCall = `
+MATCH (start {_key: 'testKey1', _integrationInstanceID: '${testInstanceID}'})
+MATCH (end {_key: 'testKey2', _integrationInstanceID: '${testInstanceID}'})
+MERGE (start)-[relationship:testRelType]->(end);`
+const wipeByIDCall = `MATCH (n {_integrationInstanceID: '${testInstanceID}'}) DETACH DELETE n`;
+const wipeAllCall = 'MATCH (n) DETACH DELETE n';
+const querySet: QuerySpec[] = [{
+  name: 'addConstraint',
+  query: constraintCall,
+  params: undefined,
+  output: {records:[]}
+},
+{
+  name: 'addEntity',
+  query: addEntityCall,
+  params: undefined,
+  output: {records:[]}
+},
+{
+  name: 'addRelationship',
+  query: addRelationshipCall,
+  params: undefined,
+  output: {records:[]}
+},
+{
+  name: 'wipeByID',
+  query: wipeByIDCall,
+  params: undefined,
+  output: {records:[]}
+},
+{
+  name: 'wipeAll',
+  query: wipeAllCall,
+  params: undefined,
+  output: {records:[]}
+}];
+
+describe('#neo4jGraphStore', () => {
+  const mockDriverResp = mockDriver();
+  const mockSession = mockSessionFromQuerySet(querySet);
+  const store = new Neo4jGraphStore({
+    uri: '',
+    username: '',
+    password: '',
+    integrationInstanceID: testInstanceID,
+    session: mockSession,
+  });
+
+  test('should generate call to create a driver connection', () => {
+    const spy = jest.spyOn(neo4j, 'driver').mockReturnValue(mockDriverResp);
+
+    const emptyStore = new Neo4jGraphStore({
+      uri: '',
+      username: '',
+      password: '',
+      integrationInstanceID: testInstanceID,
+    });
+    expect(async () => await emptyStore.close()).toReturn;
+    expect(spy).toHaveBeenCalledTimes(1);
+  });
+
+  test('should generate call to create an Entity', () => {
+    expect(async () => await store.addEntities(testEntityData)).toReturn;
+  });
+
+  test('should generate call to create a Relationship', () => {
+
+    expect(async () => await store.addRelationships(testRelationshipData)).toReturn;
+  });
+
+  test('should generate call to wipe by ID', () => {
+    expect(async () => await store.wipeInstanceIdData()).toReturn;
+  });
+
+});

package/src/neo4j/__tests__/neo4jUtilities.test.ts

@@ -0,0 +1,25 @@
+import { startsWithNumeric, sanitizePropertyName, sanitizeValue, buildPropertyParameters } from '../neo4jUtilities';
+
+
+describe('#neo4jUtilities', () => {
+  test('should return true for string starting with a numeric', () => {
+    const testNameResults: boolean = startsWithNumeric('1testname');
+    expect(testNameResults).toEqual(true);
+  });
+  test('should return false for string not starting with a numeric', () => {
+    const testTrailingNumeric: boolean = startsWithNumeric('another1testname');
+    expect(testTrailingNumeric).toEqual(false);
+  });
+  test('should sanitize property name properly', () => {
+    const testSanitize: string = sanitizePropertyName(`1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p'q"r;s:t/u?v.w,x>y<z\`1~2\t3\n4[5]6{7}8 90`);
+    expect(testSanitize).toEqual('n1a_b_c_d_e_f_g_h_i_j_k_l_m_n_o_p_q_r_s_t_u_v_w_x_y_z_1_2_3_4_5_6_7_8_90');
+  });
+  test('should sanitize value properly', () => {
+    const testSanitize: string = sanitizeValue('1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p\'q"r;s:t/u?v.w,x>y<z`1~2\t3\n4[5]6{7}8 90');
+    expect(testSanitize).toEqual('1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p\'q\\"r;s:t/u?v.w,x>y<z`1~2\t3\n4[5]6{7}8 90');
+  });
+  test('should build property string correctly including sanitization', () => {
+    const testPropResults: Object = buildPropertyParameters({test: '123', '1sanitize1hi&$abc d': '1h"i&$abc d'});
+    expect(testPropResults).toEqual({test:'123', n1sanitize1hi__abc_d:'1h\\"i&$abc d'});
+  });
+});

package/src/neo4j/index.ts

@@ -0,0 +1,2 @@
+export * from './uploadToNeo4j';
+export * from './wipeNeo4j';

package/src/neo4j/neo4jGraphStore.ts

@@ -0,0 +1,139 @@
+import { Entity, Relationship } from '@jupiterone/integration-sdk-core';
+import { sanitizeValue, buildPropertyParameters } from './neo4jUtilities';
+
+import * as neo4j from 'neo4j-driver';
+
+export interface Neo4jGraphObjectStoreParams {
+  uri: string;
+  username: string;
+  password: string;
+  integrationInstanceID: string,
+  session?: neo4j.Session
+}
+
+export class Neo4jGraphStore {
+  private neo4jDriver: neo4j.Driver;
+  private persistedSession: neo4j.Session;
+  private databaseName = 'neo4j';
+  private typeList = new Set<string>();
+  private integrationInstanceID: string;
+
+  constructor(params: Neo4jGraphObjectStoreParams) {
+    if(params.session) {
+      this.persistedSession = params.session;
+    }
+    else {
+      this.neo4jDriver = neo4j.driver(
+        params.uri,
+        neo4j.auth.basic(params.username, params.password),
+      );
+    }
+    this.integrationInstanceID = params.integrationInstanceID;
+  }
+
+  private async runCypherCommand(cypherCommand: string, cypherParameters?: any): Promise<neo4j.Result> {
+    if(this.persistedSession) {
+      const result = await this.persistedSession.run(cypherCommand);
+      return result;
+    }
+    else {
+      const session = this.neo4jDriver.session({
+        database: this.databaseName,
+        defaultAccessMode: neo4j.session.WRITE,
+      });
+      const result = await session.run(cypherCommand, cypherParameters);
+      await session.close();
+      return result;
+    }
+  }
+
+  async addEntities(newEntities: Entity[]) {
+    const nodeAlias: string = 'entityNode';
+    for (const entity of newEntities) {
+      //Add index if not already in types.  This will optimize future
+      //MATCH/MERGE calls.
+      if (!this.typeList.has(entity._type)) {
+        await this.runCypherCommand(
+          `CREATE INDEX index_${entity._type} IF NOT EXISTS FOR (n:${entity._type}) ON (n._key, n._integrationInstanceID);`,
+        );
+        this.typeList.add(entity._type);
+      }
+      const propertyParameters = buildPropertyParameters(entity);
+      const finalKeyValue = sanitizeValue(entity._key.toString());
+      const buildCommand = `
+        MERGE (${nodeAlias} {_key: $finalKeyValue, _integrationInstanceID: $integrationInstanceID}) 
+        SET ${nodeAlias} += $propertyParameters
+        SET ${nodeAlias}:${entity._type};`;
+      await this.runCypherCommand(buildCommand, {
+        propertyParameters: propertyParameters, 
+        finalKeyValue: finalKeyValue, 
+        integrationInstanceID: this.integrationInstanceID
+      });
+    }
+  }
+
+  async addRelationships(newRelationships: Relationship[]) {
+    for (const relationship of newRelationships) {
+      const relationshipAlias: string = 'relationship';
+      const propertyParameters = buildPropertyParameters(relationship);
+
+      let startEntityKey = '';
+      let endEntityKey = '';
+      //Get start and end _keys.  Will be overwritten if we're
+      //working with a mapped relationship.
+      if (relationship._fromEntityKey) {
+        startEntityKey = sanitizeValue(relationship._fromEntityKey.toString());
+      }
+      if(relationship._toEntityKey) {
+        endEntityKey = sanitizeValue(relationship._toEntityKey.toString());
+      }
+
+      if(relationship._mapping) { //Mapped Relationship
+        if(relationship._mapping['skipTargetCreation'] === false) {
+          //Create target entity first
+          const tempEntity: Entity = {
+            _class: relationship._mapping['targetEntity']._class,
+            //TODO, I think this key is wrong, but not sure what else to use
+            _key:  sanitizeValue(relationship._key.replace(relationship._mapping['sourceEntityKey'], '')),
+            _type: relationship._mapping['targetEntity']._type,
+          }
+          await this.addEntities([tempEntity]);
+        }
+        startEntityKey = sanitizeValue(relationship._mapping['sourceEntityKey']);
+        // TODO, see above.  This key might also be an issue for the same reason
+        endEntityKey = sanitizeValue(relationship._key.replace(relationship._mapping['sourceEntityKey'], ''));
+      }
+
+      const buildCommand = `
+      MERGE (start {_key: $startEntityKey, _integrationInstanceID: $integrationInstanceID})
+      MERGE (end {_key: $endEntityKey, _integrationInstanceID: $integrationInstanceID})
+      MERGE (start)-[${relationshipAlias}:${relationship._type}]->(end)
+      SET ${relationshipAlias} += $propertyParameters;`;
+      await this.runCypherCommand(buildCommand, {
+        propertyParameters: propertyParameters, 
+        startEntityKey: startEntityKey, 
+        endEntityKey: endEntityKey,
+        integrationInstanceID: this.integrationInstanceID
+      });
+    }
+  }
+
+  // TODO, if we get to very large databases we could reach a size where
+  // one or both both of the below wipe commands can't be easily executed 
+  // in memory.  At that time, we should consider requiring/using the APOC 
+  // library so we can use apoc.periodic.iterate.  Leaving out for now,
+  // since that would further complicate the Neo4j database setup.
+  async wipeInstanceIdData() {
+    const wipeCypherCommand = `MATCH (n {_integrationInstanceID: '${this.integrationInstanceID}'}) DETACH DELETE n`;
+    await this.runCypherCommand(wipeCypherCommand);
+  }
+
+  async wipeDatabase() {
+    const wipeCypherCommand = `MATCH (n) DETACH DELETE n`;
+    await this.runCypherCommand(wipeCypherCommand);
+  }
+
+  async close() {
+    await this.neo4jDriver.close();
+  }
+}

package/src/neo4j/neo4jUtilities.ts

@@ -0,0 +1,43 @@
+
+export function startsWithNumeric(str: string): boolean{
+  return /^\d/.test(str);
+}
+
+export function sanitizePropertyName(propertyName: string): string {
+  let sanitizedName = '';
+  if(startsWithNumeric(propertyName)) {
+    sanitizedName += 'n';
+  }
+  sanitizedName += propertyName;
+  sanitizedName = sanitizedName.replace(/[\s!@#$%^&*()\-=+\\|'";:/?.,><`~\t\n[\]{}]/g, "_");
+  return sanitizedName;
+}
+
+export function sanitizeValue(value: string): string {
+  return value.replace(/"/gi, '\\"')
+}
+
+export function buildPropertyParameters(propList: Object) {
+  const propertyParameters = {};
+  for (const key in propList) {
+    if (key === '_rawData') {
+      //stringify JSON in rawData so we can store it.
+      propertyParameters[key] = `"${JSON.stringify(propList[key])}"`;
+    } else {
+      // Sanitize out characters that aren't allowed in property names
+      const propertyName = sanitizePropertyName(key);
+
+      //If we're dealing with a number or boolean, leave alone, otherwise
+      //wrap in single quotes to convert to a string and escape all
+      //other single quotes so they don't terminate strings prematurely.
+      if(typeof propList[key] == 'number' || typeof propList[key] == 'boolean') {
+        propertyParameters[propertyName] = propList[key];
+      }
+      else {
+        propertyParameters[propertyName] = sanitizeValue(propList[key].toString());
+      }
+    }
+  }
+
+  return propertyParameters;
+}

package/src/neo4j/uploadToNeo4j.ts

@@ -0,0 +1,47 @@
+import { Neo4jGraphStore } from './neo4jGraphStore';
+import { iterateParsedGraphFiles, isDirectoryPresent } from '@jupiterone/integration-sdk-runtime';
+import { FlushedGraphObjectData } from '@jupiterone/integration-sdk-runtime/src/storage/types';
+
+type UploadToNeo4jParams = {
+  pathToData: string;
+  integrationInstanceID: string;
+  neo4jUri?: string;
+  neo4jUser?: string;
+  neo4jPassword?: string;
+};
+
+export async function uploadToNeo4j({
+  pathToData,
+  integrationInstanceID,
+  neo4jUri = process.env.NEO4J_URI,
+  neo4jUser = process.env.NEO4J_USER,
+  neo4jPassword = process.env.NEO4J_PASSWORD,
+}: UploadToNeo4jParams) {
+  if (!neo4jUri || !neo4jUser || !neo4jPassword) {
+    throw new Error(
+      'ERROR: must provide login information in function call or include NEO4J_URI, NEO4J_USER, and NEO4J_PASSWORD files in your .env file!',
+    );
+  }
+  if (!isDirectoryPresent(pathToData)) {
+    throw new Error('ERROR: graph directory does not exist!');
+  }
+
+  const store = new Neo4jGraphStore({
+    uri: neo4jUri,
+    username: neo4jUser,
+    password: neo4jPassword,
+    integrationInstanceID: integrationInstanceID,
+  });
+
+  async function handleGraphObjectFile(parsedData: FlushedGraphObjectData) {
+    if (parsedData.entities) await store.addEntities(parsedData.entities);
+    if (parsedData.relationships)
+      await store.addRelationships(parsedData.relationships);
+  }
+
+  try {
+    await iterateParsedGraphFiles(handleGraphObjectFile, pathToData);
+  } finally {
+    await store.close();
+  }
+}

package/src/neo4j/wipeNeo4j.ts

@@ -0,0 +1,58 @@
+import { Neo4jGraphStore } from './neo4jGraphStore';
+
+type WipeNeo4jParams = {
+  integrationInstanceID: string;
+  neo4jUri?: string;
+  neo4jUser?: string;
+  neo4jPassword?: string;
+}
+export async function wipeNeo4jByID({
+  integrationInstanceID,
+  neo4jUri = process.env.NEO4J_URI,
+  neo4jUser = process.env. NEO4J_USER,
+  neo4jPassword = process.env. NEO4J_PASSWORD,
+}: WipeNeo4jParams) {
+  if(!neo4jUri || !neo4jUser || !neo4jPassword) {
+    throw new Error('ERROR: must provide login information in function call or include NEO4J_URI, NEO4J_USER, and NEO4J_PASSWORD files in your .env file!');
+  }
+
+  const store = new Neo4jGraphStore({
+    uri: neo4jUri,
+    username: neo4jUser,
+    password: neo4jPassword,
+    integrationInstanceID: integrationInstanceID,
+  });
+  try {
+    await store.wipeInstanceIdData();
+  } finally {
+    await store.close();
+  }
+}
+
+type WipeAllNeo4jParams = {
+  neo4jUri?: string;
+  neo4jUser?: string;
+  neo4jPassword?: string;
+}
+
+export async function wipeAllNeo4j({
+  neo4jUri = process.env.NEO4J_URI,
+  neo4jUser = process.env. NEO4J_USER,
+  neo4jPassword = process.env. NEO4J_PASSWORD,
+}: WipeAllNeo4jParams) {
+  if(!neo4jUri || !neo4jUser || !neo4jPassword) {
+    throw new Error('ERROR: must provide login information in function call or include NEO4J_URI, NEO4J_USER, and NEO4J_PASSWORD files in your .env file!');
+  }
+
+  const store = new Neo4jGraphStore({
+    uri: neo4jUri,
+    username: neo4jUser,
+    password: neo4jPassword,
+    integrationInstanceID: '',
+  });
+  try {
+    await store.wipeDatabase();
+  } finally {
+    await store.close();
+  }
+}