@jupiterone/integration-sdk-cli 8.4.1 → 8.4.4

package/src/neo4j/README.md

@@ -2,16 +2,13 @@
 
 ## Installation
 
-This command assumes you have three additional values stored in your
-local .env file:
-  NEO4J_URI
-  NEO4J_USER
-  NEO4J_PASSWORD
+This command assumes you have three additional values stored in your local .env
+file: NEO4J_URI NEO4J_USER NEO4J_PASSWORD
 
-This can be used for uploading to local or remote Neo4j databases.  If
-SSL is needed for a remote connection, specify `bolt+s` or `bolt+ssc` 
-in the URI.  For easy access to a local Neo4j instance, you can launch 
-one via a Neo4j provided Docker image with the command:
+This can be used for uploading to local or remote Neo4j databases. If SSL is
+needed for a remote connection, specify `bolt+s` or `bolt+ssc` in the URI. For
+easy access to a local Neo4j instance, you can launch one via a Neo4j provided
+Docker image with the command:
 
 ```
 docker run \
@@ -25,18 +22,18 @@ docker run \
     neo4j:latest
 ```
 
-If you would like to use a different username and password, the NEO4J_AUTH
-value can be modified to whatever username/password you prefer.
+If you would like to use a different username and password, the NEO4J_AUTH value
+can be modified to whatever username/password you prefer.
 
-NOTE:  Future updates are planned to streamline this without removing 
-the option of pushing to an external Neo4j database.
+NOTE: Future updates are planned to streamline this without removing the option
+of pushing to an external Neo4j database.
 
 ## Usage
 
 Data is still collected in the same way as before with a call to `yarn start`.
 
-Once data has been collected, you can run `j1-integration neo4j push`.  This will
-push data to the Neo4j server listed in the NEO4J_URI .env parameter.  If
-running locally, you can then access data in the Neo4j database by visiting
-http://localhost:7474.  Alternatively, you can download the full Neo4j 
-Desktop application at https://neo4j.com/download/.
+Once data has been collected, you can run `j1-integration neo4j push`. This will
+push data to the Neo4j server listed in the NEO4J_URI .env parameter. If running
+locally, you can then access data in the Neo4j database by visiting
+http://localhost:7474. Alternatively, you can download the full Neo4j Desktop
+application at https://neo4j.com/download/.

package/src/neo4j/__tests__/neo4jGraphStore.test.ts

@@ -1,63 +1,66 @@
-import {
-  mockDriver,
-  mockSessionFromQuerySet,
-  QuerySpec
-} from 'neo-forgery';
+import { mockDriver, mockSessionFromQuerySet, QuerySpec } from 'neo-forgery';
 import * as neo4j from 'neo4j-driver';
 import { Neo4jGraphStore } from '../neo4jGraphStore';
 import { Entity, Relationship } from '@jupiterone/integration-sdk-core';
 
 const testInstanceID = 'testInstanceID';
-const testEntityData: Entity[] = [{
-  _type: "testType",
-  _class: "testClass",
-  _key: "testKey",
-}];
-const testRelationshipData: Relationship[] = [{
-  _fromEntityKey: "testKey1",
-  _toEntityKey: "testKey2",
-  _type: "testRelType",
-  _key: "relKey",
-  _class: "testRelationshipClass",
-}];
-const constraintCall = 'CREATE CONSTRAINT unique_testType IF NOT EXISTS ON (n:testType) ASSERT n._key IS UNIQUE;'
+const testEntityData: Entity[] = [
+  {
+    _type: 'testType',
+    _class: 'testClass',
+    _key: 'testKey',
+  },
+];
+const testRelationshipData: Relationship[] = [
+  {
+    _fromEntityKey: 'testKey1',
+    _toEntityKey: 'testKey2',
+    _type: 'testRelType',
+    _key: 'relKey',
+    _class: 'testRelationshipClass',
+  },
+];
+const constraintCall =
+  'CREATE CONSTRAINT unique_testType IF NOT EXISTS ON (n:testType) ASSERT n._key IS UNIQUE;';
 const addEntityCall = `MERGE (n:testType {_key: 'testKey', _integrationInstanceID: '${testInstanceID}'}) SET n._type = 'testType', n._class = 'testClass';`;
 const addRelationshipCall = `
 MATCH (start {_key: 'testKey1', _integrationInstanceID: '${testInstanceID}'})
 MATCH (end {_key: 'testKey2', _integrationInstanceID: '${testInstanceID}'})
-MERGE (start)-[relationship:testRelType]->(end);`
+MERGE (start)-[relationship:testRelType]->(end);`;
 const wipeByIDCall = `MATCH (n {_integrationInstanceID: '${testInstanceID}'}) DETACH DELETE n`;
 const wipeAllCall = 'MATCH (n) DETACH DELETE n';
-const querySet: QuerySpec[] = [{
-  name: 'addConstraint',
-  query: constraintCall,
-  params: undefined,
-  output: {records:[]}
-},
-{
-  name: 'addEntity',
-  query: addEntityCall,
-  params: undefined,
-  output: {records:[]}
-},
-{
-  name: 'addRelationship',
-  query: addRelationshipCall,
-  params: undefined,
-  output: {records:[]}
-},
-{
-  name: 'wipeByID',
-  query: wipeByIDCall,
-  params: undefined,
-  output: {records:[]}
-},
-{
-  name: 'wipeAll',
-  query: wipeAllCall,
-  params: undefined,
-  output: {records:[]}
-}];
+const querySet: QuerySpec[] = [
+  {
+    name: 'addConstraint',
+    query: constraintCall,
+    params: undefined,
+    output: { records: [] },
+  },
+  {
+    name: 'addEntity',
+    query: addEntityCall,
+    params: undefined,
+    output: { records: [] },
+  },
+  {
+    name: 'addRelationship',
+    query: addRelationshipCall,
+    params: undefined,
+    output: { records: [] },
+  },
+  {
+    name: 'wipeByID',
+    query: wipeByIDCall,
+    params: undefined,
+    output: { records: [] },
+  },
+  {
+    name: 'wipeAll',
+    query: wipeAllCall,
+    params: undefined,
+    output: { records: [] },
+  },
+];
 
 describe('#neo4jGraphStore', () => {
   const mockDriverResp = mockDriver();
@@ -88,12 +91,11 @@ describe('#neo4jGraphStore', () => {
   });
 
   test('should generate call to create a Relationship', () => {
-
-    expect(async () => await store.addRelationships(testRelationshipData)).toReturn;
+    expect(async () => await store.addRelationships(testRelationshipData))
+      .toReturn;
   });
 
   test('should generate call to wipe by ID', () => {
     expect(async () => await store.wipeInstanceIdData()).toReturn;
   });
-
-});
+});

package/src/neo4j/__tests__/neo4jUtilities.test.ts

@@ -1,5 +1,9 @@
-import { startsWithNumeric, sanitizePropertyName, sanitizeValue, buildPropertyParameters } from '../neo4jUtilities';
-
+import {
+  startsWithNumeric,
+  sanitizePropertyName,
+  sanitizeValue,
+  buildPropertyParameters,
+} from '../neo4jUtilities';
 
 describe('#neo4jUtilities', () => {
   test('should return true for string starting with a numeric', () => {
@@ -11,15 +15,67 @@ describe('#neo4jUtilities', () => {
     expect(testTrailingNumeric).toEqual(false);
   });
   test('should sanitize property name properly', () => {
-    const testSanitize: string = sanitizePropertyName(`1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p'q"r;s:t/u?v.w,x>y<z\`1~2\t3\n4[5]6{7}8 90`);
-    expect(testSanitize).toEqual('n1a_b_c_d_e_f_g_h_i_j_k_l_m_n_o_p_q_r_s_t_u_v_w_x_y_z_1_2_3_4_5_6_7_8_90');
+    const testSanitize: string = sanitizePropertyName(
+      `1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p'q"r;s:t/u?v.w,x>y<z\`1~2\t3\n4[5]6{7}8 90`,
+    );
+    expect(testSanitize).toEqual(
+      'n1a_b_c_d_e_f_g_h_i_j_k_l_m_n_o_p_q_r_s_t_u_v_w_x_y_z_1_2_3_4_5_6_7_8_90',
+    );
   });
   test('should sanitize value properly', () => {
-    const testSanitize: string = sanitizeValue('1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p\'q"r;s:t/u?v.w,x>y<z`1~2\t3\n4[5]6{7}8 90');
-    expect(testSanitize).toEqual('1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p\'q\\"r;s:t/u?v.w,x>y<z`1~2\t3\n4[5]6{7}8 90');
+    const testSanitize: string = sanitizeValue(
+      '1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p\'q"r;s:t/u?v.w,x>y<z`1~2\t3\n4[5]6{7}8 90',
+    );
+    expect(testSanitize).toEqual(
+      '1a!b@c#d$e%f^g&h*i(j)k-l=m+n\\o|p\'q\\"r;s:t/u?v.w,x>y<z`1~2\t3\n4[5]6{7}8 90',
+    );
   });
+});
+
+describe('#buildPropertyParameters', () => {
   test('should build property string correctly including sanitization', () => {
-    const testPropResults: Object = buildPropertyParameters({test: '123', '1sanitize1hi&$abc d': '1h"i&$abc d'});
-    expect(testPropResults).toEqual({test:'123', n1sanitize1hi__abc_d:'1h\\"i&$abc d'});
+    expect(
+      buildPropertyParameters({
+        test: '123',
+        '1sanitize1hi&$abc d': '1h"i&$abc d',
+      }),
+    ).toEqual({
+      test: '123',
+      n1sanitize1hi__abc_d: '1h\\"i&$abc d',
+    });
+  });
+
+  test('should ignore properties with the value "undefined"', () => {
+    expect(
+      buildPropertyParameters({
+        test: '123',
+        ignore: undefined,
+      }),
+    ).toEqual({
+      test: '123',
+    });
+  });
+
+  test('should ignore properties with the value "null"', () => {
+    expect(
+      buildPropertyParameters({
+        test: '123',
+        ignore: null,
+      }),
+    ).toEqual({
+      test: '123',
+    });
+  });
+
+  test('should not ignore "false" property values', () => {
+    expect(
+      buildPropertyParameters({
+        test: '123',
+        include: false,
+      }),
+    ).toEqual({
+      test: '123',
+      include: false,
+    });
   });
-});
+});

package/src/neo4j/index.ts

@@ -1,2 +1,2 @@
 export * from './uploadToNeo4j';
-export * from './wipeNeo4j';
+export * from './wipeNeo4j';

package/src/neo4j/neo4jGraphStore.ts

@@ -1,5 +1,9 @@
 import { Entity, Relationship } from '@jupiterone/integration-sdk-core';
-import { sanitizeValue, buildPropertyParameters } from './neo4jUtilities';
+import {
+  sanitizeValue,
+  buildPropertyParameters,
+  sanitizePropertyName,
+} from './neo4jUtilities';
 
 import * as neo4j from 'neo4j-driver';
 
@@ -7,8 +11,8 @@ export interface Neo4jGraphObjectStoreParams {
   uri: string;
   username: string;
   password: string;
-  integrationInstanceID: string,
-  session?: neo4j.Session
+  integrationInstanceID: string;
+  session?: neo4j.Session;
 }
 
 export class Neo4jGraphStore {
@@ -19,10 +23,9 @@ export class Neo4jGraphStore {
   private integrationInstanceID: string;
 
   constructor(params: Neo4jGraphObjectStoreParams) {
-    if(params.session) {
+    if (params.session) {
       this.persistedSession = params.session;
-    }
-    else {
+    } else {
       this.neo4jDriver = neo4j.driver(
         params.uri,
         neo4j.auth.basic(params.username, params.password),
@@ -31,12 +34,14 @@ export class Neo4jGraphStore {
     this.integrationInstanceID = params.integrationInstanceID;
   }
 
-  private async runCypherCommand(cypherCommand: string, cypherParameters?: any): Promise<neo4j.Result> {
-    if(this.persistedSession) {
+  private async runCypherCommand(
+    cypherCommand: string,
+    cypherParameters?: any,
+  ): Promise<neo4j.Result> {
+    if (this.persistedSession) {
       const result = await this.persistedSession.run(cypherCommand);
       return result;
-    }
-    else {
+    } else {
       const session = this.neo4jDriver.session({
         database: this.databaseName,
         defaultAccessMode: neo4j.session.WRITE,
@@ -50,24 +55,37 @@ export class Neo4jGraphStore {
   async addEntities(newEntities: Entity[]) {
     const nodeAlias: string = 'entityNode';
     for (const entity of newEntities) {
-      //Add index if not already in types.  This will optimize future
-      //MATCH/MERGE calls.
+      let classLabels = '';
+      if (entity._class) {
+        if (typeof entity._class === 'string') {
+          classLabels += `:${sanitizePropertyName(entity._class)}`;
+        } else {
+          for (const className of entity._class) {
+            classLabels += `:${sanitizePropertyName(className)}`;
+          }
+        }
+      }
+      // I believe we currently can't use parameters for node labels, hence the use of string
+      // interpolation in the below commands.
+      // Add index if not already in types.  This will optimize future
+      // MATCH/MERGE calls.
       if (!this.typeList.has(entity._type)) {
         await this.runCypherCommand(
           `CREATE INDEX index_${entity._type} IF NOT EXISTS FOR (n:${entity._type}) ON (n._key, n._integrationInstanceID);`,
         );
         this.typeList.add(entity._type);
       }
+      const sanitizedType = sanitizePropertyName(entity._type);
       const propertyParameters = buildPropertyParameters(entity);
       const finalKeyValue = sanitizeValue(entity._key.toString());
       const buildCommand = `
         MERGE (${nodeAlias} {_key: $finalKeyValue, _integrationInstanceID: $integrationInstanceID}) 
         SET ${nodeAlias} += $propertyParameters
-        SET ${nodeAlias}:${entity._type};`;
+        SET ${nodeAlias}:${sanitizedType}${classLabels};`;
       await this.runCypherCommand(buildCommand, {
-        propertyParameters: propertyParameters, 
-        finalKeyValue: finalKeyValue, 
-        integrationInstanceID: this.integrationInstanceID
+        propertyParameters: propertyParameters,
+        finalKeyValue: finalKeyValue,
+        integrationInstanceID: this.integrationInstanceID,
       });
     }
   }
@@ -84,43 +102,60 @@ export class Neo4jGraphStore {
       if (relationship._fromEntityKey) {
         startEntityKey = sanitizeValue(relationship._fromEntityKey.toString());
       }
-      if(relationship._toEntityKey) {
+      if (relationship._toEntityKey) {
         endEntityKey = sanitizeValue(relationship._toEntityKey.toString());
       }
 
-      if(relationship._mapping) { //Mapped Relationship
-        if(relationship._mapping['skipTargetCreation'] === false) {
+      if (relationship._mapping) {
+        //Mapped Relationship
+        if (relationship._mapping['skipTargetCreation'] === false) {
           //Create target entity first
           const tempEntity: Entity = {
             _class: relationship._mapping['targetEntity']._class,
             //TODO, I think this key is wrong, but not sure what else to use
-            _key:  sanitizeValue(relationship._key.replace(relationship._mapping['sourceEntityKey'], '')),
+            _key: sanitizeValue(
+              relationship._key.replace(
+                relationship._mapping['sourceEntityKey'],
+                '',
+              ),
+            ),
             _type: relationship._mapping['targetEntity']._type,
-          }
+          };
           await this.addEntities([tempEntity]);
         }
-        startEntityKey = sanitizeValue(relationship._mapping['sourceEntityKey']);
+        startEntityKey = sanitizeValue(
+          relationship._mapping['sourceEntityKey'],
+        );
         // TODO, see above.  This key might also be an issue for the same reason
-        endEntityKey = sanitizeValue(relationship._key.replace(relationship._mapping['sourceEntityKey'], ''));
+        endEntityKey = sanitizeValue(
+          relationship._key.replace(
+            relationship._mapping['sourceEntityKey'],
+            '',
+          ),
+        );
       }
 
+      const sanitizedRelationshipClass = sanitizePropertyName(
+        relationship._class,
+      );
+
       const buildCommand = `
       MERGE (start {_key: $startEntityKey, _integrationInstanceID: $integrationInstanceID})
       MERGE (end {_key: $endEntityKey, _integrationInstanceID: $integrationInstanceID})
-      MERGE (start)-[${relationshipAlias}:${relationship._type}]->(end)
+      MERGE (start)-[${relationshipAlias}:${sanitizedRelationshipClass}]->(end)
       SET ${relationshipAlias} += $propertyParameters;`;
       await this.runCypherCommand(buildCommand, {
-        propertyParameters: propertyParameters, 
-        startEntityKey: startEntityKey, 
+        propertyParameters: propertyParameters,
+        startEntityKey: startEntityKey,
         endEntityKey: endEntityKey,
-        integrationInstanceID: this.integrationInstanceID
+        integrationInstanceID: this.integrationInstanceID,
       });
     }
   }
 
   // TODO, if we get to very large databases we could reach a size where
-  // one or both both of the below wipe commands can't be easily executed 
-  // in memory.  At that time, we should consider requiring/using the APOC 
+  // one or both both of the below wipe commands can't be easily executed
+  // in memory.  At that time, we should consider requiring/using the APOC
   // library so we can use apoc.periodic.iterate.  Leaving out for now,
   // since that would further complicate the Neo4j database setup.
   async wipeInstanceIdData() {

package/src/neo4j/neo4jUtilities.ts

@@ -21,28 +21,29 @@ export function sanitizeValue(value: string): string {
 
 export function buildPropertyParameters(propList: Object) {
   const propertyParameters = {};
+
   for (const key in propList) {
+    const propVal = propList[key];
+
     if (key === '_rawData') {
-      //stringify JSON in rawData so we can store it.
-      propertyParameters[key] = `"${JSON.stringify(propList[key])}"`;
+      // stringify JSON in rawData so we can store it.
+      propertyParameters[key] = `"${JSON.stringify(propVal)}"`;
     } else {
       // Sanitize out characters that aren't allowed in property names
       const propertyName = sanitizePropertyName(key);
 
-      //If we're dealing with a number or boolean, leave alone, otherwise
-      //wrap in single quotes to convert to a string and escape all
-      //other single quotes so they don't terminate strings prematurely.
-      if (propList[key]) {
-        if (
-          typeof propList[key] == 'number' ||
-          typeof propList[key] == 'boolean'
-        ) {
-          propertyParameters[propertyName] = propList[key];
-        } else {
-          propertyParameters[propertyName] = sanitizeValue(
-            propList[key].toString(),
-          );
-        }
+      if (propVal === undefined || propVal === null) {
+        // Ignore properties that have the value `undefined` or `null`.
+        continue;
+      }
+
+      // If we're dealing with a number or boolean, leave alone, otherwise
+      // wrap in single quotes to convert to a string and escape all
+      // other single quotes so they don't terminate strings prematurely.
+      if (typeof propVal == 'number' || typeof propVal == 'boolean') {
+        propertyParameters[propertyName] = propVal;
+      } else {
+        propertyParameters[propertyName] = sanitizeValue(propVal.toString());
       }
     }
   }

package/src/neo4j/uploadToNeo4j.ts

@@ -1,5 +1,8 @@
 import { Neo4jGraphStore } from './neo4jGraphStore';
-import { iterateParsedGraphFiles, isDirectoryPresent } from '@jupiterone/integration-sdk-runtime';
+import {
+  iterateParsedGraphFiles,
+  isDirectoryPresent,
+} from '@jupiterone/integration-sdk-runtime';
 import { FlushedGraphObjectData } from '@jupiterone/integration-sdk-runtime/src/storage/types';
 
 type UploadToNeo4jParams = {
@@ -44,4 +47,4 @@ export async function uploadToNeo4j({
   } finally {
     await store.close();
   }
-}
+}

package/src/neo4j/wipeNeo4j.ts

@@ -5,15 +5,17 @@ type WipeNeo4jParams = {
   neo4jUri?: string;
   neo4jUser?: string;
   neo4jPassword?: string;
-}
+};
 export async function wipeNeo4jByID({
   integrationInstanceID,
   neo4jUri = process.env.NEO4J_URI,
-  neo4jUser = process.env. NEO4J_USER,
-  neo4jPassword = process.env. NEO4J_PASSWORD,
+  neo4jUser = process.env.NEO4J_USER,
+  neo4jPassword = process.env.NEO4J_PASSWORD,
 }: WipeNeo4jParams) {
-  if(!neo4jUri || !neo4jUser || !neo4jPassword) {
-    throw new Error('ERROR: must provide login information in function call or include NEO4J_URI, NEO4J_USER, and NEO4J_PASSWORD files in your .env file!');
+  if (!neo4jUri || !neo4jUser || !neo4jPassword) {
+    throw new Error(
+      'ERROR: must provide login information in function call or include NEO4J_URI, NEO4J_USER, and NEO4J_PASSWORD files in your .env file!',
+    );
   }
 
   const store = new Neo4jGraphStore({
@@ -33,15 +35,17 @@ type WipeAllNeo4jParams = {
   neo4jUri?: string;
   neo4jUser?: string;
   neo4jPassword?: string;
-}
+};
 
 export async function wipeAllNeo4j({
   neo4jUri = process.env.NEO4J_URI,
-  neo4jUser = process.env. NEO4J_USER,
-  neo4jPassword = process.env. NEO4J_PASSWORD,
+  neo4jUser = process.env.NEO4J_USER,
+  neo4jPassword = process.env.NEO4J_PASSWORD,
 }: WipeAllNeo4jParams) {
-  if(!neo4jUri || !neo4jUser || !neo4jPassword) {
-    throw new Error('ERROR: must provide login information in function call or include NEO4J_URI, NEO4J_USER, and NEO4J_PASSWORD files in your .env file!');
+  if (!neo4jUri || !neo4jUser || !neo4jPassword) {
+    throw new Error(
+      'ERROR: must provide login information in function call or include NEO4J_URI, NEO4J_USER, and NEO4J_PASSWORD files in your .env file!',
+    );
   }
 
   const store = new Neo4jGraphStore({
@@ -55,4 +59,4 @@ export async function wipeAllNeo4j({
   } finally {
     await store.close();
   }
-}
+}