@jupiterone/integration-sdk-cli 8.17.0 → 8.19.0

package/src/commands/generate-integration-graph-schema.test.ts

@@ -0,0 +1,228 @@
+import {
+  IntegrationInstanceConfig,
+  IntegrationStep,
+  RelationshipClass,
+  RelationshipDirection,
+} from '@jupiterone/integration-sdk-core';
+import { getMockIntegrationStep } from '@jupiterone/integration-sdk-private-test-utils';
+import { generateIntegrationGraphSchema } from './generate-integration-graph-schema';
+
+describe('#generateIntegrationGraphSchema', () => {
+  test('should return aggregate schema from steps', () => {
+    const steps: IntegrationStep<IntegrationInstanceConfig>[] = [
+      getMockIntegrationStep({
+        entities: [
+          {
+            _class: 'User',
+            _type: 'my_user',
+            resourceName: 'The user',
+          },
+          {
+            _class: 'Group',
+            _type: 'my_group',
+            resourceName: 'The group',
+          },
+        ],
+        mappedRelationships: [
+          {
+            _class: RelationshipClass.IS,
+            sourceType: 'my_user',
+            targetType: 'employee',
+            direction: RelationshipDirection.FORWARD,
+            _type: 'my_user_is_employee',
+          },
+        ],
+      }),
+      getMockIntegrationStep({
+        entities: [
+          {
+            _class: 'Group',
+            _type: 'my_group',
+            resourceName: 'The group',
+          },
+        ],
+        relationships: [
+          {
+            _class: RelationshipClass.HAS,
+            _type: 'my_group_has_user',
+            sourceType: 'my_user',
+            targetType: 'my_group',
+          },
+        ],
+        mappedRelationships: [],
+      }),
+    ];
+
+    expect(generateIntegrationGraphSchema(steps)).toEqual({
+      entities: [
+        {
+          _class: 'User',
+          _type: 'my_user',
+          resourceName: 'The user',
+        },
+        {
+          _class: 'Group',
+          _type: 'my_group',
+          resourceName: 'The group',
+        },
+      ],
+      relationships: [
+        {
+          _class: RelationshipClass.HAS,
+          sourceType: 'my_user',
+          targetType: 'my_group',
+        },
+      ],
+      mappedRelationships: [
+        {
+          _class: RelationshipClass.IS,
+          sourceType: 'my_user',
+          targetType: 'employee',
+          direction: RelationshipDirection.FORWARD,
+        },
+      ],
+    });
+  });
+
+  test('should dedup entity metadata', () => {
+    const steps: IntegrationStep<IntegrationInstanceConfig>[] = [
+      getMockIntegrationStep({
+        entities: [
+          {
+            _class: 'User',
+            _type: 'my_user',
+            resourceName: 'The user',
+          },
+          {
+            _class: 'User',
+            _type: 'my_user',
+            resourceName: 'The user',
+          },
+        ],
+        relationships: [],
+        mappedRelationships: [],
+      }),
+      getMockIntegrationStep({
+        entities: [
+          {
+            _class: 'User',
+            _type: 'my_user',
+            resourceName: 'The user',
+          },
+        ],
+        relationships: [],
+        mappedRelationships: [],
+      }),
+    ];
+
+    expect(generateIntegrationGraphSchema(steps)).toEqual({
+      entities: [
+        {
+          _class: 'User',
+          _type: 'my_user',
+          resourceName: 'The user',
+        },
+      ],
+      relationships: [],
+      mappedRelationships: [],
+    });
+  });
+
+  test('should dedup relationship metadata', () => {
+    const steps: IntegrationStep<IntegrationInstanceConfig>[] = [
+      getMockIntegrationStep({
+        entities: [],
+        relationships: [
+          {
+            _class: RelationshipClass.HAS,
+            _type: 'my_group_has_user',
+            sourceType: 'my_user',
+            targetType: 'my_group',
+          },
+          {
+            _class: RelationshipClass.HAS,
+            _type: 'my_group_has_user',
+            sourceType: 'my_user',
+            targetType: 'my_group',
+          },
+        ],
+        mappedRelationships: [],
+      }),
+      getMockIntegrationStep({
+        entities: [],
+        relationships: [
+          {
+            _class: RelationshipClass.HAS,
+            _type: 'my_group_has_user',
+            sourceType: 'my_user',
+            targetType: 'my_group',
+          },
+        ],
+        mappedRelationships: [],
+      }),
+    ];
+
+    expect(generateIntegrationGraphSchema(steps)).toEqual({
+      entities: [],
+      relationships: [
+        {
+          _class: RelationshipClass.HAS,
+          sourceType: 'my_user',
+          targetType: 'my_group',
+        },
+      ],
+      mappedRelationships: [],
+    });
+  });
+
+  test('should dedup mapped relationship metadata', () => {
+    const steps: IntegrationStep<IntegrationInstanceConfig>[] = [
+      getMockIntegrationStep({
+        entities: [],
+        relationships: [],
+        mappedRelationships: [
+          {
+            _class: RelationshipClass.IS,
+            sourceType: 'my_user',
+            targetType: 'employee',
+            direction: RelationshipDirection.FORWARD,
+            _type: 'my_user_is_employee',
+          },
+          {
+            _class: RelationshipClass.IS,
+            sourceType: 'my_user',
+            targetType: 'employee',
+            direction: RelationshipDirection.FORWARD,
+            _type: 'my_user_is_employee',
+          },
+        ],
+      }),
+      getMockIntegrationStep({
+        entities: [],
+        relationships: [],
+        mappedRelationships: [
+          {
+            _class: RelationshipClass.IS,
+            sourceType: 'my_user',
+            targetType: 'employee',
+            direction: RelationshipDirection.FORWARD,
+            _type: 'my_user_is_employee',
+          },
+        ],
+      }),
+    ];
+
+    expect(generateIntegrationGraphSchema(steps)).toEqual({
+      entities: [],
+      relationships: [],
+      mappedRelationships: [
+        {
+          _class: RelationshipClass.IS,
+          sourceType: 'my_user',
+          targetType: 'employee',
+          direction: RelationshipDirection.FORWARD,
+        },
+      ],
+    });
+  });
+});

package/src/commands/generate-integration-graph-schema.ts

@@ -0,0 +1,226 @@
+import {
+  IntegrationInstanceConfig,
+  IntegrationStepExecutionContext,
+  Step,
+  StepEntityMetadata,
+  StepMappedRelationshipMetadata,
+  StepRelationshipMetadata,
+} from '@jupiterone/integration-sdk-core';
+import { createCommand } from 'commander';
+import path from 'path';
+import { loadConfig } from '../config';
+import { promises as fs } from 'fs';
+import * as log from '../log';
+
+export function generateIntegrationGraphSchemaCommand() {
+  return createCommand('generate-integration-graph-schema')
+    .description(
+      'generate integration graph metadata summary from step metadata',
+    )
+    .option(
+      '-o, --output-file <path>',
+      'project relative path to generated integration graph schema file',
+    )
+    .option(
+      '-p, --project-path <directory>',
+      'path to integration project directory',
+      process.cwd(),
+    )
+    .action(async (options) => {
+      const { projectPath, outputFile } = options;
+
+      log.info(
+        `Generating integration graph schema (projectPath=${projectPath}, outputFile=${outputFile})`,
+      );
+      const config = await loadConfig(path.join(projectPath, 'src'));
+
+      const integrationGraphSchema = generateIntegrationGraphSchema(
+        config.integrationSteps,
+      );
+
+      if (outputFile) {
+        await fs.writeFile(outputFile, JSON.stringify(integrationGraphSchema), {
+          encoding: 'utf-8',
+        });
+      } else {
+        console.log(JSON.stringify(integrationGraphSchema, null, 2));
+      }
+
+      log.info('Successfully generated integration graph schema');
+    });
+}
+
+type IntegrationGraphSchemaEntityMetadata = {
+  resourceName: string;
+  _class: string | string[];
+  _type: string;
+};
+
+type IntegrationGraphSchemaRelationshipMetadata = {
+  _class: string;
+  sourceType: string;
+  targetType: string;
+};
+
+type IntegrationGraphSchemaMappedRelationshipMetadata = {
+  sourceType: string;
+  _class: string;
+  targetType: string;
+  direction: 'FORWARD' | 'REVERSE';
+};
+
+type IntegrationGraphSchema = {
+  entities: IntegrationGraphSchemaEntityMetadata[];
+  relationships: IntegrationGraphSchemaRelationshipMetadata[];
+  mappedRelationships: IntegrationGraphSchemaMappedRelationshipMetadata[];
+};
+
+export function generateIntegrationGraphSchema(
+  integrationSteps: Step<
+    IntegrationStepExecutionContext<IntegrationInstanceConfig, object>
+  >[],
+): IntegrationGraphSchema {
+  const integrationGraphSchema: IntegrationGraphSchema = {
+    entities: [],
+    relationships: [],
+    mappedRelationships: [],
+  };
+
+  // To ensure that we are generating a unique set of metadata, we dedup any
+  // entity, relationship, and mapped relationship schemas that may be
+  // duplicated across steps (rare, but possible).
+  const uniqueEntitySchemaSet = new Set<string>();
+  const uniqueRelationshipSchemaSet = new Set<string>();
+  const uniqueMappedRelationshipSchemaSet = new Set<string>();
+
+  function getIntegrationGraphSchemaEntityMetadataForStep(
+    stepEntityMetadata: StepEntityMetadata[],
+  ): IntegrationGraphSchemaEntityMetadata[] {
+    const entities: IntegrationGraphSchemaEntityMetadata[] = [];
+
+    for (const entityMetadata of stepEntityMetadata) {
+      const entitySchemaKey = getEntitySchemaKey(entityMetadata);
+
+      if (uniqueEntitySchemaSet.has(entitySchemaKey)) continue;
+      uniqueEntitySchemaSet.add(entitySchemaKey);
+
+      entities.push(toIntegrationGraphSchemaEntityMetadata(entityMetadata));
+    }
+
+    return entities;
+  }
+
+  function getIntegrationGraphSchemaRelationshipMetadataForStep(
+    stepRelationshipMetadata: StepRelationshipMetadata[],
+  ) {
+    const relationships: IntegrationGraphSchemaRelationshipMetadata[] = [];
+
+    for (const relationshipMetadata of stepRelationshipMetadata) {
+      const relationshipSchemaKey =
+        getRelationshipSchemaKey(relationshipMetadata);
+
+      if (uniqueRelationshipSchemaSet.has(relationshipSchemaKey)) continue;
+      uniqueRelationshipSchemaSet.add(relationshipSchemaKey);
+
+      relationships.push(
+        toIntegrationGraphSchemaRelationshipMetadata(relationshipMetadata),
+      );
+    }
+
+    return relationships;
+  }
+
+  function getIntegrationGraphSchemaMappedRelationshipMetadataForStep(
+    stepMappedRelationshipMetadata: StepMappedRelationshipMetadata[],
+  ) {
+    const mappedRelationships: IntegrationGraphSchemaMappedRelationshipMetadata[] =
+      [];
+
+    for (const mappedRelationshipMetadata of stepMappedRelationshipMetadata) {
+      const mappedRelationshipSchemaKey = getMappedRelationshipSchemaKey(
+        mappedRelationshipMetadata,
+      );
+
+      if (uniqueMappedRelationshipSchemaSet.has(mappedRelationshipSchemaKey))
+        continue;
+      uniqueMappedRelationshipSchemaSet.add(mappedRelationshipSchemaKey);
+
+      mappedRelationships.push(
+        toIntegrationGraphSchemaMappedRelationshipMetadata(
+          mappedRelationshipMetadata,
+        ),
+      );
+    }
+
+    return mappedRelationships;
+  }
+
+  for (const step of integrationSteps) {
+    integrationGraphSchema.entities = integrationGraphSchema.entities.concat(
+      getIntegrationGraphSchemaEntityMetadataForStep(step.entities),
+    );
+
+    integrationGraphSchema.relationships =
+      integrationGraphSchema.relationships.concat(
+        getIntegrationGraphSchemaRelationshipMetadataForStep(
+          step.relationships,
+        ),
+      );
+
+    if (step.mappedRelationships) {
+      integrationGraphSchema.mappedRelationships =
+        integrationGraphSchema.mappedRelationships.concat(
+          getIntegrationGraphSchemaMappedRelationshipMetadataForStep(
+            step.mappedRelationships,
+          ),
+        );
+    }
+  }
+
+  return integrationGraphSchema;
+}
+
+function getEntitySchemaKey(metadata: StepEntityMetadata) {
+  return `${metadata._class}|${metadata._type}`;
+}
+
+function getRelationshipSchemaKey(metadata: StepRelationshipMetadata) {
+  return `${metadata.sourceType}|${metadata._class}|${metadata.targetType}`;
+}
+
+function getMappedRelationshipSchemaKey(
+  metadata: StepMappedRelationshipMetadata,
+) {
+  return `${metadata.sourceType}|${metadata._class}|${metadata.targetType}|${metadata.direction}`;
+}
+
+function toIntegrationGraphSchemaEntityMetadata(
+  stepEntityMetadata: StepEntityMetadata,
+): IntegrationGraphSchemaEntityMetadata {
+  return {
+    resourceName: stepEntityMetadata.resourceName,
+    _class: stepEntityMetadata._class,
+    _type: stepEntityMetadata._type,
+  };
+}
+
+function toIntegrationGraphSchemaRelationshipMetadata(
+  relationshipMetadata: StepRelationshipMetadata,
+): IntegrationGraphSchemaRelationshipMetadata {
+  return {
+    _class: relationshipMetadata._class,
+    sourceType: relationshipMetadata.sourceType,
+    targetType: relationshipMetadata.targetType,
+  };
+}
+
+function toIntegrationGraphSchemaMappedRelationshipMetadata(
+  mappedRelationshipMetadata: StepMappedRelationshipMetadata,
+): IntegrationGraphSchemaMappedRelationshipMetadata {
+  return {
+    sourceType: mappedRelationshipMetadata.sourceType,
+    _class: mappedRelationshipMetadata._class,
+    targetType: mappedRelationshipMetadata.targetType,
+    direction: mappedRelationshipMetadata.direction,
+  };
+}

package/src/commands/index.ts

@@ -8,3 +8,4 @@ export * from './visualize-types';
 export * from './validate-question-file';
 export * from './neo4j';
 export * from './visualize-dependencies';
+export * from './generate-integration-graph-schema';

package/src/index.ts

@@ -11,6 +11,7 @@ import {
   validateQuestionFile,
   neo4j,
   visualizeDependencies,
+  generateIntegrationGraphSchemaCommand,
 } from './commands';
 
 export function createCli() {
@@ -24,5 +25,6 @@ export function createCli() {
     .addCommand(document())
     .addCommand(validateQuestionFile())
     .addCommand(neo4j())
-    .addCommand(visualizeDependencies());
+    .addCommand(visualizeDependencies())
+    .addCommand(generateIntegrationGraphSchemaCommand());
 }

package/src/log.ts

@@ -1,6 +1,9 @@
 import chalk from 'chalk';
 
-import { ExecuteIntegrationResult } from '@jupiterone/integration-sdk-runtime';
+import {
+  ExecuteIntegrationResult,
+  processDeclaredTypesDiff,
+} from '@jupiterone/integration-sdk-runtime';
 import {
   IntegrationStepResult,
   StepResultStatus,
@@ -36,29 +39,17 @@ export function displayExecutionResults(results: ExecuteIntegrationResult) {
 
   let undeclaredTypesDetected: boolean = false;
 
-  results.integrationStepResults.forEach((step) => {
+  processDeclaredTypesDiff(results, (step, undeclaredTypes) => {
     logStepStatus(step);
 
-    if (step.status === StepResultStatus.SUCCESS) {
-      const { declaredTypes, encounteredTypes } = step;
-
-      const declaredTypeSet = new Set(declaredTypes);
-      const undeclaredTypes = encounteredTypes.filter(
-        (type) => !declaredTypeSet.has(type),
+    if (step.status === StepResultStatus.SUCCESS && undeclaredTypes.length) {
+      undeclaredTypesDetected = true;
+      const undeclaredTypesList = undeclaredTypes.map((type) => {
+        return `\n  - ${type}`;
+      });
+      warn(
+        `The following types were encountered but are not declared in the step's "types" field:${undeclaredTypesList}`,
       );
-
-      if (undeclaredTypes.length) {
-        undeclaredTypesDetected = true;
-
-        const undeclaredTypesList = undeclaredTypes.map((type) => {
-          return `\n  - ${type}`;
-        });
-        warn(
-          `The following types were encountered but are not declared in the step's "types" field:${undeclaredTypesList}`,
-        );
-
-        console.log('');
-      }
     }
   });