@jupiterone/integration-sdk-cli 9.4.1 → 9.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupiterone/integration-sdk-cli",
-  "version": "9.4.1",
+  "version": "9.6.0",
   "description": "The SDK for developing JupiterOne integrations",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -18,27 +18,33 @@
   },
   "scripts": {
     "prebuild:dist": "rm -rf dist && mkdir dist",
-    "build:dist": "tsc -p tsconfig.dist.json --declaration",
-    "prepack": "yarn build:dist"
+    "build:dist": "tsc -p tsconfig.dist.json --declaration && yarn copy-files",
+    "prepack": "yarn build:dist",
+    "copy-files": "cp -r src/generator/template dist/src/generator/template && cp -r src/generator/stepTemplate dist/src/generator/stepTemplate",
+    "plop": "plop --plopfile dist/src/generator/newIntegration.js"
   },
   "dependencies": {
-    "@jupiterone/integration-sdk-runtime": "^9.4.1",
+    "@jupiterone/data-model": "^0.54.0",
+    "@jupiterone/integration-sdk-core": "^9.6.0",
+    "@jupiterone/integration-sdk-runtime": "^9.6.0",
     "chalk": "^4",
     "commander": "^9.4.0",
     "fs-extra": "^10.1.0",
     "globby": "^11.0.0",
+    "inquirer-checkbox-plus-prompt": "^1.4.2",
     "js-yaml": "^4.1.0",
     "json-diff": "^0.5.4",
     "lodash": "^4.17.19",
     "markdown-table": "^2.0.0",
     "neo4j-driver": "^4.3.3",
+    "plop": "^3.1.2",
     "runtypes": "5.1.0",
     "upath": "^1.2.0",
     "url-exists": "^1.0.3",
     "vis": "^4.21.0-EOL"
   },
   "devDependencies": {
-    "@jupiterone/integration-sdk-private-test-utils": "^9.4.1",
+    "@jupiterone/integration-sdk-private-test-utils": "^9.6.0",
     "@pollyjs/adapter-node-http": "^6.0.5",
     "@pollyjs/core": "^6.0.5",
     "@pollyjs/persister-fs": "^6.0.5",
@@ -50,5 +56,5 @@
     "memfs": "^3.2.0",
     "neo-forgery": "^2.0.0"
   },
-  "gitHead": "b9e06bbbd2b454182799a40083c9873dd4b76ade"
+  "gitHead": "dc14837ec81f9196b852d3ffb4e82e9789097a77"
 }

package/src/commands/generate-ingestion-sources-config.test.ts

@@ -63,6 +63,33 @@ describe('#generateIngestionSourcesConfig', () => {
   });
 
   it('should return the ingestionConfig with childIngestionSources', () => {
+    const stepFetchVulnerabilityAlerts = {
+      id: 'fetch-vulnerability-alerts',
+      name: 'Fetch Vulnerability Alerts',
+      entities: [
+        {
+          resourceName: 'GitHub Vulnerability Alerts',
+          _type: 'github_finding',
+          _class: ['Finding'],
+        },
+      ],
+      relationships: [],
+      dependsOn: ['fetch-repos'],
+      ingestionSourceId: INGESTION_SOURCE_IDS.FINDING_ALERTS,
+    };
+    const stepFetchIssues = {
+      id: 'fetch-issues',
+      name: 'Fetch Issues',
+      entities: [
+        {
+          resourceName: 'GitHub Issue',
+          _type: 'github_issue',
+          _class: ['Issue'],
+        },
+      ],
+      relationships: [],
+      dependsOn: ['fetch-repos', 'fetch-users', 'fetch-collaborators'],
+    };
     const integrationSteps: IntegrationStep<IntegrationInstanceConfig>[] = [
       {
         id: 'fetch-repos',
@@ -80,32 +107,11 @@ describe('#generateIngestionSourcesConfig', () => {
         executionHandler: jest.fn(),
       },
       {
-        id: 'fetch-vulnerability-alerts',
-        name: 'Fetch Vulnerability Alerts',
-        entities: [
-          {
-            resourceName: 'GitHub Vulnerability Alerts',
-            _type: 'github_finding',
-            _class: ['Finding'],
-          },
-        ],
-        relationships: [],
-        dependsOn: ['fetch-repos'],
-        ingestionSourceId: INGESTION_SOURCE_IDS.FINDING_ALERTS,
+        ...stepFetchVulnerabilityAlerts,
         executionHandler: jest.fn(),
       },
       {
-        id: 'fetch-issues',
-        name: 'Fetch Issues',
-        entities: [
-          {
-            resourceName: 'GitHub Issue',
-            _type: 'github_issue',
-            _class: ['Issue'],
-          },
-        ],
-        relationships: [],
-        dependsOn: ['fetch-repos', 'fetch-users', 'fetch-collaborators'],
+        ...stepFetchIssues,
         executionHandler: jest.fn(),
       },
       {
@@ -135,7 +141,9 @@ describe('#generateIngestionSourcesConfig', () => {
     expect(
       ingestionSourcesConfig[INGESTION_SOURCE_IDS.FETCH_REPOS]
         .childIngestionSources,
-    ).toEqual(['fetch-vulnerability-alerts', 'fetch-issues']);
+    ).toEqual(
+      expect.arrayContaining([stepFetchVulnerabilityAlerts, stepFetchIssues]),
+    );
     // For FINDING_ALERTS the ingestionConfig keep exactly the same
     expect(
       ingestionSourcesConfig[INGESTION_SOURCE_IDS.FINDING_ALERTS],

package/src/commands/generate-ingestion-sources-config.ts

@@ -4,6 +4,7 @@ import {
   IntegrationSourceId,
   Step,
   StepExecutionContext,
+  StepMetadata,
 } from '@jupiterone/integration-sdk-core';
 import { createCommand } from 'commander';
 import { loadConfigFromTarget } from '../config';
@@ -59,7 +60,7 @@ export function generateIngestionSourcesConfigCommand() {
 
 export type EnhancedIntegrationIngestionConfigFieldMap = Record<
   IntegrationSourceId,
-  IntegrationIngestionConfigField & { childIngestionSources?: string[] }
+  IntegrationIngestionConfigField & { childIngestionSources?: StepMetadata[] }
 >;
 
 /**
@@ -91,18 +92,19 @@ export function generateIngestionSourcesConfig<
         // Skip iteration if there are no steps pointing to the current ingestionSourceId
         return;
       }
-      // Get the stepIds that have any dependencies on the matched step ids
-      const childIngestionSources = integrationSteps
-        .filter((step) =>
-          step.dependsOn?.some((value) =>
-            matchedIntegrationStepIds.includes(value),
-          ),
-        )
-        .map(({ id }) => id);
+      // Get the dependent steps for the given matchedIntegrationStepIds
+      const childIngestionSources = integrationSteps.filter((step) =>
+        step.dependsOn?.some((value) =>
+          matchedIntegrationStepIds.includes(value),
+        ),
+      );
       // Generate ingestionConfig with the childIngestionSources
       newIngestionConfig[key] = {
         ...ingestionConfig[key],
-        childIngestionSources,
+        // Drop execution handler from returned object
+        childIngestionSources: childIngestionSources.map(
+          ({ executionHandler, ...keepAttrs }) => keepAttrs,
+        ),
       };
     } else {
       log.warn(`The key ${key} does not exist in the ingestionConfig`);

package/src/commands/generate.ts

@@ -0,0 +1,28 @@
+import path from 'path';
+import { createCommand } from 'commander';
+const dynamicImport = new Function('specifier', 'return import(specifier)');
+
+export function generate() {
+  return createCommand('generate')
+    .description('generate integrations in whole and in part')
+    .action(async (cmdOpts) => {
+      const Plop = await dynamicImport('plop');
+      const configPath = path.resolve(
+        path.join(__dirname, '../generator/newIntegration.js'),
+      );
+      Plop.Plop.prepare(
+        {
+          cwd: process.cwd(),
+          configPath,
+        },
+        (env) =>
+          Plop.Plop.execute(env, (env) => {
+            const options = {
+              ...env,
+              dest: process.cwd(),
+            };
+            return Plop.run(options, undefined, true);
+          }),
+      );
+    });
+}

package/src/commands/index.ts

@@ -11,3 +11,4 @@ export * from './visualize-dependencies';
 export * from './generate-integration-graph-schema';
 export * from './troubleshoot';
 export * from './generate-ingestion-sources-config';
+export * from './generate';

package/src/generator/actions.ts

@@ -0,0 +1,37 @@
+import { spawn } from 'child_process';
+
+async function spawnCommand(config, command, args): Promise<string> {
+  const spawnOptions = config.verbose
+    ? {
+        cwd: config.path,
+        shell: true,
+      }
+    : {
+        cwd: config.path,
+      };
+
+  return new Promise((resolve, reject) => {
+    const cmd = spawn(command, args, spawnOptions);
+    cmd.on('close', (code) => {
+      if (code === 0) {
+        resolve(`${code}`);
+      } else {
+        reject(`exited with ${code}`);
+      }
+    });
+  });
+}
+
+async function yarnInstall(_answers, config, _plop) {
+  return spawnCommand(config, 'yarn', ['install']);
+}
+
+async function yarnFormat(_answers, config, _plop): Promise<string> {
+  return spawnCommand(config, 'yarn', [`format`]);
+}
+
+async function yarnLint(_answers, config, _plop) {
+  return spawnCommand(config, 'yarn', ['lint']);
+}
+
+export { yarnInstall, yarnFormat, yarnLint };

package/src/generator/configFieldsFlow.ts

@@ -0,0 +1,60 @@
+import { askRepeatedly } from './util';
+
+type FieldType = 'json' | 'string' | 'boolean';
+
+async function configFieldsFlow(inquirer) {
+  const configFields: { field: string; type: FieldType; mask: boolean }[] = [];
+
+  const { doPrompt } = await inquirer.prompt({
+    type: 'confirm',
+    name: 'doPrompt',
+    message: 'Do you want to add config fields?',
+  });
+
+  if (!doPrompt) return configFields;
+
+  await askRepeatedly(inquirer, async () => {
+    const configField = await configFieldPrompt(inquirer);
+    configFields.push(configField);
+  });
+
+  return configFields;
+}
+
+async function configFieldPrompt(inquirer) {
+  const { field } = await inquirer.prompt({
+    type: 'input',
+    name: 'field',
+    message: 'The config field name (ex. apiKey)',
+  });
+
+  const { type } = await inquirer.prompt({
+    type: 'list',
+    name: 'type',
+    message: 'The type of the config field',
+    choices: [
+      {
+        name: 'string',
+        value: 'string',
+      },
+      {
+        name: 'boolean',
+        value: 'boolean',
+      },
+      {
+        name: 'json',
+        value: 'json',
+      },
+    ],
+  });
+
+  const { mask } = await inquirer.prompt({
+    name: 'mask',
+    type: 'confirm',
+    message: 'Should the field be masked?',
+  });
+
+  return { field, type, mask };
+}
+
+export { configFieldsFlow };

package/src/generator/entitiesFlow.ts

@@ -0,0 +1,185 @@
+import { askRepeatedly } from './util';
+import { snakeCase } from 'lodash';
+
+export type Entity = {
+  resourceName: string;
+  _type: string;
+  _class: string[];
+};
+
+async function entitiesFlow(inquirer, vendorName): Promise<Entity[]> {
+  const entities: Entity[] = [];
+  const { doPrompt } = await inquirer.prompt({
+    type: 'confirm',
+    name: 'doPrompt',
+    message: 'Do you want to declare entities?',
+  });
+  if (!doPrompt) return entities;
+
+  await askRepeatedly(inquirer, async () => {
+    const entity = await createEntityPrompt(inquirer, vendorName);
+    entities.push(entity);
+  });
+  return entities;
+}
+
+async function createEntityPrompt(inquirer, vendorName) {
+  const { resourceName } = await inquirer.prompt({
+    type: 'input',
+    name: 'resourceName',
+    message:
+      'What is the name of the entity? ex. User, EC2 Instance, Access Key',
+  });
+
+  const { _class } = await inquirer.prompt({
+    type: 'checkbox-plus',
+    name: '_class',
+    message: 'Which classes should the entity have?',
+    searchable: true,
+    pageSize: 10,
+    validate(input) {
+      if (input?.length > 0) {
+        return true;
+      } else {
+        return false;
+      }
+    },
+    source: function (ans, input) {
+      return new Promise((resolve) => {
+        if (!input) {
+          resolve(entityClassesChoices);
+          return;
+        } else {
+          const choices = entityClassesChoices.filter(({ name }) =>
+            name.toLowerCase().includes(input.toLowerCase()),
+          );
+          resolve(choices);
+        }
+      });
+    },
+  });
+
+  const { _type } = await inquirer.prompt({
+    Type: 'input',
+    name: '_type',
+    message: 'What should the _type be?',
+    default: `${snakeCase(vendorName)}_${snakeCase(resourceName)}`,
+    validate(input) {
+      const valid = /^[a-z0-9_]+$/.test(input);
+      if (!valid) {
+        return '_type must be lowercased, alphanumeric, and use underscores for spacing';
+      }
+      return valid;
+    },
+  });
+
+  return {
+    resourceName,
+    _type,
+    _class,
+  };
+}
+
+const entityClassesChoices = [
+  { name: 'AccessKey' },
+  { name: 'AccessPolicy' },
+  { name: 'AccessRole' },
+  { name: 'Account' },
+  { name: 'Alert' },
+  { name: 'Application' },
+  { name: 'ApplicationEndpoint' },
+  { name: 'Assessment' },
+  { name: 'Attacker' },
+  { name: 'Backup' },
+  { name: 'Certificate' },
+  { name: 'Channel' },
+  { name: 'Cluster' },
+  { name: 'CodeCommit' },
+  { name: 'CodeDeploy' },
+  { name: 'CodeModule' },
+  { name: 'CodeRepo' },
+  { name: 'CodeReview' },
+  { name: 'Configuration' },
+  { name: 'Container' },
+  { name: 'Control' },
+  { name: 'ControlPolicy' },
+  { name: 'CryptoKey' },
+  { name: 'DataCollection' },
+  { name: 'DataObject' },
+  { name: 'DataStore' },
+  { name: 'Database' },
+  { name: 'Deployment' },
+  { name: 'Device' },
+  { name: 'Directory' },
+  { name: 'Disk' },
+  { name: 'Document' },
+  { name: 'Domain' },
+  { name: 'DomainRecord' },
+  { name: 'DomainZone' },
+  { name: 'Entity' },
+  { name: 'Finding' },
+  { name: 'Firewall' },
+  { name: 'Framework' },
+  { name: 'Function' },
+  { name: 'Gateway' },
+  { name: 'GraphObject' },
+  { name: 'Group' },
+  { name: 'Host' },
+  { name: 'HostAgent' },
+  { name: 'Image' },
+  { name: 'Incident' },
+  { name: 'Internet' },
+  { name: 'IpAddress' },
+  { name: 'Issue' },
+  { name: 'Key' },
+  { name: 'Logs' },
+  { name: 'Model' },
+  { name: 'Module' },
+  { name: 'Network' },
+  { name: 'NetworkEndpoint' },
+  { name: 'NetworkInterface' },
+  { name: 'Organization' },
+  { name: 'PR' },
+  { name: 'PasswordPolicy' },
+  { name: 'Person' },
+  { name: 'Policy' },
+  { name: 'Problem' },
+  { name: 'Procedure' },
+  { name: 'Process' },
+  { name: 'Product' },
+  { name: 'Program' },
+  { name: 'Project' },
+  { name: 'Question' },
+  { name: 'Queue' },
+  { name: 'Record' },
+  { name: 'RecordEntity' },
+  { name: 'Repository' },
+  { name: 'Requirement' },
+  { name: 'Resource' },
+  { name: 'Review' },
+  { name: 'Risk' },
+  { name: 'Root' },
+  { name: 'Rule' },
+  { name: 'Ruleset' },
+  { name: 'Scanner' },
+  { name: 'Secret' },
+  { name: 'Section' },
+  { name: 'Service' },
+  { name: 'Site' },
+  { name: 'Standard' },
+  { name: 'Subscription' },
+  { name: 'Task' },
+  { name: 'Team' },
+  { name: 'ThreatIntel' },
+  { name: 'Training' },
+  { name: 'User' },
+  { name: 'UserGroup' },
+  { name: 'Vault' },
+  { name: 'Vendor' },
+  { name: 'Vulnerability' },
+  { name: 'Weakness' },
+  { name: 'Workflow' },
+  { name: 'Workload' },
+].map((v) => ({ name: v.name, value: v.name }));
+
+export { entitiesFlow };

package/src/generator/helpers.ts

@@ -0,0 +1,6 @@
+function generateRelationshipName(relationship) {
+  const { from, to, _class } = relationship;
+  return `${from.resourceName}_${_class}_${to.resourceName}`;
+}
+
+export { generateRelationshipName };

package/src/generator/newIntegration.ts

@@ -0,0 +1,137 @@
+import { entitiesFlow } from './entitiesFlow';
+import { relationshipsFlow } from './relationshipsFlow';
+import { configFieldsFlow } from './configFieldsFlow';
+import { stepsFlow } from './stepsFlow';
+import { generateRelationshipName } from './helpers';
+import { generateRelationshipType } from '@jupiterone/integration-sdk-core';
+import { yarnFormat, yarnInstall, yarnLint } from './actions';
+import { kebabCase } from 'lodash';
+import checkboxPlus from 'inquirer-checkbox-plus-prompt';
+import path from 'path';
+
+import { NodePlopAPI } from 'plop';
+import { Relationship } from './relationshipsFlow';
+import { Step } from './stepsFlow';
+
+function newIntegration(plop: NodePlopAPI) {
+  plop.setHelper('generateRelationshipType', function (this: any) {
+    return generateRelationshipType(this._class, this.from, this.to);
+  });
+
+  plop.setHelper('generateRelationshipName', generateRelationshipName);
+  plop.setActionType('yarnFormat', yarnFormat);
+  plop.setActionType('yarnInstall', yarnInstall);
+  plop.setActionType('yarnLint', yarnLint);
+  plop.setPrompt('checkbox-plus', checkboxPlus);
+
+  plop.setGenerator('new:integration', {
+    description: 'Create a new integration',
+    prompts: async function (inquirer) {
+      const { vendorName } = await inquirer.prompt({
+        type: 'input',
+        name: 'vendorName',
+        message:
+          'The integration vendor name (ex. AWS, Google Workspace, CrowdStrike)',
+        validate(input) {
+          if (!input) {
+            return 'Cannot be an empty string';
+          }
+          return true;
+        },
+      });
+
+      const { packageName } = await inquirer.prompt({
+        type: 'input',
+        name: 'packageName',
+        default: `@jupiterone/graph-${kebabCase(vendorName)}`,
+        message: 'The npm package name',
+        validate(input) {
+          if (!input) {
+            return 'Cannot be an empty string';
+          }
+          return true;
+        },
+      });
+
+      const { packageDescription } = await inquirer.prompt({
+        type: 'input',
+        name: 'packageDescription',
+        message: 'A description for package',
+        default: `An integration and graph conversion project for ${vendorName}`,
+        validate(input) {
+          if (!input) {
+            return 'Cannot be an empty string';
+          }
+          return true;
+        },
+      });
+
+      const configFields = await configFieldsFlow(inquirer);
+      const entities = await entitiesFlow(inquirer, vendorName);
+      let relationships: Relationship[] = [];
+      let steps: Step[] = [];
+      if (entities.length) {
+        relationships = await relationshipsFlow(inquirer, entities);
+        steps = await stepsFlow(inquirer, entities, relationships);
+      }
+
+      return {
+        vendorName,
+        packageName,
+        packageDescription,
+        configFields,
+        entities,
+        relationships,
+        steps,
+      };
+    },
+    actions: function (data) {
+      if (!data) {
+        return [];
+      }
+
+      const actions: any[] = [];
+      actions.push({
+        type: 'addMany',
+        destination: '.',
+        base: path.join(__dirname, '/template'),
+        templateFiles: path.join(__dirname + '/template/**'),
+        globOptions: { dot: true },
+        force: true,
+        data,
+      });
+
+      for (const step of data.steps) {
+        actions.push({
+          type: 'add',
+          path: path.normalize(`src/steps/${kebabCase(step.name)}/index.ts`),
+          templateFile: path.join(__dirname, 'stepTemplate/index.ts.hbs'),
+          data: step,
+          force: true,
+        });
+      }
+
+      actions.push({
+        type: 'yarnInstall',
+        path: '.',
+        verbose: true,
+      });
+
+      actions.push({
+        type: 'yarnFormat',
+        path: '.',
+        verbose: true,
+      });
+
+      actions.push({
+        type: 'yarnLint',
+        path: '.',
+        verbose: true,
+      });
+
+      return actions;
+    },
+  });
+}
+
+module.exports = newIntegration;