@jupiterone/integration-sdk-cli 8.34.0 → 9.0.0-beta.1

package/src/commands/generate-ingestion-sources-config.test.ts

@@ -0,0 +1,185 @@
+import {
+  IntegrationIngestionConfigFieldMap,
+  IntegrationStep,
+  IntegrationInstanceConfig,
+} from '@jupiterone/integration-sdk-core';
+import { generateIngestionSourcesConfig } from './generate-ingestion-sources-config';
+
+describe('#generateIngestionSourcesConfig', () => {
+  const INGESTION_SOURCE_IDS = {
+    FINDING_ALERTS: 'finding-alerts',
+    FETCH_REPOS: 'fetch-repos',
+    TEST_SOURCE: 'test-source',
+  };
+
+  const ingestionConfig: IntegrationIngestionConfigFieldMap = {
+    [INGESTION_SOURCE_IDS.FINDING_ALERTS]: {
+      title: 'Finding Alerts',
+      description:
+        'Dependabot vulnerability alert ingestion and Code scanning alerts',
+      defaultsToDisabled: true,
+    },
+    [INGESTION_SOURCE_IDS.FETCH_REPOS]: {
+      title: 'Fetch repos',
+      description: 'This is an ingestion source created for test purposes',
+      defaultsToDisabled: false,
+    },
+  };
+
+  it('should return the ingestionConfig with empty childIngestionSources', () => {
+    const integrationSteps: IntegrationStep<IntegrationInstanceConfig>[] = [
+      {
+        id: 'fetch-vulnerability-alerts',
+        name: 'Fetch Vulnerability Alerts',
+        entities: [
+          {
+            resourceName: 'GitHub Vulnerability Alerts',
+            _type: 'github_finding',
+            _class: ['Finding'],
+          },
+        ],
+        relationships: [],
+        dependsOn: ['fetch-repos'],
+        ingestionSourceId: INGESTION_SOURCE_IDS.FINDING_ALERTS,
+        executionHandler: jest.fn(),
+      },
+    ];
+    const ingestionSourcesConfig = generateIngestionSourcesConfig(
+      ingestionConfig,
+      integrationSteps,
+    );
+    expect(
+      ingestionSourcesConfig[INGESTION_SOURCE_IDS.FINDING_ALERTS],
+    ).toMatchObject(ingestionConfig[INGESTION_SOURCE_IDS.FINDING_ALERTS]);
+    // childIngestionSources is empty because there are no steps that depends on fetch-vulnerability-alerts
+    expect(
+      ingestionSourcesConfig[INGESTION_SOURCE_IDS.FINDING_ALERTS]
+        .childIngestionSources,
+    ).toBeEmpty();
+    // ingestionSourcesConfig[INGESTION_SOURCE_IDS.FETCH_REPOS] is undefined because there are no steps using that ingestionSourceId
+    expect(
+      ingestionSourcesConfig[INGESTION_SOURCE_IDS.FETCH_REPOS],
+    ).toBeUndefined();
+  });
+
+  it('should return the ingestionConfig with childIngestionSources', () => {
+    const integrationSteps: IntegrationStep<IntegrationInstanceConfig>[] = [
+      {
+        id: 'fetch-repos',
+        name: 'Fetch Repos',
+        entities: [
+          {
+            resourceName: 'Github Repo',
+            _type: 'github_repo',
+            _class: ['CodeRepo'],
+          },
+        ],
+        relationships: [],
+        dependsOn: ['fetch-account'],
+        ingestionSourceId: INGESTION_SOURCE_IDS.FETCH_REPOS,
+        executionHandler: jest.fn(),
+      },
+      {
+        id: 'fetch-vulnerability-alerts',
+        name: 'Fetch Vulnerability Alerts',
+        entities: [
+          {
+            resourceName: 'GitHub Vulnerability Alerts',
+            _type: 'github_finding',
+            _class: ['Finding'],
+          },
+        ],
+        relationships: [],
+        dependsOn: ['fetch-repos'],
+        ingestionSourceId: INGESTION_SOURCE_IDS.FINDING_ALERTS,
+        executionHandler: jest.fn(),
+      },
+      {
+        id: 'fetch-issues',
+        name: 'Fetch Issues',
+        entities: [
+          {
+            resourceName: 'GitHub Issue',
+            _type: 'github_issue',
+            _class: ['Issue'],
+          },
+        ],
+        relationships: [],
+        dependsOn: ['fetch-repos', 'fetch-users', 'fetch-collaborators'],
+        executionHandler: jest.fn(),
+      },
+      {
+        id: 'fetch-teams',
+        name: 'Fetch Teams',
+        entities: [
+          {
+            resourceName: 'GitHub Team',
+            _type: 'github_team',
+            _class: ['UserGroup'],
+          },
+        ],
+        relationships: [],
+        dependsOn: ['fetch-account'],
+        executionHandler: jest.fn(),
+      },
+    ];
+    const ingestionSourcesConfig = generateIngestionSourcesConfig(
+      ingestionConfig,
+      integrationSteps,
+    );
+    // Original object doesn't change
+    expect(
+      ingestionSourcesConfig[INGESTION_SOURCE_IDS.FETCH_REPOS],
+    ).toMatchObject(ingestionConfig[INGESTION_SOURCE_IDS.FETCH_REPOS]);
+    // New property added
+    expect(
+      ingestionSourcesConfig[INGESTION_SOURCE_IDS.FETCH_REPOS]
+        .childIngestionSources,
+    ).toEqual(['fetch-vulnerability-alerts', 'fetch-issues']);
+    // For FINDING_ALERTS the ingestionConfig keep exactly the same
+    expect(
+      ingestionSourcesConfig[INGESTION_SOURCE_IDS.FINDING_ALERTS],
+    ).toMatchObject(ingestionConfig[INGESTION_SOURCE_IDS.FINDING_ALERTS]);
+  });
+
+  it('should not add the source if it does not exist in the ingestionConfig', () => {
+    const integrationSteps: IntegrationStep<IntegrationInstanceConfig>[] = [
+      {
+        id: 'fetch-repos',
+        name: 'Fetch Repos',
+        entities: [
+          {
+            resourceName: 'Github Repo',
+            _type: 'github_repo',
+            _class: ['CodeRepo'],
+          },
+        ],
+        relationships: [],
+        dependsOn: ['fetch-account'],
+        ingestionSourceId: INGESTION_SOURCE_IDS.TEST_SOURCE,
+        executionHandler: jest.fn(),
+      },
+      {
+        id: 'fetch-issues',
+        name: 'Fetch Issues',
+        entities: [
+          {
+            resourceName: 'GitHub Issue',
+            _type: 'github_issue',
+            _class: ['Issue'],
+          },
+        ],
+        relationships: [],
+        dependsOn: ['fetch-repos', 'fetch-users', 'fetch-collaborators'],
+        executionHandler: jest.fn(),
+      },
+    ];
+    const ingestionSourcesConfig = generateIngestionSourcesConfig(
+      ingestionConfig,
+      integrationSteps,
+    );
+    expect(
+      ingestionSourcesConfig[INGESTION_SOURCE_IDS.TEST_SOURCE],
+    ).toBeUndefined();
+  });
+});

package/src/commands/generate-ingestion-sources-config.ts

@@ -0,0 +1,112 @@
+import {
+  IntegrationIngestionConfigField,
+  IntegrationIngestionConfigFieldMap,
+  IntegrationSourceId,
+  Step,
+  StepExecutionContext,
+} from '@jupiterone/integration-sdk-core';
+import { createCommand } from 'commander';
+import { loadConfigFromTarget } from '../config';
+import { promises as fs } from 'fs';
+import * as log from '../log';
+
+/* eslint-disable no-console */
+export function generateIngestionSourcesConfigCommand() {
+  return createCommand('generate-ingestion-sources-config')
+    .description(
+      'generate ingestion sources config from ingestion config and steps data',
+    )
+    .option(
+      '-o, --output-file <path>',
+      'project relative path to generated ingestion sources config file',
+    )
+    .option(
+      '-p, --project-path <directory>',
+      'path to integration project directory',
+      process.cwd(),
+    )
+    .action(async (options) => {
+      const { projectPath, outputFile } = options;
+
+      log.info(
+        `Generating ingestion sources config (projectPath=${projectPath}, outputFile=${outputFile})`,
+      );
+      const config = await loadConfigFromTarget(projectPath);
+      if (!config.ingestionConfig) {
+        log.info(
+          'Skipping the generation of ingestion sources config file as there is no ingestionConfig present.',
+        );
+      } else {
+        const ingestionSourcesConfig = generateIngestionSourcesConfig(
+          config.ingestionConfig,
+          config.integrationSteps,
+        );
+        if (outputFile) {
+          await fs.writeFile(
+            outputFile,
+            JSON.stringify(ingestionSourcesConfig),
+            {
+              encoding: 'utf-8',
+            },
+          );
+        } else {
+          console.log(JSON.stringify(ingestionSourcesConfig, null, 2));
+        }
+        log.info('Successfully generated ingestion sources config file');
+      }
+    });
+}
+
+export type EnhancedIntegrationIngestionConfigFieldMap = Record<
+  IntegrationSourceId,
+  IntegrationIngestionConfigField & { childIngestionSources?: string[] }
+>;
+
+/**
+ * Generates an ingestionConfig with childIngestionSources taking into account
+ * the integration steps that come as argument.
+ * The childIngestionSources will be the list of stepIds that have any dependencies
+ * on the steps that match the ingestion sources specified.
+ *
+ * @export
+ * @template TStepExecutionContext
+ * @param {IntegrationIngestionConfigData} ingestionConfigData ingestionData without childIngestionSources
+ * @param {Step<TStepExecutionContext>[]} integrationSteps total list of integration steps
+ * @return {*}  {IntegrationIngestionConfigFieldMap} ingestionData with childIngestionSources
+ */
+export function generateIngestionSourcesConfig<
+  TStepExecutionContext extends StepExecutionContext,
+>(
+  ingestionConfig: IntegrationIngestionConfigFieldMap,
+  integrationSteps: Step<TStepExecutionContext>[],
+): EnhancedIntegrationIngestionConfigFieldMap {
+  const newIngestionConfig: EnhancedIntegrationIngestionConfigFieldMap = {};
+  Object.keys(ingestionConfig).forEach((key) => {
+    if (ingestionConfig[key]) {
+      // Get the stepIds that match the current ingestionSourceId
+      const matchedIntegrationStepIds = integrationSteps
+        .filter((step) => step.ingestionSourceId === key)
+        .map(({ id }) => id);
+      if (!matchedIntegrationStepIds.length) {
+        // Skip iteration if there are no steps pointing to the current ingestionSourceId
+        return;
+      }
+      // Get the stepIds that have any dependencies on the matched step ids
+      const childIngestionSources = integrationSteps
+        .filter((step) =>
+          step.dependsOn?.some((value) =>
+            matchedIntegrationStepIds.includes(value),
+          ),
+        )
+        .map(({ id }) => id);
+      // Generate ingestionConfig with the childIngestionSources
+      newIngestionConfig[key] = {
+        ...ingestionConfig[key],
+        childIngestionSources,
+      };
+    } else {
+      log.warn(`The key ${key} does not exist in the ingestionConfig`);
+    }
+  });
+  return newIngestionConfig;
+}

package/src/commands/generate-integration-graph-schema.ts

@@ -7,8 +7,7 @@ import {
   StepRelationshipMetadata,
 } from '@jupiterone/integration-sdk-core';
 import { createCommand } from 'commander';
-import path from 'path';
-import { IntegrationInvocationConfigLoadError, loadConfig } from '../config';
+import { loadConfigFromTarget } from '../config';
 import { promises as fs } from 'fs';
 import * as log from '../log';
 
@@ -51,48 +50,6 @@ export function generateIntegrationGraphSchemaCommand() {
     });
 }
 
-function loadConfigFromSrc(projectPath: string) {
-  return loadConfig(path.join(projectPath, 'src'));
-}
-
-function loadConfigFromDist(projectPath: string) {
-  return loadConfig(path.join(projectPath, 'dist'));
-}
-
-/**
- * The way that integration npm packages are distributed has changed over time.
- * This function handles different cases where the invocation config has
- * traditionally lived to support backwards compatibility and make adoption
- * easier.
- */
-async function loadConfigFromTarget(projectPath: string) {
-  let configFromSrcErr: Error | undefined;
-  let configFromDistErr: Error | undefined;
-
-  try {
-    const configFromSrc = await loadConfigFromSrc(projectPath);
-    return configFromSrc;
-  } catch (err) {
-    configFromSrcErr = err;
-  }
-
-  try {
-    const configFromDist = await loadConfigFromDist(projectPath);
-    return configFromDist;
-  } catch (err) {
-    configFromDistErr = err;
-  }
-
-  const combinedError = configFromDistErr
-    ? configFromSrcErr + ', ' + configFromDistErr
-    : configFromSrcErr;
-
-  throw new IntegrationInvocationConfigLoadError(
-    'Error loading integration invocation configuration. Ensure "invocationConfig" is exported from src/index or dist/index. Additional details: ' +
-      combinedError,
-  );
-}
-
 type IntegrationGraphSchemaEntityMetadata = {
   resourceName: string;
   _class: string | string[];

package/src/commands/index.ts

@@ -10,3 +10,4 @@ export * from './neo4j';
 export * from './visualize-dependencies';
 export * from './generate-integration-graph-schema';
 export * from './troubleshoot';
+export * from './generate-ingestion-sources-config';

package/src/config.ts

@@ -51,6 +51,48 @@ export function loadInvocationConfig(
   return integrationModule.invocationConfig as IntegrationInvocationConfig;
 }
 
+function loadConfigFromSrc(projectPath: string) {
+  return loadConfig(path.join(projectPath, 'src'));
+}
+
+function loadConfigFromDist(projectPath: string) {
+  return loadConfig(path.join(projectPath, 'dist'));
+}
+
+/**
+ * The way that integration npm packages are distributed has changed over time.
+ * This function handles different cases where the invocation config has
+ * traditionally lived to support backwards compatibility and make adoption
+ * easier.
+ */
+export async function loadConfigFromTarget(projectPath: string) {
+  let configFromSrcErr: Error | undefined;
+  let configFromDistErr: Error | undefined;
+
+  try {
+    const configFromSrc = await loadConfigFromSrc(projectPath);
+    return configFromSrc;
+  } catch (err) {
+    configFromSrcErr = err;
+  }
+
+  try {
+    const configFromDist = await loadConfigFromDist(projectPath);
+    return configFromDist;
+  } catch (err) {
+    configFromDistErr = err;
+  }
+
+  const combinedError = configFromDistErr
+    ? configFromSrcErr + ', ' + configFromDistErr
+    : configFromSrcErr;
+
+  throw new IntegrationInvocationConfigLoadError(
+    'Error loading integration invocation configuration. Ensure "invocationConfig" is exported from src/index or dist/index. Additional details: ' +
+      combinedError,
+  );
+}
+
 async function isTypescriptPresent(
   projectSourceDirectory: string = path.join(process.cwd(), 'src'),
 ) {

package/src/index.ts

@@ -12,6 +12,7 @@ import {
   neo4j,
   visualizeDependencies,
   generateIntegrationGraphSchemaCommand,
+  generateIngestionSourcesConfigCommand,
   troubleshootLocalExecution,
 } from './commands';
 
@@ -28,5 +29,6 @@ export function createCli() {
     .addCommand(neo4j())
     .addCommand(visualizeDependencies())
     .addCommand(generateIntegrationGraphSchemaCommand())
+    .addCommand(generateIngestionSourcesConfigCommand())
     .addCommand(troubleshootLocalExecution());
 }