@jupiterone/integration-sdk-cli 8.23.0 → 8.24.0

package/src/__tests__/cli.test.ts

@@ -623,9 +623,15 @@ describe('document', () => {
 });
 
 describe('generate-integration-graph-schema', () => {
-  test('should generate integration graph schema', async () => {
+  test('should generate integration graph schema with TypeScript integration', async () => {
     await generateIntegrationGraphSchemaTest(
       'generate-integration-graph-schema_all-graph-data-types',
     );
   });
+
+  test('should generate integration graph schema with integration JavaScript dist', async () => {
+    await generateIntegrationGraphSchemaTest(
+      'generate-integration-graph-schema_all-graph-data-types-dist',
+    );
+  });
 });

package/src/__tests__/util/synchronization.ts

@@ -18,6 +18,13 @@ export function setupSynchronizerApi({ polly, job, baseUrl }: SetupOptions) {
       res.status(200).json({ job });
     });
 
+  polly.server
+    .get(`${baseUrl}/persister/synchronization/jobs/${job.id}`)
+    .intercept((req, res) => {
+      allowCrossOrigin(req, res);
+      res.status(200).json({ job });
+    });
+
   polly.server
     .post(`${baseUrl}/persister/synchronization/jobs/${job.id}/entities`)
     .intercept((req, res) => {
@@ -72,11 +79,18 @@ function allowCrossOrigin(req, res) {
   });
 }
 
-export function generateSynchronizationJob(): SynchronizationJob {
+export function generateSynchronizationJob(
+  options?: Pick<
+    SynchronizationJob,
+    'source' | 'scope' | 'integrationInstanceId' | 'integrationJobId'
+  >,
+): SynchronizationJob {
   return {
     id: 'test',
-    integrationJobId: 'test-job-id',
-    integrationInstanceId: 'test-instance-id',
+    source: options?.source || 'integration-managed',
+    scope: options?.scope,
+    integrationJobId: options?.integrationJobId || 'test-job-id',
+    integrationInstanceId: options?.integrationInstanceId || 'test-instance-id',
     status: SynchronizationJobStatus.AWAITING_UPLOADS,
     startTimestamp: Date.now(),
     numEntitiesUploaded: 0,

package/src/commands/generate-integration-graph-schema.ts

@@ -8,7 +8,7 @@ import {
 } from '@jupiterone/integration-sdk-core';
 import { createCommand } from 'commander';
 import path from 'path';
-import { loadConfig } from '../config';
+import { IntegrationInvocationConfigLoadError, loadConfig } from '../config';
 import { promises as fs } from 'fs';
 import * as log from '../log';
 
@@ -32,7 +32,7 @@ export function generateIntegrationGraphSchemaCommand() {
       log.info(
         `Generating integration graph schema (projectPath=${projectPath}, outputFile=${outputFile})`,
       );
-      const config = await loadConfig(path.join(projectPath, 'src'));
+      const config = await loadConfigFromTarget(projectPath);
 
       const integrationGraphSchema = generateIntegrationGraphSchema(
         config.integrationSteps,
@@ -50,6 +50,48 @@ export function generateIntegrationGraphSchemaCommand() {
     });
 }
 
+function loadConfigFromSrc(projectPath: string) {
+  return loadConfig(path.join(projectPath, 'src'));
+}
+
+function loadConfigFromDist(projectPath: string) {
+  return loadConfig(path.join(projectPath, 'dist'));
+}
+
+/**
+ * The way that integration npm packages are distributed has changed over time.
+ * This function handles different cases where the invocation config has
+ * traditionally lived to support backwards compatibility and make adoption
+ * easier.
+ */
+async function loadConfigFromTarget(projectPath: string) {
+  let configFromSrcErr: Error | undefined;
+  let configFromDistErr: Error | undefined;
+
+  try {
+    const configFromSrc = await loadConfigFromSrc(projectPath);
+    return configFromSrc;
+  } catch (err) {
+    configFromSrcErr = err;
+  }
+
+  try {
+    const configFromDist = await loadConfigFromDist(projectPath);
+    return configFromDist;
+  } catch (err) {
+    configFromDistErr = err;
+  }
+
+  const combinedError = configFromDistErr
+    ? configFromSrcErr + ', ' + configFromDistErr
+    : configFromSrcErr;
+
+  throw new IntegrationInvocationConfigLoadError(
+    'Error loading integration invocation configuration. Ensure "invocationConfig" is exported from src/index or dist/index. Additional details: ' +
+      combinedError,
+  );
+}
+
 type IntegrationGraphSchemaEntityMetadata = {
   resourceName: string;
   _class: string | string[];

package/src/commands/options.ts

@@ -0,0 +1,73 @@
+import { getApiBaseUrl } from '@jupiterone/integration-sdk-runtime';
+
+/**
+ * Determines the JupiterOne `apiBaseUrl` based on these mutually-exclusive `options`:
+ * - --api-base-url - a specific URL to use
+ * - --development - use the development API
+ *
+ * @throws Error if both --api-base-url and --development are specified
+ */
+export function getApiBaseUrlOption(options: any): string {
+  let apiBaseUrl: string;
+  if (options.apiBaseUrl) {
+    if (options.development) {
+      throw new Error(
+        'Invalid configuration supplied.  Cannot specify both --api-base-url and --development(-d) flags.',
+      );
+    }
+    apiBaseUrl = options.apiBaseUrl;
+  } else {
+    apiBaseUrl = getApiBaseUrl({
+      dev: options.development,
+    });
+  }
+  return apiBaseUrl;
+}
+
+interface SynchronizationJobSourceOptions {
+  source: 'integration-managed' | 'integration-external' | 'api';
+  scope?: string;
+  integrationInstanceId?: string;
+}
+
+/**
+ * Determines the `source` configuration for the synchronization job based on these `options`:
+ * - --source - a specific source to use
+ * - --scope - a specific scope to use, required when --source is "api"
+ * - --integrationInstanceId - a specific integrationInstanceId to use, required when --source is not "api"
+ *
+ * @throws Error if --source is "api" and --scope is not specified
+ * @throws Error if --source is "api" and --integrationInstanceId is specified
+ * @throws Error if one of --integrationInstanceId or --source is not specified
+ */
+export function getSynchronizationJobSourceOptions(
+  options: any,
+): SynchronizationJobSourceOptions {
+  const synchronizeOptions: SynchronizationJobSourceOptions = {
+    source: options.source,
+  };
+  if (options.source === 'api') {
+    if (options.integrationInstanceId)
+      throw new Error(
+        'Invalid configuration supplied. Cannot specify both --source api and --integrationInstanceId flags.',
+      );
+    if (!options.scope)
+      throw new Error(
+        'Invalid configuration supplied. --source api requires --scope flag.',
+      );
+    synchronizeOptions.scope = options.scope;
+  } else if (
+    !['integration-managed', 'integration-external'].includes(options.source)
+  ) {
+    throw new Error(
+      `Invalid configuration supplied. --source must be one of: integration-managed, integration-external, api. Received: ${options.source}.`,
+    );
+  } else if (options.integrationInstanceId) {
+    synchronizeOptions.integrationInstanceId = options.integrationInstanceId;
+  } else {
+    throw new Error(
+      'Invalid configuration supplied. --integrationInstanceId or --source api and --scope required.',
+    );
+  }
+  return synchronizeOptions;
+}

package/src/commands/run.ts

@@ -12,21 +12,25 @@ import {
   FileSystemGraphObjectStore,
   finalizeSynchronization,
   getAccountFromEnvironment,
-  getApiBaseUrl,
   getApiKeyFromEnvironment,
   initiateSynchronization,
+  synchronizationStatus,
 } from '@jupiterone/integration-sdk-runtime';
 import { createPersisterApiStepGraphObjectDataUploader } from '@jupiterone/integration-sdk-runtime/dist/src/execution/uploader';
 
 import { loadConfig } from '../config';
 import * as log from '../log';
+import {
+  getApiBaseUrlOption,
+  getSynchronizationJobSourceOptions,
+} from './options';
 
 const DEFAULT_UPLOAD_CONCURRENCY = 5;
 
 export function run() {
   return createCommand('run')
     .description('collect and sync to upload entities and relationships')
-    .requiredOption(
+    .option(
       '-i, --integrationInstanceId <id>',
       '_integrationInstanceId assigned to uploaded entities and relationships',
     )
@@ -40,7 +44,17 @@ export function run() {
       '"true" to target apps.dev.jupiterone.io',
       !!process.env.JUPITERONE_DEV,
     )
-    .option('--api-base-url <url>', 'API base URL used during run operation.')
+    .option('--api-base-url <url>', 'API base URL used during run operation')
+    .option('--skip-finalize', 'skip synchronization finalization')
+    .option(
+      '--source <integration-managed|integration-external|api>',
+      'configure the synchronization job source value',
+      'integration-managed',
+    )
+    .option(
+      '--scope <anystring>',
+      'configure the synchronization job scope value',
+    )
     .option('-V, --disable-schema-validation', 'disable schema validation')
     .option(
       '-u, --upload-batch-size <number>',
@@ -65,19 +79,7 @@ export function run() {
       log.debug('Loading account from JUPITERONE_ACCOUNT environment variable');
       const account = getAccountFromEnvironment();
 
-      let apiBaseUrl: string;
-      if (options.apiBaseUrl) {
-        if (options.development) {
-          throw new Error(
-            'Invalid configuration supplied.  Cannot specify both --api-base-url and --development(-d) flags.',
-          );
-        }
-        apiBaseUrl = options.apiBaseUrl;
-      } else {
-        apiBaseUrl = getApiBaseUrl({
-          dev: options.development,
-        });
-      }
+      const apiBaseUrl = getApiBaseUrlOption(options);
       log.debug(`Configuring client to access "${apiBaseUrl}"`);
 
       const startTime = Date.now();
@@ -88,7 +90,8 @@ export function run() {
         account,
       });
 
-      const { integrationInstanceId } = options;
+      const synchronizationJobSourceOptions =
+        getSynchronizationJobSourceOptions(options);
 
       let logger = createIntegrationLogger({
         name: 'local',
@@ -98,7 +101,7 @@ export function run() {
       const synchronizationContext = await initiateSynchronization({
         logger,
         apiClient,
-        integrationInstanceId,
+        ...synchronizationJobSourceOptions,
       });
 
       logger = synchronizationContext.logger;
@@ -150,12 +153,19 @@ export function run() {
 
         log.displayExecutionResults(executionResults);
 
-        const synchronizationResult = await finalizeSynchronization({
-          ...synchronizationContext,
-          partialDatasets: executionResults.metadata.partialDatasets,
-        });
-
-        log.displaySynchronizationResults(synchronizationResult);
+        if (!options.skipFinalize) {
+          const synchronizationResult = await finalizeSynchronization({
+            ...synchronizationContext,
+            partialDatasets: executionResults.metadata.partialDatasets,
+          });
+          log.displaySynchronizationResults(synchronizationResult);
+        } else {
+          log.info(
+            'Skipping synchronization finalization. Job will remain in "AWAITING_UPLOADS" state.',
+          );
+          const jobStatus = await synchronizationStatus(synchronizationContext);
+          log.displaySynchronizationResults(jobStatus);
+        }
       } catch (err) {
         await eventPublishingQueue.onIdle();
         if (!logger.isHandledError(err)) {

package/src/commands/sync.ts

@@ -5,19 +5,22 @@ import {
   createApiClient,
   createIntegrationLogger,
   getAccountFromEnvironment,
-  getApiBaseUrl,
   getApiKeyFromEnvironment,
   synchronizeCollectedData,
 } from '@jupiterone/integration-sdk-runtime';
 
 import * as log from '../log';
+import {
+  getApiBaseUrlOption,
+  getSynchronizationJobSourceOptions,
+} from './options';
 
 export function sync() {
   return createCommand('sync')
     .description(
       'sync collected data with JupiterOne, requires JUPITERONE_API_KEY, JUPITERONE_ACCOUNT',
     )
-    .requiredOption(
+    .option(
       '-i, --integrationInstanceId <id>',
       '_integrationInstanceId assigned to uploaded entities and relationships',
     )
@@ -32,6 +35,15 @@ export function sync() {
       !!process.env.JUPITERONE_DEV,
     )
     .option('--api-base-url <url>', 'API base URL used during sync operation.')
+    .option(
+      '--source <integration-managed|integration-external|api>',
+      'configure the synchronization job source value',
+      'integration-managed',
+    )
+    .option(
+      '--scope <anystring>',
+      'configure the synchronization job scope value',
+    )
     .option(
       '-u, --upload-batch-size <number>',
       'specify number of items per batch for upload (default 250)',
@@ -54,19 +66,7 @@ export function sync() {
       log.debug('Loading account from JUPITERONE_ACCOUNT environment variable');
       const account = getAccountFromEnvironment();
 
-      let apiBaseUrl: string;
-      if (options.apiBaseUrl) {
-        if (options.development) {
-          throw new Error(
-            'Invalid configuration supplied.  Cannot specify both --api-base-url and --development(-d) flags.',
-          );
-        }
-        apiBaseUrl = options.apiBaseUrl;
-      } else {
-        apiBaseUrl = getApiBaseUrl({
-          dev: options.development,
-        });
-      }
+      const apiBaseUrl = getApiBaseUrlOption(options);
       log.debug(`Configuring client to access "${apiBaseUrl}"`);
 
       const apiClient = createApiClient({
@@ -75,18 +75,19 @@ export function sync() {
         accessToken,
       });
 
-      const { integrationInstanceId } = options;
+      const synchronizationJobSourceOptions =
+        getSynchronizationJobSourceOptions(options);
 
       const logger = createIntegrationLogger({
         name: 'local',
         pretty: true,
       });
       const job = await synchronizeCollectedData({
-        logger: logger.child({ integrationInstanceId }),
+        logger: logger.child(synchronizationJobSourceOptions),
         apiClient,
-        integrationInstanceId,
         uploadBatchSize: options.uploadBatchSize,
         uploadRelationshipBatchSize: options.uploadRelationshipBatchSize,
+        ...synchronizationJobSourceOptions,
       });
 
       log.displaySynchronizationResults(job);