@jupiterone/integration-sdk-cli 8.23.1 → 8.24.1

package/src/__tests__/cli-sync.test.ts

@@ -64,16 +64,10 @@ test('uploads data to the synchronization api and displays the results', async (
   });
 });
 
-test('hits dev urls if JUPITERONE_DEV environment variable is set', async () => {
-  process.env.JUPITERONE_DEV = 'true';
-
+test('skips finalization with skip-finalize', async () => {
   const job = generateSynchronizationJob();
 
-  setupSynchronizerApi({
-    polly,
-    job,
-    baseUrl: 'https://api.dev.jupiterone.io',
-  });
+  setupSynchronizerApi({ polly, job, baseUrl: 'https://api.us.jupiterone.io' });
 
   await createCli().parseAsync([
     'node',
@@ -81,12 +75,13 @@ test('hits dev urls if JUPITERONE_DEV environment variable is set', async () =>
     'sync',
     '--integrationInstanceId',
     'test',
+    '--skip-finalize',
   ]);
 
   expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
   expect(log.displaySynchronizationResults).toHaveBeenCalledWith({
     ...job,
-    status: SynchronizationJobStatus.FINALIZE_PENDING,
+    status: SynchronizationJobStatus.AWAITING_UPLOADS,
     // We arrive at these numbers because of what
     // was written to disk in the 'synchronization' project fixture
     numEntitiesUploaded: 6,
@@ -94,71 +89,32 @@ test('hits dev urls if JUPITERONE_DEV environment variable is set', async () =>
   });
 });
 
-test('hits different url if --api-base-url is set', async () => {
-  const job = generateSynchronizationJob();
+test('does not publish events for source "api" since there is no integrationJobId', async () => {
+  const job = generateSynchronizationJob({ source: 'api', scope: 'test' });
 
   setupSynchronizerApi({
     polly,
     job,
-    baseUrl: 'https://api.TEST.jupiterone.io',
+    baseUrl: 'https://api.us.jupiterone.io',
   });
 
+  let eventsPublished = false;
+  polly.server
+    .post(`https://example.com/persister/synchronization/jobs/${job.id}/events`)
+    .intercept((req, res) => {
+      eventsPublished = true;
+    });
+
   await createCli().parseAsync([
     'node',
     'j1-integration',
     'sync',
-    '--integrationInstanceId',
+    '--source',
+    'api',
+    '--scope',
     'test',
-    '--api-base-url',
-    'https://api.TEST.jupiterone.io',
   ]);
 
+  expect(eventsPublished).toBe(false);
   expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
-  expect(log.displaySynchronizationResults).toHaveBeenCalledWith({
-    ...job,
-    status: SynchronizationJobStatus.FINALIZE_PENDING,
-    // We arrive at these numbers because of what
-    // was written to disk in the 'synchronization' project fixture
-    numEntitiesUploaded: 6,
-    numRelationshipsUploaded: 3,
-  });
-});
-
-test('throws an error if --api-base-url is set with --development', async () => {
-  const job = generateSynchronizationJob();
-
-  setupSynchronizerApi({
-    polly,
-    job,
-    baseUrl: 'https://api.TEST.jupiterone.io',
-  });
-
-  await expect(
-    createCli().parseAsync([
-      'node',
-      'j1-integration',
-      'sync',
-      '--integrationInstanceId',
-      'test',
-      '--development',
-      '--api-base-url',
-      'https://api.TEST.jupiterone.io',
-    ]),
-  ).rejects.toThrow(
-    'Invalid configuration supplied.  Cannot specify both --api-base-url and --development(-d) flags.',
-  );
-});
-
-test('throws if JUPITERONE_API_KEY is not set', async () => {
-  delete process.env.JUPITERONE_API_KEY;
-
-  await expect(
-    createCli().parseAsync([
-      'node',
-      'j1-integration',
-      'sync',
-      '--integrationInstanceId',
-      'test',
-    ]),
-  ).rejects.toThrow('JUPITERONE_API_KEY environment variable must be set');
 });

package/src/__tests__/options.test.ts

@@ -0,0 +1,223 @@
+import { createCommand } from 'commander';
+import {
+  addApiClientOptionsToCommand,
+  addSyncOptionsToCommand,
+  ApiClientOptions,
+  getApiClientOptions,
+  SyncOptions,
+  validateApiClientOptions,
+  validateSyncOptions,
+} from '../commands/options';
+import {
+  DEFAULT_UPLOAD_BATCH_SIZE,
+  JUPITERONE_DEV_API_BASE_URL,
+  JUPITERONE_PROD_API_BASE_URL,
+} from '@jupiterone/integration-sdk-runtime';
+
+const syncOptions = (values?: Partial<SyncOptions>): SyncOptions => ({
+  source: 'integration-managed',
+  uploadBatchSize: DEFAULT_UPLOAD_BATCH_SIZE,
+  uploadRelationshipBatchSize: DEFAULT_UPLOAD_BATCH_SIZE,
+  skipFinalize: false,
+  ...values,
+});
+
+const apiClientOptions = (
+  values?: Partial<ApiClientOptions>,
+): ApiClientOptions => ({
+  apiBaseUrl: JUPITERONE_PROD_API_BASE_URL,
+  development: false,
+  ...values,
+});
+
+describe('addSyncOptionsToCommand', () => {
+  test('default values', () => {
+    expect(
+      addSyncOptionsToCommand(createCommand())
+        .parse(['node', 'command'])
+        .opts<SyncOptions>(),
+    ).toEqual(syncOptions());
+  });
+
+  test('--integrationInstanceId', () => {
+    expect(
+      addSyncOptionsToCommand(createCommand())
+        .parse(['node', 'command', '--integrationInstanceId', 'test'])
+        .opts(),
+    ).toEqual(syncOptions({ integrationInstanceId: 'test' }));
+  });
+
+  test('--source', () => {
+    expect(
+      addSyncOptionsToCommand(createCommand())
+        .parse(['node', 'command', '--source', 'api'])
+        .opts(),
+    ).toEqual(syncOptions({ source: 'api' }));
+  });
+
+  test('--source unknown', () => {
+    expect(() => {
+      addSyncOptionsToCommand(createCommand())
+        .parse(['node', 'command', '--source', 'unknown'])
+        .opts();
+    }).toThrowError(/must be/);
+  });
+});
+
+describe('validateSyncOptions', () => {
+  test('valid', () => {
+    expect(() =>
+      validateSyncOptions(syncOptions({ integrationInstanceId: 'test' })),
+    ).not.toThrow();
+  });
+
+  test('source: api and integrationInstanceId', () => {
+    expect(() =>
+      validateSyncOptions(
+        syncOptions({ integrationInstanceId: 'test', source: 'api' }),
+      ),
+    ).toThrowError(/both --source api and --integrationInstanceId/);
+  });
+
+  test('source: api without scope', () => {
+    expect(() =>
+      validateSyncOptions(syncOptions({ source: 'api' })),
+    ).toThrowError(/--source api requires --scope/);
+  });
+
+  test('defaults with no integrationInstanceId', () => {
+    expect(() => validateSyncOptions(syncOptions())).toThrowError(
+      /--integrationInstanceId or --source api/,
+    );
+  });
+});
+
+describe('addApiClientOptionsToCommand', () => {
+  test('default values', () => {
+    expect(
+      addApiClientOptionsToCommand(createCommand())
+        .parse(['node', 'command'])
+        .opts<SyncOptions>(),
+    ).toEqual(apiClientOptions());
+  });
+
+  test('--development with default --api-base-url', () => {
+    expect(
+      addApiClientOptionsToCommand(createCommand())
+        .parse(['node', 'command', '--development'])
+        .opts(),
+    ).toEqual(
+      apiClientOptions({
+        development: true,
+        apiBaseUrl: JUPITERONE_PROD_API_BASE_URL,
+      }),
+    );
+  });
+});
+
+describe('validApiClientOptions', () => {
+  test('valid', () => {
+    expect(() => validateApiClientOptions(apiClientOptions())).not.toThrow();
+  });
+
+  test('--development with default --api-base-url', () => {
+    expect(() =>
+      validateApiClientOptions(
+        apiClientOptions({
+          development: true,
+          apiBaseUrl: JUPITERONE_PROD_API_BASE_URL,
+        }),
+      ),
+    ).not.toThrow();
+  });
+
+  test('--development with --api-base-url JUPITERONE_DEV_API_BASE_URL', () => {
+    expect(() =>
+      validateApiClientOptions(
+        apiClientOptions({
+          development: true,
+          apiBaseUrl: JUPITERONE_DEV_API_BASE_URL,
+        }),
+      ),
+    ).not.toThrow();
+  });
+
+  test('--development with --api-base-url', () => {
+    expect(() =>
+      validateApiClientOptions(
+        apiClientOptions({
+          development: true,
+          apiBaseUrl: 'https://example.com',
+        }),
+      ),
+    ).toThrow(/both --development and --api-base-url/);
+  });
+});
+
+describe('getApiClientOptions', () => {
+  beforeEach(() => {
+    process.env.JUPITERONE_ACCOUNT = 'test account';
+    process.env.JUPITERONE_API_KEY = 'test api key';
+  });
+
+  afterEach(() => {
+    delete process.env.JUPITERONE_ACCOUNT;
+    delete process.env.JUPITERONE_API_KEY;
+  });
+
+  test('defaults with valid environment', () => {
+    expect(getApiClientOptions(apiClientOptions())).toEqual({
+      apiBaseUrl: JUPITERONE_PROD_API_BASE_URL,
+      accessToken: 'test api key',
+      account: 'test account',
+    });
+  });
+
+  test('defaults with no JUPITERONE_API_KEY', () => {
+    delete process.env.JUPITERONE_API_KEY;
+    expect(() => getApiClientOptions(apiClientOptions())).toThrowError(
+      /JUPITERONE_API_KEY/,
+    );
+  });
+
+  test('defaults with no JUPITERONE_ACCOUNT', () => {
+    delete process.env.JUPITERONE_ACCOUNT;
+    expect(() => getApiClientOptions(apiClientOptions())).toThrowError(
+      /JUPITERONE_ACCOUNT/,
+    );
+  });
+
+  test('--development', () => {
+    expect(
+      getApiClientOptions(apiClientOptions({ development: true })),
+    ).toEqual({
+      apiBaseUrl: JUPITERONE_DEV_API_BASE_URL,
+      accessToken: 'test api key',
+      account: 'test account',
+    });
+  });
+
+  test(`--api-base-url ${JUPITERONE_DEV_API_BASE_URL}`, () => {
+    expect(
+      getApiClientOptions(
+        apiClientOptions({ apiBaseUrl: JUPITERONE_DEV_API_BASE_URL }),
+      ),
+    ).toEqual({
+      apiBaseUrl: JUPITERONE_DEV_API_BASE_URL,
+      accessToken: 'test api key',
+      account: 'test account',
+    });
+  });
+
+  test(`--api-base-url ${JUPITERONE_PROD_API_BASE_URL}`, () => {
+    expect(
+      getApiClientOptions(
+        apiClientOptions({ apiBaseUrl: JUPITERONE_PROD_API_BASE_URL }),
+      ),
+    ).toEqual({
+      apiBaseUrl: JUPITERONE_PROD_API_BASE_URL,
+      accessToken: 'test api key',
+      account: 'test account',
+    });
+  });
+});

package/src/__tests__/util/synchronization.ts

@@ -18,6 +18,13 @@ export function setupSynchronizerApi({ polly, job, baseUrl }: SetupOptions) {
       res.status(200).json({ job });
     });
 
+  polly.server
+    .get(`${baseUrl}/persister/synchronization/jobs/${job.id}`)
+    .intercept((req, res) => {
+      allowCrossOrigin(req, res);
+      res.status(200).json({ job });
+    });
+
   polly.server
     .post(`${baseUrl}/persister/synchronization/jobs/${job.id}/entities`)
     .intercept((req, res) => {
@@ -72,11 +79,18 @@ function allowCrossOrigin(req, res) {
   });
 }
 
-export function generateSynchronizationJob(): SynchronizationJob {
+export function generateSynchronizationJob(
+  options?: Pick<
+    SynchronizationJob,
+    'source' | 'scope' | 'integrationInstanceId' | 'integrationJobId'
+  >,
+): SynchronizationJob {
   return {
     id: 'test',
-    integrationJobId: 'test-job-id',
-    integrationInstanceId: 'test-instance-id',
+    source: options?.source || 'integration-managed',
+    scope: options?.scope,
+    integrationJobId: options?.integrationJobId || 'test-job-id',
+    integrationInstanceId: options?.integrationInstanceId || 'test-instance-id',
     status: SynchronizationJobStatus.AWAITING_UPLOADS,
     startTimestamp: Date.now(),
     numEntitiesUploaded: 0,

package/src/commands/collect.ts

@@ -11,6 +11,7 @@ import {
 
 import { loadConfig } from '../config';
 import * as log from '../log';
+import { addPathOptionsToCommand, configureRuntimeFilesystem } from './options';
 
 // coercion function to collect multiple values for a flag
 const collector = (value: string, arr: string[]) => {
@@ -19,13 +20,11 @@ const collector = (value: string, arr: string[]) => {
 };
 
 export function collect() {
-  return createCommand('collect')
+  const command = createCommand('collect');
+  addPathOptionsToCommand(command);
+
+  return command
     .description('collect data and store entities and relationships to disk')
-    .option(
-      '-p, --project-path <directory>',
-      'path to integration project directory',
-      process.cwd(),
-    )
     .option(
       '-s, --step <steps>',
       'step(s) to run, comma separated. Utilizes available caches to speed up dependent steps.',
@@ -42,6 +41,8 @@ export function collect() {
     )
     .option('-V, --disable-schema-validation', 'disable schema validation')
     .action(async (options) => {
+      configureRuntimeFilesystem(options);
+
       if (!options.cache && options.step.length === 0) {
         throw new Error(
           'Invalid option: Option --no-cache requires option --step to also be specified.',
@@ -53,13 +54,6 @@ export function collect() {
         );
       }
 
-      // Point `fileSystem.ts` functions to expected location relative to
-      // integration project path.
-      process.env.JUPITERONE_INTEGRATION_STORAGE_DIRECTORY = path.resolve(
-        options.projectPath,
-        '.j1-integration',
-      );
-
       if (options.step.length > 0 && options.cache && !options.cachePath) {
         // Step option was used, cache is wanted, and no cache path was provided
         // therefore, copy .j1-integration into .j1-integration-cache

package/src/commands/diff.ts

@@ -146,6 +146,7 @@ function diffEntities(
     };
   }
 
+  /* eslint-disable no-console */
   console.log('--- ENTITY DIFF ---');
   console.log(diffString(oldEntities, newEntities, undefined, { keysOnly }));
 }

package/src/commands/generate-integration-graph-schema.ts

@@ -12,6 +12,7 @@ import { IntegrationInvocationConfigLoadError, loadConfig } from '../config';
 import { promises as fs } from 'fs';
 import * as log from '../log';
 
+/* eslint-disable no-console */
 export function generateIntegrationGraphSchemaCommand() {
   return createCommand('generate-integration-graph-schema')
     .description(

package/src/commands/options.ts

@@ -0,0 +1,190 @@
+import {
+  DEFAULT_UPLOAD_BATCH_SIZE,
+  getAccountFromEnvironment,
+  getApiKeyFromEnvironment,
+  JUPITERONE_DEV_API_BASE_URL,
+  JUPITERONE_PROD_API_BASE_URL,
+} from '@jupiterone/integration-sdk-runtime';
+import { Command, OptionValues } from 'commander';
+import path from 'path';
+
+export interface PathOptions {
+  projectPath: string;
+}
+
+export function addPathOptionsToCommand(command: Command): Command {
+  return command.option(
+    '-p, --project-path <directory>',
+    'path to integration project directory',
+    process.cwd(),
+  );
+}
+
+/**
+ * Configure the filesystem interaction code to function against `${options.projectPath}/.j1-integration by setting the
+ * global `process.env.JUPITERONE_INTEGRATION_STORAGE_DIRECTORY` variable.
+ */
+export function configureRuntimeFilesystem(options: PathOptions): void {
+  process.env.JUPITERONE_INTEGRATION_STORAGE_DIRECTORY = path.resolve(
+    options.projectPath,
+    '.j1-integration',
+  );
+}
+
+export interface SyncOptions {
+  source: 'integration-managed' | 'integration-external' | 'api';
+  scope?: string | undefined;
+  integrationInstanceId?: string | undefined;
+  uploadBatchSize: number;
+  uploadRelationshipBatchSize: number;
+  skipFinalize: boolean;
+}
+
+export function addSyncOptionsToCommand(command: Command): Command {
+  return command
+    .option(
+      '-i, --integrationInstanceId <id>',
+      '_integrationInstanceId assigned to uploaded entities and relationships',
+    )
+    .option(
+      '--source <integration-managed|integration-external|api>',
+      'specify synchronization job source value',
+      (value: string) => {
+        if (
+          value !== 'integration-managed' &&
+          value !== 'integration-external' &&
+          value !== 'api'
+        ) {
+          throw new Error(
+            `--source must be one of "integration-managed", "integration-external", or "api"`,
+          );
+        } else {
+          return value;
+        }
+      },
+      'integration-managed',
+    )
+    .option('--scope <anystring>', 'specify synchronization job scope value')
+    .option(
+      '-u, --upload-batch-size <number>',
+      'specify number of entities and relationships per upload batch',
+      (value, _previous: Number) => Number(value),
+      DEFAULT_UPLOAD_BATCH_SIZE,
+    )
+    .option(
+      '-ur, --upload-relationship-batch-size <number>',
+      'specify number of relationships per upload batch, overrides --upload-batch-size',
+      (value, _previous: Number) => Number(value),
+      DEFAULT_UPLOAD_BATCH_SIZE,
+    )
+    .option(
+      '--skip-finalize',
+      'skip synchronization finalization to leave job open for additional uploads',
+      false,
+    );
+}
+
+/**
+ * Validates options for the synchronization job.
+ *
+ * @throws {Error} if --source is "api" and --scope is not specified
+ * @throws {Error} if --source is "api" and --integrationInstanceId is specified
+ * @throws {Error} if one of --integrationInstanceId or --source is not specified
+ */
+export function validateSyncOptions(options: SyncOptions): SyncOptions {
+  if (options.source === 'api') {
+    if (options.integrationInstanceId)
+      throw new Error(
+        'Invalid configuration supplied. Cannot specify both --source api and --integrationInstanceId flags.',
+      );
+    if (!options.scope)
+      throw new Error(
+        'Invalid configuration supplied. --source api requires --scope flag.',
+      );
+  } else if (
+    !['integration-managed', 'integration-external'].includes(options.source)
+  ) {
+    throw new Error(
+      `Invalid configuration supplied. --source must be one of: integration-managed, integration-external, api. Received: ${options.source}.`,
+    );
+  } else if (!options.integrationInstanceId) {
+    throw new Error(
+      'Invalid configuration supplied. --integrationInstanceId or --source api and --scope required.',
+    );
+  }
+  return options;
+}
+
+/**
+ * Returns an object containing only `SyncOptions` properties. This helps to ensure other properties of `OptionValues` are not passed to the `SynchronizationJob`.
+ */
+export function getSyncOptions(options: OptionValues): SyncOptions {
+  return {
+    source: options.source,
+    scope: options.scope,
+    integrationInstanceId: options.integrationInstanceId,
+    uploadBatchSize: options.uploadBatchSize,
+    uploadRelationshipBatchSize: options.uploadRelationshipBatchSize,
+    skipFinalize: options.skipFinalize,
+  };
+}
+
+export interface ApiClientOptions {
+  development: boolean;
+  apiBaseUrl: string;
+  apiKey?: string;
+  account?: string;
+}
+
+export function addApiClientOptionsToCommand(command: Command): Command {
+  return command
+    .option(
+      '--api-base-url <url>',
+      'specify synchronization API base URL',
+      JUPITERONE_PROD_API_BASE_URL,
+    )
+    .option(
+      '-d, --development',
+      '"true" to target apps.dev.jupiterone.io (JUPITERONE_DEV environment variable)',
+      !!process.env.JUPITERONE_DEV,
+    )
+    .option(
+      '--account <account>',
+      'JupiterOne account ID (JUPITERONE_ACCOUNT environment variable)',
+      process.env.JUPITERONE_ACCOUNT,
+    )
+    .option(
+      '--api-key <key>',
+      'JupiterOne API key (JUPITERONE_API_KEY environment variable)',
+      process.env.JUPITERONE_API_KEY?.replace(/./, '*'),
+    );
+}
+
+export function validateApiClientOptions(options: ApiClientOptions) {
+  if (
+    options.development &&
+    ![JUPITERONE_PROD_API_BASE_URL, JUPITERONE_DEV_API_BASE_URL].includes(
+      options.apiBaseUrl,
+    )
+  ) {
+    throw new Error(
+      `Invalid configuration supplied. Cannot specify both --development and --api-base-url flags.`,
+    );
+  }
+}
+
+/**
+ * Builds a set of options for the 'createApiClient' function.
+ *
+ * @throws IntegrationApiKeyRequiredError
+ * @throws IntegrationAccountRequiredError
+ */
+export function getApiClientOptions(options: ApiClientOptions) {
+  return {
+    apiBaseUrl: options.development
+      ? JUPITERONE_DEV_API_BASE_URL
+      : options.apiBaseUrl,
+    accessToken: options.apiKey || getApiKeyFromEnvironment(),
+    account: options.account || getAccountFromEnvironment(),
+  };
+}