@jupiterone/integration-sdk-cli 12.8.1 → 13.0.0

package/src/bocchi/templates/top-level/src/client.ts.hbs

@@ -0,0 +1,116 @@
+import { BaseAPIClient } from '@jupiterone/integration-sdk-http-client';
+import { IntegrationConfig } from './config';
+import { 
+{{#each template.steps}}
+  {{pascalCase entity.name}}, 
+{{/each}}
+} from './steps/types';
+import { Entity, IntegrationLogger } from '@jupiterone/integration-sdk-core';
+
+export type ResourceIteratee<T> = (each: T) => Promise<void> | void;
+
+export class APIClient extends BaseAPIClient {
+  private authHeaders: Record<string, string>;
+
+  constructor(readonly config: IntegrationConfig, logger: IntegrationLogger) {
+    super({
+      baseUrl: '{{template.baseUrl}}',
+      logger,
+      {{#if template.tokenBucket}}
+      tokenBucket: {
+        maximumCapacity: {{template.tokenBucket.maximumCapacity}},
+        refillRate: {{template.tokenBucket.refillRate}}
+      }
+      {{/if}}
+    });
+  }
+
+  {{#if (isNotEndpointAuth template.authentication.strategy)}}
+  protected getAuthorizationHeaders(): Record<string, string> {
+    return { {{#each (sanitizeAuthObject template.authentication.authHeaders)}}'{{@key}}': `{{this}}`,{{/each}} };
+  }  
+  {{else}}
+  protected async getAuthorizationHeaders(): Promise<Record<string, string>> {
+    const response = await (await this.retryableRequest(
+      `${this.baseUrl}{{template.authentication.params.path}}`,
+      {
+        authorize: false, // needed only for authentication call
+        headers: { {{#each (sanitizeAuthObject template.authentication.params.headers)}}'{{@key}}': `{{this}}`,{{/each}} },
+        method: '{{sanitizeHttpMethod template.authentication.params.method}}',
+      {{#if request.params}}
+        body: { {{#each (sanitizeAuthObject template.authentication.params.body)}}{{@key}}: {{this}},{{/each}} }
+      {{/if}}
+      }
+    )).json();
+    return { {{#each (sanitizeAuthObject template.authentication.authHeaders)}}'{{@key}}': `{{this}}`,{{/each}} }
+  }
+  {{/if}}
+
+  public async verifyAuthentication(): Promise<void> {
+    {{#if (isNotEndpointAuth template.authentication.strategy)}}
+    // TODO: implement
+    await new Promise(() => { throw new Error('Unimplemented: Pick an API call to use for validation!') });
+    {{else}}
+    await this.getAuthorizationHeaders();
+    {{/if}}
+  }
+{{#each template.steps}}
+
+  {{#if (isSingletonRequest response.responseType)}}
+  public async get{{pascalCase entity.name}}({{#if parentAssociation}}parentEntity: Entity{{/if}}): Promise<{{pascalCase entity.name}}> {
+    const url = `${this.baseUrl}{{{sanitizeUrlPath request.urlTemplate}}}`;
+    const response = await (await this.retryableRequest(
+      url,
+      {
+        method: '{{sanitizeHttpMethod request.method}}',
+      {{#if request.params}}
+        body: { {{#each (sanitizeHttpBody request.params)}}'{{@key}}': `{{this}}`,{{/each}} }
+      {{/if}}
+      }
+    )).json();
+    return response.{{response.dataPath}} as {{pascalCase entity.name}};
+  }
+  {{else}}
+  public async iterate{{pascalCase entity.name}}s(
+    {{#if parentAssociation}}
+    parentEntity: Entity,
+    {{/if}}
+    iteratee: ResourceIteratee<{{pascalCase entity.name}}>
+  ): Promise<void> {
+    {{#if response.nextTokenPath}}
+    let nextToken: string | undefined;
+    do {
+    {{/if}}
+      const url = `${this.baseUrl}{{{sanitizeUrlPath request.urlTemplate}}}`;
+      const response = await (await this.retryableRequest(
+        url,
+        {
+          method: '{{sanitizeHttpMethod request.method}}',
+        {{#if request.params}}
+          body: { {{#each (sanitizeHttpBody request.params)}}'{{@key}}': `{{this}}`,{{/each}} }
+        {{/if}}
+        }
+      )).json();
+      const resources = response.{{response.dataPath}} as {{pascalCase entity.name}}[];
+      for (const resource of resources) {
+        await iteratee(resource);
+      }
+    {{#if response.nextTokenPath}}
+      nextToken = response.{{response.nextTokenPath}};
+    } while (nextToken);
+    {{/if}}
+  }
+  {{/if}}
+{{/each}}
+}
+
+let client: APIClient | undefined;
+
+export function createAPIClient(
+  config: IntegrationConfig,
+  logger: IntegrationLogger
+): APIClient {
+  return client
+    ? client
+    : new APIClient(config, logger);
+}

package/src/bocchi/templates/top-level/src/config.ts.hbs

@@ -0,0 +1,41 @@
+{{#with template}}
+import {
+  IntegrationExecutionContext,
+  IntegrationValidationError,
+  IntegrationInstanceConfigFieldMap,
+} from '@jupiterone/integration-sdk-core';
+import { createAPIClient } from './client';
+
+export const instanceConfigFields = {
+{{#each instanceConfigFields}}
+    {{@key}}: {
+      {{#if type}}type: '{{type}}',{{/if}}
+      {{#if mask}}mask: {{mask}},{{/if}}
+      {{#if optional}}optional: {{optional}},{{/if}}
+    },
+{{/each}}
+} satisfies IntegrationInstanceConfigFieldMap;
+
+export interface IntegrationConfig {
+{{#each instanceConfigFields}}
+  {{@key}}: {{configTypeToType type}};
+{{/each}}
+}
+
+export async function validateInvocation(
+  context: IntegrationExecutionContext<IntegrationConfig>,
+) {
+  const { config } = context.instance;
+
+  {{#with (requiredConfig instanceConfigFields)}}
+  {{#if this.length}}
+  if ({{#each this}}!config.{{this}}{{#unless @last}} || {{/unless}}{{/each}}) {
+    throw new IntegrationValidationError('Config requires all of{{#each this}} {{this}}{{/each}}');
+  }
+  {{/if}}
+  {{/with}}
+
+  const apiClient = createAPIClient(config);
+  await apiClient.verifyAuthentication();
+}
+{{/with}}

package/src/bocchi/templates/top-level/src/index.test.ts.hbs

@@ -0,0 +1,6 @@
+import { invocationConfig as implementedConfig } from '.';
+import { invocationConfig as specConfig } from '../docs/spec';
+
+test('implemented integration should match spec', () => {
+  expect(implementedConfig).toImplementSpec(specConfig, { requireSpec: true });
+});

package/src/bocchi/templates/top-level/src/index.ts.hbs

@@ -0,0 +1,14 @@
+import { IntegrationInvocationConfig } from '@jupiterone/integration-sdk-core';
+import { integrationSteps } from './steps';
+import {
+  validateInvocation,
+  IntegrationConfig,
+  instanceConfigFields,
+} from './config';
+
+export const invocationConfig: IntegrationInvocationConfig<IntegrationConfig> =
+  {
+    instanceConfigFields,
+    validateInvocation,
+    integrationSteps,
+  };

package/src/bocchi/templates/top-level/src/steps/constants.ts.hbs

@@ -0,0 +1,60 @@
+{{#with template}}
+import {
+  RelationshipClass,
+  StepEntityMetadata,
+  StepRelationshipMetadata,
+} from '@jupiterone/integration-sdk-core';
+
+export const Steps = {
+{{#each steps}}
+  {{constantCase id}}: '{{id}}',
+  {{#if directRelationships}}
+    {{#with (getDirectRelationships this)}}
+      {{#each this}}
+  BUILD_{{constantCase sourceStep.entity.name}}_{{constantCase targetStep.entity.name}}_RELATIONSHIPS: 'build-{{kebabCase sourceStep.entity.name}}-{{kebabCase targetStep.entity.name}}-relationships',
+      {{/each}}
+    {{/with}}
+  {{/if}}
+  {{#if mappedRelationships}}
+  BUILD_{{constantCase entity.name}}_MAPPED_RELATIONSHIPS: 'build-{{kebabCase entity.name}}-mapped-relationships',
+  {{/if}}
+{{/each}}
+} satisfies Record<string, string>;
+
+export const Entities = {
+{{#each steps}}
+  {{#with entity}}
+  {{constantCase name}}: {
+    resourceName: '{{name}}',
+    _type: '{{_type}}',
+    _class: ['{{_class}}'],
+  },
+  {{/with}}
+{{/each}}
+} satisfies Record<string, StepEntityMetadata>;
+
+export const Relationships = {
+{{#each steps}}
+  {{#if parentAssociation}}
+    {{#with (getStepByType parentAssociation.parentEntityType)}}{{constantCase entity.name}}{{/with}}_{{constantCase parentAssociation.relationshipClass}}_{{constantCase entity.name}}: {
+      sourceType: Entities.{{#with (getStepByType parentAssociation.parentEntityType)}}{{constantCase entity.name}}{{/with}}._type,
+      _class: RelationshipClass.{{constantCase parentAssociation.relationshipClass}},
+      targetType: Entities.{{constantCase entity.name}}._type,
+      _type: '{{getRelationshipType parentAssociation.relationshipClass parentAssociation.parentEntityType entity._type}}',
+    },
+  {{/if}}
+  {{#if directRelationships}}
+    {{#with (getDirectRelationships this)}}
+      {{#each this}}
+  {{constantCase sourceStep.entity.name}}_{{constantCase relationshipClass}}_{{constantCase targetStep.entity.name}}: {
+    sourceType: Entities.{{constantCase sourceStep.entity.name}}._type,
+    _class: RelationshipClass.{{constantCase relationshipClass}},
+    targetType: Entities.{{constantCase targetStep.entity.name}}._type,
+    _type: '{{getRelationshipType relationshipClass sourceStep.entity._type targetStep.entity._type}}',
+  },
+      {{/each}}
+    {{/with}}
+  {{/if}}
+{{/each}}
+} satisfies Record<string, StepRelationshipMetadata>;
+{{/with}}

package/src/bocchi/templates/top-level/src/steps/converters.ts.hbs

@@ -0,0 +1,37 @@
+{{#with template}}
+import { createIntegrationEntity } from '@jupiterone/integration-sdk-core';
+import { Entities } from './constants';
+import { 
+{{#each steps}}
+  {{pascalCase entity.name}}, 
+{{/each}}
+} from './types';
+
+{{#each steps}}
+    {{#with entity}}
+export function create{{pascalCase name}}Entity(data: {{pascalCase name}}) {
+  return createIntegrationEntity({
+    entityData: {
+      source: data,
+      assign: {
+        _key: data.{{_keyPath}},
+        _type: Entities.{{constantCase name}}._type,
+        _class: Entities.{{constantCase name}}._class,
+        {{#if staticFields}}
+          {{#each staticFields}}
+        {{@key}}: {{{escape this}}},
+          {{/each}}
+        {{/if}}
+        {{#if fieldMappings}}
+          {{#each fieldMappings}}
+        {{@key}}: data.{{this}},
+          {{/each}}
+        {{/if}}
+      },
+    },
+  });
+}
+  {{/with}}
+
+{{/each}}
+{{/with}}

package/src/bocchi/templates/top-level/src/steps/index.ts.hbs

@@ -0,0 +1,13 @@
+{{#with template}}
+{{#each steps}}
+import { {{camelCase entity.name}}Steps } from './{{id}}';
+{{/each}}
+
+const integrationSteps = [
+{{#each steps}}
+  ...{{camelCase entity.name}}Steps,
+{{/each}}
+];
+
+export { integrationSteps };
+{{/with}}

package/src/bocchi/templates/top-level/src/steps/types.ts.hbs

@@ -0,0 +1,6 @@
+{{#with template}}
+{{#each steps}}
+export type {{pascalCase entity.name}} = any;
+
+{{/each}}
+{{/with}}

package/src/bocchi/templates/top-level/test/README.md.hbs

@@ -0,0 +1,5 @@
+# Testing Integrations
+
+For information on testing integrations see: 
+
+[Testing Integrations](https://github.com/JupiterOne/sdk/blob/main/docs/integrations/testing.md)

package/src/bocchi/templates/top-level/test/config.ts.hbs

@@ -0,0 +1,30 @@
+import { IntegrationInvocationConfig } from '@jupiterone/integration-sdk-core';
+import { StepTestConfig } from '@jupiterone/integration-sdk-testing';
+import * as dotenv from 'dotenv';
+import * as path from 'path';
+import { invocationConfig } from '../src';
+import { IntegrationConfig } from '../src/config';
+
+if (process.env.LOAD_ENV) {
+  dotenv.config({
+    path: path.join(__dirname, '../.env'),
+  });
+}
+
+{{#each template.instanceConfigFields}}
+const DEFAULT_{{constantCase @key}} = '';
+{{/each}}
+
+export const integrationConfig: IntegrationConfig = {
+  {{#each template.instanceConfigFields}}
+  {{camelCase @key}}: process.env.{{constantCase @key}} ?? DEFAULT_{{constantCase @key}},
+  {{/each}}
+};
+
+export function buildStepTestConfigForStep(stepId: string): StepTestConfig {
+  return {
+    stepId,
+    instanceConfig: integrationConfig,
+    invocationConfig: invocationConfig as IntegrationInvocationConfig,
+  };
+}

package/src/bocchi/templates/top-level/test/recording.ts.hbs

@@ -0,0 +1,74 @@
+import {
+  setupRecording,
+  Recording,
+  SetupRecordingInput,
+  mutations,
+} from '@jupiterone/integration-sdk-testing';
+
+export { Recording };
+
+export function setupProjectRecording(
+  input: Omit<SetupRecordingInput, 'mutateEntry'>,
+): Recording {
+  return setupRecording({
+    ...input,
+    redactedRequestHeaders: ['Authorization'],
+    redactedResponseHeaders: ['set-cookie'],
+    mutateEntry: mutations.unzipGzippedRecordingEntry,
+    /*mutateEntry: (entry) => {
+      redact(entry);
+    },*/
+  });
+}
+
+// a more sophisticated redaction example below:
+
+/*
+function getRedactedOAuthResponse() {
+  return {
+    access_token: '[REDACTED]',
+    expires_in: 9999,
+    token_type: 'Bearer',
+  };
+}
+
+function redact(entry): void {
+  if (entry.request.postData) {
+    entry.request.postData.text = '[REDACTED]';
+  }
+
+  if (!entry.response.content.text) {
+    return;
+  }
+
+  //let's unzip the entry so we can modify it
+  mutations.unzipGzippedRecordingEntry(entry);
+
+  //we can just get rid of all response content if this was the token call
+  const requestUrl = entry.request.url;
+  if (requestUrl.match(/oauth\/token/)) {
+    entry.response.content.text = JSON.stringify(getRedactedOAuthResponse());
+    return;
+  }
+
+  //if it wasn't a token call, parse the response text, removing any carriage returns or newlines
+  const responseText = entry.response.content.text;
+  const parsedResponseText = JSON.parse(responseText.replace(/\r?\n|\r/g, ''));
+
+  //now we can modify the returned object as desired
+  //in this example, if the return text is an array of objects that have the 'tenant' property...
+  if (parsedResponseText[0]?.tenant) {
+    for (let i = 0; i < parsedResponseText.length; i++) {
+      parsedResponseText[i].client_secret = '[REDACTED]';
+      parsedResponseText[i].jwt_configuration = '[REDACTED]';
+      parsedResponseText[i].signing_keys = '[REDACTED]';
+      parsedResponseText[i].encryption_key = '[REDACTED]';
+      parsedResponseText[i].addons = '[REDACTED]';
+      parsedResponseText[i].client_metadata = '[REDACTED]';
+      parsedResponseText[i].mobile = '[REDACTED]';
+      parsedResponseText[i].native_social_login = '[REDACTED]';
+    }
+  }
+
+  entry.response.content.text = JSON.stringify(parsedResponseText);
+} */

package/src/bocchi/templates/top-level/tsconfig.dist.json.hbs

@@ -0,0 +1,13 @@
+{
+  "extends": "./tsconfig.json",
+  "include": ["src", "package.json"],
+  "exclude": [
+    "dist",
+    "**/*.test.ts",
+    "**/*.test.js",
+    "**/*/__tests__/**/*.ts",
+    "**/*/__tests__/**/*.js",
+    "**/*/__mocks__/**/*.ts",
+    "**/*/__mocks__/**/*.js"
+  ]
+}

package/src/bocchi/templates/top-level/tsconfig.json.hbs

@@ -0,0 +1,7 @@
+{
+  "extends": "./node_modules/@jupiterone/integration-sdk-dev-tools/config/typescript",
+  "compilerOptions": {
+    "outDir": "dist"
+  },
+  "exclude": ["dist"]
+}

package/src/bocchi/utils/types.ts

@@ -0,0 +1,106 @@
+import { RelationshipClass } from '@jupiterone/integration-sdk-core';
+
+export const StepType = {
+  SINGLETON: 'singleton',
+  CHILD_SINGLETON: 'child-singleton',
+  FETCH_ENTITIES: 'fetch-entities',
+  FETCH_CHILD_ENTITIES: 'fetch-child-entities',
+};
+
+export interface Template {
+  instanceConfigFields: {
+    [propertyName: string]: IntegrationConfigProperty;
+  };
+  authentication: EndpointAuthentication | ConfigFieldAuthentication;
+  baseUrl: string;
+  tokenBucket?: {
+    maximumCapacity: number;
+    refillRate: number;
+  };
+  steps: Step[];
+}
+
+interface IntegrationConfigProperty {
+  type?: 'string' | 'json' | 'boolean';
+  mask?: Boolean;
+  optional?: Boolean;
+}
+
+interface EndpointAuthentication {
+  strategy: 'endpoint';
+  params: {
+    path: string;
+    method: 'GET' | 'POST';
+    body?: Record<string, any>;
+    headers?: Record<string, string>;
+  };
+  authHeaders: AuthenticationHeaders;
+}
+
+/**
+ * Authentication headers to be used in API requests.
+ * The `Authorization` header is required, and is typically
+ * a bearer token or API key.
+ */
+interface AuthenticationHeaders {
+  Authorization: string;
+  [headerName: string]: string;
+}
+
+interface ConfigFieldAuthentication {
+  strategy: 'configField';
+  authHeaders: {
+    [headerName: string]: string;
+  };
+}
+
+export interface Step {
+  id: string;
+  name: string;
+  entity: {
+    name: string;
+    _type: string;
+    _class: string | Array<string>;
+    _keyPath: string;
+    fieldMappings?: {
+      [entityProperty: string]: string;
+    };
+    staticFields?: {
+      [entityProperty: string]: string;
+    };
+  };
+  parentAssociation?: {
+    parentEntityType: string;
+    relationshipClass: string;
+  };
+  request: {
+    urlTemplate: string;
+    method?: 'GET' | 'POST';
+    params?: Record<string, any>;
+  };
+  response: {
+    dataPath: string;
+    responseType: 'SINGLETON' | 'LIST';
+    nextTokenPath?: string;
+  };
+  directRelationships?: {
+    targetKey: string;
+    targetType: string;
+    _class: RelationshipClass;
+    direction: 'FORWARD' | 'REVERSE';
+  }[];
+  mappedRelationships?: {
+    _class: RelationshipClass;
+    direction: 'FORWARD' | 'REVERSE';
+    mappings:
+      | {
+          sourceProperty: string;
+          targetProperty: string;
+        }
+      | {
+          targetValue: string;
+          targetProperty: string;
+        };
+  };
+  dependsOn?: string[];
+}

package/src/commands/bocchi.ts

@@ -0,0 +1,28 @@
+import path from 'path';
+import { createCommand } from 'commander';
+const dynamicImport = new Function('specifier', 'return import(specifier)');
+
+export function bocchi() {
+  return createCommand('bocchi')
+    .description('New graph project generator')
+    .action(async (cmdOpts) => {
+      const Plop = await dynamicImport('plop');
+      const configPath = path.resolve(
+        path.join(__dirname, '../bocchi/bocchi.js'),
+      );
+      Plop.Plop.prepare(
+        {
+          cwd: process.cwd(),
+          configPath,
+        },
+        (env) =>
+          Plop.Plop.execute(env, (env) => {
+            const options = {
+              ...env,
+              dest: process.cwd(),
+            };
+            return Plop.run(options, undefined, true);
+          }),
+      );
+    });
+}

package/src/commands/generate-ingestion-sources-config.ts

@@ -102,11 +102,10 @@ export function generateIngestionSourcesConfig<
         return;
       }
       // Get the dependent steps for the given matchedIntegrationStepIds
-      const childIngestionSources = integrationSteps.filter(
-        (step) =>
-          step.dependsOn?.some((value) =>
-            matchedIntegrationStepIds.includes(value),
-          ),
+      const childIngestionSources = integrationSteps.filter((step) =>
+        step.dependsOn?.some((value) =>
+          matchedIntegrationStepIds.includes(value),
+        ),
       );
       // Generate ingestionConfig with the childIngestionSources
       newIngestionConfig[key] = {

package/src/commands/index.ts

@@ -12,3 +12,4 @@ export * from './generate-integration-graph-schema';
 export * from './troubleshoot';
 export * from './generate-ingestion-sources-config';
 export * from './generate';
+export * from './bocchi';

package/src/generator/actions.ts

@@ -37,6 +37,10 @@ async function yarnLint(_answers, config, _plop) {
 
 // NPM commands
 
+async function j1Document(_answers, config, _plop) {
+  return spawnCommand(config, 'npm exec', ['j1-integration', 'document']);
+}
+
 async function npmInstall(_answers, config, _plop) {
   return spawnCommand(config, 'npm', ['install']);
 }
@@ -49,4 +53,12 @@ async function npmLint(_answers, config, _plop) {
   return spawnCommand(config, 'npm', ['run lint']);
 }
 
-export { yarnInstall, yarnFormat, yarnLint, npmInstall, npmFormat, npmLint };
+export {
+  yarnInstall,
+  yarnFormat,
+  yarnLint,
+  npmInstall,
+  npmFormat,
+  npmLint,
+  j1Document,
+};

package/src/index.ts

@@ -15,6 +15,7 @@ import {
   generateIngestionSourcesConfigCommand,
   troubleshootLocalExecution,
   generate,
+  bocchi,
 } from './commands';
 
 export function createCli() {
@@ -32,5 +33,6 @@ export function createCli() {
     .addCommand(generateIntegrationGraphSchemaCommand())
     .addCommand(generateIngestionSourcesConfigCommand())
     .addCommand(troubleshootLocalExecution())
-    .addCommand(generate());
+    .addCommand(generate())
+    .addCommand(bocchi());
 }