airbyte-faros-destination 0.9.56 → 0.9.58

package/README.md

@@ -168,6 +168,13 @@ Example `catalog.json`
 }
 ```
 
+To view debug logs from airbyte, set the following environment variable:
+
+```shell
+export LOG_LEVEL="debug";
+./bin/main write --config config.json --catalog catalog.json;
+```
+
 **Tip**: you can even pipe data directly from your custom source into your custom destination without Airbyte server while prefixing your streams (as expected by Faros Destination):
 
 ```shell

package/lib/converters/jira/faros_boards.d.ts

@@ -0,0 +1,7 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { JiraConverter } from './common';
+export declare class FarosBoards extends JiraConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+}

package/lib/converters/jira/faros_boards.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FarosBoards = void 0;
+const common_1 = require("./common");
+class FarosBoards extends common_1.JiraConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = [
+            'tms_TaskBoard',
+            'tms_TaskBoardProjectRelationship',
+        ];
+    }
+    async convert(record, ctx) {
+        const board = record.record.data;
+        const uid = board.uid;
+        const source = this.streamName.source;
+        return [
+            {
+                model: 'tms_TaskBoard',
+                record: { uid, name: board.name, source },
+            },
+            {
+                model: 'tms_TaskBoardProjectRelationship',
+                record: {
+                    board: { uid, source },
+                    project: { uid: board.projectKey, source },
+                },
+            },
+        ];
+    }
+}
+exports.FarosBoards = FarosBoards;
+//# sourceMappingURL=faros_boards.js.map

package/lib/converters/jira/faros_boards.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"faros_boards.js","sourceRoot":"","sources":["../../../src/converters/jira/faros_boards.ts"],"names":[],"mappings":";;;AAGA,qCAAuC;AAEvC,MAAa,WAAY,SAAQ,sBAAa;IAA9C;;QACW,sBAAiB,GAAoC;YAC5D,eAAe;YACf,kCAAkC;SACnC,CAAC;IAuBJ,CAAC;IArBC,KAAK,CAAC,OAAO,CACX,MAAqB,EACrB,GAAkB;QAElB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;QACjC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QACtC,OAAO;YACL;gBACE,KAAK,EAAE,eAAe;gBACtB,MAAM,EAAE,EAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAC;aACxC;YACD;gBACE,KAAK,EAAE,kCAAkC;gBACzC,MAAM,EAAE;oBACN,KAAK,EAAE,EAAC,GAAG,EAAE,MAAM,EAAC;oBACpB,OAAO,EAAE,EAAC,GAAG,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,EAAC;iBACzC;aACF;SACF,CAAC;IACJ,CAAC;CACF;AA3BD,kCA2BC"}

package/lib/converters/jira/faros_projects.d.ts

@@ -0,0 +1,7 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { JiraConverter } from './common';
+export declare class FarosProjects extends JiraConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+}

package/lib/converters/jira/faros_projects.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FarosProjects = void 0;
+const faros_js_client_1 = require("faros-js-client");
+const common_1 = require("./common");
+class FarosProjects extends common_1.JiraConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = ['tms_Project'];
+    }
+    async convert(record, ctx) {
+        const project = record.record.data;
+        const source = this.streamName.source;
+        return [
+            {
+                model: 'tms_Project',
+                record: {
+                    uid: project.key,
+                    name: project.name,
+                    description: faros_js_client_1.Utils.cleanAndTruncate(project.description, this.truncateLimit(ctx)),
+                    sourceSystemId: project.id,
+                    source,
+                },
+            },
+        ];
+    }
+}
+exports.FarosProjects = FarosProjects;
+//# sourceMappingURL=faros_projects.js.map

package/lib/converters/jira/faros_projects.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"faros_projects.js","sourceRoot":"","sources":["../../../src/converters/jira/faros_projects.ts"],"names":[],"mappings":";;;AACA,qDAAsC;AAGtC,qCAAuC;AAEvC,MAAa,aAAc,SAAQ,sBAAa;IAAhD;;QACW,sBAAiB,GAAoC,CAAC,aAAa,CAAC,CAAC;IAwBhF,CAAC;IAtBC,KAAK,CAAC,OAAO,CACX,MAAqB,EACrB,GAAkB;QAElB,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QACtC,OAAO;YACL;gBACE,KAAK,EAAE,aAAa;gBACpB,MAAM,EAAE;oBACN,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,WAAW,EAAE,uBAAK,CAAC,gBAAgB,CACjC,OAAO,CAAC,WAAW,EACnB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CACxB;oBACD,cAAc,EAAE,OAAO,CAAC,EAAE;oBAC1B,MAAM;iBACP;aACF;SACF,CAAC;IACJ,CAAC;CACF;AAzBD,sCAyBC"}

package/lib/converters/vanta/types.d.ts

@@ -1,10 +1,12 @@
 type OptString = string | null | undefined;
+type OptBool = boolean | null | undefined;
+type OptNumber = number | null | undefined;
 export interface BaseVulnerabilityType {
     uid: OptString;
     displayName: OptString;
     createdAt: OptString;
     externalURL: OptString;
-    severity: OptString;
+    severity: OptString | OptNumber;
 }
 export type ExtendedVulnerabilityType = BaseVulnerabilityType & {
     description?: OptString;
@@ -17,13 +19,13 @@ export interface GithubSecurityAdvisory {
     ghsaId: OptString;
 }
 export interface GithubVulnerabilityData {
-    createdAt: OptString;
+    uid: OptString;
     displayName: OptString;
+    createdAt: OptString;
     externalURL: OptString;
+    severity: OptNumber;
     repositoryName: OptString;
-    severity: OptString;
     slaDeadline: OptString;
-    uid: OptString;
     vantaDescription: OptString;
     securityAdvisory: GithubSecurityAdvisory;
 }
@@ -33,17 +35,16 @@ export interface AWSFindings {
     name: OptString;
 }
 export interface AWSVulnerabilityData {
+    uid: OptString;
+    displayName: OptString;
     createdAt: OptString;
+    externalURL: OptString;
+    severity: OptNumber;
     packageName: OptString;
     packageVersion: OptString;
-    externalURL: OptString;
-    scanType: OptString;
-    severity: OptString;
     slaDeadline: OptString;
-    uid: OptString;
     repositoryName: OptString;
     repositoryArn: OptString;
-    displayName: OptString;
     findings: AWSFindings[];
 }
 export interface AWSV2Ignored {
@@ -55,19 +56,16 @@ export interface AWSV2Asset {
     displayName: OptString;
 }
 export interface AWSV2VulnerabilityData {
+    uid: OptString;
+    displayName: OptString;
     createdAt: OptString;
     externalURL: OptString;
+    severity: OptString;
     packageName: OptString;
     packageIdentifier: OptString;
-    scanType: OptString;
-    scannerScore: OptString;
-    severity: OptString;
-    uid: OptString;
     description: OptString;
-    displayName: OptString;
     name: OptString;
-    isFixable: OptString;
-    remediation: OptString;
+    isFixable: OptBool;
     remediateBy: OptString;
     asset: AWSV2Asset;
     ignored: AWSV2Ignored;
@@ -95,4 +93,10 @@ export interface CicdArtifactKey {
     uid: string;
     repository: CicdRepoKey;
 }
+export type VulnerabilityInfo = {
+    id: string;
+    resolvedAt: OptString;
+    vulnerabilityUid: OptString;
+};
+export type FarosObjectKey = VcsOrgKey | VcsRepoKey | CicdOrgKey | CicdRepoKey | CicdArtifactKey;
 export {};

package/lib/converters/vanta/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/converters/vanta/types.ts"],"names":[],"mappings":";;;AAsFa,QAAA,eAAe,GAAa,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/converters/vanta/types.ts"],"names":[],"mappings":";;;AAqFa,QAAA,eAAe,GAAa,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC"}

package/lib/converters/vanta/utils.d.ts

@@ -1 +1,2 @@
 export declare function getQueryFromName(name: string): string;
+export declare function looksLikeGithubCommitSha(sha: string): boolean;

package/lib/converters/vanta/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getQueryFromName = void 0;
+exports.looksLikeGithubCommitSha = exports.getQueryFromName = void 0;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
 // Function to find a specific parent directory
@@ -34,4 +34,8 @@ function getQueryFromName(name) {
     return fs_extra_1.default.readFileSync(`${gql_dir}/${fn}`, 'utf8');
 }
 exports.getQueryFromName = getQueryFromName;
+function looksLikeGithubCommitSha(sha) {
+    return /^[a-f0-9]{40}$/i.test(sha);
+}
+exports.looksLikeGithubCommitSha = looksLikeGithubCommitSha;
 //# sourceMappingURL=utils.js.map

package/lib/converters/vanta/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/converters/vanta/utils.ts"],"names":[],"mappings":";;;;;;AAAA,wDAA0B;AAC1B,gDAAwB;AAExB,+CAA+C;AAC/C,SAAS,mBAAmB,CAC1B,UAAkB,EAClB,aAAqB;IAErB,IAAI,WAAW,GAAG,UAAU,CAAC;IAC7B,OAAO,WAAW,KAAK,GAAG,EAAE,CAAC;QAC3B,IAAI,cAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,aAAa,EAAE,CAAC;YACjD,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,WAAW,GAAG,cAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,gBAAgB,CAAC,IAAY;IAC3C,MAAM,OAAO,GAAG,SAAS,CAAC;IAC1B,sDAAsD;IACtD,MAAM,aAAa,GAAG,mBAAmB,CACvC,OAAO,EACP,2BAA2B,CAC5B,CAAC;IACF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAEtD,MAAM,EAAE,GAAG,GAAG,IAAI,MAAM,CAAC;IACzB,6BAA6B;IAC7B,IAAI,CAAC,kBAAE,CAAC,UAAU,CAAC,GAAG,OAAO,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,SAAS,EAAE,iBAAiB,OAAO,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,iEAAiE;IACjE,OAAO,kBAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACrD,CAAC;AApBD,4CAoBC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/converters/vanta/utils.ts"],"names":[],"mappings":";;;;;;AAAA,wDAA0B;AAC1B,gDAAwB;AAExB,+CAA+C;AAC/C,SAAS,mBAAmB,CAC1B,UAAkB,EAClB,aAAqB;IAErB,IAAI,WAAW,GAAG,UAAU,CAAC;IAC7B,OAAO,WAAW,KAAK,GAAG,EAAE,CAAC;QAC3B,IAAI,cAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,aAAa,EAAE,CAAC;YACjD,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,WAAW,GAAG,cAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,gBAAgB,CAAC,IAAY;IAC3C,MAAM,OAAO,GAAG,SAAS,CAAC;IAC1B,sDAAsD;IACtD,MAAM,aAAa,GAAG,mBAAmB,CACvC,OAAO,EACP,2BAA2B,CAC5B,CAAC;IACF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAEtD,MAAM,EAAE,GAAG,GAAG,IAAI,MAAM,CAAC;IACzB,6BAA6B;IAC7B,IAAI,CAAC,kBAAE,CAAC,UAAU,CAAC,GAAG,OAAO,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,SAAS,EAAE,iBAAiB,OAAO,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,iEAAiE;IACjE,OAAO,kBAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACrD,CAAC;AApBD,4CAoBC;AAED,SAAgB,wBAAwB,CAAC,GAAW;IAClD,OAAO,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACrC,CAAC;AAFD,4DAEC"}

package/lib/converters/vanta/vulnerabilities.d.ts

@@ -1,7 +1,7 @@
 import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { FarosClient } from 'faros-js-client';
 import { Converter, DestinationModel, DestinationRecord, StreamContext } from '../converter';
-import { AWSV2VulnerabilityData, AWSVulnerabilityData, CicdArtifactKey, ExtendedVulnerabilityType, GithubVulnerabilityData, VcsRepoKey } from './types';
-export declare function looksLikeGithubCommitSha(sha: string): boolean;
+import { AWSV2VulnerabilityData, AWSVulnerabilityData, CicdArtifactKey, ExtendedVulnerabilityType, FarosObjectKey, GithubVulnerabilityData, VcsRepoKey, VulnerabilityInfo } from './types';
 export declare abstract class Vulnerabilities extends Converter {
     source: string;
     readonly destinationModels: ReadonlyArray<DestinationModel>;
@@ -20,24 +20,54 @@ export declare abstract class Vulnerabilities extends Converter {
     vcsRepositoryQuery: string;
     cicdArtifactQueryByCommitSha: string;
     cicdArtifactQueryByRepoName: string;
+    vcsRepositoryVulnerabilityQuery: string;
+    cicdArtifactVulnerabilityQuery: string;
+    vulnerabilitiesWithMissingRepositoryNames: Set<string>;
+    vulnerabilitiesWithMissingCICDArtifacts: Set<string>;
+    noResponseFromVcsRepositoryQuery: Set<string>;
+    noResponseFromCicdArtifactQuery: Set<string>;
+    emptyResultsFromVcsRepositoryQuery: Set<string>;
+    emptyResultsFromCicdArtifactQuery: Set<string>;
+    noResponseAWSArtifactFromName: Set<string>;
+    emptyResultsAWSArtifactFromName: Set<string>;
+    nUpdatedVcsVulns: number;
+    nUpdatedCicdVulns: number;
+    missedRepositoryNames: Set<string>;
+    nFarosRequests: number;
+    skipRepoNames: Set<string>;
+    nRecordsProcessed: number;
+    maxNRecords: number | null;
+    filterLastNDays: number | null;
     /** All Vanta records should have id property */
     id(record: AirbyteRecord): any;
+    convert(record: AirbyteRecord): Promise<ReadonlyArray<DestinationRecord>>;
     addUIDToMapping(name: string, uid: string, mapping: Record<string, string[]>): void;
     getVulnRecordsFromGit(data: GithubVulnerabilityData[]): DestinationRecord[];
     getVulnRecordsFromAws(data: AWSVulnerabilityData[]): DestinationRecord[];
-    getVCSRepositoryFromName(vcsRepoName: string, ctx: StreamContext): Promise<VcsRepoKey | null>;
-    getCICDArtifactFromCommitSha(commitSha: string, ctx: StreamContext): Promise<CicdArtifactKey | null>;
-    getAWSCicdArtifactFromName(repoName: string, ctx: StreamContext): Promise<CicdArtifactKey | null>;
-    getVCSMappingsFromGit(gitRepoNamesToUIDs: Record<string, string[]>, ctx: StreamContext): Promise<DestinationRecord[]>;
+    runFunctionInBatches(names: string[], fc: FarosClient, ctx: StreamContext, batchSize: number, func: (names: string[], fc: FarosClient, ctx: StreamContext) => Promise<FarosObjectKey[]>): Promise<FarosObjectKey[]>;
+    getVCSRepositoriesFromNames(vcsRepoNames: string[], fc: FarosClient, ctx: StreamContext): Promise<VcsRepoKey[]>;
+    getCICDArtifactsFromCommitShas(commitShas: string[], fc: FarosClient, ctx: StreamContext): Promise<CicdArtifactKey[]>;
+    getVCSRepositoriesFromNamesBatches(vcsRepoNames: string[], fc: FarosClient, ctx: StreamContext, batchSize: number): Promise<VcsRepoKey[]>;
+    getCICDArtifactFromCommitShaBatches(allCommitShas: Set<string>, fc: FarosClient, ctx: StreamContext): Promise<CicdArtifactKey[]>;
+    getAWSCicdArtifactsFromRepoNamesByBatches(allRepoNames: string[], fc: FarosClient, ctx: StreamContext): Promise<CicdArtifactKey[]>;
+    getAWSCicdArtifactsFromNames(repoNames: string[], fc: FarosClient, ctx: StreamContext): Promise<CicdArtifactKey[]>;
+    getVCSMappingsFromGit(gitRepoNamesToUIDs: Record<string, string[]>, fc: FarosClient, ctx: StreamContext): Promise<DestinationRecord[]>;
     getAWSV2VulnStatusDetail(vuln: AWSV2VulnerabilityData): string;
     getAWSV2VulnStatusCategory(vuln: AWSV2VulnerabilityData): string;
-    getCICDMappingsFromAWS(ctx: StreamContext): Promise<DestinationRecord[]>;
-    getCICDMappingsFromAWSV2(aws_vulns: AWSV2VulnerabilityData[], ctx: StreamContext): Promise<DestinationRecord[]>;
+    getCICDMappingsFromAWS(fc: FarosClient, ctx: StreamContext): Promise<DestinationRecord[]>;
+    getCommitShasFromAWSV2Vulns(vulns: AWSV2VulnerabilityData[], ctx: StreamContext): [Set<string>, Record<string, AWSV2VulnerabilityData>];
+    getCICDMappingsFromAWSV2(aws_vulns: AWSV2VulnerabilityData[], fc: FarosClient, ctx: StreamContext): Promise<DestinationRecord[]>;
     getUidFromAWSV2Vuln(vuln: AWSV2VulnerabilityData): string;
     getVulnRecordsFromAwsV2(data: AWSV2VulnerabilityData[]): DestinationRecord[];
     combineAwsVulns(aws_v1_vulns: DestinationRecord[], aws_v2_vulns: DestinationRecord[], ctx: StreamContext): DestinationRecord[];
     getSecVulnerabilityRecordFromData(data: ExtendedVulnerabilityType): DestinationRecord;
-    convert(record: AirbyteRecord): Promise<ReadonlyArray<DestinationRecord>>;
     printReport(ctx: StreamContext): void;
+    getTotalNumberOfRecords(): number;
+    getPaginatedQueryResults(ctx: any, query: string, key: string): Promise<any>;
+    getAllUnresolvedObjectVulnerabilities(ctx: StreamContext, query: string, key: string): Promise<VulnerabilityInfo[]>;
+    updateVulnsInBatches(ctx: StreamContext, ids: string[], resolvedAt: string, mutationName: string, updateName: string, updatesPerRequest: number): Promise<void>;
+    updateVCSVulnsToResolved(ctx: StreamContext): Promise<number>;
+    updateCICDVulnsToResolved(ctx: StreamContext): Promise<number>;
+    updateExistingVulnerabilities(ctx: StreamContext): Promise<void>;
     onProcessingComplete(ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
 }