airbyte-faros-destination 0.9.57 → 0.9.59

package/README.md

@@ -168,6 +168,13 @@ Example `catalog.json`
 }
 ```
 
+To view debug logs from airbyte, set the following environment variable:
+
+```shell
+export LOG_LEVEL="debug";
+./bin/main write --config config.json --catalog catalog.json;
+```
+
 **Tip**: you can even pipe data directly from your custom source into your custom destination without Airbyte server while prefixing your streams (as expected by Faros Destination):
 
 ```shell

package/lib/converters/jira/faros_sprint_reports.js

@@ -5,25 +5,12 @@ const lodash_1 = require("lodash");
 const common_1 = require("./common");
 class FarosSprintReports extends common_1.JiraConverter {
     get destinationModels() {
-        return ['tms_Sprint', 'tms_SprintHistory'];
+        return ['tms_SprintHistory'];
     }
     async convert(record, ctx) {
         const sprintReport = record.record.data;
-        const uid = (0, lodash_1.toString)(sprintReport.id);
-        const results = [
-            {
-                model: 'tms_Sprint',
-                record: {
-                    uid,
-                    completedPoints: sprintReport.completedPoints,
-                    completedOutsideSprintPoints: sprintReport.completedInAnotherSprintPoints,
-                    notCompletedPoints: sprintReport.notCompletedPoints,
-                    removedPoints: sprintReport.puntedPoints,
-                    plannedPoints: sprintReport.plannedPoints,
-                    source: this.streamName.source,
-                },
-            },
-        ];
+        const uid = (0, lodash_1.toString)(sprintReport.sprintId);
+        const results = [];
         for (const issue of sprintReport.issues || []) {
             results.push({
                 model: 'tms_SprintHistory',

package/lib/converters/jira/faros_sprint_reports.js.map

@@ -1 +1 @@
-{"version":3,"file":"faros_sprint_reports.js","sourceRoot":"","sources":["../../../src/converters/jira/faros_sprint_reports.ts"],"names":[],"mappings":";;;AACA,mCAAgC;AAGhC,qCAAuC;AAEvC,MAAa,kBAAmB,SAAQ,sBAAa;IACnD,IAAI,iBAAiB;QACnB,OAAO,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;IAC7C,CAAC;IACD,KAAK,CAAC,OAAO,CACX,MAAqB,EACrB,GAAkB;QAElB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;QACxC,MAAM,GAAG,GAAG,IAAA,iBAAQ,EAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,OAAO,GAAwB;YACnC;gBACE,KAAK,EAAE,YAAY;gBACnB,MAAM,EAAE;oBACN,GAAG;oBACH,eAAe,EAAE,YAAY,CAAC,eAAe;oBAC7C,4BAA4B,EAC1B,YAAY,CAAC,8BAA8B;oBAC7C,kBAAkB,EAAE,YAAY,CAAC,kBAAkB;oBACnD,aAAa,EAAE,YAAY,CAAC,YAAY;oBACxC,aAAa,EAAE,YAAY,CAAC,aAAa;oBACzC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;iBAC/B;aACF;SACF,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC;gBACX,KAAK,EAAE,mBAAmB;gBAC1B,MAAM,EAAE;oBACN,MAAM,EAAE,EAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAC;oBAC7C,KAAK,EAAE,EAAC,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAC;oBACvD,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,EAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAC;oBAChC,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;iBAC3C;aACF,CAAC,CAAC;QACL,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAvCD,gDAuCC"}
+{"version":3,"file":"faros_sprint_reports.js","sourceRoot":"","sources":["../../../src/converters/jira/faros_sprint_reports.ts"],"names":[],"mappings":";;;AAEA,mCAAgC;AAGhC,qCAAuC;AAEvC,MAAa,kBAAmB,SAAQ,sBAAa;IACnD,IAAI,iBAAiB;QACnB,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAC/B,CAAC;IACD,KAAK,CAAC,OAAO,CACX,MAAqB,EACrB,GAAkB;QAElB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,IAAoB,CAAC;QACxD,MAAM,GAAG,GAAG,IAAA,iBAAQ,EAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAwB,EAAE,CAAC;QACxC,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC;gBACX,KAAK,EAAE,mBAAmB;gBAC1B,MAAM,EAAE;oBACN,MAAM,EAAE,EAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAC;oBAC7C,KAAK,EAAE,EAAC,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAC;oBACvD,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,EAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAC;oBAChC,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;iBAC3C;aACF,CAAC,CAAC;QACL,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAzBD,gDAyBC"}

package/lib/converters/vanta/types.d.ts

@@ -1,10 +1,12 @@
 type OptString = string | null | undefined;
+type OptBool = boolean | null | undefined;
+type OptNumber = number | null | undefined;
 export interface BaseVulnerabilityType {
     uid: OptString;
     displayName: OptString;
     createdAt: OptString;
     externalURL: OptString;
-    severity: OptString;
+    severity: OptString | OptNumber;
 }
 export type ExtendedVulnerabilityType = BaseVulnerabilityType & {
     description?: OptString;
@@ -17,13 +19,13 @@ export interface GithubSecurityAdvisory {
     ghsaId: OptString;
 }
 export interface GithubVulnerabilityData {
-    createdAt: OptString;
+    uid: OptString;
     displayName: OptString;
+    createdAt: OptString;
     externalURL: OptString;
+    severity: OptNumber;
     repositoryName: OptString;
-    severity: OptString;
     slaDeadline: OptString;
-    uid: OptString;
     vantaDescription: OptString;
     securityAdvisory: GithubSecurityAdvisory;
 }
@@ -33,17 +35,16 @@ export interface AWSFindings {
     name: OptString;
 }
 export interface AWSVulnerabilityData {
+    uid: OptString;
+    displayName: OptString;
     createdAt: OptString;
+    externalURL: OptString;
+    severity: OptNumber;
     packageName: OptString;
     packageVersion: OptString;
-    externalURL: OptString;
-    scanType: OptString;
-    severity: OptString;
     slaDeadline: OptString;
-    uid: OptString;
     repositoryName: OptString;
     repositoryArn: OptString;
-    displayName: OptString;
     findings: AWSFindings[];
 }
 export interface AWSV2Ignored {
@@ -55,19 +56,16 @@ export interface AWSV2Asset {
     displayName: OptString;
 }
 export interface AWSV2VulnerabilityData {
+    uid: OptString;
+    displayName: OptString;
     createdAt: OptString;
     externalURL: OptString;
+    severity: OptString;
     packageName: OptString;
     packageIdentifier: OptString;
-    scanType: OptString;
-    scannerScore: OptString;
-    severity: OptString;
-    uid: OptString;
     description: OptString;
-    displayName: OptString;
     name: OptString;
-    isFixable: OptString;
-    remediation: OptString;
+    isFixable: OptBool;
     remediateBy: OptString;
     asset: AWSV2Asset;
     ignored: AWSV2Ignored;
@@ -95,4 +93,10 @@ export interface CicdArtifactKey {
     uid: string;
     repository: CicdRepoKey;
 }
+export type VulnerabilityInfo = {
+    id: string;
+    resolvedAt: OptString;
+    vulnerabilityUid: OptString;
+};
+export type FarosObjectKey = VcsOrgKey | VcsRepoKey | CicdOrgKey | CicdRepoKey | CicdArtifactKey;
 export {};

package/lib/converters/vanta/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/converters/vanta/types.ts"],"names":[],"mappings":";;;AAsFa,QAAA,eAAe,GAAa,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/converters/vanta/types.ts"],"names":[],"mappings":";;;AAqFa,QAAA,eAAe,GAAa,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC"}

package/lib/converters/vanta/utils.d.ts

@@ -1 +1,2 @@
 export declare function getQueryFromName(name: string): string;
+export declare function looksLikeGithubCommitSha(sha: string): boolean;

package/lib/converters/vanta/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getQueryFromName = void 0;
+exports.looksLikeGithubCommitSha = exports.getQueryFromName = void 0;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
 // Function to find a specific parent directory
@@ -34,4 +34,8 @@ function getQueryFromName(name) {
     return fs_extra_1.default.readFileSync(`${gql_dir}/${fn}`, 'utf8');
 }
 exports.getQueryFromName = getQueryFromName;
+function looksLikeGithubCommitSha(sha) {
+    return /^[a-f0-9]{40}$/i.test(sha);
+}
+exports.looksLikeGithubCommitSha = looksLikeGithubCommitSha;
 //# sourceMappingURL=utils.js.map

package/lib/converters/vanta/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/converters/vanta/utils.ts"],"names":[],"mappings":";;;;;;AAAA,wDAA0B;AAC1B,gDAAwB;AAExB,+CAA+C;AAC/C,SAAS,mBAAmB,CAC1B,UAAkB,EAClB,aAAqB;IAErB,IAAI,WAAW,GAAG,UAAU,CAAC;IAC7B,OAAO,WAAW,KAAK,GAAG,EAAE,CAAC;QAC3B,IAAI,cAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,aAAa,EAAE,CAAC;YACjD,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,WAAW,GAAG,cAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,gBAAgB,CAAC,IAAY;IAC3C,MAAM,OAAO,GAAG,SAAS,CAAC;IAC1B,sDAAsD;IACtD,MAAM,aAAa,GAAG,mBAAmB,CACvC,OAAO,EACP,2BAA2B,CAC5B,CAAC;IACF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAEtD,MAAM,EAAE,GAAG,GAAG,IAAI,MAAM,CAAC;IACzB,6BAA6B;IAC7B,IAAI,CAAC,kBAAE,CAAC,UAAU,CAAC,GAAG,OAAO,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,SAAS,EAAE,iBAAiB,OAAO,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,iEAAiE;IACjE,OAAO,kBAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACrD,CAAC;AApBD,4CAoBC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/converters/vanta/utils.ts"],"names":[],"mappings":";;;;;;AAAA,wDAA0B;AAC1B,gDAAwB;AAExB,+CAA+C;AAC/C,SAAS,mBAAmB,CAC1B,UAAkB,EAClB,aAAqB;IAErB,IAAI,WAAW,GAAG,UAAU,CAAC;IAC7B,OAAO,WAAW,KAAK,GAAG,EAAE,CAAC;QAC3B,IAAI,cAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,aAAa,EAAE,CAAC;YACjD,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,WAAW,GAAG,cAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,gBAAgB,CAAC,IAAY;IAC3C,MAAM,OAAO,GAAG,SAAS,CAAC;IAC1B,sDAAsD;IACtD,MAAM,aAAa,GAAG,mBAAmB,CACvC,OAAO,EACP,2BAA2B,CAC5B,CAAC;IACF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAEtD,MAAM,EAAE,GAAG,GAAG,IAAI,MAAM,CAAC;IACzB,6BAA6B;IAC7B,IAAI,CAAC,kBAAE,CAAC,UAAU,CAAC,GAAG,OAAO,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,SAAS,EAAE,iBAAiB,OAAO,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,iEAAiE;IACjE,OAAO,kBAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACrD,CAAC;AApBD,4CAoBC;AAED,SAAgB,wBAAwB,CAAC,GAAW;IAClD,OAAO,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACrC,CAAC;AAFD,4DAEC"}

package/lib/converters/vanta/vulnerabilities.d.ts

@@ -1,7 +1,7 @@
 import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { FarosClient } from 'faros-js-client';
 import { Converter, DestinationModel, DestinationRecord, StreamContext } from '../converter';
-import { AWSV2VulnerabilityData, AWSVulnerabilityData, CicdArtifactKey, ExtendedVulnerabilityType, GithubVulnerabilityData, VcsRepoKey } from './types';
-export declare function looksLikeGithubCommitSha(sha: string): boolean;
+import { AWSV2VulnerabilityData, AWSVulnerabilityData, CicdArtifactKey, ExtendedVulnerabilityType, FarosObjectKey, GithubVulnerabilityData, VcsRepoKey, VulnerabilityInfo } from './types';
 export declare abstract class Vulnerabilities extends Converter {
     source: string;
     readonly destinationModels: ReadonlyArray<DestinationModel>;
@@ -20,24 +20,54 @@ export declare abstract class Vulnerabilities extends Converter {
     vcsRepositoryQuery: string;
     cicdArtifactQueryByCommitSha: string;
     cicdArtifactQueryByRepoName: string;
+    vcsRepositoryVulnerabilityQuery: string;
+    cicdArtifactVulnerabilityQuery: string;
+    vulnerabilitiesWithMissingRepositoryNames: Set<string>;
+    vulnerabilitiesWithMissingCICDArtifacts: Set<string>;
+    noResponseFromVcsRepositoryQuery: Set<string>;
+    noResponseFromCicdArtifactQuery: Set<string>;
+    emptyResultsFromVcsRepositoryQuery: Set<string>;
+    emptyResultsFromCicdArtifactQuery: Set<string>;
+    noResponseAWSArtifactFromName: Set<string>;
+    emptyResultsAWSArtifactFromName: Set<string>;
+    nUpdatedVcsVulns: number;
+    nUpdatedCicdVulns: number;
+    missedRepositoryNames: Set<string>;
+    nFarosRequests: number;
+    skipRepoNames: Set<string>;
+    nRecordsProcessed: number;
+    maxNRecords: number | null;
+    filterLastNDays: number | null;
     /** All Vanta records should have id property */
     id(record: AirbyteRecord): any;
+    convert(record: AirbyteRecord): Promise<ReadonlyArray<DestinationRecord>>;
     addUIDToMapping(name: string, uid: string, mapping: Record<string, string[]>): void;
     getVulnRecordsFromGit(data: GithubVulnerabilityData[]): DestinationRecord[];
     getVulnRecordsFromAws(data: AWSVulnerabilityData[]): DestinationRecord[];
-    getVCSRepositoryFromName(vcsRepoName: string, ctx: StreamContext): Promise<VcsRepoKey | null>;
-    getCICDArtifactFromCommitSha(commitSha: string, ctx: StreamContext): Promise<CicdArtifactKey | null>;
-    getAWSCicdArtifactFromName(repoName: string, ctx: StreamContext): Promise<CicdArtifactKey | null>;
-    getVCSMappingsFromGit(gitRepoNamesToUIDs: Record<string, string[]>, ctx: StreamContext): Promise<DestinationRecord[]>;
+    runFunctionInBatches(names: string[], fc: FarosClient, ctx: StreamContext, batchSize: number, func: (names: string[], fc: FarosClient, ctx: StreamContext) => Promise<FarosObjectKey[]>): Promise<FarosObjectKey[]>;
+    getVCSRepositoriesFromNames(vcsRepoNames: string[], fc: FarosClient, ctx: StreamContext): Promise<VcsRepoKey[]>;
+    getCICDArtifactsFromCommitShas(commitShas: string[], fc: FarosClient, ctx: StreamContext): Promise<CicdArtifactKey[]>;
+    getVCSRepositoriesFromNamesBatches(vcsRepoNames: string[], fc: FarosClient, ctx: StreamContext, batchSize: number): Promise<VcsRepoKey[]>;
+    getCICDArtifactFromCommitShaBatches(allCommitShas: Set<string>, fc: FarosClient, ctx: StreamContext): Promise<CicdArtifactKey[]>;
+    getAWSCicdArtifactsFromRepoNamesByBatches(allRepoNames: string[], fc: FarosClient, ctx: StreamContext): Promise<CicdArtifactKey[]>;
+    getAWSCicdArtifactsFromNames(repoNames: string[], fc: FarosClient, ctx: StreamContext): Promise<CicdArtifactKey[]>;
+    getVCSMappingsFromGit(gitRepoNamesToUIDs: Record<string, string[]>, fc: FarosClient, ctx: StreamContext): Promise<DestinationRecord[]>;
     getAWSV2VulnStatusDetail(vuln: AWSV2VulnerabilityData): string;
     getAWSV2VulnStatusCategory(vuln: AWSV2VulnerabilityData): string;
-    getCICDMappingsFromAWS(ctx: StreamContext): Promise<DestinationRecord[]>;
-    getCICDMappingsFromAWSV2(aws_vulns: AWSV2VulnerabilityData[], ctx: StreamContext): Promise<DestinationRecord[]>;
+    getCICDMappingsFromAWS(fc: FarosClient, ctx: StreamContext): Promise<DestinationRecord[]>;
+    getCommitShasFromAWSV2Vulns(vulns: AWSV2VulnerabilityData[], ctx: StreamContext): [Set<string>, Record<string, AWSV2VulnerabilityData>];
+    getCICDMappingsFromAWSV2(aws_vulns: AWSV2VulnerabilityData[], fc: FarosClient, ctx: StreamContext): Promise<DestinationRecord[]>;
     getUidFromAWSV2Vuln(vuln: AWSV2VulnerabilityData): string;
     getVulnRecordsFromAwsV2(data: AWSV2VulnerabilityData[]): DestinationRecord[];
     combineAwsVulns(aws_v1_vulns: DestinationRecord[], aws_v2_vulns: DestinationRecord[], ctx: StreamContext): DestinationRecord[];
     getSecVulnerabilityRecordFromData(data: ExtendedVulnerabilityType): DestinationRecord;
-    convert(record: AirbyteRecord): Promise<ReadonlyArray<DestinationRecord>>;
     printReport(ctx: StreamContext): void;
+    getTotalNumberOfRecords(): number;
+    getPaginatedQueryResults(ctx: any, query: string, key: string): Promise<any>;
+    getAllUnresolvedObjectVulnerabilities(ctx: StreamContext, query: string, key: string): Promise<VulnerabilityInfo[]>;
+    updateVulnsInBatches(ctx: StreamContext, ids: string[], resolvedAt: string, mutationName: string, updateName: string, updatesPerRequest: number): Promise<void>;
+    updateVCSVulnsToResolved(ctx: StreamContext): Promise<number>;
+    updateCICDVulnsToResolved(ctx: StreamContext): Promise<number>;
+    updateExistingVulnerabilities(ctx: StreamContext): Promise<void>;
     onProcessingComplete(ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
 }