airbyte-faros-destination 0.1.74 → 0.1.77

package/lib/converters/azure-repos/common.d.ts

@@ -0,0 +1,17 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { Converter } from '../converter';
+import { PullRequestReviewState, PullRequestState } from './models';
+export declare type ApplicationMapping = Record<string, {
+    name: string;
+    platform?: string;
+}>;
+/** Azurerepos converter base */
+export declare abstract class AzureReposConverter extends Converter {
+    source: string;
+    /** Almost every Azurerepos record have id property */
+    id(record: AirbyteRecord): any;
+    getOrganizationFromUrl(url: string): string;
+    convertStringToNumber(str: string): number;
+    convertPullRequestState(status: string): PullRequestState;
+    convertPullRequestReviewState(vote: number): PullRequestReviewState;
+}

package/lib/converters/azure-repos/common.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AzureReposConverter = void 0;
+const converter_1 = require("../converter");
+const models_1 = require("./models");
+/** Azurerepos converter base */
+class AzureReposConverter extends converter_1.Converter {
+    constructor() {
+        super(...arguments);
+        this.source = 'Azure-Repos';
+    }
+    /** Almost every Azurerepos record have id property */
+    id(record) {
+        var _a, _b;
+        return (_b = (_a = record === null || record === void 0 ? void 0 : record.record) === null || _a === void 0 ? void 0 : _a.data) === null || _b === void 0 ? void 0 : _b.id;
+    }
+    getOrganizationFromUrl(url) {
+        return url.split('/')[3];
+    }
+    convertStringToNumber(str) {
+        const lastString = str.split('-')[str.split('-').length - 1];
+        const onlyNumbers = lastString.replace(/[^\d.-]/g, '');
+        return Number(onlyNumbers);
+    }
+    //https://docs.microsoft.com/en-us/rest/api/azure/devops/git/pull-requests/get-pull-requests?view=azure-devops-rest-4.1#pullrequeststatus
+    convertPullRequestState(status) {
+        switch (status) {
+            case 'completed':
+                return {
+                    category: models_1.PullRequestStateCategory.Closed,
+                    detail: status,
+                };
+            case 'active':
+                return {
+                    category: models_1.PullRequestStateCategory.Merged,
+                    detail: status,
+                };
+            case 'notSet':
+                return {
+                    category: models_1.PullRequestStateCategory.Open,
+                    detail: status,
+                };
+            default:
+                return {
+                    category: models_1.PullRequestStateCategory.Custom,
+                    detail: status,
+                };
+        }
+    }
+    convertPullRequestReviewState(vote) {
+        if (vote > 5)
+            return {
+                category: models_1.PullRequestReviewStateCategory.Approved,
+                detail: `vote ${vote}`,
+            };
+        if (vote > 0)
+            return {
+                category: models_1.PullRequestReviewStateCategory.Commented,
+                detail: `vote ${vote}`,
+            };
+        if (vote > -5)
+            return {
+                category: models_1.PullRequestReviewStateCategory.Custom,
+                detail: `vote ${vote}`,
+            };
+        return {
+            category: models_1.PullRequestReviewStateCategory.Dismissed,
+            detail: `vote ${vote}`,
+        };
+    }
+}
+exports.AzureReposConverter = AzureReposConverter;

package/lib/converters/azure-repos/models.d.ts

@@ -0,0 +1,217 @@
+interface Project {
+    id: string;
+    name: string;
+    url: string;
+    description: string;
+    state: string;
+    revision: number;
+    visibility: string;
+    lastUpdateTime: string;
+}
+interface Creator {
+    displayName: string;
+    url: string;
+    id: string;
+    uniqueName: string;
+    imageUrl: string;
+    descriptor: string;
+}
+export interface Branch {
+    name: string;
+    aheadCount: number;
+    behindCount: number;
+    isBaseVersion: boolean;
+    commits: Commit[];
+}
+interface TagCommitObject {
+    objectId: string;
+    objectType: string;
+}
+export interface TagCommit {
+    name: string;
+    objectId: string;
+    taggedObject: TagCommitObject;
+    taggedBy: CommitAuthor;
+    message: string;
+    url: string;
+}
+export interface Tag {
+    name: string;
+    objectId: string;
+    url: string;
+    creator: Creator;
+    commit: TagCommit;
+}
+export interface Repository {
+    id: string;
+    name: string;
+    url: string;
+    project: Project;
+    defaultBranch: string;
+    size: number;
+    remoteUrl: string;
+    sshUrl: string;
+    webUrl: string;
+    isDisabled: boolean;
+    branches: Branch[];
+    tags: Tag[];
+}
+export interface PullRequestThreadComment {
+    id: number;
+    parentCommentId: number;
+    content: string;
+    publishedDate: string;
+    lastUpdatedDate: string;
+    lastContentUpdatedDate: string;
+    commentType: string;
+    author: Creator;
+}
+export interface PullRequestThread {
+    pullRequestThreadContext?: string;
+    id: number;
+    publishedDate: string;
+    lastUpdatedDate: string;
+    status: string;
+    threadContext?: string;
+    comments: PullRequestThreadComment[];
+}
+interface MergeSourceCommit {
+    commitId: string;
+    url: string;
+}
+export interface PullRequestRepository {
+    id: string;
+    name: string;
+    url: string;
+    project: Project;
+}
+export interface PullRequestCommitChangeCounts {
+    Edit?: number;
+    Add?: number;
+    Delete?: number;
+}
+export interface PullRequestCommit {
+    commitId: string;
+    author: CommitAuthor;
+    committer: CommitAuthor;
+    comment: string;
+    url: string;
+    changeCounts: PullRequestCommitChangeCounts;
+}
+export interface PullRequestReviewer {
+    reviewerUrl: string;
+    vote: number;
+    hasDeclined: boolean;
+    isFlagged: boolean;
+    displayName: string;
+    url: string;
+    id: string;
+    uniqueName: string;
+    imageUrl: string;
+}
+export interface PullRequest {
+    pullRequestId: number;
+    codeReviewId: number;
+    status: string;
+    createdBy: Creator;
+    creationDate: string;
+    closedDate?: string;
+    title: string;
+    description: string;
+    sourceRefName: string;
+    targetRefName: string;
+    mergeStatus: string;
+    isDraft: boolean;
+    mergeId: string;
+    lastMergeSourceCommit: MergeSourceCommit;
+    lastMergeCommit: MergeSourceCommit;
+    reviewers: [PullRequestReviewer];
+    url: string;
+    supportsIterations: boolean;
+    repository: PullRequestRepository;
+    commits: PullRequestCommit[];
+    threads: PullRequestThread[];
+}
+interface CommitAuthor {
+    name: string;
+    email: string;
+    date: string;
+}
+interface CommitChange {
+    Add: number;
+    Edit: number;
+    Delete: number;
+}
+export interface Commit {
+    commitId: string;
+    author: CommitAuthor;
+    committer: CommitAuthor;
+    comment: string;
+    changeCounts: CommitChange;
+    url: string;
+    remoteUrl: string;
+}
+interface Href {
+    href: string;
+}
+interface UserLink {
+    self: Href;
+    memberships: Href;
+    membershipState: Href;
+    storageKey: Href;
+    avatar: Href;
+}
+export interface User {
+    subjectKind: string;
+    domain: string;
+    principalName: string;
+    mailAddress: string;
+    origin: string;
+    originId: string;
+    displayName: string;
+    url: string;
+    descriptor: string;
+    _links: UserLink;
+}
+export interface OrgType {
+    category: OrgTypeCategory;
+    detail: string;
+}
+export declare enum OrgTypeCategory {
+    Organization = "Organization",
+    Workspace = "Workspace",
+    Group = "Group",
+    Custom = "Custom"
+}
+export interface PullRequestState {
+    category: PullRequestStateCategory;
+    detail: string;
+}
+export declare enum PullRequestStateCategory {
+    Closed = "Closed",
+    Merged = "Merged",
+    Open = "Open",
+    Custom = "Custom"
+}
+export interface PullRequestReviewState {
+    category: PullRequestReviewStateCategory;
+    detail: string;
+}
+export declare enum PullRequestReviewStateCategory {
+    Approved = "Approved",
+    Commented = "Commented",
+    ChangesRequested = "ChangesRequested",
+    Dismissed = "Dismissed",
+    Custom = "Custom"
+}
+export interface UserType {
+    category: UserTypeCategory;
+    detail: string;
+}
+export declare enum UserTypeCategory {
+    Bot = "Bot",
+    Organization = "Organization",
+    User = "User",
+    Custom = "Custom"
+}
+export {};

package/lib/converters/azure-repos/models.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UserTypeCategory = exports.PullRequestReviewStateCategory = exports.PullRequestStateCategory = exports.OrgTypeCategory = void 0;
+var OrgTypeCategory;
+(function (OrgTypeCategory) {
+    OrgTypeCategory["Organization"] = "Organization";
+    OrgTypeCategory["Workspace"] = "Workspace";
+    OrgTypeCategory["Group"] = "Group";
+    OrgTypeCategory["Custom"] = "Custom";
+})(OrgTypeCategory = exports.OrgTypeCategory || (exports.OrgTypeCategory = {}));
+var PullRequestStateCategory;
+(function (PullRequestStateCategory) {
+    PullRequestStateCategory["Closed"] = "Closed";
+    PullRequestStateCategory["Merged"] = "Merged";
+    PullRequestStateCategory["Open"] = "Open";
+    PullRequestStateCategory["Custom"] = "Custom";
+})(PullRequestStateCategory = exports.PullRequestStateCategory || (exports.PullRequestStateCategory = {}));
+var PullRequestReviewStateCategory;
+(function (PullRequestReviewStateCategory) {
+    PullRequestReviewStateCategory["Approved"] = "Approved";
+    PullRequestReviewStateCategory["Commented"] = "Commented";
+    PullRequestReviewStateCategory["ChangesRequested"] = "ChangesRequested";
+    PullRequestReviewStateCategory["Dismissed"] = "Dismissed";
+    PullRequestReviewStateCategory["Custom"] = "Custom";
+})(PullRequestReviewStateCategory = exports.PullRequestReviewStateCategory || (exports.PullRequestReviewStateCategory = {}));
+var UserTypeCategory;
+(function (UserTypeCategory) {
+    UserTypeCategory["Bot"] = "Bot";
+    UserTypeCategory["Organization"] = "Organization";
+    UserTypeCategory["User"] = "User";
+    UserTypeCategory["Custom"] = "Custom";
+})(UserTypeCategory = exports.UserTypeCategory || (exports.UserTypeCategory = {}));

package/lib/converters/azure-repos/pull_requests.d.ts

@@ -0,0 +1,7 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { AzureReposConverter } from './common';
+export declare class PullRequests extends AzureReposConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+}

package/lib/converters/azure-repos/pull_requests.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PullRequests = void 0;
+const lib_1 = require("faros-feeds-sdk/lib");
+const common_1 = require("./common");
+class PullRequests extends common_1.AzureReposConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = [
+            'vcs_PullRequest',
+            'vcs_PullRequestReview',
+            'vcs_PullRequestComment',
+        ];
+    }
+    async convert(record, ctx) {
+        var _a, _b, _c;
+        const source = this.streamName.source;
+        const pullRequestItem = record.record.data;
+        const organizationName = this.getOrganizationFromUrl(pullRequestItem.repository.url);
+        const organization = { uid: organizationName, source };
+        const repository = { name: pullRequestItem.repository.name, organization };
+        const pullRequest = { number: pullRequestItem.pullRequestId, repository };
+        const res = [];
+        const diffStats = { linesAdded: 0, linesDeleted: 0, filesChanged: 0 };
+        for (const commit of pullRequestItem.commits) {
+            diffStats.linesAdded += (_a = commit.changeCounts.Add) !== null && _a !== void 0 ? _a : 0;
+            diffStats.linesDeleted += (_b = commit.changeCounts.Delete) !== null && _b !== void 0 ? _b : 0;
+            diffStats.filesChanged += (_c = commit.changeCounts.Edit) !== null && _c !== void 0 ? _c : 0;
+        }
+        for (const thread of pullRequestItem.threads) {
+            for (const comment of thread.comments) {
+                res.push({
+                    model: 'vcs_PullRequestComment',
+                    record: {
+                        number: comment.id,
+                        comment: comment.content,
+                        createdAt: lib_1.Utils.toDate(comment.publishedDate),
+                        updatedAt: lib_1.Utils.toDate(comment.lastUpdatedDate),
+                        author: { uid: comment.author.uniqueName, source },
+                        pullRequest,
+                    },
+                });
+            }
+        }
+        res.push({
+            model: 'vcs_PullRequest',
+            record: {
+                number: pullRequestItem.pullRequestId,
+                title: pullRequestItem.title,
+                state: this.convertPullRequestState(pullRequestItem.status),
+                htmlUrl: pullRequestItem.url,
+                createdAt: lib_1.Utils.toDate(pullRequestItem.creationDate),
+                updatedAt: lib_1.Utils.toDate(pullRequestItem.creationDate),
+                mergedAt: lib_1.Utils.toDate(pullRequestItem.closedDate),
+                commitCount: pullRequestItem.commits.length,
+                commentCount: pullRequestItem.threads.length,
+                diffStats,
+                author: { uid: pullRequestItem.createdBy.uniqueName, source },
+                mergeCommit: {
+                    sha: pullRequestItem.lastMergeCommit.commitId,
+                    repository,
+                },
+                repository,
+            },
+        });
+        for (const reviewer of pullRequestItem.reviewers) {
+            res.push({
+                model: 'vcs_PullRequestReview',
+                record: {
+                    number: this.convertStringToNumber(reviewer.id),
+                    htmlUrl: reviewer.url,
+                    state: this.convertPullRequestReviewState(reviewer.vote),
+                    submittedAt: null,
+                    reviewer: { uid: reviewer.uniqueName, source },
+                    pullRequest,
+                },
+            });
+        }
+        return res;
+    }
+}
+exports.PullRequests = PullRequests;

package/lib/converters/azure-repos/repositories.d.ts

@@ -0,0 +1,8 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { AzureReposConverter } from './common';
+export declare class Repositories extends AzureReposConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    private seenOrganizations;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+}

package/lib/converters/azure-repos/repositories.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Repositories = void 0;
+const lib_1 = require("faros-feeds-sdk/lib");
+const common_1 = require("./common");
+const models_1 = require("./models");
+class Repositories extends common_1.AzureReposConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = [
+            'vcs_Branch',
+            'vcs_Commit',
+            'vcs_BranchCommitAssociation',
+            'vcs_Organization',
+            'vcs_Repository',
+            'vcs_Tag',
+        ];
+        this.seenOrganizations = new Set();
+    }
+    async convert(record, ctx) {
+        const source = this.streamName.source;
+        const repositoryItem = record.record.data;
+        const res = [];
+        const organizationName = this.getOrganizationFromUrl(repositoryItem.url);
+        const organization = { uid: organizationName, source };
+        const repository = { name: repositoryItem.name, organization };
+        if (!this.seenOrganizations.has(organizationName)) {
+            this.seenOrganizations.add(organizationName);
+            res.push({
+                model: 'vcs_Organization',
+                record: {
+                    uid: organizationName,
+                    name: organizationName,
+                    htmlUrl: `https://dev.azure.com/${organizationName}`,
+                    type: { category: models_1.OrgTypeCategory.Organization, organizationName },
+                    description: organizationName,
+                    createdAt: null,
+                    source,
+                },
+            });
+        }
+        let createdAt = null;
+        for (const branch of repositoryItem.branches) {
+            if (createdAt == null &&
+                repositoryItem.defaultBranch.endsWith(branch.name)) {
+                createdAt = lib_1.Utils.toDate(branch.commits[branch.commits.length - 1].committer.date);
+                break;
+            }
+        }
+        res.push({
+            model: 'vcs_Repository',
+            record: {
+                name: repositoryItem.name,
+                fullName: repositoryItem.name,
+                description: repositoryItem.name,
+                private: repositoryItem.project.visibility == 'private',
+                language: null,
+                size: repositoryItem.size,
+                mainBranch: repositoryItem.defaultBranch,
+                htmlUrl: repositoryItem.webUrl,
+                createdAt,
+                updatedAt: null,
+                organization,
+            },
+        });
+        for (const branch of repositoryItem.branches) {
+            res.push({
+                model: 'vcs_Branch',
+                record: {
+                    name: branch.name,
+                    repository,
+                },
+            });
+            for (const commit of branch.commits) {
+                res.push({
+                    model: 'vcs_Commit',
+                    record: {
+                        sha: commit.commitId,
+                        message: commit.comment,
+                        htmlUrl: commit.remoteUrl,
+                        createdAt: lib_1.Utils.toDate(commit.committer.date),
+                        author: { uid: commit.author.email, source },
+                        repository,
+                    },
+                });
+                res.push({
+                    model: 'vcs_BranchCommitAssociation',
+                    record: {
+                        commit: { sha: commit.commitId, repository },
+                        branch: { name: branch.name, repository },
+                    },
+                });
+            }
+        }
+        for (const tag of repositoryItem.tags) {
+            res.push({
+                model: 'vcs_Tag',
+                record: {
+                    name: tag.name,
+                    message: tag.commit.message,
+                    commit: { sha: tag.commit.taggedObject.objectId, repository },
+                    repository,
+                },
+            });
+        }
+        return res;
+    }
+}
+exports.Repositories = Repositories;

package/lib/converters/azure-repos/users.d.ts

@@ -0,0 +1,7 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { AzureReposConverter } from './common';
+export declare class Users extends AzureReposConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+}

package/lib/converters/azure-repos/users.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Users = void 0;
+const common_1 = require("./common");
+const models_1 = require("./models");
+class Users extends common_1.AzureReposConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = [
+            'vcs_Membership',
+            'vcs_User',
+        ];
+    }
+    async convert(record, ctx) {
+        const source = this.streamName.source;
+        const userItem = record.record.data;
+        const res = [];
+        const organizationName = this.getOrganizationFromUrl(userItem._links.self.href);
+        const organization = { uid: organizationName, source };
+        const type = {
+            category: models_1.UserTypeCategory.User,
+            detail: userItem.subjectKind,
+        };
+        res.push({
+            model: 'vcs_Membership',
+            record: {
+                organization,
+                user: { uid: userItem.principalName, source },
+            },
+        });
+        res.push({
+            model: 'vcs_User',
+            record: {
+                uid: userItem.principalName,
+                name: userItem.displayName,
+                type,
+                htmlUrl: userItem.url,
+                source,
+            },
+        });
+        return res;
+    }
+}
+exports.Users = Users;

package/lib/converters/bitbucket/pipelines.js

@@ -54,7 +54,11 @@ class Pipelines extends common_1.BitbucketConverter {
                 model: 'cicd_BuildCommitAssociation',
                 record: {
                     build: { uid: pipeline.uuid, pipeline: pipelineKey },
-                    commit: { repository: projectKey, sha: pipeline.target.commit.hash },
+                    commit: {
+                        repository: projectKey,
+                        sha: pipeline.target.commit.hash,
+                        uid: pipeline.target.commit.hash,
+                    },
                 },
             });
         }

package/lib/converters/buildkite/builds.js

@@ -77,7 +77,7 @@ class Builds extends common_1.BuildkiteConverter {
                     model: 'cicd_BuildCommitAssociation',
                     record: {
                         build: buildKey,
-                        commit: { repository: repoKey, sha: build.commit },
+                        commit: { repository: repoKey, sha: build.commit, uid: build.commit },
                     },
                 });
             }

package/lib/converters/{gitlab/common.d.ts → common/gitlab.d.ts}

@@ -13,11 +13,16 @@ interface RepositoryKey {
     uid: string;
     organization: OrgKey;
 }
+interface BuildKey {
+    uid: string;
+    pipeline: Omit<RepositoryKey, 'name'>;
+}
 /** Common functions shares across GitLab converters */
 export declare class GitlabCommon {
     static readonly MAX_DESCRIPTION_LENGTH = 1000;
     static tms_ProjectBoard_with_TaskBoard(projectKey: ProjectKey, name: string, description: string | null, createdAt: string | null | undefined, updatedAt: string | null | undefined): DestinationRecord[];
     static parseRepositoryKey(webUrl: string | undefined, source: string, startIndex?: number): undefined | RepositoryKey;
+    static createBuildKey(pipelineId: any, repository: RepositoryKey): BuildKey;
     static convertBuildStatus(status?: string): CategoryRef;
 }
 /** GitLab converter base */

package/lib/converters/{gitlab/common.js → common/gitlab.js}

@@ -48,6 +48,15 @@ class GitlabCommon {
             organization: { uid: organization === null || organization === void 0 ? void 0 : organization.toLowerCase(), source },
         };
     }
+    static createBuildKey(pipelineId, repository) {
+        return {
+            uid: String(pipelineId),
+            pipeline: {
+                organization: repository.organization,
+                uid: repository.name,
+            },
+        };
+    }
     // GitLab defined status for:
     // >> pipelines (aka builds): created, waiting_for_resource, preparing, pending,
     //    running, success, failed, canceled, skipped, manual, scheduled

package/lib/converters/github/assignees.js

@@ -10,7 +10,10 @@ class Assignees extends common_1.GitHubConverter {
     async convert(record, ctx) {
         const source = this.streamName.source;
         const user = record.record.data;
-        return [common_1.GitHubCommon.tms_User(user, source)];
+        const res = common_1.GitHubCommon.tms_User(user, source);
+        if (res)
+            return [res];
+        return [];
     }
 }
 exports.Assignees = Assignees;

package/lib/converters/github/common.d.ts

@@ -11,7 +11,7 @@ export declare class GitHubCommon {
     };
     static vcs_User(user: Dictionary<any>, source: string): DestinationRecord;
     static vcs_Membership(userUid: string, org: string, source: string): DestinationRecord;
-    static tms_User(user: Dictionary<any>, source: string): DestinationRecord;
+    static tms_User(user: Dictionary<any> | undefined, source: string): DestinationRecord | undefined;
     static tms_ProjectBoard_with_TaskBoard(projectKey: ProjectKey, name: string, description: string | null, createdAt: string | null | undefined, updatedAt: string | null | undefined): DestinationRecord[];
     static parseRepositoryKey(repository: string, source: string): undefined | RepositoryKey;
     static parsePRnumber(pull_request_url: string): number;