airbyte-faros-destination 0.1.62 → 0.1.65

package/README.md

@@ -6,6 +6,8 @@ Each source stream is handled by an appropriate [Converter](https://github.com/f
 
 Any additional source streams can be handled by providing a JSONata expression for the built-in [JSONataConverter](https://github.com/faros-ai/airbyte-connectors/tree/main/destinations/airbyte-faros-destination/src/converters/jsonata.ts) or by implementing converters for the streams ([read more below](#adding-support-for-additional-sources)).
 
+![](img/source_destination.png)
+
 ## Usage
 
 ### Run from Airbyte

package/lib/converters/azurepipeline/builds.d.ts

@@ -0,0 +1,8 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { AzurePipelineConverter } from './common';
+export declare class Builds extends AzurePipelineConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    private seenRepositories;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+}

package/lib/converters/azurepipeline/builds.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Builds = void 0;
+const faros_feeds_sdk_1 = require("faros-feeds-sdk");
+const common_1 = require("./common");
+class Builds extends common_1.AzurePipelineConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = [
+            'cicd_Artifact',
+            'cicd_Build',
+            'cicd_BuildStep',
+            'cicd_Repository',
+        ];
+        this.seenRepositories = new Set();
+    }
+    async convert(record, ctx) {
+        var _a;
+        const source = this.streamName.source;
+        const build = record.record.data;
+        const uid = String(build.id);
+        const organizationName = this.getOrganizationFromUrl(build.url);
+        const organization = { uid: organizationName, source };
+        const pipeline = {
+            uid: String((_a = build.definition) === null || _a === void 0 ? void 0 : _a.id),
+            organization: { uid: organizationName, source },
+        };
+        const buildUid = { uid, pipeline };
+        const createdAt = faros_feeds_sdk_1.Utils.toDate(build.queueTime);
+        const startedAt = faros_feeds_sdk_1.Utils.toDate(build.startTime);
+        const endedAt = faros_feeds_sdk_1.Utils.toDate(build.finishTime);
+        const status = this.convertBuildState(build.result);
+        const res = [];
+        const number = Number(build.buildNumber.replace('.', ''));
+        res.push({
+            model: 'cicd_Build',
+            record: {
+                uid,
+                name: build.buildNumber,
+                number,
+                createdAt,
+                startedAt,
+                endedAt,
+                status,
+                url: build.url,
+                pipeline,
+            },
+        });
+        // TODO
+        const repo = build.repository;
+        if (!this.seenRepositories.has(repo.id)) {
+            this.seenRepositories.add(repo.id);
+            res.push({
+                model: 'cicd_Repository',
+                record: {
+                    uid: repo.id,
+                    name: repo.name,
+                    description: null,
+                    url: this.getRepoUrl(repo),
+                    organization,
+                },
+            });
+        }
+        for (const job of build.jobs) {
+            const jobCreatedAt = faros_feeds_sdk_1.Utils.toDate(job.startTime);
+            const jobStartedAt = faros_feeds_sdk_1.Utils.toDate(job.startTime);
+            const jobEndedAt = faros_feeds_sdk_1.Utils.toDate(job.finishTime);
+            const jobStatus = this.convertBuildStepState(job.result);
+            const jobType = this.convertBuildStepType(job.type);
+            res.push({
+                model: 'cicd_BuildStep',
+                record: {
+                    uid: String(job.id),
+                    name: job.name,
+                    command: job.name,
+                    type: jobType,
+                    createdAt: jobCreatedAt,
+                    startedAt: jobStartedAt,
+                    endedAt: jobEndedAt,
+                    status: jobStatus,
+                    url: job.url,
+                    build: buildUid,
+                },
+            });
+        }
+        for (const artifact of build.artifacts) {
+            const artifactCreatedAt = faros_feeds_sdk_1.Utils.toDate(build.startTime);
+            const tags = [];
+            for (const [key, value] of Object.entries(artifact.resource.properties)) {
+                tags.push({ name: key, value: String(value) });
+            }
+            res.push({
+                model: 'cicd_Artifact',
+                record: {
+                    uid: String(artifact.id),
+                    name: artifact.name,
+                    url: artifact.resource.url,
+                    type: artifact.resource.type,
+                    createdAt: artifactCreatedAt,
+                    tags,
+                    build: buildUid,
+                    repository: { uid: repo.id, organization },
+                },
+            });
+        }
+        return res;
+    }
+}
+exports.Builds = Builds;

package/lib/converters/azurepipeline/common.d.ts

@@ -0,0 +1,32 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { Converter, StreamContext } from '../converter';
+import { BuildTimeline, DeploymentStatus, JobState, Repository, Timestamps } from './models';
+export declare type ApplicationMapping = Record<string, {
+    name: string;
+    platform?: string;
+}>;
+interface AzurePipelineConfig {
+    application_mapping?: ApplicationMapping;
+}
+/** Azurepipeline converter base */
+export declare abstract class AzurePipelineConverter extends Converter {
+    source: string;
+    /** Almost every Azurepipeline record have id property */
+    id(record: AirbyteRecord): any;
+    getOrganizationFromUrl(url: string): string;
+    protected azurepipleConfig(ctx: StreamContext): AzurePipelineConfig;
+    protected applicationMapping(ctx: StreamContext): ApplicationMapping;
+    convertBuildState(state: string | undefined): {
+        category: string;
+        detail: string;
+    };
+    getRepoUrl(repo: Repository): string | undefined;
+    convertBuildStepTime(buildStep: BuildTimeline): Timestamps;
+    convertBuildStepState(state: string | undefined): {
+        category: string;
+        detail: string;
+    };
+    convertBuildStepType(type: string): JobState;
+    convertDeploymentStatus(result: string): DeploymentStatus;
+}
+export {};

package/lib/converters/azurepipeline/common.js

@@ -0,0 +1,119 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AzurePipelineConverter = void 0;
+const faros_feeds_sdk_1 = require("faros-feeds-sdk");
+const converter_1 = require("../converter");
+const models_1 = require("./models");
+/** Azurepipeline converter base */
+class AzurePipelineConverter extends converter_1.Converter {
+    constructor() {
+        super(...arguments);
+        this.source = 'AzurePipeline';
+    }
+    /** Almost every Azurepipeline record have id property */
+    id(record) {
+        var _a, _b;
+        return (_b = (_a = record === null || record === void 0 ? void 0 : record.record) === null || _a === void 0 ? void 0 : _a.data) === null || _b === void 0 ? void 0 : _b.id;
+    }
+    getOrganizationFromUrl(url) {
+        return url.split('/')[3];
+    }
+    azurepipleConfig(ctx) {
+        var _a, _b;
+        return (_b = (_a = ctx.config) === null || _a === void 0 ? void 0 : _a.source_specific_configs) === null || _b === void 0 ? void 0 : _b.azurepipeline;
+    }
+    applicationMapping(ctx) {
+        var _a, _b;
+        return (_b = (_a = this.azurepipleConfig(ctx)) === null || _a === void 0 ? void 0 : _a.application_mapping) !== null && _b !== void 0 ? _b : {};
+    }
+    convertBuildState(state) {
+        if (!state) {
+            return { category: models_1.BuildStateCategory.Unknown, detail: 'undefined' };
+        }
+        const detail = state.toLowerCase();
+        // Read more on Azure pipeline build result:
+        // https://docs.microsoft.com/en-us/rest/api/azure/devops/build/builds/list?view=azure-devops-rest-6.0#buildresult
+        switch (detail) {
+            case 'canceled':
+                return { category: models_1.BuildStateCategory.Canceled, detail };
+            case 'failed':
+                return { category: models_1.BuildStateCategory.Failed, detail };
+            case 'succeeded':
+            case 'partiallySucceeded':
+                return { category: models_1.BuildStateCategory.Success, detail };
+            case 'running':
+                return { category: models_1.BuildStateCategory.Running, detail };
+            default:
+                return { category: models_1.BuildStateCategory.Custom, detail };
+        }
+    }
+    getRepoUrl(repo) {
+        switch (repo.type) {
+            case 'Bitbucket':
+                return `https://bitbucket.org/${repo.id}`;
+            case 'GitHub':
+            case 'GitHubEnterprise':
+                return `https://github.com/${repo.id}`;
+            case 'GitLab':
+                return `https://gitlab.com/${repo.id}`;
+            case 'TfsGit':
+                return repo.url;
+            default:
+                return repo.url;
+        }
+    }
+    convertBuildStepTime(buildStep) {
+        //const type = buildStep.type;
+        const result = {
+            createdAt: faros_feeds_sdk_1.Utils.toDate(buildStep.startTime),
+            startedAt: faros_feeds_sdk_1.Utils.toDate(buildStep.startTime),
+            endedAt: faros_feeds_sdk_1.Utils.toDate(buildStep.finishTime),
+        };
+        return result;
+    }
+    convertBuildStepState(state) {
+        if (!state) {
+            return { category: models_1.BuildStateCategory.Unknown, detail: 'undefined' };
+        }
+        const detail = state.toLowerCase();
+        //https://docs.microsoft.com/en-us/rest/api/azure/devops/build/timeline/get?view=azure-devops-rest-6.0#taskresult
+        switch (detail) {
+            case 'abandoned':
+            case 'canceled':
+                return { category: models_1.BuildStateCategory.Canceled, detail };
+            case 'failed':
+            case 'skipped':
+                return { category: models_1.BuildStateCategory.Failed, detail };
+            case 'succeeded':
+                return { category: models_1.BuildStateCategory.Success, detail };
+            case 'succeededWithIssues':
+                return { category: models_1.BuildStateCategory.Queued, detail };
+            default:
+                return { category: models_1.BuildStateCategory.Custom, detail };
+        }
+    }
+    // TODO
+    convertBuildStepType(type) {
+        if (!type) {
+            return { category: models_1.JobCategory.Custom, detail: 'undefined' };
+        }
+        return { category: models_1.JobCategory.Custom, detail: type };
+    }
+    convertDeploymentStatus(result) {
+        if (!result) {
+            return { category: models_1.DeploymentStatusCategory.Custom, detail: 'undefined' };
+        }
+        const detail = result;
+        switch (result) {
+            case 'canceled':
+                return { category: models_1.DeploymentStatusCategory.Canceled, detail };
+            case 'failed':
+                return { category: models_1.DeploymentStatusCategory.Failed, detail };
+            case 'succeeded':
+                return { category: models_1.DeploymentStatusCategory.Success, detail };
+            default:
+                return { category: models_1.DeploymentStatusCategory.Custom, detail };
+        }
+    }
+}
+exports.AzurePipelineConverter = AzurePipelineConverter;

package/lib/converters/azurepipeline/models.d.ts

@@ -0,0 +1,298 @@
+interface Href {
+    href: string;
+}
+interface PipelineLink {
+    self: Href;
+    web: Href;
+}
+interface RunLink {
+    self: Href;
+    web: Href;
+    pipeline: Href;
+}
+interface RunPipeline {
+    url: string;
+    id: number;
+    revision: number;
+    name: string;
+    folder: string;
+}
+interface Run {
+    _links: RunLink;
+    pipeline: RunPipeline;
+    state: string;
+    result: string;
+    createdDate: string;
+    finishedDate: string;
+    url: string;
+    id: number;
+    name: string;
+}
+export interface Pipeline {
+    id: number;
+    revision: number;
+    name?: string;
+    url?: string;
+    folder?: string;
+    _links: PipelineLink;
+    runs: Run[];
+}
+interface BuildLink {
+    self: Href;
+    web: Href;
+    sourceVersionDisplayUri: Href;
+    timeline: Href;
+    badge: Href;
+}
+interface BuildPlan {
+    planId: string;
+}
+interface BuildDefinitionProject {
+    id: string;
+    name: string;
+    description: string;
+    url: string;
+    state: string;
+    revision: number;
+    visibility: string;
+    lastUpdateTime: string;
+}
+interface BuildDefinition {
+    drafts: any[];
+    id: number;
+    name: string;
+    url: string;
+    uri: string;
+    path: string;
+    type: string;
+    queueStatus: string;
+    revision: number;
+    project: BuildDefinitionProject;
+}
+interface BuildQueuePool {
+    id: number;
+    name: string;
+    isHosted: boolean;
+}
+interface BuildQueue {
+    id: number;
+    name: string;
+    pool: BuildQueuePool;
+}
+interface BuildRequestLinkAvatar {
+    avatar: Href;
+}
+interface BuildRequestLink {
+    avatar: BuildRequestLinkAvatar;
+}
+interface BuildRequest {
+    displayName: string;
+    url: string;
+    id: string;
+    uniqueName: string;
+    imageUrl: string;
+    descriptor: string;
+    _link: BuildRequestLink;
+}
+interface BuildLog {
+    id: number;
+    type: string;
+    url: string;
+}
+export interface Repository {
+    id: string;
+    type: string;
+    name: string;
+    url: string;
+    clean?: string;
+    checkoutSubmodules: boolean;
+}
+interface ArtifactResource {
+    data: string;
+    downloadUrl: string;
+    properties: any;
+    type: string;
+    url: string;
+    _links: any;
+}
+interface BuildArtifact {
+    id: number;
+    name: string;
+    resource: ArtifactResource;
+    source: string;
+}
+interface BuildTimelineIssue {
+    type: string;
+    category?: any;
+    message: string;
+}
+export interface BuildTimeline {
+    previousAttempts: any[];
+    id: string;
+    parentId: string;
+    type: string;
+    name: string;
+    startTime?: any;
+    finishTime?: any;
+    currentOperation?: any;
+    percentComplete?: any;
+    state: string;
+    result: string;
+    resultCode?: any;
+    changeId: number;
+    lastModified: Date;
+    workerName?: any;
+    queueId: number;
+    order: number;
+    details?: any;
+    errorCount: number;
+    warningCount: number;
+    url?: any;
+    log?: any;
+    task?: any;
+    attempt: number;
+    identifier: string;
+    issues: BuildTimelineIssue[];
+}
+export interface Build {
+    _links: BuildLink;
+    properties: any;
+    tags: any[];
+    validationResults: any[];
+    plans: BuildPlan[];
+    triggerInfo: any;
+    id: number;
+    buildNumber: string;
+    status: string;
+    result: string;
+    queueTime: string;
+    startTime: string;
+    finishTime: string;
+    url: string;
+    definition: BuildDefinition;
+    buildNumberRevision: number;
+    project: BuildDefinitionProject;
+    uri: string;
+    sourceBranch: string;
+    sourceVersion: string;
+    queue: BuildQueue;
+    priority: string;
+    reason: string;
+    requestedFor: BuildRequest;
+    requestedBy: BuildRequest;
+    lastChangedDate: string;
+    lastChangedBy: BuildRequest;
+    orchestrationPlan: BuildPlan;
+    logs: BuildLog;
+    repository: Repository;
+    keepForever: boolean;
+    retainedByRelease: boolean;
+    triggeredByBuild: any;
+    artifacts: BuildArtifact[];
+    jobs: BuildTimeline[];
+}
+interface ReleaseByUser {
+    id: string;
+    displayName: string;
+    uniqueName: string;
+    url: string;
+    imageUrl?: string;
+}
+interface ReleaseLink {
+    self: Href;
+    web: Href;
+}
+interface ReleaseProjectReference {
+    id: string;
+    name: string;
+}
+export interface Release {
+    id: number;
+    name: string;
+    status: string;
+    createdOn: string;
+    modifiedOn?: string;
+    modifiedBy?: ReleaseByUser;
+    createdBy: ReleaseByUser;
+    variables: any;
+    variableGroups: [any];
+    description?: string;
+    reason?: string;
+    releaseNameFormat?: string;
+    keepForever: boolean;
+    definitionSnapshotRevision: number;
+    logsContainerUrl: string;
+    url: string;
+    _links: ReleaseLink;
+    tags: [any];
+    projectReference: ReleaseProjectReference;
+    properties: any;
+}
+export declare enum RepoSource {
+    BITBUCKET = "Bitbucket",
+    GITHUB = "GitHub",
+    GITLAB = "GitLab",
+    VCS = "VCS"
+}
+export declare enum BuildStateCategory {
+    Unknown = "Unknown",
+    Canceled = "Canceled",
+    Failed = "Failed",
+    Success = "Success",
+    Running = "Running",
+    Queued = "Queued",
+    Custom = "Custom"
+}
+export declare enum JobCategory {
+    Custom = "Custom",
+    Script = "Script",
+    Manual = "Manual"
+}
+export interface Repo {
+    readonly provider: Provider;
+    readonly url: string;
+}
+export interface RepoExtract {
+    readonly org: string;
+    readonly name: string;
+}
+export interface Provider {
+    readonly name: RepoSource;
+}
+export interface Timestamps {
+    createdAt?: Date;
+    startedAt?: Date;
+    endedAt?: Date;
+}
+export interface JobState {
+    category: string;
+    detail: string;
+}
+export interface Tag {
+    readonly name: string;
+    readonly value: string;
+}
+export declare enum UserTypeCategory {
+    BOT = "Bot",
+    ORGANIZATION = "Organization",
+    USER = "User",
+    CUSTOM = "Custom"
+}
+export declare enum DeploymentStatusCategory {
+    Canceled = "Canceled",
+    Custom = "Custom",
+    Failed = "Failed",
+    Queued = "Queued",
+    Running = "Running",
+    RolledBack = "RolledBack",
+    Success = "Success"
+}
+export interface DeploymentStatus {
+    category: DeploymentStatusCategory;
+    detail: string;
+}
+export interface ComputeApplication {
+    name: string;
+    platform: string;
+}
+export {};

package/lib/converters/azurepipeline/models.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DeploymentStatusCategory = exports.UserTypeCategory = exports.JobCategory = exports.BuildStateCategory = exports.RepoSource = void 0;
+var RepoSource;
+(function (RepoSource) {
+    RepoSource["BITBUCKET"] = "Bitbucket";
+    RepoSource["GITHUB"] = "GitHub";
+    RepoSource["GITLAB"] = "GitLab";
+    RepoSource["VCS"] = "VCS";
+})(RepoSource = exports.RepoSource || (exports.RepoSource = {}));
+var BuildStateCategory;
+(function (BuildStateCategory) {
+    BuildStateCategory["Unknown"] = "Unknown";
+    BuildStateCategory["Canceled"] = "Canceled";
+    BuildStateCategory["Failed"] = "Failed";
+    BuildStateCategory["Success"] = "Success";
+    BuildStateCategory["Running"] = "Running";
+    BuildStateCategory["Queued"] = "Queued";
+    BuildStateCategory["Custom"] = "Custom";
+})(BuildStateCategory = exports.BuildStateCategory || (exports.BuildStateCategory = {}));
+var JobCategory;
+(function (JobCategory) {
+    JobCategory["Custom"] = "Custom";
+    JobCategory["Script"] = "Script";
+    JobCategory["Manual"] = "Manual";
+})(JobCategory = exports.JobCategory || (exports.JobCategory = {}));
+var UserTypeCategory;
+(function (UserTypeCategory) {
+    UserTypeCategory["BOT"] = "Bot";
+    UserTypeCategory["ORGANIZATION"] = "Organization";
+    UserTypeCategory["USER"] = "User";
+    UserTypeCategory["CUSTOM"] = "Custom";
+})(UserTypeCategory = exports.UserTypeCategory || (exports.UserTypeCategory = {}));
+var DeploymentStatusCategory;
+(function (DeploymentStatusCategory) {
+    DeploymentStatusCategory["Canceled"] = "Canceled";
+    DeploymentStatusCategory["Custom"] = "Custom";
+    DeploymentStatusCategory["Failed"] = "Failed";
+    DeploymentStatusCategory["Queued"] = "Queued";
+    DeploymentStatusCategory["Running"] = "Running";
+    DeploymentStatusCategory["RolledBack"] = "RolledBack";
+    DeploymentStatusCategory["Success"] = "Success";
+})(DeploymentStatusCategory = exports.DeploymentStatusCategory || (exports.DeploymentStatusCategory = {}));

package/lib/converters/azurepipeline/pipelines.d.ts

@@ -0,0 +1,8 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { AzurePipelineConverter } from './common';
+export declare class Pipelines extends AzurePipelineConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    private seenOrganizations;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+}

package/lib/converters/azurepipeline/pipelines.js

@@ -0,0 +1,68 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Pipelines = void 0;
+const lib_1 = require("faros-feeds-sdk/lib");
+const common_1 = require("./common");
+class Pipelines extends common_1.AzurePipelineConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = [
+            'cicd_Deployment',
+            'cicd_Organization',
+            'cicd_Pipeline',
+        ];
+        this.seenOrganizations = new Set();
+    }
+    async convert(record, ctx) {
+        var _a;
+        const source = this.streamName.source;
+        const pipeline = record.record.data;
+        const res = [];
+        const organizationName = this.getOrganizationFromUrl(pipeline.url);
+        const organization = { uid: organizationName, source };
+        if (!this.seenOrganizations.has(organizationName)) {
+            this.seenOrganizations.add(organizationName);
+            res.push({
+                model: 'cicd_Organization',
+                record: {
+                    uid: organizationName,
+                    name: organizationName,
+                    source,
+                },
+            });
+        }
+        for (const runItem of pipeline.runs) {
+            const applicationMapping = this.applicationMapping(ctx);
+            const application = (_a = (applicationMapping && applicationMapping[runItem.name])) !== null && _a !== void 0 ? _a : null;
+            const startedAt = lib_1.Utils.toDate(runItem.createdDate);
+            const endedAt = lib_1.Utils.toDate(runItem.finishedDate);
+            const status = this.convertDeploymentStatus(runItem.result);
+            res.push({
+                model: 'cicd_Deployment',
+                record: {
+                    uid: String(runItem.id),
+                    application,
+                    build: {
+                        uid: String(runItem.id),
+                        pipeline: { uid: String(pipeline.id), organization },
+                    },
+                    startedAt,
+                    endedAt,
+                    status,
+                    source,
+                },
+            });
+        }
+        res.push({
+            model: 'cicd_Pipeline',
+            record: {
+                uid: String(pipeline.id),
+                name: pipeline.name,
+                url: pipeline.url,
+                organization,
+            },
+        });
+        return res;
+    }
+}
+exports.Pipelines = Pipelines;