airbyte-faros-destination 0.1.58 → 0.1.61

package/README.md

@@ -57,8 +57,8 @@ import {AirbyteRecord} from 'faros-airbyte-cdk';
 import {Converter, DestinationModel, DestinationRecord,FarosDestinationRunner,StreamContext} from 'airbyte-faros-destination'
 
 class Builds extends Converter {
-  source: string = 'CustomSource'
-  destinationModels: ReadonlyArray<DestinationModel> = ['cicd_Build'];
+  source = 'CustomSource'
+  destinationModels = ['cicd_Build'];
 
   id(record: AirbyteRecord): string {
     return record.record.data.id;
@@ -66,7 +66,7 @@ class Builds extends Converter {
 
   async convert(
     record: AirbyteRecord,
-    _ctx: StreamContext
+    ctx: StreamContext
   ): Promise<ReadonlyArray<DestinationRecord>> {
     const build = record.record.data
     return [
@@ -83,12 +83,13 @@ class Builds extends Converter {
     ];
   }
 }
+
 class Pipelines extends Converter {
   // similar to the Builds in the example above
   ...
 }
 
-// main entry point
+// Main entry point
 export function mainCommand(): Command {
   const destinationRunner = new FarosDestinationRunner();
 
@@ -97,7 +98,6 @@ export function mainCommand(): Command {
     new Builds(),
     new Pipelines()
   );
-
   return destinationRunner.program;
 }
 ```

package/lib/converters/azureactivedirectory/common.d.ts

@@ -0,0 +1,8 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { Converter } from '../converter';
+/** AzureActiveDirectory converter base */
+export declare abstract class AzureActiveDirectoryConverter extends Converter {
+    source: string;
+    /** Almost every Azure Active Directory record have id property */
+    id(record: AirbyteRecord): any;
+}

package/lib/converters/azureactivedirectory/common.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AzureActiveDirectoryConverter = void 0;
+const converter_1 = require("../converter");
+/** AzureActiveDirectory converter base */
+class AzureActiveDirectoryConverter extends converter_1.Converter {
+    constructor() {
+        super(...arguments);
+        this.source = 'AzureActiveDirectory';
+    }
+    /** Almost every Azure Active Directory record have id property */
+    id(record) {
+        var _a, _b;
+        return (_b = (_a = record === null || record === void 0 ? void 0 : record.record) === null || _a === void 0 ? void 0 : _a.data) === null || _b === void 0 ? void 0 : _b.id;
+    }
+}
+exports.AzureActiveDirectoryConverter = AzureActiveDirectoryConverter;

package/lib/converters/azureactivedirectory/groups.d.ts

@@ -0,0 +1,7 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { AzureActiveDirectoryConverter } from './common';
+export declare class Groups extends AzureActiveDirectoryConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+}

package/lib/converters/azureactivedirectory/groups.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Groups = void 0;
+const common_1 = require("./common");
+class Groups extends common_1.AzureActiveDirectoryConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = [
+            'org_Team',
+            'org_TeamMembership',
+        ];
+    }
+    async convert(record, ctx) {
+        var _a;
+        const source = this.streamName.source;
+        const group = record.record.data;
+        const res = [];
+        const uid = group.id;
+        const lead = group.owners.length >= 1
+            ? {
+                uid: group.owners[0],
+                source,
+            }
+            : undefined;
+        res.push({
+            model: 'org_Team',
+            record: {
+                uid,
+                name: group.displayName,
+                description: group.description,
+                lead,
+                parentTeam: { uid: 'all_teams' },
+                teamChain: ['all_teams'],
+                source,
+            },
+        });
+        for (const user of (_a = group.members) !== null && _a !== void 0 ? _a : []) {
+            res.push({
+                model: 'org_TeamMembership',
+                record: {
+                    team: { uid },
+                    member: { uid: user },
+                },
+            });
+        }
+        return res;
+    }
+}
+exports.Groups = Groups;

package/lib/converters/azureactivedirectory/models.d.ts

@@ -0,0 +1,61 @@
+interface UserIdentity {
+    signInType?: string;
+    issuer: string;
+    issuerAssignedId?: string;
+}
+export interface User {
+    businessPhones: string[];
+    displayName: string;
+    givenName: string;
+    jobTitle: string;
+    mail: string;
+    mobilePhone: string;
+    officeLocation: string;
+    preferredLanguage?: string;
+    surname: string;
+    userPrincipalName: string;
+    id: string;
+    department?: string;
+    postalCode?: string;
+    manager?: string;
+    createdDateTime?: string;
+    streetAddress?: string;
+    identities: UserIdentity[];
+}
+export interface Group {
+    id: string;
+    deletedDateTime?: string;
+    classification?: string;
+    createdDateTime: string;
+    creationOptions: any[];
+    description: string;
+    displayName: string;
+    expirationDateTime?: string;
+    groupTypes: string[];
+    isAssignableToRole?: string;
+    mail?: string;
+    mailEnabled: boolean;
+    mailNickname: string;
+    membershipRule?: string;
+    membershipRuleProcessingState?: string;
+    onPremisesDomainName?: string;
+    onPremisesLastSyncDateTime?: string;
+    onPremisesNetBiosName?: string;
+    onPremisesSamAccountName?: string;
+    onPremisesSecurityIdentifier?: string;
+    onPremisesSyncEnabled?: string;
+    preferredDataLocation?: string;
+    preferredLanguage?: string;
+    proxyAddresses: string[];
+    renewedDateTime: Date;
+    resourceBehaviorOptions: string[];
+    resourceProvisioningOptions: string[];
+    securityEnabled: boolean;
+    securityIdentifier: string;
+    theme?: string;
+    visibility?: string;
+    onPremisesProvisioningErrors: string[];
+    members: string[];
+    owners: string[];
+}
+export {};

package/lib/converters/azureactivedirectory/models.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/lib/converters/azureactivedirectory/users.d.ts

@@ -0,0 +1,8 @@
+import { AirbyteRecord } from 'faros-airbyte-cdk';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
+import { AzureActiveDirectoryConverter } from './common';
+export declare class Users extends AzureActiveDirectoryConverter {
+    readonly destinationModels: ReadonlyArray<DestinationModel>;
+    private seenDepartments;
+    convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+}

package/lib/converters/azureactivedirectory/users.js

@@ -0,0 +1,83 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Users = void 0;
+const faros_feeds_sdk_1 = require("faros-feeds-sdk");
+const common_1 = require("./common");
+class Users extends common_1.AzureActiveDirectoryConverter {
+    constructor() {
+        super(...arguments);
+        this.destinationModels = [
+            'geo_Address',
+            'geo_Location',
+            'identity_Identity',
+            'org_Department',
+            'org_Employee',
+        ];
+        this.seenDepartments = new Set();
+    }
+    async convert(record, ctx) {
+        const source = this.streamName.source;
+        const user = record.record.data;
+        const joinedAt = faros_feeds_sdk_1.Utils.toDate(user.createdDateTime);
+        const manager = user.manager ? { uid: user.manager, source } : undefined;
+        const uid = user.id;
+        const res = [];
+        if (user.department && !this.seenDepartments.has(user.department)) {
+            this.seenDepartments.add(user.department);
+            res.push({
+                model: 'org_Department',
+                record: {
+                    uid: user.department,
+                    name: user.department,
+                },
+            });
+        }
+        res.push({
+            model: 'identity_Identity',
+            record: {
+                uid,
+                fullName: `${user.givenName} ${user.surname}`,
+                lastName: user.surname,
+                primaryEmail: user.mail,
+                emails: [user.mail],
+            },
+        });
+        const location = user.streetAddress
+            ? { uid: user.streetAddress, source }
+            : undefined;
+        if (user.streetAddress) {
+            res.push({
+                model: 'geo_Address',
+                record: {
+                    uid: user.streetAddress,
+                    fullAddress: user.streetAddress,
+                    street: user.streetAddress,
+                    postalCode: user.postalCode,
+                },
+            });
+            const geo_Location = {
+                uid: user.streetAddress,
+                name: user.streetAddress,
+                raw: user.streetAddress,
+                address: { uid: user.streetAddress },
+            };
+            res.push({ model: 'geo_Location', record: geo_Location });
+        }
+        res.push({
+            model: 'org_Employee',
+            record: {
+                uid,
+                title: user.displayName,
+                level: 0,
+                joinedAt,
+                department: { uid: user.department },
+                identity: { uid, source },
+                manager,
+                location,
+                source,
+            },
+        });
+        return res;
+    }
+}
+exports.Users = Users;

package/lib/converters/bitbucket/common.d.ts

@@ -12,10 +12,15 @@ export interface CategoryRef {
     readonly category: string;
     readonly detail: string;
 }
+export interface ProjectKey {
+    uid: string;
+    source: string;
+}
 /** Common functions shares across Bitbucket converters */
 export declare class BitbucketCommon {
     static MAX_DESCRIPTION_LENGTH: number;
     static vcsUser(user: User, source: string): DestinationRecord | undefined;
+    static tms_ProjectBoard_with_TaskBoard(projectKey: ProjectKey, name: string, description: string | null, createdAt: string | null | undefined, updatedAt: string | null | undefined): DestinationRecord[];
 }
 /** Bitbucket converter base */
 export declare abstract class BitbucketConverter extends Converter {

package/lib/converters/bitbucket/common.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BitbucketConverter = exports.BitbucketCommon = void 0;
+const faros_feeds_sdk_1 = require("faros-feeds-sdk");
 const converter_1 = require("../converter");
 var UserTypeCategory;
 (function (UserTypeCategory) {
@@ -29,6 +30,34 @@ class BitbucketCommon {
             },
         };
     }
+    static tms_ProjectBoard_with_TaskBoard(projectKey, name, description, createdAt, updatedAt) {
+        return [
+            {
+                model: 'tms_Project',
+                record: {
+                    ...projectKey,
+                    name: name,
+                    description: description === null || description === void 0 ? void 0 : description.substring(0, BitbucketCommon.MAX_DESCRIPTION_LENGTH),
+                    createdAt: faros_feeds_sdk_1.Utils.toDate(createdAt),
+                    updatedAt: faros_feeds_sdk_1.Utils.toDate(updatedAt),
+                },
+            },
+            {
+                model: 'tms_TaskBoard',
+                record: {
+                    ...projectKey,
+                    name,
+                },
+            },
+            {
+                model: 'tms_TaskBoardProjectRelationship',
+                record: {
+                    board: projectKey,
+                    project: projectKey,
+                },
+            },
+        ];
+    }
 }
 exports.BitbucketCommon = BitbucketCommon;
 // max length for free-form description text field

package/lib/converters/bitbucket/issues.js

@@ -24,6 +24,8 @@ class Issues extends common_1.BitbucketConverter {
         this.destinationModels = [
             'tms_Task',
             'tms_TaskAssignment',
+            'tms_TaskProjectRelationship',
+            'tms_TaskBoardRelationship',
             'tms_User',
         ];
         this.workspacesStream = new converter_1.StreamName('bitbucket', 'workspace');
@@ -52,6 +54,21 @@ class Issues extends common_1.BitbucketConverter {
                 },
             });
         }
+        const projectRef = { uid: String(issue.repository.name), source };
+        res.push({
+            model: 'tms_TaskProjectRelationship',
+            record: {
+                task: taskRef,
+                project: projectRef,
+            },
+        });
+        res.push({
+            model: 'tms_TaskBoardRelationship',
+            record: {
+                task: taskRef,
+                board: projectRef,
+            },
+        });
         res.push({
             model: 'tms_Task',
             record: {

package/lib/converters/bitbucket/repositories.d.ts

@@ -1,9 +1,7 @@
 import { AirbyteRecord } from 'faros-airbyte-cdk';
-import { DestinationModel, DestinationRecord, StreamContext, StreamName } from '../converter';
+import { DestinationModel, DestinationRecord, StreamContext } from '../converter';
 import { BitbucketConverter } from './common';
 export declare class Repositories extends BitbucketConverter {
     readonly destinationModels: ReadonlyArray<DestinationModel>;
-    private readonly workspacesStream;
-    get dependencies(): ReadonlyArray<StreamName>;
     convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
 }

package/lib/converters/bitbucket/repositories.js

@@ -1,41 +1,55 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Repositories = void 0;
-const converter_1 = require("../converter");
+const faros_feeds_sdk_1 = require("faros-feeds-sdk");
 const common_1 = require("./common");
 class Repositories extends common_1.BitbucketConverter {
     constructor() {
         super(...arguments);
         this.destinationModels = [
             'cicd_Repository',
+            'vcs_Repository',
+            'tms_Project',
+            'tms_TaskBoard',
+            'tms_TaskBoardProjectRelationship',
         ];
-        this.workspacesStream = new converter_1.StreamName('bitbucket', 'workspace');
-    }
-    get dependencies() {
-        return [this.workspacesStream];
     }
     async convert(record, ctx) {
-        var _a, _b;
+        var _a, _b, _c, _d, _e;
         const source = this.streamName.source;
         const repository = record.record.data;
-        const workspacesStream = this.workspacesStream.asString;
-        const workspacesRecord = ctx.get(workspacesStream, repository.workspace.uuid);
-        const workspace = (_a = workspacesRecord === null || workspacesRecord === void 0 ? void 0 : workspacesRecord.record) === null || _a === void 0 ? void 0 : _a.data;
-        if (!workspace) {
-            return [];
-        }
-        return [
-            {
-                model: 'cicd_Repository',
-                record: {
-                    uid: repository.slug.toLowerCase(),
-                    name: repository.name,
-                    description: (_b = repository.description) === null || _b === void 0 ? void 0 : _b.substring(0, common_1.BitbucketCommon.MAX_DESCRIPTION_LENGTH),
-                    url: repository.links.htmlUrl,
-                    organization: { uid: workspace === null || workspace === void 0 ? void 0 : workspace.uuid, source },
-                },
+        const workspace = repository.workspace;
+        const res = [];
+        const description = (_a = repository.description) === null || _a === void 0 ? void 0 : _a.substring(0, common_1.BitbucketCommon.MAX_DESCRIPTION_LENGTH);
+        // Create a TMS Project/Board per repo that we sync
+        res.push(...common_1.BitbucketCommon.tms_ProjectBoard_with_TaskBoard({ uid: repository.name, source }, repository.name, repository.description, repository.createdOn, repository.updatedOn));
+        res.push({
+            model: 'cicd_Repository',
+            record: {
+                uid: repository.slug.toLowerCase(),
+                name: repository.fullName,
+                description,
+                url: (_b = repository === null || repository === void 0 ? void 0 : repository.links) === null || _b === void 0 ? void 0 : _b.htmlUrl,
+                organization: { uid: workspace.uuid, source },
             },
-        ];
+        });
+        res.push({
+            model: 'vcs_Repository',
+            record: {
+                name: repository.slug.toLowerCase(),
+                fullName: repository.fullName,
+                description,
+                private: repository.isPrivate,
+                language: (_c = repository.language) !== null && _c !== void 0 ? _c : null,
+                size: BigInt(repository.size),
+                htmlUrl: (_d = repository === null || repository === void 0 ? void 0 : repository.links) === null || _d === void 0 ? void 0 : _d.htmlUrl,
+                createdAt: faros_feeds_sdk_1.Utils.toDate(repository.createdOn),
+                updatedAt: faros_feeds_sdk_1.Utils.toDate(repository.updatedOn),
+                mainBranch: (_e = repository.mainBranch) === null || _e === void 0 ? void 0 : _e.name,
+                organization: { uid: workspace.uuid, source },
+            },
+        });
+        return res;
     }
 }
 exports.Repositories = Repositories;

package/lib/converters/converter-registry.d.ts

@@ -19,7 +19,7 @@ export declare class ConverterRegistry {
      */
     static getConverter(streamName: StreamName, onLoadError?: (err: Error) => void): Converter | undefined;
     /**
-     * Add a convertor to the registory.
+     * Add a convertor to the registry.
      */
     static addConverter(converter: Converter): void;
 }

package/lib/converters/converter-registry.js

@@ -53,7 +53,7 @@ class ConverterRegistry {
         }
     }
     /**
-     * Add a convertor to the registory.
+     * Add a convertor to the registry.
      */
     static addConverter(converter) {
         const name = converter.streamName.asString;

package/lib/converters/converter.d.ts

@@ -2,19 +2,28 @@ import { AirbyteConfig, AirbyteLogger, AirbyteRecord } from 'faros-airbyte-cdk';
 import { FarosClient } from 'faros-feeds-sdk';
 import { Dictionary } from 'ts-essentials';
 /** Airbyte -> Faros record converter */
-export declare abstract class Converter {
+export declare abstract class ConverterTyped<R> {
     private stream;
     /** Name of the source system that records were fetched from (e.g. GitHub) **/
     abstract readonly source: string;
     /** Input stream supported by converter */
     get streamName(): StreamName;
+    /** Dependencies on other streams (if any).
+     * !!! USE WITH CAUTION !!! Will result in increased memory usage
+     * due to accumulation of records in StreamContext (ctx) */
     get dependencies(): ReadonlyArray<StreamName>;
     /** Function to extract record id */
     abstract id(record: AirbyteRecord): any;
     /** All the record models produced by converter */
     abstract get destinationModels(): ReadonlyArray<DestinationModel>;
-    /** Function converts an input Airbyte record to Faros destination canonical record */
-    abstract convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecord>>;
+    /** Function to convert an input Airbyte record to Faros Destination canonical record(s) */
+    abstract convert(record: AirbyteRecord, ctx: StreamContext): Promise<ReadonlyArray<DestinationRecordTyped<R>>>;
+    /** On processing complete handler called by the Faros Destination
+     * after the input processing is complete.
+     * Use this to release any resources or produce any additional records if necessary. */
+    onProcessingComplete(ctx: StreamContext): Promise<ReadonlyArray<DestinationRecordTyped<R>>>;
+}
+export declare abstract class Converter extends ConverterTyped<Dictionary<any>> {
 }
 export declare function parseObjectConfig<T>(obj: any, name: string): T | undefined;
 /** Stream context to store records by stream and other helpers */
@@ -56,9 +65,10 @@ export declare class StreamName {
  *   }
  * }
  */
-export declare type DestinationRecord = {
+export declare type DestinationRecordTyped<R extends Dictionary<any>> = {
     readonly model: DestinationModel;
-    readonly record: Dictionary<any>;
+    readonly record: R;
 };
+export declare type DestinationRecord = DestinationRecordTyped<Dictionary<any>>;
 /** Faros destination model name, e.g identity_Identity, vcs_Commit */
 export declare type DestinationModel = string;

package/lib/converters/converter.js

@@ -3,12 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.StreamName = exports.StreamNameSeparator = exports.StreamContext = exports.parseObjectConfig = exports.Converter = void 0;
+exports.StreamName = exports.StreamNameSeparator = exports.StreamContext = exports.parseObjectConfig = exports.Converter = exports.ConverterTyped = void 0;
 const lodash_1 = require("lodash");
 const object_sizeof_1 = __importDefault(require("object-sizeof"));
 const verror_1 = require("verror");
 /** Airbyte -> Faros record converter */
-class Converter {
+class ConverterTyped {
     /** Input stream supported by converter */
     get streamName() {
         if (this.stream)
@@ -16,12 +16,21 @@ class Converter {
         this.stream = StreamName.fromString(`${this.source}${exports.StreamNameSeparator}${(0, lodash_1.snakeCase)(this.constructor.name)}`);
         return this.stream;
     }
-    // Dependencies on other streams (if any).
-    // !!! Use with caution !!! Will result in increased memory usage
-    // due to accumulation of records in StreamContext (ctx)
+    /** Dependencies on other streams (if any).
+     * !!! USE WITH CAUTION !!! Will result in increased memory usage
+     * due to accumulation of records in StreamContext (ctx) */
     get dependencies() {
         return [];
     }
+    /** On processing complete handler called by the Faros Destination
+     * after the input processing is complete.
+     * Use this to release any resources or produce any additional records if necessary. */
+    async onProcessingComplete(ctx) {
+        return [];
+    }
+}
+exports.ConverterTyped = ConverterTyped;
+class Converter extends ConverterTyped {
 }
 exports.Converter = Converter;
 // Helper function for reading object type configurations that

package/lib/converters/gitlab/common.d.ts

@@ -1,5 +1,5 @@
 import { AirbyteRecord } from 'faros-airbyte-cdk';
-import { Converter } from '../converter';
+import { Converter, DestinationRecord } from '../converter';
 export interface CategoryRef {
     readonly category: string;
     readonly detail: string;
@@ -16,6 +16,7 @@ interface RepositoryKey {
 /** Common functions shares across GitLab converters */
 export declare class GitlabCommon {
     static readonly MAX_DESCRIPTION_LENGTH = 1000;
+    static tms_ProjectBoard_with_TaskBoard(projectKey: ProjectKey, name: string, description: string | null, createdAt: string | null | undefined, updatedAt: string | null | undefined): DestinationRecord[];
     static parseRepositoryKey(webUrl: string | undefined, source: string, startIndex?: number): undefined | RepositoryKey;
     static convertBuildStatus(status?: string): CategoryRef;
 }
@@ -27,4 +28,8 @@ export declare abstract class GitlabConverter extends Converter {
      */
     id(record: AirbyteRecord): any;
 }
+export interface ProjectKey {
+    uid: string;
+    source: string;
+}
 export {};

package/lib/converters/gitlab/common.js

@@ -1,9 +1,38 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GitlabConverter = exports.GitlabCommon = void 0;
+const faros_feeds_sdk_1 = require("faros-feeds-sdk");
 const converter_1 = require("../converter");
 /** Common functions shares across GitLab converters */
 class GitlabCommon {
+    static tms_ProjectBoard_with_TaskBoard(projectKey, name, description, createdAt, updatedAt) {
+        return [
+            {
+                model: 'tms_Project',
+                record: {
+                    ...projectKey,
+                    name: name,
+                    description: description === null || description === void 0 ? void 0 : description.substring(0, GitlabCommon.MAX_DESCRIPTION_LENGTH),
+                    createdAt: faros_feeds_sdk_1.Utils.toDate(createdAt),
+                    updatedAt: faros_feeds_sdk_1.Utils.toDate(updatedAt),
+                },
+            },
+            {
+                model: 'tms_TaskBoard',
+                record: {
+                    ...projectKey,
+                    name,
+                },
+            },
+            {
+                model: 'tms_TaskBoardProjectRelationship',
+                record: {
+                    board: projectKey,
+                    project: projectKey,
+                },
+            },
+        ];
+    }
     static parseRepositoryKey(webUrl, source, startIndex = 3) {
         if (!webUrl)
             return undefined;