airbyte-faros-destination 0.0.1

package/lib/destination.js

@@ -0,0 +1,665 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FarosDestination = void 0;
+const analytics_node_1 = __importDefault(require("analytics-node"));
+const faros_airbyte_cdk_1 = require("faros-airbyte-cdk");
+const faros_feeds_sdk_1 = require("faros-feeds-sdk");
+const faros_js_client_1 = require("faros-js-client");
+const http_1 = __importDefault(require("http"));
+const https_1 = __importDefault(require("https"));
+const lodash_1 = require("lodash");
+const path_1 = __importDefault(require("path"));
+const readline_1 = __importDefault(require("readline"));
+const util_1 = __importDefault(require("util"));
+const uuid_1 = require("uuid");
+const verror_1 = require("verror");
+const graphql_client_1 = require("./common/graphql-client");
+const graphql_writer_1 = require("./common/graphql-writer");
+const types_1 = require("./common/types");
+const write_stats_1 = require("./common/write-stats");
+const hasura_backend_1 = require("./community/hasura-backend");
+const converter_1 = require("./converters/converter");
+const converter_registry_1 = require("./converters/converter-registry");
+const jsonata_1 = require("./converters/jsonata");
+const PACKAGE_ROOT = path_1.default.join(__dirname, '..');
+const BASE_RESOURCES_DIR = path_1.default.join(PACKAGE_ROOT, 'resources');
+const DEFAULT_API_URL = 'https://prod.api.faros.ai';
+/** Faros destination implementation. */
+class FarosDestination extends faros_airbyte_cdk_1.AirbyteDestination {
+    constructor(logger, specOverride = undefined, edition = undefined, farosClientConfig = undefined, farosClient = undefined, farosGraph = undefined, farosRevisionExpiration = undefined, jsonataConverter = undefined, jsonataMode = jsonata_1.JSONataApplyMode.FALLBACK, invalidRecordStrategy = types_1.InvalidRecordStrategy.SKIP, excludeFieldsByModel = {}, graphQLClient = undefined, analytics = undefined, segmentUserId = undefined) {
+        super();
+        this.logger = logger;
+        this.specOverride = specOverride;
+        this.edition = edition;
+        this.farosClientConfig = farosClientConfig;
+        this.farosClient = farosClient;
+        this.farosGraph = farosGraph;
+        this.farosRevisionExpiration = farosRevisionExpiration;
+        this.jsonataConverter = jsonataConverter;
+        this.jsonataMode = jsonataMode;
+        this.invalidRecordStrategy = invalidRecordStrategy;
+        this.excludeFieldsByModel = excludeFieldsByModel;
+        this.graphQLClient = graphQLClient;
+        this.analytics = analytics;
+        this.segmentUserId = segmentUserId;
+        this.onConfigCheck = undefined;
+    }
+    getFarosClient() {
+        if (this.farosClient)
+            return this.farosClient;
+        throw new verror_1.VError('Faros Client is not initialized');
+    }
+    getGraphQLClient() {
+        if (this.graphQLClient)
+            return this.graphQLClient;
+        throw new verror_1.VError('GraphQL Client is not initialized');
+    }
+    async spec() {
+        if (this.specOverride)
+            return this.specOverride;
+        return faros_airbyte_cdk_1.SpecLoader.loadSpec(path_1.default.join(BASE_RESOURCES_DIR, 'spec.json'));
+    }
+    async check(config) {
+        try {
+            await this.init(config);
+            if (this.onConfigCheck) {
+                await this.onConfigCheck(config);
+            }
+        }
+        catch (e) {
+            return new faros_airbyte_cdk_1.AirbyteConnectionStatusMessage({
+                status: faros_airbyte_cdk_1.AirbyteConnectionStatus.FAILED,
+                message: e.message,
+            });
+        }
+        return new faros_airbyte_cdk_1.AirbyteConnectionStatusMessage({
+            status: faros_airbyte_cdk_1.AirbyteConnectionStatus.SUCCEEDED,
+        });
+    }
+    makeHttpAgents(keepAlive) {
+        return keepAlive
+            ? {
+                httpAgent: new http_1.default.Agent({ keepAlive: true }),
+                httpsAgent: new https_1.default.Agent({ keepAlive: true }),
+            }
+            : {};
+    }
+    async initCommunity(config) {
+        var _a;
+        if (!config.edition_configs.hasura_url) {
+            throw new verror_1.VError('Community Edition Hasura URL is not set');
+        }
+        try {
+            const backend = new hasura_backend_1.HasuraBackend(config.edition_configs.hasura_url, config.edition_configs.hasura_admin_secret, this.makeHttpAgents(config.keep_alive));
+            const schemaLoader = new faros_js_client_1.HasuraSchemaLoader(config.edition_configs.hasura_url, config.edition_configs.hasura_admin_secret);
+            this.graphQLClient = new graphql_client_1.GraphQLClient(this.logger, schemaLoader, backend, 0, config.edition_configs.community_graphql_batch_size, false);
+        }
+        catch (e) {
+            throw new verror_1.VError(`Failed to initialize Hasura Client. Error: ${e}`);
+        }
+        try {
+            if (!config.dry_run) {
+                await this.getGraphQLClient().healthCheck();
+            }
+        }
+        catch (e) {
+            throw new verror_1.VError(`Invalid Hasura url. Error: ${e}`);
+        }
+        this.segmentUserId = config.edition_configs.segment_user_id;
+        if (this.segmentUserId) {
+            if (!(0, uuid_1.validate)(this.segmentUserId)) {
+                throw new verror_1.VError(`Segment User Id ${this.segmentUserId} is not a valid UUID. Example: ${(0, uuid_1.v4)()}`);
+            }
+            // Segment host is used for testing purposes only
+            const host = (_a = config.edition_configs) === null || _a === void 0 ? void 0 : _a.segment_test_host;
+            // Only create the client if there's a user id specified
+            this.analytics = new analytics_node_1.default('YEu7VC65n9dIR85pQ1tgV2RHQHjo2bwn', {
+                host,
+            });
+        }
+    }
+    async initCloud(config) {
+        var _a, _b;
+        if (!config.edition_configs.api_key) {
+            throw new verror_1.VError('API key is not set');
+        }
+        const useGraphQLV2 = ((_a = config.edition_configs.graphql_api) !== null && _a !== void 0 ? _a : 'v2') === 'v2';
+        try {
+            this.farosClientConfig = {
+                url: (_b = config.edition_configs.api_url) !== null && _b !== void 0 ? _b : DEFAULT_API_URL,
+                apiKey: config.edition_configs.api_key,
+                useGraphQLV2,
+            };
+            const axiosConfig = {
+                timeout: 60000,
+                ...this.makeHttpAgents(config.keep_alive),
+            };
+            this.farosClient = new faros_js_client_1.FarosClient(this.farosClientConfig, this.logger.asPino('info'), axiosConfig);
+        }
+        catch (e) {
+            throw new verror_1.VError(`Failed to initialize Faros Client. Error: ${e}`);
+        }
+        try {
+            if (!config.dry_run && config.edition_configs.check_connection) {
+                await this.getFarosClient().tenant();
+            }
+        }
+        catch (e) {
+            throw new verror_1.VError(`Invalid Faros API url or API key. Error: ${e}`);
+        }
+        this.farosGraph = config.edition_configs.graph;
+        try {
+            if (!config.dry_run && config.edition_configs.check_connection) {
+                const exists = await this.getFarosClient().graphExists(this.farosGraph);
+                if (!exists) {
+                    throw new verror_1.VError(`Faros graph ${this.farosGraph} does not exist`);
+                }
+            }
+        }
+        catch (e) {
+            throw new verror_1.VError(`Invalid Faros graph ${this.farosGraph}. Error: ${e}`);
+        }
+        this.farosRevisionExpiration = config.edition_configs.expiration;
+        if (!this.farosRevisionExpiration) {
+            this.farosRevisionExpiration = '5 seconds';
+        }
+        if (useGraphQLV2) {
+            await this.initGraphQLV2(config);
+        }
+    }
+    async initGraphQLV2(config) {
+        const client = this.getFarosClient();
+        const graph = this.farosGraph;
+        try {
+            const backend = {
+                async healthCheck() {
+                    await client.graphExists(graph);
+                },
+                async postQuery(query) {
+                    return await client.rawGql(graph, query);
+                },
+            };
+            const schemaLoader = {
+                async loadSchema() {
+                    return await client.gqlSchema(graph);
+                },
+            };
+            this.graphQLClient = new graphql_client_1.GraphQLClient(this.logger, schemaLoader, backend, config.edition_configs.cloud_graphql_upsert_batch_size, config.edition_configs.cloud_graphql_batch_size);
+        }
+        catch (e) {
+            throw new verror_1.VError(`Failed to initialize GraphQLClient. Error: ${e}`);
+        }
+        try {
+            if (!config.dry_run && config.edition_configs.check_connection) {
+                await this.getGraphQLClient().healthCheck();
+            }
+        }
+        catch (e) {
+            throw new verror_1.VError(`Failed to health check GraphQLClient. Error: ${e}`);
+        }
+    }
+    initGlobal(config) {
+        var _a, _b;
+        try {
+            this.jsonataConverter = config.jsonata_expression
+                ? jsonata_1.JSONataConverter.make(config.jsonata_expression, config.jsonata_destination_models)
+                : undefined;
+        }
+        catch (e) {
+            throw new verror_1.VError(`Failed to initialize JSONata converter. Error: ${e}`);
+        }
+        if (config.invalid_record_strategy &&
+            !Object.values(types_1.InvalidRecordStrategy).includes(config.invalid_record_strategy)) {
+            throw new verror_1.VError(`Invalid strategy ${config.invalid_record_strategy}. ` +
+                `Possible values are ${Object.values(types_1.InvalidRecordStrategy).join(',')}`);
+        }
+        if (config.invalid_record_strategy) {
+            this.invalidRecordStrategy = config.invalid_record_strategy;
+        }
+        if (config.jsonata_mode &&
+            !Object.values(jsonata_1.JSONataApplyMode).includes(config.jsonata_mode)) {
+            throw new verror_1.VError(`Invalid JSONata mode ${config.jsonata_mode}. ` +
+                `Possible values are ${Object.values(jsonata_1.JSONataApplyMode).join(',')}`);
+        }
+        if (config.jsonata_mode) {
+            this.jsonataMode = config.jsonata_mode;
+        }
+        if (config.jsonata_expression &&
+            (!Array.isArray(config.jsonata_destination_models) ||
+                !config.jsonata_destination_models.length)) {
+            throw new verror_1.VError('JSONata destination models must be set when using JSONata expression');
+        }
+        this.excludeFieldsByModel = config.exclude_fields_map
+            ? JSON.parse(config.exclude_fields_map)
+            : {};
+        const jira_configs = (_b = (_a = config.source_specific_configs) === null || _a === void 0 ? void 0 : _a.jira) !== null && _b !== void 0 ? _b : {};
+        if (typeof jira_configs.truncate_limit === 'number' &&
+            jira_configs.truncate_limit < 0) {
+            throw new verror_1.VError('Jira Truncate Limit must be a non-negative number');
+        }
+        if (typeof jira_configs.additional_fields_array_limit === 'number' &&
+            jira_configs.additional_fields_array_limit < 0) {
+            throw new verror_1.VError('Jira Additional Fields Array Limit must be a non-negative number');
+        }
+        const objectTypeConfigKeys = [
+            ['bitbucket', 'application_mapping'],
+            ['pagerduty', 'application_mapping'],
+            ['squadcast', 'application_mapping'],
+            ['statuspage', 'application_mapping'],
+            ['victorops', 'application_mapping'],
+        ];
+        objectTypeConfigKeys.forEach((k) => {
+            var _a, _b;
+            return (0, converter_1.parseObjectConfig)((_b = (_a = config.source_specific_configs) === null || _a === void 0 ? void 0 : _a[k[0]]) === null || _b === void 0 ? void 0 : _b[k[1]], k.join('.'));
+        });
+    }
+    async init(config) {
+        var _a, _b, _c;
+        const edition = (_b = (_a = config.edition_configs) === null || _a === void 0 ? void 0 : _a.edition) !== null && _b !== void 0 ? _b : types_1.Edition.CLOUD;
+        if (!((_c = config.edition_configs) === null || _c === void 0 ? void 0 : _c.edition)) {
+            this.logger.info('Edition is not set. Assuming Faros Cloud edition');
+        }
+        this.edition = edition;
+        switch (edition) {
+            case types_1.Edition.COMMUNITY:
+                await this.initCommunity(config);
+                break;
+            case types_1.Edition.CLOUD:
+                await this.initCloud(config);
+                break;
+            default:
+                throw new verror_1.VError(`Invalid edition ${edition}. ` +
+                    `Possible values are ${Object.values(types_1.Edition).join(',')}`);
+        }
+        this.initGlobal(config);
+    }
+    getOrigin(config, catalog) {
+        if (config.origin) {
+            this.logger.info(`Using origin '${config.origin}' found in config`);
+            return config.origin;
+        }
+        // Determine origin from stream prefixes
+        const origins = (0, lodash_1.uniq)(catalog.streams.map((s) => s.stream.name.split(converter_1.StreamNameSeparator, 1)[0]));
+        if (origins.length === 0) {
+            throw new verror_1.VError('Could not determine origin from catalog');
+        }
+        else if (origins.length > 1) {
+            throw new verror_1.VError(`Found multiple possible origins from catalog: ${origins.join(',')}`);
+        }
+        const origin = origins[0];
+        this.logger.info(`Determined origin '${origin}' from stream prefixes`);
+        return origin;
+    }
+    async *write(config, catalog, stdin, dryRun) {
+        await this.init(config);
+        const origin = this.getOrigin(config, catalog);
+        const { streams, deleteModelEntries, converterDependencies } = this.initStreamsCheckConverters(catalog);
+        const streamsSyncMode = {};
+        for (const stream of Object.keys(streams)) {
+            streamsSyncMode[stream] = streams[stream].destination_sync_mode;
+        }
+        let latestStateMessage = undefined;
+        const stats = new write_stats_1.WriteStats();
+        const dryRunEnabled = config.dry_run === true || dryRun;
+        // Avoid creating a new revision and writer when dry run or community edition is enabled
+        try {
+            const streamContext = new converter_1.StreamContext(this.logger, config, streamsSyncMode, this.farosGraph, origin, this.edition === types_1.Edition.COMMUNITY ? undefined : this.getFarosClient());
+            if (dryRunEnabled) {
+                this.logger.info("Dry run is ENABLED. Won't write any records");
+                for await (const stateMessage of this.writeEntries(streamContext, stdin, streams, converterDependencies, stats)) {
+                    latestStateMessage = stateMessage;
+                }
+            }
+            else if (this.graphQLClient) {
+                this.logger.info(`Using GraphQLClient for writer`);
+                const graphQLClient = this.getGraphQLClient();
+                await graphQLClient.loadSchema();
+                streamContext.resetModels = new Set(deleteModelEntries);
+                let originRemapper = undefined;
+                if (config.accept_input_records_origin && config.replace_origin_map) {
+                    const originMap = JSON.parse(config.replace_origin_map);
+                    originRemapper = (origin) => {
+                        var _a;
+                        return (_a = originMap[origin]) !== null && _a !== void 0 ? _a : origin;
+                    };
+                }
+                const writer = new graphql_writer_1.GraphQLWriter(graphQLClient, config.accept_input_records_origin
+                    ? {
+                        getOrigin: (record) => {
+                            if (!record.origin) {
+                                return origin;
+                            }
+                            if (!originRemapper) {
+                                return record.origin;
+                            }
+                            return originRemapper(record.origin);
+                        },
+                    }
+                    : { getOrigin: () => origin }, stats, this);
+                for await (const stateMessage of this.writeEntries(streamContext, stdin, streams, converterDependencies, stats, writer, async () => await graphQLClient.resetData(origin, Array.from(streamContext.resetModels), this.edition === types_1.Edition.COMMUNITY))) {
+                    await graphQLClient.flush();
+                    yield stateMessage;
+                }
+            }
+            else {
+                this.logger.info(`Opening a new revision on graph ${this.farosGraph} ` +
+                    `with expiration of ${this.farosRevisionExpiration}`);
+                // Log all models to be deleted (if any)
+                if (deleteModelEntries.length > 0) {
+                    const modelsToDelete = (0, lodash_1.sortBy)(deleteModelEntries).join(',');
+                    this.logger.info(`Deleting records in destination graph ${this.farosGraph} for models: ${modelsToDelete}`);
+                }
+                // Create an entry uploader for the destination graph
+                const entryUploaderConfig = {
+                    name: origin,
+                    url: this.farosClientConfig.url,
+                    authHeader: this.farosClientConfig.apiKey,
+                    expiration: this.farosRevisionExpiration,
+                    graphName: this.farosGraph,
+                    deleteModelEntries,
+                    logger: this.logger.asPino('debug'),
+                };
+                await (0, faros_feeds_sdk_1.withEntryUploader)(entryUploaderConfig, async (writer, state) => {
+                    try {
+                        // Log last synced time
+                        const lastSynced = (state === null || state === void 0 ? void 0 : state.lastSynced)
+                            ? `last synced at ${state.lastSynced}`
+                            : 'not synced yet';
+                        this.logger.info(`Destination graph ${this.farosGraph} was ${lastSynced}`);
+                        // Process input and write entries
+                        for await (const stateMessage of this.writeEntries(streamContext, stdin, streams, converterDependencies, stats, writer)) {
+                            latestStateMessage = stateMessage;
+                        }
+                        // Return the current time
+                        return { lastSynced: new Date().toISOString() };
+                    }
+                    finally {
+                        // Don't forget to close the writer
+                        if (!writer.writableEnded)
+                            writer.end();
+                    }
+                });
+            }
+            if (this.analytics) {
+                this.logger.debug('Sending write stats to Segment');
+                const fn = (callback) => {
+                    this.analytics
+                        .track({
+                        event: 'Write Stats',
+                        userId: this.segmentUserId,
+                        properties: stats,
+                    }, callback)
+                        .flush(callback);
+                };
+                await util_1.default
+                    .promisify(fn)()
+                    .catch((err) => this.logger.error(`Failed to send write stats to Segment: ${err.message}`, err.stack));
+            }
+            // Airbyte updates connection state whenever the destination emits a state
+            // message, indicating that prior records have been processed.  Since we
+            // are writing all records in a single revision, only return the final
+            // state message at the end, once the revision has been closed.
+            if (latestStateMessage) {
+                yield latestStateMessage;
+            }
+        }
+        finally {
+            // Log collected statistics
+            stats.log(this.logger, dryRunEnabled ? 'Would write' : 'Wrote');
+        }
+    }
+    async *writeEntries(ctx, stdin, streams, converterDependencies, stats, writer, resetData) {
+        const recordsToBeProcessedLast = [];
+        const convertersUsed = new Map();
+        // NOTE: readline.createInterface() will start to consume the input stream once invoked.
+        // Having asynchronous operations between interface creation and asynchronous iteration may
+        // result in missed lines.
+        const input = readline_1.default.createInterface({
+            input: stdin,
+            terminal: stdin.isTTY,
+        });
+        try {
+            let sourceFailed = undefined;
+            // Process input & write records
+            for await (const line of input) {
+                let stateMessage = undefined;
+                await this.handleRecordProcessingError(stats, async () => {
+                    var _a;
+                    const msg = (0, faros_airbyte_cdk_1.parseAirbyteMessage)(line);
+                    stats.messagesRead++;
+                    if (msg.type === faros_airbyte_cdk_1.AirbyteMessageType.STATE) {
+                        const message = msg;
+                        if (((_a = message.sourceStatus) === null || _a === void 0 ? void 0 : _a.status) === 'ERRORED') {
+                            this.logger.error('Airbyte Source has failed: ' + message.sourceStatus.error);
+                            sourceFailed = message.sourceStatus;
+                        }
+                        else {
+                            stateMessage = message;
+                        }
+                    }
+                    else if (msg.type === faros_airbyte_cdk_1.AirbyteMessageType.RECORD) {
+                        stats.recordsRead++;
+                        const recordMessage = msg;
+                        if (!recordMessage.record) {
+                            throw new verror_1.VError('Empty record');
+                        }
+                        if (!streams[recordMessage.record.stream]) {
+                            throw new verror_1.VError(`Undefined stream ${recordMessage.record.stream}`);
+                        }
+                        const unpacked = recordMessage.unpackRaw();
+                        if (!unpacked.record) {
+                            throw new verror_1.VError('Empty unpacked record');
+                        }
+                        const stream = unpacked.record.stream;
+                        stats.incrementProcessedByStream(stream);
+                        const converter = this.getConverter(stream);
+                        // No need to fail on records we don't have a converter yet
+                        if (!converter) {
+                            stats.recordsSkipped++;
+                            return;
+                        }
+                        // Collect all the used converters
+                        if (!convertersUsed.has(stream)) {
+                            convertersUsed.set(stream, converter);
+                        }
+                        const writeRecord = async (context) => {
+                            stats.recordsWritten += await this.writeRecord(converter, unpacked, stats, context, writer);
+                            stats.recordsProcessed++;
+                        };
+                        // Check if any converters depend on this record stream.
+                        // If yes, keep the record in the stream context for other converters to get.
+                        const streamName = converter_1.StreamName.fromString(stream).asString;
+                        const recordId = converter.id(unpacked);
+                        if (converterDependencies.has(streamName) && recordId) {
+                            ctx.set(streamName, String(recordId), unpacked);
+                            // Print stream context stats every so often
+                            if (stats.recordsProcessed % 1000 == 0) {
+                                this.logger.info(`Stream context stats: ${ctx.stats()}`);
+                            }
+                        }
+                        // Process the record immediately if converter has no dependencies,
+                        // otherwise process it later once all streams are processed.
+                        if (converter.dependencies.length === 0) {
+                            await writeRecord(ctx);
+                        }
+                        else {
+                            recordsToBeProcessedLast.push(writeRecord);
+                        }
+                    }
+                });
+                if (stateMessage) {
+                    yield stateMessage;
+                }
+            }
+            // Process all the remaining records
+            if (recordsToBeProcessedLast.length > 0) {
+                this.logger.info(`Stdin processing completed, but still have ${recordsToBeProcessedLast.length} records to process`);
+                this.logger.info(`Stream context stats: ${ctx.stats()}`);
+                for await (const process of recordsToBeProcessedLast) {
+                    await this.handleRecordProcessingError(stats, () => process(ctx));
+                }
+            }
+            // Invoke on processing complete handler on all the active converters
+            for (const converter of convertersUsed.values()) {
+                const results = await converter.onProcessingComplete(ctx);
+                stats.recordsWritten += await this.writeConvertedRecords(converter, results, stats, writer);
+            }
+            if (sourceFailed) {
+                this.logger.error('Skipping reset of non-incremental models due to' +
+                    ` Airbyte Source failure: ${sourceFailed.error}`);
+            }
+            else {
+                await (resetData === null || resetData === void 0 ? void 0 : resetData());
+            }
+            // Don't forget to close the writer
+            await (writer === null || writer === void 0 ? void 0 : writer.end());
+        }
+        finally {
+            input.close();
+        }
+    }
+    async handleRecordProcessingError(stats, processRecord) {
+        var _a;
+        try {
+            await processRecord();
+        }
+        catch (e) {
+            stats.recordsErrored++;
+            this.logger.error(`Error processing input: ${(_a = e.message) !== null && _a !== void 0 ? _a : JSON.stringify(e)}`, e.stack);
+            switch (this.invalidRecordStrategy) {
+                case types_1.InvalidRecordStrategy.SKIP:
+                    stats.recordsSkipped++;
+                    break;
+                case types_1.InvalidRecordStrategy.FAIL:
+                    throw e;
+            }
+        }
+    }
+    initStreamsCheckConverters(catalog) {
+        const streams = (0, lodash_1.keyBy)(catalog.streams, (s) => s.stream.name);
+        const streamKeys = Object.keys(streams);
+        const deleteModelEntries = [];
+        const dependenciesByStream = {};
+        const incrementalModels = [];
+        // Check input streams & initialize record converters
+        for (const stream of streamKeys) {
+            const destinationSyncMode = streams[stream].destination_sync_mode;
+            if (!destinationSyncMode) {
+                throw new verror_1.VError(`Undefined destination sync mode for stream ${stream}`);
+            }
+            const converter = this.getConverter(stream, (err) => {
+                if (err.message.includes('Cannot find module ')) {
+                    this.logger.warn(`No converter found for ${stream}`);
+                }
+                else {
+                    this.logger.error(err.message, err.stack);
+                }
+            });
+            if (converter) {
+                this.logger.info(`Using ${converter.constructor.name} converter to convert ${stream} stream records`);
+                // Collect all converter dependencies
+                if (converter.dependencies.length > 0) {
+                    const streamName = converter.streamName.asString;
+                    if (!dependenciesByStream[streamName]) {
+                        dependenciesByStream[streamName] = new Set();
+                    }
+                    const deps = dependenciesByStream[streamName];
+                    converter.dependencies.forEach((d) => deps.add(d.asString));
+                }
+                // Prepare destination models to delete if any
+                if (destinationSyncMode === faros_airbyte_cdk_1.DestinationSyncMode.OVERWRITE) {
+                    deleteModelEntries.push(...converter.destinationModels);
+                }
+                else if (streams[stream].sync_mode === faros_airbyte_cdk_1.SyncMode.INCREMENTAL) {
+                    incrementalModels.push(...converter.destinationModels);
+                }
+            }
+        }
+        // Check for circular dependencies and error early if any
+        this.checkForCircularDependencies(dependenciesByStream);
+        // Collect all converter dependencies
+        const converterDependencies = new Set();
+        Object.keys(dependenciesByStream).forEach((k) => dependenciesByStream[k].forEach((v) => converterDependencies.add(v)));
+        return {
+            streams,
+            deleteModelEntries: (0, lodash_1.difference)((0, lodash_1.uniq)(deleteModelEntries), (0, lodash_1.uniq)(incrementalModels)),
+            converterDependencies,
+        };
+    }
+    getConverter(stream, onLoadError) {
+        var _a;
+        const converter = converter_registry_1.ConverterRegistry.getConverter(converter_1.StreamName.fromString(stream), onLoadError);
+        return this.jsonataMode === jsonata_1.JSONataApplyMode.OVERRIDE
+            ? (_a = this.jsonataConverter) !== null && _a !== void 0 ? _a : converter
+            : converter !== null && converter !== void 0 ? converter : this.jsonataConverter;
+    }
+    async writeRecord(converter, recordMessage, stats, ctx, writer) {
+        // Apply conversion on the input record
+        const results = await converter.convert(recordMessage, ctx);
+        // Write the converted records
+        return this.writeConvertedRecords(converter, results, stats, writer);
+    }
+    async writeConvertedRecords(converter, results, stats, writer) {
+        if (!Array.isArray(results))
+            throw new verror_1.VError('Invalid results: not an array');
+        let recordsWritten = 0;
+        // Write out the results to the output stream
+        for (const result of results) {
+            if (!result.model)
+                throw new verror_1.VError('Invalid result: undefined model');
+            if (!result.record)
+                throw new verror_1.VError('Invalid result: undefined record');
+            if (typeof result.record !== 'object')
+                throw new verror_1.VError('Invalid result: record is not an object');
+            // Set the source if missing
+            if (!result.record['source']) {
+                result.record['source'] = converter.streamName.source;
+            }
+            // Exclude record fields if necessary
+            const exclusions = this.excludeFieldsByModel[result.model];
+            if ((exclusions === null || exclusions === void 0 ? void 0 : exclusions.length) > 0) {
+                result.record = (0, lodash_1.pickBy)(result.record, (_v, k) => !exclusions.includes(k));
+            }
+            // Write the record & increment the stats
+            let isTimestamped = false;
+            if (writer) {
+                if (writer instanceof graphql_writer_1.GraphQLWriter) {
+                    isTimestamped = await writer.write(result);
+                }
+                else {
+                    writer.write({ [result.model]: result.record });
+                }
+            }
+            if (!isTimestamped) {
+                recordsWritten++;
+                stats.incrementWrittenByModel(result.model);
+            }
+        }
+        return recordsWritten;
+    }
+    addToPath(depsByStream, dep, path) {
+        if (path.includes(dep)) {
+            throw new verror_1.VError(`Circular converter dependency detected: ${path
+                .slice(path.indexOf(dep))
+                .concat(dep)
+                .join(',')}`);
+        }
+        path.push(dep);
+        if (depsByStream[dep]) {
+            return depsByStream[dep].forEach((next) => this.addToPath(depsByStream, next, [...path]));
+        }
+    }
+    checkForCircularDependencies(depsByStream) {
+        Object.keys(depsByStream).forEach((dep) => {
+            const dd = [...depsByStream[dep].values()];
+            this.logger.info(`Records of stream ${dep} will be accumulated and processed last, ` +
+                `since their converter has dependencies on streams: ${dd.join(',')}`);
+            this.addToPath(depsByStream, dep, []);
+        });
+    }
+}
+exports.FarosDestination = FarosDestination;
+//# sourceMappingURL=destination.js.map