airbyte-faros-destination 0.1.53

package/README.md

@@ -0,0 +1,161 @@
+# Faros Destination for Airbyte
+
+Faros Destination is a universal destination to import data from any Airbyte source into Faros.
+
+Each source stream is handled by an appropriate [Converter](src/converters/converter.ts) implementation. Here are all the [supported sources](src/converters).
+
+Any additional source streams can be handled by providing a JSONata expression for the built-in [JSONataConverter](src/converters/jsonata.ts) or by implementing converters for the streams ([read more below](#adding-support-for-additional-sources)).
+
+## Usage
+
+### Run from Airbyte
+
+1. Open Airbyte UI
+2. Go to Settings -> Destinations -> New Connector
+3. Set image - `farosai/airbyte-faros-destination`
+4. Set documentation - `https://github.com/faros-ai/airbyte-connectors`
+5. Set the desired version `x.y.z`, e.g the latest [![](https://img.shields.io/docker/v/farosai/airbyte-faros-destination?color=blue&label=docker)](https://hub.docker.com/r/farosai/airbyte-faros-destination/tags)
+6. Click Add
+
+More details on how to add a custom connector can be found in the official Airbyte [guide](https://docs.airbyte.com/integrations/custom-connectors).
+
+### Run from Terminal
+
+```shell
+docker pull farosai/airbyte-faros-destination
+docker run farosai/airbyte-faros-destination
+```
+
+## Adding Support for Additional Sources
+
+Faros Destination is built to be easily extensible to support additional sources by adding stream converters.
+
+### Well-Known Sources
+
+For well-known sources you can simply open a PR against this repository and add any new stream converters.
+We are always happy for you to contribute them to the community.
+
+When developing a converter, there is a specific naming convention that must be followed to ensure that the converter is correctly picked up by the Faros Destination. The converter class name must be the PascalCase (or UpperCamelCase) version of the stream name, e.g. [PullRequestStats](src/converters/github/pull_request_stats.ts) for GitHub.
+
+For testing, create the `all-streams.log` file and make sure that no real data is included. Also, ensure that the stream names are prefixed with the stream prefix specified in the test file. For example: `mytestsource__github__`.
+
+We have implemented [a lot of converters](src/converters) already so you have plenty examples to follow.
+
+### Custom Sources
+
+For custom or internal sources you can use Faros Destination as npm package in your project without having to fork or open PRs against this repo:
+```
+npm i airbyte-faros-destination
+```
+
+The library currently provides a main class `FarosDestinationRunner` which allows you to register the custom converters you created. Additionally, use the program property to get a default CLI app that provides the basic commands needed to write records to Faros.
+
+Example `index.ts`:
+```typescript
+import {Command} from 'commander';
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+import {Converter, DestinationModel, DestinationRecord,FarosDestinationRunner,StreamContext} from 'airbyte-faros-destination'
+
+class Builds extends Converter {
+  source: string = 'CustomSource'
+  destinationModels: ReadonlyArray<DestinationModel> = ['cicd_Build'];
+
+  id(record: AirbyteRecord): string {
+    return record.record.data.id;
+  }
+
+  async convert(
+    record: AirbyteRecord,
+    _ctx: StreamContext
+  ): Promise<ReadonlyArray<DestinationRecord>> {
+    const build = record.record.data
+    return [
+      {
+        model: 'cicd_Build',
+        record: {
+          uid: String(build.bid),
+          startedAt: new Date(build.created_at),
+          endedAt: new Date(build.finished_at),
+          status: build.status,
+          source: this.source
+        },
+      },
+    ];
+  }
+}
+class Pipelines extends Converter {
+  // similar to the Builds in the example above
+  ...
+}
+
+// main entry point
+export function mainCommand(): Command {
+  const destinationRunner = new FarosDestinationRunner();
+
+  // Register your custom converter(s)
+  destinationRunner.registerConverters(
+    new Builds(),
+    new Pipelines()
+  );
+
+  return destinationRunner.program;
+}
+```
+
+Example shell script to run file `bin/main`:
+```shell
+#!/usr/bin/env node
+
+const {mainCommand} = require('../lib');
+
+mainCommand().parseAsync(process.argv).catch((err) => {
+  console.error(err.message);
+  process.exit(1);
+});
+```
+
+### Writing Records into Faros
+```shell
+./bin/main write --config config.json --catalog catalog.json
+```
+
+You will need to provide two parameters, a `config.json` which specifies the
+details of the Faros destination to write to, and `catalog.json` to
+document source streams to write records for.
+
+Example `config.json`
+```json
+{
+  "edition_configs": {
+    "edition": "cloud",
+    "api_url": "https://prod.api.faros.ai",
+    "api_key": "<my_faros_api_key>",
+    "graph": "default"
+  },
+  "origin": "mydatasource"
+}
+```
+See [spec.json](resources/spec.json) for more properties for the `config.json`
+
+Example `catalog.json`
+```json
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "mydatasource__CustomSource__builds"
+      },
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "mydatasource__CustomSource__pipelines"
+      },
+      "destination_sync_mode": "append"
+    }
+  ]
+}
+```
+
+### Additional Commands
+Run `./bin/main --help` for detailed information on available commands.

package/lib/common/write-stats.d.ts

@@ -0,0 +1,16 @@
+import { AirbyteLogger } from 'faros-airbyte-cdk';
+import { Dictionary } from 'ts-essentials';
+export declare class WriteStats {
+    messagesRead: number;
+    recordsRead: number;
+    recordsProcessed: number;
+    recordsWritten: number;
+    recordsErrored: number;
+    recordsSkipped: number;
+    processedByStream: Dictionary<number>;
+    writtenByModel: Dictionary<number>;
+    constructor(messagesRead?: number, recordsRead?: number, recordsProcessed?: number, recordsWritten?: number, recordsErrored?: number, recordsSkipped?: number, processedByStream?: Dictionary<number>, writtenByModel?: Dictionary<number>);
+    incrementProcessedByStream(stream: string): void;
+    incrementWrittenByModel(model: string): void;
+    log(logger: AirbyteLogger, writeMsg: 'Wrote' | 'Would write'): void;
+}

package/lib/common/write-stats.js

@@ -0,0 +1,48 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WriteStats = void 0;
+const lodash_1 = __importDefault(require("lodash"));
+class WriteStats {
+    constructor(messagesRead = 0, recordsRead = 0, recordsProcessed = 0, recordsWritten = 0, recordsErrored = 0, recordsSkipped = 0, processedByStream = {}, writtenByModel = {}) {
+        this.messagesRead = messagesRead;
+        this.recordsRead = recordsRead;
+        this.recordsProcessed = recordsProcessed;
+        this.recordsWritten = recordsWritten;
+        this.recordsErrored = recordsErrored;
+        this.recordsSkipped = recordsSkipped;
+        this.processedByStream = processedByStream;
+        this.writtenByModel = writtenByModel;
+    }
+    incrementProcessedByStream(stream) {
+        const count = this.processedByStream[stream];
+        this.processedByStream[stream] = count ? count + 1 : 1;
+    }
+    incrementWrittenByModel(model) {
+        const count = this.writtenByModel[model];
+        this.writtenByModel[model] = count ? count + 1 : 1;
+    }
+    log(logger, writeMsg) {
+        logger.info(`Read ${this.messagesRead} messages`);
+        logger.info(`Read ${this.recordsRead} records`);
+        logger.info(`Processed ${this.recordsProcessed} records`);
+        const processed = (0, lodash_1.default)(this.processedByStream)
+            .toPairs()
+            .orderBy(0, 'asc')
+            .fromPairs()
+            .value();
+        logger.info(`Processed records by stream: ${JSON.stringify(processed)}`);
+        logger.info(`${writeMsg} ${this.recordsWritten} records`);
+        const written = (0, lodash_1.default)(this.writtenByModel)
+            .toPairs()
+            .orderBy(0, 'asc')
+            .fromPairs()
+            .value();
+        logger.info(`${writeMsg} records by model: ${JSON.stringify(written)}`);
+        logger.info(`Skipped ${this.recordsSkipped} records`);
+        logger.info(`Errored ${this.recordsErrored} records`);
+    }
+}
+exports.WriteStats = WriteStats;

package/lib/community/hasura-client.d.ts

@@ -0,0 +1,27 @@
+import { Dictionary } from 'ts-essentials';
+import { TimestampedRecord } from './types';
+export declare class HasuraClient {
+    private readonly api;
+    private readonly logger;
+    private readonly primaryKeys;
+    private readonly scalars;
+    private readonly references;
+    private readonly backReferences;
+    private sortedModelDependencies;
+    constructor(url: string);
+    healthCheck(): Promise<void>;
+    private fetchDbSource;
+    loadSchema(): Promise<void>;
+    private fetchPrimaryKeys;
+    private backReferenceOriginCheck;
+    resetData(origin: string, models: ReadonlyArray<string>): Promise<void>;
+    writeRecord(model: string, record: Dictionary<any>, origin: string): Promise<void>;
+    writeTimestampedRecord(record: TimestampedRecord): Promise<void>;
+    private writeUpdateRecord;
+    private writeDeletionRecord;
+    private postQuery;
+    private createWhereClause;
+    private createMutationObject;
+    private formatFieldValue;
+    private createConflictClause;
+}

package/lib/community/hasura-client.js

@@ -0,0 +1,278 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HasuraClient = void 0;
+const axios_1 = __importDefault(require("axios"));
+const date_format_1 = __importDefault(require("date-format"));
+const faros_airbyte_cdk_1 = require("faros-airbyte-cdk");
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const json_to_graphql_query_1 = require("json-to-graphql-query");
+const lodash_1 = require("lodash");
+const path_1 = __importDefault(require("path"));
+const toposort_1 = __importDefault(require("toposort"));
+const traverse_1 = __importDefault(require("traverse"));
+const verror_1 = require("verror");
+const types_1 = require("./types");
+class HasuraClient {
+    constructor(url) {
+        this.logger = new faros_airbyte_cdk_1.AirbyteLogger();
+        this.primaryKeys = {};
+        this.scalars = {};
+        this.references = {};
+        this.backReferences = {};
+        this.api = axios_1.default.create({
+            baseURL: url,
+            headers: { 'X-Hasura-Role': 'admin' },
+        });
+    }
+    async healthCheck() {
+        try {
+            await this.api.get('/healthz');
+        }
+        catch (e) {
+            throw new verror_1.VError(e, 'Failed to check Hasura health');
+        }
+    }
+    async fetchDbSource() {
+        const response = await this.api.post('/v1/metadata', {
+            type: 'export_metadata',
+            version: 2,
+            args: {},
+        });
+        const sources = response.data.metadata.sources;
+        const defaultSource = (0, lodash_1.find)(sources, (source) => source.name === 'default');
+        if (!defaultSource) {
+            throw new verror_1.VError('Faros database not connected to Hasura');
+        }
+        return defaultSource;
+    }
+    async loadSchema() {
+        var _a, _b, _c, _d;
+        await this.fetchPrimaryKeys();
+        const source = await this.fetchDbSource();
+        const query = await fs_extra_1.default.readFile(path_1.default.join(__dirname, '../../resources/introspection-query.gql'), 'utf8');
+        const response = await this.api.post('/v1/graphql', { query });
+        const schema = response.data.data.__schema;
+        for (const table of source.tables) {
+            const tableName = table.table.name;
+            const type = (0, lodash_1.find)(schema.types, (t) => t.name === tableName && t.kind === 'OBJECT');
+            if (!type)
+                continue;
+            const scalarTypes = type.fields.filter((t) => (t.type.kind === 'SCALAR' ||
+                (t.type.kind === 'NON_NULL' && t.type.ofType.kind === 'SCALAR')) &&
+                t.description !== 'generated');
+            const scalars = {};
+            for (const scalar of scalarTypes) {
+                scalars[scalar.name] = (_b = (_a = scalar.type.ofType) === null || _a === void 0 ? void 0 : _a.name) !== null && _b !== void 0 ? _b : scalar.type.name;
+            }
+            this.scalars[tableName] = scalars;
+            const references = {};
+            for (const rel of (_c = table.object_relationships) !== null && _c !== void 0 ? _c : []) {
+                const [refType] = rel.name.split('__');
+                references[rel.using.foreign_key_constraint_on] = {
+                    field: rel.name,
+                    model: refType,
+                };
+            }
+            this.references[tableName] = references;
+            this.backReferences[tableName] = ((_d = table.array_relationships) !== null && _d !== void 0 ? _d : []).map((rel) => {
+                return {
+                    field: rel.name,
+                    model: rel.using.foreign_key_constraint_on.table.name,
+                };
+            });
+        }
+        const modelDeps = [];
+        for (const model of Object.keys(this.references)) {
+            for (const ref of Object.values(this.references[model])) {
+                if (model !== ref.model) {
+                    modelDeps.push([model, ref.model]);
+                }
+            }
+        }
+        this.sortedModelDependencies = (0, toposort_1.default)(modelDeps);
+    }
+    async fetchPrimaryKeys() {
+        const response = await this.api.post('/v2/query', {
+            type: 'run_sql',
+            args: {
+                source: 'default',
+                sql: await fs_extra_1.default.readFile(path_1.default.join(__dirname, '../../resources/fetch-primary-keys.sql'), 'utf8'),
+                cascade: false,
+                read_only: true,
+            },
+        });
+        const result = response.data.result;
+        result
+            .filter((row) => row[0] !== 'table_name')
+            .forEach(([table, exp]) => {
+            // TODO: better way to do this?
+            const columns = exp
+                .replace('pkey(VARIADIC ARRAY[', '')
+                .replace('])', '')
+                .split(', ')
+                .map((col) => col.replace(/"/g, ''));
+            this.primaryKeys[table] = columns;
+        });
+    }
+    backReferenceOriginCheck(br, origin) {
+        const base = { origin: { _neq: origin } };
+        const nestedChecks = this.backReferences[br.model]
+            .filter((nbr) => nbr.field != br.field)
+            .map((nbr) => this.backReferenceOriginCheck(nbr, origin));
+        return {
+            [br.field]: {
+                _or: [base].concat(nestedChecks),
+            },
+        };
+    }
+    async resetData(origin, models) {
+        for (const model of (0, lodash_1.intersection)(this.sortedModelDependencies, models)) {
+            this.logger.info(`Resetting ${model} data for origin ${origin}`);
+            const deleteConditions = {
+                origin: { _eq: origin },
+                _not: {
+                    _or: this.backReferences[model].map((br) => this.backReferenceOriginCheck(br, origin)),
+                },
+            };
+            const mutation = {
+                [`delete_${model}`]: {
+                    __args: {
+                        where: {
+                            _and: { ...deleteConditions },
+                        },
+                    },
+                    affected_rows: true,
+                },
+            };
+            await this.postQuery({ mutation }, `Failed to reset ${model} data for origin ${origin}`);
+        }
+    }
+    // TODO: find alternate batching strategy
+    // cannot use Hasura batching due to https://github.com/hasura/graphql-engine/issues/4633
+    async writeRecord(model, record, origin) {
+        const obj = this.createMutationObject(model, origin, record);
+        const mutation = {
+            [`insert_${model}_one`]: { __args: obj, id: true },
+        };
+        await this.postQuery({ mutation }, `Failed to write ${model} record`);
+    }
+    async writeTimestampedRecord(record) {
+        switch (record.operation) {
+            case types_1.Operation.UPSERT:
+                await this.writeRecord(record.model, record.data, record.origin);
+                break;
+            case types_1.Operation.UPDATE:
+                await this.writeUpdateRecord(record);
+                break;
+            case types_1.Operation.DELETION:
+                await this.writeDeletionRecord(record);
+                break;
+            default:
+                throw new verror_1.VError(`Unuspported operation ${record.operation} for ${record}`);
+        }
+    }
+    async writeUpdateRecord(record) {
+        const mutation = {
+            [`update_${record.model}`]: {
+                __args: {
+                    where: this.createWhereClause(record.model, record.where),
+                    _set: this.createMutationObject(record.model, record.origin, record.patch).object,
+                },
+                returning: {
+                    id: true,
+                },
+            },
+        };
+        await this.postQuery({ mutation }, `Failed to update ${record.model} record`);
+    }
+    async writeDeletionRecord(record) {
+        const mutation = {
+            [`delete_${record.model}`]: {
+                __args: {
+                    where: this.createWhereClause(record.model, record.where),
+                },
+                affected_rows: true,
+            },
+        };
+        await this.postQuery({ mutation }, `Failed to delete ${record.model} record`);
+    }
+    async postQuery(query, errorMsg) {
+        const response = await this.api.post('/v1/graphql', {
+            query: (0, json_to_graphql_query_1.jsonToGraphQLQuery)(query),
+        });
+        if (response.data.errors) {
+            throw new verror_1.VError(`${errorMsg}: ${JSON.stringify(response.data.errors)}`);
+        }
+        return response.data;
+    }
+    createWhereClause(model, record) {
+        const obj = {};
+        for (const [field, value] of Object.entries(record)) {
+            const nestedModel = this.references[model][field];
+            if (nestedModel && value) {
+                obj[nestedModel.field] = this.createWhereClause(nestedModel.model, value);
+            }
+            else {
+                const val = this.formatFieldValue(model, field, value);
+                if (val)
+                    obj[field] = { _eq: val };
+            }
+        }
+        return obj;
+    }
+    createMutationObject(model, origin, record, nested) {
+        const obj = {};
+        for (const [field, value] of Object.entries(record)) {
+            const nestedModel = this.references[model][field];
+            if (nestedModel && value) {
+                obj[nestedModel.field] = this.createMutationObject(nestedModel.model, origin, value, true);
+            }
+            else {
+                const val = this.formatFieldValue(model, field, value);
+                if (val)
+                    obj[field] = val;
+            }
+        }
+        obj['origin'] = origin;
+        return {
+            [nested ? 'data' : 'object']: obj,
+            on_conflict: this.createConflictClause(model, nested),
+        };
+    }
+    formatFieldValue(model, field, value) {
+        if (!value)
+            return undefined;
+        const type = this.scalars[model][field];
+        if (!type) {
+            this.logger.debug(`Could not find type of ${field} in ${model}`);
+            return undefined;
+        }
+        if (type === 'timestamptz') {
+            return timestamptz(value);
+        }
+        else if (typeof value === 'object' || Array.isArray(value)) {
+            return (0, traverse_1.default)(value).map(function (val) {
+                if (val instanceof Date) {
+                    this.update(val.getTime());
+                }
+            });
+        }
+        return value;
+    }
+    createConflictClause(model, nested) {
+        const updateColumns = nested
+            ? ['refreshedAt']
+            : (0, lodash_1.difference)(Object.keys(this.scalars[model]), this.primaryKeys[model]);
+        return {
+            constraint: new json_to_graphql_query_1.EnumType(`${model}_pkey`),
+            update_columns: updateColumns.map((c) => new json_to_graphql_query_1.EnumType(c)),
+        };
+    }
+}
+exports.HasuraClient = HasuraClient;
+function timestamptz(date) {
+    return date_format_1.default.asString(date_format_1.default.ISO8601_WITH_TZ_OFFSET_FORMAT, date);
+}

package/lib/community/hasura-writer.d.ts

@@ -0,0 +1,12 @@
+import { WriteStats } from '../common/write-stats';
+import { HasuraClient } from './hasura-client';
+export declare class HasuraWriter {
+    private readonly hasuraClient;
+    private readonly origin;
+    private readonly stats;
+    private readonly handleRecordProcessingError;
+    private readonly timestampedRecords;
+    constructor(hasuraClient: HasuraClient, origin: string, stats: WriteStats, handleRecordProcessingError: (stats: WriteStats, processRecord: () => Promise<void>) => Promise<void>);
+    write(result: any): Promise<boolean>;
+    end(): Promise<void>;
+}

package/lib/community/hasura-writer.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HasuraWriter = void 0;
+const lodash_1 = require("lodash");
+const verror_1 = require("verror");
+const types_1 = require("./types");
+class HasuraWriter {
+    constructor(hasuraClient, origin, stats, handleRecordProcessingError) {
+        this.hasuraClient = hasuraClient;
+        this.origin = origin;
+        this.stats = stats;
+        this.handleRecordProcessingError = handleRecordProcessingError;
+        this.timestampedRecords = [];
+    }
+    async write(result) {
+        const [baseModel, operation] = result.model.split('__', 2);
+        if (!operation) {
+            await this.hasuraClient.writeRecord(result.model, result.record, this.origin);
+            return false;
+        }
+        else if (Object.values(types_1.Operation).includes(operation)) {
+            this.timestampedRecords.push({
+                model: baseModel,
+                operation,
+                origin: this.origin,
+                ...result.record,
+            });
+            return true;
+        }
+        else {
+            throw new verror_1.VError(`Unuspported model operation ${operation} for ${result.model}: ${result.record}`);
+        }
+    }
+    async end() {
+        for (const record of (0, lodash_1.sortBy)(this.timestampedRecords, (r) => r.at)) {
+            await this.handleRecordProcessingError(this.stats, async () => {
+                await this.hasuraClient.writeTimestampedRecord(record);
+                this.stats.recordsWritten++;
+                this.stats.incrementWrittenByModel(`${record.model}__${record.operation}`);
+            });
+        }
+    }
+}
+exports.HasuraWriter = HasuraWriter;

package/lib/community/types.d.ts

@@ -0,0 +1,26 @@
+import { Dictionary } from 'ts-essentials';
+export declare enum Operation {
+    UPSERT = "Upsert",
+    UPDATE = "Update",
+    DELETION = "Deletion"
+}
+export interface TimestampedRecord {
+    model: string;
+    origin: string;
+    at: number;
+    operation: Operation;
+}
+export interface UpsertRecord extends TimestampedRecord {
+    operation: Operation.UPSERT;
+    data: Dictionary<any>;
+}
+export interface UpdateRecord extends TimestampedRecord {
+    operation: Operation.UPDATE;
+    where: Dictionary<any>;
+    mask: string[];
+    patch: Dictionary<any>;
+}
+export interface DeletionRecord extends TimestampedRecord {
+    operation: Operation.DELETION;
+    where: Dictionary<any>;
+}

package/lib/community/types.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Operation = void 0;
+var Operation;
+(function (Operation) {
+    Operation["UPSERT"] = "Upsert";
+    Operation["UPDATE"] = "Update";
+    Operation["DELETION"] = "Deletion";
+})(Operation = exports.Operation || (exports.Operation = {}));