airbyte-faros-destination 0.0.1

package/README.md

@@ -0,0 +1,178 @@
+# Faros Destination for Airbyte
+
+Faros Destination is a universal destination to import data from any Airbyte source into Faros.
+
+Each source stream is handled by an appropriate [Converter](https://github.com/faros-ai/airbyte-connectors/tree/main/destinations/airbyte-faros-destination/src/converters/converter.ts) implementation. Here are all the [supported sources](https://github.com/faros-ai/airbyte-connectors/tree/main/destinations/airbyte-faros-destination/src/converters).
+
+Any additional source streams can be handled by providing a JSONata expression for the built-in [JSONataConverter](https://github.com/faros-ai/airbyte-connectors/tree/main/destinations/airbyte-faros-destination/src/converters/jsonata.ts) or by implementing converters for the streams ([read more below](#adding-support-for-additional-sources)).
+
+![](img/source_destination.png)
+
+## Usage
+
+### Run from Airbyte
+
+1. Open Airbyte UI
+2. Go to Settings -> Destinations -> New Connector
+3. Set image - `farosai/airbyte-faros-destination`
+4. Set documentation - `https://github.com/faros-ai/airbyte-connectors`
+5. Set the desired version `x.y.z`, e.g the latest [![](https://img.shields.io/docker/v/farosai/airbyte-faros-destination?color=blue&label=docker)](https://hub.docker.com/r/farosai/airbyte-faros-destination/tags)
+6. Click Add
+
+More details on how to add a custom connector can be found in the [official Airbyte guide](https://docs.airbyte.com/integrations/custom-connectors).
+
+### Run from Terminal
+
+```shell
+docker pull farosai/airbyte-faros-destination
+docker run farosai/airbyte-faros-destination
+```
+
+## Adding Support for Additional Sources
+
+Faros Destination is built to be easily extensible to support additional sources by adding stream converters.
+
+### Well-Known Sources
+
+For well-known sources you can simply open a PR against this repository and add any new stream converters.
+We are always happy for you to contribute them to the community.
+
+When developing a converter, there is a specific naming convention that must be followed to ensure that the converter is correctly picked up by the Faros Destination. The converter class name must be the PascalCase (or UpperCamelCase) version of the stream name, e.g. [PullRequestStats](https://github.com/faros-ai/airbyte-connectors/tree/main/destinations/airbyte-faros-destination/src/converters/github/pull_request_stats.ts) for GitHub.
+
+For testing, create the `all-streams.log` file and make sure that no real data is included. Also, ensure that the stream names are prefixed with the stream prefix specified in the test file. For example: `mytestsource__github__`.
+
+We have implemented [a lot of converters](https://github.com/faros-ai/airbyte-connectors/tree/main/destinations/airbyte-faros-destination/src/converters) already so you have plenty examples to follow.
+
+### Custom Sources
+
+For custom or internal sources you can use Faros Destination as npm package in your project without having to fork or open PRs against this repo:
+
+```
+npm i airbyte-faros-destination
+```
+
+The library currently provides a main class `FarosDestinationRunner` which allows you to register the custom converters you created. Additionally, use the program property to get a default CLI app that provides the basic commands needed to write records to Faros.
+
+Example `index.ts`:
+
+```typescript
+import {Command} from 'commander';
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+import {Converter,DestinationModel,DestinationRecord,FarosDestinationRunner,StreamContext} from 'airbyte-faros-destination'
+
+class Builds extends Converter {
+  source = 'CustomSource'
+  destinationModels = ['cicd_Build'];
+
+  id(record: AirbyteRecord): string {
+    return record.record.data.id;
+  }
+
+  async convert(
+    record: AirbyteRecord,
+    ctx: StreamContext
+  ): Promise<ReadonlyArray<DestinationRecord>> {
+    const build = record.record.data
+    return [
+      {
+        model: 'cicd_Build',
+        record: {
+          uid: String(build.bid),
+          startedAt: new Date(build.created_at),
+          endedAt: new Date(build.finished_at),
+          status: build.status,
+          source: this.source
+        },
+      },
+    ];
+  }
+}
+
+class Pipelines extends Converter {
+  // similar to the Builds in the example above
+  ...
+}
+
+// Main entry point
+export function mainCommand(): Command {
+  const destinationRunner = new FarosDestinationRunner();
+
+  // Register your custom converter(s)
+  destinationRunner.registerConverters(
+    new Builds(),
+    new Pipelines()
+  );
+
+  return destinationRunner.program;
+}
+```
+
+Example shell script to run file `bin/main`:
+
+```shell
+#!/usr/bin/env node
+
+const {mainCommand} = require('../lib');
+
+mainCommand().parseAsync(process.argv).catch((err) => {
+  console.error(err.message);
+  process.exit(1);
+});
+```
+
+### Writing Records into Faros
+
+```shell
+./bin/main write --config config.json --catalog catalog.json
+```
+
+You will need to provide two parameters, a `config.json` which specifies the
+details of the Faros destination to write to, and `catalog.json` to
+document source streams to write records for.
+
+Example `config.json`
+
+```json
+{
+  "edition_configs": {
+    "edition": "cloud",
+    "api_url": "https://prod.api.faros.ai",
+    "api_key": "<my_faros_api_key>",
+    "graph": "default"
+  },
+  "origin": "mydatasource"
+}
+```
+
+See [spec.json](https://github.com/faros-ai/airbyte-connectors/tree/main/destinations/airbyte-faros-destination/resources/spec.json) for more properties for the `config.json`
+
+Example `catalog.json`
+
+```json
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "mydatasource__CustomSource__builds"
+      },
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "mydatasource__CustomSource__pipelines"
+      },
+      "destination_sync_mode": "append"
+    }
+  ]
+}
+```
+
+**Tip**: you can even pipe data directly from your custom source into your custom destination without Airbyte server while prefixing your streams (as expected by Faros Destination):
+
+```shell
+<my-source-command> | jq -c -R 'fromjson? | select(.type == "RECORD") | .record.stream = "mydatasource__CustomSource__\(.record.stream)"' | <my-destination-command>
+```
+
+### Additional Commands
+
+Run `./bin/main --help` for detailed information on available commands.

package/lib/common/graphql-client.d.ts

@@ -0,0 +1,134 @@
+import { AirbyteLogger } from 'faros-airbyte-cdk';
+import { SchemaLoader } from 'faros-js-client';
+import { Dictionary } from 'ts-essentials';
+import { TimestampedRecord } from './types';
+export interface GraphQLBackend {
+    healthCheck(): Promise<void>;
+    postQuery(query: any): Promise<any>;
+}
+export interface Upsert {
+    id?: string;
+    readonly model: string;
+    readonly object: Dictionary<any>;
+    readonly foreignKeys: Dictionary<Upsert>;
+}
+export declare class UpsertBuffer {
+    private readonly upsertBuffer;
+    add(upsert: Upsert): void;
+    size(): number;
+    pop(model: string): Upsert[] | undefined;
+    get(model: string): Upsert[] | undefined;
+    private getInternal;
+    clear(): number;
+}
+export declare function serialize(obj: Dictionary<number | string>): string;
+/**
+ * Like lodash pick but with null value replacement.
+ */
+export declare function strictPick(obj: any, keys: string[], nullValue?: string): any;
+/**
+ * Groups objects by primary key and then merges all related objects into
+ * a single object where last-wins for overlapping properties.
+ */
+export declare function mergeByPrimaryKey(objects: any[], primaryKeys: string[]): any[];
+/**
+ * Execute async callback for each batch of upserts.
+ */
+export declare function batchIterator<T>(batches: Upsert[][], callback: (batch: Upsert[]) => Promise<T>): AsyncIterable<T>;
+/**
+ * Separates objects into arrays of objects
+ * that all have the same keys.
+ */
+export declare function groupByKeys(objects: any[]): any[][];
+/**
+ * Separates upserts into levels. Level 0 has no dependencies on other levels.
+ * Level 1 depends on Level 0, 2 on 1 and so on.
+ */
+export declare function toLevels(upserts: Upsert[]): Upsert[][];
+export declare function toPostgresArrayLiteral(value: any[]): string;
+/**
+ * Client for writing records as GraphQL mutations.  The client supports 3
+ * kinds of writes: Upserts, Updates and Deletes.
+ *
+ * For upserts (when upsertBatchSize is greater than 0), the high-level algorithm is:
+ *
+ * > For each record, build a tree of Upserts. The tree has more than one node if the current record represents a
+ *   nested entity (e.g. branch record referencing repo which, in-turn, references org).
+ * > Buffer Upserts and index each by model (e.g. vcs_Branch)
+ * > When batch size limit is reached, execute a single insert mutation per model. Start with leaves of Upsert tree.
+ * > After inserting a batch, copy the id of each record back to the Upsert object. When inserting subsequent batches,
+ *   required foreign keys will be read from the Upsert tree and copied to the appropriate batch mutation.
+ *
+ * Note: there is a complication in this algorithm for self-referent models (i.e. org_Employee's manager relationship).
+ * For these models, we split the batch into "levels". The first level consists of records with no dependencies on
+ * records of the same type. The second depends on the first and so on.
+ *
+ * For Updates and Deletes (which are much less frequent) we have a separate write buffer.  This buffer is
+ * flushed if it reaches capacity.  There is no attempt to combine these into bulk mutations (as done w/ upserts).
+ */
+export declare class GraphQLClient {
+    private readonly logger;
+    private readonly schemaLoader;
+    private readonly backend;
+    private schema;
+    private tableNames;
+    private tableDependencies;
+    private readonly mutationBatchSize;
+    private readonly upsertBatchSize;
+    private readonly writeBuffer;
+    private readonly upsertBuffer;
+    private readonly selfReferentModels;
+    private readonly updateResetLimit;
+    private resetLimitMillis;
+    constructor(logger: AirbyteLogger, schemaLoader: SchemaLoader, backend: GraphQLBackend, upsertBatchSize: any, mutationBatchSize: any, updateResetLimit?: boolean);
+    healthCheck(): Promise<void>;
+    loadSchema(): Promise<void>;
+    private checkSchema;
+    resetData(origin: string, models: ReadonlyArray<string>, keepReferencedRecords: boolean): Promise<void>;
+    writeRecord(model: string, record: Dictionary<any>, origin: string): Promise<void>;
+    private flushUpsertBuffer;
+    private execUpsert;
+    private doFlushUpsertBuffer;
+    writeTimestampedRecord(record: TimestampedRecord): Promise<void>;
+    private writeUpdateRecord;
+    private writeDeletionRecord;
+    private postQuery;
+    flush(): Promise<void>;
+    private flushWriteBuffer;
+    /**
+     * Constructs a gql query from an array of json mutations.
+     * The outputted qql mutation might look like:
+     *
+     *   mutation  {
+     *     i1: insert_cicd_Artifact_one(object: {uid: "u1b"}) {
+     *       id
+     *       refreshedAt
+     *     }
+     *     i2: insert_cicd_Artifact_one(object: {uid: "u2b"}) {
+     *       id
+     *       refreshedAt
+     *     }
+     *   }
+     *
+     *  Notable here are the i1/i2 aliases.  These are required when multiple operations
+     *  share the same name (e.g. insert_cicd_Artifact_one) and are supported in
+     *  jsonToGraphQLQuery with __aliasFor directive.
+     *
+     *  @return batch gql mutation or undefined if the input is undefined, empty
+     *  or doesn't contain any mutations.
+     */
+    static batchMutation(queries: any[]): string | undefined;
+    private createWhereClause;
+    private createMutationObject;
+    private addUpsert;
+    private objectWithForeignKeys;
+    /**
+     * returns serialized version of keys fields
+     */
+    private serializedPrimaryKey;
+    private toUpsertOps;
+    private isValidField;
+    private formatFieldValue;
+    private createConflictClause;
+    private createUpsertConflictClause;
+}