airbyte-faros-destination 0.15.4 → 0.16.1

package/lib/index.d.ts

@@ -14,3 +14,4 @@ export declare function mainCommand(options?: {
 export { Edition, InvalidRecordStrategy } from './common/types';
 export { Converter, DestinationRecord, DestinationModel, StreamContext, StreamName, } from './converters/converter';
 export { FarosDestinationRunner } from './destination-runner';
+export * from './testing-tools/testing-tools';

package/lib/index.js

@@ -1,4 +1,18 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.FarosDestinationRunner = exports.StreamName = exports.StreamContext = exports.Converter = exports.InvalidRecordStrategy = exports.Edition = void 0;
 exports.mainCommand = mainCommand;
@@ -36,4 +50,5 @@ Object.defineProperty(exports, "StreamContext", { enumerable: true, get: functio
 Object.defineProperty(exports, "StreamName", { enumerable: true, get: function () { return converter_1.StreamName; } });
 var destination_runner_2 = require("./destination-runner");
 Object.defineProperty(exports, "FarosDestinationRunner", { enumerable: true, get: function () { return destination_runner_2.FarosDestinationRunner; } });
+__exportStar(require("./testing-tools/testing-tools"), exports);
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAYA,kCAoBC;AA9BD,6DAA4D;AAE5D;;;;;;;GAOG;AACH,SAAgB,WAAW,CAAC,OAG3B;IACC,MAAM,iBAAiB,GAAG,IAAI,2CAAsB,EAAE,CAAC;IACvD,MAAM,OAAO,GAAG,iBAAiB,CAAC,WAAW,EAAE,CAAC;IAEhD,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,YAAY,EAAE,CAAC;QAC1B,OAAO,CAAC,YAAY,EAAE,CAAC;IACzB,CAAC;IACD,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,cAAc,EAAE,CAAC;QAC5B,OAAO,CAAC,eAAe,CAAC;YACtB,gEAAgE;YAChE,QAAQ,EAAE,GAAG,EAAE,GAAE,CAAC;YAClB,gEAAgE;YAChE,QAAQ,EAAE,GAAG,EAAE,GAAE,CAAC;SACnB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,wCAA8D;AAAtD,gGAAA,OAAO,OAAA;AAAE,8GAAA,qBAAqB,OAAA;AACtC,oDAMgC;AAL9B,sGAAA,SAAS,OAAA;AAGT,0GAAA,aAAa,OAAA;AACb,uGAAA,UAAU,OAAA;AAEZ,2DAA4D;AAApD,4HAAA,sBAAsB,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAYA,kCAoBC;AA9BD,6DAA4D;AAE5D;;;;;;;GAOG;AACH,SAAgB,WAAW,CAAC,OAG3B;IACC,MAAM,iBAAiB,GAAG,IAAI,2CAAsB,EAAE,CAAC;IACvD,MAAM,OAAO,GAAG,iBAAiB,CAAC,WAAW,EAAE,CAAC;IAEhD,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,YAAY,EAAE,CAAC;QAC1B,OAAO,CAAC,YAAY,EAAE,CAAC;IACzB,CAAC;IACD,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,cAAc,EAAE,CAAC;QAC5B,OAAO,CAAC,eAAe,CAAC;YACtB,gEAAgE;YAChE,QAAQ,EAAE,GAAG,EAAE,GAAE,CAAC;YAClB,gEAAgE;YAChE,QAAQ,EAAE,GAAG,EAAE,GAAE,CAAC;SACnB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,wCAA8D;AAAtD,gGAAA,OAAO,OAAA;AAAE,8GAAA,qBAAqB,OAAA;AACtC,oDAMgC;AAL9B,sGAAA,SAAS,OAAA;AAGT,0GAAA,aAAa,OAAA;AACb,uGAAA,UAAU,OAAA;AAEZ,2DAA4D;AAApD,4HAAA,sBAAsB,OAAA;AAE9B,gEAA8C"}

package/lib/testing-tools/cli.d.ts

@@ -0,0 +1,37 @@
+import { ChildProcess } from 'child_process';
+import { Readable, Writable } from 'stream';
+export declare function read(s: Readable): Promise<string>;
+export declare function readLines(s: Readable): Promise<ReadonlyArray<string>>;
+export interface CLIOptions {
+    env?: {
+        [key: string]: string;
+    };
+}
+/** A convenience class for testing CLI inputs and outputs. */
+export declare class CLI {
+    private readonly cp;
+    constructor(cp: ChildProcess);
+    get stderr(): Readable;
+    get stdout(): Readable;
+    get stdin(): Writable;
+    /** Waits for the child process to terminate and returns the exit code. */
+    wait(): Promise<number>;
+    /**
+     * Checks stdin for the given prompt string, then writes the given response to
+     * stdout.
+     */
+    respondToPrompt(prompt: string, response: string): void;
+    /**
+     * Runs CLI as a subprocess with the given arguments.
+     *
+     * If a home directory is not provided, the CLI's home directory will be set
+     * to an empty, unique, and temporary directory. In particular, this means
+     * that runs created by this function have separate caches and can run
+     * concurrently without interference.
+     *
+     * If a home diectory is not provided, the newly generated temporary directory
+     * will be deleted after the CLI command has returned. Otherwise, it is the
+     * caller's responsibility to clean up their chosen home directory.
+     */
+    static runWith(args: string[], opts?: CLIOptions, bin?: string): Promise<CLI>;
+}

package/lib/testing-tools/cli.js

@@ -0,0 +1,86 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CLI = void 0;
+exports.read = read;
+exports.readLines = readLines;
+const cross_spawn_1 = __importDefault(require("cross-spawn"));
+const path_1 = __importDefault(require("path"));
+const consumers_1 = __importDefault(require("stream/consumers"));
+async function read(s) {
+    return await consumers_1.default.text(s);
+}
+async function readLines(s) {
+    return (await read(s)).split('\n');
+}
+/** A convenience class for testing CLI inputs and outputs. */
+class CLI {
+    constructor(cp) {
+        this.cp = cp;
+    }
+    get stderr() {
+        // `stderr` is always defined with the options in `runWith`.
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        return this.cp.stderr;
+    }
+    get stdout() {
+        // `stdout` is always defined with the options in `runWith`.
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        return this.cp.stdout;
+    }
+    get stdin() {
+        // `stdin` is always defined with the options in `runWith`.
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        return this.cp.stdin;
+    }
+    /** Waits for the child process to terminate and returns the exit code. */
+    async wait() {
+        if (this.cp.exitCode !== null) {
+            return this.cp.exitCode;
+        }
+        return new Promise((ok) => {
+            this.cp.on('exit', ok);
+        });
+    }
+    /**
+     * Checks stdin for the given prompt string, then writes the given response to
+     * stdout.
+     */
+    respondToPrompt(prompt, response) {
+        // Use flag to only write to stdin once, since the Enquirer library prompt
+        // triggers stdout.on() multiple times.
+        let shouldRespond = true;
+        this.stdout.on('data', (data) => {
+            if (shouldRespond && data.toString().includes(prompt)) {
+                this.stdin.write(response, 'utf8');
+                shouldRespond = false;
+            }
+        });
+    }
+    /**
+     * Runs CLI as a subprocess with the given arguments.
+     *
+     * If a home directory is not provided, the CLI's home directory will be set
+     * to an empty, unique, and temporary directory. In particular, this means
+     * that runs created by this function have separate caches and can run
+     * concurrently without interference.
+     *
+     * If a home diectory is not provided, the newly generated temporary directory
+     * will be deleted after the CLI command has returned. Otherwise, it is the
+     * caller's responsibility to clean up their chosen home directory.
+     */
+    static async runWith(args, opts = {}, bin = 'main') {
+        const spawnOpts = {
+            env: {
+                ...process.env,
+                ...opts.env,
+            },
+        };
+        const cp = (0, cross_spawn_1.default)(path_1.default.resolve('bin', bin), args, spawnOpts);
+        return new CLI(cp);
+    }
+}
+exports.CLI = CLI;
+//# sourceMappingURL=cli.js.map

package/lib/testing-tools/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/testing-tools/cli.ts"],"names":[],"mappings":";;;;;;AAMA,oBAEC;AAED,8BAEC;AAXD,8DAAgC;AAChC,gDAAwB;AAExB,iEAAyC;AAElC,KAAK,UAAU,IAAI,CAAC,CAAW;IACpC,OAAO,MAAM,mBAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAEM,KAAK,UAAU,SAAS,CAAC,CAAW;IACzC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACrC,CAAC;AAMD,8DAA8D;AAC9D,MAAa,GAAG;IACd,YAA6B,EAAgB;QAAhB,OAAE,GAAF,EAAE,CAAc;IAAG,CAAC;IAEjD,IAAI,MAAM;QACR,4DAA4D;QAC5D,oEAAoE;QACpE,OAAO,IAAI,CAAC,EAAE,CAAC,MAAO,CAAC;IACzB,CAAC;IAED,IAAI,MAAM;QACR,4DAA4D;QAC5D,oEAAoE;QACpE,OAAO,IAAI,CAAC,EAAE,CAAC,MAAO,CAAC;IACzB,CAAC;IAED,IAAI,KAAK;QACP,2DAA2D;QAC3D,oEAAoE;QACpE,OAAO,IAAI,CAAC,EAAE,CAAC,KAAM,CAAC;IACxB,CAAC;IAED,0EAA0E;IAC1E,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,EAAE,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC;QAC1B,CAAC;QACD,OAAO,IAAI,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YACxB,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,MAAc,EAAE,QAAgB;QAC9C,0EAA0E;QAC1E,uCAAuC;QACvC,IAAI,aAAa,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;YACnC,IAAI,aAAa,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtD,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACnC,aAAa,GAAG,KAAK,CAAC;YACxB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAClB,IAAc,EACd,OAAmB,EAAE,EACrB,GAAG,GAAG,MAAM;QAEZ,MAAM,SAAS,GAAG;YAChB,GAAG,EAAE;gBACH,GAAG,OAAO,CAAC,GAAG;gBACd,GAAG,IAAI,CAAC,GAAG;aACZ;SACF,CAAC;QACF,MAAM,EAAE,GAAG,IAAA,qBAAK,EAAC,cAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;QAC5D,OAAO,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC;IACrB,CAAC;CACF;AAzED,kBAyEC"}

package/lib/testing-tools/index.d.ts

@@ -0,0 +1,3 @@
+export * from './cli';
+export * from './testing-tools';
+export * from './utils';

package/lib/testing-tools/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./cli"), exports);
+__exportStar(require("./testing-tools"), exports);
+__exportStar(require("./utils"), exports);
+//# sourceMappingURL=index.js.map

package/lib/testing-tools/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/testing-tools/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,wCAAsB;AACtB,kDAAgC;AAChC,0CAAwB"}

package/lib/testing-tools/testing-tools.d.ts

@@ -0,0 +1,36 @@
+import { Mockttp } from 'mockttp';
+import pino from 'pino';
+import { Dictionary } from 'ts-essentials';
+import { Edition, InvalidRecordStrategy } from '../common/types';
+/**
+ * Read a test resource by name
+ */
+export declare function readTestResourceFile(fileName: string): string;
+/**
+ * Parse a test resource into JSON
+ */
+export declare function readTestResourceAsJSON(fileName: string): any;
+/**
+ * Creates a temporary file
+ * @return path to the temporary file
+ */
+export declare function tempFile(data: string, postfix: string): Promise<string>;
+export interface TempConfigOptions {
+    readonly api_url: string;
+    readonly invalid_record_strategy?: InvalidRecordStrategy;
+    readonly edition?: Edition;
+    readonly edition_configs?: Dictionary<any>;
+    readonly source_specific_configs?: Dictionary<any>;
+    readonly replace_origin_map?: Dictionary<any>;
+    readonly exclude_fields_map?: Dictionary<any>;
+    readonly log_records?: boolean;
+}
+/**
+ * Creates a temporary file with testing configuration
+ * @return path to the temporary config file to delete
+ */
+export declare function tempConfig(options: TempConfigOptions): Promise<string>;
+export declare function getConf(options: TempConfigOptions): any;
+export declare function sourceSpecificTempConfig(url: string, source_specific_configs: Dictionary<any>): Promise<string>;
+export declare function initMockttp(mockttp: Mockttp): Promise<void>;
+export declare function testLogger(name?: string): pino.Logger;

package/lib/testing-tools/testing-tools.js

@@ -0,0 +1,138 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readTestResourceFile = readTestResourceFile;
+exports.readTestResourceAsJSON = readTestResourceAsJSON;
+exports.tempFile = tempFile;
+exports.tempConfig = tempConfig;
+exports.getConf = getConf;
+exports.sourceSpecificTempConfig = sourceSpecificTempConfig;
+exports.initMockttp = initMockttp;
+exports.testLogger = testLogger;
+const fs_1 = __importDefault(require("fs"));
+const pino_1 = __importDefault(require("pino"));
+const tmp_promise_1 = __importDefault(require("tmp-promise"));
+const util_1 = __importDefault(require("util"));
+const types_1 = require("../common/types");
+const TEST_SOURCE_ID = 'mytestsource';
+// Remove all controlled temporary objects on process exit
+tmp_promise_1.default.setGracefulCleanup();
+/**
+ * Read a test resource by name
+ */
+function readTestResourceFile(fileName) {
+    return fs_1.default.readFileSync(`test/resources/${fileName}`, 'utf8');
+}
+const writeFile = util_1.default.promisify(fs_1.default.write);
+/**
+ * Parse a test resource into JSON
+ */
+function readTestResourceAsJSON(fileName) {
+    return JSON.parse(readTestResourceFile(fileName));
+}
+/**
+ * Creates a temporary file
+ * @return path to the temporary file
+ */
+async function tempFile(data, postfix) {
+    const file = await tmp_promise_1.default.file({ postfix });
+    await writeFile(file.fd, data, null, 'utf-8');
+    return file.path;
+}
+/**
+ * Creates a temporary file with testing configuration
+ * @return path to the temporary config file to delete
+ */
+async function tempConfig(options) {
+    const conf = getConf(options);
+    return tempFile(JSON.stringify(conf), '.json');
+}
+function getConf(options) {
+    const { api_url, invalid_record_strategy = types_1.InvalidRecordStrategy.FAIL, edition = types_1.Edition.CLOUD, edition_configs = {}, source_specific_configs = {}, replace_origin_map = {}, exclude_fields_map = {}, log_records = false, } = options;
+    const edition_configs_defaults = edition === types_1.Edition.CLOUD
+        ? {
+            edition,
+            api_url,
+            api_key: 'test-api-key',
+            graph: 'test-graph',
+        }
+        : {
+            edition,
+            hasura_url: api_url,
+            segment_user_id: 'bacaf6e6-41d8-4102-a3a4-5d28100e642f',
+            segment_test_host: api_url,
+        };
+    const conf = {
+        edition_configs: { ...edition_configs_defaults, ...edition_configs },
+        invalid_record_strategy,
+        origin: 'test-origin',
+        jsonata_destination_models: ['generic_Record'],
+        jsonata_expression: `
+    data.{
+      "model": "generic_Record",
+      "record": {
+        "uid": foo
+      }
+    }`,
+        source_specific_configs,
+        replace_origin_map: JSON.stringify(replace_origin_map),
+        exclude_fields_map: JSON.stringify(exclude_fields_map),
+        faros_source_id: TEST_SOURCE_ID,
+        log_records,
+    };
+    return conf;
+}
+function sourceSpecificTempConfig(url, source_specific_configs) {
+    return tempConfig({ api_url: url, source_specific_configs });
+}
+async function initMockttp(mockttp) {
+    await mockttp.start({ startPort: 30000, endPort: 50000 });
+    // Faros GraphQL auth check
+    await mockttp
+        .forGet('/users/me')
+        .once()
+        .thenReply(200, JSON.stringify({ tenantId: '1' }));
+    // Faros GraphQL graph stats
+    await mockttp
+        .forGet('/graphs/test-graph/statistics')
+        .once()
+        .thenReply(200, JSON.stringify({}));
+    // Hasura health check
+    await mockttp.forGet('/healthz').once().thenReply(200, JSON.stringify({}));
+    // Get Faros Account
+    await mockttp
+        .forGet(`/accounts/${TEST_SOURCE_ID}`)
+        .once()
+        .thenReply(200, JSON.stringify({ account: { accountId: TEST_SOURCE_ID } }));
+    // Faros Account Sync
+    const mockSyncResult = {
+        sync: {
+            syncId: '1',
+            logId: '1',
+            startedAt: new Date().toISOString(),
+            status: 'running',
+        },
+    };
+    await mockttp
+        .forPut(`/accounts/${TEST_SOURCE_ID}/syncs`)
+        .once()
+        .thenReply(200, JSON.stringify(mockSyncResult));
+    await mockttp
+        .forPatch(`/accounts/${TEST_SOURCE_ID}/syncs/1`)
+        .once()
+        .thenReply(200, JSON.stringify(mockSyncResult));
+}
+function testLogger(name = 'test') {
+    var _a, _b;
+    return (0, pino_1.default)({
+        name,
+        level: (_a = process.env.LOG_LEVEL) !== null && _a !== void 0 ? _a : 'info',
+        // pino-pretty leaves threads open which can prevent Jest from exiting properly
+        transport: ((_b = process.env.LOG_LEVEL) === null || _b === void 0 ? void 0 : _b.toLowerCase()) === 'debug'
+            ? { target: 'pino-pretty', options: { levelFirst: true } }
+            : undefined,
+    });
+}
+//# sourceMappingURL=testing-tools.js.map

package/lib/testing-tools/testing-tools.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testing-tools.js","sourceRoot":"","sources":["../../src/testing-tools/testing-tools.ts"],"names":[],"mappings":";;;;;AAiBA,oDAEC;AAOD,wDAEC;AAMD,4BAIC;AAiBD,gCAIC;AAED,0BA+CC;AAED,4DAKC;AAED,kCA0CC;AAED,gCAUC;AA3KD,4CAAoB;AAEpB,gDAAwB;AACxB,8DAA8B;AAE9B,gDAAwB;AACxB,2CAAiE;AAGjE,MAAM,cAAc,GAAG,cAAc,CAAC;AAEtC,0DAA0D;AAC1D,qBAAG,CAAC,kBAAkB,EAAE,CAAC;AAEzB;;GAEG;AACH,SAAgB,oBAAoB,CAAC,QAAgB;IACnD,OAAO,YAAE,CAAC,YAAY,CAAC,kBAAkB,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,SAAS,GAAG,cAAI,CAAC,SAAS,CAAC,YAAE,CAAC,KAAK,CAAC,CAAC;AAE3C;;GAEG;AACH,SAAgB,sBAAsB,CAAC,QAAgB;IACrD,OAAO,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC,CAAC;AACpD,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,QAAQ,CAAC,IAAY,EAAE,OAAe;IAC1D,MAAM,IAAI,GAAG,MAAM,qBAAG,CAAC,IAAI,CAAC,EAAC,OAAO,EAAC,CAAC,CAAC;IACvC,MAAM,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,IAAI,CAAC;AACnB,CAAC;AAaD;;;GAGG;AACI,KAAK,UAAU,UAAU,CAAC,OAA0B;IACzD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE9B,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;AACjD,CAAC;AAED,SAAgB,OAAO,CAAC,OAA0B;IAChD,MAAM,EACJ,OAAO,EACP,uBAAuB,GAAG,6BAAqB,CAAC,IAAI,EACpD,OAAO,GAAG,eAAO,CAAC,KAAK,EACvB,eAAe,GAAG,EAAE,EACpB,uBAAuB,GAAG,EAAE,EAC5B,kBAAkB,GAAG,EAAE,EACvB,kBAAkB,GAAG,EAAE,EACvB,WAAW,GAAG,KAAK,GACpB,GAAG,OAAO,CAAC;IAEZ,MAAM,wBAAwB,GAC5B,OAAO,KAAK,eAAO,CAAC,KAAK;QACvB,CAAC,CAAC;YACE,OAAO;YACP,OAAO;YACP,OAAO,EAAE,cAAc;YACvB,KAAK,EAAE,YAAY;SACpB;QACH,CAAC,CAAC;YACE,OAAO;YACP,UAAU,EAAE,OAAO;YACnB,eAAe,EAAE,sCAAsC;YACvD,iBAAiB,EAAE,OAAO;SAC3B,CAAC;IAER,MAAM,IAAI,GAAG;QACX,eAAe,EAAE,EAAC,GAAG,wBAAwB,EAAE,GAAG,eAAe,EAAC;QAClE,uBAAuB;QACvB,MAAM,EAAE,aAAa;QACrB,0BAA0B,EAAE,CAAC,gBAAgB,CAAC;QAC9C,kBAAkB,EAAE;;;;;;MAMlB;QACF,uBAAuB;QACvB,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC;QACtD,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC;QACtD,eAAe,EAAE,cAAc;QAC/B,WAAW;KACZ,CAAC;IAEF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,wBAAwB,CACtC,GAAW,EACX,uBAAwC;IAExC,OAAO,UAAU,CAAC,EAAC,OAAO,EAAE,GAAG,EAAE,uBAAuB,EAAC,CAAC,CAAC;AAC7D,CAAC;AAEM,KAAK,UAAU,WAAW,CAAC,OAAgB;IAChD,MAAM,OAAO,CAAC,KAAK,CAAC,EAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAC,CAAC,CAAC;IAExD,2BAA2B;IAC3B,MAAM,OAAO;SACV,MAAM,CAAC,WAAW,CAAC;SACnB,IAAI,EAAE;SACN,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,EAAC,QAAQ,EAAE,GAAG,EAAC,CAAC,CAAC,CAAC;IAEnD,4BAA4B;IAC5B,MAAM,OAAO;SACV,MAAM,CAAC,+BAA+B,CAAC;SACvC,IAAI,EAAE;SACN,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;IAEtC,sBAAsB;IACtB,MAAM,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;IAE3E,oBAAoB;IACpB,MAAM,OAAO;SACV,MAAM,CAAC,aAAa,cAAc,EAAE,CAAC;SACrC,IAAI,EAAE;SACN,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,EAAC,OAAO,EAAE,EAAC,SAAS,EAAE,cAAc,EAAC,EAAC,CAAC,CAAC,CAAC;IAE1E,qBAAqB;IACrB,MAAM,cAAc,GAAG;QACrB,IAAI,EAAE;YACJ,MAAM,EAAE,GAAG;YACX,KAAK,EAAE,GAAG;YACV,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,SAAS;SAClB;KACF,CAAC;IACF,MAAM,OAAO;SACV,MAAM,CAAC,aAAa,cAAc,QAAQ,CAAC;SAC3C,IAAI,EAAE;SACN,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC;IAElD,MAAM,OAAO;SACV,QAAQ,CAAC,aAAa,cAAc,UAAU,CAAC;SAC/C,IAAI,EAAE;SACN,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,SAAgB,UAAU,CAAC,IAAI,GAAG,MAAM;;IACtC,OAAO,IAAA,cAAI,EAAC;QACV,IAAI;QACJ,KAAK,EAAE,MAAA,OAAO,CAAC,GAAG,CAAC,SAAS,mCAAI,MAAM;QACtC,+EAA+E;QAC/E,SAAS,EACP,CAAA,MAAA,OAAO,CAAC,GAAG,CAAC,SAAS,0CAAE,WAAW,EAAE,MAAK,OAAO;YAC9C,CAAC,CAAC,EAAC,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,EAAC,UAAU,EAAE,IAAI,EAAC,EAAC;YACtD,CAAC,CAAC,SAAS;KAChB,CAAC,CAAC;AACL,CAAC"}

package/lib/testing-tools/utils.d.ts

@@ -0,0 +1,15 @@
+import { Dictionary } from 'ts-essentials';
+export interface DestinationWriteTestOptions {
+    configPath: string;
+    catalogPath: string;
+    inputRecordsPath: string;
+    checkRecordsData?: (records: ReadonlyArray<Dictionary<any>>) => void;
+}
+export interface GenerateBasicTestSuiteOptions {
+    sourceName: string;
+    catalogPath?: string;
+    inputRecordsPath?: string;
+    checkRecordsData?: (records: ReadonlyArray<Dictionary<any>>) => void;
+}
+export declare const destinationWriteTest: (options: DestinationWriteTestOptions) => Promise<void>;
+export declare function generateBasicTestSuite({ sourceName, catalogPath, inputRecordsPath, checkRecordsData, }: GenerateBasicTestSuiteOptions): void;

package/lib/testing-tools/utils.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.destinationWriteTest = void 0;
+exports.generateBasicTestSuite = generateBasicTestSuite;
+const faros_airbyte_cdk_1 = require("faros-airbyte-cdk");
+const mockttp_1 = require("mockttp");
+const cli_1 = require("./cli");
+const testing_tools_1 = require("./testing-tools");
+// Executes the destination write command in dry-run mode and optionally checks:
+// - The processed and written records count
+// - The records data
+const destinationWriteTest = async (options) => {
+    const { configPath, catalogPath, inputRecordsPath, checkRecordsData = undefined, } = options;
+    const cli = await cli_1.CLI.runWith([
+        'write',
+        '--config',
+        configPath,
+        '--catalog',
+        catalogPath,
+        '--dry-run',
+    ]);
+    cli.stdin.end((0, testing_tools_1.readTestResourceFile)(inputRecordsPath), 'utf8');
+    const stdoutLines = await (0, cli_1.readLines)(cli.stdout);
+    const matches = [];
+    stdoutLines.forEach((line) => {
+        const regexes = [
+            /Processed (\d+) records/,
+            /Would write (\d+) records/,
+            /Errored (\d+) records/,
+            /Skipped (\d+) records/,
+            /Processed records by stream: {(.*)}/,
+            /Would write records by model: {(.*)}/,
+        ];
+        const matchedLine = regexes.find((regex) => regex.test(line));
+        if (matchedLine) {
+            matches.push(matchedLine.exec(line)[0]);
+        }
+    });
+    expect(matches).toMatchSnapshot();
+    if (checkRecordsData) {
+        const records = readRecordData(stdoutLines);
+        checkRecordsData(records);
+    }
+    expect(await (0, cli_1.read)(cli.stderr)).toBe('');
+    expect(await cli.wait()).toBe(0);
+};
+exports.destinationWriteTest = destinationWriteTest;
+function readRecordData(lines) {
+    const records = [];
+    for (const line of lines) {
+        try {
+            const msg = (0, faros_airbyte_cdk_1.parseAirbyteMessage)(line);
+            if (msg.type === faros_airbyte_cdk_1.AirbyteMessageType.RECORD) {
+                const recordMessage = msg;
+                records.push(recordMessage.record.data);
+            }
+        }
+        catch (error) {
+            // Not a record message. Ignore.
+        }
+    }
+    return records;
+}
+function generateBasicTestSuite({ sourceName, catalogPath = `test/resources/${sourceName}/catalog.json`, inputRecordsPath = `${sourceName}/all-streams.log`, checkRecordsData, }) {
+    describe(`${sourceName} basic test`, () => {
+        const mockttp = (0, mockttp_1.getLocal)({ debug: false, recordTraffic: false });
+        let configPath;
+        beforeEach(async () => {
+            await (0, testing_tools_1.initMockttp)(mockttp);
+            configPath = await (0, testing_tools_1.tempConfig)({
+                api_url: mockttp.url,
+                log_records: !!checkRecordsData,
+            });
+        });
+        afterEach(async () => {
+            await mockttp.stop();
+        });
+        test('process records from all streams', async () => {
+            await (0, exports.destinationWriteTest)({
+                configPath,
+                catalogPath,
+                inputRecordsPath,
+                checkRecordsData,
+            });
+        });
+    });
+}
+//# sourceMappingURL=utils.js.map

package/lib/testing-tools/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/testing-tools/utils.ts"],"names":[],"mappings":";;;AAgGA,wDA+BC;AA/HD,yDAI2B;AAC3B,qCAAiC;AAGjC,+BAA2C;AAC3C,mDAA8E;AAgB9E,gFAAgF;AAChF,4CAA4C;AAC5C,qBAAqB;AACd,MAAM,oBAAoB,GAAG,KAAK,EACvC,OAAoC,EACrB,EAAE;IACjB,MAAM,EACJ,UAAU,EACV,WAAW,EACX,gBAAgB,EAChB,gBAAgB,GAAG,SAAS,GAC7B,GAAG,OAAO,CAAC;IACZ,MAAM,GAAG,GAAG,MAAM,SAAG,CAAC,OAAO,CAAC;QAC5B,OAAO;QACP,UAAU;QACV,UAAU;QACV,WAAW;QACX,WAAW;QACX,WAAW;KACZ,CAAC,CAAC;IAEH,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAA,oCAAoB,EAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC,CAAC;IAE9D,MAAM,WAAW,GAAG,MAAM,IAAA,eAAS,EAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,MAAM,OAAO,GAAG;YACd,yBAAyB;YACzB,2BAA2B;YAC3B,uBAAuB;YACvB,uBAAuB;YACvB,qCAAqC;YACrC,sCAAsC;SACvC,CAAC;QAEF,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9D,IAAI,WAAW,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,CAAC;IAElC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QAC5C,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,CAAC,MAAM,IAAA,UAAI,EAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACnC,CAAC,CAAC;AAhDW,QAAA,oBAAoB,wBAgD/B;AAEF,SAAS,cAAc,CACrB,KAA4B;IAE5B,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAA,uCAAmB,EAAC,IAAI,CAAC,CAAC;YACtC,IAAI,GAAG,CAAC,IAAI,KAAK,sCAAkB,CAAC,MAAM,EAAE,CAAC;gBAC3C,MAAM,aAAa,GAAG,GAAoB,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gCAAgC;QAClC,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAgB,sBAAsB,CAAC,EACrC,UAAU,EACV,WAAW,GAAG,kBAAkB,UAAU,eAAe,EACzD,gBAAgB,GAAG,GAAG,UAAU,kBAAkB,EAClD,gBAAgB,GACc;IAC9B,QAAQ,CAAC,GAAG,UAAU,aAAa,EAAE,GAAG,EAAE;QACxC,MAAM,OAAO,GAAG,IAAA,kBAAQ,EAAC,EAAC,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAC,CAAC,CAAC;QAC/D,IAAI,UAAU,CAAC;QAEf,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,IAAA,2BAAW,EAAC,OAAO,CAAC,CAAC;YAC3B,UAAU,GAAG,MAAM,IAAA,0BAAU,EAAC;gBAC5B,OAAO,EAAE,OAAO,CAAC,GAAG;gBACpB,WAAW,EAAE,CAAC,CAAC,gBAAgB;aAChC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,KAAK,IAAI,EAAE;YACnB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,IAAA,4BAAoB,EAAC;gBACzB,UAAU;gBACV,WAAW;gBACX,gBAAgB;gBAChB,gBAAgB;aACjB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}