@flowcore/data-pump 0.21.4 → 0.23.0

package/package.json

@@ -1,39 +1,56 @@
 {
   "name": "@flowcore/data-pump",
-  "version": "0.21.4",
   "description": "Flowcore Data Pump",
+  "version": "0.23.0",
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/mod.d.ts",
+      "import": "./dist/mod.js"
+    }
+  },
+  "types": "./dist/mod.d.ts",
+  "files": [
+    "dist",
+    "README.md",
+    "CHANGELOG.md"
+  ],
+  "sideEffects": false,
   "homepage": "https://github.com/flowcore-io/flowcore-data-pump#readme",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/flowcore-io/flowcore-data-pump.git"
   },
-  "license": "MIT",
   "bugs": {
     "url": "https://github.com/flowcore-io/flowcore-data-pump/issues"
   },
-  "main": "./script/mod.js",
-  "module": "./esm/mod.js",
-  "exports": {
-    ".": {
-      "import": "./esm/mod.js",
-      "require": "./script/mod.js"
-    }
+  "scripts": {
+    "build": "tsup src/mod.ts --format esm --dts --clean --out-dir dist",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "lint": "tsc --noEmit",
+    "test": "bun test",
+    "typecheck": "tsc --noEmit"
   },
   "dependencies": {
     "@date-fns/utc": "^2.1.0",
+    "@flowcore/sdk": "^4.2.3",
+    "@flowcore/sdk-oidc-client": "^1.3.1",
+    "@flowcore/time-uuid": "^0.2.0",
     "@sinclair/typebox": "0.32.15",
     "date-fns": "^4.1.0",
-    "long": "^5.3.1",
     "nats": "^2.29.1",
     "prom-client": "^15.1.3",
-    "rxjs": "^7.8.1",
-    "@deno/shim-deno": "~0.18.0",
-    "ws": "^8.13.0"
+    "rxjs": "^7.8.1"
   },
   "devDependencies": {
-    "@types/node": "^20.9.0",
+    "@types/bun": "^1.2.19",
+    "@types/pg": "^8.11.10",
     "@types/ws": "^8.5.10",
-    "@types/long": "^5.0.0"
-  },
-  "_generatedBy": "dnt@dev"
-}
+    "pg": "^8.13.0",
+    "prettier": "^3.3.3",
+    "tsup": "^8.3.0",
+    "typescript": "^5.6.3"
+  }
+}

package/esm/_dnt.polyfills.d.ts

@@ -1,50 +0,0 @@
-declare global {
-    interface Array<T> {
-        /**
-         * Returns the value of the last element in the array where predicate is true, and undefined
-         * otherwise.
-         * @param predicate find calls predicate once for each element of the array, in ascending
-         * order, until it finds one where predicate returns true. If such an element is found, find
-         * immediately returns that element value. Otherwise, find returns undefined.
-         * @param thisArg If provided, it will be used as the this value for each invocation of
-         * predicate. If it is not provided, undefined is used instead.
-         */
-        findLast<S extends T>(predicate: (this: void, value: T, index: number, obj: T[]) => value is S, thisArg?: any): S | undefined;
-        findLast(predicate: (value: T, index: number, obj: T[]) => unknown, thisArg?: any): T | undefined;
-        /**
-         * Returns the index of the last element in the array where predicate is true, and -1
-         * otherwise.
-         * @param predicate find calls predicate once for each element of the array, in ascending
-         * order, until it finds one where predicate returns true. If such an element is found,
-         * findIndex immediately returns that element index. Otherwise, findIndex returns -1.
-         * @param thisArg If provided, it will be used as the this value for each invocation of
-         * predicate. If it is not provided, undefined is used instead.
-         */
-        findLastIndex(predicate: (value: T, index: number, obj: T[]) => unknown, thisArg?: any): number;
-    }
-    interface Uint8Array {
-        /**
-         * Returns the value of the last element in the array where predicate is true, and undefined
-         * otherwise.
-         * @param predicate findLast calls predicate once for each element of the array, in descending
-         * order, until it finds one where predicate returns true. If such an element is found, findLast
-         * immediately returns that element value. Otherwise, findLast returns undefined.
-         * @param thisArg If provided, it will be used as the this value for each invocation of
-         * predicate. If it is not provided, undefined is used instead.
-         */
-        findLast<S extends number>(predicate: (value: number, index: number, array: Uint8Array) => value is S, thisArg?: any): S | undefined;
-        findLast(predicate: (value: number, index: number, array: Uint8Array) => unknown, thisArg?: any): number | undefined;
-        /**
-         * Returns the index of the last element in the array where predicate is true, and -1
-         * otherwise.
-         * @param predicate findLastIndex calls predicate once for each element of the array, in descending
-         * order, until it finds one where predicate returns true. If such an element is found,
-         * findLastIndex immediately returns that element index. Otherwise, findLastIndex returns -1.
-         * @param thisArg If provided, it will be used as the this value for each invocation of
-         * predicate. If it is not provided, undefined is used instead.
-         */
-        findLastIndex(predicate: (value: number, index: number, array: Uint8Array) => unknown, thisArg?: any): number;
-    }
-}
-export {};
-//# sourceMappingURL=_dnt.polyfills.d.ts.map

package/esm/_dnt.polyfills.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"_dnt.polyfills.d.ts","sourceRoot":"","sources":["../src/_dnt.polyfills.ts"],"names":[],"mappings":"AACA,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,KAAK,CAAC,CAAC;QACf;;;;;;;;WAQG;QACH,QAAQ,CAAC,CAAC,SAAS,CAAC,EAClB,SAAS,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC,EACxE,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QACjB,QAAQ,CACN,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,OAAO,EACzD,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QAEjB;;;;;;;;WAQG;QACH,aAAa,CACX,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,OAAO,EACzD,OAAO,CAAC,EAAE,GAAG,GACZ,MAAM,CAAC;KACX;IACD,UAAU,UAAU;QAClB;;;;;;;;WAQG;QACH,QAAQ,CAAC,CAAC,SAAS,MAAM,EACvB,SAAS,EAAE,CACP,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,UAAU,KAChB,KAAK,IAAI,CAAC,EACf,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QACjB,QAAQ,CACJ,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,EACvE,OAAO,CAAC,EAAE,GAAG,GACd,MAAM,GAAG,SAAS,CAAC;QAEtB;;;;;;;;WAQG;QACH,aAAa,CACT,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,EACvE,OAAO,CAAC,EAAE,GAAG,GACd,MAAM,CAAC;KACX;CACF;AA4CD,OAAO,EAAE,CAAC"}

package/esm/_dnt.polyfills.js

@@ -1,37 +0,0 @@
-function findLastIndex(self, callbackfn, that) {
-    const boundFunc = that === undefined ? callbackfn : callbackfn.bind(that);
-    let index = self.length - 1;
-    while (index >= 0) {
-        const result = boundFunc(self[index], index, self);
-        if (result) {
-            return index;
-        }
-        index--;
-    }
-    return -1;
-}
-function findLast(self, callbackfn, that) {
-    const index = self.findLastIndex(callbackfn, that);
-    return index === -1 ? undefined : self[index];
-}
-if (!Array.prototype.findLastIndex) {
-    Array.prototype.findLastIndex = function (callbackfn, that) {
-        return findLastIndex(this, callbackfn, that);
-    };
-}
-if (!Array.prototype.findLast) {
-    Array.prototype.findLast = function (callbackfn, that) {
-        return findLast(this, callbackfn, that);
-    };
-}
-if (!Uint8Array.prototype.findLastIndex) {
-    Uint8Array.prototype.findLastIndex = function (callbackfn, that) {
-        return findLastIndex(this, callbackfn, that);
-    };
-}
-if (!Uint8Array.prototype.findLast) {
-    Uint8Array.prototype.findLast = function (callbackfn, that) {
-        return findLast(this, callbackfn, that);
-    };
-}
-export {};

package/esm/_dnt.shims.d.ts

@@ -1,9 +0,0 @@
-import { Deno } from "@deno/shim-deno";
-export { Deno } from "@deno/shim-deno";
-import { default as WebSocket } from "ws";
-export { default as WebSocket } from "ws";
-export declare const dntGlobalThis: Omit<typeof globalThis, "WebSocket" | "Deno"> & {
-    Deno: typeof Deno;
-    WebSocket: typeof WebSocket;
-};
-//# sourceMappingURL=_dnt.shims.d.ts.map

package/esm/_dnt.shims.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"_dnt.shims.d.ts","sourceRoot":"","sources":["../src/_dnt.shims.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EAAE,OAAO,IAAI,SAAS,EAAE,MAAM,IAAI,CAAC;AAC1C,OAAO,EAAE,OAAO,IAAI,SAAS,EAAE,MAAM,IAAI,CAAC;AAM1C,eAAO,MAAM,aAAa;;;CAA2C,CAAC"}

package/esm/_dnt.shims.js

@@ -1,64 +0,0 @@
-import { Deno } from "@deno/shim-deno";
-export { Deno } from "@deno/shim-deno";
-import { default as WebSocket } from "ws";
-export { default as WebSocket } from "ws";
-const dntGlobals = {
-    Deno,
-    WebSocket,
-};
-export const dntGlobalThis = createMergeProxy(globalThis, dntGlobals);
-function createMergeProxy(baseObj, extObj) {
-    return new Proxy(baseObj, {
-        get(_target, prop, _receiver) {
-            if (prop in extObj) {
-                return extObj[prop];
-            }
-            else {
-                return baseObj[prop];
-            }
-        },
-        set(_target, prop, value) {
-            if (prop in extObj) {
-                delete extObj[prop];
-            }
-            baseObj[prop] = value;
-            return true;
-        },
-        deleteProperty(_target, prop) {
-            let success = false;
-            if (prop in extObj) {
-                delete extObj[prop];
-                success = true;
-            }
-            if (prop in baseObj) {
-                delete baseObj[prop];
-                success = true;
-            }
-            return success;
-        },
-        ownKeys(_target) {
-            const baseKeys = Reflect.ownKeys(baseObj);
-            const extKeys = Reflect.ownKeys(extObj);
-            const extKeysSet = new Set(extKeys);
-            return [...baseKeys.filter((k) => !extKeysSet.has(k)), ...extKeys];
-        },
-        defineProperty(_target, prop, desc) {
-            if (prop in extObj) {
-                delete extObj[prop];
-            }
-            Reflect.defineProperty(baseObj, prop, desc);
-            return true;
-        },
-        getOwnPropertyDescriptor(_target, prop) {
-            if (prop in extObj) {
-                return Reflect.getOwnPropertyDescriptor(extObj, prop);
-            }
-            else {
-                return Reflect.getOwnPropertyDescriptor(baseObj, prop);
-            }
-        },
-        has(_target, prop) {
-            return prop in extObj || prop in baseObj;
-        },
-    });
-}

package/esm/data-pump/data-pump-cluster.d.ts

@@ -1,91 +0,0 @@
-import * as dntShim from "../_dnt.shims.js";
-import { type FlowcoreDataPumpOptions } from "./data-pump.js";
-import type { FlowcoreDataSource } from "./data-source.js";
-import type { FlowcoreDataPumpCoordinator } from "./types.js";
-export interface FlowcoreDataPumpClusterOptions extends FlowcoreDataPumpOptions {
-    coordinator: FlowcoreDataPumpCoordinator;
-    advertisedAddress?: string;
-    dataSourceOverride?: FlowcoreDataSource;
-    leaseTtlMs?: number;
-    leaseRenewIntervalMs?: number;
-    heartbeatIntervalMs?: number;
-    workerConcurrency?: number;
-    clusterKey?: string;
-}
-export declare class FlowcoreDataPumpCluster {
-    private readonly options;
-    private readonly instanceId;
-    private readonly coordinator;
-    private readonly leaseTtlMs;
-    private readonly leaseRenewIntervalMs;
-    private readonly heartbeatIntervalMs;
-    private readonly workerConcurrency;
-    private readonly logger?;
-    private running;
-    private isLeader;
-    private pump?;
-    private leaderConnection?;
-    private workers;
-    private workerRoundRobinIndex;
-    private leaseRenewInterval?;
-    private discoveryInterval?;
-    private heartbeatInterval?;
-    private electionInterval?;
-    private workerHandler?;
-    private workerFailedHandler?;
-    private pumpRestartAttempts;
-    private natsConnectionManager?;
-    private natsDistLeader?;
-    private natsDistWorker?;
-    constructor(options: FlowcoreDataPumpClusterOptions);
-    private get useNatsDistribution();
-    private get natsDistributionSubject();
-    get isRunning(): boolean;
-    get isLeaderInstance(): boolean;
-    get activeWorkerCount(): number;
-    get id(): string;
-    /**
-     * Handle an incoming WebSocket connection from a peer.
-     * Call this from your HTTP server's WebSocket upgrade handler.
-     *
-     * Example (Deno):
-     * ```typescript
-     * Deno.serve({ port: 8080 }, (req) => {
-     *   if (req.headers.get("upgrade") === "websocket") {
-     *     const { socket, response } = Deno.upgradeWebSocket(req)
-     *     cluster.handleConnection(socket)
-     *     return response
-     *   }
-     *   return new Response("Not found", { status: 404 })
-     * })
-     * ```
-     *
-     * Example (Node.js with ws):
-     * ```typescript
-     * import { WebSocketServer } from "ws"
-     * const wss = new WebSocketServer({ port: 8080 })
-     * wss.on("connection", (ws) => cluster.handleConnection(ws))
-     * ```
-     */
-    handleConnection(ws: dntShim.WebSocket): void;
-    start(): Promise<void>;
-    stop(): Promise<void>;
-    private startHeartbeat;
-    private startElectionLoop;
-    private attemptElection;
-    private becomeLeader;
-    private stepDownAsLeader;
-    private startPumpAsLeader;
-    private distributeEvents;
-    private distributeEventsNats;
-    private distributeEventsWs;
-    private selectWorker;
-    private startWorkerDiscovery;
-    private discoverWorkers;
-    private connectToWorker;
-    private handleWorkerDisconnect;
-    private scheduleReconnect;
-    private handleLeaderMessage;
-    private handleWorkerMessage;
-}
-//# sourceMappingURL=data-pump-cluster.d.ts.map

package/esm/data-pump/data-pump-cluster.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"data-pump-cluster.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-pump-cluster.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAoB,KAAK,uBAAuB,EAAE,MAAM,gBAAgB,CAAA;AAC/E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAI1D,OAAO,KAAK,EAAE,2BAA2B,EAAkB,MAAM,YAAY,CAAA;AAc7E,MAAM,WAAW,8BAA+B,SAAQ,uBAAuB;IAC7E,WAAW,EAAE,2BAA2B,CAAA;IACxC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,kBAAkB,CAAC,EAAE,kBAAkB,CAAA;IACvC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,oBAAoB,CAAC,EAAE,MAAM,CAAA;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAUD,qBAAa,uBAAuB;IAmCtB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAlCpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA6B;IACzD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IAC5C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAQ;IAC1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAgB;IAExC,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,IAAI,CAAC,CAAkB;IAC/B,OAAO,CAAC,gBAAgB,CAAC,CAAc;IAGvC,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,qBAAqB,CAAI;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAgC;IAC3D,OAAO,CAAC,iBAAiB,CAAC,CAAgC;IAG1D,OAAO,CAAC,iBAAiB,CAAC,CAAgC;IAC1D,OAAO,CAAC,gBAAgB,CAAC,CAAgC;IAGzD,OAAO,CAAC,aAAa,CAAC,CAA4C;IAClE,OAAO,CAAC,mBAAmB,CAAC,CAAmD;IAE/E,OAAO,CAAC,mBAAmB,CAAI;IAG/B,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IACrD,OAAO,CAAC,cAAc,CAAC,CAAwB;IAC/C,OAAO,CAAC,cAAc,CAAC,CAAwB;gBAElB,OAAO,EAAE,8BAA8B;IAmBpE,OAAO,KAAK,mBAAmB,GAE9B;IAED,OAAO,KAAK,uBAAuB,GAIlC;IAED,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,IAAI,EAAE,IAAI,MAAM,CAEf;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,gBAAgB,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,GAAG,IAAI;IAevC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAuCtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD3B,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,iBAAiB;YAyBX,eAAe;YAgBf,YAAY;IA0B1B,OAAO,CAAC,gBAAgB;IAqCxB,OAAO,CAAC,iBAAiB;IA4BzB,OAAO,CAAC,gBAAgB;YAOV,oBAAoB;YA2BpB,kBAAkB;IAqChC,OAAO,CAAC,YAAY;IAoBpB,OAAO,CAAC,oBAAoB;YAOd,eAAe;IA8B7B,OAAO,CAAC,eAAe;IAqCvB,OAAO,CAAC,sBAAsB;IAiB9B,OAAO,CAAC,iBAAiB;IAiCzB,OAAO,CAAC,mBAAmB;IAY3B,OAAO,CAAC,mBAAmB;CA6B5B"}