@stilyng94/athena-query-client 1.0.1 → 1.1.0

package/README.md

@@ -25,9 +25,9 @@ import {
   JsonFileAppender,
 } from "@stilyng94/athena-query-client";
 
-const query = "SELECT * from api_logs";
+const athenaQuery = "SELECT * from api_logs";
 
-const execute = async (query) => {
+const execute = async (query: string) => {
   const athena = new AthenaQueryClient({
     Catalog: "catalog",
     Database: "logs",
@@ -37,7 +37,7 @@ const execute = async (query) => {
     s3Region: "us-west-2",
   });
 
-  const queryExecutionId = await athena.query();
+  const queryExecutionId = await athena.query(query);
 
   const jsonFileAppender = new JsonFileAppender({
     directory: "",
@@ -60,7 +60,7 @@ const execute = async (query) => {
     .catch((error) => console.error(error));
 };
 
-execute().catch(console.error);
+execute(athenaQuery).catch(console.error);
 ```
 
 ## Get query results and map to Js object
@@ -71,9 +71,9 @@ import {
   MappedQueryResultProcessor,
 } from "@stilyng94/athena-query-client";
 
-const query = "SELECT * from api_logs";
+const athenaQuery = "SELECT * from api_logs";
 
-const execute = async (query) => {
+const execute = async (query: string) => {
   const athena = new AthenaQueryClient({
     Catalog: "catalog",
     Database: "logs",
@@ -83,7 +83,7 @@ const execute = async (query) => {
     s3Region: "us-west-2",
   });
 
-  const queryExecutionId = await athena.query();
+  const queryExecutionId = await athena.query(query);
 
   const mappedQueryResultProcessor = new MappedQueryResultProcessor({
     athenaClient: athena,
@@ -93,5 +93,5 @@ const execute = async (query) => {
   });
 };
 
-execute().catch(console.error);
+execute(athenaQuery).catch(console.error);
 ```

package/dist/demo.js

@@ -1,15 +1,32 @@
-import { AthenaQueryClient } from "./athena.js";
-const client = new AthenaQueryClient({
-    Catalog: "",
-    s3OutputLocation: "s3://sjka/s",
-    s3Region: "us-east-1",
-    Database: "",
-    ClientConfig: {
-        region: "us-east-1",
-    },
-});
-client
-    .query("select * from db")
-    .then((d) => console.log(d))
-    .then((e) => console.error({ e }));
+import { DemoS3QueryResultProcessor } from "./demo.processor.js";
+import { JsonFileAppender } from "./json-file-appender.js";
+import { processToJson } from "./utils.js";
+const execute = async () => {
+    const fileNamesSet = new Set();
+    const s3QueryResultProcessor = new DemoS3QueryResultProcessor({
+        s3OutputLocation: "s3://results/queries",
+        s3Region: "us-west-2",
+        batchSize: 999,
+        onData: async (rows) => {
+            const fileNames = await processToJson(rows, "conversions");
+            // biome-ignore lint/complexity/noForEach: <explanation>
+            fileNames.forEach((fileName) => {
+                fileNamesSet.add(fileName);
+            });
+        },
+        onComplete: async () => {
+            await Promise.all(Array.from(fileNamesSet).map((fileName) => {
+                const jsonFileAppender = new JsonFileAppender({
+                    fileName,
+                    directory: "conversions",
+                });
+                return jsonFileAppender.closeFileWithBracket();
+            }));
+        },
+    });
+    s3QueryResultProcessor
+        .processResults("094cedb4-8739-4957-81c1-ecf3e4d8c783")
+        .catch((error) => console.error(error));
+};
+execute().catch(console.error);
 //# sourceMappingURL=demo.js.map

package/dist/demo.js.map

@@ -1 +1 @@
-{"version":3,"file":"demo.js","sourceRoot":"","sources":["../src/demo.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAAC;IACnC,OAAO,EAAE,EAAE;IACX,gBAAgB,EAAE,aAAa;IAC/B,QAAQ,EAAE,WAAW;IACrB,QAAQ,EAAE,EAAE;IACZ,YAAY,EAAE;QACZ,MAAM,EAAE,WAAW;KACpB;CACF,CAAC,CAAC;AACH,MAAM;KACH,KAAK,CAAC,kBAAkB,CAAC;KACzB,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;KAC3B,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC"}
+{"version":3,"file":"demo.js","sourceRoot":"","sources":["../src/demo.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE;IACzB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IAEvC,MAAM,sBAAsB,GAAG,IAAI,0BAA0B,CAAC;QAC5D,gBAAgB,EAAE,sBAAsB;QACxC,QAAQ,EAAE,WAAW;QACrB,SAAS,EAAE,GAAG;QACd,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACrB,MAAM,SAAS,GAAG,MAAM,aAAa,CACnC,IASE,EACF,aAAa,CACd,CAAC;YACF,wDAAwD;YACxD,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAC7B,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC7B,CAAC,CAAC,CAAC;QACL,CAAC;QACD,UAAU,EAAE,KAAK,IAAI,EAAE;YACrB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACxC,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,CAAC;oBAC5C,QAAQ;oBACR,SAAS,EAAE,aAAa;iBACzB,CAAC,CAAC;gBACH,OAAO,gBAAgB,CAAC,oBAAoB,EAAE,CAAC;YACjD,CAAC,CAAC,CACH,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;IACH,sBAAsB;SACnB,cAAc,CAAC,sCAAsC,CAAC;SACtD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,OAAO,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC"}

package/dist/demo.processor.d.ts

@@ -0,0 +1,88 @@
+import type { Readable } from "node:stream";
+import { type ResultSet, type Row } from "@aws-sdk/client-athena";
+import { type MappedQueryResultProcessorParams, type QueryResultProcessor, type S3QueryResultProcessorParams } from "./types.js";
+/**
+ * Processes Athena query results stored in S3
+ */
+export declare class DemoS3QueryResultProcessor implements QueryResultProcessor {
+    private readonly config;
+    readonly batchSize: number;
+    readonly s3OutputLocation: string;
+    /**
+     * Creates a new S3QueryResultProcessor
+     * @param {S3QueryResultProcessorParams} config - Configuration parameters
+     */
+    constructor(config: S3QueryResultProcessorParams);
+    /**
+     * Processes query results from S3
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<void>} Promise that resolves when processing is complete
+     */
+    processResults(queryExecutionId: string): Promise<void>;
+    /**
+     * Processes streaming results from S3
+     * @param {Readable} stream - The readable stream of results
+     * @returns {Promise<void>} Promise that resolves when processing is complete
+     */
+    processStreamingResults(stream: Readable): Promise<void>;
+    /**
+     * Processes a batch of records
+     * @param {unknown[]} batch - Array of records to process
+     * @returns {Promise<void>} Promise that resolves when batch is processed
+     */
+    processBatch(batch: unknown[]): Promise<void>;
+    /**
+     * Validates the batch size configuration
+     * @param {number} [batchSize] - The batch size to validate
+     * @returns {number} The validated batch size
+     * @throws {Error} If batch size exceeds maximum allowed
+     */
+    validateBatchSize(batchSize?: number): number;
+}
+/**
+ * Processes Athena query results by mapping them to objects
+ */
+export declare class MappedQueryResultProcessor implements QueryResultProcessor {
+    private readonly config;
+    /**
+     * Creates a new MappedQueryResultProcessor
+     * @param {MappedQueryResultProcessorParams} config - Configuration parameters
+     */
+    constructor(config: MappedQueryResultProcessorParams);
+    /**
+     * Processes query results by mapping them to objects
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<Record<string, string>[]>} Promise resolving to array of mapped objects
+     */
+    processResults(queryExecutionId: string): Promise<Record<string, string>[]>;
+    /**
+     * Fetches query results from Athena
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<ResultSet>} Promise resolving to Athena ResultSet
+     * @throws {Error} If results are empty or undefined
+     */
+    fetchQueryResults(queryExecutionId: string): Promise<ResultSet>;
+    /**
+     * Extracts column headers from result rows
+     * @param {Row[]} rows - Array of result rows
+     * @returns {string[]} Array of header names
+     * @throws {Error} If no headers are found
+     */
+    extractHeaders(rows: Row[]): string[];
+    /**
+     * Maps a row to an object using column headers as keys
+     * @param {Row} row - The row to map
+     * @param {string[]} headers - Array of column headers
+     * @returns {Record<string, string>} Object with header-value pairs
+     */
+    mapRowToObject(row: Row, headers: string[]): Record<string, string>;
+    /**
+     * Extracts  Athena ResultSet data into an array of key-value objects
+     * Extracts Athena ResultSet data into an array of key-value objects
+     * @private
+     * @param {ResultSet} resultSet - The ResultSet object returned by Athena
+     * @returns {Record<string, string>[]} Array of objects representing query rows
+     * @throws {Error} If no headers are found
+     */
+    extractRows(resultSet: ResultSet): Record<string, string>[];
+}

package/dist/demo.processor.js

@@ -0,0 +1,192 @@
+import { createReadStream } from "node:fs";
+import path from "node:path";
+import { cwd } from "node:process";
+import { finished } from "node:stream/promises";
+import { GetQueryResultsCommand, } from "@aws-sdk/client-athena";
+import { parse } from "csv-parse";
+import { MAX_BATCH_SIZE, } from "./types.js";
+/**
+ * Processes Athena query results stored in S3
+ */
+export class DemoS3QueryResultProcessor {
+    config;
+    batchSize = MAX_BATCH_SIZE;
+    s3OutputLocation;
+    /**
+     * Creates a new S3QueryResultProcessor
+     * @param {S3QueryResultProcessorParams} config - Configuration parameters
+     */
+    constructor(config) {
+        this.config = config;
+        this.batchSize = this.validateBatchSize(config.batchSize);
+        this.s3OutputLocation = config.s3OutputLocation;
+    }
+    /**
+     * Processes query results from S3
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<void>} Promise that resolves when processing is complete
+     */
+    async processResults(queryExecutionId) {
+        console.log("Fetching query results from S3:", queryExecutionId);
+        const responseStream = createReadStream(path.join(cwd(), "athena.csv"));
+        return this.processStreamingResults(responseStream);
+    }
+    /**
+     * Processes streaming results from S3
+     * @param {Readable} stream - The readable stream of results
+     * @returns {Promise<void>} Promise that resolves when processing is complete
+     */
+    async processStreamingResults(stream) {
+        const batch = [];
+        let totalProcessed = 0;
+        const parser = stream.pipe(parse({ ...(this.config.csvParseOptions ?? { columns: true }) }));
+        try {
+            parser.on("readable", async () => {
+                let record = parser.read();
+                while (record !== null) {
+                    // Work with each record
+                    batch.push(record);
+                    if (batch.length >= this.batchSize) {
+                        console.log(`Processing batch of ${batch.length} records...`);
+                        await this.processBatch(batch);
+                        totalProcessed += batch.length;
+                        batch.length = 0; // Clear array efficiently
+                        console.log(`Total records processed: ${totalProcessed}`);
+                    }
+                    record = parser.read();
+                }
+            });
+            // Wait for parsing to complete
+            await finished(parser);
+            // Process remaining records
+            if (batch.length > 0) {
+                console.log(`Processing final batch of ${batch.length} records...`);
+                await this.processBatch(batch);
+                totalProcessed += batch.length;
+                console.log(`Final total records processed: ${totalProcessed}`);
+            }
+            await this.config?.onComplete?.();
+        }
+        catch (error) {
+            throw new Error(`Error processing results: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    }
+    /**
+     * Processes a batch of records
+     * @param {unknown[]} batch - Array of records to process
+     * @returns {Promise<void>} Promise that resolves when batch is processed
+     */
+    async processBatch(batch) {
+        try {
+            await this.config.onData(batch);
+        }
+        catch (error) {
+            throw new Error(`Error processing batch: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    }
+    /**
+     * Validates the batch size configuration
+     * @param {number} [batchSize] - The batch size to validate
+     * @returns {number} The validated batch size
+     * @throws {Error} If batch size exceeds maximum allowed
+     */
+    validateBatchSize(batchSize) {
+        if (!batchSize)
+            return MAX_BATCH_SIZE;
+        if (batchSize > MAX_BATCH_SIZE) {
+            throw new Error(`Batch size cannot be greater than ${MAX_BATCH_SIZE}`);
+        }
+        return batchSize;
+    }
+}
+/**
+ * Processes Athena query results by mapping them to objects
+ */
+export class MappedQueryResultProcessor {
+    config;
+    /**
+     * Creates a new MappedQueryResultProcessor
+     * @param {MappedQueryResultProcessorParams} config - Configuration parameters
+     */
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Processes query results by mapping them to objects
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<Record<string, string>[]>} Promise resolving to array of mapped objects
+     */
+    async processResults(queryExecutionId) {
+        console.log("Fetching results for QueryExecutionId:", queryExecutionId);
+        try {
+            const resultSet = await this.fetchQueryResults(queryExecutionId);
+            return this.extractRows(resultSet);
+        }
+        catch (error) {
+            throw new Error(`Error processing results: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    }
+    /**
+     * Fetches query results from Athena
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<ResultSet>} Promise resolving to Athena ResultSet
+     * @throws {Error} If results are empty or undefined
+     */
+    async fetchQueryResults(queryExecutionId) {
+        const response = await this.config.athenaClient.send(new GetQueryResultsCommand({ QueryExecutionId: queryExecutionId }));
+        if (!response.ResultSet) {
+            throw new Error("Query results are empty or undefined");
+        }
+        return response.ResultSet;
+    }
+    /**
+     * Extracts column headers from result rows
+     * @param {Row[]} rows - Array of result rows
+     * @returns {string[]} Array of header names
+     * @throws {Error} If no headers are found
+     */
+    extractHeaders(rows) {
+        const headers = rows[0]?.Data?.map((column) => column.VarCharValue || "");
+        if (!headers || headers.length === 0) {
+            throw new Error("No headers found in the result set");
+        }
+        return headers;
+    }
+    /**
+     * Maps a row to an object using column headers as keys
+     * @param {Row} row - The row to map
+     * @param {string[]} headers - Array of column headers
+     * @returns {Record<string, string>} Object with header-value pairs
+     */
+    mapRowToObject(row, headers) {
+        const mappedRow = {};
+        row.Data?.forEach((value, index) => {
+            const header = headers[index];
+            if (header) {
+                mappedRow[header] = value.VarCharValue || "";
+            }
+        });
+        return mappedRow;
+    }
+    /**
+     * Extracts  Athena ResultSet data into an array of key-value objects
+     * Extracts Athena ResultSet data into an array of key-value objects
+     * @private
+     * @param {ResultSet} resultSet - The ResultSet object returned by Athena
+     * @returns {Record<string, string>[]} Array of objects representing query rows
+     * @throws {Error} If no headers are found
+     */
+    extractRows(resultSet) {
+        const { Rows } = resultSet;
+        if (!Rows || Rows.length === 0) {
+            console.error("No rows found in result set");
+            return [];
+        }
+        const headers = this.extractHeaders(Rows);
+        if (!headers.length) {
+            throw new Error("No headers found in the result set");
+        }
+        return Rows.slice(1).map((row) => this.mapRowToObject(row, headers));
+    }
+}
+//# sourceMappingURL=demo.processor.js.map

package/dist/demo.processor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"demo.processor.js","sourceRoot":"","sources":["../src/demo.processor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EACL,sBAAsB,GAGvB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,EACL,cAAc,GAIf,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAC;AAEnC;;GAEG;AACH,MAAM,OAAO,0BAA0B;IAQR;IAPpB,SAAS,GAAG,cAAc,CAAC;IAC3B,gBAAgB,CAAS;IAElC;;;OAGG;IACH,YAA6B,MAAoC;QAApC,WAAM,GAAN,MAAM,CAA8B;QAC/D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1D,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAClD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAAC,gBAAwB;QAC3C,OAAO,CAAC,GAAG,CAAC,iCAAiC,EAAE,gBAAgB,CAAC,CAAC;QAEjE,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;QACxE,OAAO,IAAI,CAAC,uBAAuB,CAAC,cAAc,CAAC,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,uBAAuB,CAAC,MAAgB;QAC5C,MAAM,KAAK,GAAc,EAAE,CAAC;QAC5B,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,KAAK,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CACjE,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE;gBAC/B,IAAI,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3B,OAAO,MAAM,KAAK,IAAI,EAAE,CAAC;oBACvB,wBAAwB;oBACxB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACnB,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;wBACnC,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,CAAC,MAAM,aAAa,CAAC,CAAC;wBAC9D,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;wBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAC;wBAC/B,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,0BAA0B;wBAC5C,OAAO,CAAC,GAAG,CAAC,4BAA4B,cAAc,EAAE,CAAC,CAAC;oBAC5D,CAAC;oBACD,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,+BAA+B;YAC/B,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;YACvB,4BAA4B;YAC5B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,6BAA6B,KAAK,CAAC,MAAM,aAAa,CAAC,CAAC;gBACpE,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAC;gBAC/B,OAAO,CAAC,GAAG,CAAC,kCAAkC,cAAc,EAAE,CAAC,CAAC;YAClE,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,6BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,KAAgB;QACjC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,2BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,iBAAiB,CAAC,SAAkB;QAClC,IAAI,CAAC,SAAS;YAAE,OAAO,cAAc,CAAC;QACtC,IAAI,SAAS,GAAG,cAAc,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,qCAAqC,cAAc,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,0BAA0B;IAKR;IAJ7B;;;OAGG;IACH,YAA6B,MAAwC;QAAxC,WAAM,GAAN,MAAM,CAAkC;IAAG,CAAC;IAEzE;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAClB,gBAAwB;QAExB,OAAO,CAAC,GAAG,CAAC,wCAAwC,EAAE,gBAAgB,CAAC,CAAC;QAExE,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,6BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,iBAAiB,CAAC,gBAAwB;QAC9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAClD,IAAI,sBAAsB,CAAC,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,CAAC,CACnE,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,OAAO,QAAQ,CAAC,SAAS,CAAC;IAC5B,CAAC;IAED;;;;;OAKG;IACH,cAAc,CAAC,IAAW;QACxB,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;QAE1E,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACH,cAAc,CAAC,GAAQ,EAAE,OAAiB;QACxC,MAAM,SAAS,GAA2B,EAAE,CAAC;QAE7C,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACjC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,MAAM,EAAE,CAAC;gBACX,SAAS,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC;YAC/C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;;OAOG;IACH,WAAW,CAAC,SAAoB;QAC9B,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;IACvE,CAAC;CACF"}

package/dist/query-results-processor.js

@@ -42,10 +42,10 @@ export class S3QueryResultProcessor {
     async #processStreamingResults(stream) {
         const batch = [];
         let totalProcessed = 0;
-        const parser = stream.pipe(parse({ ...(this.config.csvParseOptions ?? {}) }));
+        const parser = stream.pipe(parse({ columns: true, ...(this.config.csvParseOptions ?? {}) }));
         try {
             parser.on('readable', async () => {
-                const record = parser.read();
+                let record = parser.read();
                 while (record !== null) {
                     // Work with each record
                     batch.push(record);
@@ -56,6 +56,7 @@ export class S3QueryResultProcessor {
                         batch.length = 0; // Clear array efficiently
                         console.log(`Total records processed: ${totalProcessed}`);
                     }
+                    record = parser.read();
                 }
             });
             // Wait for parsing to complete

package/dist/query-results-processor.js.map

@@ -1 +1 @@
-{"version":3,"file":"query-results-processor.js","sourceRoot":"","sources":["../src/query-results-processor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EACL,sBAAsB,GAGvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC/D,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EACL,cAAc,GAIf,MAAM,YAAY,CAAA;AAEnB;;GAEG;AACH,MAAM,OAAO,sBAAsB;IASJ;IARpB,UAAU,GAAG,cAAc,CAAA;IAC3B,SAAS,CAAU;IACnB,iBAAiB,CAAQ;IAElC;;;OAGG;IACH,YAA6B,MAAoC;QAApC,WAAM,GAAN,MAAM,CAA8B;QAC/D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QAC3D,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,gBAAgB,CAAA;QAChD,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC5D,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAAC,gBAAwB;QAC3C,iCAAiC;QACjC,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,iBAAiB,IAAI,gBAAgB,MAAM,CAAA;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAA;QACpD,OAAO,CAAC,GAAG,CAAC,iCAAiC,EAAE,UAAU,CAAC,CAAA;QAE1D,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC7D,OAAO,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,wBAAwB,CAAC,MAAgB;QAC7C,MAAM,KAAK,GAAc,EAAE,CAAA;QAC3B,IAAI,cAAc,GAAG,CAAC,CAAA;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,KAAK,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,EAAE,CAAC,CAClD,CAAA;QAED,IAAI,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE;gBAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;gBAC5B,OAAO,MAAM,KAAK,IAAI,EAAE,CAAC;oBACvB,wBAAwB;oBACxB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;oBAClB,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpC,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,CAAC,MAAM,aAAa,CAAC,CAAA;wBAC7D,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;wBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAA;wBAC9B,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA,CAAC,0BAA0B;wBAC3C,OAAO,CAAC,GAAG,CAAC,4BAA4B,cAAc,EAAE,CAAC,CAAA;oBAC3D,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAA;YAEF,+BAA+B;YAC/B,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAA;YACtB,4BAA4B;YAC5B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,6BAA6B,KAAK,CAAC,MAAM,aAAa,CAAC,CAAA;gBACnE,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;gBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAA;gBAC9B,OAAO,CAAC,GAAG,CAAC,kCAAkC,cAAc,EAAE,CAAC,CAAA;YACjE,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAA;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,6BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAAC,KAAgB;QAClC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,2BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,kBAAkB,CAAC,SAAkB;QACnC,IAAI,CAAC,SAAS;YAAE,OAAO,cAAc,CAAA;QACrC,IAAI,SAAS,GAAG,cAAc,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,qCAAqC,cAAc,EAAE,CAAC,CAAA;QACxE,CAAC;QACD,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,MAAc,EAAE,GAAW;QAC9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CACxC,IAAI,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CACtC,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,IAAI,GAAG,EAAE,CAAC,CAAA;QAC3D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAA;QAExD,OAAO,QAAQ,CAAC,IAAgB,CAAA;IAClC,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,GAAW;QACrB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;YAC9B,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YAE/C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;YACxD,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;aACjC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,mBACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,0BAA0B;IAKR;IAJ7B;;;OAGG;IACH,YAA6B,MAAwC;QAAxC,WAAM,GAAN,MAAM,CAAkC;IAAG,CAAC;IAEzE;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAClB,gBAAwB;QAExB,OAAO,CAAC,GAAG,CAAC,wCAAwC,EAAE,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAA;YACjE,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAA;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,6BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,gBAAwB;QAC/C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAClD,IAAI,sBAAsB,CAAC,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,CAAC,CACnE,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QACzD,CAAC;QAED,OAAO,QAAQ,CAAC,SAAS,CAAA;IAC3B,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,IAAW;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAQ,EAAE,OAAiB;QACzC,MAAM,SAAS,GAA2B,EAAE,CAAA;QAE5C,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACjC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;YAC7B,IAAI,MAAM,EAAE,CAAC;gBACX,SAAS,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,IAAI,EAAE,CAAA;YAC9C,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;;;;OAOG;IACH,YAAY,CAAC,SAAoB;QAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;YAC5C,OAAO,EAAE,CAAA;QACX,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAC1C,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAA;IACvE,CAAC;CACF"}
+{"version":3,"file":"query-results-processor.js","sourceRoot":"","sources":["../src/query-results-processor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EACL,sBAAsB,GAGvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC/D,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EACL,cAAc,GAIf,MAAM,YAAY,CAAA;AAEnB;;GAEG;AACH,MAAM,OAAO,sBAAsB;IASJ;IARpB,UAAU,GAAG,cAAc,CAAA;IAC3B,SAAS,CAAU;IACnB,iBAAiB,CAAQ;IAElC;;;OAGG;IACH,YAA6B,MAAoC;QAApC,WAAM,GAAN,MAAM,CAA8B;QAC/D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QAC3D,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,gBAAgB,CAAA;QAChD,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC5D,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAAC,gBAAwB;QAC3C,iCAAiC;QACjC,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,iBAAiB,IAAI,gBAAgB,MAAM,CAAA;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAA;QACpD,OAAO,CAAC,GAAG,CAAC,iCAAiC,EAAE,UAAU,CAAC,CAAA;QAE1D,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC7D,OAAO,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,wBAAwB,CAAC,MAAgB;QAC7C,MAAM,KAAK,GAAc,EAAE,CAAA;QAC3B,IAAI,cAAc,GAAG,CAAC,CAAA;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,EAAE,CAAC,CACjE,CAAA;QAED,IAAI,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE;gBAC/B,IAAI,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;gBAC1B,OAAO,MAAM,KAAK,IAAI,EAAE,CAAC;oBACvB,wBAAwB;oBACxB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;oBAClB,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpC,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,CAAC,MAAM,aAAa,CAAC,CAAA;wBAC7D,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;wBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAA;wBAC9B,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA,CAAC,0BAA0B;wBAC3C,OAAO,CAAC,GAAG,CAAC,4BAA4B,cAAc,EAAE,CAAC,CAAA;oBAC3D,CAAC;oBACD,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;gBACxB,CAAC;YACH,CAAC,CAAC,CAAA;YAEF,+BAA+B;YAC/B,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAA;YACtB,4BAA4B;YAC5B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,6BAA6B,KAAK,CAAC,MAAM,aAAa,CAAC,CAAA;gBACnE,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;gBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAA;gBAC9B,OAAO,CAAC,GAAG,CAAC,kCAAkC,cAAc,EAAE,CAAC,CAAA;YACjE,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAA;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,6BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAAC,KAAgB;QAClC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,2BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,kBAAkB,CAAC,SAAkB;QACnC,IAAI,CAAC,SAAS;YAAE,OAAO,cAAc,CAAA;QACrC,IAAI,SAAS,GAAG,cAAc,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,qCAAqC,cAAc,EAAE,CAAC,CAAA;QACxE,CAAC;QACD,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,MAAc,EAAE,GAAW;QAC9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CACxC,IAAI,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CACtC,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,IAAI,GAAG,EAAE,CAAC,CAAA;QAC3D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAA;QAExD,OAAO,QAAQ,CAAC,IAAgB,CAAA;IAClC,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,GAAW;QACrB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;YAC9B,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YAE/C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;YACxD,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;aACjC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,mBACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,0BAA0B;IAKR;IAJ7B;;;OAGG;IACH,YAA6B,MAAwC;QAAxC,WAAM,GAAN,MAAM,CAAkC;IAAG,CAAC;IAEzE;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAClB,gBAAwB;QAExB,OAAO,CAAC,GAAG,CAAC,wCAAwC,EAAE,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAA;YACjE,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAA;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,6BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,gBAAwB;QAC/C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAClD,IAAI,sBAAsB,CAAC,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,CAAC,CACnE,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QACzD,CAAC;QAED,OAAO,QAAQ,CAAC,SAAS,CAAA;IAC3B,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,IAAW;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAQ,EAAE,OAAiB;QACzC,MAAM,SAAS,GAA2B,EAAE,CAAA;QAE5C,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACjC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;YAC7B,IAAI,MAAM,EAAE,CAAC;gBACX,SAAS,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,IAAI,EAAE,CAAA;YAC9C,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;;;;OAOG;IACH,YAAY,CAAC,SAAoB;QAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;YAC5C,OAAO,EAAE,CAAA;QACX,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAC1C,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAA;IACvE,CAAC;CACF"}

package/dist/schema.d.ts

@@ -0,0 +1,72 @@
+import { z } from "zod";
+export declare const safeUrlDecode: (value: string) => string;
+export declare const urlEncodedString: (schema: z.ZodString) => z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>;
+export declare const datePreprocessor: z.ZodEffects<z.ZodString, string, unknown>;
+export declare const CsiUriQuerySchema: z.ZodObject<{
+    type: z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>;
+    siteId: z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>;
+    currency: z.ZodDefault<z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>>;
+    idorder: z.ZodDefault<z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>>;
+    revenue: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    created_at: z.ZodDefault<z.ZodOptional<z.ZodEffects<z.ZodString, string, unknown>>>;
+    shipping: z.ZodDefault<z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>>;
+    discount: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    referrer: z.ZodDefault<z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>>;
+    _idv: z.ZodDefault<z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>>;
+    _id: z.ZodDefault<z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>>;
+    idsite: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    quicktransaction: z.ZodOptional<z.ZodEnum<["0", "1"]>>;
+    agent: z.ZodDefault<z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>>;
+    audit_key: z.ZodNullable<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>;
+    device_type: z.ZodDefault<z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>>;
+    parent_idv: z.ZodDefault<z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>>;
+    country: z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>;
+    items_count: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    url: z.ZodOptional<z.ZodEffects<z.ZodDefault<z.ZodOptional<z.ZodString>>, string, unknown>>;
+    date: z.ZodOptional<z.ZodEffects<z.ZodString, string, unknown>>;
+}, "strip", z.ZodTypeAny, {
+    currency: string;
+    idorder: string;
+    revenue: string;
+    created_at: string;
+    shipping: string;
+    discount: string;
+    referrer: string;
+    _idv: string;
+    _id: string;
+    idsite: string;
+    agent: string;
+    audit_key: string | null;
+    device_type: string;
+    parent_idv: string;
+    items_count: string;
+    type?: string | undefined;
+    siteId?: string | undefined;
+    quicktransaction?: "0" | "1" | undefined;
+    country?: string | undefined;
+    url?: string | undefined;
+    date?: string | undefined;
+}, {
+    type?: unknown;
+    siteId?: unknown;
+    currency?: unknown;
+    idorder?: unknown;
+    revenue?: string | undefined;
+    created_at?: unknown;
+    shipping?: unknown;
+    discount?: string | undefined;
+    referrer?: unknown;
+    _idv?: unknown;
+    _id?: unknown;
+    idsite?: string | undefined;
+    quicktransaction?: "0" | "1" | undefined;
+    agent?: unknown;
+    audit_key?: unknown;
+    device_type?: unknown;
+    parent_idv?: unknown;
+    country?: unknown;
+    items_count?: string | undefined;
+    url?: unknown;
+    date?: unknown;
+}>;
+export type CsiUriQuery = z.infer<typeof CsiUriQuerySchema>;

package/dist/schema.js

@@ -0,0 +1,62 @@
+import { z } from "zod";
+// Helper function to decode URL components safely
+export const safeUrlDecode = (value) => {
+    try {
+        // Handle different levels of URL encoding
+        return decodeURIComponent(decodeURIComponent(value.replace(/\+/g, " ")));
+        // biome-ignore lint/correctness/noUnusedVariables: <explanation>
+    }
+    catch (e) {
+        try {
+            // Try single decoding if double fails
+            return decodeURIComponent(value.replace(/\+/g, " "));
+        }
+        catch {
+            // Return original if both fail
+            return value;
+        }
+    }
+};
+// Create a preprocessor for URL-encoded strings
+export const urlEncodedString = (schema) => z.preprocess((val) => {
+    if (typeof val !== "string")
+        return val;
+    return safeUrlDecode(val);
+}, schema.optional().default(""));
+// Create a preprocessor for dates
+export const datePreprocessor = z.preprocess((val) => {
+    if (typeof val !== "string")
+        return val;
+    const decoded = safeUrlDecode(val);
+    try {
+        return new Date(decoded).toISOString();
+    }
+    catch {
+        return val;
+    }
+}, z.string());
+// Zod schema with all fields optional
+export const CsiUriQuerySchema = z.object({
+    type: urlEncodedString(z.string()).optional(),
+    siteId: urlEncodedString(z.string().regex(/^\d+$/, "Site ID must be numeric")).optional(),
+    currency: urlEncodedString(z.string()).optional().default(""),
+    idorder: urlEncodedString(z.string()).optional().default(""),
+    revenue: z.string().optional().default(""),
+    created_at: datePreprocessor.optional().default(""),
+    shipping: urlEncodedString(z.string()).optional().default(""),
+    discount: z.string().optional().default(""),
+    referrer: urlEncodedString(z.string()).optional().default(""),
+    _idv: urlEncodedString(z.string()).optional().default(""),
+    _id: urlEncodedString(z.string()).optional().default(""),
+    idsite: z.string().optional().default(""),
+    quicktransaction: z.enum(["0", "1"]).optional(),
+    agent: urlEncodedString(z.string()).optional().default(""),
+    audit_key: urlEncodedString(z.string()).nullable(),
+    device_type: urlEncodedString(z.string()).optional().default(""),
+    parent_idv: urlEncodedString(z.string()).optional().default(""),
+    country: urlEncodedString(z.string().length(2, "Country must be a 2-letter code")).optional(),
+    items_count: z.string().optional().default(""),
+    url: urlEncodedString(z.string().url()).optional(),
+    date: datePreprocessor.optional(),
+});
+//# sourceMappingURL=schema.js.map

package/dist/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,kDAAkD;AAClD,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,KAAa,EAAE,EAAE;IAC7C,IAAI,CAAC;QACH,0CAA0C;QAC1C,OAAO,kBAAkB,CAAC,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QACzE,iEAAiE;IACnE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC;YACH,sCAAsC;YACtC,OAAO,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;YAC/B,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,gDAAgD;AAChD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,MAAmB,EAAE,EAAE,CACtD,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACxC,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC;AAC5B,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;AAEpC,kCAAkC;AAClC,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,EAAE;IACnD,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACxC,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;AAEf,sCAAsC;AACtC,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,IAAI,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC7C,MAAM,EAAE,gBAAgB,CACtB,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,yBAAyB,CAAC,CACrD,CAAC,QAAQ,EAAE;IACZ,QAAQ,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7D,OAAO,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC5D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1C,UAAU,EAAE,gBAAgB,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACnD,QAAQ,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7D,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC3C,QAAQ,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7D,IAAI,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACzD,GAAG,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACxD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACzC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/C,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1D,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAClD,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAChE,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC/D,OAAO,EAAE,gBAAgB,CACvB,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,iCAAiC,CAAC,CACxD,CAAC,QAAQ,EAAE;IACZ,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC9C,GAAG,EAAE,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;IAClD,IAAI,EAAE,gBAAgB,CAAC,QAAQ,EAAE;CAClC,CAAC,CAAC"}

package/dist/utils.d.ts

@@ -0,0 +1,21 @@
+/**
+ *@description Interface for mapped data row structure
+ */
+interface MappedDataRow {
+    partition_date: string;
+    hour: string;
+    idorder: string;
+    idsite: string;
+    siteid: string;
+    sid: string;
+    cs_uri_query: string;
+    cs_referer: string;
+}
+/**
+ * @description Processes mapped data and writes to JSON files grouped by date
+ * @param {MappedDataRow[]} mappedData - Data to be processed
+ * @param {string} directory - Output directory for files
+ * @returns {Promise<string[]>} Array of created filenames
+ */
+export declare function processToJson(mappedData: MappedDataRow[], directory: string): Promise<string[]>;
+export {};

package/dist/utils.js

@@ -0,0 +1,98 @@
+import { JsonFileAppender } from "./json-file-appender.js";
+import { CsiUriQuerySchema } from "./schema.js";
+/**
+ *@description Parses a CSI_URI query string into a structured object
+ * @param {string} queryString - The query string to parse
+ * @returns {CsiUriQuery} Parsed and validated query object
+ */
+const parseCsiUriQuery = (queryString) => {
+    const cleanQueryString = queryString
+        .replace(/^cs_uri_query:\s*['"]?/, "")
+        .replace(/['"]$/, "");
+    // Create params object
+    const params = {};
+    // Split and parse, handling both & and ; separators
+    // biome-ignore lint/complexity/noForEach: <explanation>
+    cleanQueryString.split(/[&;]/).forEach((param) => {
+        const [key, ...values] = param.split("=");
+        if (key) {
+            // Join values back together in case the value contained = signs
+            params[key.trim()] = values.join("=");
+        }
+    });
+    // Validate and return
+    const parsed = CsiUriQuerySchema.parse(params);
+    return parsed;
+};
+/**
+ *@description Converts pixel log data to conversion format
+ * @param {MappedDataRow} mappedRow - Raw mapped data
+ * @param {CsiUriQuery} csiUriQuery - Parsed URI query parameters
+ */
+const pixelLog = (mappedRow, csiUriQuery) => {
+    const conversion = {
+        created_at: `${new Date(mappedRow.partition_date).toISOString().split("T")[0]} ${mappedRow.hour}:00`,
+        date: new Date(mappedRow.partition_date).toISOString().split("T")[0] ?? "",
+        idorder: mappedRow.idorder,
+        revenue: csiUriQuery.revenue,
+        currency: csiUriQuery.currency,
+        parent_idv: csiUriQuery._idv,
+        _idv: csiUriQuery._idv,
+        nginxLogs: true,
+    };
+    return conversion;
+};
+/**
+ * @description Processes mapped data and writes to JSON files grouped by date
+ * @param {MappedDataRow[]} mappedData - Data to be processed
+ * @param {string} directory - Output directory for files
+ * @returns {Promise<string[]>} Array of created filenames
+ */
+export async function processToJson(mappedData, directory) {
+    console.log(`Processing ${mappedData.length} rows to directory: ${directory}`);
+    const fileNames = [];
+    try {
+        // Group data by date
+        const dateGroup = mappedData.reduce((acc, row) => {
+            const csiUriQuery = parseCsiUriQuery(row.cs_uri_query);
+            const log = pixelLog(row, csiUriQuery);
+            // biome-ignore lint/style/noNonNullAssertion: <explanation>
+            const date = log.created_at.split(" ")[0];
+            // ignore log if idorder is empty or _idv is empty or idv value is undefined or revenue cannot be parsed as number
+            if (!log.idorder ||
+                log.idorder === "undefined" ||
+                !log._idv ||
+                log._idv === "undefined" ||
+                !log.revenue ||
+                Number.isNaN(Number(log.revenue))) {
+                return acc;
+            }
+            if (!acc[date]) {
+                acc[date] = [];
+            }
+            // biome-ignore lint/style/noNonNullAssertion: <explanation>
+            acc[date].push(log);
+            return acc;
+        }, {});
+        console.log(`Grouped data into ${Object.keys(dateGroup).length} dates`);
+        const jsonFileAppenderPromises = Object.entries(dateGroup).map(async ([date, data]) => {
+            const filename = `conversion_${date}.json`;
+            console.log(`Processing ${date}: ${data.length} records`);
+            fileNames.push(filename);
+            const jsonFileAppender = new JsonFileAppender({
+                fileName: filename,
+                directory: directory,
+            });
+            await jsonFileAppender.flush(data);
+            console.log(`Completed writing ${filename}`);
+        });
+        await Promise.all(jsonFileAppenderPromises);
+        console.log("All files processed successfully");
+        return fileNames;
+    }
+    catch (error) {
+        console.error("Error in processToJson:", error);
+        process.exit(1);
+    }
+}
+//# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAoB,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAElE;;;;GAIG;AACH,MAAM,gBAAgB,GAAG,CAAC,WAAmB,EAAe,EAAE;IAC5D,MAAM,gBAAgB,GAAG,WAAW;SACjC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC;SACrC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAExB,uBAAuB;IACvB,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,oDAAoD;IACpD,wDAAwD;IACxD,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAC/C,MAAM,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,GAAG,EAAE,CAAC;YACR,gEAAgE;YAChE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,sBAAsB;IACtB,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAgBF;;;;GAIG;AACH,MAAM,QAAQ,GAAG,CAAC,SAAwB,EAAE,WAAwB,EAAE,EAAE;IACtE,MAAM,UAAU,GAAG;QACjB,UAAU,EAAE,GACV,IAAI,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAC/D,IAAI,SAAS,CAAC,IAAI,KAAK;QACvB,IAAI,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;QAC1E,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,QAAQ,EAAE,WAAW,CAAC,QAAQ;QAC9B,UAAU,EAAE,WAAW,CAAC,IAAI;QAC5B,IAAI,EAAE,WAAW,CAAC,IAAI;QACtB,SAAS,EAAE,IAAI;KAChB,CAAC;IACF,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF;;;;;GAKG;AAEH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA2B,EAC3B,SAAiB;IAEjB,OAAO,CAAC,GAAG,CACT,cAAc,UAAU,CAAC,MAAM,uBAAuB,SAAS,EAAE,CAClE,CAAC;IAEF,MAAM,SAAS,GAAa,EAAE,CAAC;IAE/B,IAAI,CAAC;QACH,qBAAqB;QACrB,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC/C,MAAM,WAAW,GAAG,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YACvC,4DAA4D;YAC5D,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;YAE3C,kHAAkH;YAClH,IACE,CAAC,GAAG,CAAC,OAAO;gBACZ,GAAG,CAAC,OAAO,KAAK,WAAW;gBAC3B,CAAC,GAAG,CAAC,IAAI;gBACT,GAAG,CAAC,IAAI,KAAK,WAAW;gBACxB,CAAC,GAAG,CAAC,OAAO;gBACZ,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EACjC,CAAC;gBACD,OAAO,GAAG,CAAC;YACb,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACf,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACjB,CAAC;YACD,4DAA4D;YAC5D,GAAG,CAAC,IAAI,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAmD,CAAC,CAAC;QAExD,OAAO,CAAC,GAAG,CAAC,qBAAqB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,QAAQ,CAAC,CAAC;QAExE,MAAM,wBAAwB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,GAAG,CAC5D,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;YACrB,MAAM,QAAQ,GAAG,cAAc,IAAI,OAAO,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,KAAK,IAAI,CAAC,MAAM,UAAU,CAAC,CAAC;YAC1D,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEzB,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,CAAC;gBAC5C,QAAQ,EAAE,QAAQ;gBAClB,SAAS,EAAE,SAAS;aACrB,CAAC,CAAC;YAEH,MAAM,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;QAC/C,CAAC,CACF,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;QAChD,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stilyng94/athena-query-client",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "An Athena client to query data from Athena utilizing Aws-S3 or direct results from Athena query",
   "type": "module",
   "main": "dist/index.js",