@stilyng94/athena-query-client 1.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [2024] [Paul Osei Kofi]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,97 @@
+**athena-query-client**
+
+# @stilyng94/athena-query-client
+
+A lightweight client library for executing Amazon Athena queries.
+
+## Installation
+
+```bash
+npm install @stilyng94/athena-query-client
+```
+
+### !!Notice
+
+Ensure aws `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` envs are set
+
+### Usage
+
+## Get query results via S3 and write to JsonFile
+
+```typescript
+import {
+  AthenaQueryClient,
+  S3QueryResultProcessor,
+  JsonFileAppender,
+} from "@stilyng94/athena-query-client";
+
+const query = "SELECT * from api_logs";
+
+const execute = async (query) => {
+  const athena = new AthenaQueryClient({
+    Catalog: "catalog",
+    Database: "logs",
+    ClientConfig: { region: "us-west-2" },
+    WorkGroup: "primary",
+    s3OutputLocation: "s3://results/queries",
+    s3Region: "us-west-2",
+  });
+
+  const queryExecutionId = await athena.query();
+
+  const jsonFileAppender = new JsonFileAppender({
+    directory: "",
+    fileName: "",
+  });
+
+  const s3QueryResultProcessor = new S3QueryResultProcessor({
+    s3OutputLocation: "s3://results/queries",
+    s3Region: "us-west-2",
+    batchSize: 999,
+    onData: async (data) => {
+      await jsonFileAppender.flush(data);
+    },
+    onComplete: async () => {
+      await jsonFileAppender.closeFileWithBracket();
+    },
+  });
+  s3QueryResultProcessor
+    .processResults(queryExecutionId)
+    .catch((error) => console.error(error));
+};
+
+execute().catch(console.error);
+```
+
+## Get query results and map to Js object
+
+```typescript
+import {
+  AthenaQueryClient,
+  MappedQueryResultProcessor,
+} from "@stilyng94/athena-query-client";
+
+const query = "SELECT * from api_logs";
+
+const execute = async (query) => {
+  const athena = new AthenaQueryClient({
+    Catalog: "catalog",
+    Database: "logs",
+    ClientConfig: { region: "us-west-2" },
+    WorkGroup: "primary",
+    s3OutputLocation: "s3://results/queries",
+    s3Region: "us-west-2",
+  });
+
+  const queryExecutionId = await athena.query();
+
+  const mappedQueryResultProcessor = new MappedQueryResultProcessor({
+    athenaClient: athena,
+  });
+  mappedQueryResultProcessor.processResults(queryExecutionId).then((data) => {
+    console.log(data);
+  });
+};
+
+execute().catch(console.error);
+```

package/dist/athena.d.ts

@@ -0,0 +1,20 @@
+import type { AthenaQueryClientConfig } from './types.js';
+/**
+ * Client for executing queries against AWS Athena and retrieving results
+ * @class AthenaQueryClient
+ */
+export declare class AthenaQueryClient {
+    #private;
+    /**
+     * Creates an instance of AthenaQueryClient
+     * @param {AthenaQueryClientConfig} config - Configuration object for the client
+     */
+    constructor(config: AthenaQueryClientConfig);
+    /**
+     * Executes an SQL query in Athena and returns the QueryExecutionId
+     * @param {string} sqlQuery - The SQL query string to execute
+     * @returns {Promise<string>} Promise resolving to QueryExecutionId
+     * @throws {Error} If query execution fails or encounters an unknown state
+     */
+    query(sqlQuery: string): Promise<string>;
+}

package/dist/athena.js

@@ -0,0 +1,92 @@
+import { AthenaClient, GetQueryExecutionCommand, QueryExecutionState, StartQueryExecutionCommand, } from '@aws-sdk/client-athena';
+import { EMPTY, defer, firstValueFrom, from, of, throwError, timer, } from 'rxjs';
+import { expand, filter, switchMap, take } from 'rxjs/operators';
+import { AthenaQueryError } from './errors.js';
+/**
+ * Client for executing queries against AWS Athena and retrieving results
+ * @class AthenaQueryClient
+ */
+export class AthenaQueryClient {
+    #database;
+    #client;
+    #catalog;
+    #workGroup;
+    #resultReuseConfiguration;
+    #s3OutputLocation;
+    /**
+     * Creates an instance of AthenaQueryClient
+     * @param {AthenaQueryClientConfig} config - Configuration object for the client
+     */
+    constructor(config) {
+        this.#client = new AthenaClient({
+            ...config.ClientConfig,
+            region: config.s3Region,
+        });
+        this.#database = config.Database;
+        this.#catalog = config.Catalog;
+        this.#workGroup = config.WorkGroup || 'primary';
+        this.#resultReuseConfiguration = config.ResultReuseConfiguration || {
+            ResultReuseByAgeConfiguration: { Enabled: true, MaxAgeInMinutes: 60 },
+        };
+        this.#s3OutputLocation = config.s3OutputLocation;
+    }
+    /**
+     * Executes an SQL query in Athena and returns the QueryExecutionId
+     * @param {string} sqlQuery - The SQL query string to execute
+     * @returns {Promise<string>} Promise resolving to QueryExecutionId
+     * @throws {Error} If query execution fails or encounters an unknown state
+     */
+    async query(sqlQuery) {
+        console.log('Executing query:', sqlQuery);
+        const queryExecutionInput = {
+            QueryString: sqlQuery,
+            QueryExecutionContext: {
+                Database: this.#database,
+                Catalog: this.#catalog,
+            },
+            WorkGroup: this.#workGroup,
+            ResultReuseConfiguration: this.#resultReuseConfiguration,
+            ResultConfiguration: { OutputLocation: this.#s3OutputLocation },
+        };
+        const { QueryExecutionId } = await this.#client.send(new StartQueryExecutionCommand(queryExecutionInput));
+        console.log('Query execution started with ID:', QueryExecutionId);
+        if (!QueryExecutionId) {
+            throw new AthenaQueryError('Failed to start query: QueryExecutionId is undefined');
+        }
+        return await firstValueFrom(this.#checkQueryExecutionState(QueryExecutionId)).catch((error) => {
+            console.error('Query execution failed:', error);
+            throw new AthenaQueryError(`Query execution failed: ${error.message}`);
+        });
+    }
+    /**
+     * Polls the Athena query execution state until completion
+     * @param {string} QueryExecutionId - The ID of the query being executed
+     * @returns {Observable<string>} Observable emitting QueryExecutionId on success
+     */
+    #checkQueryExecutionState(QueryExecutionId) {
+        return defer(() => from(this.#client.send(new GetQueryExecutionCommand({ QueryExecutionId })))).pipe(switchMap((response) => {
+            const state = response.QueryExecution?.Status?.State;
+            console.log(`Query execution state: ${state}`);
+            switch (state) {
+                case QueryExecutionState.SUCCEEDED:
+                    return of(QueryExecutionId);
+                case QueryExecutionState.FAILED: {
+                    const errorMessage = response.QueryExecution?.Status?.StateChangeReason ||
+                        'Unknown reason';
+                    console.error('Query failed:', errorMessage);
+                    return throwError(() => new Error(`Query failed: ${errorMessage}`));
+                }
+                case QueryExecutionState.CANCELLED:
+                    return throwError(() => new Error('Query was cancelled'));
+                case QueryExecutionState.QUEUED:
+                case QueryExecutionState.RUNNING:
+                    return of(null); // Continue polling
+                default:
+                    return throwError(() => new Error(`Unknown query state: ${state}`));
+            }
+        }), expand((result) => result === null
+            ? timer(1000).pipe(switchMap(() => this.#checkQueryExecutionState(QueryExecutionId)))
+            : EMPTY), filter((result) => result !== null), take(1));
+    }
+}
+//# sourceMappingURL=athena.js.map

package/dist/athena.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"athena.js","sourceRoot":"","sources":["../src/athena.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,YAAY,EACZ,wBAAwB,EACxB,mBAAmB,EAEnB,0BAA0B,GAE3B,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EACL,KAAK,EAEL,KAAK,EACL,cAAc,EACd,IAAI,EACJ,EAAE,EACF,UAAU,EACV,KAAK,GACN,MAAM,MAAM,CAAA;AACb,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAA;AAG9C;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACnB,SAAS,CAAQ;IACjB,OAAO,CAAc;IACrB,QAAQ,CAAQ;IAChB,UAAU,CAAQ;IAClB,yBAAyB,CAA0B;IACnD,iBAAiB,CAAQ;IAElC;;;OAGG;IACH,YAAY,MAA+B;QACzC,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC;YAC9B,GAAG,MAAM,CAAC,YAAY;YACtB,MAAM,EAAE,MAAM,CAAC,QAAQ;SACxB,CAAC,CAAA;QACF,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAA;QAChC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAA;QAC9B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,SAAS,IAAI,SAAS,CAAA;QAC/C,IAAI,CAAC,yBAAyB,GAAG,MAAM,CAAC,wBAAwB,IAAI;YAClE,6BAA6B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE,EAAE;SACtE,CAAA;QACD,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,gBAAgB,CAAA;IAClD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK,CAAC,QAAgB;QAC1B,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,QAAQ,CAAC,CAAA;QAEzC,MAAM,mBAAmB,GAAoC;YAC3D,WAAW,EAAE,QAAQ;YACrB,qBAAqB,EAAE;gBACrB,QAAQ,EAAE,IAAI,CAAC,SAAS;gBACxB,OAAO,EAAE,IAAI,CAAC,QAAQ;aACvB;YACD,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,wBAAwB,EAAE,IAAI,CAAC,yBAAyB;YACxD,mBAAmB,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,iBAAiB,EAAE;SAChE,CAAA;QAED,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAClD,IAAI,0BAA0B,CAAC,mBAAmB,CAAC,CACpD,CAAA;QACD,OAAO,CAAC,GAAG,CAAC,kCAAkC,EAAE,gBAAgB,CAAC,CAAA;QAEjE,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,MAAM,IAAI,gBAAgB,CACxB,sDAAsD,CACvD,CAAA;QACH,CAAC;QAED,OAAO,MAAM,cAAc,CACzB,IAAI,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CACjD,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YAChB,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAA;YAC/C,MAAM,IAAI,gBAAgB,CAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;QACxE,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;;;OAIG;IACH,yBAAyB,CAAC,gBAAwB;QAChD,OAAO,KAAK,CAAC,GAAG,EAAE,CAChB,IAAI,CACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,wBAAwB,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,CACtE,CACF,CAAC,IAAI,CACJ,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE;YACrB,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,KAAK,CAAA;YACpD,OAAO,CAAC,GAAG,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAA;YAE9C,QAAQ,KAAK,EAAE,CAAC;gBACd,KAAK,mBAAmB,CAAC,SAAS;oBAChC,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAA;gBAC7B,KAAK,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;oBAChC,MAAM,YAAY,GAChB,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,iBAAiB;wBAClD,gBAAgB,CAAA;oBAClB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,YAAY,CAAC,CAAA;oBAC5C,OAAO,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,KAAK,CAAC,iBAAiB,YAAY,EAAE,CAAC,CAAC,CAAA;gBACrE,CAAC;gBACD,KAAK,mBAAmB,CAAC,SAAS;oBAChC,OAAO,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAA;gBAC3D,KAAK,mBAAmB,CAAC,MAAM,CAAC;gBAChC,KAAK,mBAAmB,CAAC,OAAO;oBAC9B,OAAO,EAAE,CAAC,IAAI,CAAC,CAAA,CAAC,mBAAmB;gBACrC;oBACE,OAAO,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,KAAK,CAAC,wBAAwB,KAAK,EAAE,CAAC,CAAC,CAAA;YACvE,CAAC;QACH,CAAC,CAAC,EACF,MAAM,CACJ,CAAC,MAAM,EAAE,EAAE,CACT,MAAM,KAAK,IAAI;YACb,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CACd,SAAS,CAAC,GAAG,EAAE,CACb,IAAI,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CACjD,CACF;YACH,CAAC,CAAC,KAAK,CACZ,EACD,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,IAAI,CAAC,EACnC,IAAI,CAAC,CAAC,CAAC,CACR,CAAA;IACH,CAAC;CACF"}

package/dist/demo.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/demo.js

@@ -0,0 +1,15 @@
+import { AthenaQueryClient } from "./athena.js";
+const client = new AthenaQueryClient({
+    Catalog: "",
+    s3OutputLocation: "s3://sjka/s",
+    s3Region: "us-east-1",
+    Database: "",
+    ClientConfig: {
+        region: "us-east-1",
+    },
+});
+client
+    .query("select * from db")
+    .then((d) => console.log(d))
+    .then((e) => console.error({ e }));
+//# sourceMappingURL=demo.js.map

package/dist/demo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"demo.js","sourceRoot":"","sources":["../src/demo.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAAC;IACnC,OAAO,EAAE,EAAE;IACX,gBAAgB,EAAE,aAAa;IAC/B,QAAQ,EAAE,WAAW;IACrB,QAAQ,EAAE,EAAE;IACZ,YAAY,EAAE;QACZ,MAAM,EAAE,WAAW;KACpB;CACF,CAAC,CAAC;AACH,MAAM;KACH,KAAK,CAAC,kBAAkB,CAAC;KACzB,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;KAC3B,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC"}

package/dist/errors.d.ts

@@ -0,0 +1,4 @@
+export declare class AthenaQueryError extends Error {
+    readonly queryExecutionId?: string | undefined;
+    constructor(message: string, queryExecutionId?: string | undefined);
+}

package/dist/errors.js

@@ -0,0 +1,9 @@
+export class AthenaQueryError extends Error {
+    queryExecutionId;
+    constructor(message, queryExecutionId) {
+        super(message);
+        this.queryExecutionId = queryExecutionId;
+        this.name = 'AthenaQueryError';
+    }
+}
+//# sourceMappingURL=errors.js.map

package/dist/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAGvB;IAFlB,YACE,OAAe,EACC,gBAAyB;QAEzC,KAAK,CAAC,OAAO,CAAC,CAAA;QAFE,qBAAgB,GAAhB,gBAAgB,CAAS;QAGzC,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAA;IAChC,CAAC;CACF"}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export { AthenaQueryClient } from './athena.js';
+export { AthenaQueryError } from './errors.js';
+export { MappedQueryResultProcessor, S3QueryResultProcessor, } from './query-results-processor.js';
+export type { AthenaQueryClientConfig, MappedQueryResultProcessorParams, QueryResultProcessor, S3QueryResultProcessorParams, JsonFileAppenderOptions, } from './types.js';
+export { JsonFileAppender } from './json-file-appender.js';

package/dist/index.js

@@ -0,0 +1,5 @@
+export { AthenaQueryClient } from './athena.js';
+export { AthenaQueryError } from './errors.js';
+export { MappedQueryResultProcessor, S3QueryResultProcessor, } from './query-results-processor.js';
+export { JsonFileAppender } from './json-file-appender.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAA;AAC9C,OAAO,EACL,0BAA0B,EAC1B,sBAAsB,GACvB,MAAM,8BAA8B,CAAA;AAQrC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA"}

package/dist/json-file-appender.d.ts

@@ -0,0 +1,24 @@
+import type { JsonFileAppenderOptions } from './types.js';
+/**
+ *@description Handles appending JSON data to files with proper formatting
+ */
+export declare class JsonFileAppender {
+    #private;
+    private readonly options;
+    /**
+     *@description Creates a new JsonFileAppender instance
+     * @param {AppendOptions} options - Configuration options
+     */
+    constructor(options: JsonFileAppenderOptions);
+    /**
+     *@description Flushes the current batch of data to the file
+     * @param {unknown[]} currentBatch - Array of items to write
+     * @returns {Promise<void>}
+     */
+    flush(currentBatch: unknown[]): Promise<void>;
+    /**
+     * @description Adds closing bracket to JSON array file
+     * @returns {void}
+     */
+    closeFileWithBracket(): void;
+}

package/dist/json-file-appender.js

@@ -0,0 +1,105 @@
+import { createWriteStream } from 'node:fs';
+import { access, mkdir, stat, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+/**
+ *@description Handles appending JSON data to files with proper formatting
+ */
+export class JsonFileAppender {
+    options;
+    #isFirstWrite = true;
+    #filePath;
+    /**
+     *@description Creates a new JsonFileAppender instance
+     * @param {AppendOptions} options - Configuration options
+     */
+    constructor(options) {
+        this.options = options;
+        this.#filePath = path.join(options.directory, options.fileName);
+    }
+    /**
+     *@description Flushes the current batch of data to the file
+     * @param {unknown[]} currentBatch - Array of items to write
+     * @returns {Promise<void>}
+     */
+    async flush(currentBatch) {
+        console.log(`Flushing batch of ${currentBatch.length} items to ${this.options.fileName}`);
+        await this.#ensureDirectoryExists(this.options.directory);
+        const filePath = path.join(this.options.directory, this.options.fileName);
+        await this.#initializeFile();
+        const writeStream = createWriteStream(filePath, {
+            flags: 'a',
+        });
+        try {
+            for (let i = 0; i < currentBatch.length; i++) {
+                const item = currentBatch[i];
+                // Add comma if not first item and not first write
+                if (!this.#isFirstWrite || i > 0) {
+                    writeStream.write(',\n');
+                }
+                writeStream.write(JSON.stringify(item, null));
+            }
+            this.#isFirstWrite = false;
+            console.log(`Successfully flushed ${currentBatch.length} items`);
+        }
+        catch (error) {
+            console.error('Error while flushing data:', error);
+            throw error;
+        }
+        finally {
+            writeStream.end();
+        }
+    }
+    /**
+     * @description Adds closing bracket to JSON array file
+     * @returns {void}
+     */
+    closeFileWithBracket() {
+        console.log(`Closing JSON array in file: ${this.#filePath}`);
+        const writeStream = createWriteStream(this.#filePath, { flags: 'a' });
+        try {
+            writeStream.write('\n]');
+        }
+        catch (error) {
+            console.error('Error while closing JSON array:', error);
+            throw error;
+        }
+        finally {
+            writeStream.end();
+        }
+    }
+    /**
+     *@description Initializes the file with proper JSON array structure
+     * @returns {Promise<void>}
+     */
+    async #initializeFile() {
+        try {
+            await access(this.#filePath);
+            // File exists, check if it's empty or has content
+            const statistics = await stat(this.#filePath);
+            this.#isFirstWrite = statistics.size === 0;
+            console.log(`File exists. Size: ${statistics.size} bytes. First write: ${this.#isFirstWrite}`);
+        }
+        catch {
+            // File doesn't exist, create it with initial array bracket
+            console.log(`Creating new file: ${this.#filePath}`);
+            await writeFile(this.#filePath, '[\n', 'utf-8');
+            this.#isFirstWrite = true;
+        }
+    }
+    /**
+     *@description Ensures the specified directory exists, creates if it doesn't
+     * @param {string} directory - Directory path to check/create
+     * @returns {Promise<void>}
+     */
+    async #ensureDirectoryExists(directory) {
+        try {
+            await access(directory);
+            console.log(`Directory ${directory} exists`);
+        }
+        catch {
+            await mkdir(directory, { recursive: true });
+            console.log(`Created directory ${directory}`);
+        }
+    }
+}
+//# sourceMappingURL=json-file-appender.js.map

package/dist/json-file-appender.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-file-appender.js","sourceRoot":"","sources":["../src/json-file-appender.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC3C,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACjE,OAAO,IAAI,MAAM,WAAW,CAAA;AAG5B;;GAEG;AACH,MAAM,OAAO,gBAAgB;IAQE;IAP7B,aAAa,GAAG,IAAI,CAAA;IACX,SAAS,CAAQ;IAE1B;;;OAGG;IACH,YAA6B,OAAgC;QAAhC,YAAO,GAAP,OAAO,CAAyB;QAC3D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA;IACjE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK,CAAC,YAAuB;QACjC,OAAO,CAAC,GAAG,CACT,qBAAqB,YAAY,CAAC,MAAM,aAAa,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAC7E,CAAA;QACD,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACzE,MAAM,IAAI,CAAC,eAAe,EAAE,CAAA;QAE5B,MAAM,WAAW,GAAG,iBAAiB,CAAC,QAAQ,EAAE;YAC9C,KAAK,EAAE,GAAG;SACX,CAAC,CAAA;QAEF,IAAI,CAAC;YACH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;gBAC5B,kDAAkD;gBAClD,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBACjC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;gBAED,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAC/C,CAAC;YAED,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;YAC1B,OAAO,CAAC,GAAG,CAAC,wBAAwB,YAAY,CAAC,MAAM,QAAQ,CAAC,CAAA;QAClE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC,CAAA;YAClD,MAAM,KAAK,CAAA;QACb,CAAC;gBAAS,CAAC;YACT,WAAW,CAAC,GAAG,EAAE,CAAA;QACnB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,oBAAoB;QAClB,OAAO,CAAC,GAAG,CAAC,+BAA+B,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;QAC5D,MAAM,WAAW,GAAG,iBAAiB,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;QACrE,IAAI,CAAC;YACH,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAA;YACvD,MAAM,KAAK,CAAA;QACb,CAAC;gBAAS,CAAC;YACT,WAAW,CAAC,GAAG,EAAE,CAAA;QACnB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAC5B,kDAAkD;YAClD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAC7C,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,IAAI,KAAK,CAAC,CAAA;YAC1C,OAAO,CAAC,GAAG,CACT,sBAAsB,UAAU,CAAC,IAAI,wBACnC,IAAI,CAAC,aACP,EAAE,CACH,CAAA;QACH,CAAC;QAAC,MAAM,CAAC;YACP,2DAA2D;YAC3D,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;YACnD,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;YAC/C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;QAC3B,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,sBAAsB,CAAC,SAAiB;QAC5C,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAA;YACvB,OAAO,CAAC,GAAG,CAAC,aAAa,SAAS,SAAS,CAAC,CAAA;QAC9C,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YAC3C,OAAO,CAAC,GAAG,CAAC,qBAAqB,SAAS,EAAE,CAAC,CAAA;QAC/C,CAAC;IACH,CAAC;CACF"}

package/dist/query-results-processor.d.ts

@@ -0,0 +1,37 @@
+import { type MappedQueryResultProcessorParams, type QueryResultProcessor, type S3QueryResultProcessorParams } from './types.js';
+/**
+ * Processes Athena query results stored in S3
+ */
+export declare class S3QueryResultProcessor implements QueryResultProcessor {
+    #private;
+    private readonly config;
+    /**
+     * Creates a new S3QueryResultProcessor
+     * @param {S3QueryResultProcessorParams} config - Configuration parameters
+     */
+    constructor(config: S3QueryResultProcessorParams);
+    /**
+     * Processes query results from S3
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<void>} Promise that resolves when processing is complete
+     */
+    processResults(queryExecutionId: string): Promise<void>;
+}
+/**
+ * Processes Athena query results by mapping them to objects
+ */
+export declare class MappedQueryResultProcessor implements QueryResultProcessor {
+    #private;
+    private readonly config;
+    /**
+     * Creates a new MappedQueryResultProcessor
+     * @param {MappedQueryResultProcessorParams} config - Configuration parameters
+     */
+    constructor(config: MappedQueryResultProcessorParams);
+    /**
+     * Processes query results by mapping them to objects
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<Record<string, string>[]>} Promise resolving to array of mapped objects
+     */
+    processResults(queryExecutionId: string): Promise<Record<string, string>[]>;
+}

package/dist/query-results-processor.js

@@ -0,0 +1,231 @@
+import { finished } from 'node:stream/promises';
+import { GetQueryResultsCommand, } from '@aws-sdk/client-athena';
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { parse } from 'csv-parse';
+import { MAX_BATCH_SIZE, } from './types.js';
+/**
+ * Processes Athena query results stored in S3
+ */
+export class S3QueryResultProcessor {
+    config;
+    #batchSize = MAX_BATCH_SIZE;
+    #s3Client;
+    #s3OutputLocation;
+    /**
+     * Creates a new S3QueryResultProcessor
+     * @param {S3QueryResultProcessorParams} config - Configuration parameters
+     */
+    constructor(config) {
+        this.config = config;
+        this.#batchSize = this.#validateBatchSize(config.batchSize);
+        this.#s3OutputLocation = config.s3OutputLocation;
+        this.#s3Client = new S3Client({ region: config.s3Region });
+    }
+    /**
+     * Processes query results from S3
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<void>} Promise that resolves when processing is complete
+     */
+    async processResults(queryExecutionId) {
+        // Get the S3 location of results
+        const s3Location = `${this.#s3OutputLocation}/${queryExecutionId}.csv`;
+        const { Bucket, Key } = this.#parseS3Url(s3Location);
+        console.log('Fetching query results from S3:', s3Location);
+        const responseStream = await this.#fetchS3Object(Bucket, Key);
+        return this.#processStreamingResults(responseStream);
+    }
+    /**
+     * Processes streaming results from S3
+     * @param {Readable} stream - The readable stream of results
+     * @returns {Promise<void>} Promise that resolves when processing is complete
+     */
+    async #processStreamingResults(stream) {
+        const batch = [];
+        let totalProcessed = 0;
+        const parser = stream.pipe(parse({ ...(this.config.csvParseOptions ?? {}) }));
+        try {
+            parser.on('readable', async () => {
+                const record = parser.read();
+                while (record !== null) {
+                    // Work with each record
+                    batch.push(record);
+                    if (batch.length >= this.#batchSize) {
+                        console.log(`Processing batch of ${batch.length} records...`);
+                        await this.#processBatch(batch);
+                        totalProcessed += batch.length;
+                        batch.length = 0; // Clear array efficiently
+                        console.log(`Total records processed: ${totalProcessed}`);
+                    }
+                }
+            });
+            // Wait for parsing to complete
+            await finished(parser);
+            // Process remaining records
+            if (batch.length > 0) {
+                console.log(`Processing final batch of ${batch.length} records...`);
+                await this.#processBatch(batch);
+                totalProcessed += batch.length;
+                console.log(`Final total records processed: ${totalProcessed}`);
+            }
+            await this.config?.onComplete?.();
+        }
+        catch (error) {
+            throw new Error(`Error processing results: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Processes a batch of records
+     * @param {unknown[]} batch - Array of records to process
+     * @returns {Promise<void>} Promise that resolves when batch is processed
+     */
+    async #processBatch(batch) {
+        try {
+            await this.config.onData(batch);
+        }
+        catch (error) {
+            throw new Error(`Error processing batch: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Validates the batch size configuration
+     * @param {number} [batchSize] - The batch size to validate
+     * @returns {number} The validated batch size
+     * @throws {Error} If batch size exceeds maximum allowed
+     */
+    #validateBatchSize(batchSize) {
+        if (!batchSize)
+            return MAX_BATCH_SIZE;
+        if (batchSize > MAX_BATCH_SIZE) {
+            throw new Error(`Batch size cannot be greater than ${MAX_BATCH_SIZE}`);
+        }
+        return batchSize;
+    }
+    /**
+     * Fetches an object from S3
+     * @param {string} Bucket - The S3 bucket name
+     * @param {string} Key - The S3 object key
+     * @returns {Promise<Readable>} Promise resolving to readable stream
+     * @throws {Error} If object fetch fails
+     */
+    async #fetchS3Object(Bucket, Key) {
+        const response = await this.#s3Client.send(new GetObjectCommand({ Bucket, Key }));
+        if (!response.Body) {
+            throw new Error(`Failed to fetch file: ${Bucket}/${Key}`);
+        }
+        console.log('S3 response metadata:', response.$metadata);
+        return response.Body;
+    }
+    /**
+     * Parses an S3 URL into bucket and key components
+     * @param {string} url - The S3 URL to parse
+     * @returns {{ Bucket: string; Key: string }} Object containing bucket and key
+     * @throws {Error} If URL is invalid or missing bucket name
+     */
+    #parseS3Url(url) {
+        try {
+            const parsedUrl = new URL(url);
+            const bucket = parsedUrl.hostname.split('.')[0];
+            if (!bucket) {
+                throw new Error('Invalid S3 URL: missing bucket name');
+            }
+            return {
+                Bucket: bucket,
+                Key: parsedUrl.pathname.slice(1),
+            };
+        }
+        catch (error) {
+            throw new Error(`Invalid S3 URL: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+}
+/**
+ * Processes Athena query results by mapping them to objects
+ */
+export class MappedQueryResultProcessor {
+    config;
+    /**
+     * Creates a new MappedQueryResultProcessor
+     * @param {MappedQueryResultProcessorParams} config - Configuration parameters
+     */
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Processes query results by mapping them to objects
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<Record<string, string>[]>} Promise resolving to array of mapped objects
+     */
+    async processResults(queryExecutionId) {
+        console.log('Fetching results for QueryExecutionId:', queryExecutionId);
+        try {
+            const resultSet = await this.#fetchQueryResults(queryExecutionId);
+            return this.#extractRows(resultSet);
+        }
+        catch (error) {
+            throw new Error(`Error processing results: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Fetches query results from Athena
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<ResultSet>} Promise resolving to Athena ResultSet
+     * @throws {Error} If results are empty or undefined
+     */
+    async #fetchQueryResults(queryExecutionId) {
+        const response = await this.config.athenaClient.send(new GetQueryResultsCommand({ QueryExecutionId: queryExecutionId }));
+        if (!response.ResultSet) {
+            throw new Error('Query results are empty or undefined');
+        }
+        return response.ResultSet;
+    }
+    /**
+     * Extracts column headers from result rows
+     * @param {Row[]} rows - Array of result rows
+     * @returns {string[]} Array of header names
+     * @throws {Error} If no headers are found
+     */
+    #extractHeaders(rows) {
+        const headers = rows[0]?.Data?.map((column) => column.VarCharValue || '');
+        if (!headers || headers.length === 0) {
+            throw new Error('No headers found in the result set');
+        }
+        return headers;
+    }
+    /**
+     * Maps a row to an object using column headers as keys
+     * @param {Row} row - The row to map
+     * @param {string[]} headers - Array of column headers
+     * @returns {Record<string, string>} Object with header-value pairs
+     */
+    #mapRowToObject(row, headers) {
+        const mappedRow = {};
+        row.Data?.forEach((value, index) => {
+            const header = headers[index];
+            if (header) {
+                mappedRow[header] = value.VarCharValue || '';
+            }
+        });
+        return mappedRow;
+    }
+    /**
+     * Extracts  Athena ResultSet data into an array of key-value objects
+     * Extracts Athena ResultSet data into an array of key-value objects
+     * @private
+     * @param {ResultSet} resultSet - The ResultSet object returned by Athena
+     * @returns {Record<string, string>[]} Array of objects representing query rows
+     * @throws {Error} If no headers are found
+     */
+    #extractRows(resultSet) {
+        const { Rows } = resultSet;
+        if (!Rows || Rows.length === 0) {
+            console.error('No rows found in result set');
+            return [];
+        }
+        const headers = this.#extractHeaders(Rows);
+        if (!headers.length) {
+            throw new Error('No headers found in the result set');
+        }
+        return Rows.slice(1).map((row) => this.#mapRowToObject(row, headers));
+    }
+}
+//# sourceMappingURL=query-results-processor.js.map

package/dist/query-results-processor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"query-results-processor.js","sourceRoot":"","sources":["../src/query-results-processor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EACL,sBAAsB,GAGvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC/D,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EACL,cAAc,GAIf,MAAM,YAAY,CAAA;AAEnB;;GAEG;AACH,MAAM,OAAO,sBAAsB;IASJ;IARpB,UAAU,GAAG,cAAc,CAAA;IAC3B,SAAS,CAAU;IACnB,iBAAiB,CAAQ;IAElC;;;OAGG;IACH,YAA6B,MAAoC;QAApC,WAAM,GAAN,MAAM,CAA8B;QAC/D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QAC3D,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,gBAAgB,CAAA;QAChD,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC5D,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAAC,gBAAwB;QAC3C,iCAAiC;QACjC,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,iBAAiB,IAAI,gBAAgB,MAAM,CAAA;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAA;QACpD,OAAO,CAAC,GAAG,CAAC,iCAAiC,EAAE,UAAU,CAAC,CAAA;QAE1D,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC7D,OAAO,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,wBAAwB,CAAC,MAAgB;QAC7C,MAAM,KAAK,GAAc,EAAE,CAAA;QAC3B,IAAI,cAAc,GAAG,CAAC,CAAA;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,KAAK,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,EAAE,CAAC,CAClD,CAAA;QAED,IAAI,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE;gBAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;gBAC5B,OAAO,MAAM,KAAK,IAAI,EAAE,CAAC;oBACvB,wBAAwB;oBACxB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;oBAClB,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpC,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,CAAC,MAAM,aAAa,CAAC,CAAA;wBAC7D,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;wBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAA;wBAC9B,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA,CAAC,0BAA0B;wBAC3C,OAAO,CAAC,GAAG,CAAC,4BAA4B,cAAc,EAAE,CAAC,CAAA;oBAC3D,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAA;YAEF,+BAA+B;YAC/B,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAA;YACtB,4BAA4B;YAC5B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,6BAA6B,KAAK,CAAC,MAAM,aAAa,CAAC,CAAA;gBACnE,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;gBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAA;gBAC9B,OAAO,CAAC,GAAG,CAAC,kCAAkC,cAAc,EAAE,CAAC,CAAA;YACjE,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAA;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,6BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAAC,KAAgB;QAClC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,2BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,kBAAkB,CAAC,SAAkB;QACnC,IAAI,CAAC,SAAS;YAAE,OAAO,cAAc,CAAA;QACrC,IAAI,SAAS,GAAG,cAAc,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,qCAAqC,cAAc,EAAE,CAAC,CAAA;QACxE,CAAC;QACD,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,MAAc,EAAE,GAAW;QAC9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CACxC,IAAI,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CACtC,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,IAAI,GAAG,EAAE,CAAC,CAAA;QAC3D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAA;QAExD,OAAO,QAAQ,CAAC,IAAgB,CAAA;IAClC,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,GAAW;QACrB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;YAC9B,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YAE/C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;YACxD,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;aACjC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,mBACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,0BAA0B;IAKR;IAJ7B;;;OAGG;IACH,YAA6B,MAAwC;QAAxC,WAAM,GAAN,MAAM,CAAkC;IAAG,CAAC;IAEzE;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAClB,gBAAwB;QAExB,OAAO,CAAC,GAAG,CAAC,wCAAwC,EAAE,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAA;YACjE,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAA;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,6BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,gBAAwB;QAC/C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAClD,IAAI,sBAAsB,CAAC,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,CAAC,CACnE,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QACzD,CAAC;QAED,OAAO,QAAQ,CAAC,SAAS,CAAA;IAC3B,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,IAAW;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAQ,EAAE,OAAiB;QACzC,MAAM,SAAS,GAA2B,EAAE,CAAA;QAE5C,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACjC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;YAC7B,IAAI,MAAM,EAAE,CAAC;gBACX,SAAS,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,IAAI,EAAE,CAAA;YAC9C,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;;;;OAOG;IACH,YAAY,CAAC,SAAoB;QAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;YAC5C,OAAO,EAAE,CAAA;QACX,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAC1C,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAA;IACvE,CAAC;CACF"}

package/dist/types.d.ts

@@ -0,0 +1,76 @@
+import type { AthenaClient, AthenaClientConfig, ResultReuseConfiguration } from '@aws-sdk/client-athena';
+import type { Options } from 'csv-parse';
+/** Maximum number of records to process in a single batch */
+export declare const MAX_BATCH_SIZE: number;
+/**
+ * Configuration interface for AthenaQueryClient
+ * @interface AthenaQueryClientConfig
+ */
+export interface AthenaQueryClientConfig {
+    /** AWS SDK Athena client configuration */
+    ClientConfig: AthenaClientConfig;
+    /** AWS Athena database name. The database must exist in the catalog */
+    Database: string;
+    /** AWS Athena data source catalog name */
+    Catalog: string;
+    /** The name of the workgroup in which the query is being started. Optional, defaults to 'primary' */
+    WorkGroup?: string;
+    /** Configuration for result reuse in Athena */
+    ResultReuseConfiguration?: ResultReuseConfiguration;
+    /** AWS region for S3 operations */
+    s3Region: string;
+    /** S3 location where query results will be stored */
+    s3OutputLocation: string;
+}
+/**
+ * Interface for processing Athena query results
+ */
+export interface QueryResultProcessor {
+    /**
+     * Processes the results of an Athena query execution
+     * @param {string} queryExecutionId - The ID of the query execution
+     * @returns {Promise<unknown>} Promise resolving to the processed results
+     */
+    processResults(queryExecutionId: string): Promise<unknown>;
+}
+export interface S3QueryResultProcessorParams {
+    /**
+     * Callback function to process each batch of data
+     */
+    onData: (data: unknown[]) => Promise<void>;
+    /**
+     * Optional Callback function to call when processing is done
+     * @param data Optional data to pass to the callback function
+     * @returns Promise resolving to void
+     */
+    onComplete?: (data?: unknown) => Promise<void>;
+    /**
+     * Optional: The maximum number of records to process in a single batch
+     * @default 999
+     */
+    batchSize?: number;
+    /**
+     * Options for CSV parsing
+     * @default { columns: true }
+     */
+    csvParseOptions?: Options;
+    /** S3 location where query results are being stored */
+    s3OutputLocation: string;
+    /** AWS region for S3 operations */
+    s3Region: string;
+}
+export interface MappedQueryResultProcessorParams {
+    /**
+     *
+     */
+    athenaClient: AthenaClient;
+}
+/**
+ *@description Options for configuring the JsonFileAppender
+ */
+export interface JsonFileAppenderOptions {
+    /** Name of the file to append to */
+    fileName: string;
+    /** Directory path where the file is located */
+    directory: string;
+}

package/dist/types.js

@@ -0,0 +1,3 @@
+/** Maximum number of records to process in a single batch */
+export const MAX_BATCH_SIZE = 999;
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAOA,6DAA6D;AAC7D,MAAM,CAAC,MAAM,cAAc,GAAW,GAAG,CAAA"}

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@stilyng94/athena-query-client",
+  "version": "1.0.1",
+  "description": "An Athena client to query data from Athena utilizing Aws-S3 or direct results from Athena query",
+  "type": "module",
+  "main": "dist/index.js",
+  "homepage": "https://github.com/stilyng94/athena-query-client",
+  "bugs": {
+    "url": "https://github.com/stilyng94/athena-query-client"
+  },
+  "keywords": [
+    "athena",
+    "s3",
+    "aws"
+  ],
+  "author": {
+    "name": "Paul Osei Kofi",
+    "email": "oseipaulkofi@gmail.com"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/stilyng94/athena-query-client.git"
+  },
+  "files": [
+    "dist"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "@aws-sdk/client-athena": "^3.696.0",
+    "@aws-sdk/client-s3": "^3.717.0",
+    "csv-parse": "^5.6.0",
+    "rxjs": "^7.8.1"
+  },
+  "devDependencies": {
+    "@arethetypeswrong/cli": "^0.17.2",
+    "@biomejs/biome": "1.9.4",
+    "@changesets/cli": "^2.27.11",
+    "@total-typescript/tsconfig": "^1.0.4",
+    "@types/node": "^22.10.2",
+    "typescript": "^5.6.3"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "tsc",
+    "local-release": "changeset version && changeset publish",
+    "format-and-fix": "biome format --write .",
+    "lint-and-fix": "biome lint --write .",
+    "check": "biome check --write .",
+    "check-exports": "attw --pack . --ignore-rules=cjs-resolves-to-esm",
+    "pre-ci": "pnpm format-and-fix && pnpm lint-and-fix && pnpm check",
+    "ci": "pnpm build && pnpm check-exports"
+  }
+}