@stilyng94/athena-query-client 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -2,7 +2,17 @@ import { finished } from 'node:stream/promises';
|
|
|
2
2
|
import { GetQueryResultsCommand, } from '@aws-sdk/client-athena';
|
|
3
3
|
import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
|
4
4
|
import { parse } from 'csv-parse';
|
|
5
|
+
import { EMPTY, defer, firstValueFrom, from, of, throwError, timer, } from 'rxjs';
|
|
5
6
|
import { MAX_BATCH_SIZE, } from './types.js';
|
|
7
|
+
import { catchError, concatWith, expand, filter, switchMap, } from 'rxjs/operators';
|
|
8
|
+
function validateBatchSize(maxBatchSize, batchSize) {
|
|
9
|
+
if (!batchSize)
|
|
10
|
+
return maxBatchSize;
|
|
11
|
+
if (batchSize > maxBatchSize) {
|
|
12
|
+
throw new Error(`Batch size cannot be greater than ${maxBatchSize}`);
|
|
13
|
+
}
|
|
14
|
+
return batchSize;
|
|
15
|
+
}
|
|
6
16
|
/**
|
|
7
17
|
* Processes Athena query results stored in S3
|
|
8
18
|
*/
|
|
@@ -94,12 +104,7 @@ export class S3QueryResultProcessor {
|
|
|
94
104
|
* @throws {Error} If batch size exceeds maximum allowed
|
|
95
105
|
*/
|
|
96
106
|
#validateBatchSize(batchSize) {
|
|
97
|
-
|
|
98
|
-
return MAX_BATCH_SIZE;
|
|
99
|
-
if (batchSize > MAX_BATCH_SIZE) {
|
|
100
|
-
throw new Error(`Batch size cannot be greater than ${MAX_BATCH_SIZE}`);
|
|
101
|
-
}
|
|
102
|
-
return batchSize;
|
|
107
|
+
return validateBatchSize(this.#batchSize, batchSize);
|
|
103
108
|
}
|
|
104
109
|
/**
|
|
105
110
|
* Fetches an object from S3
|
|
@@ -144,12 +149,16 @@ export class S3QueryResultProcessor {
|
|
|
144
149
|
*/
|
|
145
150
|
export class MappedQueryResultProcessor {
|
|
146
151
|
config;
|
|
152
|
+
#batchSize = MAX_BATCH_SIZE;
|
|
153
|
+
#shouldPaginate = true;
|
|
147
154
|
/**
|
|
148
155
|
* Creates a new MappedQueryResultProcessor
|
|
149
156
|
* @param {MappedQueryResultProcessorParams} config - Configuration parameters
|
|
150
157
|
*/
|
|
151
158
|
constructor(config) {
|
|
152
159
|
this.config = config;
|
|
160
|
+
this.#batchSize = this.#validateBatchSize(config.MaxResults);
|
|
161
|
+
this.#shouldPaginate = config.paginateResults ?? true;
|
|
153
162
|
}
|
|
154
163
|
/**
|
|
155
164
|
* Processes query results by mapping them to objects
|
|
@@ -159,8 +168,8 @@ export class MappedQueryResultProcessor {
|
|
|
159
168
|
async processResults(queryExecutionId) {
|
|
160
169
|
console.log('Fetching results for QueryExecutionId:', queryExecutionId);
|
|
161
170
|
try {
|
|
162
|
-
const
|
|
163
|
-
return this.#extractRows(resultSet);
|
|
171
|
+
const resultSets = await firstValueFrom(this.#fetchQueryResults(queryExecutionId, undefined));
|
|
172
|
+
return resultSets.flatMap((resultSet) => this.#extractRows(resultSet));
|
|
164
173
|
}
|
|
165
174
|
catch (error) {
|
|
166
175
|
throw new Error(`Error processing results: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
@@ -169,15 +178,32 @@ export class MappedQueryResultProcessor {
|
|
|
169
178
|
/**
|
|
170
179
|
* Fetches query results from Athena
|
|
171
180
|
* @param {string} queryExecutionId - The ID of the query execution
|
|
172
|
-
* @
|
|
173
|
-
* @
|
|
181
|
+
* @param {string | undefined} nextToken - The pagination token
|
|
182
|
+
* @returns {Promise<ResultSet[]>} Promise resolving to Athena ResultSet
|
|
183
|
+
* @throws {Error} If results are undefined
|
|
174
184
|
*/
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
185
|
+
#fetchQueryResults(queryExecutionId, nextToken) {
|
|
186
|
+
return defer(() => from(this.config.athenaClient.send(new GetQueryResultsCommand({
|
|
187
|
+
QueryExecutionId: queryExecutionId,
|
|
188
|
+
MaxResults: this.#batchSize,
|
|
189
|
+
NextToken: nextToken,
|
|
190
|
+
})))).pipe(switchMap((response) => {
|
|
191
|
+
if (!response.ResultSet) {
|
|
192
|
+
return throwError(() => new Error(`Query results are undefined for QueryExecutionId: ${queryExecutionId}`));
|
|
193
|
+
}
|
|
194
|
+
const resultObservable = of(response.ResultSet);
|
|
195
|
+
if (this.#shouldPaginate && response.NextToken) {
|
|
196
|
+
console.log('Fetching next page of results...');
|
|
197
|
+
return resultObservable.pipe(concatWith(of(response.NextToken)));
|
|
198
|
+
}
|
|
199
|
+
console.log('Query results fetching complete');
|
|
200
|
+
return resultObservable; // Only emit the current ResultSet
|
|
201
|
+
}), expand((resultOrNextToken) => typeof resultOrNextToken === 'string' // If it's a NextToken, fetch the next page
|
|
202
|
+
? timer(500).pipe(switchMap(() => this.#fetchQueryResults(queryExecutionId, resultOrNextToken)))
|
|
203
|
+
: EMPTY), filter((resultOrNextToken) => typeof resultOrNextToken !== 'string'), catchError((error) => {
|
|
204
|
+
console.error(`Error fetching query results for QueryExecutionId: ${queryExecutionId}`, error);
|
|
205
|
+
return throwError(() => new Error(`Failed to fetch query results for QueryExecutionId: ${queryExecutionId}. ${error.message}`));
|
|
206
|
+
}));
|
|
181
207
|
}
|
|
182
208
|
/**
|
|
183
209
|
* Extracts column headers from result rows
|
|
@@ -228,5 +254,14 @@ export class MappedQueryResultProcessor {
|
|
|
228
254
|
}
|
|
229
255
|
return Rows.slice(1).map((row) => this.#mapRowToObject(row, headers));
|
|
230
256
|
}
|
|
257
|
+
/**
|
|
258
|
+
* Validates the batch size configuration
|
|
259
|
+
* @param {number} [batchSize] - The batch size to validate
|
|
260
|
+
* @returns {number} The validated batch size
|
|
261
|
+
* @throws {Error} If batch size exceeds maximum allowed
|
|
262
|
+
*/
|
|
263
|
+
#validateBatchSize(batchSize) {
|
|
264
|
+
return validateBatchSize(this.#batchSize, batchSize);
|
|
265
|
+
}
|
|
231
266
|
}
|
|
232
267
|
//# sourceMappingURL=query-results-processor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"query-results-processor.js","sourceRoot":"","sources":["../src/query-results-processor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EACL,sBAAsB,GAGvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC/D,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EACL,cAAc,GAIf,MAAM,YAAY,CAAA;AAEnB;;GAEG;AACH,MAAM,OAAO,sBAAsB;IASJ;IARpB,UAAU,GAAG,cAAc,CAAA;IAC3B,SAAS,CAAU;IACnB,iBAAiB,CAAQ;IAElC;;;OAGG;IACH,YAA6B,MAAoC;QAApC,WAAM,GAAN,MAAM,CAA8B;QAC/D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QAC3D,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,gBAAgB,CAAA;QAChD,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC5D,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAAC,gBAAwB;QAC3C,iCAAiC;QACjC,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,iBAAiB,IAAI,gBAAgB,MAAM,CAAA;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAA;QACpD,OAAO,CAAC,GAAG,CAAC,iCAAiC,EAAE,UAAU,CAAC,CAAA;QAE1D,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC7D,OAAO,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,wBAAwB,CAAC,MAAgB;QAC7C,MAAM,KAAK,GAAc,EAAE,CAAA;QAC3B,IAAI,cAAc,GAAG,CAAC,CAAA;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,EAAE,CAAC,CACjE,CAAA;QAED,IAAI,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE;gBAC/B,IAAI,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;gBAC1B,OAAO,MAAM,KAAK,IAAI,EAAE,CAAC;oBACvB,wBAAwB;oBACxB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;oBAClB,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpC,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,CAAC,MAAM,aAAa,CAAC,CAAA;wBAC7D,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;wBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAA;wBAC9B,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA,CAAC,0BAA0B;wBAC3C,OAAO,CAAC,GAAG,CAAC,4BAA4B,cAAc,EAAE,CAAC,CAAA;oBAC3D,CAAC;oBACD,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;gBACxB,CAAC;YACH,CAAC,CAAC,CAAA;YAEF,+BAA+B;YAC/B,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAA;YACtB,4BAA4B;YAC5B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,6BAA6B,KAAK,CAAC,MAAM,aAAa,CAAC,CAAA;gBACnE,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;gBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAA;gBAC9B,OAAO,CAAC,GAAG,CAAC,kCAAkC,cAAc,EAAE,CAAC,CAAA;YACjE,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAA;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,6BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAAC,KAAgB;QAClC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,2BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,kBAAkB,CAAC,SAAkB;QACnC,
|
|
1
|
+
{"version":3,"file":"query-results-processor.js","sourceRoot":"","sources":["../src/query-results-processor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EACL,sBAAsB,GAGvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC/D,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EACL,KAAK,EAEL,KAAK,EACL,cAAc,EACd,IAAI,EACJ,EAAE,EACF,UAAU,EACV,KAAK,GACN,MAAM,MAAM,CAAA;AACb,OAAO,EACL,cAAc,GAIf,MAAM,YAAY,CAAA;AAEnB,OAAO,EACL,UAAU,EACV,UAAU,EACV,MAAM,EACN,MAAM,EACN,SAAS,GACV,MAAM,gBAAgB,CAAA;AAEvB,SAAS,iBAAiB,CAAC,YAAoB,EAAE,SAAkB;IACjE,IAAI,CAAC,SAAS;QAAE,OAAO,YAAY,CAAA;IACnC,IAAI,SAAS,GAAG,YAAY,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,qCAAqC,YAAY,EAAE,CAAC,CAAA;IACtE,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,sBAAsB;IASJ;IARpB,UAAU,GAAG,cAAc,CAAA;IAC3B,SAAS,CAAU;IACnB,iBAAiB,CAAQ;IAElC;;;OAGG;IACH,YAA6B,MAAoC;QAApC,WAAM,GAAN,MAAM,CAA8B;QAC/D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QAC3D,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,gBAAgB,CAAA;QAChD,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC5D,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAAC,gBAAwB;QAC3C,iCAAiC;QACjC,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,iBAAiB,IAAI,gBAAgB,MAAM,CAAA;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAA;QACpD,OAAO,CAAC,GAAG,CAAC,iCAAiC,EAAE,UAAU,CAAC,CAAA;QAE1D,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC7D,OAAO,IAAI,CAAC,wBAAwB,CAAC,cAAc,CAAC,CAAA;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,wBAAwB,CAAC,MAAgB;QAC7C,MAAM,KAAK,GAAc,EAAE,CAAA;QAC3B,IAAI,cAAc,GAAG,CAAC,CAAA;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,EAAE,CAAC,CACjE,CAAA;QAED,IAAI,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE;gBAC/B,IAAI,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;gBAC1B,OAAO,MAAM,KAAK,IAAI,EAAE,CAAC;oBACvB,wBAAwB;oBACxB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;oBAClB,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpC,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,CAAC,MAAM,aAAa,CAAC,CAAA;wBAC7D,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;wBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAA;wBAC9B,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA,CAAC,0BAA0B;wBAC3C,OAAO,CAAC,GAAG,CAAC,4BAA4B,cAAc,EAAE,CAAC,CAAA;oBAC3D,CAAC;oBACD,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;gBACxB,CAAC;YACH,CAAC,CAAC,CAAA;YAEF,+BAA+B;YAC/B,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAA;YACtB,4BAA4B;YAC5B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,6BAA6B,KAAK,CAAC,MAAM,aAAa,CAAC,CAAA;gBACnE,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;gBAC/B,cAAc,IAAI,KAAK,CAAC,MAAM,CAAA;gBAC9B,OAAO,CAAC,GAAG,CAAC,kCAAkC,cAAc,EAAE,CAAC,CAAA;YACjE,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAA;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,6BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAAC,KAAgB;QAClC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,2BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,kBAAkB,CAAC,SAAkB;QACnC,OAAO,iBAAiB,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;IACtD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,MAAc,EAAE,GAAW;QAC9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CACxC,IAAI,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CACtC,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,IAAI,GAAG,EAAE,CAAC,CAAA;QAC3D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAA;QAExD,OAAO,QAAQ,CAAC,IAAgB,CAAA;IAClC,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,GAAW;QACrB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;YAC9B,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YAE/C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;YACxD,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;aACjC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,mBACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,0BAA0B;IAQR;IAPpB,UAAU,GAAG,cAAc,CAAA;IAC3B,eAAe,GAAY,IAAI,CAAA;IAExC;;;OAGG;IACH,YAA6B,MAAwC;QAAxC,WAAM,GAAN,MAAM,CAAkC;QACnE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;QAC5D,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,IAAI,CAAA;IACvD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAClB,gBAAwB;QAExB,OAAO,CAAC,GAAG,CAAC,wCAAwC,EAAE,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,cAAc,CACrC,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,SAAS,CAAC,CACrD,CAAA;YACD,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAA;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,6BACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC3C,EAAE,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,kBAAkB,CAChB,gBAAwB,EACxB,SAA6B;QAE7B,OAAO,KAAK,CAAC,GAAG,EAAE,CAChB,IAAI,CACF,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAC3B,IAAI,sBAAsB,CAAC;YACzB,gBAAgB,EAAE,gBAAgB;YAClC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,SAAS;SACrB,CAAC,CACH,CACF,CACF,CAAC,IAAI,CACJ,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE;YACrB,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;gBACxB,OAAO,UAAU,CACf,GAAG,EAAE,CACH,IAAI,KAAK,CACP,qDAAqD,gBAAgB,EAAE,CACxE,CACJ,CAAA;YACH,CAAC;YAED,MAAM,gBAAgB,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;YAE/C,IAAI,IAAI,CAAC,eAAe,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;gBAC/C,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAA;gBAC/C,OAAO,gBAAgB,CAAC,IAAI,CAC1B,UAAU,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CACnC,CAAA;YACH,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAA;YAC9C,OAAO,gBAAgB,CAAA,CAAC,kCAAkC;QAC5D,CAAC,CAAC,EACF,MAAM,CACJ,CAAC,iBAAiB,EAAE,EAAE,CACpB,OAAO,iBAAiB,KAAK,QAAQ,CAAC,2CAA2C;YAC/E,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CACb,SAAS,CAAC,GAAG,EAAE,CACb,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAC7D,CACF;YACH,CAAC,CAAC,KAAK,CACZ,EACD,MAAM,CACJ,CAAC,iBAAiB,EAAE,EAAE,CAAC,OAAO,iBAAiB,KAAK,QAAQ,CAC7D,EACD,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE;YACnB,OAAO,CAAC,KAAK,CACX,sDAAsD,gBAAgB,EAAE,EACxE,KAAK,CACN,CAAA;YACD,OAAO,UAAU,CACf,GAAG,EAAE,CACH,IAAI,KAAK,CACP,uDAAuD,gBAAgB,KAAK,KAAK,CAAC,OAAO,EAAE,CAC5F,CACJ,CAAA;QACH,CAAC,CAAC,CACH,CAAA;IACH,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,IAAW;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAQ,EAAE,OAAiB;QACzC,MAAM,SAAS,GAA2B,EAAE,CAAA;QAE5C,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACjC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;YAC7B,IAAI,MAAM,EAAE,CAAC;gBACX,SAAS,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,IAAI,EAAE,CAAA;YAC9C,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;;;;OAOG;IACH,YAAY,CAAC,SAAoB;QAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;YAC5C,OAAO,EAAE,CAAA;QACX,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAC1C,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAA;IACvE,CAAC;IAED;;;;;OAKG;IACH,kBAAkB,CAAC,SAAkB;QACnC,OAAO,iBAAiB,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;IACtD,CAAC;CACF"}
|
package/dist/types.d.ts
CHANGED
|
@@ -46,6 +46,7 @@ export interface S3QueryResultProcessorParams {
|
|
|
46
46
|
onComplete?: (data?: unknown) => Promise<void>;
|
|
47
47
|
/**
|
|
48
48
|
* Optional: The maximum number of records to process in a single batch
|
|
49
|
+
* should be less than or equal to 999
|
|
49
50
|
* @default 999
|
|
50
51
|
*/
|
|
51
52
|
batchSize?: number;
|
|
@@ -61,9 +62,20 @@ export interface S3QueryResultProcessorParams {
|
|
|
61
62
|
}
|
|
62
63
|
export interface MappedQueryResultProcessorParams {
|
|
63
64
|
/**
|
|
64
|
-
*
|
|
65
|
+
* Athena client for executing queries
|
|
65
66
|
*/
|
|
66
67
|
athenaClient: AthenaClient;
|
|
68
|
+
/**
|
|
69
|
+
* Optional: Maximum number of records per query
|
|
70
|
+
* should be less than or equal to 999
|
|
71
|
+
* @default 999
|
|
72
|
+
*/
|
|
73
|
+
MaxResults?: number;
|
|
74
|
+
/**
|
|
75
|
+
* Should results be paginated. Athena has a limit of 1000 records per query, so this is useful for large datasets
|
|
76
|
+
* @default true
|
|
77
|
+
*/
|
|
78
|
+
paginateResults?: boolean;
|
|
67
79
|
}
|
|
68
80
|
/**
|
|
69
81
|
*@description Options for configuring the JsonFileAppender
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@stilyng94/athena-query-client",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "An Athena client to query data from Athena utilizing Aws-S3 or direct results from Athena query",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
"@aws-sdk/client-athena": "^3.696.0",
|
|
30
30
|
"@aws-sdk/client-s3": "^3.717.0",
|
|
31
31
|
"csv-parse": "^5.6.0",
|
|
32
|
+
"pino": "^9.6.0",
|
|
32
33
|
"rxjs": "^7.8.1"
|
|
33
34
|
},
|
|
34
35
|
"devDependencies": {
|