@mshick/dyno 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,231 @@
1
+ import createDebug from 'debug';
2
+ import fastq from 'fastq';
3
+ import { calculateDelay, delay, ensureError, reduceCapacity } from "../util.js";
4
+ const log = createDebug('dyno-requests');
5
+ /**
6
+ * Compacts results, removing the often redundant table name key
7
+ */
8
+ function compactSendCompletely(data) {
9
+ if ('Responses' in data) {
10
+ const responses = data.Responses;
11
+ data.Responses = responses
12
+ ? Object.values(responses)[0]
13
+ : responses;
14
+ }
15
+ if ('UnprocessedKeys' in data) {
16
+ const unprocessed = data.UnprocessedKeys;
17
+ data.UnprocessedKeys = unprocessed
18
+ ? Object.values(unprocessed)[0]
19
+ : unprocessed;
20
+ }
21
+ if ('UnprocessedItems' in data) {
22
+ const unprocessed = data.UnprocessedItems;
23
+ data.UnprocessedItems = unprocessed
24
+ ? Object.values(unprocessed)[0]
25
+ : unprocessed;
26
+ }
27
+ return data;
28
+ }
29
+ function _sendCompletely(requests, requestFactory, options, callback) {
30
+ const l = log.extend('sendCompletely');
31
+ const { concurrency = 1, maxRetries = 5 } = options;
32
+ l('requests:', requests.length, 'concurrency:', concurrency, 'maxRetries:', maxRetries);
33
+ const worker = (request, done) => {
34
+ const result = { error: null, data: {} };
35
+ let retryCount = 0;
36
+ const send = async (req) => {
37
+ const res = await req.send();
38
+ if (!res.data) {
39
+ l('send:error', res.error);
40
+ result.error = res.error;
41
+ return result;
42
+ }
43
+ if ('Responses' in res.data) {
44
+ if (res.data.Responses) {
45
+ result.data.Responses ??= {};
46
+ const responses = result.data.Responses;
47
+ for (const [table, item] of Object.entries(res.data.Responses)) {
48
+ responses[table] ??= [];
49
+ const bucket = responses[table];
50
+ for (const r of item) {
51
+ bucket.push(r);
52
+ }
53
+ }
54
+ }
55
+ }
56
+ if (res.data.ConsumedCapacity) {
57
+ result.data.ConsumedCapacity = reduceCapacity(result.data.ConsumedCapacity ?? {}, res.data.ConsumedCapacity);
58
+ }
59
+ let retry;
60
+ if ('UnprocessedItems' in res.data) {
61
+ if (res.data.UnprocessedItems && Object.keys(res.data.UnprocessedItems).length) {
62
+ retry = requestFactory({
63
+ RequestItems: res.data.UnprocessedItems,
64
+ ReturnConsumedCapacity: req.params.ReturnConsumedCapacity,
65
+ });
66
+ }
67
+ }
68
+ if ('UnprocessedKeys' in res.data) {
69
+ if (res.data.UnprocessedKeys && Object.keys(res.data.UnprocessedKeys).length) {
70
+ retry = requestFactory({
71
+ RequestItems: res.data.UnprocessedKeys,
72
+ ReturnConsumedCapacity: req.params.ReturnConsumedCapacity,
73
+ });
74
+ }
75
+ }
76
+ if (retry && retryCount < maxRetries) {
77
+ l('send:retry', retryCount);
78
+ await delay(calculateDelay(retryCount));
79
+ retryCount += 1;
80
+ return send(retry);
81
+ }
82
+ if ('UnprocessedKeys' in res.data) {
83
+ if (res.data.UnprocessedKeys) {
84
+ result.data.UnprocessedKeys ??= {};
85
+ const unprocessedKeys = result.data.UnprocessedKeys;
86
+ for (const [table, item] of Object.entries(res.data.UnprocessedKeys)) {
87
+ unprocessedKeys[table] ??= { Keys: [] };
88
+ const bucket = unprocessedKeys[table];
89
+ bucket.Keys ??= [];
90
+ for (const r of item.Keys ?? []) {
91
+ bucket.Keys.push(r);
92
+ }
93
+ }
94
+ }
95
+ }
96
+ if ('UnprocessedItems' in res.data) {
97
+ if (res.data.UnprocessedItems) {
98
+ result.data.UnprocessedItems ??= {};
99
+ const unprocessedItems = result.data.UnprocessedItems;
100
+ for (const [table, items] of Object.entries(res.data.UnprocessedItems)) {
101
+ unprocessedItems[table] ??= [];
102
+ const bucket = unprocessedItems[table];
103
+ for (const r of items) {
104
+ bucket.push(r);
105
+ }
106
+ }
107
+ }
108
+ }
109
+ return result;
110
+ };
111
+ void send(request)
112
+ .then((res) => {
113
+ done(null, res);
114
+ })
115
+ .catch((err) => {
116
+ result.error = ensureError(err);
117
+ done(null, result);
118
+ });
119
+ };
120
+ const drain = (results) => {
121
+ return () => {
122
+ l('drain:results', results.length);
123
+ const errors = [];
124
+ const data = {};
125
+ if (results.length) {
126
+ for (const res of results) {
127
+ if (res.error) {
128
+ errors.push(res.error);
129
+ }
130
+ if (!res.data) {
131
+ continue;
132
+ }
133
+ if ('Responses' in res.data) {
134
+ if (res.data.Responses) {
135
+ data.Responses ??= {};
136
+ const responses = data.Responses;
137
+ for (const [table, response] of Object.entries(res.data.Responses)) {
138
+ responses[table] ??= [];
139
+ const bucket = responses[table];
140
+ for (const r of response) {
141
+ bucket.push(r);
142
+ }
143
+ }
144
+ }
145
+ }
146
+ if ('UnprocessedItems' in res.data) {
147
+ if (res.data.UnprocessedItems && Object.keys(res.data.UnprocessedItems ?? {}).length) {
148
+ data.UnprocessedItems ??= {};
149
+ const unprocessedItems = data.UnprocessedItems;
150
+ for (const [table, items] of Object.entries(res.data.UnprocessedItems)) {
151
+ unprocessedItems[table] ??= [];
152
+ const bucket = unprocessedItems[table];
153
+ for (const r of items) {
154
+ bucket.push(r);
155
+ }
156
+ }
157
+ }
158
+ }
159
+ if ('UnprocessedKeys' in res.data) {
160
+ if (res.data.UnprocessedKeys && Object.keys(res.data.UnprocessedKeys ?? {}).length) {
161
+ data.UnprocessedKeys ??= {};
162
+ const unprocessedKeys = data.UnprocessedKeys;
163
+ for (const [table, items] of Object.entries(res.data.UnprocessedKeys)) {
164
+ unprocessedKeys[table] ??= { Keys: [] };
165
+ const bucket = unprocessedKeys[table];
166
+ bucket.Keys ??= [];
167
+ for (const r of items.Keys ?? []) {
168
+ bucket.Keys.push(r);
169
+ }
170
+ }
171
+ }
172
+ }
173
+ if (res.data.ConsumedCapacity) {
174
+ data.ConsumedCapacity = reduceCapacity(data.ConsumedCapacity ?? {}, res.data.ConsumedCapacity);
175
+ }
176
+ }
177
+ }
178
+ if (options.compact === true) {
179
+ callback(errors.length ? new AggregateError(errors, 'SendCompletely batch error') : undefined, compactSendCompletely(data));
180
+ return;
181
+ }
182
+ callback(errors.length ? new AggregateError(errors, 'SendCompletely batch error') : undefined, data);
183
+ };
184
+ };
185
+ const q = fastq(worker, concurrency);
186
+ const results = [];
187
+ q.drain = drain(results);
188
+ for (const req of requests) {
189
+ q.push(req, (_err, res) => {
190
+ if (res) {
191
+ results.push(res);
192
+ }
193
+ });
194
+ }
195
+ }
196
+ async function _sendCompletelyAsync(requests, requestFactory, options) {
197
+ return new Promise((resolve) => {
198
+ _sendCompletely(requests, requestFactory, options, (error, data) => {
199
+ resolve({ error, data });
200
+ });
201
+ });
202
+ }
203
+ export class SendCompletelyBatch {
204
+ requestFactory;
205
+ items;
206
+ options;
207
+ requests;
208
+ constructor(requestFactory, items, options) {
209
+ this.requestFactory = requestFactory;
210
+ this.items = items;
211
+ this.options = options;
212
+ this.requests = items.map((params) => requestFactory(params));
213
+ }
214
+ load(items) {
215
+ return new SendCompletelyBatch(this.requestFactory, items, this.options);
216
+ }
217
+ sendAll(optionsOrCb, cb) {
218
+ if (typeof optionsOrCb === 'function') {
219
+ _sendCompletely(this.requests, this.requestFactory, this.options, optionsOrCb);
220
+ }
221
+ else if (typeof cb === 'function') {
222
+ _sendCompletely(this.requests, this.requestFactory, { ...this.options, ...optionsOrCb }, cb);
223
+ }
224
+ else {
225
+ return _sendCompletelyAsync(this.requests, this.requestFactory, {
226
+ ...this.options,
227
+ ...optionsOrCb,
228
+ });
229
+ }
230
+ }
231
+ }
@@ -0,0 +1,32 @@
1
+ import type { ConsumedCapacity } from '@aws-sdk/client-dynamodb';
2
+ import type { BatchGetCommandInput, BatchGetCommandOutput, BatchWriteCommandInput, BatchWriteCommandOutput } from '@aws-sdk/lib-dynamodb';
3
+ import type { NativeAttributeValue } from '@aws-sdk/util-dynamodb';
4
+ import type { NativeAttributeMap } from '../types.ts';
5
+ export type BatchCommandInput = BatchGetCommandInput | BatchWriteCommandInput;
6
+ export type BatchCommandOutput = BatchGetCommandOutput | BatchWriteCommandOutput;
7
+ export type FetcherResponse<T extends BatchCommandOutput> = {
8
+ data: T;
9
+ error?: undefined;
10
+ } | {
11
+ error: unknown;
12
+ data?: undefined;
13
+ };
14
+ export type Fetcher<T extends BatchCommandInput> = {
15
+ params: T;
16
+ send: () => Promise<FetcherResponse<T extends BatchGetCommandInput ? BatchGetCommandOutput : BatchWriteCommandOutput>>;
17
+ };
18
+ export type BatchResult = {
19
+ error?: unknown;
20
+ data: {
21
+ Responses?: Record<string, Array<Record<string, NativeAttributeValue>>>;
22
+ ConsumedCapacity?: ConsumedCapacity;
23
+ UnprocessedKeys?: Record<string, {
24
+ Keys: NativeAttributeMap[];
25
+ }>;
26
+ UnprocessedItems?: Record<string, NativeAttributeMap[]>;
27
+ };
28
+ };
29
+ export type RequestFactory<U extends BatchCommandInput> = (params: U) => Fetcher<U>;
30
+ export type CompactOptions = {
31
+ compact?: boolean;
32
+ };
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,11 @@
1
+ import type { BatchGetCommandOutput, GetCommandOutput, QueryCommandOutput, ScanCommandOutput } from '@aws-sdk/lib-dynamodb';
2
+ export type ParseResponseOptions = {
3
+ /**
4
+ * Set to true to _not_ upgrade Uint8Arrays in responses to Buffers.
5
+ */
6
+ noBuffers?: boolean;
7
+ };
8
+ /**
9
+ * Parse the items in a variety of responses. Converts Uint8Arrays to Buffers. Mutates response items.
10
+ */
11
+ export declare function parseResponse<T extends Pick<GetCommandOutput, 'Item'> | Pick<ScanCommandOutput, 'Items'> | Pick<QueryCommandOutput, 'Items'> | Pick<BatchGetCommandOutput, 'Responses'>>(response: T, options: ParseResponseOptions): T;
@@ -0,0 +1,47 @@
1
+ import isPlainObject from 'lodash/isPlainObject.js';
2
+ function restoreItemBuffers(item) {
3
+ for (const [k, v] of Object.entries(item)) {
4
+ if (isPlainObject(v)) {
5
+ restoreItemBuffers(v);
6
+ continue;
7
+ }
8
+ if (Array.isArray(v)) {
9
+ for (const vv of v) {
10
+ if (isPlainObject(vv)) {
11
+ restoreItemBuffers(vv);
12
+ }
13
+ }
14
+ continue;
15
+ }
16
+ if (v instanceof Uint8Array) {
17
+ item[k] = Buffer.from(v);
18
+ }
19
+ }
20
+ return item;
21
+ }
22
+ /**
23
+ * Parse the items in a variety of responses. Converts Uint8Arrays to Buffers. Mutates response items.
24
+ */
25
+ export function parseResponse(response, options) {
26
+ if (options.noBuffers) {
27
+ return response;
28
+ }
29
+ if ('Item' in response) {
30
+ if (response.Item) {
31
+ restoreItemBuffers(response.Item);
32
+ }
33
+ }
34
+ if ('Items' in response) {
35
+ if (response.Items) {
36
+ response.Items.forEach(restoreItemBuffers);
37
+ }
38
+ }
39
+ if ('Responses' in response) {
40
+ if (response.Responses) {
41
+ for (const items of Object.values(response.Responses)) {
42
+ items.forEach(restoreItemBuffers);
43
+ }
44
+ }
45
+ }
46
+ return response;
47
+ }
@@ -0,0 +1,44 @@
1
+ import type { ReadableOptions } from 'node:stream';
2
+ import { type BatchWriteCommandInput, type DynamoDBDocument, type QueryCommandInput, type QueryCommandOutput, type ScanCommandInput, type ScanCommandOutput } from '@aws-sdk/lib-dynamodb';
3
+ import { type ParseResponseOptions } from './responses.ts';
4
+ import { DynoReadableStream } from './streams/dyno-readable-stream.ts';
5
+ import { DynoWritableStream } from './streams/dyno-writable-stream.ts';
6
+ import type { ParallelWritableOptions } from './streams/parallel-writable.ts';
7
+ export type ScanInput = Omit<ScanCommandInput, 'TableName'> & {
8
+ TableName: string;
9
+ /**
10
+ * Maximum number of pages of scan results to request. Set to `Infinity` to return all available data.
11
+ */
12
+ Pages?: number;
13
+ };
14
+ export type ScanOutput = Omit<ScanCommandOutput, '$metadata'>;
15
+ export type QueryInput = Omit<QueryCommandInput, 'TableName'> & {
16
+ TableName: string;
17
+ /**
18
+ * Maximum number of pages of scan results to request. Set to `Infinity` to return all available data.
19
+ */
20
+ Pages?: number;
21
+ };
22
+ export type QueryOutput = Omit<QueryCommandOutput, '$metadata'>;
23
+ export type ReadStreamOptions = ReadableOptions & ParseResponseOptions & {
24
+ mode?: 'scan' | 'query';
25
+ pageSize?: number;
26
+ };
27
+ /**
28
+ * Create a scan stream, reading the whole table unless limited.
29
+ */
30
+ export declare function createReadStream(client: DynamoDBDocument, { TableName, Pages, Limit, ExclusiveStartKey, ...commandInput }: ScanInput | QueryInput, { mode, pageSize, noBuffers, ...readableOptions }?: ReadStreamOptions): DynoReadableStream;
31
+ export type PutStreamInput = Omit<BatchWriteCommandInput, 'TableName' | 'RequestItems'> & {
32
+ TableName: string;
33
+ };
34
+ export type PutStreamOptions = ParallelWritableOptions & {
35
+ pageSize?: number;
36
+ maxRetries?: number;
37
+ maxBatchSize?: number;
38
+ docMode?: boolean;
39
+ };
40
+ /**
41
+ * Create a put stream, writing batches to DynamoDB with support for valid
42
+ * request size checking. Supports parallel writes and delayed retry of unprocessed items.
43
+ */
44
+ export declare function createPutStream(client: DynamoDBDocument, commandInput: PutStreamInput, { concurrency, pageSize, maxRetries, maxBatchSize, ...writableOptions }?: PutStreamOptions): DynoWritableStream;
package/dist/stream.js ADDED
@@ -0,0 +1,205 @@
1
+ import assert from 'node:assert';
2
+ import { BatchWriteCommand, QueryCommand, ScanCommand, } from '@aws-sdk/lib-dynamodb';
3
+ import { isThrottlingError } from '@smithy/service-error-classification';
4
+ import createDebug from 'debug';
5
+ import { parseResponse } from "./responses.js";
6
+ import { DynoReadableStream } from "./streams/dyno-readable-stream.js";
7
+ import { DynoWritableStream } from "./streams/dyno-writable-stream.js";
8
+ import { calculateDelay, getItemSize } from "./util.js";
9
+ const log = createDebug('dyno-stream');
10
+ const logRead = log.extend('read');
11
+ const logPut = log.extend('put');
12
+ function getLimit(limit, pageSize) {
13
+ if (limit !== undefined && pageSize !== undefined) {
14
+ return Math.min(limit, pageSize);
15
+ }
16
+ return limit ?? pageSize;
17
+ }
18
+ /**
19
+ * Create a scan stream, reading the whole table unless limited.
20
+ */
21
+ export function createReadStream(client, { TableName, Pages = Number.POSITIVE_INFINITY, Limit, ExclusiveStartKey, ...commandInput }, { mode = 'scan', pageSize, noBuffers, ...readableOptions } = {}) {
22
+ logRead('createReadStream');
23
+ assert(Pages > 0, 'Pages must be an integer greater than 0');
24
+ let pending = false;
25
+ let items = [];
26
+ function push() {
27
+ logRead('push', 'items:', items.length);
28
+ let status = true;
29
+ while (status && items.length) {
30
+ status = readable.push(items.shift());
31
+ }
32
+ return status;
33
+ }
34
+ function finish() {
35
+ logRead('finish', 'count:', readable.Count, 'scanned:', readable.ScannedCount, 'readableLength:', readable.readableLength, 'highWater:', readable.readableHighWaterMark);
36
+ readable.push(null);
37
+ }
38
+ function request(nextLimit) {
39
+ logRead('request', 'limit:', nextLimit);
40
+ pending = true;
41
+ let command;
42
+ if (mode === 'scan') {
43
+ command = new ScanCommand({
44
+ ...commandInput,
45
+ TableName,
46
+ ExclusiveStartKey: readable.LastEvaluatedKey,
47
+ Limit: getLimit(pageSize, nextLimit),
48
+ });
49
+ }
50
+ else {
51
+ command = new QueryCommand({
52
+ ...commandInput,
53
+ TableName,
54
+ ExclusiveStartKey: readable.LastEvaluatedKey,
55
+ Limit: getLimit(pageSize, nextLimit),
56
+ });
57
+ }
58
+ logRead('sending command input:', command.input);
59
+ client.send(command, (error, response) => {
60
+ logRead('got response');
61
+ pending = false;
62
+ if (error) {
63
+ logRead('error', { error, response });
64
+ readable.emit('error', error);
65
+ return;
66
+ }
67
+ if (!response) {
68
+ logRead('empty');
69
+ readable.update();
70
+ read();
71
+ return;
72
+ }
73
+ parseResponse(response, { noBuffers });
74
+ if (response.Items) {
75
+ items = [...response.Items];
76
+ }
77
+ readable.update(response);
78
+ if (readable.hasNextPage()) {
79
+ logRead('has another page');
80
+ read();
81
+ return;
82
+ }
83
+ read();
84
+ });
85
+ }
86
+ function read() {
87
+ logRead('read', 'pending:', pending, 'items:', items.length);
88
+ /**
89
+ * If status is false, highwater mark has been reached and we should not
90
+ * push more data until `read` is called again.
91
+ */
92
+ const status = push();
93
+ if (items.length) {
94
+ return;
95
+ }
96
+ const nextLimit = readable.getNextLimit();
97
+ if (readable.isLastPage() || (nextLimit !== undefined && nextLimit <= 0)) {
98
+ finish();
99
+ return;
100
+ }
101
+ if (status && !pending) {
102
+ request(nextLimit);
103
+ }
104
+ }
105
+ const readable = new DynoReadableStream({ objectMode: true, read, ...readableOptions }, { TableName, LastEvaluatedKey: ExclusiveStartKey, Limit, Pages });
106
+ return readable;
107
+ }
108
+ /**
109
+ * Create a put stream, writing batches to DynamoDB with support for valid
110
+ * request size checking. Supports parallel writes and delayed retry of unprocessed items.
111
+ */
112
+ export function createPutStream(client, commandInput, { concurrency = 5, pageSize = 25, maxRetries = 5, maxBatchSize = 16 * 1024 * 1024, ...writableOptions } = {}) {
113
+ const { TableName } = commandInput;
114
+ let pageItems = [];
115
+ let pageItemsSize = 0;
116
+ const resetPageItems = () => {
117
+ pageItemsSize = 0;
118
+ pageItems = [];
119
+ };
120
+ const pushPageItem = (item, itemSize) => {
121
+ pageItemsSize += itemSize;
122
+ pageItems.push({ PutRequest: { Item: item } });
123
+ };
124
+ function write(item, _encoding, callback) {
125
+ const nextItemSize = getItemSize(item);
126
+ const nextPageItemsSize = pageItemsSize + nextItemSize;
127
+ // If adding the next item pushes the batch over the limit, write and push onto the next page
128
+ if (nextPageItemsSize > maxBatchSize) {
129
+ logPut('write - writing page', { maxBatchSize, nextPageItemsSize });
130
+ writePage(pageItems, 0, callback);
131
+ resetPageItems();
132
+ pushPageItem(item, nextItemSize);
133
+ return;
134
+ }
135
+ pushPageItem(item, nextItemSize);
136
+ // If the page size is hit, write
137
+ if (pageItems.length === pageSize) {
138
+ logPut('write - writing page', {
139
+ pageSize,
140
+ pageItemsLength: pageItems.length,
141
+ });
142
+ writePage(pageItems, 0, callback);
143
+ resetPageItems();
144
+ return;
145
+ }
146
+ callback();
147
+ }
148
+ function _done(callback) {
149
+ return (error) => {
150
+ writable.done();
151
+ callback(error);
152
+ };
153
+ }
154
+ function final(callback) {
155
+ const done = _done(callback);
156
+ if (!pageItems.length) {
157
+ done();
158
+ return;
159
+ }
160
+ logPut('final - writing page', { pageItemsLength: pageItems.length });
161
+ writePage(pageItems, 0, done);
162
+ pageItems = [];
163
+ }
164
+ function writePage(items, retryCount, callback) {
165
+ const command = new BatchWriteCommand({
166
+ RequestItems: {
167
+ [TableName]: items,
168
+ },
169
+ ...commandInput,
170
+ });
171
+ logPut('sending command', { retryCount });
172
+ client.send(command, (error, response) => {
173
+ if (error) {
174
+ logPut({ error, response });
175
+ }
176
+ // In these cases the items were not processed, so retry the items originally provided
177
+ if (error && isThrottlingError(error) && !response) {
178
+ setTimeout(() => {
179
+ writePage(items, retryCount + 1, callback);
180
+ }, calculateDelay(retryCount));
181
+ return;
182
+ }
183
+ writable.update(response, items);
184
+ const writableUnprocessedItems = writable.UnprocessedItems;
185
+ // If there are any dangling items and retries are left, attempt to process them
186
+ if (writableUnprocessedItems?.length && retryCount < maxRetries) {
187
+ const page = writableUnprocessedItems.splice(0, pageSize);
188
+ setTimeout(() => {
189
+ writePage(page, retryCount + 1, callback);
190
+ }, calculateDelay(retryCount));
191
+ return;
192
+ }
193
+ callback(error);
194
+ });
195
+ }
196
+ const writable = new DynoWritableStream({
197
+ TableName,
198
+ concurrency,
199
+ write,
200
+ final,
201
+ objectMode: true,
202
+ ...writableOptions,
203
+ });
204
+ return writable;
205
+ }
@@ -0,0 +1,24 @@
1
+ import { Readable, type ReadableOptions } from 'node:stream';
2
+ import type { ConsumedCapacity } from '@aws-sdk/client-dynamodb';
3
+ import type { QueryCommandOutput, ScanCommandOutput } from '@aws-sdk/lib-dynamodb';
4
+ export type DynoReadableStreamOptions = {
5
+ TableName: string;
6
+ LastEvaluatedKey?: ScanCommandOutput['LastEvaluatedKey'];
7
+ Limit?: number;
8
+ Pages?: number;
9
+ };
10
+ export declare class DynoReadableStream extends Readable {
11
+ Count: number;
12
+ ScannedCount: number;
13
+ LastEvaluatedKey: ScanCommandOutput['LastEvaluatedKey'];
14
+ ConsumedCapacity?: ConsumedCapacity;
15
+ Limit: number | undefined;
16
+ Pages: number;
17
+ TableName: string;
18
+ private _updated;
19
+ constructor(opts: ReadableOptions, dynoOpts: DynoReadableStreamOptions);
20
+ update(response?: ScanCommandOutput | QueryCommandOutput): void;
21
+ hasNextPage(): boolean;
22
+ isLastPage(): boolean;
23
+ getNextLimit(): number | undefined;
24
+ }
@@ -0,0 +1,41 @@
1
+ import { Readable } from 'node:stream';
2
+ import { reduceCapacity } from "../util.js";
3
+ export class DynoReadableStream extends Readable {
4
+ Count = 0;
5
+ ScannedCount = 0;
6
+ LastEvaluatedKey;
7
+ ConsumedCapacity;
8
+ Limit;
9
+ Pages;
10
+ TableName;
11
+ _updated = false;
12
+ constructor(opts, dynoOpts) {
13
+ super(opts);
14
+ this.TableName = dynoOpts.TableName;
15
+ this.Limit = dynoOpts.Limit;
16
+ this.Pages = dynoOpts.Pages ?? Number.POSITIVE_INFINITY;
17
+ this.LastEvaluatedKey = dynoOpts.LastEvaluatedKey;
18
+ }
19
+ update(response) {
20
+ this.Count += response?.Count ?? 0;
21
+ this.ScannedCount += response?.ScannedCount ?? 0;
22
+ this.LastEvaluatedKey = response?.LastEvaluatedKey;
23
+ this.ConsumedCapacity = response?.ConsumedCapacity
24
+ ? reduceCapacity(this.ConsumedCapacity ?? {}, response.ConsumedCapacity)
25
+ : this.ConsumedCapacity;
26
+ this.Pages -= 1;
27
+ this._updated = true;
28
+ }
29
+ hasNextPage() {
30
+ return this._updated ? Boolean(this.LastEvaluatedKey) : true;
31
+ }
32
+ isLastPage() {
33
+ return this._updated && (!this.LastEvaluatedKey || this.Pages <= 0);
34
+ }
35
+ getNextLimit() {
36
+ if (this.Limit === undefined) {
37
+ return;
38
+ }
39
+ return this.Limit - this.Count;
40
+ }
41
+ }
@@ -0,0 +1,18 @@
1
+ import type { ConsumedCapacity } from '@aws-sdk/client-dynamodb';
2
+ import type { BatchWriteCommandInput, BatchWriteCommandOutput } from '@aws-sdk/lib-dynamodb';
3
+ import { ParallelWritable, type ParallelWritableOptions } from './parallel-writable.ts';
4
+ export type BatchWriteRequestItem = NonNullable<BatchWriteCommandInput['RequestItems']>[string][number];
5
+ export type BatchWriteUnprocessedItem = NonNullable<BatchWriteCommandOutput['UnprocessedItems']>[string][number];
6
+ export type DynoWritableStreamOptions = ParallelWritableOptions & {
7
+ TableName: string;
8
+ };
9
+ export declare class DynoWritableStream extends ParallelWritable {
10
+ Count: number;
11
+ UnprocessedItems: BatchWriteUnprocessedItem[] | undefined;
12
+ ConsumedCapacity?: ConsumedCapacity;
13
+ TableName: string;
14
+ private readonly _consumedCapacity;
15
+ constructor(opts: DynoWritableStreamOptions);
16
+ update(response: BatchWriteCommandOutput | undefined, items: BatchWriteRequestItem[]): void;
17
+ done(): void;
18
+ }