@mshick/dyno 0.1.0

package/dist/requests/requests.js

@@ -0,0 +1,225 @@
+import createDebug from 'debug';
+import { parseResponse } from "../responses.js";
+import { getItemSize } from "../util.js";
+import { SendAllBatch } from "./send-all.js";
+import { SendCompletelyBatch } from "./send-completely.js";
+const log = createDebug('dyno-requests');
+function _batchGetItemRequests(params) {
+    const requestItems = params.RequestItems;
+    if (!requestItems) {
+        throw new Error('RequestItems is required');
+    }
+    const batchedItems = Object.keys(requestItems).reduce((paramSet, tableName) => {
+        const keys = requestItems[tableName]?.Keys;
+        if (!keys) {
+            return paramSet;
+        }
+        const toMake = [...keys];
+        return chop(toMake);
+        function chop(requestsToMake) {
+            // request set that we're building — paramSet is seeded with one entry
+            // and only grows, so paramSet[paramSet.length - 1] is always defined.
+            const lastParams = paramSet[paramSet.length - 1];
+            if (!lastParams) {
+                return paramSet;
+            }
+            const requests = lastParams.RequestItems ?? {};
+            const requestsTable = requests[tableName] ?? { Keys: [] };
+            requests[tableName] = requestsTable;
+            // count existing requests in the current params
+            const count = Object.keys(requests).reduce((count, tableName) => {
+                return count + (requests[tableName]?.Keys?.length ?? 0);
+            }, 0);
+            // gather more from the requested params
+            const more = requestsToMake.splice(0, 100 - count);
+            // add them to the request set
+            requestsTable.Keys = (requestsTable.Keys ?? []).concat(more);
+            // if there are no requests left, return the modified paramSet
+            if (!requestsToMake.length) {
+                return paramSet;
+            }
+            // otherwise start a new request set
+            paramSet.push({
+                RequestItems: {},
+                ReturnConsumedCapacity: params.ReturnConsumedCapacity,
+            });
+            return chop(requestsToMake);
+        }
+    }, [
+        {
+            RequestItems: {},
+            ReturnConsumedCapacity: params.ReturnConsumedCapacity,
+        },
+    ]);
+    return batchedItems;
+}
+function _batchWriteItemRequests(params, { maxLength = 25, maxSize = 16 * 1024 * 1024 } = {}) {
+    const requestItems = params.RequestItems;
+    if (!requestItems) {
+        throw new Error('RequestItems is required');
+    }
+    const batchedItems = Object.keys(requestItems).reduce((paramSet, tableName) => {
+        const reqs = requestItems[tableName];
+        if (!reqs) {
+            return paramSet;
+        }
+        return chop([...reqs]);
+        function chop(requestsToMake) {
+            // paramSet is seeded with one entry and only grows.
+            const lastParams = paramSet[paramSet.length - 1];
+            if (!lastParams) {
+                return paramSet;
+            }
+            const requests = lastParams.RequestItems ?? {};
+            const requestsTable = requests[tableName] ?? [];
+            requests[tableName] = requestsTable;
+            // count existing requests in the current params
+            const count = Object.values(requests).reduce((c, r) => {
+                return c + r.length;
+            }, 0);
+            // find existing requests size
+            let size = Object.values(requests).reduce((s, request) => {
+                return (s +
+                    request.reduce((s, r) => {
+                        return s + (r.PutRequest?.Item ? getItemSize(r.PutRequest.Item) : 0);
+                    }, 0));
+            }, 0);
+            // Add requests one by one until it would put us over the size limit
+            const startingCount = count;
+            for (let i = 0; i < maxLength - count; i++) {
+                const next = requestsToMake[0];
+                if (!next) {
+                    return paramSet;
+                }
+                const nextSize = next.PutRequest?.Item ? getItemSize(next.PutRequest.Item) : 0;
+                if (size + nextSize > maxSize) {
+                    break;
+                }
+                size += nextSize;
+                const itemToPush = requestsToMake.shift();
+                if (itemToPush) {
+                    requestsTable.push(itemToPush);
+                }
+            }
+            // if there are no requests left, return the modified paramSet
+            if (!requestsToMake.length) {
+                return paramSet;
+            }
+            // If the next item couldn't fit in an empty params set, it's larger than
+            // maxSize on its own — recursing would loop forever.
+            if (startingCount === 0 && requestsTable.length === 0) {
+                const next = requestsToMake[0];
+                const nextSize = next?.PutRequest?.Item ? getItemSize(next.PutRequest.Item) : 0;
+                throw new Error(`Item exceeds maxSize: item is ~${nextSize} bytes, limit is ${maxSize}`);
+            }
+            // otherwise start a new request set
+            paramSet.push({
+                RequestItems: {},
+                ReturnConsumedCapacity: params.ReturnConsumedCapacity,
+            });
+            return chop(requestsToMake);
+        }
+    }, [
+        {
+            RequestItems: {},
+            ReturnConsumedCapacity: params.ReturnConsumedCapacity,
+        },
+    ]);
+    return batchedItems;
+}
+export function batchWriteItemRequests(client, params, options = {}) {
+    const l = log.extend('batchWriteItemRequests');
+    l(params, options);
+    const requestFactory = (params) => ({
+        params,
+        async send() {
+            try {
+                const data = await client.batchWrite(params, options);
+                return { data };
+            }
+            catch (error) {
+                return { error };
+            }
+        },
+    });
+    const items = _batchWriteItemRequests(params, options);
+    return new SendAllBatch(requestFactory, items, options);
+}
+export function batchGetItemRequests(client, params, options = {}) {
+    const l = log.extend('batchGetItemRequests');
+    l(params, options);
+    const requestFactory = (params) => ({
+        params,
+        async send() {
+            try {
+                const data = parseResponse(await client.batchGet(params, options), options);
+                return { data };
+            }
+            catch (error) {
+                return { error };
+            }
+        },
+    });
+    const items = _batchGetItemRequests(params);
+    return new SendAllBatch(requestFactory, items, options);
+}
+export function batchWriteAll(client, params, options = {}) {
+    const l = log.extend('batchWriteAll');
+    l(params, options);
+    const requestFactory = (params) => ({
+        params,
+        async send() {
+            try {
+                const data = await client.batchWrite(params, options);
+                return { data };
+            }
+            catch (error) {
+                return { error };
+            }
+        },
+    });
+    const items = _batchWriteItemRequests(params);
+    return new SendCompletelyBatch(requestFactory, items, options);
+}
+export function batchPutAll(client, tableName, items, options = {}) {
+    const params = {
+        RequestItems: {
+            [tableName]: items.map((item) => ({
+                PutRequest: {
+                    Item: item,
+                },
+            })),
+        },
+    };
+    return batchWriteAll(client, params, options);
+}
+export function batchDeleteAll(client, tableName, items, options = {}) {
+    const params = {
+        RequestItems: {
+            [tableName]: items.map((item) => ({
+                DeleteRequest: {
+                    Key: item,
+                },
+            })),
+        },
+    };
+    return batchWriteAll(client, params, options);
+}
+export function batchGetAll(client, params, options = {}) {
+    const l = log.extend('batchGetAll');
+    l(params, options);
+    const requestFactory = (params) => ({
+        params,
+        async send() {
+            try {
+                const data = parseResponse(await client.batchGet(params, options), options);
+                return { data };
+            }
+            catch (error) {
+                return { error };
+            }
+        },
+    });
+    const items = _batchGetItemRequests(params);
+    return new SendCompletelyBatch(requestFactory, items, options);
+}

package/dist/requests/send-all.d.ts

@@ -0,0 +1,43 @@
+import type { BatchGetCommandInput, BatchGetCommandOutput, BatchWriteCommandOutput } from '@aws-sdk/lib-dynamodb';
+import type { BatchCommandInput, BatchCommandOutput, Fetcher, RequestFactory } from './types.ts';
+type SendAllCallback<I extends BatchCommandInput, O extends BatchCommandOutput = I extends BatchGetCommandInput ? BatchGetCommandOutput : BatchWriteCommandOutput> = (error: Array<Error | null> | undefined, data: Array<SendAllResult<O> | null>, unprocessed?: SendAllBatch<I>) => void;
+type SendAllCompactCallback<I extends BatchCommandInput> = (error: Array<Error | null> | undefined, data: Array<SendAllCompactResult<I> | null>, unprocessed?: SendAllBatch<I>) => void;
+export type SendAllResult<O extends BatchCommandOutput> = Omit<O, '$metadata'>;
+export type SendAllGetOutput = SendAllResult<BatchGetCommandOutput>;
+export type SendAllWriteOutput = SendAllResult<BatchWriteCommandOutput>;
+export type SendAllCompactGetOutput = Omit<SendAllResult<BatchGetCommandOutput>, 'Responses' | 'UnprocessedKeys'> & {
+    Responses?: NonNullable<BatchGetCommandOutput['Responses']>[string];
+    UnprocessedKeys?: NonNullable<BatchGetCommandOutput['UnprocessedKeys']>[string];
+};
+export type SendAllCompactWriteOutput = Omit<SendAllResult<BatchWriteCommandOutput>, 'UnprocessedItems'> & {
+    UnprocessedItems?: NonNullable<BatchWriteCommandOutput['UnprocessedItems']>[string];
+};
+export type SendAllCompactResult<I extends BatchCommandInput> = I extends BatchGetCommandInput ? SendAllCompactGetOutput : SendAllCompactWriteOutput;
+export type SendAllOptions = {
+    concurrency?: number;
+};
+export declare class SendAllBatch<I extends BatchCommandInput, O extends BatchCommandOutput = I extends BatchGetCommandInput ? BatchGetCommandOutput : BatchWriteCommandOutput> {
+    private readonly requestFactory;
+    private readonly options;
+    readonly requests: Array<Fetcher<I> | null>;
+    constructor(requestFactory: RequestFactory<I>, items: Array<Pick<I, 'RequestItems' | 'ReturnConsumedCapacity'> | null>, options: SendAllOptions);
+    load(items: Array<Pick<I, 'RequestItems' | 'ReturnConsumedCapacity'> | null>): SendAllBatch<I, I extends BatchGetCommandInput ? BatchGetCommandOutput : BatchWriteCommandOutput>;
+    sendAll(options: SendAllOptions & {
+        compact: true;
+    }): Promise<{
+        error: Array<Error | null> | undefined;
+        data: Array<SendAllCompactResult<I> | null>;
+        unprocessed?: SendAllBatch<I>;
+    }>;
+    sendAll(options?: SendAllOptions): Promise<{
+        error: Array<Error | null> | undefined;
+        data: Array<SendAllResult<O> | null>;
+        unprocessed?: SendAllBatch<I>;
+    }>;
+    sendAll(cb: SendAllCallback<I>): void;
+    sendAll(options: SendAllOptions, cb: SendAllCallback<I>): void;
+    sendAll(options: SendAllOptions & {
+        compact: true;
+    }, cb: SendAllCompactCallback<I>): void;
+}
+export {};

package/dist/requests/send-all.js

@@ -0,0 +1,163 @@
+import createDebug from 'debug';
+import fastq from 'fastq';
+import { ensureError } from "../util.js";
+const log = createDebug('dyno-requests');
+function _sendAll(requests, load, options, callback) {
+    const l = log.extend('sendAll');
+    const { concurrency = 1 } = options;
+    l('requests:', requests.length, 'concurrency:', concurrency);
+    const worker = (request, done) => {
+        if (!request) {
+            done(null);
+            return;
+        }
+        void request
+            .send()
+            .then((res) => {
+            done(null, res);
+        })
+            .catch((err) => {
+            done(null, { error: ensureError(err) });
+        });
+    };
+    const drain = (results) => {
+        return () => {
+            l('done:results', results?.length);
+            const errors = [];
+            const data = [];
+            const unprocessed = [];
+            for (const res of results) {
+                errors.push(res?.error ? ensureError(res.error) : null);
+                data.push(res?.data ? res.data : null);
+                if (!res?.data) {
+                    unprocessed.push(null);
+                    continue;
+                }
+                if ('UnprocessedItems' in res.data &&
+                    res.data.UnprocessedItems &&
+                    Object.keys(res.data.UnprocessedItems).length) {
+                    unprocessed.push({
+                        RequestItems: res.data.UnprocessedItems,
+                    });
+                }
+                else if ('UnprocessedKeys' in res.data &&
+                    res.data.UnprocessedKeys &&
+                    Object.keys(res.data.UnprocessedKeys).length) {
+                    unprocessed.push({
+                        RequestItems: res.data.UnprocessedKeys,
+                    });
+                }
+                else {
+                    unprocessed.push(null);
+                }
+            }
+            if (options.compact === true) {
+                callback(errors.filter((e) => Boolean(e)).length ? errors : undefined, compactSendAll(data), unprocessed.filter((u) => Boolean(u)).length ? load(unprocessed) : undefined);
+                return;
+            }
+            callback(errors.filter((e) => Boolean(e)).length ? errors : undefined, data, unprocessed.filter((u) => Boolean(u)).length ? load(unprocessed) : undefined);
+        };
+    };
+    const q = fastq(worker, concurrency);
+    const results = [];
+    q.drain = drain(results);
+    for (const req of requests) {
+        q.push(req, (_err, res) => {
+            results.push(res ?? null);
+        });
+    }
+}
+function hasResponses(data) {
+    if (!data) {
+        return false;
+    }
+    if ('Responses' in data) {
+        return true;
+    }
+    return false;
+}
+function hasUnprocessedKeys(data) {
+    if (!data) {
+        return false;
+    }
+    if ('UnprocessedKeys' in data) {
+        return true;
+    }
+    return false;
+}
+function hasUnprocessedItems(data) {
+    if (!data) {
+        return false;
+    }
+    if ('UnprocessedItems' in data) {
+        return true;
+    }
+    return false;
+}
+function compactSendAll(data) {
+    if (data.some(hasResponses)) {
+        for (const d of data) {
+            if (d) {
+                const responses = d.Responses;
+                d.Responses = responses
+                    ? Object.values(responses)[0]
+                    : responses;
+            }
+        }
+    }
+    if (data.some(hasUnprocessedKeys)) {
+        for (const d of data) {
+            if (d) {
+                const unprocessed = d.UnprocessedKeys;
+                d.UnprocessedKeys = unprocessed
+                    ? Object.values(unprocessed)[0]
+                    : unprocessed;
+            }
+        }
+    }
+    if (data.some(hasUnprocessedItems)) {
+        for (const d of data) {
+            if (d) {
+                const unprocessed = d.UnprocessedItems;
+                d.UnprocessedItems = unprocessed
+                    ? Object.values(unprocessed)[0]
+                    : unprocessed;
+            }
+        }
+    }
+    return data;
+}
+async function _sendAllAsync(requests, load, options) {
+    return new Promise((resolve) => {
+        _sendAll(requests, load, options, (error, data, unprocessed) => {
+            resolve({ error, data, unprocessed });
+        });
+    });
+}
+export class SendAllBatch {
+    requestFactory;
+    options;
+    requests;
+    constructor(requestFactory, items, options) {
+        this.requestFactory = requestFactory;
+        this.options = options;
+        this.requests = items.map((params) => (params ? requestFactory(params) : null));
+    }
+    load(items) {
+        return new SendAllBatch(this.requestFactory, items, this.options);
+    }
+    sendAll(optionsOrCb, cb) {
+        if (typeof optionsOrCb === 'function') {
+            _sendAll(this.requests, this.load.bind(this), this.options, optionsOrCb);
+        }
+        else if (typeof cb === 'function') {
+            _sendAll(this.requests, this.load.bind(this), { ...this.options, ...optionsOrCb }, cb);
+        }
+        else {
+            return _sendAllAsync(this.requests, this.load.bind(this), {
+                ...this.options,
+                ...optionsOrCb,
+            });
+        }
+    }
+}

package/dist/requests/send-completely.d.ts

@@ -0,0 +1,48 @@
+import type { ConsumedCapacity } from '@aws-sdk/client-dynamodb';
+import type { BatchGetCommandInput, BatchGetCommandOutput, BatchWriteCommandOutput } from '@aws-sdk/lib-dynamodb';
+import type { BatchCommandInput, BatchCommandOutput, Fetcher, RequestFactory } from './types.ts';
+type SendCompletelyCallback<I extends BatchCommandInput, O extends BatchCommandOutput = I extends BatchGetCommandInput ? BatchGetCommandOutput : BatchWriteCommandOutput> = (error: AggregateError | undefined, data: SendCompletelyResult<O>) => void;
+type SendCompletelyCompactCallback<I extends BatchCommandInput> = (error: AggregateError | undefined, data: SendCompletelyCompactResult<I>) => void;
+export type SendCompletelyResult<O extends BatchCommandOutput> = Omit<O, '$metadata' | 'ConsumedCapacity'> & {
+    ConsumedCapacity?: ConsumedCapacity;
+};
+export type SendCompletelyGetOutput = SendCompletelyResult<BatchGetCommandOutput>;
+export type SendCompletelyWriteOutput = SendCompletelyResult<BatchWriteCommandOutput>;
+export type SendCompletelyCompactGetOutput = Omit<SendCompletelyResult<BatchGetCommandOutput>, 'Responses' | 'UnprocessedKeys'> & {
+    Responses?: NonNullable<BatchGetCommandOutput['Responses']>[string];
+    UnprocessedKeys?: NonNullable<BatchGetCommandOutput['UnprocessedKeys']>[string];
+    ConsumedCapacity?: ConsumedCapacity;
+};
+export type SendCompletelyCompactWriteOutput = Omit<SendCompletelyResult<BatchWriteCommandOutput>, 'UnprocessedItems'> & {
+    UnprocessedItems?: NonNullable<BatchWriteCommandOutput['UnprocessedItems']>[string];
+    ConsumedCapacity?: ConsumedCapacity;
+};
+export type SendCompletelyCompactResult<I extends BatchCommandInput> = I extends BatchGetCommandInput ? SendCompletelyCompactGetOutput : SendCompletelyCompactWriteOutput;
+export type SendCompletelyOptions = {
+    concurrency?: number;
+    maxRetries?: number;
+};
+export declare class SendCompletelyBatch<I extends BatchCommandInput, O extends BatchCommandOutput = I extends BatchGetCommandInput ? BatchGetCommandOutput : BatchWriteCommandOutput> {
+    private readonly requestFactory;
+    items: Array<Pick<I, 'RequestItems' | 'ReturnConsumedCapacity'>>;
+    private readonly options;
+    readonly requests: Array<Fetcher<I>>;
+    constructor(requestFactory: RequestFactory<I>, items: Array<Pick<I, 'RequestItems' | 'ReturnConsumedCapacity'>>, options: SendCompletelyOptions);
+    load(items: Array<Pick<I, 'RequestItems' | 'ReturnConsumedCapacity'>>): SendCompletelyBatch<I, I extends BatchGetCommandInput ? BatchGetCommandOutput : BatchWriteCommandOutput>;
+    sendAll(options: SendCompletelyOptions & {
+        compact: true;
+    }): Promise<{
+        data: SendCompletelyCompactResult<I>;
+        error: AggregateError | undefined;
+    }>;
+    sendAll(options?: SendCompletelyOptions): Promise<{
+        data: SendCompletelyResult<O>;
+        error: AggregateError | undefined;
+    }>;
+    sendAll(cb: SendCompletelyCallback<I>): void;
+    sendAll(options: SendCompletelyOptions, cb: SendCompletelyCallback<I>): void;
+    sendAll(options: SendCompletelyOptions & {
+        compact: true;
+    }, cb: SendCompletelyCompactCallback<I>): void;
+}
+export {};