@aws-blocks/bb-distributed-table 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +174 -0
- package/README.md +292 -0
- package/dist/errors.d.ts +111 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +135 -0
- package/dist/gsi-manager-lambda/index.js +221 -0
- package/dist/gsi-manager-lambda.d.ts +26 -0
- package/dist/gsi-manager-lambda.d.ts.map +1 -0
- package/dist/gsi-manager-lambda.js +244 -0
- package/dist/index.aws.d.ts +59 -0
- package/dist/index.aws.d.ts.map +1 -0
- package/dist/index.aws.js +310 -0
- package/dist/index.browser.d.ts +5 -0
- package/dist/index.browser.d.ts.map +1 -0
- package/dist/index.browser.js +7 -0
- package/dist/index.cdk.d.ts +27 -0
- package/dist/index.cdk.d.ts.map +1 -0
- package/dist/index.cdk.js +180 -0
- package/dist/index.cdk.test.d.ts +2 -0
- package/dist/index.cdk.test.d.ts.map +1 -0
- package/dist/index.cdk.test.js +93 -0
- package/dist/index.mock.d.ts +101 -0
- package/dist/index.mock.d.ts.map +1 -0
- package/dist/index.mock.js +301 -0
- package/dist/index.test.d.ts +2 -0
- package/dist/index.test.d.ts.map +1 -0
- package/dist/index.test.js +555 -0
- package/dist/parity.test.d.ts +2 -0
- package/dist/parity.test.d.ts.map +1 -0
- package/dist/parity.test.js +557 -0
- package/dist/types.d.ts +143 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +3 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +3 -0
- package/package.json +49 -0
- package/src/errors.ts +145 -0
- package/src/gsi-manager-lambda.ts +305 -0
- package/src/index.aws.ts +400 -0
- package/src/index.browser.ts +8 -0
- package/src/index.cdk.test.ts +107 -0
- package/src/index.cdk.ts +220 -0
- package/src/index.mock.ts +363 -0
- package/src/index.test.ts +657 -0
- package/src/parity.test.ts +763 -0
- package/src/types.ts +163 -0
- package/src/version.ts +3 -0
package/dist/errors.js
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Typed error constants for DistributedTable. Use with `isBlocksError()` in catch blocks.
|
|
5
|
+
*
|
|
6
|
+
* @example
|
|
7
|
+
* ```typescript
|
|
8
|
+
* import { isBlocksError } from '@aws-blocks/core';
|
|
9
|
+
* import { DistributedTableErrors } from '@aws-blocks/bb-distributed-table';
|
|
10
|
+
*
|
|
11
|
+
* try {
|
|
12
|
+
* await table.put(item, { ifNotExists: true });
|
|
13
|
+
* } catch (e: unknown) {
|
|
14
|
+
* if (isBlocksError(e, DistributedTableErrors.ConditionalCheckFailed)) {
|
|
15
|
+
* // item already exists
|
|
16
|
+
* }
|
|
17
|
+
* throw e;
|
|
18
|
+
* }
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
export const DistributedTableErrors = {
|
|
22
|
+
ConditionalCheckFailed: 'ConditionalCheckFailedException',
|
|
23
|
+
ValidationFailed: 'ValidationFailedException',
|
|
24
|
+
/**
|
|
25
|
+
* The query or condition shape is invalid and was rejected before reaching
|
|
26
|
+
* DynamoDB: a missing `where` clause, a partition key not given as
|
|
27
|
+
* `{ equals: value }`, an unknown index, more than one sort-key condition, or
|
|
28
|
+
* an empty `ifFieldEquals`. These are all caller bugs — something the caller
|
|
29
|
+
* can fix by correcting the call. Catchable via
|
|
30
|
+
* `isBlocksError(e, DistributedTableErrors.InvalidQuery)`.
|
|
31
|
+
*
|
|
32
|
+
* Kept distinct from {@link ItemTooLarge} (a runtime data condition) so a
|
|
33
|
+
* customer can tell "my query is wrong" from "this item is too big" by name
|
|
34
|
+
* alone rather than string-matching the message.
|
|
35
|
+
*/
|
|
36
|
+
InvalidQuery: 'InvalidQueryException',
|
|
37
|
+
/**
|
|
38
|
+
* An item exceeds DynamoDB's 400 KB per-item size limit. Unlike an invalid
|
|
39
|
+
* query, this is not necessarily a caller bug — the size of a given item may
|
|
40
|
+
* be outside the caller's control — so callers may want to branch on it
|
|
41
|
+
* (skip, split, or store a reference instead). Catchable via
|
|
42
|
+
* `isBlocksError(e, DistributedTableErrors.ItemTooLarge)`.
|
|
43
|
+
*
|
|
44
|
+
* The mock checks serialized byte length client-side and throws this directly.
|
|
45
|
+
* On AWS, DynamoDB raises a generic `ValidationException` for oversized items;
|
|
46
|
+
* the runtime detects the size-specific message and re-maps it to this name so
|
|
47
|
+
* both layers are catchable with the same code. Other `ValidationException`
|
|
48
|
+
* causes (malformed expressions, type mismatches) propagate as-is.
|
|
49
|
+
*/
|
|
50
|
+
ItemTooLarge: 'ItemTooLargeException',
|
|
51
|
+
/**
|
|
52
|
+
* A batch operation could not complete all entries within the retry budget.
|
|
53
|
+
* DynamoDB batch APIs return UnprocessedKeys/UnprocessedItems (HTTP 200) under
|
|
54
|
+
* sustained throttling; when retries are exhausted we surface this so callers
|
|
55
|
+
* can back off and resubmit rather than silently losing writes or mistaking a
|
|
56
|
+
* throttled read for a missing item.
|
|
57
|
+
*
|
|
58
|
+
* The in-memory mock never throttles, so it never produces this error — the
|
|
59
|
+
* constant is shared purely so catch-site handling is identical across both.
|
|
60
|
+
*/
|
|
61
|
+
BatchIncomplete: 'BatchIncompleteException',
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* @internal Build an Error whose `name` carries the typed error code (so callers
|
|
65
|
+
* can match it with `isBlocksError`). Shared by the mock and AWS runtime so both
|
|
66
|
+
* produce identically shaped errors.
|
|
67
|
+
*/
|
|
68
|
+
export function blocksError(name, message) {
|
|
69
|
+
const err = new Error(`${name}: ${message}`);
|
|
70
|
+
err.name = name;
|
|
71
|
+
return err;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* @internal Normalize a sort-key condition before it drives a query. Shared by
|
|
75
|
+
* the mock and AWS runtime so both treat the same inputs identically:
|
|
76
|
+
*
|
|
77
|
+
* - **Zero defined fields** (`undefined`, or a present-but-empty `{}` /
|
|
78
|
+
* `{ createdAt: undefined }`) → returns `undefined`, i.e. "no sort-key filter,
|
|
79
|
+
* query the whole partition". A present-but-empty object would otherwise
|
|
80
|
+
* diverge: the mock's per-item matcher accepts everything (returns the whole
|
|
81
|
+
* partition) while the AWS runtime registers `#sk` in `ExpressionAttributeNames`
|
|
82
|
+
* with no clause that uses it, which DynamoDB rejects with `ValidationException`.
|
|
83
|
+
* - **Exactly one defined field** → returns the condition unchanged.
|
|
84
|
+
* - **More than one defined field** → throws `InvalidQuery`, because DynamoDB allows
|
|
85
|
+
* only one sort-key condition per `KeyConditionExpression` (use `between` for ranges).
|
|
86
|
+
*
|
|
87
|
+
* @throws {DistributedTableErrors.InvalidQuery} If more than one sort-key field is defined.
|
|
88
|
+
*/
|
|
89
|
+
export function normalizeSortKeyCondition(condition) {
|
|
90
|
+
if (!condition)
|
|
91
|
+
return undefined;
|
|
92
|
+
const definedKeys = Object.keys(condition).filter(k => condition[k] !== undefined);
|
|
93
|
+
if (definedKeys.length === 0)
|
|
94
|
+
return undefined;
|
|
95
|
+
if (definedKeys.length > 1) {
|
|
96
|
+
throw blocksError(DistributedTableErrors.InvalidQuery, DistributedTableMessages.multipleSortKeyConditions(definedKeys));
|
|
97
|
+
}
|
|
98
|
+
return condition;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* @internal Validation messages shared by the mock and AWS runtime. Centralised
|
|
102
|
+
* here so the two implementations stay byte-for-byte in lockstep — parity tests
|
|
103
|
+
* assert the same wording against both.
|
|
104
|
+
*/
|
|
105
|
+
export const DistributedTableMessages = {
|
|
106
|
+
indexNotFound: (index) => `Index '${index}' not found`,
|
|
107
|
+
whereRequired: (pkField) => `query() requires a 'where' clause with partition key field '${pkField}'`,
|
|
108
|
+
partitionKeyEqualsRequired: (pkField) => `query() requires '${pkField}: { equals: value }' in the where clause (partition key must be an exact match)`,
|
|
109
|
+
multipleSortKeyConditions: (conditionKeys) => `Only one sort key condition is allowed per query (DynamoDB limitation). ` +
|
|
110
|
+
`Got: ${conditionKeys.join(', ')}. Use "between" for range queries.`,
|
|
111
|
+
emptyIfFieldEquals: 'ifFieldEquals must contain at least one field with a non-undefined value',
|
|
112
|
+
itemTooLarge: (bytes) => `Item size has exceeded the maximum allowed size of 400 KB (got ${bytes} bytes)`,
|
|
113
|
+
batchIncomplete: (operation, remaining, attempts) => `${operation} did not complete: ${remaining} entr${remaining === 1 ? 'y' : 'ies'} still unprocessed ` +
|
|
114
|
+
`after ${attempts} attempts (DynamoDB throttling or response-size limits). Retry with backoff.`,
|
|
115
|
+
};
|
|
116
|
+
/**
|
|
117
|
+
* @internal Re-map DynamoDB's generic `ValidationException` to the intent-revealing
|
|
118
|
+
* `ItemTooLarge` name when (and only when) it was raised for an oversized item.
|
|
119
|
+
*
|
|
120
|
+
* DynamoDB raises a single `ValidationException` for many unrelated conditions, so
|
|
121
|
+
* we narrow on the size-specific message ("size has exceeded") before re-mapping —
|
|
122
|
+
* other `ValidationException` causes (malformed expressions, type mismatches) are
|
|
123
|
+
* left untouched and propagate as-is. This mirrors the mock's client-side size
|
|
124
|
+
* check so both layers are catchable with `isBlocksError(e, ItemTooLarge)`. The
|
|
125
|
+
* original DynamoDB error is preserved as `cause` (kept server-side per D-003) so
|
|
126
|
+
* its stack and requestId remain available for debugging.
|
|
127
|
+
*/
|
|
128
|
+
export function remapItemTooLarge(err) {
|
|
129
|
+
if (err instanceof Error && err.name === 'ValidationException' && /size has exceeded/i.test(err.message)) {
|
|
130
|
+
const remapped = new Error(err.message, { cause: err });
|
|
131
|
+
remapped.name = DistributedTableErrors.ItemTooLarge;
|
|
132
|
+
return remapped;
|
|
133
|
+
}
|
|
134
|
+
return err;
|
|
135
|
+
}
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/gsi-manager-lambda.ts
|
|
21
|
+
var gsi_manager_lambda_exports = {};
|
|
22
|
+
__export(gsi_manager_lambda_exports, {
|
|
23
|
+
handler: () => handler,
|
|
24
|
+
isCompleteHandler: () => isCompleteHandler
|
|
25
|
+
});
|
|
26
|
+
module.exports = __toCommonJS(gsi_manager_lambda_exports);
|
|
27
|
+
var import_client_dynamodb = require("@aws-sdk/client-dynamodb");
|
|
28
|
+
var dynamodb = new import_client_dynamodb.DynamoDBClient({});
|
|
29
|
+
async function describeTable(tableName) {
|
|
30
|
+
const result = await dynamodb.send(new import_client_dynamodb.DescribeTableCommand({ TableName: tableName }));
|
|
31
|
+
return result.Table;
|
|
32
|
+
}
|
|
33
|
+
function getCurrentGSIs(table) {
|
|
34
|
+
const gsis = {};
|
|
35
|
+
for (const gsi of table.GlobalSecondaryIndexes ?? []) {
|
|
36
|
+
const pk = gsi.KeySchema?.find((k) => k.KeyType === "HASH")?.AttributeName;
|
|
37
|
+
const sk = gsi.KeySchema?.find((k) => k.KeyType === "RANGE")?.AttributeName;
|
|
38
|
+
if (pk) gsis[gsi.IndexName] = { pk, sk };
|
|
39
|
+
}
|
|
40
|
+
return gsis;
|
|
41
|
+
}
|
|
42
|
+
function gsiMatchesDesired(table, desired) {
|
|
43
|
+
const current = getCurrentGSIs(table);
|
|
44
|
+
const currentNames = Object.keys(current);
|
|
45
|
+
const desiredNames = Object.keys(desired);
|
|
46
|
+
if (currentNames.length !== desiredNames.length) return false;
|
|
47
|
+
for (const name of desiredNames) {
|
|
48
|
+
if (!current[name]) return false;
|
|
49
|
+
if (current[name].pk !== desired[name].partitionKey) return false;
|
|
50
|
+
if (current[name].sk !== desired[name].sortKey) return false;
|
|
51
|
+
}
|
|
52
|
+
return true;
|
|
53
|
+
}
|
|
54
|
+
function isTableBusy(table) {
|
|
55
|
+
if (table.TableStatus !== "ACTIVE") return true;
|
|
56
|
+
for (const gsi of table.GlobalSecondaryIndexes ?? []) {
|
|
57
|
+
if (gsi.IndexStatus !== "ACTIVE") return true;
|
|
58
|
+
}
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
async function recreateTableWithIndexes(tableName, desired) {
|
|
62
|
+
console.log("\u26A0\uFE0F SANDBOX MODE: Recreating table with all GSIs (fast path)");
|
|
63
|
+
const table = await describeTable(tableName);
|
|
64
|
+
const items = [];
|
|
65
|
+
let lastKey;
|
|
66
|
+
do {
|
|
67
|
+
const result = await dynamodb.send(new import_client_dynamodb.ScanCommand({ TableName: tableName, ExclusiveStartKey: lastKey }));
|
|
68
|
+
items.push(...result.Items ?? []);
|
|
69
|
+
lastKey = result.LastEvaluatedKey;
|
|
70
|
+
} while (lastKey);
|
|
71
|
+
console.log(`Backed up ${items.length} items`);
|
|
72
|
+
await dynamodb.send(new import_client_dynamodb.DeleteTableCommand({ TableName: tableName }));
|
|
73
|
+
while (true) {
|
|
74
|
+
try {
|
|
75
|
+
await dynamodb.send(new import_client_dynamodb.DescribeTableCommand({ TableName: tableName }));
|
|
76
|
+
await new Promise((r) => setTimeout(r, 3e3));
|
|
77
|
+
} catch (e) {
|
|
78
|
+
if (e.name === "ResourceNotFoundException") break;
|
|
79
|
+
throw e;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
const usedAttrs = /* @__PURE__ */ new Set();
|
|
83
|
+
table.KeySchema.forEach((k) => usedAttrs.add(k.AttributeName));
|
|
84
|
+
for (const cfg of Object.values(desired)) {
|
|
85
|
+
usedAttrs.add(cfg.partitionKey);
|
|
86
|
+
if (cfg.sortKey) usedAttrs.add(cfg.sortKey);
|
|
87
|
+
}
|
|
88
|
+
const existingAttrMap = /* @__PURE__ */ new Map();
|
|
89
|
+
for (const attr of table.AttributeDefinitions ?? []) {
|
|
90
|
+
existingAttrMap.set(attr.AttributeName, attr.AttributeType);
|
|
91
|
+
}
|
|
92
|
+
const attrDefs = [...usedAttrs].map((name) => ({
|
|
93
|
+
AttributeName: name,
|
|
94
|
+
AttributeType: existingAttrMap.get(name) ?? Object.values(desired).find((c) => c.partitionKey === name)?.partitionKeyType ?? Object.values(desired).find((c) => c.sortKey === name)?.sortKeyType ?? import_client_dynamodb.ScalarAttributeType.S
|
|
95
|
+
}));
|
|
96
|
+
await dynamodb.send(new import_client_dynamodb.CreateTableCommand({
|
|
97
|
+
TableName: tableName,
|
|
98
|
+
KeySchema: table.KeySchema,
|
|
99
|
+
AttributeDefinitions: attrDefs,
|
|
100
|
+
BillingMode: import_client_dynamodb.BillingMode.PAY_PER_REQUEST,
|
|
101
|
+
GlobalSecondaryIndexes: Object.entries(desired).map(([name, cfg]) => ({
|
|
102
|
+
IndexName: name,
|
|
103
|
+
KeySchema: [
|
|
104
|
+
{ AttributeName: cfg.partitionKey, KeyType: import_client_dynamodb.KeyType.HASH },
|
|
105
|
+
...cfg.sortKey ? [{ AttributeName: cfg.sortKey, KeyType: import_client_dynamodb.KeyType.RANGE }] : []
|
|
106
|
+
],
|
|
107
|
+
Projection: { ProjectionType: "ALL" }
|
|
108
|
+
}))
|
|
109
|
+
}));
|
|
110
|
+
while (true) {
|
|
111
|
+
const t = await describeTable(tableName);
|
|
112
|
+
if (!isTableBusy(t)) break;
|
|
113
|
+
await new Promise((r) => setTimeout(r, 3e3));
|
|
114
|
+
}
|
|
115
|
+
for (let i = 0; i < items.length; i += 25) {
|
|
116
|
+
await dynamodb.send(new import_client_dynamodb.BatchWriteItemCommand({
|
|
117
|
+
RequestItems: { [tableName]: items.slice(i, i + 25).map((item) => ({ PutRequest: { Item: item } })) }
|
|
118
|
+
}));
|
|
119
|
+
}
|
|
120
|
+
console.log(`\u2705 Sandbox: table recreated with ${Object.keys(desired).length} GSIs, ${items.length} items restored`);
|
|
121
|
+
}
|
|
122
|
+
async function handler(event) {
|
|
123
|
+
console.log("onEvent:", JSON.stringify(event, null, 2));
|
|
124
|
+
const { TableName, Indexes, SandboxMode } = event.ResourceProperties;
|
|
125
|
+
const desired = event.RequestType === "Delete" ? {} : Indexes ?? {};
|
|
126
|
+
const isSandbox = SandboxMode === "true";
|
|
127
|
+
const physicalId = event.PhysicalResourceId ?? TableName;
|
|
128
|
+
const table = await describeTable(TableName);
|
|
129
|
+
if (gsiMatchesDesired(table, desired) && !isTableBusy(table)) {
|
|
130
|
+
console.log("Already in desired state");
|
|
131
|
+
return { PhysicalResourceId: physicalId, Data: { Status: "COMPLETE" } };
|
|
132
|
+
}
|
|
133
|
+
if (isSandbox && Object.keys(desired).length > 0) {
|
|
134
|
+
await recreateTableWithIndexes(TableName, desired);
|
|
135
|
+
return { PhysicalResourceId: physicalId, Data: { Status: "COMPLETE" } };
|
|
136
|
+
}
|
|
137
|
+
if (!isTableBusy(table)) {
|
|
138
|
+
await initiateNextChange(TableName, table, desired);
|
|
139
|
+
}
|
|
140
|
+
return { PhysicalResourceId: physicalId, Data: { Status: "IN_PROGRESS" } };
|
|
141
|
+
}
|
|
142
|
+
async function isCompleteHandler(event) {
|
|
143
|
+
console.log("isComplete:", JSON.stringify(event, null, 2));
|
|
144
|
+
const { TableName, Indexes, SandboxMode } = event.ResourceProperties;
|
|
145
|
+
const desired = event.RequestType === "Delete" ? {} : Indexes ?? {};
|
|
146
|
+
const table = await describeTable(TableName);
|
|
147
|
+
if (isTableBusy(table)) {
|
|
148
|
+
console.log("Table busy, waiting...");
|
|
149
|
+
return { IsComplete: false };
|
|
150
|
+
}
|
|
151
|
+
if (gsiMatchesDesired(table, desired)) {
|
|
152
|
+
console.log("\u2705 All GSIs match desired state");
|
|
153
|
+
return { IsComplete: true };
|
|
154
|
+
}
|
|
155
|
+
await initiateNextChange(TableName, table, desired);
|
|
156
|
+
return { IsComplete: false };
|
|
157
|
+
}
|
|
158
|
+
async function initiateNextChange(tableName, table, desired) {
|
|
159
|
+
const current = getCurrentGSIs(table);
|
|
160
|
+
const existingAttrMap = /* @__PURE__ */ new Map();
|
|
161
|
+
for (const attr of table.AttributeDefinitions ?? []) {
|
|
162
|
+
existingAttrMap.set(attr.AttributeName, attr.AttributeType);
|
|
163
|
+
}
|
|
164
|
+
for (const [name, cur] of Object.entries(current)) {
|
|
165
|
+
const des = desired[name];
|
|
166
|
+
if (des && (cur.pk !== des.partitionKey || cur.sk !== des.sortKey)) {
|
|
167
|
+
console.log(`Deleting GSI '${name}' (schema mismatch \u2014 must delete before recreating)`);
|
|
168
|
+
await dynamodb.send(new import_client_dynamodb.UpdateTableCommand({
|
|
169
|
+
TableName: tableName,
|
|
170
|
+
GlobalSecondaryIndexUpdates: [{ Delete: { IndexName: name } }]
|
|
171
|
+
}));
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
for (const [name, cfg] of Object.entries(desired)) {
|
|
176
|
+
if (!current[name]) {
|
|
177
|
+
console.log(`Creating GSI '${name}'`);
|
|
178
|
+
const attrDefs = [];
|
|
179
|
+
attrDefs.push({
|
|
180
|
+
AttributeName: cfg.partitionKey,
|
|
181
|
+
AttributeType: existingAttrMap.get(cfg.partitionKey) ?? cfg.partitionKeyType ?? import_client_dynamodb.ScalarAttributeType.S
|
|
182
|
+
});
|
|
183
|
+
if (cfg.sortKey) {
|
|
184
|
+
attrDefs.push({
|
|
185
|
+
AttributeName: cfg.sortKey,
|
|
186
|
+
AttributeType: existingAttrMap.get(cfg.sortKey) ?? cfg.sortKeyType ?? import_client_dynamodb.ScalarAttributeType.S
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
await dynamodb.send(new import_client_dynamodb.UpdateTableCommand({
|
|
190
|
+
TableName: tableName,
|
|
191
|
+
AttributeDefinitions: attrDefs,
|
|
192
|
+
GlobalSecondaryIndexUpdates: [{
|
|
193
|
+
Create: {
|
|
194
|
+
IndexName: name,
|
|
195
|
+
KeySchema: [
|
|
196
|
+
{ AttributeName: cfg.partitionKey, KeyType: import_client_dynamodb.KeyType.HASH },
|
|
197
|
+
...cfg.sortKey ? [{ AttributeName: cfg.sortKey, KeyType: import_client_dynamodb.KeyType.RANGE }] : []
|
|
198
|
+
],
|
|
199
|
+
Projection: { ProjectionType: "ALL" }
|
|
200
|
+
}
|
|
201
|
+
}]
|
|
202
|
+
}));
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
for (const name of Object.keys(current)) {
|
|
207
|
+
if (!desired[name]) {
|
|
208
|
+
console.log(`Deleting GSI '${name}' (no longer desired)`);
|
|
209
|
+
await dynamodb.send(new import_client_dynamodb.UpdateTableCommand({
|
|
210
|
+
TableName: tableName,
|
|
211
|
+
GlobalSecondaryIndexUpdates: [{ Delete: { IndexName: name } }]
|
|
212
|
+
}));
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
218
|
+
0 && (module.exports = {
|
|
219
|
+
handler,
|
|
220
|
+
isCompleteHandler
|
|
221
|
+
});
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
interface IndexConfig {
|
|
2
|
+
partitionKey: string;
|
|
3
|
+
sortKey?: string;
|
|
4
|
+
partitionKeyType?: 'S' | 'N' | 'B';
|
|
5
|
+
sortKeyType?: 'S' | 'N' | 'B';
|
|
6
|
+
}
|
|
7
|
+
interface CfnEvent {
|
|
8
|
+
RequestType: 'Create' | 'Update' | 'Delete';
|
|
9
|
+
PhysicalResourceId?: string;
|
|
10
|
+
ResourceProperties: {
|
|
11
|
+
TableName: string;
|
|
12
|
+
Indexes: Record<string, IndexConfig>;
|
|
13
|
+
SandboxMode?: string;
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
export declare function handler(event: CfnEvent): Promise<{
|
|
17
|
+
PhysicalResourceId: string;
|
|
18
|
+
Data: {
|
|
19
|
+
Status: string;
|
|
20
|
+
};
|
|
21
|
+
}>;
|
|
22
|
+
export declare function isCompleteHandler(event: any): Promise<{
|
|
23
|
+
IsComplete: boolean;
|
|
24
|
+
}>;
|
|
25
|
+
export {};
|
|
26
|
+
//# sourceMappingURL=gsi-manager-lambda.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gsi-manager-lambda.d.ts","sourceRoot":"","sources":["../src/gsi-manager-lambda.ts"],"names":[],"mappings":"AAkBA,UAAU,WAAW;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;IACnC,WAAW,CAAC,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;CAC9B;AAED,UAAU,QAAQ;IACjB,WAAW,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC5C,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACrC,WAAW,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACF;AA+HD,wBAAsB,OAAO,CAAC,KAAK,EAAE,QAAQ;;;;;GA+B5C;AAQD,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,GAAG;;GAuBjD"}
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
import { DynamoDBClient, DescribeTableCommand, UpdateTableCommand, DeleteTableCommand, CreateTableCommand, ScanCommand, BatchWriteItemCommand, BillingMode, ScalarAttributeType, KeyType, } from '@aws-sdk/client-dynamodb';
|
|
4
|
+
const dynamodb = new DynamoDBClient({});
|
|
5
|
+
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
6
|
+
async function describeTable(tableName) {
|
|
7
|
+
const result = await dynamodb.send(new DescribeTableCommand({ TableName: tableName }));
|
|
8
|
+
return result.Table;
|
|
9
|
+
}
|
|
10
|
+
function getCurrentGSIs(table) {
|
|
11
|
+
const gsis = {};
|
|
12
|
+
for (const gsi of table.GlobalSecondaryIndexes ?? []) {
|
|
13
|
+
const pk = gsi.KeySchema?.find((k) => k.KeyType === 'HASH')?.AttributeName;
|
|
14
|
+
const sk = gsi.KeySchema?.find((k) => k.KeyType === 'RANGE')?.AttributeName;
|
|
15
|
+
if (pk)
|
|
16
|
+
gsis[gsi.IndexName] = { pk, sk };
|
|
17
|
+
}
|
|
18
|
+
return gsis;
|
|
19
|
+
}
|
|
20
|
+
function gsiMatchesDesired(table, desired) {
|
|
21
|
+
const current = getCurrentGSIs(table);
|
|
22
|
+
const currentNames = Object.keys(current);
|
|
23
|
+
const desiredNames = Object.keys(desired);
|
|
24
|
+
if (currentNames.length !== desiredNames.length)
|
|
25
|
+
return false;
|
|
26
|
+
for (const name of desiredNames) {
|
|
27
|
+
if (!current[name])
|
|
28
|
+
return false;
|
|
29
|
+
if (current[name].pk !== desired[name].partitionKey)
|
|
30
|
+
return false;
|
|
31
|
+
if (current[name].sk !== desired[name].sortKey)
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
return true;
|
|
35
|
+
}
|
|
36
|
+
function isTableBusy(table) {
|
|
37
|
+
if (table.TableStatus !== 'ACTIVE')
|
|
38
|
+
return true;
|
|
39
|
+
for (const gsi of table.GlobalSecondaryIndexes ?? []) {
|
|
40
|
+
if (gsi.IndexStatus !== 'ACTIVE')
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
// ── Sandbox fast path ───────────────────────────────────────────────────────
|
|
46
|
+
async function recreateTableWithIndexes(tableName, desired) {
|
|
47
|
+
console.log('⚠️ SANDBOX MODE: Recreating table with all GSIs (fast path)');
|
|
48
|
+
const table = await describeTable(tableName);
|
|
49
|
+
// Backup data
|
|
50
|
+
const items = [];
|
|
51
|
+
let lastKey;
|
|
52
|
+
do {
|
|
53
|
+
const result = await dynamodb.send(new ScanCommand({ TableName: tableName, ExclusiveStartKey: lastKey }));
|
|
54
|
+
items.push(...(result.Items ?? []));
|
|
55
|
+
lastKey = result.LastEvaluatedKey;
|
|
56
|
+
} while (lastKey);
|
|
57
|
+
console.log(`Backed up ${items.length} items`);
|
|
58
|
+
// Delete table
|
|
59
|
+
await dynamodb.send(new DeleteTableCommand({ TableName: tableName }));
|
|
60
|
+
while (true) {
|
|
61
|
+
try {
|
|
62
|
+
await dynamodb.send(new DescribeTableCommand({ TableName: tableName }));
|
|
63
|
+
await new Promise(r => setTimeout(r, 3000));
|
|
64
|
+
}
|
|
65
|
+
catch (e) {
|
|
66
|
+
if (e.name === 'ResourceNotFoundException')
|
|
67
|
+
break;
|
|
68
|
+
throw e;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// Collect all attribute definitions needed for keys
|
|
72
|
+
const usedAttrs = new Set();
|
|
73
|
+
table.KeySchema.forEach((k) => usedAttrs.add(k.AttributeName));
|
|
74
|
+
for (const cfg of Object.values(desired)) {
|
|
75
|
+
usedAttrs.add(cfg.partitionKey);
|
|
76
|
+
if (cfg.sortKey)
|
|
77
|
+
usedAttrs.add(cfg.sortKey);
|
|
78
|
+
}
|
|
79
|
+
const existingAttrMap = new Map();
|
|
80
|
+
for (const attr of table.AttributeDefinitions ?? []) {
|
|
81
|
+
existingAttrMap.set(attr.AttributeName, attr.AttributeType);
|
|
82
|
+
}
|
|
83
|
+
const attrDefs = [...usedAttrs].map(name => ({
|
|
84
|
+
AttributeName: name,
|
|
85
|
+
AttributeType: (existingAttrMap.get(name) ??
|
|
86
|
+
Object.values(desired).find(c => c.partitionKey === name)?.partitionKeyType ??
|
|
87
|
+
Object.values(desired).find(c => c.sortKey === name)?.sortKeyType ??
|
|
88
|
+
ScalarAttributeType.S),
|
|
89
|
+
}));
|
|
90
|
+
// Recreate with all GSIs
|
|
91
|
+
await dynamodb.send(new CreateTableCommand({
|
|
92
|
+
TableName: tableName,
|
|
93
|
+
KeySchema: table.KeySchema,
|
|
94
|
+
AttributeDefinitions: attrDefs,
|
|
95
|
+
BillingMode: BillingMode.PAY_PER_REQUEST,
|
|
96
|
+
GlobalSecondaryIndexes: Object.entries(desired).map(([name, cfg]) => ({
|
|
97
|
+
IndexName: name,
|
|
98
|
+
KeySchema: [
|
|
99
|
+
{ AttributeName: cfg.partitionKey, KeyType: KeyType.HASH },
|
|
100
|
+
...(cfg.sortKey ? [{ AttributeName: cfg.sortKey, KeyType: KeyType.RANGE }] : []),
|
|
101
|
+
],
|
|
102
|
+
Projection: { ProjectionType: 'ALL' },
|
|
103
|
+
})),
|
|
104
|
+
}));
|
|
105
|
+
// Wait for active
|
|
106
|
+
while (true) {
|
|
107
|
+
const t = await describeTable(tableName);
|
|
108
|
+
if (!isTableBusy(t))
|
|
109
|
+
break;
|
|
110
|
+
await new Promise(r => setTimeout(r, 3000));
|
|
111
|
+
}
|
|
112
|
+
// Restore data
|
|
113
|
+
for (let i = 0; i < items.length; i += 25) {
|
|
114
|
+
await dynamodb.send(new BatchWriteItemCommand({
|
|
115
|
+
RequestItems: { [tableName]: items.slice(i, i + 25).map(item => ({ PutRequest: { Item: item } })) },
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
118
|
+
console.log(`✅ Sandbox: table recreated with ${Object.keys(desired).length} GSIs, ${items.length} items restored`);
|
|
119
|
+
}
|
|
120
|
+
// ── onEvent handler ─────────────────────────────────────────────────────────
|
|
121
|
+
// Called once per Create/Update/Delete. Kicks off the first GSI change (or
|
|
122
|
+
// the sandbox fast path). Returns immediately — isComplete polls for progress.
|
|
123
|
+
export async function handler(event) {
|
|
124
|
+
console.log('onEvent:', JSON.stringify(event, null, 2));
|
|
125
|
+
const { TableName, Indexes, SandboxMode } = event.ResourceProperties;
|
|
126
|
+
const desired = event.RequestType === 'Delete' ? {} : (Indexes ?? {});
|
|
127
|
+
const isSandbox = SandboxMode === 'true';
|
|
128
|
+
// On Create, we set the physical resource ID. On Update/Delete, we must
|
|
129
|
+
// echo back the original ID — CloudFormation rejects changes to it.
|
|
130
|
+
const physicalId = event.PhysicalResourceId ?? TableName;
|
|
131
|
+
// Check if already done
|
|
132
|
+
const table = await describeTable(TableName);
|
|
133
|
+
if (gsiMatchesDesired(table, desired) && !isTableBusy(table)) {
|
|
134
|
+
console.log('Already in desired state');
|
|
135
|
+
return { PhysicalResourceId: physicalId, Data: { Status: 'COMPLETE' } };
|
|
136
|
+
}
|
|
137
|
+
// Sandbox fast path: drop and recreate with all GSIs at once
|
|
138
|
+
if (isSandbox && Object.keys(desired).length > 0) {
|
|
139
|
+
await recreateTableWithIndexes(TableName, desired);
|
|
140
|
+
return { PhysicalResourceId: physicalId, Data: { Status: 'COMPLETE' } };
|
|
141
|
+
}
|
|
142
|
+
// Production path: initiate the first GSI change if table is idle.
|
|
143
|
+
// If table is busy (prior GSI still updating), just return — isComplete will poll.
|
|
144
|
+
if (!isTableBusy(table)) {
|
|
145
|
+
await initiateNextChange(TableName, table, desired);
|
|
146
|
+
}
|
|
147
|
+
return { PhysicalResourceId: physicalId, Data: { Status: 'IN_PROGRESS' } };
|
|
148
|
+
}
|
|
149
|
+
// ── isComplete handler ──────────────────────────────────────────────────────
|
|
150
|
+
// Called periodically by the Provider framework. Checks if the table matches
|
|
151
|
+
// the desired state. If a GSI update is in progress, returns IsComplete=false.
|
|
152
|
+
// If the table is idle but doesn't match, initiates the next change.
|
|
153
|
+
// Creations are performed before deletions when possible.
|
|
154
|
+
export async function isCompleteHandler(event) {
|
|
155
|
+
console.log('isComplete:', JSON.stringify(event, null, 2));
|
|
156
|
+
const { TableName, Indexes, SandboxMode } = event.ResourceProperties;
|
|
157
|
+
const desired = event.RequestType === 'Delete' ? {} : (Indexes ?? {});
|
|
158
|
+
const table = await describeTable(TableName);
|
|
159
|
+
// If a GSI operation is in progress, wait for it
|
|
160
|
+
if (isTableBusy(table)) {
|
|
161
|
+
console.log('Table busy, waiting...');
|
|
162
|
+
return { IsComplete: false };
|
|
163
|
+
}
|
|
164
|
+
// If we match the desired state, we're done
|
|
165
|
+
if (gsiMatchesDesired(table, desired)) {
|
|
166
|
+
console.log('✅ All GSIs match desired state');
|
|
167
|
+
return { IsComplete: true };
|
|
168
|
+
}
|
|
169
|
+
// Table is idle but doesn't match — initiate the next change
|
|
170
|
+
await initiateNextChange(TableName, table, desired);
|
|
171
|
+
return { IsComplete: false };
|
|
172
|
+
}
|
|
173
|
+
// ── Initiate next GSI change ────────────────────────────────────────────────
|
|
174
|
+
// Performs creations before deletions when possible.
|
|
175
|
+
//
|
|
176
|
+
// Edge case where deletion must happen first: if a desired GSI has the same
|
|
177
|
+
// name as an existing GSI but different key schema. DynamoDB doesn't support
|
|
178
|
+
// in-place GSI modification — the old one must be deleted before the new one
|
|
179
|
+
// can be created. We detect this by checking if a current GSI name exists in
|
|
180
|
+
// desired but with a different key schema.
|
|
181
|
+
async function initiateNextChange(tableName, table, desired) {
|
|
182
|
+
const current = getCurrentGSIs(table);
|
|
183
|
+
const existingAttrMap = new Map();
|
|
184
|
+
for (const attr of table.AttributeDefinitions ?? []) {
|
|
185
|
+
existingAttrMap.set(attr.AttributeName, attr.AttributeType);
|
|
186
|
+
}
|
|
187
|
+
// 1. Check for schema-mismatched GSIs that must be deleted before recreation.
|
|
188
|
+
// These take priority because the creation of the replacement can't proceed
|
|
189
|
+
// until the old one is gone.
|
|
190
|
+
for (const [name, cur] of Object.entries(current)) {
|
|
191
|
+
const des = desired[name];
|
|
192
|
+
if (des && (cur.pk !== des.partitionKey || cur.sk !== des.sortKey)) {
|
|
193
|
+
console.log(`Deleting GSI '${name}' (schema mismatch — must delete before recreating)`);
|
|
194
|
+
await dynamodb.send(new UpdateTableCommand({
|
|
195
|
+
TableName: tableName,
|
|
196
|
+
GlobalSecondaryIndexUpdates: [{ Delete: { IndexName: name } }],
|
|
197
|
+
}));
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
// 2. Create missing GSIs (creations before deletions)
|
|
202
|
+
for (const [name, cfg] of Object.entries(desired)) {
|
|
203
|
+
if (!current[name]) {
|
|
204
|
+
console.log(`Creating GSI '${name}'`);
|
|
205
|
+
const attrDefs = [];
|
|
206
|
+
attrDefs.push({
|
|
207
|
+
AttributeName: cfg.partitionKey,
|
|
208
|
+
AttributeType: (existingAttrMap.get(cfg.partitionKey) ?? cfg.partitionKeyType ?? ScalarAttributeType.S),
|
|
209
|
+
});
|
|
210
|
+
if (cfg.sortKey) {
|
|
211
|
+
attrDefs.push({
|
|
212
|
+
AttributeName: cfg.sortKey,
|
|
213
|
+
AttributeType: (existingAttrMap.get(cfg.sortKey) ?? cfg.sortKeyType ?? ScalarAttributeType.S),
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
await dynamodb.send(new UpdateTableCommand({
|
|
217
|
+
TableName: tableName,
|
|
218
|
+
AttributeDefinitions: attrDefs,
|
|
219
|
+
GlobalSecondaryIndexUpdates: [{
|
|
220
|
+
Create: {
|
|
221
|
+
IndexName: name,
|
|
222
|
+
KeySchema: [
|
|
223
|
+
{ AttributeName: cfg.partitionKey, KeyType: KeyType.HASH },
|
|
224
|
+
...(cfg.sortKey ? [{ AttributeName: cfg.sortKey, KeyType: KeyType.RANGE }] : []),
|
|
225
|
+
],
|
|
226
|
+
Projection: { ProjectionType: 'ALL' },
|
|
227
|
+
},
|
|
228
|
+
}],
|
|
229
|
+
}));
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
// 3. Delete extra GSIs (only after all creations are done)
|
|
234
|
+
for (const name of Object.keys(current)) {
|
|
235
|
+
if (!desired[name]) {
|
|
236
|
+
console.log(`Deleting GSI '${name}' (no longer desired)`);
|
|
237
|
+
await dynamodb.send(new UpdateTableCommand({
|
|
238
|
+
TableName: tableName,
|
|
239
|
+
GlobalSecondaryIndexUpdates: [{ Delete: { IndexName: name } }],
|
|
240
|
+
}));
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|