@bitblit/ratchet-aws 6.0.146-alpha → 6.0.147-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/package.json +5 -4
  2. package/src/batch/aws-batch-background-processor.spec.ts +22 -0
  3. package/src/batch/aws-batch-background-processor.ts +71 -0
  4. package/src/batch/aws-batch-ratchet.spec.ts +42 -0
  5. package/src/batch/aws-batch-ratchet.ts +70 -0
  6. package/src/build/ratchet-aws-info.ts +19 -0
  7. package/src/cache/memory-storage-provider.ts +39 -0
  8. package/src/cache/simple-cache-object-wrapper.ts +11 -0
  9. package/src/cache/simple-cache-read-options.ts +9 -0
  10. package/src/cache/simple-cache-storage-provider.ts +15 -0
  11. package/src/cache/simple-cache.spec.ts +42 -0
  12. package/src/cache/simple-cache.ts +81 -0
  13. package/src/cloudwatch/cloud-watch-log-group-ratchet.spec.ts +26 -0
  14. package/src/cloudwatch/cloud-watch-log-group-ratchet.ts +105 -0
  15. package/src/cloudwatch/cloud-watch-logs-ratchet.spec.ts +123 -0
  16. package/src/cloudwatch/cloud-watch-logs-ratchet.ts +232 -0
  17. package/src/cloudwatch/cloud-watch-metrics-ratchet.spec.ts +30 -0
  18. package/src/cloudwatch/cloud-watch-metrics-ratchet.ts +98 -0
  19. package/src/dao/example-prototype-dao-item.ts +8 -0
  20. package/src/dao/memory-prototype-dao-provider.ts +16 -0
  21. package/src/dao/prototype-dao-config.ts +8 -0
  22. package/src/dao/prototype-dao-db.ts +4 -0
  23. package/src/dao/prototype-dao-provider.ts +6 -0
  24. package/src/dao/prototype-dao.spec.ts +33 -0
  25. package/src/dao/prototype-dao.ts +110 -0
  26. package/src/dao/s3-simple-dao.ts +96 -0
  27. package/src/dao/simple-dao-item.ts +13 -0
  28. package/src/dynamodb/dynamo-ratchet-like.ts +61 -0
  29. package/src/dynamodb/dynamo-ratchet.spec.ts +206 -0
  30. package/src/dynamodb/dynamo-ratchet.ts +850 -0
  31. package/src/dynamodb/dynamo-table-ratchet.spec.ts +23 -0
  32. package/src/dynamodb/dynamo-table-ratchet.ts +189 -0
  33. package/src/dynamodb/hash-spreader.spec.ts +22 -0
  34. package/src/dynamodb/hash-spreader.ts +89 -0
  35. package/src/dynamodb/impl/dynamo-db-storage-provider.spec.ts +60 -0
  36. package/src/dynamodb/impl/dynamo-db-storage-provider.ts +140 -0
  37. package/src/dynamodb/impl/dynamo-db-sync-lock.spec.ts +41 -0
  38. package/src/dynamodb/impl/dynamo-db-sync-lock.ts +78 -0
  39. package/src/dynamodb/impl/dynamo-expiring-code-provider.ts +31 -0
  40. package/src/dynamodb/impl/dynamo-runtime-parameter-provider.spec.ts +65 -0
  41. package/src/dynamodb/impl/dynamo-runtime-parameter-provider.ts +44 -0
  42. package/src/ec2/ec2-ratchet.spec.ts +45 -0
  43. package/src/ec2/ec2-ratchet.ts +169 -0
  44. package/src/ecr/ecr-unused-image-cleaner-options.ts +9 -0
  45. package/src/ecr/ecr-unused-image-cleaner-output.ts +8 -0
  46. package/src/ecr/ecr-unused-image-cleaner-repository-output.ts +10 -0
  47. package/src/ecr/ecr-unused-image-cleaner.spec.ts +40 -0
  48. package/src/ecr/ecr-unused-image-cleaner.ts +183 -0
  49. package/src/ecr/retained-image-descriptor.ts +7 -0
  50. package/src/ecr/retained-image-reason.ts +4 -0
  51. package/src/ecr/used-image-finder.ts +6 -0
  52. package/src/ecr/used-image-finders/aws-batch-used-image-finder.ts +40 -0
  53. package/src/ecr/used-image-finders/lambda-used-image-finder.ts +51 -0
  54. package/src/environment/cascade-environment-service-provider.ts +28 -0
  55. package/src/environment/env-var-environment-service-provider.ts +36 -0
  56. package/src/environment/environment-service-config.ts +7 -0
  57. package/src/environment/environment-service-provider.ts +7 -0
  58. package/src/environment/environment-service.spec.ts +41 -0
  59. package/src/environment/environment-service.ts +89 -0
  60. package/src/environment/fixed-environment-service-provider.ts +26 -0
  61. package/src/environment/ssm-environment-service-provider.spec.ts +18 -0
  62. package/src/environment/ssm-environment-service-provider.ts +71 -0
  63. package/src/expiring-code/expiring-code-params.ts +7 -0
  64. package/src/expiring-code/expiring-code-provider.ts +6 -0
  65. package/src/expiring-code/expiring-code-ratchet.spec.ts +10 -0
  66. package/src/expiring-code/expiring-code-ratchet.ts +44 -0
  67. package/src/expiring-code/expiring-code.ts +6 -0
  68. package/src/iam/aws-credentials-ratchet.ts +25 -0
  69. package/src/lambda/lambda-event-detector.ts +55 -0
  70. package/src/lambda/lambda-event-type-guards.ts +38 -0
  71. package/src/model/cloud-watch-metrics-minute-level-dynamo-count-request.ts +18 -0
  72. package/src/model/dynamo-count-result.ts +8 -0
  73. package/src/route53/route-53-ratchet.ts +77 -0
  74. package/src/runtime-parameter/cached-stored-runtime-parameter.ts +5 -0
  75. package/src/runtime-parameter/global-variable-override-runtime-parameter-provider.spec.ts +41 -0
  76. package/src/runtime-parameter/global-variable-override-runtime-parameter-provider.ts +82 -0
  77. package/src/runtime-parameter/memory-runtime-parameter-provider.ts +42 -0
  78. package/src/runtime-parameter/runtime-parameter-provider.ts +12 -0
  79. package/src/runtime-parameter/runtime-parameter-ratchet.spec.ts +53 -0
  80. package/src/runtime-parameter/runtime-parameter-ratchet.ts +84 -0
  81. package/src/runtime-parameter/stored-runtime-parameter.ts +6 -0
  82. package/src/s3/expanded-file-children.ts +5 -0
  83. package/src/s3/impl/s3-environment-service-provider.ts +41 -0
  84. package/src/s3/impl/s3-expiring-code-provider.spec.ts +63 -0
  85. package/src/s3/impl/s3-expiring-code-provider.ts +71 -0
  86. package/src/s3/impl/s3-prototype-dao-provider.spec.ts +45 -0
  87. package/src/s3/impl/s3-prototype-dao-provider.ts +37 -0
  88. package/src/s3/impl/s3-remote-file-tracking-provider-options.ts +6 -0
  89. package/src/s3/impl/s3-remote-file-tracking-provider.spec.ts +67 -0
  90. package/src/s3/impl/s3-remote-file-tracking-provider.ts +157 -0
  91. package/src/s3/impl/s3-storage-provider.spec.ts +32 -0
  92. package/src/s3/impl/s3-storage-provider.ts +60 -0
  93. package/src/s3/s3-cache-ratchet-like.ts +64 -0
  94. package/src/s3/s3-cache-ratchet.spec.ts +150 -0
  95. package/src/s3/s3-cache-ratchet.ts +476 -0
  96. package/src/s3/s3-location-sync-ratchet.ts +207 -0
  97. package/src/s3/s3-ratchet.spec.ts +26 -0
  98. package/src/s3/s3-ratchet.ts +26 -0
  99. package/src/ses/ses-mail-sending-provider.ts +85 -0
  100. package/src/sns/sns-ratchet.spec.ts +24 -0
  101. package/src/sns/sns-ratchet.ts +52 -0
  102. package/src/sync-lock/memory-sync-lock.ts +48 -0
  103. package/src/sync-lock/sync-lock-provider.ts +5 -0
@@ -0,0 +1,23 @@
1
+ import { Logger } from '@bitblit/ratchet-common/logger/logger';
2
+ import { DescribeTableCommandOutput, DynamoDBClient } from '@aws-sdk/client-dynamodb';
3
+
4
+ import { mockClient } from 'aws-sdk-client-mock';
5
+ import { DynamoTableRatchet } from './dynamo-table-ratchet.js';
6
+ import { beforeEach, describe, test } from 'vitest';
7
+
8
+ let mockDynamo;
9
+
10
+ describe('#dynamoTableRatchet', function () {
11
+ mockDynamo = mockClient(DynamoDBClient);
12
+
13
+ beforeEach(() => {
14
+ mockDynamo.reset();
15
+ });
16
+
17
+ test.skip('should copy a table', async () => {
18
+ const dr: DynamoTableRatchet = new DynamoTableRatchet(mockDynamo);
19
+ //const tn: string[] = await dr.listAllTables();
20
+ const result: DescribeTableCommandOutput = await dr.copyTable('src-dev', 'dst-dev');
21
+ Logger.info('result was : %j', result);
22
+ }, 300_000_000);
23
+ });
@@ -0,0 +1,189 @@
1
+ import {
2
+ AttributeDefinition,
3
+ CreateTableCommand,
4
+ CreateTableCommandInput,
5
+ CreateTableCommandOutput,
6
+ DeleteTableCommand,
7
+ DeleteTableCommandInput,
8
+ DeleteTableCommandOutput,
9
+ DescribeTableCommand,
10
+ DescribeTableCommandOutput,
11
+ DynamoDBClient,
12
+ GlobalSecondaryIndex,
13
+ GlobalSecondaryIndexDescription,
14
+ KeySchemaElement,
15
+ ListTablesCommand,
16
+ ListTablesCommandInput,
17
+ ListTablesCommandOutput,
18
+ ProvisionedThroughput,
19
+ ResourceNotFoundException,
20
+ } from '@aws-sdk/client-dynamodb';
21
+ import { LocalSecondaryIndex } from '@aws-sdk/client-dynamodb/dist-types/models/models_0.js';
22
+ import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
23
+ import { Logger } from '@bitblit/ratchet-common/logger/logger';
24
+ import { ErrorRatchet } from '@bitblit/ratchet-common/lang/error-ratchet';
25
+ import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
26
+
27
+ export class DynamoTableRatchet {
28
+ constructor(private awsDDB: DynamoDBClient) {
29
+ if (!awsDDB) {
30
+ throw 'awsDDB may not be null';
31
+ }
32
+ }
33
+
34
+ public async deleteTable(tableName: string, waitForDelete = true): Promise<DeleteTableCommandOutput> {
35
+ RequireRatchet.notNullOrUndefined(tableName);
36
+ const input: DeleteTableCommandInput = {
37
+ TableName: tableName,
38
+ };
39
+ Logger.debug('Deleting ddb table %s', tableName);
40
+ const rval: DeleteTableCommandOutput = await this.awsDDB.send(new DeleteTableCommand(input));
41
+
42
+ if (waitForDelete) {
43
+ Logger.debug('Table marked for delete, waiting for deletion');
44
+ await this.waitForTableDelete(tableName);
45
+ }
46
+
47
+ return rval;
48
+ }
49
+
50
+ public async createTable(
51
+ input: CreateTableCommandInput,
52
+ waitForReady = true,
53
+ replaceIfExists = false,
54
+ ): Promise<CreateTableCommandOutput> {
55
+ RequireRatchet.notNullOrUndefined(input);
56
+ RequireRatchet.notNullOrUndefined(input.TableName);
57
+
58
+ Logger.debug('Creating new table : %j', input);
59
+ const exists: boolean = await this.tableExists(input.TableName);
60
+
61
+ if (exists) {
62
+ if (replaceIfExists) {
63
+ Logger.debug('Table %s exists and replace specified - deleting', input.TableName);
64
+ await this.deleteTable(input.TableName);
65
+ } else {
66
+ ErrorRatchet.throwFormattedErr('Cannot create table %s - exists already and replace not specified', input.TableName);
67
+ }
68
+ }
69
+
70
+ const rval: CreateTableCommandOutput = await this.awsDDB.send(new CreateTableCommand(input));
71
+
72
+ if (waitForReady) {
73
+ Logger.debug('Table created, awaiting ready');
74
+ await this.waitForTableReady(input.TableName);
75
+ }
76
+
77
+ return rval;
78
+ }
79
+
80
+ public async waitForTableReady(tableName: string): Promise<boolean> {
81
+ let rval = true;
82
+ let out: DescribeTableCommandOutput = await this.safeDescribeTable(tableName);
83
+
84
+ while (!!out && !!out.Table && out.Table.TableStatus !== 'ACTIVE') {
85
+ Logger.silly('Table not ready - waiting 2 seconds');
86
+ await PromiseRatchet.wait(2000);
87
+ out = await this.safeDescribeTable(tableName);
88
+ }
89
+
90
+ if (!out && !out.Table) {
91
+ Logger.warn('Cannot wait for %s to be ready - table does not exist', tableName);
92
+ rval = false;
93
+ }
94
+
95
+ return rval;
96
+ }
97
+
98
+ public async waitForTableDelete(tableName: string): Promise<void> {
99
+ let out: DescribeTableCommandOutput = await this.safeDescribeTable(tableName);
100
+
101
+ while (out) {
102
+ Logger.silly('Table %s still exists, waiting 2 seconds (State is %s)', tableName, out.Table.TableStatus);
103
+ await PromiseRatchet.wait(2000);
104
+ out = await this.safeDescribeTable(tableName);
105
+ }
106
+ }
107
+
108
+ public async tableExists(tableName: string): Promise<boolean> {
109
+ const desc: DescribeTableCommandOutput = await this.safeDescribeTable(tableName);
110
+ return !!desc;
111
+ }
112
+
113
+ public async listAllTables(): Promise<string[]> {
114
+ const input: ListTablesCommandInput = {};
115
+ let rval: string[] = [];
116
+
117
+ do {
118
+ const out: ListTablesCommandOutput = await this.awsDDB.send(new ListTablesCommand(input));
119
+ rval = rval.concat(out.TableNames);
120
+ input.ExclusiveStartTableName = out.LastEvaluatedTableName;
121
+ } while (input.ExclusiveStartTableName);
122
+ return rval;
123
+ }
124
+
125
+ public async safeDescribeTable(tableName: string): Promise<DescribeTableCommandOutput> {
126
+ try {
127
+ const out: DescribeTableCommandOutput = await this.awsDDB.send(new DescribeTableCommand({ TableName: tableName }));
128
+ return out;
129
+ } catch (err) {
130
+ if (err instanceof ResourceNotFoundException) {
131
+ return null;
132
+ } else {
133
+ throw err;
134
+ }
135
+ }
136
+ }
137
+
138
+ public async copyTable(
139
+ srcTableName: string,
140
+ dstTableName: string,
141
+ overrides?: CreateTableCommandInput,
142
+ copyData?: boolean,
143
+ ): Promise<CreateTableCommandOutput> {
144
+ RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(srcTableName, 'srcTableName');
145
+ RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(dstTableName, 'dstTableName');
146
+ if (copyData) {
147
+ throw ErrorRatchet.fErr('Cannot copy %s to %s - copy data not supported yet', srcTableName, dstTableName);
148
+ }
149
+ const srcTableDef: DescribeTableCommandOutput = await this.safeDescribeTable(srcTableName);
150
+ if (await this.tableExists(dstTableName)) {
151
+ throw ErrorRatchet.fErr('Cannot copy to %s - table already exists', dstTableName);
152
+ }
153
+ if (!srcTableDef) {
154
+ throw ErrorRatchet.fErr('Cannot copy %s - doesnt exist', srcTableName);
155
+ }
156
+
157
+ const _ads: AttributeDefinition[] = srcTableDef.Table.AttributeDefinitions;
158
+ const _ks: KeySchemaElement[] = srcTableDef.Table.KeySchema;
159
+ const _gi: GlobalSecondaryIndexDescription[] = srcTableDef.Table.GlobalSecondaryIndexes;
160
+
161
+ const createInput: CreateTableCommandInput = Object.assign({}, overrides || {}, {
162
+ AttributeDefinitions: srcTableDef.Table.AttributeDefinitions,
163
+ TableName: dstTableName,
164
+ KeySchema: srcTableDef.Table.KeySchema,
165
+ LocalSecondaryIndexes: srcTableDef.Table.LocalSecondaryIndexes as LocalSecondaryIndex[],
166
+ GlobalSecondaryIndexes: srcTableDef.Table.GlobalSecondaryIndexes.map((gi) => {
167
+ const output: GlobalSecondaryIndex = gi as GlobalSecondaryIndex;
168
+ if (output.ProvisionedThroughput?.WriteCapacityUnits === 0 || output.ProvisionedThroughput?.ReadCapacityUnits === 0) {
169
+ output.ProvisionedThroughput = undefined;
170
+ }
171
+ return output;
172
+ }),
173
+ BillingMode: srcTableDef.Table.BillingModeSummary.BillingMode,
174
+ ProvisionedThroughput:
175
+ srcTableDef.Table.BillingModeSummary.BillingMode === 'PROVISIONED'
176
+ ? (srcTableDef.Table.ProvisionedThroughput as ProvisionedThroughput)
177
+ : undefined,
178
+ StreamSpecification: srcTableDef.Table.StreamSpecification,
179
+ SSESpecification: srcTableDef.Table.SSEDescription,
180
+ Tags: undefined,
181
+ TableClass: srcTableDef.Table.TableClassSummary?.TableClass,
182
+ DeletionProtectionEnabled: srcTableDef.Table.DeletionProtectionEnabled,
183
+ });
184
+
185
+ const rval: CreateTableCommandOutput = await this.awsDDB.send(new CreateTableCommand(createInput));
186
+
187
+ return rval;
188
+ }
189
+ }
@@ -0,0 +1,22 @@
1
+ import { HashSpreader } from './hash-spreader.js';
2
+ import { describe, expect, test } from 'vitest';
3
+
4
+ describe('#hashSpreader', function () {
5
+ test('should enumerate spread', async () => {
6
+ const spread: HashSpreader = new HashSpreader(3, 16);
7
+ expect(spread.allBuckets.length).toEqual(16);
8
+
9
+ const spread2: HashSpreader = new HashSpreader(3, 19);
10
+ expect(spread2.allBuckets.length).toEqual(19);
11
+ });
12
+
13
+ test('should spread1', async () => {
14
+ const spread: HashSpreader = new HashSpreader(3, 16);
15
+ expect(spread.allSpreadValues('x').length).toEqual(16);
16
+ });
17
+
18
+ test('should spread multi', async () => {
19
+ const spread: HashSpreader = new HashSpreader(3, 16);
20
+ expect(spread.allSpreadValuesForArray(['x', 'y']).length).toEqual(32);
21
+ });
22
+ });
@@ -0,0 +1,89 @@
1
+ /**
2
+ * A helper for adding "spreaders" suffixes to keys to prevent hot key problems on indexes and
3
+ * hash keys.
4
+ *
5
+ * If you need to, you can easily increase the buckets number later, but may NOT decrease it. Increasing
6
+ * spots is also hard (although I have an idea for implementing that later...)
7
+ */
8
+ import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
9
+ import { ErrorRatchet } from '@bitblit/ratchet-common/lang/error-ratchet';
10
+ import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
11
+
12
+ export class HashSpreader {
13
+ private _allSlots: string[];
14
+
15
+ /*
16
+ Spots is how many locations to use, buckets is the number of unique strings to use, and the alphabet
17
+ are the strings to use in the spreader. Every element in the alphabet must be unique and alphanumeric. If there
18
+ are N items in the alphabet, then buckets must be less than N^spots
19
+
20
+ The separator is placed between the source data and the spreader
21
+ */
22
+ constructor(
23
+ private spots: number = 3,
24
+ private buckets: number = 16,
25
+ private separator: string = '_',
26
+ private alphabet: string = '0123456789ABCDEF',
27
+ ) {
28
+ RequireRatchet.true(spots > 0, 'Spots must be larger than 0');
29
+ RequireRatchet.true(buckets > 1, 'Buckets must be larger than 1');
30
+ RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(alphabet), 'Alphabet may not be null or empty');
31
+ RequireRatchet.true(StringRatchet.allUnique(alphabet), 'Alphabet must be unique');
32
+ RequireRatchet.true(StringRatchet.stringContainsOnlyAlphanumeric(alphabet), 'Alphabet must be alphanumeric');
33
+ const permutations: number = Math.pow(alphabet.length, spots);
34
+ RequireRatchet.true(buckets < permutations, 'Buckets must be less than permutations (' + buckets + ' / ' + permutations + ')');
35
+ RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(this.separator), 'Separator must be nonnull and nonempty');
36
+ const allPerms: string[] = StringRatchet.allPermutationsOfLength(spots, alphabet);
37
+ this._allSlots = allPerms.slice(0, buckets);
38
+ }
39
+
40
+ public get allBuckets(): string[] {
41
+ return Object.assign([], this._allSlots);
42
+ }
43
+
44
+ public get randomBucket(): string {
45
+ return this._allSlots[Math.floor(Math.random() * this.buckets)];
46
+ }
47
+
48
+ public allSpreadValues(input: string): string[] {
49
+ RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(input), 'Cannot spread null/empty value');
50
+ const rval: string[] = this._allSlots.map((s) => input + this.separator + s);
51
+ return rval;
52
+ }
53
+
54
+ public allSpreadValuesForArray(inputs: string[]): string[] {
55
+ RequireRatchet.true(inputs && inputs.length > 0, 'Cannot spread null/empty array');
56
+ let rval: string[] = [];
57
+ inputs.forEach((i) => {
58
+ rval = rval.concat(this.allSpreadValues(i));
59
+ });
60
+ return rval;
61
+ }
62
+
63
+ public addSpreader(input: string): string {
64
+ RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(input), 'Cannot spread null/empty value');
65
+ return input + this.separator + this.randomBucket;
66
+ }
67
+
68
+ public extractBucket(input: string): string {
69
+ RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(input), 'Cannot extract from null or empty value');
70
+ const loc: number = input.length - this.spots;
71
+ if (loc < 0 || input.charAt(loc) !== this.separator) {
72
+ ErrorRatchet.throwFormattedErr(
73
+ 'Cannot extract bucket, not created by this spreader (missing %s at location %d)',
74
+ this.separator,
75
+ loc,
76
+ );
77
+ }
78
+ return input.substring(loc);
79
+ }
80
+
81
+ public removeBucket(input: string): string {
82
+ RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(input), 'Cannot extract from null or empty value');
83
+ const loc: number = input.length - this.spots;
84
+ if (loc < 0 || input.charAt(loc) !== this.separator) {
85
+ ErrorRatchet.throwFormattedErr('Cannot remove bucket, not created by this spreader (missing %s at location %d)', this.separator, loc);
86
+ }
87
+ return input.substring(0, loc);
88
+ }
89
+ }
@@ -0,0 +1,60 @@
1
+ import { DynamoDbSimpleCacheOptions, DynamoDbStorageProvider } from './dynamo-db-storage-provider.js';
2
+ import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb';
3
+ import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
4
+ import { DynamoRatchet } from '../dynamo-ratchet.js';
5
+ import { SimpleCache } from '../../cache/simple-cache.js';
6
+ import { SimpleCacheObjectWrapper } from '../../cache/simple-cache-object-wrapper.js';
7
+
8
+ import { describe, expect, test } from 'vitest';
9
+
10
+ describe('#DynamoDbStorageProvider', function () {
11
+ test.skip('should read/write/delete with an dynamo handler', async () => {
12
+ const dr: DynamoRatchet = new DynamoRatchet(DynamoDBDocument.from(new DynamoDBClient({ region: 'us-east-1' })));
13
+ const opts: DynamoDbSimpleCacheOptions = DynamoDbStorageProvider.createDefaultOptions();
14
+ opts.tableName = 'test-table';
15
+ opts.useRangeKeys = false;
16
+ opts.hashKeyName = 'cacheUid';
17
+ opts.dynamoExpiresColumnName = 'expiresEpochSeconds';
18
+
19
+ const ddbStorage: DynamoDbStorageProvider = new DynamoDbStorageProvider(dr, opts);
20
+
21
+ const simpleCache: SimpleCache = new SimpleCache(ddbStorage, 2000000);
22
+
23
+ await simpleCache.removeFromCache('test1'); // Make sure clear
24
+
25
+ const test1a: SimpleCacheObjectWrapper<any> = await simpleCache.fetchWrapper<any>('test1', () => Promise.resolve({ x: 1 }));
26
+ expect(test1a).not.toBeNull();
27
+ expect(test1a.generated).toBeTruthy();
28
+ expect(test1a.value).not.toBeNull();
29
+ expect(test1a.value['x']).toEqual(1);
30
+
31
+ const test1b: SimpleCacheObjectWrapper<any> = await simpleCache.fetchWrapper<any>('test1', () => Promise.resolve({ x: 1 }));
32
+ expect(test1b).not.toBeNull();
33
+ expect(test1b.generated).toBeFalsy();
34
+ expect(test1b.value).not.toBeNull();
35
+ expect(test1b.value['x']).toEqual(1);
36
+
37
+ await simpleCache.removeFromCache('test1'); // Make sure clear
38
+ }, 60_000);
39
+
40
+ test.skip('should write a bunch', async () => {
41
+ const dr: DynamoRatchet = new DynamoRatchet(DynamoDBDocument.from(new DynamoDBClient({ region: 'us-east-1' })));
42
+ const opts: DynamoDbSimpleCacheOptions = DynamoDbStorageProvider.createDefaultOptions();
43
+ opts.tableName = 'test-table';
44
+ opts.useRangeKeys = false;
45
+ opts.hashKeyName = 'cacheUid';
46
+ opts.dynamoExpiresColumnName = 'expiresEpochSeconds';
47
+
48
+ const ddbStorage: DynamoDbStorageProvider = new DynamoDbStorageProvider(dr, opts);
49
+ const simpleCache: SimpleCache = new SimpleCache(ddbStorage, 1000);
50
+
51
+ for (let i = 0; i < 10; i++) {
52
+ const _tests: SimpleCacheObjectWrapper<any> = await simpleCache.fetchWrapper<any>('test' + i, () => Promise.resolve({ x: i }));
53
+ }
54
+
55
+ const all: SimpleCacheObjectWrapper<any>[] = await simpleCache.readAll();
56
+ expect(all).not.toBeNull();
57
+ expect(all.length).toBeGreaterThan(9);
58
+ await simpleCache.clearCache();
59
+ }, 60_000);
60
+ });
@@ -0,0 +1,140 @@
1
+ /*
2
+ Objects implementing this interface can store and retrieve objects in a cache, using a read-thru
3
+ approach.
4
+ */
5
+
6
+ import { PutCommandOutput, QueryCommandInput, ScanCommandInput } from '@aws-sdk/lib-dynamodb';
7
+ import { SimpleCacheObjectWrapper } from '../../cache/simple-cache-object-wrapper.js';
8
+ import { SimpleCacheStorageProvider } from '../../cache/simple-cache-storage-provider.js';
9
+ import { DynamoRatchet } from '../dynamo-ratchet.js';
10
+ import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
11
+
12
+ export class DynamoDbStorageProvider implements SimpleCacheStorageProvider {
13
+ // If hash key is provided, then the cache key is the range, otherwise the cache key is the hash
14
+ constructor(
15
+ private dynamo: DynamoRatchet,
16
+ private opts: DynamoDbSimpleCacheOptions,
17
+ ) {
18
+ RequireRatchet.notNullOrUndefined(this.dynamo, 'dynamo');
19
+ RequireRatchet.notNullOrUndefined(this.opts, 'opts');
20
+ RequireRatchet.notNullOrUndefined(this.opts.tableName, 'opts.tableName');
21
+ RequireRatchet.notNullOrUndefined(this.opts.hashKeyName, 'opts.hashKeyName');
22
+ RequireRatchet.true(!this.opts.useRangeKeys || (!!this.opts.rangeKeyName && !!this.opts.hashKeyValue), 'invalid range configuration');
23
+ }
24
+
25
+ public static createDefaultOptions(): DynamoDbSimpleCacheOptions {
26
+ const rval: DynamoDbSimpleCacheOptions = {
27
+ tableName: 'simple-cache',
28
+ useRangeKeys: false,
29
+ hashKeyName: 'cache-key',
30
+ rangeKeyName: null,
31
+ hashKeyValue: null,
32
+ };
33
+ return rval;
34
+ }
35
+
36
+ public createKeyObject(cacheKey: string): any {
37
+ const keys: any = {};
38
+ if (this.opts.useRangeKeys) {
39
+ keys[this.opts.hashKeyName] = this.opts.hashKeyValue;
40
+ keys[this.opts.rangeKeyName] = cacheKey;
41
+ } else {
42
+ keys[this.opts.hashKeyName] = cacheKey;
43
+ }
44
+ return keys;
45
+ }
46
+
47
+ public cleanDynamoFieldsFromObjectInPlace(rval: any): void {
48
+ if (rval) {
49
+ //eslint-disable-next-line @typescript-eslint/no-dynamic-delete
50
+ delete rval[this.opts.hashKeyName];
51
+ if (this.opts.rangeKeyName) {
52
+ //eslint-disable-next-line @typescript-eslint/no-dynamic-delete
53
+ delete rval[this.opts.rangeKeyName];
54
+ }
55
+ if (this.opts.dynamoExpiresColumnName) {
56
+ //eslint-disable-next-line @typescript-eslint/no-dynamic-delete
57
+ delete rval[this.opts.dynamoExpiresColumnName];
58
+ }
59
+ }
60
+ }
61
+
62
+ public extractKeysFromObject(rval: SimpleCacheObjectWrapper<any>): any {
63
+ let keys: any = null;
64
+ if (rval) {
65
+ keys = {};
66
+ if (this.opts.useRangeKeys) {
67
+ keys[this.opts.hashKeyName] = this.opts.hashKeyValue;
68
+ keys[this.opts.rangeKeyName] = rval.cacheKey;
69
+ } else {
70
+ keys[this.opts.hashKeyName] = rval.cacheKey;
71
+ }
72
+ }
73
+ return keys;
74
+ }
75
+
76
+ public async readFromCache<T>(cacheKey: string): Promise<SimpleCacheObjectWrapper<T>> {
77
+ const dKey: any = this.createKeyObject(cacheKey);
78
+ const rval: SimpleCacheObjectWrapper<T> = await this.dynamo.simpleGet<SimpleCacheObjectWrapper<T>>(this.opts.tableName, dKey);
79
+ this.cleanDynamoFieldsFromObjectInPlace(rval);
80
+
81
+ return rval;
82
+ }
83
+
84
+ public async storeInCache<T>(value: SimpleCacheObjectWrapper<T>): Promise<boolean> {
85
+ RequireRatchet.notNullOrUndefined(value, 'value');
86
+ RequireRatchet.notNullOrUndefined(value.cacheKey, 'value.cacheKey');
87
+ const toSave: any = Object.assign({}, value, this.createKeyObject(value.cacheKey));
88
+ if (this.opts.dynamoExpiresColumnName && value.expiresEpochMS) {
89
+ toSave[this.opts.dynamoExpiresColumnName] = Math.floor(value.expiresEpochMS / 1000);
90
+ }
91
+ const wrote: PutCommandOutput = await this.dynamo.simplePut(this.opts.tableName, toSave);
92
+ return !!wrote;
93
+ }
94
+
95
+ public async removeFromCache(cacheKey: string): Promise<void> {
96
+ await this.dynamo.simpleDelete(this.opts.tableName, this.createKeyObject(cacheKey));
97
+ }
98
+
99
+ public async clearCache(): Promise<number> {
100
+ // This ain't super efficient, I can make it more so later with a projection expression
101
+ const allValues: SimpleCacheObjectWrapper<any>[] = await this.readAll();
102
+ const allKeys: any[] = allValues.map((a) => this.extractKeysFromObject(a));
103
+ const rval: number = await this.dynamo.deleteAllInBatches(this.opts.tableName, allKeys, 25);
104
+ return rval;
105
+ }
106
+
107
+ public async readAll(): Promise<SimpleCacheObjectWrapper<any>[]> {
108
+ let rval: SimpleCacheObjectWrapper<any>[] = null;
109
+ if (this.opts.useRangeKeys) {
110
+ const qry: QueryCommandInput = {
111
+ TableName: this.opts.tableName,
112
+ KeyConditionExpression: '#cacheKey = :cacheKey',
113
+ ExpressionAttributeNames: {
114
+ '#cacheKey': this.opts.hashKeyName,
115
+ },
116
+ ExpressionAttributeValues: {
117
+ ':cacheKey': this.opts.hashKeyValue,
118
+ },
119
+ };
120
+ rval = await this.dynamo.fullyExecuteQuery<SimpleCacheObjectWrapper<any>>(qry);
121
+ } else {
122
+ const scan: ScanCommandInput = {
123
+ TableName: this.opts.tableName,
124
+ };
125
+ rval = await this.dynamo.fullyExecuteScan<SimpleCacheObjectWrapper<any>>(scan);
126
+ }
127
+ rval.forEach((r) => this.cleanDynamoFieldsFromObjectInPlace(r));
128
+
129
+ return rval;
130
+ }
131
+ }
132
+
133
+ export interface DynamoDbSimpleCacheOptions {
134
+ tableName: string;
135
+ useRangeKeys: boolean;
136
+ hashKeyName: string;
137
+ rangeKeyName?: string;
138
+ hashKeyValue?: string;
139
+ dynamoExpiresColumnName?: string;
140
+ }
@@ -0,0 +1,41 @@
1
+ import { DynamoDbSyncLock } from './dynamo-db-sync-lock.js';
2
+ import { DynamoRatchet } from '../dynamo-ratchet.js';
3
+ import { Logger } from '@bitblit/ratchet-common/logger/logger';
4
+
5
+ import { beforeEach, describe, expect, test } from 'vitest';
6
+ import { mock, MockProxy } from 'vitest-mock-extended';
7
+
8
+ let mockDR: MockProxy<DynamoRatchet>;
9
+
10
+ describe('#syncLockService', () => {
11
+ beforeEach(() => {
12
+ mockDR = mock<DynamoRatchet>();
13
+ });
14
+
15
+ test.skip('should test sync locks', async () => {
16
+ const svc: DynamoDbSyncLock = new DynamoDbSyncLock(mockDR, 'test-table');
17
+
18
+ const lockTestValue: string = 'SYNC_LOCK_TEST';
19
+
20
+ const aq1: boolean = await svc.acquireLock(lockTestValue);
21
+ expect(aq1).toBe(true);
22
+ const aq2: boolean = await svc.acquireLock(lockTestValue);
23
+ expect(aq2).toBe(false);
24
+ await svc.releaseLock(lockTestValue);
25
+ const aq3: boolean = await svc.acquireLock(lockTestValue);
26
+ expect(aq3).toBe(true);
27
+ await svc.releaseLock(lockTestValue);
28
+ });
29
+
30
+ test('should clear expired sync locks', async () => {
31
+ mockDR.fullyExecuteScan.mockResolvedValue([{ lockingKey: 'aa' }, { lockingKey: 'ab' }]);
32
+ mockDR.deleteAllInBatches.mockResolvedValue(2);
33
+
34
+ const svc: DynamoDbSyncLock = new DynamoDbSyncLock(mockDR, 'test-table');
35
+
36
+ const res: number = await svc.clearExpiredSyncLocks();
37
+ Logger.info('Got : %s', res);
38
+
39
+ expect(res).toEqual(2);
40
+ });
41
+ });
@@ -0,0 +1,78 @@
1
+ import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
2
+ import { Logger } from '@bitblit/ratchet-common/logger/logger';
3
+ import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
4
+ import { DynamoRatchet } from '../dynamo-ratchet.js';
5
+ import { DeleteCommandOutput, PutCommand, PutCommandOutput, ScanCommandInput } from '@aws-sdk/lib-dynamodb';
6
+ import { SyncLockProvider } from '../../sync-lock/sync-lock-provider.js';
7
+ import { ConditionalCheckFailedException, ReturnConsumedCapacity } from '@aws-sdk/client-dynamodb';
8
+
9
+ export class DynamoDbSyncLock implements SyncLockProvider {
10
+ constructor(
11
+ private ratchet: DynamoRatchet,
12
+ private tableName: string,
13
+ ) {
14
+ RequireRatchet.notNullOrUndefined(ratchet, 'ratchet');
15
+ RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(this.tableName), 'tableName');
16
+ }
17
+
18
+ public async acquireLock(lockKey: string, expirationSeconds: number = 30): Promise<boolean> {
19
+ let rval: boolean = false;
20
+ if (!!lockKey && !!expirationSeconds) {
21
+ const nowSeconds: number = Math.floor(new Date().getTime() / 1000);
22
+ const row: any = {
23
+ lockingKey: lockKey,
24
+ timestamp: nowSeconds,
25
+ expires: nowSeconds + expirationSeconds,
26
+ };
27
+
28
+ const params = {
29
+ Item: row,
30
+ ReturnConsumedCapacity: ReturnConsumedCapacity.TOTAL,
31
+ TableName: this.tableName,
32
+ ConditionExpression: 'attribute_not_exists(lockingKey)',
33
+ };
34
+
35
+ try {
36
+ const _pio: PutCommandOutput = await this.ratchet.getDDB().send(new PutCommand(params));
37
+ rval = true;
38
+ } catch (err) {
39
+ if (err instanceof ConditionalCheckFailedException) {
40
+ Logger.silly('Unable to acquire lock on %s', lockKey);
41
+ }
42
+ }
43
+ }
44
+
45
+ return rval;
46
+ }
47
+
48
+ public async releaseLock(lockKey: string): Promise<void> {
49
+ if (StringRatchet.trimToNull(lockKey)) {
50
+ try {
51
+ const dio: DeleteCommandOutput = await this.ratchet.simpleDelete(this.tableName, { lockingKey: lockKey });
52
+ Logger.silly('Released lock %s : %s', lockKey, dio);
53
+ } catch (err) {
54
+ Logger.warn('Failed to release lock key : %s : %s', lockKey, err, err);
55
+ }
56
+ }
57
+ }
58
+
59
+ public async clearExpiredSyncLocks(): Promise<number> {
60
+ const nowSeconds: number = Math.floor(new Date().getTime() / 1000);
61
+ const scan: ScanCommandInput = {
62
+ TableName: this.tableName,
63
+ FilterExpression: 'expires < :now',
64
+ ExpressionAttributeValues: {
65
+ ':now': nowSeconds,
66
+ },
67
+ };
68
+
69
+ const vals: any[] = await this.ratchet.fullyExecuteScan(scan);
70
+ const keysOnly: any[] = vals.map((v) => {
71
+ const next: any = { lockingKey: v['lockingKey'] };
72
+ return next;
73
+ });
74
+ const removed: number = await this.ratchet.deleteAllInBatches(this.tableName, keysOnly, 25);
75
+
76
+ return removed;
77
+ }
78
+ }
@@ -0,0 +1,31 @@
1
+ import { ExpiringCodeProvider } from '../../expiring-code/expiring-code-provider.js';
2
+ import { DynamoRatchet } from '../dynamo-ratchet.js';
3
+ import { ExpiringCode } from '../../expiring-code/expiring-code.js';
4
+ import { DynamoTableRatchet } from '../dynamo-table-ratchet.js';
5
+ import { PutCommandOutput } from '@aws-sdk/lib-dynamodb';
6
+
7
+ export class DynamoExpiringCodeProvider implements ExpiringCodeProvider {
8
+ constructor(
9
+ private tableName: string,
10
+ private dynamoRatchet: DynamoRatchet,
11
+ ) {}
12
+
13
+ public async checkCode(code: string, context: string, deleteOnMatch?: boolean): Promise<boolean> {
14
+ const keys: any = { code: code, context: context };
15
+ const expCode: ExpiringCode = await this.dynamoRatchet.simpleGet<ExpiringCode>(this.tableName, keys);
16
+ const rval: boolean = expCode && expCode.expiresEpochMS > Date.now();
17
+ if (rval && deleteOnMatch) {
18
+ await this.dynamoRatchet.simpleDelete(this.tableName, keys);
19
+ }
20
+ return rval;
21
+ }
22
+
23
+ public async storeCode(code: ExpiringCode): Promise<boolean> {
24
+ const output: PutCommandOutput = await this.dynamoRatchet.simplePut(this.tableName, code);
25
+ return output && output.ConsumedCapacity.CapacityUnits > 0;
26
+ }
27
+
28
+ public async createTableIfMissing(_dtr: DynamoTableRatchet): Promise<any> {
29
+ return null; // TODO: Impl
30
+ }
31
+ }