@bitblit/ratchet-aws 6.0.146-alpha → 6.0.148-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/package.json +5 -4
  2. package/src/batch/aws-batch-background-processor.spec.ts +22 -0
  3. package/src/batch/aws-batch-background-processor.ts +71 -0
  4. package/src/batch/aws-batch-ratchet.spec.ts +42 -0
  5. package/src/batch/aws-batch-ratchet.ts +70 -0
  6. package/src/build/ratchet-aws-info.ts +19 -0
  7. package/src/cache/memory-storage-provider.ts +39 -0
  8. package/src/cache/simple-cache-object-wrapper.ts +11 -0
  9. package/src/cache/simple-cache-read-options.ts +9 -0
  10. package/src/cache/simple-cache-storage-provider.ts +15 -0
  11. package/src/cache/simple-cache.spec.ts +42 -0
  12. package/src/cache/simple-cache.ts +81 -0
  13. package/src/cloudwatch/cloud-watch-log-group-ratchet.spec.ts +26 -0
  14. package/src/cloudwatch/cloud-watch-log-group-ratchet.ts +105 -0
  15. package/src/cloudwatch/cloud-watch-logs-ratchet.spec.ts +123 -0
  16. package/src/cloudwatch/cloud-watch-logs-ratchet.ts +232 -0
  17. package/src/cloudwatch/cloud-watch-metrics-ratchet.spec.ts +30 -0
  18. package/src/cloudwatch/cloud-watch-metrics-ratchet.ts +98 -0
  19. package/src/dao/example-prototype-dao-item.ts +8 -0
  20. package/src/dao/memory-prototype-dao-provider.ts +16 -0
  21. package/src/dao/prototype-dao-config.ts +8 -0
  22. package/src/dao/prototype-dao-db.ts +4 -0
  23. package/src/dao/prototype-dao-provider.ts +6 -0
  24. package/src/dao/prototype-dao.spec.ts +33 -0
  25. package/src/dao/prototype-dao.ts +110 -0
  26. package/src/dao/s3-simple-dao.ts +96 -0
  27. package/src/dao/simple-dao-item.ts +13 -0
  28. package/src/dynamodb/dynamo-ratchet-like.ts +61 -0
  29. package/src/dynamodb/dynamo-ratchet.spec.ts +206 -0
  30. package/src/dynamodb/dynamo-ratchet.ts +850 -0
  31. package/src/dynamodb/dynamo-table-ratchet.spec.ts +23 -0
  32. package/src/dynamodb/dynamo-table-ratchet.ts +189 -0
  33. package/src/dynamodb/hash-spreader.spec.ts +22 -0
  34. package/src/dynamodb/hash-spreader.ts +89 -0
  35. package/src/dynamodb/impl/dynamo-db-storage-provider.spec.ts +60 -0
  36. package/src/dynamodb/impl/dynamo-db-storage-provider.ts +140 -0
  37. package/src/dynamodb/impl/dynamo-db-sync-lock.spec.ts +41 -0
  38. package/src/dynamodb/impl/dynamo-db-sync-lock.ts +78 -0
  39. package/src/dynamodb/impl/dynamo-expiring-code-provider.ts +31 -0
  40. package/src/dynamodb/impl/dynamo-runtime-parameter-provider.spec.ts +65 -0
  41. package/src/dynamodb/impl/dynamo-runtime-parameter-provider.ts +44 -0
  42. package/src/ec2/ec2-ratchet.spec.ts +45 -0
  43. package/src/ec2/ec2-ratchet.ts +169 -0
  44. package/src/ecr/ecr-unused-image-cleaner-options.ts +9 -0
  45. package/src/ecr/ecr-unused-image-cleaner-output.ts +8 -0
  46. package/src/ecr/ecr-unused-image-cleaner-repository-output.ts +10 -0
  47. package/src/ecr/ecr-unused-image-cleaner.spec.ts +40 -0
  48. package/src/ecr/ecr-unused-image-cleaner.ts +183 -0
  49. package/src/ecr/retained-image-descriptor.ts +7 -0
  50. package/src/ecr/retained-image-reason.ts +4 -0
  51. package/src/ecr/used-image-finder.ts +6 -0
  52. package/src/ecr/used-image-finders/aws-batch-used-image-finder.ts +40 -0
  53. package/src/ecr/used-image-finders/lambda-used-image-finder.ts +51 -0
  54. package/src/environment/cascade-environment-service-provider.ts +28 -0
  55. package/src/environment/env-var-environment-service-provider.ts +36 -0
  56. package/src/environment/environment-service-config.ts +7 -0
  57. package/src/environment/environment-service-provider.ts +7 -0
  58. package/src/environment/environment-service.spec.ts +41 -0
  59. package/src/environment/environment-service.ts +89 -0
  60. package/src/environment/fixed-environment-service-provider.ts +26 -0
  61. package/src/environment/ssm-environment-service-provider.spec.ts +18 -0
  62. package/src/environment/ssm-environment-service-provider.ts +71 -0
  63. package/src/expiring-code/expiring-code-params.ts +7 -0
  64. package/src/expiring-code/expiring-code-provider.ts +6 -0
  65. package/src/expiring-code/expiring-code-ratchet.spec.ts +10 -0
  66. package/src/expiring-code/expiring-code-ratchet.ts +44 -0
  67. package/src/expiring-code/expiring-code.ts +6 -0
  68. package/src/iam/aws-credentials-ratchet.ts +25 -0
  69. package/src/lambda/lambda-event-detector.ts +55 -0
  70. package/src/lambda/lambda-event-type-guards.ts +38 -0
  71. package/src/model/cloud-watch-metrics-minute-level-dynamo-count-request.ts +18 -0
  72. package/src/model/dynamo-count-result.ts +8 -0
  73. package/src/route53/route-53-ratchet.ts +77 -0
  74. package/src/runtime-parameter/cached-stored-runtime-parameter.ts +5 -0
  75. package/src/runtime-parameter/global-variable-override-runtime-parameter-provider.spec.ts +41 -0
  76. package/src/runtime-parameter/global-variable-override-runtime-parameter-provider.ts +82 -0
  77. package/src/runtime-parameter/memory-runtime-parameter-provider.ts +42 -0
  78. package/src/runtime-parameter/runtime-parameter-provider.ts +12 -0
  79. package/src/runtime-parameter/runtime-parameter-ratchet.spec.ts +53 -0
  80. package/src/runtime-parameter/runtime-parameter-ratchet.ts +84 -0
  81. package/src/runtime-parameter/stored-runtime-parameter.ts +6 -0
  82. package/src/s3/expanded-file-children.ts +5 -0
  83. package/src/s3/impl/s3-environment-service-provider.ts +41 -0
  84. package/src/s3/impl/s3-expiring-code-provider.spec.ts +63 -0
  85. package/src/s3/impl/s3-expiring-code-provider.ts +71 -0
  86. package/src/s3/impl/s3-prototype-dao-provider.spec.ts +45 -0
  87. package/src/s3/impl/s3-prototype-dao-provider.ts +37 -0
  88. package/src/s3/impl/s3-remote-file-tracking-provider-options.ts +6 -0
  89. package/src/s3/impl/s3-remote-file-tracking-provider.spec.ts +67 -0
  90. package/src/s3/impl/s3-remote-file-tracking-provider.ts +157 -0
  91. package/src/s3/impl/s3-storage-provider.spec.ts +32 -0
  92. package/src/s3/impl/s3-storage-provider.ts +60 -0
  93. package/src/s3/s3-cache-ratchet-like.ts +64 -0
  94. package/src/s3/s3-cache-ratchet.spec.ts +150 -0
  95. package/src/s3/s3-cache-ratchet.ts +476 -0
  96. package/src/s3/s3-location-sync-ratchet.ts +207 -0
  97. package/src/s3/s3-ratchet.spec.ts +26 -0
  98. package/src/s3/s3-ratchet.ts +26 -0
  99. package/src/ses/ses-mail-sending-provider.ts +85 -0
  100. package/src/sns/sns-ratchet.spec.ts +24 -0
  101. package/src/sns/sns-ratchet.ts +52 -0
  102. package/src/sync-lock/memory-sync-lock.ts +48 -0
  103. package/src/sync-lock/sync-lock-provider.ts +5 -0
package/package.json CHANGED
@@ -1,10 +1,11 @@
1
1
  {
2
2
  "name": "@bitblit/ratchet-aws",
3
- "version": "6.0.146-alpha",
3
+ "version": "6.0.148-alpha",
4
4
  "description": "Common tools for use with AWS browser and node",
5
5
  "sideEffects": false,
6
6
  "type": "module",
7
7
  "files": [
8
+ "src/**",
8
9
  "lib/**",
9
10
  "bin/**"
10
11
  ],
@@ -51,7 +52,7 @@
51
52
  },
52
53
  "license": "Apache-2.0",
53
54
  "dependencies": {
54
- "@bitblit/ratchet-common": "6.0.146-alpha"
55
+ "@bitblit/ratchet-common": "6.0.148-alpha"
55
56
  },
56
57
  "optionalDependencies": {
57
58
  "@aws-sdk/client-athena": "3.922.0",
@@ -99,7 +100,7 @@
99
100
  "@aws-sdk/lib-storage": "^3.922.0",
100
101
  "@aws-sdk/s3-request-presigner": "^3.922.0",
101
102
  "@aws-sdk/types": "^3.922.0",
102
- "@bitblit/ratchet-common": "6.0.146-alpha",
103
+ "@bitblit/ratchet-common": "6.0.148-alpha",
103
104
  "@smithy/abort-controller": "^4.2.4",
104
105
  "@smithy/smithy-client": "^4.9.2",
105
106
  "@smithy/util-waiter": "^4.2.4"
@@ -176,7 +177,7 @@
176
177
  }
177
178
  },
178
179
  "devDependencies": {
179
- "@bitblit/ratchet-node-only": "6.0.146-alpha",
180
+ "@bitblit/ratchet-node-only": "6.0.148-alpha",
180
181
  "@types/aws-lambda": "8.10.157",
181
182
  "aws-sdk-client-mock": "4.1.0"
182
183
  }
@@ -0,0 +1,22 @@
1
+ import { AwsBatchRatchet } from './aws-batch-ratchet.js';
2
+ import { SubmitJobCommandOutput } from '@aws-sdk/client-batch';
3
+ import { AwsBatchBackgroundProcessor } from './aws-batch-background-processor.js';
4
+ import { beforeEach, describe, expect, test } from 'vitest';
5
+ import { mock, MockProxy } from 'vitest-mock-extended';
6
+
7
+ let mockBatchRatchet: MockProxy<AwsBatchRatchet>;
8
+
9
+ describe('#AwsBatchBackgroundProcessor', () => {
10
+ beforeEach(() => {
11
+ mockBatchRatchet = mock<AwsBatchRatchet>();
12
+ });
13
+
14
+ test('Should schedule background task', async () => {
15
+ //mockBatchRatchet.defaultQueueName = 'a';
16
+ mockBatchRatchet.scheduleJob.mockResolvedValue({ jobId: 'newID', jobName: 'name', $metadata: null });
17
+ const svc: AwsBatchBackgroundProcessor = new AwsBatchBackgroundProcessor(mockBatchRatchet, null);
18
+
19
+ const res: SubmitJobCommandOutput = await svc.scheduleBackgroundTask('BACKGROUND_TASK_NAME', {}, {});
20
+ expect(res).not.toBeNull();
21
+ });
22
+ });
@@ -0,0 +1,71 @@
1
+ import { SubmitJobCommandInput, SubmitJobCommandOutput } from '@aws-sdk/client-batch';
2
+ import { DateTime } from 'luxon';
3
+ import { AwsBatchRatchet } from './aws-batch-ratchet.js';
4
+ import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
5
+ import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
6
+ import { ErrorRatchet } from '@bitblit/ratchet-common/lang/error-ratchet';
7
+ import { Logger } from '@bitblit/ratchet-common/logger/logger';
8
+
9
+ /**
10
+ * Class to simplify using AWS batch as a background processor
11
+ * (Primarily used by Epsilon)
12
+ */
13
+ export class AwsBatchBackgroundProcessor {
14
+ constructor(
15
+ private batchRatchet: AwsBatchRatchet,
16
+ private validTaskNames?: string[],
17
+ ) {
18
+ RequireRatchet.notNullOrUndefined(this.batchRatchet, 'batchRatchet');
19
+ RequireRatchet.notNullOrUndefined(this.batchRatchet.batchClient, 'batchRatchet.batchClient');
20
+ RequireRatchet.notNullOrUndefined(this.batchRatchet.defaultJobDefinition, 'batchRatchet.defaultJobDefinition');
21
+ RequireRatchet.notNullOrUndefined(this.batchRatchet.defaultQueueName, 'batchRatchet.defaultQueueName');
22
+ }
23
+
24
+ public async scheduleBackgroundTask(
25
+ taskName: string,
26
+ data: Record<string, any> = {},
27
+ meta: Record<string, any> = {},
28
+ ): Promise<SubmitJobCommandOutput> {
29
+ if (
30
+ this.validTaskNames &&
31
+ this.validTaskNames.length &&
32
+ (!StringRatchet.trimToNull(taskName) || !this.validTaskNames.includes(taskName))
33
+ ) {
34
+ ErrorRatchet.throwFormattedErr('Cannot start task %s - not found in valid task list', taskName);
35
+ }
36
+
37
+ Logger.info('Submitting background task to AWS batch: %s %j %s', taskName, data, this.batchRatchet.defaultQueueName);
38
+
39
+ let rval: SubmitJobCommandOutput = null;
40
+
41
+ const jobName: string = `${this.batchRatchet.defaultJobDefinition}-${taskName}_${DateTime.utc().toFormat('yyyy-MM-dd-HH-mm')}`;
42
+
43
+ const options: SubmitJobCommandInput = {
44
+ jobName: jobName,
45
+ jobDefinition: this.batchRatchet.defaultJobDefinition,
46
+ jobQueue: this.batchRatchet.defaultQueueName,
47
+ parameters: {
48
+ taskName,
49
+ taskData: JSON.stringify(data),
50
+ taskMetadata: JSON.stringify(meta),
51
+ },
52
+ };
53
+
54
+ try {
55
+ rval = await this.batchRatchet.scheduleJob(options);
56
+ Logger.info('Job %s(%s) submitted', rval.jobName, rval.jobId);
57
+ } catch (err) {
58
+ Logger.error(
59
+ 'Cannot submit batch job taskName: %s jobDef: %s queue: %s jobName: %s data: %j',
60
+ taskName,
61
+ this.batchRatchet.defaultJobDefinition,
62
+ this.batchRatchet.defaultQueueName,
63
+ jobName,
64
+ data,
65
+ err,
66
+ );
67
+ }
68
+
69
+ return rval;
70
+ }
71
+ }
@@ -0,0 +1,42 @@
1
+ import { AwsBatchRatchet } from './aws-batch-ratchet.js';
2
+ import { BatchClient, JobStatus, JobSummary, ListJobsCommand, SubmitJobCommand, SubmitJobCommandOutput } from '@aws-sdk/client-batch';
3
+ import { mockClient } from 'aws-sdk-client-mock';
4
+ import { beforeEach, describe, expect, test } from 'vitest';
5
+
6
+ let mockBatch;
7
+
8
+ describe('#AwsBatchService', () => {
9
+ mockBatch = mockClient(BatchClient);
10
+ beforeEach(() => {
11
+ mockBatch.reset();
12
+ });
13
+
14
+ test('Should schedule batch job', async () => {
15
+ const svc: AwsBatchRatchet = new AwsBatchRatchet(mockBatch);
16
+ mockBatch.on(SubmitJobCommand).resolves({ jobName: 'b' });
17
+
18
+ const res: SubmitJobCommandOutput = await svc.scheduleJob({
19
+ jobName: 'testName',
20
+ jobDefinition: 'testDefinition',
21
+ jobQueue: 'testQueue',
22
+ });
23
+ expect(res).not.toBeNull();
24
+ });
25
+
26
+ test('Should list jobs', async () => {
27
+ const svc: AwsBatchRatchet = new AwsBatchRatchet(mockBatch);
28
+ mockBatch.on(ListJobsCommand).resolves([{}] as JobSummary[]);
29
+
30
+ const res: JobSummary[] = await svc.listJobs('testQueue');
31
+ expect(res).not.toBeNull();
32
+ expect(res.length).toEqual(1);
33
+ });
34
+
35
+ test('Should count jobs in state', async () => {
36
+ const svc: AwsBatchRatchet = new AwsBatchRatchet(mockBatch);
37
+ mockBatch.on(ListJobsCommand).resolves([{}] as JobSummary[]);
38
+
39
+ const res: number = await svc.jobCountInState(JobStatus.RUNNABLE, 'testQueue');
40
+ expect(res).toEqual(1);
41
+ });
42
+ });
@@ -0,0 +1,70 @@
1
+ import {
2
+ BatchClient,
3
+ JobStatus,
4
+ JobSummary,
5
+ ListJobsCommand,
6
+ ListJobsCommandInput,
7
+ ListJobsCommandOutput,
8
+ SubmitJobCommand,
9
+ SubmitJobCommandInput,
10
+ SubmitJobCommandOutput,
11
+ } from '@aws-sdk/client-batch';
12
+ import { Logger } from '@bitblit/ratchet-common/logger/logger';
13
+ import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
14
+
15
+ /**
16
+ * Ratchet for simplifying interacting with AWS Batch
17
+ */
18
+ export class AwsBatchRatchet {
19
+ constructor(
20
+ private _batchClient: BatchClient,
21
+ private _defaultQueueName?: string,
22
+ private _defaultJobDefinition?: string,
23
+ ) {}
24
+
25
+ public get batchClient(): BatchClient {
26
+ return this._batchClient;
27
+ }
28
+ public get defaultQueueName(): string {
29
+ return this._defaultQueueName;
30
+ }
31
+ public get defaultJobDefinition(): string {
32
+ return this._defaultJobDefinition;
33
+ }
34
+
35
+ public async scheduleJob(options: SubmitJobCommandInput): Promise<SubmitJobCommandOutput> {
36
+ Logger.info('Submitting batch job %s', options.jobName);
37
+ try {
38
+ const rval: SubmitJobCommandOutput = await this._batchClient.send(new SubmitJobCommand(options));
39
+ Logger.info('Job %s(%s) submitted', rval.jobName, rval.jobId);
40
+ return rval;
41
+ } catch (err) {
42
+ Logger.error('Cannot submit batch job %s: %s', options.jobName, err);
43
+ }
44
+ return null;
45
+ }
46
+
47
+ public async jobCountInState(jobStatus: JobStatus, queueName: string = this.defaultQueueName): Promise<number> {
48
+ const all: JobSummary[] = await this.listJobs(queueName, jobStatus);
49
+ return all.length;
50
+ }
51
+
52
+ public async listJobs(queueName: string = this.defaultQueueName, jobStatus: JobStatus = null): Promise<JobSummary[]> {
53
+ RequireRatchet.notNullOrUndefined(queueName, 'queueName');
54
+ let rval: JobSummary[] = [];
55
+ const request: ListJobsCommandInput = {
56
+ jobQueue: queueName,
57
+ jobStatus: jobStatus,
58
+ nextToken: null,
59
+ };
60
+ Logger.info('Fetching %j', request);
61
+ do {
62
+ Logger.info('Pulling page...');
63
+ const tmp: ListJobsCommandOutput = await this._batchClient.send(new ListJobsCommand(request));
64
+ rval = rval.concat(tmp.jobSummaryList);
65
+ request.nextToken = tmp.nextToken;
66
+ } while (request.nextToken);
67
+
68
+ return rval;
69
+ }
70
+ }
@@ -0,0 +1,19 @@
1
+ import { BuildInformation } from '@bitblit/ratchet-common/build/build-information';
2
+
3
+ export class RatchetAwsInfo {
4
+ // Empty constructor prevents instantiation
5
+ // eslint-disable-next-line @typescript-eslint/no-empty-function
6
+ private constructor() {}
7
+
8
+ public static buildInformation(): BuildInformation {
9
+ const val: BuildInformation = {
10
+ version: 'LOCAL-SNAPSHOT',
11
+ hash: 'LOCAL-HASH',
12
+ branch: 'LOCAL-BRANCH',
13
+ tag: 'LOCAL-TAG',
14
+ timeBuiltISO: 'LOCAL-TIME-ISO',
15
+ notes: 'LOCAL-NOTES',
16
+ };
17
+ return val;
18
+ }
19
+ }
@@ -0,0 +1,39 @@
1
+ /*
2
+ Simple memory-based version, mainly for testing
3
+ */
4
+
5
+ import { SimpleCacheObjectWrapper } from './simple-cache-object-wrapper.js';
6
+ import { SimpleCacheStorageProvider } from './simple-cache-storage-provider.js';
7
+
8
+ export class MemoryStorageProvider implements SimpleCacheStorageProvider {
9
+ private _cache: Map<string, SimpleCacheObjectWrapper<any>> = new Map<string, SimpleCacheObjectWrapper<any>>();
10
+
11
+ public async readFromCache<T>(cacheKey: string): Promise<SimpleCacheObjectWrapper<T>> {
12
+ return this._cache.get(cacheKey);
13
+ }
14
+
15
+ public async storeInCache<T>(value: SimpleCacheObjectWrapper<T>): Promise<boolean> {
16
+ let rval: boolean = false;
17
+ if (value?.cacheKey) {
18
+ this._cache.set(value.cacheKey, value);
19
+ rval = true;
20
+ }
21
+ return rval;
22
+ }
23
+
24
+ public async removeFromCache(cacheKey: string): Promise<void> {
25
+ if (cacheKey) {
26
+ this._cache.delete(cacheKey);
27
+ }
28
+ }
29
+
30
+ public async clearCache(): Promise<number> {
31
+ const rval: number = this._cache.size;
32
+ this._cache = new Map<string, SimpleCacheObjectWrapper<any>>();
33
+ return rval;
34
+ }
35
+
36
+ public async readAll(): Promise<SimpleCacheObjectWrapper<any>[]> {
37
+ return Array.from(this._cache.values());
38
+ }
39
+ }
@@ -0,0 +1,11 @@
1
+ /*
2
+ Wraps an object to be stored in the cache along with the cache data needed to store/filter it
3
+ */
4
+
5
+ export interface SimpleCacheObjectWrapper<T> {
6
+ cacheKey: string;
7
+ createdEpochMS: number;
8
+ expiresEpochMS: number;
9
+ value: T;
10
+ generated: boolean;
11
+ }
@@ -0,0 +1,9 @@
1
+ /*
2
+ Controls how the object will be read from cache
3
+ */
4
+
5
+ export interface SimpleCacheReadOptions {
6
+ maxStalenessMS: number;
7
+ timeToLiveMS: number;
8
+ cacheNullValues: boolean;
9
+ }
@@ -0,0 +1,15 @@
1
+ /*
2
+ Objects implementing this interface can store and retrieve objects in a cache, using a read-thru
3
+ approach.
4
+ */
5
+
6
+ import { SimpleCacheObjectWrapper } from './simple-cache-object-wrapper.js';
7
+
8
+ export interface SimpleCacheStorageProvider {
9
+ // Reads the object from the cache and returns the wrapper. If the
10
+ readFromCache<T>(cacheKey: string): Promise<SimpleCacheObjectWrapper<T>>;
11
+ storeInCache<T>(value: SimpleCacheObjectWrapper<T>): Promise<boolean>;
12
+ removeFromCache(cacheKey: string): Promise<void>;
13
+ clearCache(): Promise<number>;
14
+ readAll(): Promise<SimpleCacheObjectWrapper<any>[]>;
15
+ }
@@ -0,0 +1,42 @@
1
+ import { SimpleCache } from './simple-cache.js';
2
+ import { SimpleCacheObjectWrapper } from './simple-cache-object-wrapper.js';
3
+ import { describe, expect, test } from 'vitest';
4
+ import { MemoryStorageProvider } from './memory-storage-provider.js';
5
+
6
+ describe('#simpleCache', function () {
7
+ test.skip('should read/write/delete with a memory', async () => {
8
+ const storageProvider: MemoryStorageProvider = new MemoryStorageProvider();
9
+
10
+ const simpleCache: SimpleCache = new SimpleCache(storageProvider, 2000000);
11
+
12
+ await simpleCache.removeFromCache('test1'); // Make sure clear
13
+
14
+ const test1a: SimpleCacheObjectWrapper<any> = await simpleCache.fetchWrapper<any>('test1', () => Promise.resolve({ x: 1 }));
15
+ expect(test1a).not.toBeNull();
16
+ expect(test1a.generated).toBeTruthy();
17
+ expect(test1a.value).not.toBeNull();
18
+ expect(test1a.value['x']).toEqual(1);
19
+
20
+ const test1b: SimpleCacheObjectWrapper<any> = await simpleCache.fetchWrapper<any>('test1', () => Promise.resolve({ x: 1 }));
21
+ expect(test1b).not.toBeNull();
22
+ expect(test1b.generated).toBeFalsy();
23
+ expect(test1b.value).not.toBeNull();
24
+ expect(test1b.value['x']).toEqual(1);
25
+
26
+ await simpleCache.removeFromCache('test1'); // Make sure clear
27
+ }, 60_000);
28
+
29
+ test.skip('should write a bunch', async () => {
30
+ const storageProvider: MemoryStorageProvider = new MemoryStorageProvider();
31
+ const simpleCache: SimpleCache = new SimpleCache(storageProvider, 1000);
32
+
33
+ for (let i = 0; i < 10; i++) {
34
+ const _tests: SimpleCacheObjectWrapper<any> = await simpleCache.fetchWrapper<any>('test' + i, () => Promise.resolve({ x: i }));
35
+ }
36
+
37
+ const all: SimpleCacheObjectWrapper<any>[] = await simpleCache.readAll();
38
+ expect(all).not.toBeNull();
39
+ expect(all.length).toBeGreaterThan(9);
40
+ await simpleCache.clearCache();
41
+ }, 60_000);
42
+ });
@@ -0,0 +1,81 @@
1
+ /*
2
+ Wraps up a simple cache storage provider and gives helper methods
3
+ */
4
+
5
+ import { Logger } from '@bitblit/ratchet-common/logger/logger';
6
+ import { SimpleCacheStorageProvider } from './simple-cache-storage-provider.js';
7
+ import { SimpleCacheObjectWrapper } from './simple-cache-object-wrapper.js';
8
+ import { SimpleCacheReadOptions } from './simple-cache-read-options.js';
9
+
10
+ export class SimpleCache {
11
+ // Default 1 minute expiration
12
+ constructor(
13
+ private provider: SimpleCacheStorageProvider,
14
+ private defaultTimeToLiveMS: number = 1_000 * 60,
15
+ ) {}
16
+
17
+ public createDefaultReadOptions(): SimpleCacheReadOptions {
18
+ return {
19
+ maxStalenessMS: null,
20
+ timeToLiveMS: this.defaultTimeToLiveMS,
21
+ cacheNullValues: false,
22
+ };
23
+ }
24
+
25
+ public async fetchWrapper<T>(
26
+ cacheKey: string,
27
+ producer: () => Promise<T>,
28
+ opts: SimpleCacheReadOptions = this.createDefaultReadOptions(),
29
+ ): Promise<SimpleCacheObjectWrapper<T>> {
30
+ Logger.silly('Fetching %s', cacheKey);
31
+ const now: number = new Date().getTime();
32
+ let rval: SimpleCacheObjectWrapper<T> = await this.provider.readFromCache(cacheKey);
33
+ if (rval && rval.expiresEpochMS < now) {
34
+ Logger.debug('Object found, but expired - removing');
35
+ rval = null;
36
+ }
37
+ if (rval && opts && opts.maxStalenessMS && now - rval.createdEpochMS > opts.maxStalenessMS) {
38
+ Logger.debug('Object found by too stale - removing');
39
+ rval = null;
40
+ }
41
+ if (!rval) {
42
+ Logger.debug('%s not found in cache, generating', cacheKey);
43
+ const tmp: T = await producer();
44
+ if (tmp || opts?.cacheNullValues) {
45
+ Logger.debug('Writing %j to cache');
46
+ rval = {
47
+ cacheKey: cacheKey,
48
+ createdEpochMS: now,
49
+ expiresEpochMS: opts && opts.timeToLiveMS ? now + opts.timeToLiveMS : null,
50
+ value: tmp,
51
+ generated: false, // Always STORE false, overwrite it when it should be true
52
+ };
53
+ await this.provider.storeInCache(rval);
54
+ rval.generated = true;
55
+ }
56
+ }
57
+ return rval;
58
+ }
59
+
60
+ public async fetch<T>(cacheKey: string, producer: () => Promise<T>, opts: SimpleCacheReadOptions = null): Promise<T> {
61
+ const wrapper: SimpleCacheObjectWrapper<T> = await this.fetchWrapper(cacheKey, producer, opts);
62
+ return wrapper ? wrapper.value : null;
63
+ }
64
+
65
+ public async removeFromCache<T>(cacheKey: string, returnOldValue?: boolean): Promise<SimpleCacheObjectWrapper<T>> {
66
+ let rval: SimpleCacheObjectWrapper<T> = null;
67
+ if (returnOldValue) {
68
+ rval = await this.fetchWrapper(cacheKey, () => null);
69
+ }
70
+ await this.provider.removeFromCache(cacheKey);
71
+ return rval;
72
+ }
73
+
74
+ public async clearCache(): Promise<number> {
75
+ return this.provider.clearCache();
76
+ }
77
+
78
+ public async readAll(): Promise<SimpleCacheObjectWrapper<any>[]> {
79
+ return this.provider.readAll();
80
+ }
81
+ }
@@ -0,0 +1,26 @@
1
+ import { CloudWatchLogsClient, DescribeLogStreamsCommand, DescribeLogStreamsCommandOutput } from '@aws-sdk/client-cloudwatch-logs';
2
+ import { CloudWatchLogGroupRatchet } from './cloud-watch-log-group-ratchet.js';
3
+ import { mockClient } from 'aws-sdk-client-mock';
4
+ import { beforeEach, describe, expect, test } from 'vitest';
5
+
6
+ let mockCW;
7
+
8
+ describe('#CloudWatchLogGroupRatchet', function () {
9
+ mockCW = mockClient(CloudWatchLogsClient);
10
+ beforeEach(() => {
11
+ mockCW.reset();
12
+ });
13
+
14
+ test('should read log stream names', async () => {
15
+ mockCW.on(DescribeLogStreamsCommand).resolves({
16
+ logStreams: [{ logStreamName: '1' }, { logStreamName: '2' }],
17
+ } as DescribeLogStreamsCommandOutput);
18
+
19
+ const cw: CloudWatchLogGroupRatchet = new CloudWatchLogGroupRatchet('testGroup', mockCW);
20
+
21
+ const res: string[] = await cw.readLogStreamNames();
22
+
23
+ expect(res).toBeTruthy();
24
+ expect(res.length).toEqual(2);
25
+ });
26
+ });
@@ -0,0 +1,105 @@
1
+ import {
2
+ CloudWatchLogsClient,
3
+ DescribeLogStreamsCommand,
4
+ DescribeLogStreamsCommandInput,
5
+ DescribeLogStreamsCommandOutput,
6
+ FilteredLogEvent,
7
+ FilterLogEventsCommand,
8
+ FilterLogEventsCommandInput,
9
+ FilterLogEventsCommandOutput,
10
+ LogStream,
11
+ } from '@aws-sdk/client-cloudwatch-logs';
12
+ import { Logger } from '@bitblit/ratchet-common/logger/logger';
13
+ import { StopWatch } from '@bitblit/ratchet-common/lang/stop-watch';
14
+
15
+ export class CloudWatchLogGroupRatchet {
16
+ constructor(
17
+ private logGroup: string,
18
+ private awsCWLogs: CloudWatchLogsClient = new CloudWatchLogsClient({ region: 'us-east-1' }),
19
+ ) {}
20
+
21
+ public get cloudWatchLogsClient(): CloudWatchLogsClient {
22
+ return this.awsCWLogs;
23
+ }
24
+ public async readLogStreams(startTimestamp: number = null, endTimestamp: number = null): Promise<LogStream[]> {
25
+ const params: DescribeLogStreamsCommandInput = {
26
+ logGroupName: this.logGroup,
27
+ // logStreamNamePrefix: prefix,
28
+ orderBy: 'LastEventTime',
29
+ };
30
+
31
+ const rval: LogStream[] = [];
32
+
33
+ do {
34
+ Logger.debug('Pulling more log streams (%d found so far)', rval.length);
35
+ const temp: DescribeLogStreamsCommandOutput = await this.awsCWLogs.send(new DescribeLogStreamsCommand(params));
36
+
37
+ temp.logStreams.forEach((s) => {
38
+ if (s.lastEventTimestamp !== null) {
39
+ if (!startTimestamp || s.lastEventTimestamp >= startTimestamp) {
40
+ if (!endTimestamp || s.firstEventTimestamp <= endTimestamp) {
41
+ rval.push(s);
42
+ }
43
+ }
44
+ }
45
+ });
46
+
47
+ params.nextToken = temp.nextToken;
48
+ } while (params.nextToken);
49
+
50
+ Logger.debug('Found %d total, returning', rval.length);
51
+ return rval;
52
+ }
53
+
54
+ public async readLogStreamNames(startTimestamp: number = null, endTimestamp: number = null): Promise<string[]> {
55
+ const streams: LogStream[] = await this.readLogStreams(startTimestamp, endTimestamp);
56
+ const rval: string[] = streams.map((s) => s.logStreamName);
57
+ return rval;
58
+ }
59
+
60
+ public async readEvents(
61
+ filter: string,
62
+ startTimestamp: number = null,
63
+ endTimestamp: number = null,
64
+ sortEvents = true,
65
+ maxEvents: number = null,
66
+ ): Promise<FilteredLogEvent[]> {
67
+ const sw: StopWatch = new StopWatch();
68
+ const params: FilterLogEventsCommandInput = {
69
+ logGroupName: this.logGroup,
70
+ endTime: endTimestamp,
71
+ startTime: startTimestamp,
72
+ };
73
+
74
+ if (filter) {
75
+ params.filterPattern = filter;
76
+ }
77
+
78
+ Logger.debug('Reading log events matching : %j', params);
79
+
80
+ let rval: FilteredLogEvent[] = [];
81
+
82
+ do {
83
+ Logger.debug('Pulling more log events (%d found so far) : %s', rval.length, sw.dump());
84
+ const temp: FilterLogEventsCommandOutput = await this.awsCWLogs.send(new FilterLogEventsCommand(params));
85
+ rval = rval.concat(temp.events);
86
+ params.nextToken = temp.nextToken;
87
+ } while (!!params.nextToken && (!maxEvents || rval.length < maxEvents));
88
+
89
+ Logger.debug('Found %d total in %s', rval.length, sw.dump());
90
+
91
+ if (sortEvents) {
92
+ Logger.debug('Sorting events by timestamp');
93
+ rval = rval.sort((a, b) => {
94
+ let rval = a.timestamp - b.timestamp;
95
+ if (rval === 0) {
96
+ rval = a.message.localeCompare(b.message);
97
+ }
98
+ return rval;
99
+ });
100
+ }
101
+
102
+ sw.log();
103
+ return rval;
104
+ }
105
+ }