@bitblit/ratchet-aws 4.0.80-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/batch/aws-batch-background-processor.js +44 -0
- package/dist/cjs/batch/aws-batch-ratchet.js +55 -0
- package/dist/cjs/build/ratchet-aws-info.js +18 -0
- package/dist/cjs/cache/dynamo-db-storage-provider.js +111 -0
- package/dist/cjs/cache/s3-storage-provider.js +44 -0
- package/dist/cjs/cache/simple-cache-object-wrapper.js +2 -0
- package/dist/cjs/cache/simple-cache-read-options.js +2 -0
- package/dist/cjs/cache/simple-cache-storage-provider.js +2 -0
- package/dist/cjs/cache/simple-cache.js +66 -0
- package/dist/cjs/cloudwatch/cloud-watch-log-group-ratchet.js +73 -0
- package/dist/cjs/cloudwatch/cloud-watch-logs-ratchet.js +173 -0
- package/dist/cjs/cloudwatch/cloud-watch-metrics-ratchet.js +57 -0
- package/dist/cjs/daemon/daemon-like.js +2 -0
- package/dist/cjs/daemon/daemon-process-create-options.js +2 -0
- package/dist/cjs/daemon/daemon-process-state-public-token.js +2 -0
- package/dist/cjs/daemon/daemon-process-state.js +2 -0
- package/dist/cjs/daemon/daemon-util.js +152 -0
- package/dist/cjs/daemon/daemon.js +126 -0
- package/dist/cjs/dao/prototype-dao-config.js +2 -0
- package/dist/cjs/dao/prototype-dao-db.js +2 -0
- package/dist/cjs/dao/prototype-dao-provider.js +2 -0
- package/dist/cjs/dao/prototype-dao.js +88 -0
- package/dist/cjs/dao/s3-prototype-dao-provider.js +28 -0
- package/dist/cjs/dao/s3-simple-dao.js +78 -0
- package/dist/cjs/dao/simple-dao-item.js +2 -0
- package/dist/cjs/dynamodb/dynamo-ratchet-like.js +2 -0
- package/dist/cjs/dynamodb/dynamo-ratchet.js +667 -0
- package/dist/cjs/dynamodb/dynamo-table-ratchet.js +91 -0
- package/dist/cjs/dynamodb/hash-spreader.js +62 -0
- package/dist/cjs/ec2/ec2-ratchet.js +107 -0
- package/dist/cjs/environment/cascade-environment-service-provider.js +27 -0
- package/dist/cjs/environment/env-var-environment-service-provider.js +33 -0
- package/dist/cjs/environment/environment-service-config.js +2 -0
- package/dist/cjs/environment/environment-service-provider.js +2 -0
- package/dist/cjs/environment/environment-service.js +52 -0
- package/dist/cjs/environment/fixed-environment-service-provider.js +24 -0
- package/dist/cjs/environment/s3-environment-service-provider.js +29 -0
- package/dist/cjs/environment/ssm-environment-service-provider.js +61 -0
- package/dist/cjs/expiring-code/dynamo-expiring-code-provider.js +26 -0
- package/dist/cjs/expiring-code/expiring-code-params.js +2 -0
- package/dist/cjs/expiring-code/expiring-code-provider.js +2 -0
- package/dist/cjs/expiring-code/expiring-code-ratchet.js +37 -0
- package/dist/cjs/expiring-code/expiring-code.js +2 -0
- package/dist/cjs/expiring-code/s3-expiring-code-provider.js +49 -0
- package/dist/cjs/iam/aws-credentials-ratchet.js +21 -0
- package/dist/cjs/index.js +81 -0
- package/dist/cjs/lambda/lambda-event-detector.js +42 -0
- package/dist/cjs/lambda/lambda-event-type-guards.js +28 -0
- package/dist/cjs/model/cloud-watch-metrics-minute-level-dynamo-count-request.js +2 -0
- package/dist/cjs/model/cloud-watch-metrics-unit.js +33 -0
- package/dist/cjs/model/dynamo/doc-put-item-command-input.js +2 -0
- package/dist/cjs/model/dynamo/doc-query-command-input.js +2 -0
- package/dist/cjs/model/dynamo/doc-scan-command-input.js +2 -0
- package/dist/cjs/model/dynamo/doc-update-item-command-input.js +2 -0
- package/dist/cjs/model/dynamo-count-result.js +2 -0
- package/dist/cjs/route53/route-53-ratchet.js +57 -0
- package/dist/cjs/runtime-parameter/cached-stored-runtime-parameter.js +2 -0
- package/dist/cjs/runtime-parameter/dynamo-runtime-parameter-provider.js +38 -0
- package/dist/cjs/runtime-parameter/global-variable-override-runtime-parameter-provider.js +54 -0
- package/dist/cjs/runtime-parameter/memory-runtime-parameter-provider.js +30 -0
- package/dist/cjs/runtime-parameter/runtime-parameter-provider.js +2 -0
- package/dist/cjs/runtime-parameter/runtime-parameter-ratchet.js +74 -0
- package/dist/cjs/runtime-parameter/stored-runtime-parameter.js +2 -0
- package/dist/cjs/s3/s3-cache-ratchet.js +332 -0
- package/dist/cjs/s3/s3-cache-to-local-disk-ratchet.js +105 -0
- package/dist/cjs/s3/s3-location-sync-ratchet.js +142 -0
- package/dist/cjs/s3/s3-ratchet.js +26 -0
- package/dist/cjs/ses/email-attachment.js +2 -0
- package/dist/cjs/ses/mailer-config.js +2 -0
- package/dist/cjs/ses/mailer-like.js +2 -0
- package/dist/cjs/ses/mailer.js +208 -0
- package/dist/cjs/ses/ratchet-template-renderer.js +2 -0
- package/dist/cjs/ses/ready-to-send-email.js +2 -0
- package/dist/cjs/ses/remote-handlebars-template-renderer.js +79 -0
- package/dist/cjs/ses/resolved-ready-to-send-email.js +2 -0
- package/dist/cjs/sns/sns-ratchet.js +47 -0
- package/dist/cjs/sync-lock/dynamo-db-sync-lock.js +69 -0
- package/dist/cjs/sync-lock/memory-sync-lock.js +40 -0
- package/dist/cjs/sync-lock/sync-lock-provider.js +2 -0
- package/dist/es/batch/aws-batch-background-processor.js +40 -0
- package/dist/es/batch/aws-batch-ratchet.js +51 -0
- package/dist/es/build/ratchet-aws-info.js +14 -0
- package/dist/es/cache/dynamo-db-storage-provider.js +107 -0
- package/dist/es/cache/s3-storage-provider.js +40 -0
- package/dist/es/cache/simple-cache-object-wrapper.js +1 -0
- package/dist/es/cache/simple-cache-read-options.js +1 -0
- package/dist/es/cache/simple-cache-storage-provider.js +1 -0
- package/dist/es/cache/simple-cache.js +62 -0
- package/dist/es/cloudwatch/cloud-watch-log-group-ratchet.js +69 -0
- package/dist/es/cloudwatch/cloud-watch-logs-ratchet.js +169 -0
- package/dist/es/cloudwatch/cloud-watch-metrics-ratchet.js +53 -0
- package/dist/es/daemon/daemon-like.js +1 -0
- package/dist/es/daemon/daemon-process-create-options.js +1 -0
- package/dist/es/daemon/daemon-process-state-public-token.js +1 -0
- package/dist/es/daemon/daemon-process-state.js +1 -0
- package/dist/es/daemon/daemon-util.js +148 -0
- package/dist/es/daemon/daemon.js +122 -0
- package/dist/es/dao/prototype-dao-config.js +1 -0
- package/dist/es/dao/prototype-dao-db.js +1 -0
- package/dist/es/dao/prototype-dao-provider.js +1 -0
- package/dist/es/dao/prototype-dao.js +84 -0
- package/dist/es/dao/s3-prototype-dao-provider.js +24 -0
- package/dist/es/dao/s3-simple-dao.js +74 -0
- package/dist/es/dao/simple-dao-item.js +1 -0
- package/dist/es/dynamodb/dynamo-ratchet-like.js +1 -0
- package/dist/es/dynamodb/dynamo-ratchet.js +663 -0
- package/dist/es/dynamodb/dynamo-table-ratchet.js +87 -0
- package/dist/es/dynamodb/hash-spreader.js +58 -0
- package/dist/es/ec2/ec2-ratchet.js +103 -0
- package/dist/es/environment/cascade-environment-service-provider.js +23 -0
- package/dist/es/environment/env-var-environment-service-provider.js +29 -0
- package/dist/es/environment/environment-service-config.js +1 -0
- package/dist/es/environment/environment-service-provider.js +1 -0
- package/dist/es/environment/environment-service.js +48 -0
- package/dist/es/environment/fixed-environment-service-provider.js +20 -0
- package/dist/es/environment/s3-environment-service-provider.js +25 -0
- package/dist/es/environment/ssm-environment-service-provider.js +56 -0
- package/dist/es/expiring-code/dynamo-expiring-code-provider.js +22 -0
- package/dist/es/expiring-code/expiring-code-params.js +1 -0
- package/dist/es/expiring-code/expiring-code-provider.js +1 -0
- package/dist/es/expiring-code/expiring-code-ratchet.js +33 -0
- package/dist/es/expiring-code/expiring-code.js +1 -0
- package/dist/es/expiring-code/s3-expiring-code-provider.js +45 -0
- package/dist/es/iam/aws-credentials-ratchet.js +17 -0
- package/dist/es/index.js +78 -0
- package/dist/es/lambda/lambda-event-detector.js +38 -0
- package/dist/es/lambda/lambda-event-type-guards.js +24 -0
- package/dist/es/model/cloud-watch-metrics-minute-level-dynamo-count-request.js +1 -0
- package/dist/es/model/cloud-watch-metrics-unit.js +30 -0
- package/dist/es/model/dynamo/doc-put-item-command-input.js +1 -0
- package/dist/es/model/dynamo/doc-query-command-input.js +1 -0
- package/dist/es/model/dynamo/doc-scan-command-input.js +1 -0
- package/dist/es/model/dynamo/doc-update-item-command-input.js +1 -0
- package/dist/es/model/dynamo-count-result.js +1 -0
- package/dist/es/route53/route-53-ratchet.js +53 -0
- package/dist/es/runtime-parameter/cached-stored-runtime-parameter.js +1 -0
- package/dist/es/runtime-parameter/dynamo-runtime-parameter-provider.js +34 -0
- package/dist/es/runtime-parameter/global-variable-override-runtime-parameter-provider.js +49 -0
- package/dist/es/runtime-parameter/memory-runtime-parameter-provider.js +26 -0
- package/dist/es/runtime-parameter/runtime-parameter-provider.js +1 -0
- package/dist/es/runtime-parameter/runtime-parameter-ratchet.js +70 -0
- package/dist/es/runtime-parameter/stored-runtime-parameter.js +1 -0
- package/dist/es/s3/s3-cache-ratchet.js +328 -0
- package/dist/es/s3/s3-cache-to-local-disk-ratchet.js +100 -0
- package/dist/es/s3/s3-location-sync-ratchet.js +137 -0
- package/dist/es/s3/s3-ratchet.js +22 -0
- package/dist/es/ses/email-attachment.js +1 -0
- package/dist/es/ses/mailer-config.js +1 -0
- package/dist/es/ses/mailer-like.js +1 -0
- package/dist/es/ses/mailer.js +204 -0
- package/dist/es/ses/ratchet-template-renderer.js +1 -0
- package/dist/es/ses/ready-to-send-email.js +1 -0
- package/dist/es/ses/remote-handlebars-template-renderer.js +74 -0
- package/dist/es/ses/resolved-ready-to-send-email.js +1 -0
- package/dist/es/sns/sns-ratchet.js +43 -0
- package/dist/es/sync-lock/dynamo-db-sync-lock.js +65 -0
- package/dist/es/sync-lock/memory-sync-lock.js +36 -0
- package/dist/es/sync-lock/sync-lock-provider.js +1 -0
- package/dist/tsconfig.cjs.tsbuildinfo +1 -0
- package/dist/tsconfig.es.tsbuildinfo +1 -0
- package/dist/tsconfig.types.tsbuildinfo +1 -0
- package/dist/types/batch/aws-batch-background-processor.d.ts +12 -0
- package/dist/types/batch/aws-batch-ratchet.d.ts +16 -0
- package/dist/types/build/ratchet-aws-info.d.ts +5 -0
- package/dist/types/cache/dynamo-db-storage-provider.d.ts +25 -0
- package/dist/types/cache/s3-storage-provider.d.ts +14 -0
- package/dist/types/cache/simple-cache-object-wrapper.d.ts +7 -0
- package/dist/types/cache/simple-cache-read-options.d.ts +5 -0
- package/dist/types/cache/simple-cache-storage-provider.d.ts +8 -0
- package/dist/types/cache/simple-cache.d.ts +14 -0
- package/dist/types/cloudwatch/cloud-watch-log-group-ratchet.d.ts +9 -0
- package/dist/types/cloudwatch/cloud-watch-logs-ratchet.d.ts +14 -0
- package/dist/types/cloudwatch/cloud-watch-metrics-ratchet.d.ts +10 -0
- package/dist/types/daemon/daemon-like.d.ts +17 -0
- package/dist/types/daemon/daemon-process-create-options.d.ts +7 -0
- package/dist/types/daemon/daemon-process-state-public-token.d.ts +4 -0
- package/dist/types/daemon/daemon-process-state.d.ts +13 -0
- package/dist/types/daemon/daemon-util.d.ts +24 -0
- package/dist/types/daemon/daemon.d.ts +33 -0
- package/dist/types/dao/prototype-dao-config.d.ts +8 -0
- package/dist/types/dao/prototype-dao-db.d.ts +4 -0
- package/dist/types/dao/prototype-dao-provider.d.ts +5 -0
- package/dist/types/dao/prototype-dao.d.ts +15 -0
- package/dist/types/dao/s3-prototype-dao-provider.d.ts +10 -0
- package/dist/types/dao/s3-simple-dao.d.ts +15 -0
- package/dist/types/dao/simple-dao-item.d.ts +5 -0
- package/dist/types/dynamodb/dynamo-ratchet-like.d.ts +27 -0
- package/dist/types/dynamodb/dynamo-ratchet.d.ts +36 -0
- package/dist/types/dynamodb/dynamo-table-ratchet.d.ts +11 -0
- package/dist/types/dynamodb/hash-spreader.d.ts +15 -0
- package/dist/types/ec2/ec2-ratchet.d.ts +25 -0
- package/dist/types/environment/cascade-environment-service-provider.d.ts +9 -0
- package/dist/types/environment/env-var-environment-service-provider.d.ts +10 -0
- package/dist/types/environment/environment-service-config.d.ts +7 -0
- package/dist/types/environment/environment-service-provider.d.ts +7 -0
- package/dist/types/environment/environment-service.d.ts +14 -0
- package/dist/types/environment/fixed-environment-service-provider.d.ts +10 -0
- package/dist/types/environment/s3-environment-service-provider.d.ts +18 -0
- package/dist/types/environment/ssm-environment-service-provider.d.ts +12 -0
- package/dist/types/expiring-code/dynamo-expiring-code-provider.d.ts +12 -0
- package/dist/types/expiring-code/expiring-code-params.d.ts +7 -0
- package/dist/types/expiring-code/expiring-code-provider.d.ts +5 -0
- package/dist/types/expiring-code/expiring-code-ratchet.d.ts +13 -0
- package/dist/types/expiring-code/expiring-code.d.ts +6 -0
- package/dist/types/expiring-code/s3-expiring-code-provider.d.ts +17 -0
- package/dist/types/iam/aws-credentials-ratchet.d.ts +9 -0
- package/dist/types/index.d.ts +81 -0
- package/dist/types/lambda/lambda-event-detector.d.ts +14 -0
- package/dist/types/lambda/lambda-event-type-guards.d.ts +10 -0
- package/dist/types/model/cloud-watch-metrics-minute-level-dynamo-count-request.d.ts +12 -0
- package/dist/types/model/cloud-watch-metrics-unit.d.ts +29 -0
- package/dist/types/model/dynamo/doc-put-item-command-input.d.ts +4 -0
- package/dist/types/model/dynamo/doc-query-command-input.d.ts +5 -0
- package/dist/types/model/dynamo/doc-scan-command-input.d.ts +5 -0
- package/dist/types/model/dynamo/doc-update-item-command-input.d.ts +5 -0
- package/dist/types/model/dynamo-count-result.d.ts +5 -0
- package/dist/types/route53/route-53-ratchet.d.ts +7 -0
- package/dist/types/runtime-parameter/cached-stored-runtime-parameter.d.ts +4 -0
- package/dist/types/runtime-parameter/dynamo-runtime-parameter-provider.d.ts +11 -0
- package/dist/types/runtime-parameter/global-variable-override-runtime-parameter-provider.d.ts +24 -0
- package/dist/types/runtime-parameter/memory-runtime-parameter-provider.d.ts +13 -0
- package/dist/types/runtime-parameter/runtime-parameter-provider.d.ts +11 -0
- package/dist/types/runtime-parameter/runtime-parameter-ratchet.d.ts +15 -0
- package/dist/types/runtime-parameter/stored-runtime-parameter.d.ts +6 -0
- package/dist/types/s3/s3-cache-ratchet.d.ts +38 -0
- package/dist/types/s3/s3-cache-to-local-disk-ratchet.d.ts +21 -0
- package/dist/types/s3/s3-location-sync-ratchet.d.ts +21 -0
- package/dist/types/s3/s3-ratchet.d.ts +5 -0
- package/dist/types/ses/email-attachment.d.ts +23 -0
- package/dist/types/ses/mailer-config.d.ts +15 -0
- package/dist/types/ses/mailer-like.d.ts +18 -0
- package/dist/types/ses/mailer.d.ts +26 -0
- package/dist/types/ses/ratchet-template-renderer.d.ts +8 -0
- package/dist/types/ses/ready-to-send-email.d.ts +66 -0
- package/dist/types/ses/remote-handlebars-template-renderer.d.ts +15 -0
- package/dist/types/ses/resolved-ready-to-send-email.d.ts +16 -0
- package/dist/types/sns/sns-ratchet.d.ts +8 -0
- package/dist/types/sync-lock/dynamo-db-sync-lock.d.ts +10 -0
- package/dist/types/sync-lock/memory-sync-lock.d.ts +11 -0
- package/dist/types/sync-lock/sync-lock-provider.d.ts +5 -0
- package/package.json +112 -0
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.S3CacheRatchet = void 0;
|
|
4
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
5
|
+
const ratchet_common_1 = require("@bitblit/ratchet-common");
|
|
6
|
+
const ratchet_common_2 = require("@bitblit/ratchet-common");
|
|
7
|
+
const ratchet_common_3 = require("@bitblit/ratchet-common");
|
|
8
|
+
const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
|
|
9
|
+
const lib_storage_1 = require("@aws-sdk/lib-storage");
|
|
10
|
+
const ratchet_common_4 = require("@bitblit/ratchet-common");
|
|
11
|
+
const ratchet_common_5 = require("@bitblit/ratchet-common");
|
|
12
|
+
class S3CacheRatchet {
|
|
13
|
+
constructor(s3, defaultBucket = null) {
|
|
14
|
+
this.s3 = s3;
|
|
15
|
+
this.defaultBucket = defaultBucket;
|
|
16
|
+
ratchet_common_2.RequireRatchet.notNullOrUndefined(this.s3, 's3');
|
|
17
|
+
}
|
|
18
|
+
static applyCacheControlMaxAge(input, seconds) {
|
|
19
|
+
if (input && seconds) {
|
|
20
|
+
input.CacheControl = 'max-age=' + seconds;
|
|
21
|
+
}
|
|
22
|
+
return input;
|
|
23
|
+
}
|
|
24
|
+
static applyUserMetaData(input, key, value) {
|
|
25
|
+
if (input && ratchet_common_4.StringRatchet.trimToNull(key) && ratchet_common_4.StringRatchet.trimToNull(value)) {
|
|
26
|
+
input.Metadata = input.Metadata || {};
|
|
27
|
+
input.Metadata[key] = value;
|
|
28
|
+
}
|
|
29
|
+
return input;
|
|
30
|
+
}
|
|
31
|
+
getDefaultBucket() {
|
|
32
|
+
return this.defaultBucket;
|
|
33
|
+
}
|
|
34
|
+
getS3Client() {
|
|
35
|
+
return this.s3;
|
|
36
|
+
}
|
|
37
|
+
async fileExists(key, bucket = null) {
|
|
38
|
+
try {
|
|
39
|
+
const head = await this.fetchMetaForCacheFile(key, this.bucketVal(bucket));
|
|
40
|
+
return !!head;
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
ratchet_common_1.Logger.silly('Error calling file exists (as expected) %s', err);
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
async fetchCacheFileAsS3GetObjectCommandOutput(key, bucket = null) {
|
|
48
|
+
let rval = null;
|
|
49
|
+
try {
|
|
50
|
+
const params = {
|
|
51
|
+
Bucket: this.bucketVal(bucket),
|
|
52
|
+
Key: key,
|
|
53
|
+
};
|
|
54
|
+
rval = await this.s3.send(new client_s3_1.GetObjectCommand(params));
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
if (err instanceof client_s3_1.NoSuchKey) {
|
|
58
|
+
ratchet_common_1.Logger.debug('Key %s not found - returning null', key);
|
|
59
|
+
rval = null;
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
throw err;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return rval;
|
|
66
|
+
}
|
|
67
|
+
async fetchCacheFileAsReadableStream(key, bucket = null) {
|
|
68
|
+
const out = await this.fetchCacheFileAsS3GetObjectCommandOutput(key, bucket);
|
|
69
|
+
return out.Body.transformToWebStream();
|
|
70
|
+
}
|
|
71
|
+
async fetchCacheFileAsBuffer(key, bucket = null) {
|
|
72
|
+
let rval = null;
|
|
73
|
+
const out = await this.fetchCacheFileAsS3GetObjectCommandOutput(key, bucket);
|
|
74
|
+
if (out === null || out === void 0 ? void 0 : out.Body) {
|
|
75
|
+
const tmp = await out.Body.transformToByteArray();
|
|
76
|
+
rval = Buffer.from(tmp);
|
|
77
|
+
}
|
|
78
|
+
return rval;
|
|
79
|
+
}
|
|
80
|
+
async fetchCacheFileAsString(key, bucket = null) {
|
|
81
|
+
let rval = null;
|
|
82
|
+
const out = await this.fetchCacheFileAsS3GetObjectCommandOutput(key, bucket);
|
|
83
|
+
if (out === null || out === void 0 ? void 0 : out.Body) {
|
|
84
|
+
rval = await out.Body.transformToString();
|
|
85
|
+
}
|
|
86
|
+
return rval;
|
|
87
|
+
}
|
|
88
|
+
async fetchCacheFileAsObject(key, bucket = null) {
|
|
89
|
+
const value = await this.fetchCacheFileAsString(key, bucket);
|
|
90
|
+
return value ? JSON.parse(value) : null;
|
|
91
|
+
}
|
|
92
|
+
async removeCacheFile(key, bucket = null) {
|
|
93
|
+
let rval = null;
|
|
94
|
+
const params = {
|
|
95
|
+
Bucket: this.bucketVal(bucket),
|
|
96
|
+
Key: key,
|
|
97
|
+
};
|
|
98
|
+
try {
|
|
99
|
+
rval = await this.s3.send(new client_s3_1.DeleteObjectCommand(params));
|
|
100
|
+
}
|
|
101
|
+
catch (err) {
|
|
102
|
+
if (err && err['statusCode'] == 404) {
|
|
103
|
+
ratchet_common_1.Logger.info('Swallowing 404 deleting missing object %s %s', bucket, key);
|
|
104
|
+
rval = null;
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
throw err;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return rval;
|
|
111
|
+
}
|
|
112
|
+
async writeObjectToCacheFile(key, dataObject, template, bucket) {
|
|
113
|
+
const json = JSON.stringify(dataObject);
|
|
114
|
+
return this.writeStringToCacheFile(key, json, template, bucket);
|
|
115
|
+
}
|
|
116
|
+
async writeStringToCacheFile(key, dataString, template, bucket) {
|
|
117
|
+
const stream = ratchet_common_5.StreamRatchet.stringToWebReadableStream(dataString);
|
|
118
|
+
return this.writeStreamToCacheFile(key, stream, template, bucket);
|
|
119
|
+
}
|
|
120
|
+
async writeStreamToCacheFile(key, data, template, bucket) {
|
|
121
|
+
const params = Object.assign({}, template || {}, {
|
|
122
|
+
Bucket: this.bucketVal(bucket),
|
|
123
|
+
Key: key,
|
|
124
|
+
Body: data,
|
|
125
|
+
});
|
|
126
|
+
const upload = new lib_storage_1.Upload({
|
|
127
|
+
client: this.s3,
|
|
128
|
+
params: params,
|
|
129
|
+
tags: [],
|
|
130
|
+
queueSize: 4,
|
|
131
|
+
partSize: 1024 * 1024 * 5,
|
|
132
|
+
leavePartsOnError: false,
|
|
133
|
+
});
|
|
134
|
+
upload.on('httpUploadProgress', (progress) => {
|
|
135
|
+
ratchet_common_1.Logger.info('Uploading : %s', progress);
|
|
136
|
+
});
|
|
137
|
+
const result = await upload.done();
|
|
138
|
+
return result;
|
|
139
|
+
}
|
|
140
|
+
async synchronize(srcPrefix, targetPrefix, targetRatchet = this, recurseSubFolders = false) {
|
|
141
|
+
ratchet_common_2.RequireRatchet.notNullOrUndefined(srcPrefix, 'srcPrefix');
|
|
142
|
+
ratchet_common_2.RequireRatchet.notNullOrUndefined(targetPrefix, 'targetPrefix');
|
|
143
|
+
ratchet_common_2.RequireRatchet.true(srcPrefix.endsWith('/'), 'srcPrefix must end in /');
|
|
144
|
+
ratchet_common_2.RequireRatchet.true(targetPrefix.endsWith('/'), 'targetPrefix must end in /');
|
|
145
|
+
let rval = [];
|
|
146
|
+
const sourceFiles = await this.directChildrenOfPrefix(srcPrefix);
|
|
147
|
+
const targetFiles = await targetRatchet.directChildrenOfPrefix(targetPrefix);
|
|
148
|
+
const sw = new ratchet_common_3.StopWatch();
|
|
149
|
+
for (let i = 0; i < sourceFiles.length; i++) {
|
|
150
|
+
const sourceFile = sourceFiles[i];
|
|
151
|
+
ratchet_common_1.Logger.info('Processing %s : %s', sourceFile, sw.dumpExpected(i / sourceFiles.length));
|
|
152
|
+
if (sourceFile.endsWith('/')) {
|
|
153
|
+
if (recurseSubFolders) {
|
|
154
|
+
ratchet_common_1.Logger.info('%s is a subfolder - recursing');
|
|
155
|
+
const subs = await this.synchronize(srcPrefix + sourceFile, targetPrefix + sourceFile, targetRatchet, recurseSubFolders);
|
|
156
|
+
ratchet_common_1.Logger.info('Got %d back from %s', subs.length, sourceFile);
|
|
157
|
+
rval = rval.concat(subs);
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
ratchet_common_1.Logger.info('%s is a subfolder and recurse not specified - skipping', sourceFile);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
let shouldCopy = true;
|
|
165
|
+
const srcMeta = await this.fetchMetaForCacheFile(srcPrefix + sourceFile);
|
|
166
|
+
if (targetFiles.includes(sourceFile)) {
|
|
167
|
+
const targetMeta = await targetRatchet.fetchMetaForCacheFile(targetPrefix + sourceFile);
|
|
168
|
+
if (srcMeta.ETag === targetMeta.ETag) {
|
|
169
|
+
ratchet_common_1.Logger.debug('Skipping - identical');
|
|
170
|
+
shouldCopy = false;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (shouldCopy) {
|
|
174
|
+
ratchet_common_1.Logger.debug('Copying...');
|
|
175
|
+
const srcStream = await this.fetchCacheFileAsReadableStream(srcPrefix + sourceFile);
|
|
176
|
+
try {
|
|
177
|
+
const written = await targetRatchet.writeStreamToCacheFile(targetPrefix + sourceFile, srcStream, srcMeta, undefined);
|
|
178
|
+
ratchet_common_1.Logger.silly('Write result : %j', written);
|
|
179
|
+
rval.push(sourceFile);
|
|
180
|
+
}
|
|
181
|
+
catch (err) {
|
|
182
|
+
ratchet_common_1.Logger.error('Failed to sync : %s : %s', sourceFile, err);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
ratchet_common_1.Logger.info('Found %d files, copied %d', sourceFiles.length, rval.length);
|
|
188
|
+
sw.log();
|
|
189
|
+
return rval;
|
|
190
|
+
}
|
|
191
|
+
async fetchMetaForCacheFile(key, bucket = null) {
|
|
192
|
+
let rval = null;
|
|
193
|
+
try {
|
|
194
|
+
rval = await this.s3.send(new client_s3_1.HeadObjectCommand({
|
|
195
|
+
Bucket: this.bucketVal(bucket),
|
|
196
|
+
Key: key,
|
|
197
|
+
}));
|
|
198
|
+
}
|
|
199
|
+
catch (err) {
|
|
200
|
+
if (err && err['statusCode'] == 404) {
|
|
201
|
+
ratchet_common_1.Logger.info('Cache file %s %s not found returning null', this.bucketVal(bucket), key);
|
|
202
|
+
rval = null;
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
ratchet_common_1.Logger.error('Unrecognized error, rethrowing : %s', err, err);
|
|
206
|
+
throw err;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return rval;
|
|
210
|
+
}
|
|
211
|
+
async cacheFileAgeInSeconds(key, bucket = null) {
|
|
212
|
+
try {
|
|
213
|
+
const res = await this.fetchMetaForCacheFile(key, bucket);
|
|
214
|
+
if (res && res.LastModified) {
|
|
215
|
+
return Math.floor((new Date().getTime() - res.LastModified.getTime()) / 1000);
|
|
216
|
+
}
|
|
217
|
+
else {
|
|
218
|
+
ratchet_common_1.Logger.warn('Cache file %s %s had no last modified returning null', this.bucketVal(bucket), key);
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
catch (err) {
|
|
223
|
+
if (err && err['statusCode'] == 404) {
|
|
224
|
+
ratchet_common_1.Logger.warn('Cache file %s %s not found returning null', this.bucketVal(bucket), key);
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
throw err;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
async copyFile(srcKey, dstKey, srcBucket = null, dstBucket = null) {
|
|
233
|
+
const params = {
|
|
234
|
+
CopySource: '/' + this.bucketVal(srcBucket) + '/' + srcKey,
|
|
235
|
+
Bucket: this.bucketVal(dstBucket),
|
|
236
|
+
Key: dstKey,
|
|
237
|
+
MetadataDirective: 'COPY',
|
|
238
|
+
};
|
|
239
|
+
const rval = await this.s3.send(new client_s3_1.CopyObjectCommand(params));
|
|
240
|
+
return rval;
|
|
241
|
+
}
|
|
242
|
+
async quietCopyFile(srcKey, dstKey, srcBucket = null, dstBucket = null) {
|
|
243
|
+
let rval = false;
|
|
244
|
+
try {
|
|
245
|
+
const tmp = await this.copyFile(srcKey, dstKey, srcBucket, dstBucket);
|
|
246
|
+
rval = true;
|
|
247
|
+
}
|
|
248
|
+
catch (err) {
|
|
249
|
+
ratchet_common_1.Logger.silly('Failed to copy file in S3 : %s', err);
|
|
250
|
+
}
|
|
251
|
+
return rval;
|
|
252
|
+
}
|
|
253
|
+
async preSignedDownloadUrlForCacheFile(key, expirationSeconds = 3600, bucket = null) {
|
|
254
|
+
const getCommand = {
|
|
255
|
+
Bucket: this.bucketVal(bucket),
|
|
256
|
+
Key: key,
|
|
257
|
+
};
|
|
258
|
+
const link = await (0, s3_request_presigner_1.getSignedUrl)(this.s3, new client_s3_1.GetObjectCommand(getCommand), { expiresIn: expirationSeconds });
|
|
259
|
+
return link;
|
|
260
|
+
}
|
|
261
|
+
async directChildrenOfPrefix(prefix, expandFiles = false, bucket = null, maxToReturn = null) {
|
|
262
|
+
const returnValue = [];
|
|
263
|
+
const params = {
|
|
264
|
+
Bucket: this.bucketVal(bucket),
|
|
265
|
+
Prefix: prefix,
|
|
266
|
+
Delimiter: '/',
|
|
267
|
+
};
|
|
268
|
+
let response = null;
|
|
269
|
+
do {
|
|
270
|
+
response = await this.s3.send(new client_s3_1.ListObjectsCommand(params));
|
|
271
|
+
const prefixLength = prefix.length;
|
|
272
|
+
if (response['CommonPrefixes']) {
|
|
273
|
+
response['CommonPrefixes'].forEach((cp) => {
|
|
274
|
+
if (!maxToReturn || returnValue.length < maxToReturn) {
|
|
275
|
+
const value = cp['Prefix'].substring(prefixLength);
|
|
276
|
+
returnValue.push(value);
|
|
277
|
+
}
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
if (response['Contents']) {
|
|
281
|
+
await Promise.all(response['Contents'].map(async (cp) => {
|
|
282
|
+
if (!maxToReturn || returnValue.length < maxToReturn) {
|
|
283
|
+
if (expandFiles) {
|
|
284
|
+
const expanded = {
|
|
285
|
+
link: await this.preSignedDownloadUrlForCacheFile(cp['Key'], 3600, bucket),
|
|
286
|
+
name: cp['Key'].substring(prefixLength),
|
|
287
|
+
size: cp['Size'],
|
|
288
|
+
};
|
|
289
|
+
returnValue.push(expanded);
|
|
290
|
+
}
|
|
291
|
+
else {
|
|
292
|
+
returnValue.push(cp['Key'].substring(prefixLength));
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}));
|
|
296
|
+
}
|
|
297
|
+
params.Marker = response.NextMarker;
|
|
298
|
+
} while (params.Marker && (!maxToReturn || returnValue.length < maxToReturn));
|
|
299
|
+
return returnValue;
|
|
300
|
+
}
|
|
301
|
+
async allSubFoldersOfPrefix(prefix, bucket = null) {
|
|
302
|
+
const returnValue = [prefix];
|
|
303
|
+
let idx = 0;
|
|
304
|
+
while (idx < returnValue.length) {
|
|
305
|
+
const next = returnValue[idx++];
|
|
306
|
+
ratchet_common_1.Logger.debug('Pulling %s (%d remaining)', next, returnValue.length - idx);
|
|
307
|
+
const req = {
|
|
308
|
+
Bucket: this.bucketVal(bucket),
|
|
309
|
+
Prefix: next,
|
|
310
|
+
Delimiter: '/',
|
|
311
|
+
};
|
|
312
|
+
let resp = null;
|
|
313
|
+
do {
|
|
314
|
+
req.ContinuationToken = resp ? resp.NextContinuationToken : null;
|
|
315
|
+
resp = await this.s3.send(new client_s3_1.ListObjectsCommand(req));
|
|
316
|
+
resp.CommonPrefixes.forEach((p) => {
|
|
317
|
+
returnValue.push(p.Prefix);
|
|
318
|
+
});
|
|
319
|
+
ratchet_common_1.Logger.debug('g:%j', resp);
|
|
320
|
+
} while (resp.NextContinuationToken);
|
|
321
|
+
}
|
|
322
|
+
return returnValue;
|
|
323
|
+
}
|
|
324
|
+
bucketVal(explicitBucket) {
|
|
325
|
+
const rval = explicitBucket ? explicitBucket : this.defaultBucket;
|
|
326
|
+
if (!rval) {
|
|
327
|
+
throw 'You must set either the default bucket or pass it explicitly';
|
|
328
|
+
}
|
|
329
|
+
return rval;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
exports.S3CacheRatchet = S3CacheRatchet;
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.S3CacheToLocalDiskRatchet = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const crypto_1 = tslib_1.__importDefault(require("crypto"));
|
|
6
|
+
const fs_1 = tslib_1.__importDefault(require("fs"));
|
|
7
|
+
const path_1 = tslib_1.__importDefault(require("path"));
|
|
8
|
+
const ratchet_common_1 = require("@bitblit/ratchet-common");
|
|
9
|
+
class S3CacheToLocalDiskRatchet {
|
|
10
|
+
constructor(s3, tmpFolder, cacheTimeoutSeconds = S3CacheToLocalDiskRatchet.DEFAULT_CACHE_TIMEOUT_SEC) {
|
|
11
|
+
this.s3 = s3;
|
|
12
|
+
this.tmpFolder = tmpFolder;
|
|
13
|
+
this.cacheTimeoutSeconds = cacheTimeoutSeconds;
|
|
14
|
+
this.currentlyLoading = new Map();
|
|
15
|
+
ratchet_common_1.RequireRatchet.notNullOrUndefined(s3, 's3');
|
|
16
|
+
ratchet_common_1.RequireRatchet.notNullOrUndefined(ratchet_common_1.StringRatchet.trimToNull(tmpFolder));
|
|
17
|
+
ratchet_common_1.RequireRatchet.true(fs_1.default.existsSync(tmpFolder), 'folder must exist : ' + tmpFolder);
|
|
18
|
+
}
|
|
19
|
+
async getFileString(key) {
|
|
20
|
+
const buf = await this.getFileBuffer(key);
|
|
21
|
+
return buf ? buf.toString() : null;
|
|
22
|
+
}
|
|
23
|
+
keyToLocalCachePath(key) {
|
|
24
|
+
const cachedHash = this.generateCacheHash(this.s3.getDefaultBucket() + '/' + key);
|
|
25
|
+
const rval = path_1.default.join(this.tmpFolder, cachedHash);
|
|
26
|
+
return rval;
|
|
27
|
+
}
|
|
28
|
+
removeCacheFileForKey(key) {
|
|
29
|
+
const localCachePath = this.keyToLocalCachePath(key);
|
|
30
|
+
ratchet_common_1.Logger.info('Removing cache file for %s : %s', key, localCachePath);
|
|
31
|
+
if (fs_1.default.existsSync(localCachePath)) {
|
|
32
|
+
fs_1.default.unlinkSync(localCachePath);
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
ratchet_common_1.Logger.debug('Skipping delete for %s - does not exist', localCachePath);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
async getFileBuffer(key) {
|
|
39
|
+
const localCachePath = this.keyToLocalCachePath(key);
|
|
40
|
+
let rval = null;
|
|
41
|
+
rval = this.getCacheFileAsBuffer(localCachePath);
|
|
42
|
+
if (!rval) {
|
|
43
|
+
ratchet_common_1.Logger.info('No cache. Downloading File s3://%s/%s to %s', this.s3.getDefaultBucket(), key, localCachePath);
|
|
44
|
+
try {
|
|
45
|
+
let prom = this.currentlyLoading.get(key);
|
|
46
|
+
if (prom) {
|
|
47
|
+
ratchet_common_1.Logger.info('Already running - wait for that');
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
ratchet_common_1.Logger.info('Not running - start');
|
|
51
|
+
prom = this.updateLocalCacheFile(key, localCachePath);
|
|
52
|
+
this.currentlyLoading.set(key, prom);
|
|
53
|
+
}
|
|
54
|
+
rval = await prom;
|
|
55
|
+
this.currentlyLoading.delete(key);
|
|
56
|
+
}
|
|
57
|
+
catch (err) {
|
|
58
|
+
ratchet_common_1.Logger.warn('File %s/%s does not exist. Err code: %s', this.s3.getDefaultBucket(), key, err);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
ratchet_common_1.Logger.info('Found cache file for s3://%s/%s. Local path %s', this.s3.getDefaultBucket(), key, localCachePath);
|
|
63
|
+
}
|
|
64
|
+
return rval;
|
|
65
|
+
}
|
|
66
|
+
async updateLocalCacheFile(key, localCachePath) {
|
|
67
|
+
const rval = await this.s3.fetchCacheFileAsBuffer(key);
|
|
68
|
+
if (rval && rval.length > 0) {
|
|
69
|
+
ratchet_common_1.Logger.info('Saving %d bytes to disk for cache', rval.length);
|
|
70
|
+
fs_1.default.writeFileSync(localCachePath, rval);
|
|
71
|
+
}
|
|
72
|
+
return rval;
|
|
73
|
+
}
|
|
74
|
+
getCacheFileAsString(filePath) {
|
|
75
|
+
const buf = this.getCacheFileAsBuffer(filePath);
|
|
76
|
+
return buf ? buf.toString() : null;
|
|
77
|
+
}
|
|
78
|
+
getCacheFileAsBuffer(filePath) {
|
|
79
|
+
if (!fs_1.default.existsSync(filePath)) {
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
try {
|
|
83
|
+
const stats = fs_1.default.statSync(filePath);
|
|
84
|
+
const now = new Date().getTime();
|
|
85
|
+
const duration = (now - stats.ctimeMs) / 1000;
|
|
86
|
+
if (duration >= this.cacheTimeoutSeconds) {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
const rval = fs_1.default.readFileSync(filePath);
|
|
91
|
+
return rval;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
catch (err) {
|
|
95
|
+
ratchet_common_1.Logger.warn('Error getting s3 cache file %s', err);
|
|
96
|
+
}
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
generateCacheHash(hashVal) {
|
|
100
|
+
const rval = crypto_1.default.createHash('md5').update(hashVal).digest('hex');
|
|
101
|
+
return rval;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
exports.S3CacheToLocalDiskRatchet = S3CacheToLocalDiskRatchet;
|
|
105
|
+
S3CacheToLocalDiskRatchet.DEFAULT_CACHE_TIMEOUT_SEC = 7 * 24 * 3600;
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.S3LocationSyncRatchet = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
6
|
+
const lodash_1 = tslib_1.__importDefault(require("lodash"));
|
|
7
|
+
const ratchet_common_1 = require("@bitblit/ratchet-common");
|
|
8
|
+
const lib_storage_1 = require("@aws-sdk/lib-storage");
|
|
9
|
+
class S3LocationSyncRatchet {
|
|
10
|
+
constructor(config) {
|
|
11
|
+
ratchet_common_1.RequireRatchet.notNullOrUndefined(config, 'config');
|
|
12
|
+
this.config = config;
|
|
13
|
+
if (!this.config.maxNumThreads) {
|
|
14
|
+
this.config.maxNumThreads = 15;
|
|
15
|
+
}
|
|
16
|
+
if (!this.config.maxRetries) {
|
|
17
|
+
this.config.maxRetries = 5;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
updateSrcPrefix(prefix) {
|
|
21
|
+
this.config.srcPrefix = prefix;
|
|
22
|
+
}
|
|
23
|
+
updateDstPrefix(prefix) {
|
|
24
|
+
this.config.dstPrefix = prefix;
|
|
25
|
+
}
|
|
26
|
+
async copyObject(key, size, express = false) {
|
|
27
|
+
const dstKey = key.replace(this.config.srcPrefix, this.config.dstPrefix);
|
|
28
|
+
let completedCopying = false;
|
|
29
|
+
let retries = 0;
|
|
30
|
+
while (!completedCopying && retries < this.config.maxRetries) {
|
|
31
|
+
ratchet_common_1.Logger.debug(`${retries > 0 ? `Retry ${retries} ` : ''}${express ? 'Express' : 'Slow'} copying
|
|
32
|
+
[${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]`);
|
|
33
|
+
try {
|
|
34
|
+
if (express) {
|
|
35
|
+
const params = {
|
|
36
|
+
CopySource: encodeURIComponent([this.config.srcBucket, key].join('/')),
|
|
37
|
+
Bucket: this.config.dstBucket,
|
|
38
|
+
Key: dstKey,
|
|
39
|
+
MetadataDirective: 'COPY',
|
|
40
|
+
};
|
|
41
|
+
await this.config.dstS3.send(new client_s3_1.CopyObjectCommand(params));
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
const fetched = await this.config.srcS3.send(new client_s3_1.GetObjectCommand({ Bucket: this.config.srcBucket, Key: key }));
|
|
45
|
+
const params = {
|
|
46
|
+
Bucket: this.config.dstBucket,
|
|
47
|
+
Key: dstKey,
|
|
48
|
+
Body: fetched.Body,
|
|
49
|
+
ContentLength: size,
|
|
50
|
+
};
|
|
51
|
+
const upload = new lib_storage_1.Upload({
|
|
52
|
+
client: this.config.dstS3,
|
|
53
|
+
params: params,
|
|
54
|
+
tags: [],
|
|
55
|
+
queueSize: 4,
|
|
56
|
+
partSize: 1024 * 1024 * 5,
|
|
57
|
+
leavePartsOnError: false,
|
|
58
|
+
});
|
|
59
|
+
upload.on('httpUploadProgress', (progress) => {
|
|
60
|
+
ratchet_common_1.Logger.info('Uploading : %s', progress);
|
|
61
|
+
});
|
|
62
|
+
await upload.done();
|
|
63
|
+
}
|
|
64
|
+
completedCopying = true;
|
|
65
|
+
}
|
|
66
|
+
catch (err) {
|
|
67
|
+
ratchet_common_1.Logger.warn(`Can't ${express ? 'express' : 'slow'} copy
|
|
68
|
+
[${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]: %j`, err);
|
|
69
|
+
retries++;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
ratchet_common_1.Logger.debug(`Finished ${express ? 'express' : 'slow'} copying
|
|
73
|
+
[${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]`);
|
|
74
|
+
}
|
|
75
|
+
async listObjects(bucket, prefix, s3) {
|
|
76
|
+
ratchet_common_1.Logger.info(`Scanning bucket [${[bucket, prefix].join('/')}]`);
|
|
77
|
+
const params = {
|
|
78
|
+
Bucket: bucket,
|
|
79
|
+
Prefix: prefix,
|
|
80
|
+
};
|
|
81
|
+
let more = true;
|
|
82
|
+
const rval = {};
|
|
83
|
+
while (more) {
|
|
84
|
+
const response = await s3.send(new client_s3_1.ListObjectsV2Command(params));
|
|
85
|
+
more = response.IsTruncated;
|
|
86
|
+
lodash_1.default.each(response.Contents, (obj) => {
|
|
87
|
+
rval[obj.Key] = { Key: obj.Key, LastModified: obj.LastModified, ETag: obj.ETag, Size: obj.Size };
|
|
88
|
+
});
|
|
89
|
+
if (more) {
|
|
90
|
+
params.ContinuationToken = response.NextContinuationToken;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return rval;
|
|
94
|
+
}
|
|
95
|
+
async startSyncing() {
|
|
96
|
+
ratchet_common_1.Logger.info(`Syncing [${this.config.srcBucket}/${this.config.srcPrefix}
|
|
97
|
+
---> ${this.config.dstBucket}/${this.config.dstPrefix}]`);
|
|
98
|
+
const cp = async (obj) => {
|
|
99
|
+
await this.copyObject(obj.Key, obj.Size);
|
|
100
|
+
};
|
|
101
|
+
let cmpResult = await this.compareSrcAndDst();
|
|
102
|
+
if (cmpResult.needCopy.length > 0 || cmpResult.diff.length > 0) {
|
|
103
|
+
await ratchet_common_1.PromiseRatchet.runBoundedParallelSingleParam(cp, cmpResult.needCopy, this, this.config.maxNumThreads);
|
|
104
|
+
await ratchet_common_1.PromiseRatchet.runBoundedParallelSingleParam(cp, cmpResult.diff, this, this.config.maxNumThreads);
|
|
105
|
+
ratchet_common_1.Logger.info('Verifying...');
|
|
106
|
+
cmpResult = await this.compareSrcAndDst();
|
|
107
|
+
ratchet_common_1.Logger.debug('Compare result %j', cmpResult);
|
|
108
|
+
}
|
|
109
|
+
return cmpResult.needCopy.length === 0 && cmpResult.diff.length === 0;
|
|
110
|
+
}
|
|
111
|
+
async compareSrcAndDst() {
|
|
112
|
+
const getSrc = this.listObjects(this.config.srcBucket, this.config.srcPrefix, this.config.srcS3);
|
|
113
|
+
const getDst = this.listObjects(this.config.dstBucket, this.config.dstPrefix, this.config.dstS3);
|
|
114
|
+
const srcObjs = await getSrc;
|
|
115
|
+
const dstObjs = await getDst;
|
|
116
|
+
const rval = {
|
|
117
|
+
needCopy: [],
|
|
118
|
+
existed: [],
|
|
119
|
+
diff: [],
|
|
120
|
+
};
|
|
121
|
+
await ratchet_common_1.PromiseRatchet.runBoundedParallelSingleParam((key) => {
|
|
122
|
+
const sObj = srcObjs[key];
|
|
123
|
+
const dstKey = key.replace(this.config.srcPrefix, this.config.dstPrefix);
|
|
124
|
+
const dObj = dstObjs.hasOwnProperty(dstKey) ? dstObjs[dstKey] : undefined;
|
|
125
|
+
if (!dObj) {
|
|
126
|
+
rval.needCopy.push(sObj);
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
if (sObj.Size !== dObj.Size) {
|
|
130
|
+
rval.diff.push(sObj);
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
if (sObj.LastModified.getTime() <= dObj.LastModified.getTime()) {
|
|
134
|
+
rval.existed.push(sObj);
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
rval.diff.push(sObj);
|
|
138
|
+
}, Object.keys(srcObjs), this, this.config.maxNumThreads);
|
|
139
|
+
return rval;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
exports.S3LocationSyncRatchet = S3LocationSyncRatchet;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.S3Ratchet = void 0;
|
|
4
|
+
const ratchet_common_1 = require("@bitblit/ratchet-common");
|
|
5
|
+
class S3Ratchet {
|
|
6
|
+
static checkS3UrlForValidity(value) {
|
|
7
|
+
let rval = false;
|
|
8
|
+
if (value) {
|
|
9
|
+
rval = value.startsWith('s3://') && value.trim().length > 5;
|
|
10
|
+
}
|
|
11
|
+
return rval;
|
|
12
|
+
}
|
|
13
|
+
static extractBucketFromURL(value) {
|
|
14
|
+
ratchet_common_1.RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(value), 'invalid s3 url');
|
|
15
|
+
const idx1 = value.indexOf('/', 5);
|
|
16
|
+
const rval = idx1 > 0 ? value.substring(5, idx1) : value.substring(5);
|
|
17
|
+
return rval;
|
|
18
|
+
}
|
|
19
|
+
static extractKeyFromURL(value) {
|
|
20
|
+
ratchet_common_1.RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(value), 'invalid s3 url');
|
|
21
|
+
const idx1 = value.indexOf('/', 5);
|
|
22
|
+
const rval = idx1 > 0 ? value.substring(idx1 + 1) : null;
|
|
23
|
+
return rval;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
exports.S3Ratchet = S3Ratchet;
|