@bitblit/ratchet-aws-node-only 5.0.110 → 5.0.112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +17 -17
- package/lib/s3/s3-synced-file/s3-synced-file-config-init-mode.d.ts +0 -4
- package/lib/s3/s3-synced-file/s3-synced-file-config-init-mode.js +0 -6
- package/lib/s3/s3-synced-file/s3-synced-file-config-init-mode.js.map +0 -1
- package/lib/s3/s3-synced-file/s3-synced-file-config.d.ts +0 -15
- package/lib/s3/s3-synced-file/s3-synced-file-config.js +0 -2
- package/lib/s3/s3-synced-file/s3-synced-file-config.js.map +0 -1
- package/lib/s3/s3-synced-file/s3-synced-file-optimization.d.ts +0 -3
- package/lib/s3/s3-synced-file/s3-synced-file-optimization.js +0 -5
- package/lib/s3/s3-synced-file/s3-synced-file-optimization.js.map +0 -1
- package/lib/s3/s3-synced-file/s3-synced-file-remote-backup-mode.d.ts +0 -4
- package/lib/s3/s3-synced-file/s3-synced-file-remote-backup-mode.js +0 -6
- package/lib/s3/s3-synced-file/s3-synced-file-remote-backup-mode.js.map +0 -1
- package/lib/s3/s3-synced-file/s3-synced-file.d.ts +0 -34
- package/lib/s3/s3-synced-file/s3-synced-file.js +0 -218
- package/lib/s3/s3-synced-file/s3-synced-file.js.map +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bitblit/ratchet-aws-node-only",
|
|
3
|
-
"version": "5.0.
|
|
3
|
+
"version": "5.0.112",
|
|
4
4
|
"description": "Common tools for use with AWS (Node only)",
|
|
5
5
|
"note-on-side-effects": "Technically the entries in 'bin' below might be side effects, but they are called explicitly",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -56,16 +56,16 @@
|
|
|
56
56
|
},
|
|
57
57
|
"license": "Apache-2.0",
|
|
58
58
|
"dependencies": {
|
|
59
|
-
"@bitblit/ratchet-aws": "5.0.
|
|
60
|
-
"@bitblit/ratchet-common": "5.0.
|
|
59
|
+
"@bitblit/ratchet-aws": "5.0.112",
|
|
60
|
+
"@bitblit/ratchet-common": "5.0.112"
|
|
61
61
|
},
|
|
62
62
|
"optionalDependencies": {
|
|
63
|
-
"@aws-sdk/client-athena": "3.
|
|
64
|
-
"@aws-sdk/client-sts": "3.
|
|
65
|
-
"@aws-sdk/types": "3.
|
|
66
|
-
"@smithy/abort-controller": "3.1.
|
|
67
|
-
"@smithy/smithy-client": "3.
|
|
68
|
-
"@smithy/util-waiter": "3.1.
|
|
63
|
+
"@aws-sdk/client-athena": "3.666.0",
|
|
64
|
+
"@aws-sdk/client-sts": "3.666.0",
|
|
65
|
+
"@aws-sdk/types": "3.664.0",
|
|
66
|
+
"@smithy/abort-controller": "3.1.5",
|
|
67
|
+
"@smithy/smithy-client": "3.4.0",
|
|
68
|
+
"@smithy/util-waiter": "3.1.6",
|
|
69
69
|
"mailparser": "3.7.1",
|
|
70
70
|
"mime-types": "2.1.35",
|
|
71
71
|
"tmp": "0.2.3",
|
|
@@ -73,14 +73,14 @@
|
|
|
73
73
|
"walk": "2.3.15"
|
|
74
74
|
},
|
|
75
75
|
"peerDependencies": {
|
|
76
|
-
"@aws-sdk/client-athena": "^3.
|
|
77
|
-
"@aws-sdk/client-sts": "^3.
|
|
78
|
-
"@aws-sdk/types": "^3.
|
|
79
|
-
"@bitblit/ratchet-aws": "5.0.
|
|
80
|
-
"@bitblit/ratchet-common": "5.0.
|
|
81
|
-
"@smithy/abort-controller": "^3.1.
|
|
82
|
-
"@smithy/smithy-client": "^3.
|
|
83
|
-
"@smithy/util-waiter": "^3.1.
|
|
76
|
+
"@aws-sdk/client-athena": "^3.666.0",
|
|
77
|
+
"@aws-sdk/client-sts": "^3.666.0",
|
|
78
|
+
"@aws-sdk/types": "^3.664.0",
|
|
79
|
+
"@bitblit/ratchet-aws": "5.0.112",
|
|
80
|
+
"@bitblit/ratchet-common": "5.0.112",
|
|
81
|
+
"@smithy/abort-controller": "^3.1.5",
|
|
82
|
+
"@smithy/smithy-client": "^3.4.0",
|
|
83
|
+
"@smithy/util-waiter": "^3.1.6",
|
|
84
84
|
"mailparser": "^3.7.1",
|
|
85
85
|
"mime-types": "^2.1.35",
|
|
86
86
|
"tmp": "^0.2.3",
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
export var S3SyncedFileConfigInitMode;
|
|
2
|
-
(function (S3SyncedFileConfigInitMode) {
|
|
3
|
-
S3SyncedFileConfigInitMode["Lazy"] = "Lazy";
|
|
4
|
-
S3SyncedFileConfigInitMode["OnStartup"] = "OnStartup";
|
|
5
|
-
})(S3SyncedFileConfigInitMode || (S3SyncedFileConfigInitMode = {}));
|
|
6
|
-
//# sourceMappingURL=s3-synced-file-config-init-mode.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"s3-synced-file-config-init-mode.js","sourceRoot":"","sources":["../../../src/s3/s3-synced-file/s3-synced-file-config-init-mode.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,0BAGX;AAHD,WAAY,0BAA0B;IACpC,2CAAa,CAAA;IACb,qDAAuB,CAAA;AACzB,CAAC,EAHW,0BAA0B,KAA1B,0BAA0B,QAGrC"}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import { S3CacheRatchetLike } from '@bitblit/ratchet-aws/s3/s3-cache-ratchet-like';
|
|
2
|
-
import { S3SyncedFileConfigInitMode } from './s3-synced-file-config-init-mode.js';
|
|
3
|
-
import { S3SyncedFileRemoteBackupMode } from './s3-synced-file-remote-backup-mode.js';
|
|
4
|
-
import { S3SyncedFileOptimization } from './s3-synced-file-optimization.js';
|
|
5
|
-
export interface S3SyncedFileConfig {
|
|
6
|
-
s3CacheRatchetLike: S3CacheRatchetLike;
|
|
7
|
-
s3Path: string;
|
|
8
|
-
forceLocalFileFullPath?: string;
|
|
9
|
-
initMode: S3SyncedFileConfigInitMode;
|
|
10
|
-
backupMode?: S3SyncedFileRemoteBackupMode;
|
|
11
|
-
fetchOptimizations?: S3SyncedFileOptimization[];
|
|
12
|
-
pushOptimizations?: S3SyncedFileOptimization[];
|
|
13
|
-
remoteStatusTtlMs?: number;
|
|
14
|
-
leaveTempFileOnDisk?: boolean;
|
|
15
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"s3-synced-file-config.js","sourceRoot":"","sources":["../../../src/s3/s3-synced-file/s3-synced-file-config.ts"],"names":[],"mappings":""}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
export var S3SyncedFileOptimization;
|
|
2
|
-
(function (S3SyncedFileOptimization) {
|
|
3
|
-
S3SyncedFileOptimization["TreatSameSizeAsNoChange"] = "TreatSameSizeAsNoChange";
|
|
4
|
-
})(S3SyncedFileOptimization || (S3SyncedFileOptimization = {}));
|
|
5
|
-
//# sourceMappingURL=s3-synced-file-optimization.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"s3-synced-file-optimization.js","sourceRoot":"","sources":["../../../src/s3/s3-synced-file/s3-synced-file-optimization.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,wBAEX;AAFD,WAAY,wBAAwB;IAClC,+EAAmD,CAAA;AACrD,CAAC,EAFW,wBAAwB,KAAxB,wBAAwB,QAEnC"}
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
export var S3SyncedFileRemoteBackupMode;
|
|
2
|
-
(function (S3SyncedFileRemoteBackupMode) {
|
|
3
|
-
S3SyncedFileRemoteBackupMode["EveryUpload"] = "EveryUpload";
|
|
4
|
-
S3SyncedFileRemoteBackupMode["Disabled"] = "Disabled";
|
|
5
|
-
})(S3SyncedFileRemoteBackupMode || (S3SyncedFileRemoteBackupMode = {}));
|
|
6
|
-
//# sourceMappingURL=s3-synced-file-remote-backup-mode.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"s3-synced-file-remote-backup-mode.js","sourceRoot":"","sources":["../../../src/s3/s3-synced-file/s3-synced-file-remote-backup-mode.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,4BAGX;AAHD,WAAY,4BAA4B;IACtC,2DAA2B,CAAA;IAC3B,qDAAqB,CAAA;AACvB,CAAC,EAHW,4BAA4B,KAA5B,4BAA4B,QAGvC"}
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
import fs from 'fs';
|
|
2
|
-
import { HeadObjectCommandOutput } from '@aws-sdk/client-s3';
|
|
3
|
-
import { S3SyncedFileConfig } from './s3-synced-file-config.js';
|
|
4
|
-
import { RemoteFileSyncLike } from '@bitblit/ratchet-common/network/remote-file-sync/remote-file-sync-like';
|
|
5
|
-
import { RemoteStatusData } from '@bitblit/ratchet-common/network/remote-file-sync/remote-status-data';
|
|
6
|
-
import { FileTransferResult } from '@bitblit/ratchet-common/network/remote-file-sync/file-transfer-result';
|
|
7
|
-
import { BackupResult } from '@bitblit/ratchet-common/network/remote-file-sync/backup-result';
|
|
8
|
-
export declare class S3SyncedFile implements RemoteFileSyncLike {
|
|
9
|
-
private config;
|
|
10
|
-
private readonly _localFileName;
|
|
11
|
-
private _remoteStatus;
|
|
12
|
-
private _lastSyncEpochMS;
|
|
13
|
-
private _loadingRemoteSignal;
|
|
14
|
-
constructor(config: S3SyncedFileConfig);
|
|
15
|
-
get remoteStatusData(): Promise<RemoteStatusData>;
|
|
16
|
-
private initialize;
|
|
17
|
-
localAndRemoteAreSameSize(): Promise<boolean>;
|
|
18
|
-
directWriteValueToLocalFile(value: string | Uint8Array): void;
|
|
19
|
-
get lastSyncEpochMS(): number;
|
|
20
|
-
get localFileName(): string;
|
|
21
|
-
get localFileStats(): fs.Stats;
|
|
22
|
-
get localFileBytes(): number;
|
|
23
|
-
get localFileUpdatedEpochMS(): number;
|
|
24
|
-
fetchRemoteMeta(): Promise<HeadObjectCommandOutput>;
|
|
25
|
-
get wouldFetch(): Promise<boolean>;
|
|
26
|
-
get wouldPush(): Promise<boolean>;
|
|
27
|
-
sendLocalToRemote(): Promise<FileTransferResult>;
|
|
28
|
-
backupRemote(): Promise<BackupResult>;
|
|
29
|
-
fetchRemoteToLocal(): Promise<FileTransferResult>;
|
|
30
|
-
fetchRemoteToLocalIfNewerThan(epochMS: number): Promise<FileTransferResult>;
|
|
31
|
-
private hasFetchOptimization;
|
|
32
|
-
private hasPushOptimization;
|
|
33
|
-
private innerFetchRemoteToLocalIfNewerThan;
|
|
34
|
-
}
|
|
@@ -1,218 +0,0 @@
|
|
|
1
|
-
import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
|
|
2
|
-
import { Logger } from '@bitblit/ratchet-common/logger/logger';
|
|
3
|
-
import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
|
|
4
|
-
import { StopWatch } from '@bitblit/ratchet-common/lang/stop-watch';
|
|
5
|
-
import tmp from 'tmp';
|
|
6
|
-
import fs from 'fs';
|
|
7
|
-
import { S3SyncedFileConfigInitMode } from './s3-synced-file-config-init-mode.js';
|
|
8
|
-
import { DateTime } from 'luxon';
|
|
9
|
-
import { S3SyncedFileRemoteBackupMode } from './s3-synced-file-remote-backup-mode.js';
|
|
10
|
-
import { S3SyncedFileOptimization } from './s3-synced-file-optimization.js';
|
|
11
|
-
import { FileTransferResult } from '@bitblit/ratchet-common/network/remote-file-sync/file-transfer-result';
|
|
12
|
-
import { BackupResult } from '@bitblit/ratchet-common/network/remote-file-sync/backup-result';
|
|
13
|
-
export class S3SyncedFile {
|
|
14
|
-
config;
|
|
15
|
-
_localFileName;
|
|
16
|
-
_remoteStatus;
|
|
17
|
-
_lastSyncEpochMS;
|
|
18
|
-
_loadingRemoteSignal;
|
|
19
|
-
constructor(config) {
|
|
20
|
-
this.config = config;
|
|
21
|
-
RequireRatchet.notNullOrUndefined(config, 'config');
|
|
22
|
-
RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(config.s3Path, 's3Path');
|
|
23
|
-
RequireRatchet.notNullOrUndefined(config.s3CacheRatchetLike, 's3CacheRatchetLike');
|
|
24
|
-
RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(config.s3CacheRatchetLike.getDefaultBucket(), 's3CacheRatchetLike.getDefaultBucket()');
|
|
25
|
-
if (config.forceLocalFileFullPath) {
|
|
26
|
-
this._localFileName = config.forceLocalFileFullPath;
|
|
27
|
-
}
|
|
28
|
-
else {
|
|
29
|
-
const extension = config.s3Path.includes('.') ? config.s3Path.substring(config.s3Path.lastIndexOf('.') + 1) : undefined;
|
|
30
|
-
this._localFileName = config.forceLocalFileFullPath ?? tmp.fileSync({ postfix: extension, keep: config.leaveTempFileOnDisk }).name;
|
|
31
|
-
}
|
|
32
|
-
Logger.info('Using local file %s for remote path %s %s', this._localFileName, this.config.s3CacheRatchetLike.getDefaultBucket(), this.config.s3Path);
|
|
33
|
-
this.initialize().then(() => {
|
|
34
|
-
Logger.info('Initialized');
|
|
35
|
-
});
|
|
36
|
-
}
|
|
37
|
-
get remoteStatusData() {
|
|
38
|
-
if (!this._remoteStatus ||
|
|
39
|
-
(this.config.remoteStatusTtlMs && Date.now() - this._remoteStatus.updatedEpochMs > this.config.remoteStatusTtlMs)) {
|
|
40
|
-
return (async () => {
|
|
41
|
-
const meta = await this.fetchRemoteMeta();
|
|
42
|
-
this._remoteStatus = {
|
|
43
|
-
updatedEpochMs: Date.now(),
|
|
44
|
-
remoteSizeInBytes: meta.ContentLength,
|
|
45
|
-
remoteLastUpdatedEpochMs: meta.LastModified.getTime(),
|
|
46
|
-
remoteHash: meta.ETag,
|
|
47
|
-
};
|
|
48
|
-
return this._remoteStatus;
|
|
49
|
-
})();
|
|
50
|
-
}
|
|
51
|
-
else {
|
|
52
|
-
return Promise.resolve(this._remoteStatus);
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
async initialize() {
|
|
56
|
-
Logger.info('Using local path %s to sync %s / %s', this._localFileName, this.config.s3CacheRatchetLike.getDefaultBucket(), this.config.s3Path);
|
|
57
|
-
if (this.config.initMode === S3SyncedFileConfigInitMode.OnStartup) {
|
|
58
|
-
Logger.info('Initial loading');
|
|
59
|
-
this.fetchRemoteToLocal().then(() => {
|
|
60
|
-
Logger.info('Finished initial load');
|
|
61
|
-
});
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
async localAndRemoteAreSameSize() {
|
|
65
|
-
let rval = false;
|
|
66
|
-
const localBytes = this.localFileBytes;
|
|
67
|
-
if (localBytes !== null) {
|
|
68
|
-
const remoteBytes = (await this.remoteStatusData).remoteSizeInBytes;
|
|
69
|
-
rval = localBytes === remoteBytes;
|
|
70
|
-
Logger.info('Local size is %s, remote is %s, same is %s', localBytes, remoteBytes, rval);
|
|
71
|
-
}
|
|
72
|
-
return rval;
|
|
73
|
-
}
|
|
74
|
-
directWriteValueToLocalFile(value) {
|
|
75
|
-
RequireRatchet.notNullOrUndefined(value, 'value');
|
|
76
|
-
fs.writeFileSync(this._localFileName, value);
|
|
77
|
-
}
|
|
78
|
-
get lastSyncEpochMS() {
|
|
79
|
-
return this._lastSyncEpochMS;
|
|
80
|
-
}
|
|
81
|
-
get localFileName() {
|
|
82
|
-
return this._localFileName;
|
|
83
|
-
}
|
|
84
|
-
get localFileStats() {
|
|
85
|
-
let rval = null;
|
|
86
|
-
if (fs.existsSync(this._localFileName)) {
|
|
87
|
-
const stat = fs.statSync(this._localFileName);
|
|
88
|
-
rval = stat;
|
|
89
|
-
}
|
|
90
|
-
return rval;
|
|
91
|
-
}
|
|
92
|
-
get localFileBytes() {
|
|
93
|
-
const st = this.localFileStats;
|
|
94
|
-
return st ? st.size : null;
|
|
95
|
-
}
|
|
96
|
-
get localFileUpdatedEpochMS() {
|
|
97
|
-
const st = this.localFileStats;
|
|
98
|
-
return st ? st.mtime.getTime() : null;
|
|
99
|
-
}
|
|
100
|
-
async fetchRemoteMeta() {
|
|
101
|
-
return this.config.s3CacheRatchetLike.fetchMetaForCacheFile(this.config.s3Path);
|
|
102
|
-
}
|
|
103
|
-
get wouldFetch() {
|
|
104
|
-
return (async () => {
|
|
105
|
-
const rval = !this.hasFetchOptimization(S3SyncedFileOptimization.TreatSameSizeAsNoChange) || !(await this.localAndRemoteAreSameSize());
|
|
106
|
-
return rval;
|
|
107
|
-
})();
|
|
108
|
-
}
|
|
109
|
-
get wouldPush() {
|
|
110
|
-
return (async () => {
|
|
111
|
-
const rval = !this.hasPushOptimization(S3SyncedFileOptimization.TreatSameSizeAsNoChange) || !(await this.localAndRemoteAreSameSize());
|
|
112
|
-
return rval;
|
|
113
|
-
})();
|
|
114
|
-
}
|
|
115
|
-
async sendLocalToRemote() {
|
|
116
|
-
const sw = new StopWatch();
|
|
117
|
-
Logger.info('Sending local file to remote');
|
|
118
|
-
let rval = null;
|
|
119
|
-
if (await this.wouldPush) {
|
|
120
|
-
try {
|
|
121
|
-
if (this.config.backupMode === S3SyncedFileRemoteBackupMode.EveryUpload) {
|
|
122
|
-
Logger.info('EveryUpload mode set - backing up');
|
|
123
|
-
const backupRes = await this.backupRemote();
|
|
124
|
-
Logger.info('Backup result : %s', backupRes);
|
|
125
|
-
}
|
|
126
|
-
const out = await this.config.s3CacheRatchetLike.writeStreamToCacheFile(this.config.s3Path, fs.createReadStream(this._localFileName));
|
|
127
|
-
Logger.silly('SendLocalToRemote: %j', out);
|
|
128
|
-
this._remoteStatus = null;
|
|
129
|
-
rval = FileTransferResult.Updated;
|
|
130
|
-
}
|
|
131
|
-
catch (err) {
|
|
132
|
-
Logger.error('Failed to transfer %s : %s', this._localFileName, err, err);
|
|
133
|
-
rval = FileTransferResult.Error;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
else {
|
|
137
|
-
Logger.info('TreatSameSizeAsNoChange set and files are same size - skipping');
|
|
138
|
-
rval = FileTransferResult.Skipped;
|
|
139
|
-
}
|
|
140
|
-
Logger.info('Sent %d bytes to remote in %s', this.localFileBytes, sw.dump());
|
|
141
|
-
return rval;
|
|
142
|
-
}
|
|
143
|
-
async backupRemote() {
|
|
144
|
-
let rval = null;
|
|
145
|
-
try {
|
|
146
|
-
const lastSlash = this.config.s3Path.lastIndexOf('/');
|
|
147
|
-
const datePart = '/backup/' + DateTime.now().toFormat('yyyy/MM/dd/HH/mm/ss') + '/';
|
|
148
|
-
const newPath = lastSlash > -1
|
|
149
|
-
? this.config.s3Path.substring(0, lastSlash) + datePart + this.config.s3Path.substring(lastSlash + 1)
|
|
150
|
-
: datePart + this.config.s3Path;
|
|
151
|
-
Logger.info('Backing up path %s to %s', this.config.s3Path, newPath);
|
|
152
|
-
await this.config.s3CacheRatchetLike.copyFile(this.config.s3Path, newPath);
|
|
153
|
-
rval = BackupResult.Success;
|
|
154
|
-
}
|
|
155
|
-
catch (err) {
|
|
156
|
-
Logger.error('Failed to backup %s : %s', this.config.s3Path, err, err);
|
|
157
|
-
rval = BackupResult.Error;
|
|
158
|
-
}
|
|
159
|
-
return rval;
|
|
160
|
-
}
|
|
161
|
-
async fetchRemoteToLocal() {
|
|
162
|
-
Logger.info('Called fetchRemoteToLocal');
|
|
163
|
-
return this.fetchRemoteToLocalIfNewerThan(0);
|
|
164
|
-
}
|
|
165
|
-
async fetchRemoteToLocalIfNewerThan(epochMS) {
|
|
166
|
-
if (!this._loadingRemoteSignal) {
|
|
167
|
-
this._loadingRemoteSignal = this.innerFetchRemoteToLocalIfNewerThan(epochMS);
|
|
168
|
-
}
|
|
169
|
-
return this._loadingRemoteSignal;
|
|
170
|
-
}
|
|
171
|
-
hasFetchOptimization(opt) {
|
|
172
|
-
const rval = !!opt && (this?.config?.fetchOptimizations || []).includes(opt);
|
|
173
|
-
Logger.info('hasFetchOptimization %s returning %s', opt, rval);
|
|
174
|
-
return rval;
|
|
175
|
-
}
|
|
176
|
-
hasPushOptimization(opt) {
|
|
177
|
-
const rval = !!opt && (this?.config?.pushOptimizations || []).includes(opt);
|
|
178
|
-
Logger.info('hasPushOptimization %s returning %s', opt, rval);
|
|
179
|
-
return rval;
|
|
180
|
-
}
|
|
181
|
-
async innerFetchRemoteToLocalIfNewerThan(epochMS) {
|
|
182
|
-
try {
|
|
183
|
-
const sw = new StopWatch();
|
|
184
|
-
if (await this.wouldFetch) {
|
|
185
|
-
const req = {
|
|
186
|
-
Bucket: this.config.s3CacheRatchetLike.getDefaultBucket(),
|
|
187
|
-
Key: this.config.s3Path,
|
|
188
|
-
IfModifiedSince: new Date(epochMS),
|
|
189
|
-
};
|
|
190
|
-
const output = await this.config.s3CacheRatchetLike.fetchCacheFilePassThru(req);
|
|
191
|
-
const fileStream = fs.createWriteStream(this._localFileName);
|
|
192
|
-
const readStream = output.Body;
|
|
193
|
-
readStream.pipe(fileStream);
|
|
194
|
-
Logger.info('Waiting for pipe completion');
|
|
195
|
-
await PromiseRatchet.resolveOnEvent(fileStream, ['close', 'finish'], ['error']);
|
|
196
|
-
Logger.info('Pipe completed');
|
|
197
|
-
this._lastSyncEpochMS = Date.now();
|
|
198
|
-
this._remoteStatus = {
|
|
199
|
-
updatedEpochMs: Date.now(),
|
|
200
|
-
remoteSizeInBytes: output.ContentLength,
|
|
201
|
-
remoteLastUpdatedEpochMs: output.LastModified.getTime(),
|
|
202
|
-
remoteHash: output.ETag,
|
|
203
|
-
};
|
|
204
|
-
Logger.info('Fetched remote to local, %d bytes in %s', output.ContentLength, sw.dump());
|
|
205
|
-
}
|
|
206
|
-
else {
|
|
207
|
-
Logger.info('TreatSameSizeAsNoChange not enabled OR files are same size - skipping');
|
|
208
|
-
this._lastSyncEpochMS = Date.now();
|
|
209
|
-
}
|
|
210
|
-
return FileTransferResult.Updated;
|
|
211
|
-
}
|
|
212
|
-
catch (err) {
|
|
213
|
-
Logger.error('Failed to fetch %s / %s : %s', this.config.s3CacheRatchetLike.getDefaultBucket(), this.config.s3Path, err, err);
|
|
214
|
-
return FileTransferResult.Error;
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
//# sourceMappingURL=s3-synced-file.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"s3-synced-file.js","sourceRoot":"","sources":["../../../src/s3/s3-synced-file/s3-synced-file.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAC9E,OAAO,EAAE,MAAM,EAAE,MAAM,uCAAuC,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAmB,MAAM,IAAI,CAAC;AASrC,OAAO,EAAE,0BAA0B,EAAE,MAAM,sCAAsC,CAAC;AAClF,OAAO,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AACjC,OAAO,EAAE,4BAA4B,EAAE,MAAM,wCAAwC,CAAC;AACtF,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAG5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,uEAAuE,CAAC;AAC3G,OAAO,EAAE,YAAY,EAAE,MAAM,gEAAgE,CAAC;AAG9F,MAAM,OAAO,YAAY;IAOH;IANH,cAAc,CAAS;IAEhC,aAAa,CAAmB;IAChC,gBAAgB,CAAS;IACzB,oBAAoB,CAA8B;IAE1D,YAAoB,MAA0B;QAA1B,WAAM,GAAN,MAAM,CAAoB;QAC5C,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,sCAAsC,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC/E,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;QACnF,cAAc,CAAC,sCAAsC,CACnD,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,EAC5C,uCAAuC,CACxC,CAAC;QAEF,IAAI,MAAM,CAAC,sBAAsB,EAAE,CAAC;YAClC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,sBAAsB,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,MAAM,SAAS,GAAW,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAChI,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,sBAAsB,IAAI,GAAG,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC,IAAI,CAAC;QACrI,CAAC;QACD,MAAM,CAAC,IAAI,CACT,2CAA2C,EAC3C,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,EACjD,IAAI,CAAC,MAAM,CAAC,MAAM,CACnB,CAAC;QAEF,IAAI,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;YAC1B,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAW,gBAAgB;QACzB,IACE,CAAC,IAAI,CAAC,aAAa;YACnB,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,EACjH,CAAC;YACD,OAAO,CAAC,KAAK,IAAI,EAAE;gBACjB,MAAM,IAAI,GAA4B,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;gBACnE,IAAI,CAAC,aAAa,GAAG;oBACnB,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE;oBAC1B,iBAAiB,EAAE,IAAI,CAAC,aAAa;oBACrC,wBAAwB,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;oBACrD,UAAU,EAAE,IAAI,CAAC,IAAI;iBACtB,CAAC;gBACF,OAAO,IAAI,CAAC,aAAa,CAAC;YAC5B,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;aAAM,CAAC;YACN,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,MAAM,CAAC,IAAI,CACT,qCAAqC,EACrC,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,EACjD,IAAI,CAAC,MAAM,CAAC,MAAM,CACnB,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,0BAA0B,CAAC,SAAS,EAAE,CAAC;YAClE,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAC/B,IAAI,CAAC,kBAAkB,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;gBAClC,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,yBAAyB;QACpC,IAAI,IAAI,GAAY,KAAK,CAAC;QAC1B,MAAM,UAAU,GAAW,IAAI,CAAC,cAAc,CAAC;QAC/C,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,WAAW,GAAW,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,CAAC,iBAAiB,CAAC;YAC5E,IAAI,GAAG,UAAU,KAAK,WAAW,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,4CAA4C,EAAE,UAAU,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;QAC3F,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,2BAA2B,CAAC,KAA0B;QAC3D,cAAc,CAAC,kBAAkB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAClD,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,IAAW,eAAe;QACxB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IACD,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED,IAAW,cAAc;QACvB,IAAI,IAAI,GAAa,IAAI,CAAC;QAC1B,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACvC,MAAM,IAAI,GAAa,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACxD,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAW,cAAc;QACvB,MAAM,EAAE,GAAa,IAAI,CAAC,cAAc,CAAC;QACzC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7B,CAAC;IAED,IAAW,uBAAuB;QAChC,MAAM,EAAE,GAAa,IAAI,CAAC,cAAc,CAAC;QACzC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACxC,CAAC;IAEM,KAAK,CAAC,eAAe;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAClF,CAAC;IAGD,IAAW,UAAU;QACnB,OAAO,CAAC,KAAK,IAAI,EAAE;YACjB,MAAM,IAAI,GACR,CAAC,IAAI,CAAC,oBAAoB,CAAC,wBAAwB,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC,CAAC;YAC5H,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IACD,IAAW,SAAS;QAClB,OAAO,CAAC,KAAK,IAAI,EAAE;YACjB,MAAM,IAAI,GACR,CAAC,IAAI,CAAC,mBAAmB,CAAC,wBAAwB,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC,CAAC;YAC3H,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,iBAAiB;QAC5B,MAAM,EAAE,GAAc,IAAI,SAAS,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;QAC5C,IAAI,IAAI,GAAuB,IAAI,CAAC;QACpC,IAAI,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK,4BAA4B,CAAC,WAAW,EAAE,CAAC;oBACxE,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;oBACjD,MAAM,SAAS,GAAiB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;oBAC1D,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,SAAS,CAAC,CAAC;gBAC/C,CAAC;gBACD,MAAM,GAAG,GAAyC,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,sBAAsB,CAC3G,IAAI,CAAC,MAAM,CAAC,MAAM,EAClB,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CACzC,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAC;gBAE3C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAC1B,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC;YACpC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,IAAI,CAAC,cAAc,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;gBAC1E,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC;YAClC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;YAC9E,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC;QACpC,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7E,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,YAAY;QACvB,IAAI,IAAI,GAAiB,IAAI,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,SAAS,GAAW,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC9D,MAAM,QAAQ,GAAW,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC,GAAG,GAAG,CAAC;YAC3F,MAAM,OAAO,GACX,SAAS,GAAG,CAAC,CAAC;gBACZ,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC;gBACrG,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;YAEpC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACrE,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC3E,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC;QAC9B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACvE,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC;QAC5B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,kBAAkB;QAC7B,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC,6BAA6B,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IAEM,KAAK,CAAC,6BAA6B,CAAC,OAAe;QACxD,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,kCAAkC,CAAC,OAAO,CAAC,CAAC;QAC/E,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACnC,CAAC;IAEO,oBAAoB,CAAC,GAA6B;QACxD,MAAM,IAAI,GAAY,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACtF,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAC/D,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,mBAAmB,CAAC,GAA6B;QACvD,MAAM,IAAI,GAAY,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACrF,MAAM,CAAC,IAAI,CAAC,qCAAqC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,kCAAkC,CAAC,OAAe;QAC9D,IAAI,CAAC;YACH,MAAM,EAAE,GAAc,IAAI,SAAS,EAAE,CAAC;YAEtC,IAAI,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC1B,MAAM,GAAG,GAA0B;oBACjC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;oBACzD,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;oBACvB,eAAe,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC;iBACnC,CAAC;gBAEF,MAAM,MAAM,GAA2B,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;gBACxG,MAAM,UAAU,GAAgB,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC1E,MAAM,UAAU,GAAa,MAAM,CAAC,IAAgB,CAAC;gBACrD,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;gBAC3C,MAAM,cAAc,CAAC,cAAc,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;gBAChF,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBAC9B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAEnC,IAAI,CAAC,aAAa,GAAG;oBAEnB,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE;oBAC1B,iBAAiB,EAAE,MAAM,CAAC,aAAa;oBACvC,wBAAwB,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE;oBACvD,UAAU,EAAE,MAAM,CAAC,IAAI;iBACxB,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,yCAAyC,EAAE,MAAM,CAAC,aAAa,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1F,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAC;gBACrF,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACrC,CAAC;YACD,OAAO,kBAAkB,CAAC,OAAO,CAAC;QACpC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAC9H,OAAO,kBAAkB,CAAC,KAAK,CAAC;QAClC,CAAC;IACH,CAAC;CACF"}
|