@expo/build-tools 21.0.1 → 21.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builders/android.js +12 -1
- package/dist/builders/ios.js +13 -1
- package/dist/common/easBuildInternal.js +3 -0
- package/dist/common/hookableBuildPhase.d.ts +18 -0
- package/dist/common/hookableBuildPhase.js +72 -0
- package/dist/common/jobHooks.d.ts +16 -0
- package/dist/common/jobHooks.js +95 -0
- package/dist/common/setup.d.ts +14 -2
- package/dist/common/setup.js +40 -21
- package/dist/context.js +1 -31
- package/dist/index.d.ts +4 -3
- package/dist/index.js +5 -5
- package/dist/{gcs/LoggerStream.d.ts → logging/RemoteLoggerStream.d.ts} +6 -10
- package/dist/{gcs/LoggerStream.js → logging/RemoteLoggerStream.js} +13 -27
- package/dist/steps/functions/maestroResultParser.d.ts +1 -0
- package/dist/steps/functions/maestroResultParser.js +12 -0
- package/dist/steps/functions/maestroScreenshots.d.ts +1 -0
- package/dist/steps/functions/maestroScreenshots.js +167 -34
- package/dist/steps/functions/maestroTests.js +4 -0
- package/dist/steps/functions/startServeSimRemoteSession.js +2 -3
- package/dist/steps/functions/uploadDeviceRunSessionScreenRecordings.js +20 -1
- package/dist/steps/utils/IosSimulatorRecordingUtils.js +6 -2
- package/dist/steps/utils/remoteDeviceRunSession.d.ts +0 -1
- package/dist/steps/utils/remoteDeviceRunSession.js +7 -8
- package/dist/{gcs → storage}/retry.d.ts +5 -4
- package/dist/{gcs → storage}/retry.js +13 -9
- package/dist/storage/uploadWithSignedUrl.d.ts +12 -0
- package/dist/storage/uploadWithSignedUrl.js +68 -0
- package/package.json +7 -8
- package/resources/record-sim/Sources/RecordSim/RecordingModels.swift +7 -0
- package/resources/record-sim/Sources/RecordSim/RecordingOutputWriter.swift +5 -1
- package/resources/record-sim/Sources/RecordSim/SimulatorRecorder.swift +13 -1
- package/resources/record-sim/Sources/record-sim/main.swift +20 -4
- package/dist/gcs/__unit__/gcs.test.d.ts +0 -1
- package/dist/gcs/__unit__/gcs.test.js +0 -335
- package/dist/gcs/client.d.ts +0 -58
- package/dist/gcs/client.js +0 -173
|
@@ -7,6 +7,7 @@ import UniformTypeIdentifiers
|
|
|
7
7
|
public final class SimulatorRecorder {
|
|
8
8
|
public var onSegment: ((SegmentOutput) -> Void)?
|
|
9
9
|
public var onSimulatorStopped: ((String) -> Void)?
|
|
10
|
+
public var onFinalizationStage: ((RecordingFinalizationStage) -> Void)?
|
|
10
11
|
|
|
11
12
|
private static let callbackStalenessTimeout: TimeInterval = 5
|
|
12
13
|
private static let firstFrameRewireInterval: TimeInterval = 1
|
|
@@ -26,6 +27,8 @@ public final class SimulatorRecorder {
|
|
|
26
27
|
private var monotonicClock = MonotonicClock()
|
|
27
28
|
private var firstAcceptedCaptureTime: CMTime?
|
|
28
29
|
private var firstAcceptedWallClock: Date?
|
|
30
|
+
private var recordingWidth: Int?
|
|
31
|
+
private var recordingHeight: Int?
|
|
29
32
|
private var lastPTS: CMTime?
|
|
30
33
|
private var lastSeed: UInt32?
|
|
31
34
|
private var lastFrameCallbackElapsed: TimeInterval?
|
|
@@ -113,6 +116,7 @@ public final class SimulatorRecorder {
|
|
|
113
116
|
displaySource?.stop()
|
|
114
117
|
displaySource = nil
|
|
115
118
|
}
|
|
119
|
+
onFinalizationStage?(.captureStopped)
|
|
116
120
|
|
|
117
121
|
return try writerQueue.sync {
|
|
118
122
|
if let firstError {
|
|
@@ -139,12 +143,18 @@ public final class SimulatorRecorder {
|
|
|
139
143
|
if let error = outputWriter.error {
|
|
140
144
|
throw error
|
|
141
145
|
}
|
|
146
|
+
onFinalizationStage?(.videoSaved)
|
|
142
147
|
guard let firstAcceptedWallClock else {
|
|
143
148
|
throw RecorderError.make(25, "Missing first frame wall-clock timestamp")
|
|
144
149
|
}
|
|
150
|
+
guard let recordingWidth, let recordingHeight else {
|
|
151
|
+
throw RecorderError.make(27, "Missing recording dimensions")
|
|
152
|
+
}
|
|
145
153
|
return try outputWriter.writeManifest(
|
|
146
154
|
configuration: configuration,
|
|
147
|
-
firstFrameWallClock: firstAcceptedWallClock
|
|
155
|
+
firstFrameWallClock: firstAcceptedWallClock,
|
|
156
|
+
width: recordingWidth,
|
|
157
|
+
height: recordingHeight
|
|
148
158
|
)
|
|
149
159
|
}
|
|
150
160
|
}
|
|
@@ -590,6 +600,8 @@ public final class SimulatorRecorder {
|
|
|
590
600
|
self.writer = writer
|
|
591
601
|
self.input = input
|
|
592
602
|
self.adaptor = adaptor
|
|
603
|
+
recordingWidth = width
|
|
604
|
+
recordingHeight = height
|
|
593
605
|
}
|
|
594
606
|
|
|
595
607
|
private func assetWriterSettings(width: Int, height: Int) -> [String: Any] {
|
|
@@ -10,15 +10,15 @@ struct CLIOptions {
|
|
|
10
10
|
var codec: RecorderCodec = .h264
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
private var
|
|
13
|
+
private var requestedStopSignal: CInt = 0
|
|
14
14
|
private var simulatorStopped: CInt = 0
|
|
15
15
|
|
|
16
|
-
private func handleStopSignal(_: CInt) {
|
|
17
|
-
|
|
16
|
+
private func handleStopSignal(_ signal: CInt) {
|
|
17
|
+
requestedStopSignal = signal
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
private func isStopRequested() -> Bool {
|
|
21
|
-
|
|
21
|
+
requestedStopSignal != 0 || simulatorStopped != 0
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
func usage() -> String {
|
|
@@ -112,6 +112,14 @@ do {
|
|
|
112
112
|
simulatorStopped = 1
|
|
113
113
|
fputs("[record-sim] simulator stopped state=\(state); finalizing recording\n", stderr)
|
|
114
114
|
}
|
|
115
|
+
recorder.onFinalizationStage = { stage in
|
|
116
|
+
switch stage {
|
|
117
|
+
case .captureStopped:
|
|
118
|
+
fputs("[record-sim] screen capture stopped; saving final video\n", stderr)
|
|
119
|
+
case .videoSaved:
|
|
120
|
+
fputs("[record-sim] final video saved; writing recording details\n", stderr)
|
|
121
|
+
}
|
|
122
|
+
}
|
|
115
123
|
recorder.onSegment = { segment in
|
|
116
124
|
switch segment.kind {
|
|
117
125
|
case .initialization:
|
|
@@ -130,6 +138,14 @@ do {
|
|
|
130
138
|
Thread.sleep(forTimeInterval: 0.1)
|
|
131
139
|
}
|
|
132
140
|
|
|
141
|
+
if requestedStopSignal != 0 {
|
|
142
|
+
let signalName = switch requestedStopSignal {
|
|
143
|
+
case SIGINT: "SIGINT"
|
|
144
|
+
case SIGTERM: "SIGTERM"
|
|
145
|
+
default: "signal \(requestedStopSignal)"
|
|
146
|
+
}
|
|
147
|
+
fputs("[record-sim] received \(signalName); finishing recording\n", stderr)
|
|
148
|
+
}
|
|
133
149
|
let manifest = try recorder.stop()
|
|
134
150
|
if let recording = manifest.recording {
|
|
135
151
|
fputs("[record-sim] done recording=\(recording)\n", stderr)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,335 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const crypto_1 = require("crypto");
|
|
7
|
-
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
8
|
-
const node_fetch_1 = __importDefault(require("node-fetch"));
|
|
9
|
-
const path_1 = __importDefault(require("path"));
|
|
10
|
-
const stream_1 = require("stream");
|
|
11
|
-
const client_1 = __importDefault(require("../client"));
|
|
12
|
-
jest.mock('node-fetch');
|
|
13
|
-
class ErrorWithCode extends Error {
|
|
14
|
-
_code;
|
|
15
|
-
constructor(message, code) {
|
|
16
|
-
super(message);
|
|
17
|
-
if (code) {
|
|
18
|
-
this._code = code;
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
get code() {
|
|
22
|
-
return this._code;
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
class DNSError extends ErrorWithCode {
|
|
26
|
-
_code = 'ENOTFOUND';
|
|
27
|
-
}
|
|
28
|
-
const TEST_BUCKET = 'turtle-v2-test';
|
|
29
|
-
let googleApplicationCredentials;
|
|
30
|
-
beforeAll(() => {
|
|
31
|
-
googleApplicationCredentials = process.env.GOOGLE_APPLICATION_CREDENTIALS;
|
|
32
|
-
process.env.GOOGLE_APPLICATION_CREDENTIALS = path_1.default.join(__dirname, 'mock-credentials.json');
|
|
33
|
-
});
|
|
34
|
-
afterAll(() => {
|
|
35
|
-
process.env.GOOGLE_APPLICATION_CREDENTIALS = googleApplicationCredentials;
|
|
36
|
-
});
|
|
37
|
-
describe('GCS client', () => {
|
|
38
|
-
describe('uploadWithPresignedURL function', () => {
|
|
39
|
-
it('should throw an error if upload fails', async () => {
|
|
40
|
-
const fetchMock = jest.mocked(node_fetch_1.default);
|
|
41
|
-
const res = {
|
|
42
|
-
ok: false,
|
|
43
|
-
status: 500,
|
|
44
|
-
};
|
|
45
|
-
const localImagePath = path_1.default.join(__dirname, 'cat.jpg');
|
|
46
|
-
const key = 'cat.jpg';
|
|
47
|
-
const gcs = new client_1.default(TEST_BUCKET);
|
|
48
|
-
const signedUrl = await gcs.createSignedUploadUrl({
|
|
49
|
-
key,
|
|
50
|
-
expirationTime: 30000,
|
|
51
|
-
contentType: 'image/jpeg',
|
|
52
|
-
});
|
|
53
|
-
fetchMock.mockImplementation(async () => res);
|
|
54
|
-
await expect(client_1.default.uploadWithSignedUrl({
|
|
55
|
-
signedUrl,
|
|
56
|
-
srcGeneratorAsync: async () => fs_extra_1.default.createReadStream(localImagePath),
|
|
57
|
-
retryIntervalMs: 500,
|
|
58
|
-
})).rejects.toThrow();
|
|
59
|
-
});
|
|
60
|
-
it('should return stripped URL if successful', async () => {
|
|
61
|
-
const fetchMock = jest.mocked(node_fetch_1.default);
|
|
62
|
-
const res = {
|
|
63
|
-
ok: true,
|
|
64
|
-
status: 200,
|
|
65
|
-
};
|
|
66
|
-
const localImagePath = path_1.default.join(__dirname, 'cat.jpg');
|
|
67
|
-
const key = 'cat.jpg';
|
|
68
|
-
const gcs = new client_1.default(TEST_BUCKET);
|
|
69
|
-
const signedUrl = await gcs.createSignedUploadUrl({
|
|
70
|
-
key,
|
|
71
|
-
expirationTime: 30000,
|
|
72
|
-
contentType: 'image/jpeg',
|
|
73
|
-
});
|
|
74
|
-
fetchMock.mockImplementation(async () => res);
|
|
75
|
-
const result = await client_1.default.uploadWithSignedUrl({
|
|
76
|
-
signedUrl,
|
|
77
|
-
srcGeneratorAsync: async () => fs_extra_1.default.createReadStream(localImagePath),
|
|
78
|
-
retryIntervalMs: 500,
|
|
79
|
-
});
|
|
80
|
-
expect(result).toEqual('https://storage.googleapis.com/turtle-v2-test/cat.jpg');
|
|
81
|
-
});
|
|
82
|
-
it('should retry upload on DNS error up to 2 times', async () => {
|
|
83
|
-
const fetchMock = jest.mocked(node_fetch_1.default);
|
|
84
|
-
const res = {
|
|
85
|
-
ok: true,
|
|
86
|
-
status: 200,
|
|
87
|
-
};
|
|
88
|
-
const localImagePath = path_1.default.join(__dirname, 'cat.jpg');
|
|
89
|
-
const key = 'cat.jpg';
|
|
90
|
-
const gcs = new client_1.default(TEST_BUCKET);
|
|
91
|
-
const signedUrl = await gcs.createSignedUploadUrl({
|
|
92
|
-
key,
|
|
93
|
-
expirationTime: 30000,
|
|
94
|
-
contentType: 'image/jpeg',
|
|
95
|
-
});
|
|
96
|
-
fetchMock
|
|
97
|
-
.mockImplementationOnce(async () => {
|
|
98
|
-
throw new DNSError('failed once');
|
|
99
|
-
})
|
|
100
|
-
.mockImplementationOnce(async () => {
|
|
101
|
-
throw new DNSError('failed twice', 'EAI_AGAIN');
|
|
102
|
-
})
|
|
103
|
-
.mockImplementation(async () => res);
|
|
104
|
-
const result = await client_1.default.uploadWithSignedUrl({
|
|
105
|
-
signedUrl,
|
|
106
|
-
srcGeneratorAsync: async () => fs_extra_1.default.createReadStream(localImagePath),
|
|
107
|
-
retryIntervalMs: 500,
|
|
108
|
-
});
|
|
109
|
-
expect(result).toEqual('https://storage.googleapis.com/turtle-v2-test/cat.jpg');
|
|
110
|
-
expect(fetchMock).toHaveBeenCalledTimes(3);
|
|
111
|
-
});
|
|
112
|
-
it('should retry upload on retriable status codes error up to 2 times', async () => {
|
|
113
|
-
const fetchMock = jest.mocked(node_fetch_1.default);
|
|
114
|
-
const res = {
|
|
115
|
-
ok: true,
|
|
116
|
-
status: 200,
|
|
117
|
-
};
|
|
118
|
-
const localImagePath = path_1.default.join(__dirname, 'cat.jpg');
|
|
119
|
-
const key = 'cat.jpg';
|
|
120
|
-
const gcs = new client_1.default(TEST_BUCKET);
|
|
121
|
-
const signedUrl = await gcs.createSignedUploadUrl({
|
|
122
|
-
key,
|
|
123
|
-
expirationTime: 30000,
|
|
124
|
-
contentType: 'image/jpeg',
|
|
125
|
-
});
|
|
126
|
-
fetchMock
|
|
127
|
-
.mockImplementationOnce(async () => {
|
|
128
|
-
return {
|
|
129
|
-
ok: false,
|
|
130
|
-
status: 503,
|
|
131
|
-
};
|
|
132
|
-
})
|
|
133
|
-
.mockImplementationOnce(async () => {
|
|
134
|
-
return {
|
|
135
|
-
ok: false,
|
|
136
|
-
status: 408,
|
|
137
|
-
};
|
|
138
|
-
})
|
|
139
|
-
.mockImplementation(async () => res);
|
|
140
|
-
const result = await client_1.default.uploadWithSignedUrl({
|
|
141
|
-
signedUrl,
|
|
142
|
-
srcGeneratorAsync: async () => fs_extra_1.default.createReadStream(localImagePath),
|
|
143
|
-
retryIntervalMs: 500,
|
|
144
|
-
});
|
|
145
|
-
expect(result).toEqual('https://storage.googleapis.com/turtle-v2-test/cat.jpg');
|
|
146
|
-
expect(fetchMock).toHaveBeenCalledTimes(3);
|
|
147
|
-
});
|
|
148
|
-
it('should retry upload on retriable status codes and DNS errors error up to 2 times', async () => {
|
|
149
|
-
const fetchMock = jest.mocked(node_fetch_1.default);
|
|
150
|
-
const res = {
|
|
151
|
-
ok: true,
|
|
152
|
-
status: 200,
|
|
153
|
-
};
|
|
154
|
-
const localImagePath = path_1.default.join(__dirname, 'cat.jpg');
|
|
155
|
-
const key = 'cat.jpg';
|
|
156
|
-
const gcs = new client_1.default(TEST_BUCKET);
|
|
157
|
-
const signedUrl = await gcs.createSignedUploadUrl({
|
|
158
|
-
key,
|
|
159
|
-
expirationTime: 30000,
|
|
160
|
-
contentType: 'image/jpeg',
|
|
161
|
-
});
|
|
162
|
-
fetchMock
|
|
163
|
-
.mockImplementationOnce(async () => {
|
|
164
|
-
return {
|
|
165
|
-
ok: false,
|
|
166
|
-
status: 503,
|
|
167
|
-
};
|
|
168
|
-
})
|
|
169
|
-
.mockImplementationOnce(async () => {
|
|
170
|
-
throw new DNSError('failed once');
|
|
171
|
-
})
|
|
172
|
-
.mockImplementation(async () => res);
|
|
173
|
-
const result = await client_1.default.uploadWithSignedUrl({
|
|
174
|
-
signedUrl,
|
|
175
|
-
srcGeneratorAsync: async () => fs_extra_1.default.createReadStream(localImagePath),
|
|
176
|
-
retryIntervalMs: 500,
|
|
177
|
-
});
|
|
178
|
-
expect(result).toEqual('https://storage.googleapis.com/turtle-v2-test/cat.jpg');
|
|
179
|
-
expect(fetchMock).toHaveBeenCalledTimes(3);
|
|
180
|
-
});
|
|
181
|
-
it('should not retry upload on DNS error more than 2 times', async () => {
|
|
182
|
-
const fetchMock = jest.mocked(node_fetch_1.default);
|
|
183
|
-
const res = {
|
|
184
|
-
ok: true,
|
|
185
|
-
status: 200,
|
|
186
|
-
};
|
|
187
|
-
const localImagePath = path_1.default.join(__dirname, 'cat.jpg');
|
|
188
|
-
const key = 'cat.jpg';
|
|
189
|
-
const gcs = new client_1.default(TEST_BUCKET);
|
|
190
|
-
const signedUrl = await gcs.createSignedUploadUrl({
|
|
191
|
-
key,
|
|
192
|
-
expirationTime: 30000,
|
|
193
|
-
contentType: 'image/jpeg',
|
|
194
|
-
});
|
|
195
|
-
const lastDNSError = new DNSError('failed thrice');
|
|
196
|
-
fetchMock
|
|
197
|
-
.mockImplementationOnce(async () => {
|
|
198
|
-
throw new DNSError('failed once');
|
|
199
|
-
})
|
|
200
|
-
.mockImplementationOnce(async () => {
|
|
201
|
-
throw new DNSError('failed twice', 'EAI_AGAIN');
|
|
202
|
-
})
|
|
203
|
-
.mockImplementationOnce(async () => {
|
|
204
|
-
throw lastDNSError;
|
|
205
|
-
})
|
|
206
|
-
.mockImplementation(async () => res);
|
|
207
|
-
await expect(client_1.default.uploadWithSignedUrl({
|
|
208
|
-
signedUrl,
|
|
209
|
-
srcGeneratorAsync: async () => fs_extra_1.default.createReadStream(localImagePath),
|
|
210
|
-
retryIntervalMs: 500,
|
|
211
|
-
})).rejects.toThrow(lastDNSError);
|
|
212
|
-
expect(fetchMock).toHaveBeenCalledTimes(3);
|
|
213
|
-
});
|
|
214
|
-
it('should not retry upload on retriable status code more than 2 times', async () => {
|
|
215
|
-
const fetchMock = jest.mocked(node_fetch_1.default);
|
|
216
|
-
const res = {
|
|
217
|
-
ok: true,
|
|
218
|
-
status: 200,
|
|
219
|
-
};
|
|
220
|
-
const localImagePath = path_1.default.join(__dirname, 'cat.jpg');
|
|
221
|
-
const key = 'cat.jpg';
|
|
222
|
-
const gcs = new client_1.default(TEST_BUCKET);
|
|
223
|
-
const signedUrl = await gcs.createSignedUploadUrl({
|
|
224
|
-
key,
|
|
225
|
-
expirationTime: 30000,
|
|
226
|
-
contentType: 'image/jpeg',
|
|
227
|
-
});
|
|
228
|
-
fetchMock
|
|
229
|
-
.mockImplementationOnce(async () => {
|
|
230
|
-
return {
|
|
231
|
-
ok: false,
|
|
232
|
-
status: 503,
|
|
233
|
-
};
|
|
234
|
-
})
|
|
235
|
-
.mockImplementationOnce(async () => {
|
|
236
|
-
return {
|
|
237
|
-
ok: false,
|
|
238
|
-
status: 408,
|
|
239
|
-
};
|
|
240
|
-
})
|
|
241
|
-
.mockImplementationOnce(async () => {
|
|
242
|
-
return {
|
|
243
|
-
ok: false,
|
|
244
|
-
status: 504,
|
|
245
|
-
};
|
|
246
|
-
})
|
|
247
|
-
.mockImplementation(async () => res);
|
|
248
|
-
await expect(client_1.default.uploadWithSignedUrl({
|
|
249
|
-
signedUrl,
|
|
250
|
-
srcGeneratorAsync: async () => fs_extra_1.default.createReadStream(localImagePath),
|
|
251
|
-
retryIntervalMs: 500,
|
|
252
|
-
})).rejects.toThrow();
|
|
253
|
-
expect(fetchMock).toHaveBeenCalledTimes(3);
|
|
254
|
-
});
|
|
255
|
-
it('should not retry upload on other error codes', async () => {
|
|
256
|
-
const fetchMock = jest.mocked(node_fetch_1.default);
|
|
257
|
-
const res = {
|
|
258
|
-
ok: true,
|
|
259
|
-
status: 200,
|
|
260
|
-
};
|
|
261
|
-
const localImagePath = path_1.default.join(__dirname, 'cat.jpg');
|
|
262
|
-
const key = 'cat.jpg';
|
|
263
|
-
const gcs = new client_1.default(TEST_BUCKET);
|
|
264
|
-
const signedUrl = await gcs.createSignedUploadUrl({
|
|
265
|
-
key,
|
|
266
|
-
expirationTime: 30000,
|
|
267
|
-
contentType: 'image/jpeg',
|
|
268
|
-
});
|
|
269
|
-
const nonDNSError = new ErrorWithCode('failed once', 'A_DIFFERENT_CODE');
|
|
270
|
-
fetchMock
|
|
271
|
-
.mockImplementationOnce(async () => {
|
|
272
|
-
throw nonDNSError;
|
|
273
|
-
})
|
|
274
|
-
.mockImplementation(async () => res);
|
|
275
|
-
await expect(client_1.default.uploadWithSignedUrl({
|
|
276
|
-
signedUrl,
|
|
277
|
-
srcGeneratorAsync: async () => fs_extra_1.default.createReadStream(localImagePath),
|
|
278
|
-
retryIntervalMs: 500,
|
|
279
|
-
})).rejects.toThrow(nonDNSError);
|
|
280
|
-
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
281
|
-
});
|
|
282
|
-
it('retries upload with full stream', async () => {
|
|
283
|
-
const fetchMock = jest.mocked(node_fetch_1.default);
|
|
284
|
-
const receivedBodies = [];
|
|
285
|
-
const recordBody = async (request) => {
|
|
286
|
-
let body = Buffer.from([]);
|
|
287
|
-
for await (const chunk of request.body) {
|
|
288
|
-
body = Buffer.concat([body, chunk]);
|
|
289
|
-
}
|
|
290
|
-
receivedBodies.push(body);
|
|
291
|
-
};
|
|
292
|
-
fetchMock
|
|
293
|
-
.mockImplementationOnce(async (_path, request) => {
|
|
294
|
-
await recordBody(request);
|
|
295
|
-
return {
|
|
296
|
-
ok: false,
|
|
297
|
-
status: 503,
|
|
298
|
-
};
|
|
299
|
-
})
|
|
300
|
-
.mockImplementationOnce(async (_path, request) => {
|
|
301
|
-
await recordBody(request);
|
|
302
|
-
throw new DNSError('failed once');
|
|
303
|
-
})
|
|
304
|
-
.mockImplementation(async (_path, request) => {
|
|
305
|
-
await recordBody(request);
|
|
306
|
-
return {
|
|
307
|
-
ok: true,
|
|
308
|
-
status: 201,
|
|
309
|
-
};
|
|
310
|
-
});
|
|
311
|
-
const gcs = new client_1.default(TEST_BUCKET);
|
|
312
|
-
const signedUrl = await gcs.createSignedUploadUrl({
|
|
313
|
-
key: 'text.txt',
|
|
314
|
-
expirationTime: 30000,
|
|
315
|
-
contentType: 'text/plain',
|
|
316
|
-
});
|
|
317
|
-
const bufferToUpload = (0, crypto_1.randomBytes)(16);
|
|
318
|
-
const result = await client_1.default.uploadWithSignedUrl({
|
|
319
|
-
signedUrl,
|
|
320
|
-
srcGeneratorAsync: async () => stream_1.Readable.from(bufferToUpload),
|
|
321
|
-
retryIntervalMs: 500,
|
|
322
|
-
});
|
|
323
|
-
expect(result).toEqual('https://storage.googleapis.com/turtle-v2-test/text.txt');
|
|
324
|
-
expect(fetchMock).toHaveBeenCalledTimes(3);
|
|
325
|
-
expect(receivedBodies.length).toBe(3);
|
|
326
|
-
for (const body of receivedBodies) {
|
|
327
|
-
// Here we're testing that each body received in every retry is the same.
|
|
328
|
-
// If we passed the same body instance to each of the retries
|
|
329
|
-
// each retry would consume a chunk of the same stream,
|
|
330
|
-
// causing the uploaded file to be corrupted (missing first bytes).
|
|
331
|
-
expect(body).toStrictEqual(bufferToUpload);
|
|
332
|
-
}
|
|
333
|
-
});
|
|
334
|
-
});
|
|
335
|
-
});
|
package/dist/gcs/client.d.ts
DELETED
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
import { CreateWriteStreamOptions, File } from '@google-cloud/storage';
|
|
2
|
-
import { Readable } from 'stream';
|
|
3
|
-
import { RetryOptions } from './retry';
|
|
4
|
-
interface SignedUrlParams {
|
|
5
|
-
key: string;
|
|
6
|
-
expirationTime: number;
|
|
7
|
-
contentType?: string;
|
|
8
|
-
extensionHeaders?: {
|
|
9
|
-
[key: string]: number | string | string[];
|
|
10
|
-
};
|
|
11
|
-
}
|
|
12
|
-
interface UploadWithSignedUrlParams {
|
|
13
|
-
signedUrl: GCS.SignedUrl;
|
|
14
|
-
srcGeneratorAsync: () => Promise<Readable>;
|
|
15
|
-
retryIntervalMs?: RetryOptions['retryIntervalMs'];
|
|
16
|
-
retries?: RetryOptions['retries'];
|
|
17
|
-
}
|
|
18
|
-
declare class GCS {
|
|
19
|
-
private readonly bucket;
|
|
20
|
-
private readonly client;
|
|
21
|
-
constructor(bucket: string);
|
|
22
|
-
static uploadWithSignedUrl({ signedUrl, srcGeneratorAsync, retries, retryIntervalMs, }: UploadWithSignedUrlParams): Promise<string>;
|
|
23
|
-
formatHttpUrl(key: string): string;
|
|
24
|
-
uploadFile({ key, src, streamOptions, }: {
|
|
25
|
-
key: string;
|
|
26
|
-
src: Readable;
|
|
27
|
-
streamOptions?: CreateWriteStreamOptions;
|
|
28
|
-
}): Promise<{
|
|
29
|
-
Location: string;
|
|
30
|
-
}>;
|
|
31
|
-
deleteFile(key: string): Promise<void>;
|
|
32
|
-
createSignedUploadUrl({ key, expirationTime, contentType, extensionHeaders, }: SignedUrlParams): Promise<GCS.SignedUrl>;
|
|
33
|
-
createSignedDownloadUrl({ key, expirationTime, }: {
|
|
34
|
-
key: string;
|
|
35
|
-
expirationTime: number;
|
|
36
|
-
}): Promise<string>;
|
|
37
|
-
checkIfFileExists(key: string, fileHash?: string): Promise<boolean>;
|
|
38
|
-
listDirectory(prefix: string): Promise<string[]>;
|
|
39
|
-
moveFile(src: string, dest: string): Promise<void>;
|
|
40
|
-
deleteFiles(keys: string[]): Promise<void>;
|
|
41
|
-
downloadFile(key: string, destinationPath: string): Promise<void>;
|
|
42
|
-
getFile(key: string): File;
|
|
43
|
-
}
|
|
44
|
-
declare namespace GCS {
|
|
45
|
-
interface SignedUrl {
|
|
46
|
-
url: string;
|
|
47
|
-
headers: {
|
|
48
|
-
[key: string]: string;
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
interface Config {
|
|
52
|
-
accessKeyId: string;
|
|
53
|
-
secretAccessKey: string;
|
|
54
|
-
region: string;
|
|
55
|
-
bucket: string;
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
export default GCS;
|
package/dist/gcs/client.js
DELETED
|
@@ -1,173 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
-
};
|
|
38
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
const storage_1 = require("@google-cloud/storage");
|
|
40
|
-
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
41
|
-
const node_fetch_1 = __importStar(require("node-fetch"));
|
|
42
|
-
const node_stream_1 = __importDefault(require("node:stream"));
|
|
43
|
-
const node_util_1 = require("node:util");
|
|
44
|
-
const url_1 = require("url");
|
|
45
|
-
const retry_1 = require("./retry");
|
|
46
|
-
const pipeline = (0, node_util_1.promisify)(node_stream_1.default.pipeline);
|
|
47
|
-
class GCS {
|
|
48
|
-
bucket;
|
|
49
|
-
client = new storage_1.Storage();
|
|
50
|
-
constructor(bucket) {
|
|
51
|
-
this.bucket = bucket;
|
|
52
|
-
this.bucket = bucket;
|
|
53
|
-
}
|
|
54
|
-
static async uploadWithSignedUrl({ signedUrl, srcGeneratorAsync, retries = 2, retryIntervalMs = 30_000, }) {
|
|
55
|
-
let resp;
|
|
56
|
-
try {
|
|
57
|
-
resp = await (0, retry_1.retryOnGCSUploadFailure)(async () => {
|
|
58
|
-
const src = await srcGeneratorAsync();
|
|
59
|
-
return await (0, node_fetch_1.default)(signedUrl.url, {
|
|
60
|
-
method: 'PUT',
|
|
61
|
-
headers: signedUrl.headers,
|
|
62
|
-
body: src,
|
|
63
|
-
});
|
|
64
|
-
}, {
|
|
65
|
-
retries,
|
|
66
|
-
retryIntervalMs,
|
|
67
|
-
});
|
|
68
|
-
}
|
|
69
|
-
catch (err) {
|
|
70
|
-
if (err instanceof node_fetch_1.FetchError) {
|
|
71
|
-
throw new Error(`Failed to upload the file, reason: ${err.code}`);
|
|
72
|
-
}
|
|
73
|
-
else {
|
|
74
|
-
throw err;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
if (!resp.ok) {
|
|
78
|
-
let body;
|
|
79
|
-
try {
|
|
80
|
-
body = await resp.text();
|
|
81
|
-
}
|
|
82
|
-
catch { }
|
|
83
|
-
throw new Error(`Failed to upload file: status: ${resp.status} status text: ${resp.statusText}, body: ${body}`);
|
|
84
|
-
}
|
|
85
|
-
const url = new url_1.URL(signedUrl.url);
|
|
86
|
-
return `${url.protocol}//${url.host}${url.pathname}`; // strip query string
|
|
87
|
-
}
|
|
88
|
-
formatHttpUrl(key) {
|
|
89
|
-
return this.client.bucket(this.bucket).file(key).publicUrl();
|
|
90
|
-
}
|
|
91
|
-
async uploadFile({ key, src, streamOptions, }) {
|
|
92
|
-
const file = this.client.bucket(this.bucket).file(key);
|
|
93
|
-
await new Promise((res, rej) => {
|
|
94
|
-
src.pipe(file
|
|
95
|
-
.createWriteStream(streamOptions)
|
|
96
|
-
.on('error', err => {
|
|
97
|
-
rej(err);
|
|
98
|
-
})
|
|
99
|
-
.on('finish', () => {
|
|
100
|
-
res();
|
|
101
|
-
}));
|
|
102
|
-
});
|
|
103
|
-
return { Location: file.publicUrl() };
|
|
104
|
-
}
|
|
105
|
-
async deleteFile(key) {
|
|
106
|
-
try {
|
|
107
|
-
await this.client.bucket(this.bucket).file(key).delete();
|
|
108
|
-
}
|
|
109
|
-
catch (err) {
|
|
110
|
-
if (err.response?.statusCode === 404) {
|
|
111
|
-
return;
|
|
112
|
-
}
|
|
113
|
-
throw err;
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
async createSignedUploadUrl({ key, expirationTime, contentType, extensionHeaders = {}, }) {
|
|
117
|
-
const config = {
|
|
118
|
-
version: 'v4',
|
|
119
|
-
action: 'write',
|
|
120
|
-
expires: Date.now() + expirationTime,
|
|
121
|
-
contentType,
|
|
122
|
-
extensionHeaders,
|
|
123
|
-
};
|
|
124
|
-
const [url] = await this.client.bucket(this.bucket).file(key).getSignedUrl(config);
|
|
125
|
-
return {
|
|
126
|
-
url,
|
|
127
|
-
headers: {
|
|
128
|
-
...(contentType ? { 'content-type': contentType } : {}),
|
|
129
|
-
...extensionHeaders,
|
|
130
|
-
},
|
|
131
|
-
};
|
|
132
|
-
}
|
|
133
|
-
async createSignedDownloadUrl({ key, expirationTime, }) {
|
|
134
|
-
const options = {
|
|
135
|
-
version: 'v4',
|
|
136
|
-
action: 'read',
|
|
137
|
-
expires: Date.now() + expirationTime,
|
|
138
|
-
};
|
|
139
|
-
const [url] = await this.client.bucket(this.bucket).file(key).getSignedUrl(options);
|
|
140
|
-
return url;
|
|
141
|
-
}
|
|
142
|
-
async checkIfFileExists(key, fileHash) {
|
|
143
|
-
let metadata;
|
|
144
|
-
try {
|
|
145
|
-
[metadata] = await this.client.bucket(this.bucket).file(key).getMetadata();
|
|
146
|
-
}
|
|
147
|
-
catch (error) {
|
|
148
|
-
if (error.code === 404) {
|
|
149
|
-
return false;
|
|
150
|
-
}
|
|
151
|
-
throw error;
|
|
152
|
-
}
|
|
153
|
-
return fileHash ? metadata.etag === fileHash : true;
|
|
154
|
-
}
|
|
155
|
-
async listDirectory(prefix) {
|
|
156
|
-
const [files] = await this.client.bucket(this.bucket).getFiles({ prefix });
|
|
157
|
-
return files.map(x => this.formatHttpUrl(x.name));
|
|
158
|
-
}
|
|
159
|
-
async moveFile(src, dest) {
|
|
160
|
-
await this.client.bucket(this.bucket).file(src).move(dest);
|
|
161
|
-
}
|
|
162
|
-
async deleteFiles(keys) {
|
|
163
|
-
await Promise.all(keys.map(key => this.deleteFile(key)));
|
|
164
|
-
}
|
|
165
|
-
async downloadFile(key, destinationPath) {
|
|
166
|
-
const stream = this.client.bucket(this.bucket).file(key).createReadStream();
|
|
167
|
-
await pipeline(stream, fs_extra_1.default.createWriteStream(destinationPath));
|
|
168
|
-
}
|
|
169
|
-
getFile(key) {
|
|
170
|
-
return this.client.bucket(this.bucket).file(key);
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
exports.default = GCS;
|