@qrvey/object-storage 1.0.6-953 → 2.0.0-1158-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cjs/index-v2.js +1699 -0
- package/dist/cjs/index-v2.js.map +1 -0
- package/dist/cjs/index.js +8 -1
- package/dist/cjs/index.js.map +1 -1
- package/dist/esm/chunk-66BQA4WX.mjs +96 -0
- package/dist/esm/chunk-66BQA4WX.mjs.map +1 -0
- package/dist/esm/index-v2.d.mts +417 -0
- package/dist/esm/index-v2.mjs +1582 -0
- package/dist/esm/index-v2.mjs.map +1 -0
- package/dist/esm/index.mjs +9 -44
- package/dist/esm/index.mjs.map +1 -1
- package/dist/types/index-v2.d.ts +417 -0
- package/package.json +19 -4
|
@@ -0,0 +1,1699 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var exponentialBackoff = require('exponential-backoff');
|
|
4
|
+
var telemetry = require('@qrvey/telemetry');
|
|
5
|
+
var clientS3 = require('@aws-sdk/client-s3');
|
|
6
|
+
var credentialProviders = require('@aws-sdk/credential-providers');
|
|
7
|
+
var nodeHttpHandler = require('@smithy/node-http-handler');
|
|
8
|
+
var https = require('https');
|
|
9
|
+
var crypto = require('crypto');
|
|
10
|
+
var s3RequestPresigner = require('@aws-sdk/s3-request-presigner');
|
|
11
|
+
var libStorage = require('@aws-sdk/lib-storage');
|
|
12
|
+
var stream = require('stream');
|
|
13
|
+
var storageBlob = require('@azure/storage-blob');
|
|
14
|
+
var identity = require('@azure/identity');
|
|
15
|
+
|
|
16
|
+
var __defProp = Object.defineProperty;
|
|
17
|
+
var __defProps = Object.defineProperties;
|
|
18
|
+
var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
|
|
19
|
+
var __getOwnPropSymbols = Object.getOwnPropertySymbols;
|
|
20
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
21
|
+
var __propIsEnum = Object.prototype.propertyIsEnumerable;
|
|
22
|
+
var __knownSymbol = (name, symbol) => {
|
|
23
|
+
if (symbol = Symbol[name])
|
|
24
|
+
return symbol;
|
|
25
|
+
throw Error("Symbol." + name + " is not defined");
|
|
26
|
+
};
|
|
27
|
+
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
28
|
+
var __spreadValues = (a, b) => {
|
|
29
|
+
for (var prop in b || (b = {}))
|
|
30
|
+
if (__hasOwnProp.call(b, prop))
|
|
31
|
+
__defNormalProp(a, prop, b[prop]);
|
|
32
|
+
if (__getOwnPropSymbols)
|
|
33
|
+
for (var prop of __getOwnPropSymbols(b)) {
|
|
34
|
+
if (__propIsEnum.call(b, prop))
|
|
35
|
+
__defNormalProp(a, prop, b[prop]);
|
|
36
|
+
}
|
|
37
|
+
return a;
|
|
38
|
+
};
|
|
39
|
+
var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
|
|
40
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
41
|
+
var __publicField = (obj, key, value) => {
|
|
42
|
+
__defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
43
|
+
return value;
|
|
44
|
+
};
|
|
45
|
+
var __await = function(promise, isYieldStar) {
|
|
46
|
+
this[0] = promise;
|
|
47
|
+
this[1] = isYieldStar;
|
|
48
|
+
};
|
|
49
|
+
var __asyncGenerator = (__this, __arguments, generator) => {
|
|
50
|
+
var resume = (k, v, yes, no) => {
|
|
51
|
+
try {
|
|
52
|
+
var x = generator[k](v), isAwait = (v = x.value) instanceof __await, done = x.done;
|
|
53
|
+
Promise.resolve(isAwait ? v[0] : v).then((y) => isAwait ? resume(k === "return" ? k : "next", v[1] ? { done: y.done, value: y.value } : y, yes, no) : yes({ value: y, done })).catch((e) => resume("throw", e, yes, no));
|
|
54
|
+
} catch (e) {
|
|
55
|
+
no(e);
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
var method = (k) => it[k] = (x) => new Promise((yes, no) => resume(k, x, yes, no));
|
|
59
|
+
var it = {};
|
|
60
|
+
return generator = generator.apply(__this, __arguments), it[Symbol.asyncIterator] = () => it, method("next"), method("throw"), method("return"), it;
|
|
61
|
+
};
|
|
62
|
+
var __yieldStar = (value) => {
|
|
63
|
+
var obj = value[__knownSymbol("asyncIterator")];
|
|
64
|
+
var isAwait = false;
|
|
65
|
+
var method;
|
|
66
|
+
var it = {};
|
|
67
|
+
if (obj == null) {
|
|
68
|
+
obj = value[__knownSymbol("iterator")]();
|
|
69
|
+
method = (k) => it[k] = (x) => obj[k](x);
|
|
70
|
+
} else {
|
|
71
|
+
obj = obj.call(value);
|
|
72
|
+
method = (k) => it[k] = (v) => {
|
|
73
|
+
if (isAwait) {
|
|
74
|
+
isAwait = false;
|
|
75
|
+
if (k === "throw")
|
|
76
|
+
throw v;
|
|
77
|
+
return v;
|
|
78
|
+
}
|
|
79
|
+
isAwait = true;
|
|
80
|
+
return {
|
|
81
|
+
done: false,
|
|
82
|
+
value: new __await(new Promise((resolve) => {
|
|
83
|
+
var x = obj[k](v);
|
|
84
|
+
if (!(x instanceof Object))
|
|
85
|
+
throw TypeError("Object expected");
|
|
86
|
+
resolve(x);
|
|
87
|
+
}), 1)
|
|
88
|
+
};
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
return it[__knownSymbol("iterator")] = () => it, method("next"), "throw" in obj ? method("throw") : it.throw = (x) => {
|
|
92
|
+
throw x;
|
|
93
|
+
}, "return" in obj && method("return"), it;
|
|
94
|
+
};
|
|
95
|
+
var __forAwait = (obj, it, method) => (it = obj[__knownSymbol("asyncIterator")]) ? it.call(obj) : (obj = obj[__knownSymbol("iterator")](), it = {}, method = (key, fn) => (fn = obj[key]) && (it[key] = (arg) => new Promise((yes, no, done) => (arg = fn.call(obj, arg), done = arg.done, Promise.resolve(arg.value).then((value) => yes({ value, done }), no)))), method("next"), method("return"), it);
|
|
96
|
+
|
|
97
|
+
// src/core/errors.ts
|
|
98
|
+
var _StorageError = class _StorageError extends Error {
|
|
99
|
+
constructor(options) {
|
|
100
|
+
super(options.message);
|
|
101
|
+
__publicField(this, "code");
|
|
102
|
+
__publicField(this, "retryable");
|
|
103
|
+
__publicField(this, "provider");
|
|
104
|
+
__publicField(this, "requestId");
|
|
105
|
+
__publicField(this, "cause");
|
|
106
|
+
this.name = new.target.name;
|
|
107
|
+
this.code = options.code;
|
|
108
|
+
this.retryable = options.retryable;
|
|
109
|
+
this.provider = options.provider;
|
|
110
|
+
this.requestId = options.requestId;
|
|
111
|
+
this.cause = options.cause;
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
__name(_StorageError, "StorageError");
|
|
115
|
+
var StorageError = _StorageError;
|
|
116
|
+
var _ThrottledError = class _ThrottledError extends StorageError {
|
|
117
|
+
constructor(options) {
|
|
118
|
+
super(__spreadProps(__spreadValues({
|
|
119
|
+
code: "THROTTLED"
|
|
120
|
+
}, options), {
|
|
121
|
+
retryable: true
|
|
122
|
+
}));
|
|
123
|
+
__publicField(this, "retryAfterMs");
|
|
124
|
+
this.retryAfterMs = options.retryAfterMs;
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
__name(_ThrottledError, "ThrottledError");
|
|
128
|
+
var ThrottledError = _ThrottledError;
|
|
129
|
+
var _NotFoundError = class _NotFoundError extends StorageError {
|
|
130
|
+
constructor(options) {
|
|
131
|
+
super(__spreadValues({
|
|
132
|
+
code: "NOT_FOUND",
|
|
133
|
+
retryable: false
|
|
134
|
+
}, options));
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
__name(_NotFoundError, "NotFoundError");
|
|
138
|
+
var NotFoundError = _NotFoundError;
|
|
139
|
+
var _AuthError = class _AuthError extends StorageError {
|
|
140
|
+
constructor(options) {
|
|
141
|
+
super(__spreadValues({
|
|
142
|
+
code: "AUTH",
|
|
143
|
+
retryable: false
|
|
144
|
+
}, options));
|
|
145
|
+
}
|
|
146
|
+
};
|
|
147
|
+
__name(_AuthError, "AuthError");
|
|
148
|
+
var AuthError = _AuthError;
|
|
149
|
+
var _ValidationError = class _ValidationError extends StorageError {
|
|
150
|
+
constructor(options) {
|
|
151
|
+
super(__spreadValues({
|
|
152
|
+
code: "VALIDATION",
|
|
153
|
+
retryable: false
|
|
154
|
+
}, options));
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
__name(_ValidationError, "ValidationError");
|
|
158
|
+
var ValidationError = _ValidationError;
|
|
159
|
+
var _ConfigurationError = class _ConfigurationError extends StorageError {
|
|
160
|
+
constructor(options) {
|
|
161
|
+
super(__spreadValues({
|
|
162
|
+
code: "CONFIGURATION",
|
|
163
|
+
retryable: false
|
|
164
|
+
}, options));
|
|
165
|
+
}
|
|
166
|
+
};
|
|
167
|
+
__name(_ConfigurationError, "ConfigurationError");
|
|
168
|
+
var ConfigurationError = _ConfigurationError;
|
|
169
|
+
var _NetworkError = class _NetworkError extends StorageError {
|
|
170
|
+
constructor(options) {
|
|
171
|
+
super(__spreadValues({
|
|
172
|
+
code: "NETWORK",
|
|
173
|
+
retryable: true
|
|
174
|
+
}, options));
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
__name(_NetworkError, "NetworkError");
|
|
178
|
+
var NetworkError = _NetworkError;
|
|
179
|
+
var _IntegrityError = class _IntegrityError extends StorageError {
|
|
180
|
+
constructor(options) {
|
|
181
|
+
super(__spreadValues({
|
|
182
|
+
code: "INTEGRITY",
|
|
183
|
+
retryable: false
|
|
184
|
+
}, options));
|
|
185
|
+
}
|
|
186
|
+
};
|
|
187
|
+
__name(_IntegrityError, "IntegrityError");
|
|
188
|
+
var IntegrityError = _IntegrityError;
|
|
189
|
+
var _MultipartSessionError = class _MultipartSessionError extends StorageError {
|
|
190
|
+
constructor(options) {
|
|
191
|
+
super(__spreadValues({
|
|
192
|
+
code: "MULTIPART_SESSION",
|
|
193
|
+
retryable: false
|
|
194
|
+
}, options));
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
__name(_MultipartSessionError, "MultipartSessionError");
|
|
198
|
+
var MultipartSessionError = _MultipartSessionError;
|
|
199
|
+
|
|
200
|
+
// src/core/providerRegistry.ts
|
|
201
|
+
var envDefaults = /* @__PURE__ */ new Map();
|
|
202
|
+
function registerProviderEnvDefaults(providerId, factory) {
|
|
203
|
+
envDefaults.set(providerId, factory);
|
|
204
|
+
}
|
|
205
|
+
__name(registerProviderEnvDefaults, "registerProviderEnvDefaults");
|
|
206
|
+
function resolveOptionsFromEnv(env = process.env) {
|
|
207
|
+
var _a3, _b;
|
|
208
|
+
const providerId = (_a3 = env.OBJECT_STORAGE_SERVICE) == null ? void 0 : _a3.toLowerCase();
|
|
209
|
+
const factory = providerId ? envDefaults.get(providerId) : void 0;
|
|
210
|
+
if (!factory) {
|
|
211
|
+
throw new ConfigurationError({
|
|
212
|
+
message: `Unsupported or missing OBJECT_STORAGE_SERVICE: "${(_b = env.OBJECT_STORAGE_SERVICE) != null ? _b : ""}". Registered providers: ${[
|
|
213
|
+
...envDefaults.keys()
|
|
214
|
+
].join(", ")}`
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
return factory(env);
|
|
218
|
+
}
|
|
219
|
+
__name(resolveOptionsFromEnv, "resolveOptionsFromEnv");
|
|
220
|
+
function resolveObjectStorageOptions(options, env = process.env) {
|
|
221
|
+
return options != null ? options : resolveOptionsFromEnv(env);
|
|
222
|
+
}
|
|
223
|
+
__name(resolveObjectStorageOptions, "resolveObjectStorageOptions");
|
|
224
|
+
var logger = new telemetry.LoggerService("@qrvey/object-storage");
|
|
225
|
+
|
|
226
|
+
// src/core/instrumentation.ts
|
|
227
|
+
function recordRetry(code) {
|
|
228
|
+
logger.warn("objectstorage.retry", {
|
|
229
|
+
code: code != null ? code : "unknown"
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
__name(recordRetry, "recordRetry");
|
|
233
|
+
function recordThrottle(provider) {
|
|
234
|
+
logger.warn("objectstorage.throttle", {
|
|
235
|
+
provider: provider != null ? provider : "unknown"
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
__name(recordThrottle, "recordThrottle");
|
|
239
|
+
async function instrument(provider, operation, fn) {
|
|
240
|
+
const startedAt = Date.now();
|
|
241
|
+
try {
|
|
242
|
+
const result = await fn();
|
|
243
|
+
logger.debug("objectstorage.operation", {
|
|
244
|
+
provider,
|
|
245
|
+
operation,
|
|
246
|
+
result: "ok",
|
|
247
|
+
durationMs: Date.now() - startedAt
|
|
248
|
+
});
|
|
249
|
+
return result;
|
|
250
|
+
} catch (error) {
|
|
251
|
+
const meta = {
|
|
252
|
+
provider,
|
|
253
|
+
operation,
|
|
254
|
+
result: "error",
|
|
255
|
+
durationMs: Date.now() - startedAt
|
|
256
|
+
};
|
|
257
|
+
if (error instanceof StorageError) {
|
|
258
|
+
meta.code = error.code;
|
|
259
|
+
meta.retryable = error.retryable;
|
|
260
|
+
if (error.requestId)
|
|
261
|
+
meta.requestId = error.requestId;
|
|
262
|
+
}
|
|
263
|
+
logger.error("objectstorage.operation", meta);
|
|
264
|
+
throw error;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
__name(instrument, "instrument");
|
|
268
|
+
|
|
269
|
+
// src/core/retry.ts
|
|
270
|
+
var DEFAULTS = {
|
|
271
|
+
numOfAttempts: 5,
|
|
272
|
+
startingDelayMs: 100,
|
|
273
|
+
maxDelayMs: 5e3
|
|
274
|
+
};
|
|
275
|
+
function isRetryableError(error) {
|
|
276
|
+
return error instanceof StorageError && error.retryable;
|
|
277
|
+
}
|
|
278
|
+
__name(isRetryableError, "isRetryableError");
|
|
279
|
+
async function withRetry(fn, policy = {}) {
|
|
280
|
+
var _a3;
|
|
281
|
+
const { numOfAttempts, startingDelayMs, maxDelayMs } = __spreadValues(__spreadValues({}, DEFAULTS), policy);
|
|
282
|
+
const shouldRetry = (_a3 = policy.retry) != null ? _a3 : isRetryableError;
|
|
283
|
+
const options = {
|
|
284
|
+
numOfAttempts,
|
|
285
|
+
startingDelay: startingDelayMs,
|
|
286
|
+
maxDelay: maxDelayMs,
|
|
287
|
+
jitter: "full",
|
|
288
|
+
retry: async (error, attempt) => {
|
|
289
|
+
if (!shouldRetry(error, attempt))
|
|
290
|
+
return false;
|
|
291
|
+
const code = error instanceof StorageError ? error.code : void 0;
|
|
292
|
+
recordRetry(code);
|
|
293
|
+
if (error instanceof ThrottledError) {
|
|
294
|
+
recordThrottle(error.provider);
|
|
295
|
+
}
|
|
296
|
+
logger.warn("object-storage operation retry", {
|
|
297
|
+
attempt,
|
|
298
|
+
numOfAttempts,
|
|
299
|
+
code
|
|
300
|
+
});
|
|
301
|
+
if (error instanceof ThrottledError && error.retryAfterMs !== void 0) {
|
|
302
|
+
await sleep(Math.min(error.retryAfterMs, maxDelayMs));
|
|
303
|
+
}
|
|
304
|
+
return true;
|
|
305
|
+
}
|
|
306
|
+
};
|
|
307
|
+
return exponentialBackoff.backOff(fn, options);
|
|
308
|
+
}
|
|
309
|
+
__name(withRetry, "withRetry");
|
|
310
|
+
function sleep(ms) {
|
|
311
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
312
|
+
}
|
|
313
|
+
__name(sleep, "sleep");
|
|
314
|
+
|
|
315
|
+
// src/core/concurrency.ts
|
|
316
|
+
function createLimiter(options) {
|
|
317
|
+
var _a3, _b;
|
|
318
|
+
const ceiling = options.max;
|
|
319
|
+
const floor = (_a3 = options.minMax) != null ? _a3 : 1;
|
|
320
|
+
const growthThreshold = (_b = options.growthThreshold) != null ? _b : 10;
|
|
321
|
+
let max = options.max;
|
|
322
|
+
let inFlight = 0;
|
|
323
|
+
let consecutiveSuccesses = 0;
|
|
324
|
+
const queue = [];
|
|
325
|
+
function release() {
|
|
326
|
+
var _a4;
|
|
327
|
+
inFlight--;
|
|
328
|
+
(_a4 = queue.shift()) == null ? void 0 : _a4();
|
|
329
|
+
}
|
|
330
|
+
__name(release, "release");
|
|
331
|
+
function acquire() {
|
|
332
|
+
if (inFlight < max) {
|
|
333
|
+
inFlight++;
|
|
334
|
+
return Promise.resolve();
|
|
335
|
+
}
|
|
336
|
+
return new Promise((resolve) => queue.push(() => {
|
|
337
|
+
inFlight++;
|
|
338
|
+
resolve();
|
|
339
|
+
}));
|
|
340
|
+
}
|
|
341
|
+
__name(acquire, "acquire");
|
|
342
|
+
function onThrottle() {
|
|
343
|
+
if (!options.adaptive)
|
|
344
|
+
return;
|
|
345
|
+
consecutiveSuccesses = 0;
|
|
346
|
+
const reduced = Math.max(floor, Math.floor(max / 2));
|
|
347
|
+
if (reduced < max) {
|
|
348
|
+
max = reduced;
|
|
349
|
+
logger.warn("object-storage limiter shrunk after throttle", {
|
|
350
|
+
max
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
__name(onThrottle, "onThrottle");
|
|
355
|
+
function onSuccess() {
|
|
356
|
+
if (!options.adaptive || max >= ceiling)
|
|
357
|
+
return;
|
|
358
|
+
consecutiveSuccesses++;
|
|
359
|
+
if (consecutiveSuccesses >= growthThreshold) {
|
|
360
|
+
consecutiveSuccesses = 0;
|
|
361
|
+
max = Math.min(ceiling, max + 1);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
__name(onSuccess, "onSuccess");
|
|
365
|
+
return {
|
|
366
|
+
async run(fn) {
|
|
367
|
+
await acquire();
|
|
368
|
+
try {
|
|
369
|
+
const result = await fn();
|
|
370
|
+
onSuccess();
|
|
371
|
+
return result;
|
|
372
|
+
} catch (error) {
|
|
373
|
+
if (error instanceof ThrottledError)
|
|
374
|
+
onThrottle();
|
|
375
|
+
throw error;
|
|
376
|
+
} finally {
|
|
377
|
+
release();
|
|
378
|
+
}
|
|
379
|
+
},
|
|
380
|
+
get inFlight() {
|
|
381
|
+
return inFlight;
|
|
382
|
+
},
|
|
383
|
+
get pending() {
|
|
384
|
+
return queue.length;
|
|
385
|
+
},
|
|
386
|
+
get currentMax() {
|
|
387
|
+
return max;
|
|
388
|
+
}
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
__name(createLimiter, "createLimiter");
|
|
392
|
+
|
|
393
|
+
// src/core/batch.ts
|
|
394
|
+
function chunk(items, size) {
|
|
395
|
+
const chunks = [];
|
|
396
|
+
for (let i = 0; i < items.length; i += size) {
|
|
397
|
+
chunks.push(items.slice(i, i + size));
|
|
398
|
+
}
|
|
399
|
+
return chunks;
|
|
400
|
+
}
|
|
401
|
+
__name(chunk, "chunk");
|
|
402
|
+
async function mapBatched(items, batchSize, limiter, fn) {
|
|
403
|
+
const batches = chunk(items, batchSize);
|
|
404
|
+
return Promise.all(batches.map((batch, index) => limiter.run(() => fn(batch, index))));
|
|
405
|
+
}
|
|
406
|
+
__name(mapBatched, "mapBatched");
|
|
407
|
+
|
|
408
|
+
// src/core/range.ts
|
|
409
|
+
var BOUNDED = /^bytes=(\d+)-(\d+)?$/;
|
|
410
|
+
var SUFFIX = /^bytes=-(\d+)$/;
|
|
411
|
+
function parseRange(input) {
|
|
412
|
+
const bounded = BOUNDED.exec(input);
|
|
413
|
+
if (bounded && bounded[1] !== void 0) {
|
|
414
|
+
const start = Number(bounded[1]);
|
|
415
|
+
const end = bounded[2] !== void 0 ? Number(bounded[2]) : void 0;
|
|
416
|
+
if (end !== void 0 && end < start) {
|
|
417
|
+
throw new ValidationError({
|
|
418
|
+
message: `Invalid range "${input}": end must be >= start`
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
return {
|
|
422
|
+
start,
|
|
423
|
+
end
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
const suffix = SUFFIX.exec(input);
|
|
427
|
+
if (suffix && suffix[1] !== void 0) {
|
|
428
|
+
return {
|
|
429
|
+
suffixLength: Number(suffix[1])
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
throw new ValidationError({
|
|
433
|
+
message: `Invalid range "${input}": expected "bytes=start-[end]" or "bytes=-suffix"`
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
__name(parseRange, "parseRange");
|
|
437
|
+
function toRangeHeader(range) {
|
|
438
|
+
var _a3;
|
|
439
|
+
if (range.suffixLength !== void 0)
|
|
440
|
+
return `bytes=-${range.suffixLength}`;
|
|
441
|
+
return `bytes=${range.start}-${(_a3 = range.end) != null ? _a3 : ""}`;
|
|
442
|
+
}
|
|
443
|
+
__name(toRangeHeader, "toRangeHeader");
|
|
444
|
+
|
|
445
|
+
// src/providers/aws-s3/config.ts
|
|
446
|
+
var AWS_S3_PROVIDER_ID = "aws_s3";
|
|
447
|
+
function isAwsS3StorageOptions(options) {
|
|
448
|
+
return options.provider === AWS_S3_PROVIDER_ID;
|
|
449
|
+
}
|
|
450
|
+
__name(isAwsS3StorageOptions, "isAwsS3StorageOptions");
|
|
451
|
+
function getDefaultAwsS3OptionsFromEnv(env = process.env) {
|
|
452
|
+
const region = env.AWS_REGION || env.AWS_DEFAULT_REGION;
|
|
453
|
+
if (!region) {
|
|
454
|
+
throw new ConfigurationError({
|
|
455
|
+
provider: AWS_S3_PROVIDER_ID,
|
|
456
|
+
message: "Missing AWS_REGION (or AWS_DEFAULT_REGION) for aws_s3 provider"
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
return {
|
|
460
|
+
provider: AWS_S3_PROVIDER_ID,
|
|
461
|
+
region,
|
|
462
|
+
auth: {
|
|
463
|
+
kind: "default"
|
|
464
|
+
}
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
__name(getDefaultAwsS3OptionsFromEnv, "getDefaultAwsS3OptionsFromEnv");
|
|
468
|
+
registerProviderEnvDefaults(AWS_S3_PROVIDER_ID, getDefaultAwsS3OptionsFromEnv);
|
|
469
|
+
|
|
470
|
+
// src/core/clientCache.ts
|
|
471
|
+
var DEFAULT_MAX_SIZE = 32;
|
|
472
|
+
var _ClientCache = class _ClientCache {
|
|
473
|
+
constructor(options = {}) {
|
|
474
|
+
__publicField(this, "options");
|
|
475
|
+
__publicField(this, "entries");
|
|
476
|
+
this.options = options;
|
|
477
|
+
this.entries = /* @__PURE__ */ new Map();
|
|
478
|
+
}
|
|
479
|
+
getOrCreate(key, factory) {
|
|
480
|
+
var _a3, _b, _c;
|
|
481
|
+
const existing = this.entries.get(key);
|
|
482
|
+
if (existing !== void 0) {
|
|
483
|
+
this.entries.delete(key);
|
|
484
|
+
this.entries.set(key, existing);
|
|
485
|
+
return existing;
|
|
486
|
+
}
|
|
487
|
+
const client = factory();
|
|
488
|
+
this.entries.set(key, client);
|
|
489
|
+
logger.debug("object-storage client created", {
|
|
490
|
+
size: this.entries.size
|
|
491
|
+
});
|
|
492
|
+
const maxSize = (_a3 = this.options.maxSize) != null ? _a3 : DEFAULT_MAX_SIZE;
|
|
493
|
+
if (this.entries.size > maxSize) {
|
|
494
|
+
const oldestKey = this.entries.keys().next().value;
|
|
495
|
+
const oldest = this.entries.get(oldestKey);
|
|
496
|
+
this.entries.delete(oldestKey);
|
|
497
|
+
(_c = (_b = this.options).onEvict) == null ? void 0 : _c.call(_b, oldest, oldestKey);
|
|
498
|
+
logger.warn("object-storage client evicted from cache", {
|
|
499
|
+
maxSize
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
return client;
|
|
503
|
+
}
|
|
504
|
+
get size() {
|
|
505
|
+
return this.entries.size;
|
|
506
|
+
}
|
|
507
|
+
clear() {
|
|
508
|
+
this.entries.clear();
|
|
509
|
+
}
|
|
510
|
+
};
|
|
511
|
+
__name(_ClientCache, "ClientCache");
|
|
512
|
+
var ClientCache = _ClientCache;
|
|
513
|
+
function stableCacheKey(parts) {
|
|
514
|
+
return JSON.stringify(sortDeep(parts));
|
|
515
|
+
}
|
|
516
|
+
__name(stableCacheKey, "stableCacheKey");
|
|
517
|
+
function hashSecret(value) {
|
|
518
|
+
return crypto.createHash("sha256").update(value).digest("hex").slice(0, 16);
|
|
519
|
+
}
|
|
520
|
+
__name(hashSecret, "hashSecret");
|
|
521
|
+
function sortDeep(value) {
|
|
522
|
+
if (Array.isArray(value))
|
|
523
|
+
return value.map(sortDeep);
|
|
524
|
+
if (value && typeof value === "object") {
|
|
525
|
+
return Object.keys(value).sort().reduce((acc, key) => {
|
|
526
|
+
acc[key] = sortDeep(value[key]);
|
|
527
|
+
return acc;
|
|
528
|
+
}, {});
|
|
529
|
+
}
|
|
530
|
+
return value;
|
|
531
|
+
}
|
|
532
|
+
__name(sortDeep, "sortDeep");
|
|
533
|
+
|
|
534
|
+
// src/providers/aws-s3/clientFactory.ts
|
|
535
|
+
var DEFAULT_HTTP = {
|
|
536
|
+
connectionTimeoutMs: 5e3,
|
|
537
|
+
requestTimeoutMs: 12e4,
|
|
538
|
+
maxSockets: 64
|
|
539
|
+
};
|
|
540
|
+
var cache = new ClientCache({
|
|
541
|
+
onEvict: (client) => client.destroy()
|
|
542
|
+
});
|
|
543
|
+
function getAwsS3Client(options) {
|
|
544
|
+
var _a3;
|
|
545
|
+
const auth = (_a3 = options.auth) != null ? _a3 : {
|
|
546
|
+
kind: "default"
|
|
547
|
+
};
|
|
548
|
+
const key = stableCacheKey({
|
|
549
|
+
provider: options.provider,
|
|
550
|
+
region: options.region,
|
|
551
|
+
endpoint: options.endpoint,
|
|
552
|
+
http: options.http,
|
|
553
|
+
auth: authKeyParts(auth)
|
|
554
|
+
});
|
|
555
|
+
return cache.getOrCreate(key, () => buildClient(options, auth));
|
|
556
|
+
}
|
|
557
|
+
__name(getAwsS3Client, "getAwsS3Client");
|
|
558
|
+
function clearAwsS3ClientCache() {
|
|
559
|
+
cache.clear();
|
|
560
|
+
}
|
|
561
|
+
__name(clearAwsS3ClientCache, "clearAwsS3ClientCache");
|
|
562
|
+
function buildClient(options, auth) {
|
|
563
|
+
const http = __spreadValues(__spreadValues({}, DEFAULT_HTTP), options.http);
|
|
564
|
+
return new clientS3.S3Client({
|
|
565
|
+
region: options.region,
|
|
566
|
+
endpoint: options.endpoint,
|
|
567
|
+
credentials: credentialsFor(auth),
|
|
568
|
+
requestHandler: new nodeHttpHandler.NodeHttpHandler({
|
|
569
|
+
connectionTimeout: http.connectionTimeoutMs,
|
|
570
|
+
requestTimeout: http.requestTimeoutMs,
|
|
571
|
+
httpsAgent: new https.Agent({
|
|
572
|
+
keepAlive: true,
|
|
573
|
+
maxSockets: http.maxSockets
|
|
574
|
+
})
|
|
575
|
+
})
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
__name(buildClient, "buildClient");
|
|
579
|
+
function credentialsFor(auth) {
|
|
580
|
+
switch (auth.kind) {
|
|
581
|
+
case "default":
|
|
582
|
+
return credentialProviders.fromNodeProviderChain();
|
|
583
|
+
case "static":
|
|
584
|
+
return auth.credentials;
|
|
585
|
+
case "assumeRole":
|
|
586
|
+
return credentialProviders.fromTemporaryCredentials({
|
|
587
|
+
params: {
|
|
588
|
+
RoleArn: auth.roleArn,
|
|
589
|
+
ExternalId: auth.externalId,
|
|
590
|
+
RoleSessionName: "qrvey-object-storage"
|
|
591
|
+
}
|
|
592
|
+
});
|
|
593
|
+
case "webIdentity":
|
|
594
|
+
return credentialProviders.fromTokenFile(__spreadValues({
|
|
595
|
+
roleArn: auth.roleArn
|
|
596
|
+
}, auth.tokenFile && {
|
|
597
|
+
webIdentityTokenFile: auth.tokenFile
|
|
598
|
+
}));
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
__name(credentialsFor, "credentialsFor");
|
|
602
|
+
function authKeyParts(auth) {
|
|
603
|
+
switch (auth.kind) {
|
|
604
|
+
case "default":
|
|
605
|
+
return {
|
|
606
|
+
kind: auth.kind
|
|
607
|
+
};
|
|
608
|
+
case "static":
|
|
609
|
+
return {
|
|
610
|
+
kind: auth.kind,
|
|
611
|
+
accessKeyId: auth.credentials.accessKeyId,
|
|
612
|
+
secret: hashSecret(auth.credentials.secretAccessKey)
|
|
613
|
+
};
|
|
614
|
+
case "assumeRole":
|
|
615
|
+
return {
|
|
616
|
+
kind: auth.kind,
|
|
617
|
+
roleArn: auth.roleArn,
|
|
618
|
+
externalId: auth.externalId
|
|
619
|
+
};
|
|
620
|
+
case "webIdentity":
|
|
621
|
+
return {
|
|
622
|
+
kind: auth.kind,
|
|
623
|
+
roleArn: auth.roleArn,
|
|
624
|
+
tokenFile: auth.tokenFile
|
|
625
|
+
};
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
__name(authKeyParts, "authKeyParts");
|
|
629
|
+
|
|
630
|
+
// src/providers/aws-s3/errors.ts
|
|
631
|
+
var THROTTLE_NAMES = /* @__PURE__ */ new Set([
|
|
632
|
+
"SlowDown",
|
|
633
|
+
"ThrottlingException",
|
|
634
|
+
"Throttling",
|
|
635
|
+
"TooManyRequestsException",
|
|
636
|
+
"RequestLimitExceeded",
|
|
637
|
+
"ProvisionedThroughputExceededException"
|
|
638
|
+
]);
|
|
639
|
+
var NOT_FOUND_NAMES = /* @__PURE__ */ new Set([
|
|
640
|
+
"NoSuchKey",
|
|
641
|
+
"NoSuchBucket",
|
|
642
|
+
"NotFound",
|
|
643
|
+
"NoSuchUpload"
|
|
644
|
+
]);
|
|
645
|
+
var AUTH_NAMES = /* @__PURE__ */ new Set([
|
|
646
|
+
"AccessDenied",
|
|
647
|
+
"InvalidAccessKeyId",
|
|
648
|
+
"SignatureDoesNotMatch",
|
|
649
|
+
"ExpiredToken",
|
|
650
|
+
"TokenRefreshRequired",
|
|
651
|
+
"CredentialsProviderError"
|
|
652
|
+
]);
|
|
653
|
+
function mapAwsError(error) {
|
|
654
|
+
var _a3, _b, _c, _d, _e, _f;
|
|
655
|
+
if (error instanceof StorageError)
|
|
656
|
+
return error;
|
|
657
|
+
const aws = error != null ? error : {};
|
|
658
|
+
const name = (_a3 = aws.name) != null ? _a3 : "UnknownError";
|
|
659
|
+
const message = (_b = aws.message) != null ? _b : String(error);
|
|
660
|
+
const status = (_c = aws.$metadata) == null ? void 0 : _c.httpStatusCode;
|
|
661
|
+
const base = {
|
|
662
|
+
message: `${name}: ${message}`,
|
|
663
|
+
provider: AWS_S3_PROVIDER_ID,
|
|
664
|
+
requestId: (_d = aws.$metadata) == null ? void 0 : _d.requestId,
|
|
665
|
+
cause: error
|
|
666
|
+
};
|
|
667
|
+
if (THROTTLE_NAMES.has(name) || status === 429) {
|
|
668
|
+
const retryAfter = (_f = (_e = aws.$response) == null ? void 0 : _e.headers) == null ? void 0 : _f["retry-after"];
|
|
669
|
+
return new ThrottledError(__spreadProps(__spreadValues({}, base), {
|
|
670
|
+
code: name,
|
|
671
|
+
retryAfterMs: retryAfter !== void 0 ? Number(retryAfter) * 1e3 : void 0
|
|
672
|
+
}));
|
|
673
|
+
}
|
|
674
|
+
if (NOT_FOUND_NAMES.has(name) || status === 404) {
|
|
675
|
+
return new NotFoundError(__spreadProps(__spreadValues({}, base), {
|
|
676
|
+
code: name
|
|
677
|
+
}));
|
|
678
|
+
}
|
|
679
|
+
if (AUTH_NAMES.has(name) || status === 403) {
|
|
680
|
+
return new AuthError(__spreadProps(__spreadValues({}, base), {
|
|
681
|
+
code: name
|
|
682
|
+
}));
|
|
683
|
+
}
|
|
684
|
+
if (status !== void 0 && status >= 500 || name === "TimeoutError") {
|
|
685
|
+
return new NetworkError(__spreadProps(__spreadValues({}, base), {
|
|
686
|
+
code: name
|
|
687
|
+
}));
|
|
688
|
+
}
|
|
689
|
+
return new StorageError(__spreadProps(__spreadValues({}, base), {
|
|
690
|
+
code: name,
|
|
691
|
+
retryable: false
|
|
692
|
+
}));
|
|
693
|
+
}
|
|
694
|
+
__name(mapAwsError, "mapAwsError");
|
|
695
|
+
var DELETE_BATCH_SIZE = 1e3;
|
|
696
|
+
var DEFAULT_PRESIGN_SECONDS = 3600;
|
|
697
|
+
var _AwsS3Provider = class _AwsS3Provider {
|
|
698
|
+
constructor(config) {
|
|
699
|
+
__publicField(this, "bucketName");
|
|
700
|
+
__publicField(this, "multipart");
|
|
701
|
+
__publicField(this, "client");
|
|
702
|
+
__publicField(this, "retryPolicy");
|
|
703
|
+
__publicField(this, "limiter");
|
|
704
|
+
var _a3, _b;
|
|
705
|
+
const resolved = resolveObjectStorageOptions(config.options);
|
|
706
|
+
if (!isAwsS3StorageOptions(resolved)) {
|
|
707
|
+
throw new ConfigurationError({
|
|
708
|
+
message: `AwsS3Provider requires aws_s3 options, got provider "${resolved.provider}"`
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
this.bucketName = config.bucketName;
|
|
712
|
+
this.client = getAwsS3Client(resolved);
|
|
713
|
+
this.retryPolicy = (_a3 = config.retry) != null ? _a3 : {};
|
|
714
|
+
this.limiter = (_b = config.limiter) != null ? _b : createLimiter({
|
|
715
|
+
max: 10,
|
|
716
|
+
adaptive: true
|
|
717
|
+
});
|
|
718
|
+
this.multipart = new AwsS3MultipartClient(this.client, this.bucketName, this.retryPolicy, (operation, fn) => this.send(operation, fn));
|
|
719
|
+
}
|
|
720
|
+
send(operation, fn) {
|
|
721
|
+
return instrument(AWS_S3_PROVIDER_ID, operation, () => withRetry(() => fn().catch((error) => Promise.reject(mapAwsError(error))), this.retryPolicy));
|
|
722
|
+
}
|
|
723
|
+
async getObject(key, options) {
|
|
724
|
+
const range = (options == null ? void 0 : options.range) ? toRangeHeader(parseRange(options.range)) : void 0;
|
|
725
|
+
const response = await this.send("getObject", () => this.client.send(new clientS3.GetObjectCommand({
|
|
726
|
+
Bucket: this.bucketName,
|
|
727
|
+
Key: key,
|
|
728
|
+
Range: range,
|
|
729
|
+
VersionId: options == null ? void 0 : options.versionId
|
|
730
|
+
})));
|
|
731
|
+
return {
|
|
732
|
+
body: response.Body,
|
|
733
|
+
contentLength: response.ContentLength,
|
|
734
|
+
contentType: response.ContentType,
|
|
735
|
+
contentEncoding: response.ContentEncoding,
|
|
736
|
+
contentDisposition: response.ContentDisposition,
|
|
737
|
+
contentLanguage: response.ContentLanguage,
|
|
738
|
+
cacheControl: response.CacheControl,
|
|
739
|
+
etag: response.ETag,
|
|
740
|
+
lastModified: response.LastModified,
|
|
741
|
+
metadata: response.Metadata
|
|
742
|
+
};
|
|
743
|
+
}
|
|
744
|
+
async getObjectProperties(key) {
|
|
745
|
+
const response = await this.send("getObjectProperties", () => this.client.send(new clientS3.HeadObjectCommand({
|
|
746
|
+
Bucket: this.bucketName,
|
|
747
|
+
Key: key
|
|
748
|
+
})));
|
|
749
|
+
return {
|
|
750
|
+
contentLength: response.ContentLength,
|
|
751
|
+
contentType: response.ContentType,
|
|
752
|
+
etag: response.ETag,
|
|
753
|
+
lastModified: response.LastModified,
|
|
754
|
+
metadata: response.Metadata
|
|
755
|
+
};
|
|
756
|
+
}
|
|
757
|
+
async upload(key, body, options) {
|
|
758
|
+
if (body instanceof stream.Readable) {
|
|
759
|
+
return this.uploadStream(key, body, options);
|
|
760
|
+
}
|
|
761
|
+
const response = await this.send("upload", () => this.client.send(new clientS3.PutObjectCommand(__spreadValues({
|
|
762
|
+
Bucket: this.bucketName,
|
|
763
|
+
Key: key,
|
|
764
|
+
Body: body
|
|
765
|
+
}, mapUploadOptions(options)))));
|
|
766
|
+
return {
|
|
767
|
+
key,
|
|
768
|
+
etag: response.ETag,
|
|
769
|
+
versionId: response.VersionId
|
|
770
|
+
};
|
|
771
|
+
}
|
|
772
|
+
async uploadStream(key, body, options) {
|
|
773
|
+
var _a3, _b;
|
|
774
|
+
const upload = new libStorage.Upload({
|
|
775
|
+
client: this.client,
|
|
776
|
+
params: __spreadValues({
|
|
777
|
+
Bucket: this.bucketName,
|
|
778
|
+
Key: key,
|
|
779
|
+
Body: body
|
|
780
|
+
}, mapUploadOptions(options)),
|
|
781
|
+
queueSize: (_a3 = options == null ? void 0 : options.queueSize) != null ? _a3 : 5,
|
|
782
|
+
partSize: (_b = options == null ? void 0 : options.partSizeBytes) != null ? _b : 8 * 1024 * 1024
|
|
783
|
+
});
|
|
784
|
+
return instrument(AWS_S3_PROVIDER_ID, "upload", async () => {
|
|
785
|
+
try {
|
|
786
|
+
const response = await upload.done();
|
|
787
|
+
return {
|
|
788
|
+
key,
|
|
789
|
+
etag: response.ETag,
|
|
790
|
+
versionId: response.VersionId
|
|
791
|
+
};
|
|
792
|
+
} catch (error) {
|
|
793
|
+
throw mapAwsError(error);
|
|
794
|
+
}
|
|
795
|
+
});
|
|
796
|
+
}
|
|
797
|
+
createWriteStream(key, options) {
|
|
798
|
+
var _a3, _b;
|
|
799
|
+
const stream$1 = new stream.PassThrough();
|
|
800
|
+
const upload = new libStorage.Upload({
|
|
801
|
+
client: this.client,
|
|
802
|
+
params: __spreadValues({
|
|
803
|
+
Bucket: this.bucketName,
|
|
804
|
+
Key: key,
|
|
805
|
+
Body: stream$1
|
|
806
|
+
}, mapUploadOptions(options)),
|
|
807
|
+
queueSize: (_a3 = options == null ? void 0 : options.queueSize) != null ? _a3 : 5,
|
|
808
|
+
partSize: (_b = options == null ? void 0 : options.partSizeBytes) != null ? _b : 8 * 1024 * 1024
|
|
809
|
+
});
|
|
810
|
+
const done = instrument(AWS_S3_PROVIDER_ID, "createWriteStream", () => upload.done().then((response) => ({
|
|
811
|
+
key,
|
|
812
|
+
etag: response.ETag,
|
|
813
|
+
versionId: response.VersionId
|
|
814
|
+
})).catch((error) => Promise.reject(mapAwsError(error))));
|
|
815
|
+
return {
|
|
816
|
+
stream: stream$1,
|
|
817
|
+
done,
|
|
818
|
+
abort: () => upload.abort()
|
|
819
|
+
};
|
|
820
|
+
}
|
|
821
|
+
async delete(key) {
|
|
822
|
+
await this.send("delete", () => this.client.send(new clientS3.DeleteObjectCommand({
|
|
823
|
+
Bucket: this.bucketName,
|
|
824
|
+
Key: key
|
|
825
|
+
})));
|
|
826
|
+
}
|
|
827
|
+
async deleteMany(keys) {
|
|
828
|
+
const batchResults = await mapBatched(keys, DELETE_BATCH_SIZE, this.limiter, async (batch) => {
|
|
829
|
+
var _a3;
|
|
830
|
+
const response = await this.send("deleteMany", () => this.client.send(new clientS3.DeleteObjectsCommand({
|
|
831
|
+
Bucket: this.bucketName,
|
|
832
|
+
Delete: {
|
|
833
|
+
Objects: batch.map((key) => ({
|
|
834
|
+
Key: key
|
|
835
|
+
})),
|
|
836
|
+
Quiet: false
|
|
837
|
+
}
|
|
838
|
+
})));
|
|
839
|
+
const failed = new Map(((_a3 = response.Errors) != null ? _a3 : []).map((e) => {
|
|
840
|
+
var _a4, _b;
|
|
841
|
+
return [
|
|
842
|
+
(_a4 = e.Key) != null ? _a4 : "",
|
|
843
|
+
(_b = e.Message) != null ? _b : "delete failed"
|
|
844
|
+
];
|
|
845
|
+
}));
|
|
846
|
+
return batch.map((key) => ({
|
|
847
|
+
key,
|
|
848
|
+
deleted: !failed.has(key),
|
|
849
|
+
error: failed.get(key)
|
|
850
|
+
}));
|
|
851
|
+
});
|
|
852
|
+
return batchResults.flat();
|
|
853
|
+
}
|
|
854
|
+
async list(options) {
|
|
855
|
+
var _a3;
|
|
856
|
+
const response = await this.send("list", () => this.client.send(new clientS3.ListObjectsV2Command({
|
|
857
|
+
Bucket: this.bucketName,
|
|
858
|
+
Prefix: options == null ? void 0 : options.prefix,
|
|
859
|
+
MaxKeys: options == null ? void 0 : options.limit,
|
|
860
|
+
ContinuationToken: options == null ? void 0 : options.continuationToken
|
|
861
|
+
})));
|
|
862
|
+
return {
|
|
863
|
+
items: ((_a3 = response.Contents) != null ? _a3 : []).map(toObjectSummary),
|
|
864
|
+
nextToken: response.NextContinuationToken
|
|
865
|
+
};
|
|
866
|
+
}
|
|
867
|
+
iterate(options) {
|
|
868
|
+
return __asyncGenerator(this, null, function* () {
|
|
869
|
+
let token;
|
|
870
|
+
do {
|
|
871
|
+
const page = yield new __await(this.list({
|
|
872
|
+
prefix: options == null ? void 0 : options.prefix,
|
|
873
|
+
limit: options == null ? void 0 : options.pageSize,
|
|
874
|
+
continuationToken: token
|
|
875
|
+
}));
|
|
876
|
+
yield* __yieldStar(page.items);
|
|
877
|
+
token = page.nextToken;
|
|
878
|
+
} while (token);
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
async listAll(options) {
|
|
882
|
+
const items = [];
|
|
883
|
+
try {
|
|
884
|
+
for (var iter = __forAwait(this.iterate(options)), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
|
|
885
|
+
const item = temp.value;
|
|
886
|
+
items.push(item);
|
|
887
|
+
}
|
|
888
|
+
} catch (temp) {
|
|
889
|
+
error = [temp];
|
|
890
|
+
} finally {
|
|
891
|
+
try {
|
|
892
|
+
more && (temp = iter.return) && await temp.call(iter);
|
|
893
|
+
} finally {
|
|
894
|
+
if (error)
|
|
895
|
+
throw error[0];
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
return items;
|
|
899
|
+
}
|
|
900
|
+
async headBucket() {
|
|
901
|
+
try {
|
|
902
|
+
await this.send("headBucket", () => this.client.send(new clientS3.HeadBucketCommand({
|
|
903
|
+
Bucket: this.bucketName
|
|
904
|
+
})));
|
|
905
|
+
return {
|
|
906
|
+
exists: true
|
|
907
|
+
};
|
|
908
|
+
} catch (error) {
|
|
909
|
+
if (error instanceof NotFoundError)
|
|
910
|
+
return {
|
|
911
|
+
exists: false
|
|
912
|
+
};
|
|
913
|
+
throw error;
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
getDownloadUrl(key, options) {
|
|
917
|
+
var _a3;
|
|
918
|
+
return s3RequestPresigner.getSignedUrl(this.client, new clientS3.GetObjectCommand({
|
|
919
|
+
Bucket: this.bucketName,
|
|
920
|
+
Key: key
|
|
921
|
+
}), {
|
|
922
|
+
expiresIn: (_a3 = options == null ? void 0 : options.expiresInSeconds) != null ? _a3 : DEFAULT_PRESIGN_SECONDS
|
|
923
|
+
});
|
|
924
|
+
}
|
|
925
|
+
async getUploadUrl(key, options) {
|
|
926
|
+
var _a3;
|
|
927
|
+
const url = await s3RequestPresigner.getSignedUrl(this.client, new clientS3.PutObjectCommand({
|
|
928
|
+
Bucket: this.bucketName,
|
|
929
|
+
Key: key,
|
|
930
|
+
ContentType: options == null ? void 0 : options.contentType
|
|
931
|
+
}), {
|
|
932
|
+
expiresIn: (_a3 = options == null ? void 0 : options.expiresInSeconds) != null ? _a3 : DEFAULT_PRESIGN_SECONDS
|
|
933
|
+
});
|
|
934
|
+
return {
|
|
935
|
+
key,
|
|
936
|
+
url
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
};
|
|
940
|
+
__name(_AwsS3Provider, "AwsS3Provider");
|
|
941
|
+
var AwsS3Provider = _AwsS3Provider;
|
|
942
|
+
var _a;
|
|
943
|
+
var AwsS3MultipartClient = (_a = class {
|
|
944
|
+
constructor(client, bucketName, retryPolicy, send) {
|
|
945
|
+
__publicField(this, "client");
|
|
946
|
+
__publicField(this, "bucketName");
|
|
947
|
+
__publicField(this, "retryPolicy");
|
|
948
|
+
__publicField(this, "send");
|
|
949
|
+
this.client = client;
|
|
950
|
+
this.bucketName = bucketName;
|
|
951
|
+
this.retryPolicy = retryPolicy;
|
|
952
|
+
this.send = send;
|
|
953
|
+
}
|
|
954
|
+
async create(key, options) {
|
|
955
|
+
const response = await this.send("multipart.create", () => this.client.send(new clientS3.CreateMultipartUploadCommand(__spreadValues({
|
|
956
|
+
Bucket: this.bucketName,
|
|
957
|
+
Key: key
|
|
958
|
+
}, mapUploadOptions(options)))));
|
|
959
|
+
return {
|
|
960
|
+
uploadId: response.UploadId
|
|
961
|
+
};
|
|
962
|
+
}
|
|
963
|
+
async uploadPart(key, uploadId, partNumber, body, contentLengthBytes) {
|
|
964
|
+
const response = await this.send("multipart.uploadPart", () => this.client.send(new clientS3.UploadPartCommand({
|
|
965
|
+
Bucket: this.bucketName,
|
|
966
|
+
Key: key,
|
|
967
|
+
UploadId: uploadId,
|
|
968
|
+
PartNumber: partNumber,
|
|
969
|
+
Body: body,
|
|
970
|
+
ContentLength: contentLengthBytes
|
|
971
|
+
})));
|
|
972
|
+
return {
|
|
973
|
+
partNumber,
|
|
974
|
+
etag: response.ETag
|
|
975
|
+
};
|
|
976
|
+
}
|
|
977
|
+
getPartUploadUrl(key, uploadId, partNumber, options) {
|
|
978
|
+
var _a3;
|
|
979
|
+
return s3RequestPresigner.getSignedUrl(this.client, new clientS3.UploadPartCommand({
|
|
980
|
+
Bucket: this.bucketName,
|
|
981
|
+
Key: key,
|
|
982
|
+
UploadId: uploadId,
|
|
983
|
+
PartNumber: partNumber
|
|
984
|
+
}), {
|
|
985
|
+
expiresIn: (_a3 = options == null ? void 0 : options.expiresInSeconds) != null ? _a3 : DEFAULT_PRESIGN_SECONDS
|
|
986
|
+
});
|
|
987
|
+
}
|
|
988
|
+
async complete(key, uploadId, parts) {
|
|
989
|
+
const response = await this.send("multipart.complete", () => this.client.send(new clientS3.CompleteMultipartUploadCommand({
|
|
990
|
+
Bucket: this.bucketName,
|
|
991
|
+
Key: key,
|
|
992
|
+
UploadId: uploadId,
|
|
993
|
+
MultipartUpload: {
|
|
994
|
+
Parts: [
|
|
995
|
+
...parts
|
|
996
|
+
].sort((a, b) => a.partNumber - b.partNumber).map((p) => ({
|
|
997
|
+
PartNumber: p.partNumber,
|
|
998
|
+
ETag: p.etag
|
|
999
|
+
}))
|
|
1000
|
+
}
|
|
1001
|
+
})));
|
|
1002
|
+
return {
|
|
1003
|
+
key,
|
|
1004
|
+
etag: response.ETag,
|
|
1005
|
+
versionId: response.VersionId
|
|
1006
|
+
};
|
|
1007
|
+
}
|
|
1008
|
+
async abort(key, uploadId) {
|
|
1009
|
+
await this.send("multipart.abort", () => this.client.send(new clientS3.AbortMultipartUploadCommand({
|
|
1010
|
+
Bucket: this.bucketName,
|
|
1011
|
+
Key: key,
|
|
1012
|
+
UploadId: uploadId
|
|
1013
|
+
})));
|
|
1014
|
+
}
|
|
1015
|
+
async listParts(key, uploadId) {
|
|
1016
|
+
var _a3;
|
|
1017
|
+
const response = await this.send("multipart.listParts", () => this.client.send(new clientS3.ListPartsCommand({
|
|
1018
|
+
Bucket: this.bucketName,
|
|
1019
|
+
Key: key,
|
|
1020
|
+
UploadId: uploadId
|
|
1021
|
+
})));
|
|
1022
|
+
return ((_a3 = response.Parts) != null ? _a3 : []).map((p) => ({
|
|
1023
|
+
partNumber: p.PartNumber,
|
|
1024
|
+
etag: p.ETag,
|
|
1025
|
+
size: p.Size,
|
|
1026
|
+
lastModified: p.LastModified
|
|
1027
|
+
}));
|
|
1028
|
+
}
|
|
1029
|
+
async listUploads(prefix) {
|
|
1030
|
+
var _a3;
|
|
1031
|
+
const response = await this.send("multipart.listUploads", () => this.client.send(new clientS3.ListMultipartUploadsCommand({
|
|
1032
|
+
Bucket: this.bucketName,
|
|
1033
|
+
Prefix: prefix
|
|
1034
|
+
})));
|
|
1035
|
+
return ((_a3 = response.Uploads) != null ? _a3 : []).map((u) => ({
|
|
1036
|
+
key: u.Key,
|
|
1037
|
+
uploadId: u.UploadId,
|
|
1038
|
+
initiatedAt: u.Initiated
|
|
1039
|
+
}));
|
|
1040
|
+
}
|
|
1041
|
+
}, __name(_a, "AwsS3MultipartClient"), _a);
|
|
1042
|
+
function mapUploadOptions(options) {
|
|
1043
|
+
return {
|
|
1044
|
+
ContentType: options == null ? void 0 : options.contentType,
|
|
1045
|
+
ContentEncoding: options == null ? void 0 : options.contentEncoding,
|
|
1046
|
+
CacheControl: options == null ? void 0 : options.cacheControl,
|
|
1047
|
+
ContentDisposition: options == null ? void 0 : options.contentDisposition,
|
|
1048
|
+
Metadata: options == null ? void 0 : options.metadata
|
|
1049
|
+
};
|
|
1050
|
+
}
|
|
1051
|
+
__name(mapUploadOptions, "mapUploadOptions");
|
|
1052
|
+
function toObjectSummary(content) {
|
|
1053
|
+
return {
|
|
1054
|
+
key: content.Key,
|
|
1055
|
+
size: content.Size,
|
|
1056
|
+
etag: content.ETag,
|
|
1057
|
+
lastModified: content.LastModified
|
|
1058
|
+
};
|
|
1059
|
+
}
|
|
1060
|
+
__name(toObjectSummary, "toObjectSummary");
|
|
1061
|
+
|
|
1062
|
+
// src/providers/azure-blob/config.ts
|
|
1063
|
+
var AZURE_BLOB_PROVIDER_ID = "azure_blob_storage";
|
|
1064
|
+
function isAzureBlobStorageOptions(options) {
|
|
1065
|
+
return options.provider === AZURE_BLOB_PROVIDER_ID;
|
|
1066
|
+
}
|
|
1067
|
+
__name(isAzureBlobStorageOptions, "isAzureBlobStorageOptions");
|
|
1068
|
+
function getDefaultAzureBlobOptionsFromEnv(env = process.env) {
|
|
1069
|
+
const connectionString = env.AZURE_DATALAKE_CONNECTION_STRING || env.AZURE_STORAGE_CONNECTION_STRING;
|
|
1070
|
+
if (!connectionString) {
|
|
1071
|
+
throw new ConfigurationError({
|
|
1072
|
+
provider: AZURE_BLOB_PROVIDER_ID,
|
|
1073
|
+
message: "Missing AZURE_DATALAKE_CONNECTION_STRING (or AZURE_STORAGE_CONNECTION_STRING) for azure_blob_storage provider"
|
|
1074
|
+
});
|
|
1075
|
+
}
|
|
1076
|
+
return {
|
|
1077
|
+
provider: AZURE_BLOB_PROVIDER_ID,
|
|
1078
|
+
auth: {
|
|
1079
|
+
kind: "connectionString",
|
|
1080
|
+
connectionString
|
|
1081
|
+
}
|
|
1082
|
+
};
|
|
1083
|
+
}
|
|
1084
|
+
__name(getDefaultAzureBlobOptionsFromEnv, "getDefaultAzureBlobOptionsFromEnv");
|
|
1085
|
+
registerProviderEnvDefaults(AZURE_BLOB_PROVIDER_ID, getDefaultAzureBlobOptionsFromEnv);
|
|
1086
|
+
var cache2 = new ClientCache();
|
|
1087
|
+
function getAzureBlobServiceClient(options) {
|
|
1088
|
+
const key = stableCacheKey({
|
|
1089
|
+
provider: options.provider,
|
|
1090
|
+
endpoint: options.endpoint,
|
|
1091
|
+
auth: authKeyParts2(options.auth)
|
|
1092
|
+
});
|
|
1093
|
+
return cache2.getOrCreate(key, () => buildClient2(options));
|
|
1094
|
+
}
|
|
1095
|
+
__name(getAzureBlobServiceClient, "getAzureBlobServiceClient");
|
|
1096
|
+
function clearAzureBlobClientCache() {
|
|
1097
|
+
cache2.clear();
|
|
1098
|
+
}
|
|
1099
|
+
__name(clearAzureBlobClientCache, "clearAzureBlobClientCache");
|
|
1100
|
+
function buildClient2(options) {
|
|
1101
|
+
const auth = options.auth;
|
|
1102
|
+
switch (auth.kind) {
|
|
1103
|
+
case "connectionString":
|
|
1104
|
+
return storageBlob.BlobServiceClient.fromConnectionString(auth.connectionString);
|
|
1105
|
+
case "accountKey":
|
|
1106
|
+
return new storageBlob.BlobServiceClient(endpointFor(options, auth.accountName), new storageBlob.StorageSharedKeyCredential(auth.accountName, auth.accountKey));
|
|
1107
|
+
case "sas":
|
|
1108
|
+
return new storageBlob.BlobServiceClient(`${endpointFor(options, auth.accountName)}?${auth.sasToken.replace(/^\?/, "")}`);
|
|
1109
|
+
case "workloadIdentity":
|
|
1110
|
+
return new storageBlob.BlobServiceClient(endpointFor(options, auth.accountName), new identity.DefaultAzureCredential());
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
__name(buildClient2, "buildClient");
|
|
1114
|
+
function endpointFor(options, accountName) {
|
|
1115
|
+
var _a3;
|
|
1116
|
+
return (_a3 = options.endpoint) != null ? _a3 : `https://${accountName}.blob.core.windows.net`;
|
|
1117
|
+
}
|
|
1118
|
+
__name(endpointFor, "endpointFor");
|
|
1119
|
+
function authKeyParts2(auth) {
|
|
1120
|
+
switch (auth.kind) {
|
|
1121
|
+
case "connectionString":
|
|
1122
|
+
return {
|
|
1123
|
+
kind: auth.kind,
|
|
1124
|
+
connection: hashSecret(auth.connectionString)
|
|
1125
|
+
};
|
|
1126
|
+
case "accountKey":
|
|
1127
|
+
return {
|
|
1128
|
+
kind: auth.kind,
|
|
1129
|
+
accountName: auth.accountName,
|
|
1130
|
+
key: hashSecret(auth.accountKey)
|
|
1131
|
+
};
|
|
1132
|
+
case "sas":
|
|
1133
|
+
return {
|
|
1134
|
+
kind: auth.kind,
|
|
1135
|
+
accountName: auth.accountName,
|
|
1136
|
+
token: hashSecret(auth.sasToken)
|
|
1137
|
+
};
|
|
1138
|
+
case "workloadIdentity":
|
|
1139
|
+
return {
|
|
1140
|
+
kind: auth.kind,
|
|
1141
|
+
accountName: auth.accountName
|
|
1142
|
+
};
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
__name(authKeyParts2, "authKeyParts");
|
|
1146
|
+
|
|
1147
|
+
// src/providers/azure-blob/errors.ts
|
|
1148
|
+
var AUTH_CODES = /* @__PURE__ */ new Set([
|
|
1149
|
+
"AuthenticationFailed",
|
|
1150
|
+
"AuthorizationFailure",
|
|
1151
|
+
"AuthorizationPermissionMismatch",
|
|
1152
|
+
"InsufficientAccountPermissions",
|
|
1153
|
+
"InvalidAuthenticationInfo"
|
|
1154
|
+
]);
|
|
1155
|
+
var NOT_FOUND_CODES = /* @__PURE__ */ new Set([
|
|
1156
|
+
"BlobNotFound",
|
|
1157
|
+
"ContainerNotFound",
|
|
1158
|
+
"ResourceNotFound"
|
|
1159
|
+
]);
|
|
1160
|
+
function mapAzureError(error) {
|
|
1161
|
+
var _a3, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
|
|
1162
|
+
if (error instanceof StorageError)
|
|
1163
|
+
return error;
|
|
1164
|
+
const azure = error != null ? error : {};
|
|
1165
|
+
const code = (_d = (_c = (_b = (_a3 = azure.details) == null ? void 0 : _a3.errorCode) != null ? _b : azure.code) != null ? _c : azure.name) != null ? _d : "UnknownError";
|
|
1166
|
+
const status = azure.statusCode;
|
|
1167
|
+
const base = {
|
|
1168
|
+
message: `${code}: ${(_e = azure.message) != null ? _e : String(error)}`,
|
|
1169
|
+
provider: AZURE_BLOB_PROVIDER_ID,
|
|
1170
|
+
requestId: (_h = (_g = (_f = azure.response) == null ? void 0 : _f.headers) == null ? void 0 : _g.get) == null ? void 0 : _h.call(_g, "x-ms-request-id"),
|
|
1171
|
+
cause: error
|
|
1172
|
+
};
|
|
1173
|
+
if (status === 429 || code === "ServerBusy") {
|
|
1174
|
+
const retryAfter = (_k = (_j = (_i = azure.response) == null ? void 0 : _i.headers) == null ? void 0 : _j.get) == null ? void 0 : _k.call(_j, "retry-after");
|
|
1175
|
+
return new ThrottledError(__spreadProps(__spreadValues({}, base), {
|
|
1176
|
+
code,
|
|
1177
|
+
retryAfterMs: retryAfter !== void 0 ? Number(retryAfter) * 1e3 : void 0
|
|
1178
|
+
}));
|
|
1179
|
+
}
|
|
1180
|
+
if (NOT_FOUND_CODES.has(code) || status === 404) {
|
|
1181
|
+
return new NotFoundError(__spreadProps(__spreadValues({}, base), {
|
|
1182
|
+
code
|
|
1183
|
+
}));
|
|
1184
|
+
}
|
|
1185
|
+
if (AUTH_CODES.has(code) || status === 403 || status === 401) {
|
|
1186
|
+
return new AuthError(__spreadProps(__spreadValues({}, base), {
|
|
1187
|
+
code
|
|
1188
|
+
}));
|
|
1189
|
+
}
|
|
1190
|
+
if (status !== void 0 && status >= 500) {
|
|
1191
|
+
return new NetworkError(__spreadProps(__spreadValues({}, base), {
|
|
1192
|
+
code
|
|
1193
|
+
}));
|
|
1194
|
+
}
|
|
1195
|
+
return new StorageError(__spreadProps(__spreadValues({}, base), {
|
|
1196
|
+
code,
|
|
1197
|
+
retryable: false
|
|
1198
|
+
}));
|
|
1199
|
+
}
|
|
1200
|
+
__name(mapAzureError, "mapAzureError");
|
|
1201
|
+
|
|
1202
|
+
// src/providers/azure-blob/multipartSessionStore.ts
|
|
1203
|
+
var DEFAULT_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
1204
|
+
var _InMemoryMultipartSessionStore = class _InMemoryMultipartSessionStore {
|
|
1205
|
+
constructor(ttlMs = DEFAULT_TTL_MS) {
|
|
1206
|
+
__publicField(this, "ttlMs");
|
|
1207
|
+
__publicField(this, "sessions");
|
|
1208
|
+
this.ttlMs = ttlMs;
|
|
1209
|
+
this.sessions = /* @__PURE__ */ new Map();
|
|
1210
|
+
}
|
|
1211
|
+
async create(session) {
|
|
1212
|
+
this.evictExpired();
|
|
1213
|
+
this.sessions.set(session.uploadId, session);
|
|
1214
|
+
}
|
|
1215
|
+
async get(uploadId) {
|
|
1216
|
+
const session = this.sessions.get(uploadId);
|
|
1217
|
+
if (!session)
|
|
1218
|
+
return void 0;
|
|
1219
|
+
if (this.isExpired(session)) {
|
|
1220
|
+
this.sessions.delete(uploadId);
|
|
1221
|
+
return void 0;
|
|
1222
|
+
}
|
|
1223
|
+
return session;
|
|
1224
|
+
}
|
|
1225
|
+
async appendBlock(uploadId, block) {
|
|
1226
|
+
const session = await this.get(uploadId);
|
|
1227
|
+
if (session)
|
|
1228
|
+
session.blocks.push(block);
|
|
1229
|
+
}
|
|
1230
|
+
async delete(uploadId) {
|
|
1231
|
+
this.sessions.delete(uploadId);
|
|
1232
|
+
}
|
|
1233
|
+
async list(containerName, blobNamePrefix) {
|
|
1234
|
+
this.evictExpired();
|
|
1235
|
+
return [
|
|
1236
|
+
...this.sessions.values()
|
|
1237
|
+
].filter((s) => s.containerName === containerName && (blobNamePrefix === void 0 || s.blobName.startsWith(blobNamePrefix)));
|
|
1238
|
+
}
|
|
1239
|
+
isExpired(session) {
|
|
1240
|
+
return Date.now() - session.createdAt > this.ttlMs;
|
|
1241
|
+
}
|
|
1242
|
+
evictExpired() {
|
|
1243
|
+
for (const [uploadId, session] of this.sessions) {
|
|
1244
|
+
if (this.isExpired(session))
|
|
1245
|
+
this.sessions.delete(uploadId);
|
|
1246
|
+
}
|
|
1247
|
+
}
|
|
1248
|
+
};
|
|
1249
|
+
__name(_InMemoryMultipartSessionStore, "InMemoryMultipartSessionStore");
|
|
1250
|
+
var InMemoryMultipartSessionStore = _InMemoryMultipartSessionStore;
|
|
1251
|
+
var DELETE_BATCH_SIZE2 = 256;
|
|
1252
|
+
var DEFAULT_PRESIGN_SECONDS2 = 3600;
|
|
1253
|
+
var PART_NUMBER_WIDTH = 5;
|
|
1254
|
+
var _AzureBlobStorageProvider = class _AzureBlobStorageProvider {
|
|
1255
|
+
constructor(config) {
|
|
1256
|
+
__publicField(this, "bucketName");
|
|
1257
|
+
__publicField(this, "multipart");
|
|
1258
|
+
__publicField(this, "serviceClient");
|
|
1259
|
+
__publicField(this, "container");
|
|
1260
|
+
__publicField(this, "retryPolicy");
|
|
1261
|
+
__publicField(this, "limiter");
|
|
1262
|
+
var _a3, _b, _c;
|
|
1263
|
+
const resolved = resolveObjectStorageOptions(config.options);
|
|
1264
|
+
if (!isAzureBlobStorageOptions(resolved)) {
|
|
1265
|
+
throw new ConfigurationError({
|
|
1266
|
+
message: `AzureBlobStorageProvider requires azure_blob_storage options, got provider "${resolved.provider}"`
|
|
1267
|
+
});
|
|
1268
|
+
}
|
|
1269
|
+
this.bucketName = config.bucketName;
|
|
1270
|
+
this.serviceClient = getAzureBlobServiceClient(resolved);
|
|
1271
|
+
this.container = this.serviceClient.getContainerClient(config.bucketName);
|
|
1272
|
+
this.retryPolicy = (_a3 = config.retry) != null ? _a3 : {};
|
|
1273
|
+
this.limiter = (_b = config.limiter) != null ? _b : createLimiter({
|
|
1274
|
+
max: 10,
|
|
1275
|
+
adaptive: true
|
|
1276
|
+
});
|
|
1277
|
+
this.multipart = new AzureBlobMultipartClient(this.container, this.bucketName, (_c = config.multipartStore) != null ? _c : new InMemoryMultipartSessionStore(), (operation, fn) => this.send(operation, fn));
|
|
1278
|
+
}
|
|
1279
|
+
send(operation, fn) {
|
|
1280
|
+
return instrument(AZURE_BLOB_PROVIDER_ID, operation, () => withRetry(() => fn().catch((error) => Promise.reject(mapAzureError(error))), this.retryPolicy));
|
|
1281
|
+
}
|
|
1282
|
+
block(key) {
|
|
1283
|
+
return this.container.getBlockBlobClient(key);
|
|
1284
|
+
}
|
|
1285
|
+
async getObject(key, options) {
|
|
1286
|
+
const { offset, count } = await this.resolveRange(key, options == null ? void 0 : options.range);
|
|
1287
|
+
const response = await this.send("getObject", () => this.block(key).download(offset, count));
|
|
1288
|
+
if (!response.readableStreamBody) {
|
|
1289
|
+
throw new NotFoundError({
|
|
1290
|
+
message: `Empty body downloading "${key}"`,
|
|
1291
|
+
provider: "azure_blob_storage"
|
|
1292
|
+
});
|
|
1293
|
+
}
|
|
1294
|
+
return {
|
|
1295
|
+
body: response.readableStreamBody,
|
|
1296
|
+
contentLength: response.contentLength,
|
|
1297
|
+
contentType: response.contentType,
|
|
1298
|
+
contentEncoding: response.contentEncoding,
|
|
1299
|
+
contentDisposition: response.contentDisposition,
|
|
1300
|
+
contentLanguage: response.contentLanguage,
|
|
1301
|
+
cacheControl: response.cacheControl,
|
|
1302
|
+
etag: response.etag,
|
|
1303
|
+
lastModified: response.lastModified,
|
|
1304
|
+
metadata: response.metadata
|
|
1305
|
+
};
|
|
1306
|
+
}
|
|
1307
|
+
async resolveRange(key, rangeInput) {
|
|
1308
|
+
var _a3;
|
|
1309
|
+
if (!rangeInput)
|
|
1310
|
+
return {
|
|
1311
|
+
offset: 0
|
|
1312
|
+
};
|
|
1313
|
+
const range = parseRange(rangeInput);
|
|
1314
|
+
if (range.suffixLength !== void 0) {
|
|
1315
|
+
const props = await this.getObjectProperties(key);
|
|
1316
|
+
const total = (_a3 = props.contentLength) != null ? _a3 : 0;
|
|
1317
|
+
const count2 = Math.min(range.suffixLength, total);
|
|
1318
|
+
return {
|
|
1319
|
+
offset: total - count2,
|
|
1320
|
+
count: count2
|
|
1321
|
+
};
|
|
1322
|
+
}
|
|
1323
|
+
const start = range.start;
|
|
1324
|
+
const count = range.end !== void 0 ? range.end - start + 1 : void 0;
|
|
1325
|
+
return {
|
|
1326
|
+
offset: start,
|
|
1327
|
+
count
|
|
1328
|
+
};
|
|
1329
|
+
}
|
|
1330
|
+
async getObjectProperties(key) {
|
|
1331
|
+
const response = await this.send("getObjectProperties", () => this.block(key).getProperties());
|
|
1332
|
+
return {
|
|
1333
|
+
contentLength: response.contentLength,
|
|
1334
|
+
contentType: response.contentType,
|
|
1335
|
+
etag: response.etag,
|
|
1336
|
+
lastModified: response.lastModified,
|
|
1337
|
+
metadata: response.metadata
|
|
1338
|
+
};
|
|
1339
|
+
}
|
|
1340
|
+
async upload(key, body, options) {
|
|
1341
|
+
const blob = this.block(key);
|
|
1342
|
+
const headers = mapHttpHeaders(options);
|
|
1343
|
+
if (body instanceof stream.Readable) {
|
|
1344
|
+
const response2 = await this.send("upload", () => blob.uploadStream(body, options == null ? void 0 : options.partSizeBytes, options == null ? void 0 : options.queueSize, {
|
|
1345
|
+
blobHTTPHeaders: headers,
|
|
1346
|
+
metadata: options == null ? void 0 : options.metadata
|
|
1347
|
+
}));
|
|
1348
|
+
return {
|
|
1349
|
+
key,
|
|
1350
|
+
etag: response2.etag
|
|
1351
|
+
};
|
|
1352
|
+
}
|
|
1353
|
+
const data = typeof body === "string" ? Buffer.from(body) : Buffer.from(body);
|
|
1354
|
+
const response = await this.send("upload", () => blob.upload(data, data.length, {
|
|
1355
|
+
blobHTTPHeaders: headers,
|
|
1356
|
+
metadata: options == null ? void 0 : options.metadata
|
|
1357
|
+
}));
|
|
1358
|
+
return {
|
|
1359
|
+
key,
|
|
1360
|
+
etag: response.etag
|
|
1361
|
+
};
|
|
1362
|
+
}
|
|
1363
|
+
createWriteStream(key, options) {
|
|
1364
|
+
const stream$1 = new stream.PassThrough();
|
|
1365
|
+
let aborted = false;
|
|
1366
|
+
const done = this.send("createWriteStream", () => this.block(key).uploadStream(stream$1, options == null ? void 0 : options.partSizeBytes, options == null ? void 0 : options.queueSize, {
|
|
1367
|
+
blobHTTPHeaders: mapHttpHeaders(options),
|
|
1368
|
+
metadata: options == null ? void 0 : options.metadata
|
|
1369
|
+
})).then((response) => ({
|
|
1370
|
+
key,
|
|
1371
|
+
etag: response.etag
|
|
1372
|
+
}));
|
|
1373
|
+
return {
|
|
1374
|
+
stream: stream$1,
|
|
1375
|
+
done,
|
|
1376
|
+
abort: async () => {
|
|
1377
|
+
if (!aborted) {
|
|
1378
|
+
aborted = true;
|
|
1379
|
+
stream$1.destroy(new Error("upload aborted"));
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
};
|
|
1383
|
+
}
|
|
1384
|
+
async delete(key) {
|
|
1385
|
+
await this.send("delete", () => this.block(key).delete());
|
|
1386
|
+
}
|
|
1387
|
+
async deleteMany(keys) {
|
|
1388
|
+
const batchResults = await mapBatched(keys, DELETE_BATCH_SIZE2, this.limiter, async (batch) => Promise.all(batch.map(async (key) => {
|
|
1389
|
+
try {
|
|
1390
|
+
await this.delete(key);
|
|
1391
|
+
return {
|
|
1392
|
+
key,
|
|
1393
|
+
deleted: true
|
|
1394
|
+
};
|
|
1395
|
+
} catch (error) {
|
|
1396
|
+
return {
|
|
1397
|
+
key,
|
|
1398
|
+
deleted: false,
|
|
1399
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1400
|
+
};
|
|
1401
|
+
}
|
|
1402
|
+
})));
|
|
1403
|
+
return batchResults.flat();
|
|
1404
|
+
}
|
|
1405
|
+
async list(options) {
|
|
1406
|
+
var _a3, _b;
|
|
1407
|
+
const iterator = this.container.listBlobsFlat({
|
|
1408
|
+
prefix: options == null ? void 0 : options.prefix
|
|
1409
|
+
}).byPage({
|
|
1410
|
+
maxPageSize: options == null ? void 0 : options.limit,
|
|
1411
|
+
continuationToken: (options == null ? void 0 : options.continuationToken) || void 0
|
|
1412
|
+
});
|
|
1413
|
+
const page = await this.send("list", async () => (await iterator.next()).value);
|
|
1414
|
+
const items = ((_b = (_a3 = page == null ? void 0 : page.segment) == null ? void 0 : _a3.blobItems) != null ? _b : []).map((blob) => ({
|
|
1415
|
+
key: blob.name,
|
|
1416
|
+
size: blob.properties.contentLength,
|
|
1417
|
+
etag: blob.properties.etag,
|
|
1418
|
+
lastModified: blob.properties.lastModified
|
|
1419
|
+
}));
|
|
1420
|
+
return {
|
|
1421
|
+
items,
|
|
1422
|
+
nextToken: (page == null ? void 0 : page.continuationToken) || void 0
|
|
1423
|
+
};
|
|
1424
|
+
}
|
|
1425
|
+
iterate(options) {
|
|
1426
|
+
return __asyncGenerator(this, null, function* () {
|
|
1427
|
+
let token;
|
|
1428
|
+
do {
|
|
1429
|
+
const page = yield new __await(this.list({
|
|
1430
|
+
prefix: options == null ? void 0 : options.prefix,
|
|
1431
|
+
limit: options == null ? void 0 : options.pageSize,
|
|
1432
|
+
continuationToken: token
|
|
1433
|
+
}));
|
|
1434
|
+
yield* __yieldStar(page.items);
|
|
1435
|
+
token = page.nextToken;
|
|
1436
|
+
} while (token);
|
|
1437
|
+
});
|
|
1438
|
+
}
|
|
1439
|
+
async listAll(options) {
|
|
1440
|
+
const items = [];
|
|
1441
|
+
try {
|
|
1442
|
+
for (var iter = __forAwait(this.iterate(options)), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
|
|
1443
|
+
const item = temp.value;
|
|
1444
|
+
items.push(item);
|
|
1445
|
+
}
|
|
1446
|
+
} catch (temp) {
|
|
1447
|
+
error = [temp];
|
|
1448
|
+
} finally {
|
|
1449
|
+
try {
|
|
1450
|
+
more && (temp = iter.return) && await temp.call(iter);
|
|
1451
|
+
} finally {
|
|
1452
|
+
if (error)
|
|
1453
|
+
throw error[0];
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
return items;
|
|
1457
|
+
}
|
|
1458
|
+
async headBucket() {
|
|
1459
|
+
try {
|
|
1460
|
+
const response = await this.send("headBucket", () => this.container.getProperties());
|
|
1461
|
+
return {
|
|
1462
|
+
exists: true,
|
|
1463
|
+
metadata: response.metadata
|
|
1464
|
+
};
|
|
1465
|
+
} catch (error) {
|
|
1466
|
+
if (error instanceof NotFoundError)
|
|
1467
|
+
return {
|
|
1468
|
+
exists: false
|
|
1469
|
+
};
|
|
1470
|
+
throw error;
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
async getDownloadUrl(key, options) {
|
|
1474
|
+
return this.generateSasUrl(key, "r", options);
|
|
1475
|
+
}
|
|
1476
|
+
async getUploadUrl(key, options) {
|
|
1477
|
+
const url = await this.generateSasUrl(key, "cw", options);
|
|
1478
|
+
return {
|
|
1479
|
+
key,
|
|
1480
|
+
url
|
|
1481
|
+
};
|
|
1482
|
+
}
|
|
1483
|
+
async generateSasUrl(key, permissions, options) {
|
|
1484
|
+
var _a3;
|
|
1485
|
+
const expiresInSeconds = (_a3 = options == null ? void 0 : options.expiresInSeconds) != null ? _a3 : DEFAULT_PRESIGN_SECONDS2;
|
|
1486
|
+
try {
|
|
1487
|
+
return await this.block(key).generateSasUrl({
|
|
1488
|
+
permissions: storageBlob.BlobSASPermissions.parse(permissions),
|
|
1489
|
+
expiresOn: new Date(Date.now() + expiresInSeconds * 1e3)
|
|
1490
|
+
});
|
|
1491
|
+
} catch (error) {
|
|
1492
|
+
throw new ConfigurationError({
|
|
1493
|
+
message: "SAS generation requires shared key credentials (connectionString or accountKey auth)",
|
|
1494
|
+
provider: "azure_blob_storage",
|
|
1495
|
+
cause: error
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
};
|
|
1500
|
+
__name(_AzureBlobStorageProvider, "AzureBlobStorageProvider");
|
|
1501
|
+
var AzureBlobStorageProvider = _AzureBlobStorageProvider;
|
|
1502
|
+
var _a2;
|
|
1503
|
+
var AzureBlobMultipartClient = (_a2 = class {
|
|
1504
|
+
constructor(container, containerName, store, send) {
|
|
1505
|
+
__publicField(this, "container");
|
|
1506
|
+
__publicField(this, "containerName");
|
|
1507
|
+
__publicField(this, "store");
|
|
1508
|
+
__publicField(this, "send");
|
|
1509
|
+
this.container = container;
|
|
1510
|
+
this.containerName = containerName;
|
|
1511
|
+
this.store = store;
|
|
1512
|
+
this.send = send;
|
|
1513
|
+
}
|
|
1514
|
+
block(key) {
|
|
1515
|
+
return this.container.getBlockBlobClient(key);
|
|
1516
|
+
}
|
|
1517
|
+
blockIdFor(uploadId, partNumber) {
|
|
1518
|
+
return Buffer.from(`${uploadId}:${String(partNumber).padStart(PART_NUMBER_WIDTH, "0")}`).toString("base64");
|
|
1519
|
+
}
|
|
1520
|
+
async create(key) {
|
|
1521
|
+
const uploadId = crypto.randomUUID();
|
|
1522
|
+
await this.store.create({
|
|
1523
|
+
uploadId,
|
|
1524
|
+
blobName: key,
|
|
1525
|
+
containerName: this.containerName,
|
|
1526
|
+
blocks: [],
|
|
1527
|
+
createdAt: Date.now()
|
|
1528
|
+
});
|
|
1529
|
+
return {
|
|
1530
|
+
uploadId
|
|
1531
|
+
};
|
|
1532
|
+
}
|
|
1533
|
+
async session(uploadId) {
|
|
1534
|
+
const session = await this.store.get(uploadId);
|
|
1535
|
+
if (!session) {
|
|
1536
|
+
throw new MultipartSessionError({
|
|
1537
|
+
message: `Unknown or expired multipart uploadId "${uploadId}"`,
|
|
1538
|
+
provider: "azure_blob_storage"
|
|
1539
|
+
});
|
|
1540
|
+
}
|
|
1541
|
+
return session;
|
|
1542
|
+
}
|
|
1543
|
+
async uploadPart(key, uploadId, partNumber, body, contentLengthBytes) {
|
|
1544
|
+
await this.session(uploadId);
|
|
1545
|
+
const blockId = this.blockIdFor(uploadId, partNumber);
|
|
1546
|
+
const { data, length } = resolveBlockBody(body, contentLengthBytes);
|
|
1547
|
+
await this.send("multipart.uploadPart", () => this.block(key).stageBlock(blockId, data, length));
|
|
1548
|
+
await this.store.appendBlock(uploadId, {
|
|
1549
|
+
blockId,
|
|
1550
|
+
partNumber
|
|
1551
|
+
});
|
|
1552
|
+
return {
|
|
1553
|
+
partNumber,
|
|
1554
|
+
etag: blockId
|
|
1555
|
+
};
|
|
1556
|
+
}
|
|
1557
|
+
async getPartUploadUrl() {
|
|
1558
|
+
throw new ConfigurationError({
|
|
1559
|
+
message: "Azure presigned part upload not supported yet; upload parts server-side via multipart.uploadPart",
|
|
1560
|
+
provider: "azure_blob_storage"
|
|
1561
|
+
});
|
|
1562
|
+
}
|
|
1563
|
+
async complete(key, uploadId, parts) {
|
|
1564
|
+
await this.session(uploadId);
|
|
1565
|
+
const blockIds = [
|
|
1566
|
+
...parts
|
|
1567
|
+
].sort((a, b) => a.partNumber - b.partNumber).map((p) => p.etag);
|
|
1568
|
+
const response = await this.send("multipart.complete", () => this.block(key).commitBlockList(blockIds));
|
|
1569
|
+
await this.store.delete(uploadId);
|
|
1570
|
+
return {
|
|
1571
|
+
key,
|
|
1572
|
+
etag: response.etag
|
|
1573
|
+
};
|
|
1574
|
+
}
|
|
1575
|
+
async abort(_key, uploadId) {
|
|
1576
|
+
await this.store.delete(uploadId);
|
|
1577
|
+
}
|
|
1578
|
+
async listParts(_key, uploadId) {
|
|
1579
|
+
const session = await this.session(uploadId);
|
|
1580
|
+
return session.blocks.map((b) => ({
|
|
1581
|
+
partNumber: b.partNumber,
|
|
1582
|
+
etag: b.blockId
|
|
1583
|
+
}));
|
|
1584
|
+
}
|
|
1585
|
+
async listUploads(prefix) {
|
|
1586
|
+
const sessions = await this.store.list(this.containerName, prefix);
|
|
1587
|
+
return sessions.map((s) => ({
|
|
1588
|
+
key: s.blobName,
|
|
1589
|
+
uploadId: s.uploadId,
|
|
1590
|
+
initiatedAt: new Date(s.createdAt)
|
|
1591
|
+
}));
|
|
1592
|
+
}
|
|
1593
|
+
}, __name(_a2, "AzureBlobMultipartClient"), _a2);
|
|
1594
|
+
function resolveBlockBody(body, contentLengthBytes) {
|
|
1595
|
+
if (body instanceof stream.Readable) {
|
|
1596
|
+
if (contentLengthBytes === void 0) {
|
|
1597
|
+
throw new ValidationError({
|
|
1598
|
+
message: "contentLengthBytes is required when uploading a part from a stream",
|
|
1599
|
+
provider: "azure_blob_storage"
|
|
1600
|
+
});
|
|
1601
|
+
}
|
|
1602
|
+
return {
|
|
1603
|
+
data: body,
|
|
1604
|
+
length: contentLengthBytes
|
|
1605
|
+
};
|
|
1606
|
+
}
|
|
1607
|
+
const data = typeof body === "string" ? Buffer.from(body) : Buffer.from(body);
|
|
1608
|
+
return {
|
|
1609
|
+
data,
|
|
1610
|
+
length: data.length
|
|
1611
|
+
};
|
|
1612
|
+
}
|
|
1613
|
+
__name(resolveBlockBody, "resolveBlockBody");
|
|
1614
|
+
function mapHttpHeaders(options) {
|
|
1615
|
+
return {
|
|
1616
|
+
blobContentType: options == null ? void 0 : options.contentType,
|
|
1617
|
+
blobContentEncoding: options == null ? void 0 : options.contentEncoding,
|
|
1618
|
+
blobCacheControl: options == null ? void 0 : options.cacheControl,
|
|
1619
|
+
blobContentDisposition: options == null ? void 0 : options.contentDisposition,
|
|
1620
|
+
blobContentLanguage: void 0
|
|
1621
|
+
};
|
|
1622
|
+
}
|
|
1623
|
+
__name(mapHttpHeaders, "mapHttpHeaders");
|
|
1624
|
+
var cache3 = new ClientCache();
|
|
1625
|
+
function getObjectStorageClient(bucket, options) {
|
|
1626
|
+
const resolved = resolveObjectStorageOptions(options);
|
|
1627
|
+
const optionsHash = crypto.createHash("sha256").update(stableCacheKey(resolved)).digest("hex");
|
|
1628
|
+
const key = `${resolved.provider}:${bucket}:${optionsHash}`;
|
|
1629
|
+
return cache3.getOrCreate(key, () => buildProvider(bucket, resolved));
|
|
1630
|
+
}
|
|
1631
|
+
__name(getObjectStorageClient, "getObjectStorageClient");
|
|
1632
|
+
function buildProvider(bucket, options) {
|
|
1633
|
+
var _a3;
|
|
1634
|
+
if (isAwsS3StorageOptions(options)) {
|
|
1635
|
+
return new AwsS3Provider({
|
|
1636
|
+
bucketName: bucket,
|
|
1637
|
+
options
|
|
1638
|
+
});
|
|
1639
|
+
}
|
|
1640
|
+
if (isAzureBlobStorageOptions(options)) {
|
|
1641
|
+
return new AzureBlobStorageProvider({
|
|
1642
|
+
bucketName: bucket,
|
|
1643
|
+
options
|
|
1644
|
+
});
|
|
1645
|
+
}
|
|
1646
|
+
throw new ConfigurationError({
|
|
1647
|
+
message: `Unsupported object storage provider: "${(_a3 = options == null ? void 0 : options.provider) != null ? _a3 : "unknown"}"`
|
|
1648
|
+
});
|
|
1649
|
+
}
|
|
1650
|
+
__name(buildProvider, "buildProvider");
|
|
1651
|
+
function clearObjectStorageClientCache() {
|
|
1652
|
+
cache3.clear();
|
|
1653
|
+
}
|
|
1654
|
+
__name(clearObjectStorageClientCache, "clearObjectStorageClientCache");
|
|
1655
|
+
|
|
1656
|
+
// src/index-v2.ts
|
|
1657
|
+
var OBJECT_STORAGE_PROVIDERS = Object.freeze({
|
|
1658
|
+
AWS_S3: AWS_S3_PROVIDER_ID,
|
|
1659
|
+
AZURE_BLOB_STORAGE: AZURE_BLOB_PROVIDER_ID
|
|
1660
|
+
});
|
|
1661
|
+
|
|
1662
|
+
exports.AWS_S3_PROVIDER_ID = AWS_S3_PROVIDER_ID;
|
|
1663
|
+
exports.AZURE_BLOB_PROVIDER_ID = AZURE_BLOB_PROVIDER_ID;
|
|
1664
|
+
exports.AuthError = AuthError;
|
|
1665
|
+
exports.AwsS3Provider = AwsS3Provider;
|
|
1666
|
+
exports.AzureBlobStorageProvider = AzureBlobStorageProvider;
|
|
1667
|
+
exports.ConfigurationError = ConfigurationError;
|
|
1668
|
+
exports.InMemoryMultipartSessionStore = InMemoryMultipartSessionStore;
|
|
1669
|
+
exports.IntegrityError = IntegrityError;
|
|
1670
|
+
exports.MultipartSessionError = MultipartSessionError;
|
|
1671
|
+
exports.NetworkError = NetworkError;
|
|
1672
|
+
exports.NotFoundError = NotFoundError;
|
|
1673
|
+
exports.OBJECT_STORAGE_PROVIDERS = OBJECT_STORAGE_PROVIDERS;
|
|
1674
|
+
exports.StorageError = StorageError;
|
|
1675
|
+
exports.ThrottledError = ThrottledError;
|
|
1676
|
+
exports.ValidationError = ValidationError;
|
|
1677
|
+
exports.chunk = chunk;
|
|
1678
|
+
exports.clearAwsS3ClientCache = clearAwsS3ClientCache;
|
|
1679
|
+
exports.clearAzureBlobClientCache = clearAzureBlobClientCache;
|
|
1680
|
+
exports.clearObjectStorageClientCache = clearObjectStorageClientCache;
|
|
1681
|
+
exports.createLimiter = createLimiter;
|
|
1682
|
+
exports.getAwsS3Client = getAwsS3Client;
|
|
1683
|
+
exports.getAzureBlobServiceClient = getAzureBlobServiceClient;
|
|
1684
|
+
exports.getDefaultAwsS3OptionsFromEnv = getDefaultAwsS3OptionsFromEnv;
|
|
1685
|
+
exports.getDefaultAzureBlobOptionsFromEnv = getDefaultAzureBlobOptionsFromEnv;
|
|
1686
|
+
exports.getObjectStorageClient = getObjectStorageClient;
|
|
1687
|
+
exports.isAwsS3StorageOptions = isAwsS3StorageOptions;
|
|
1688
|
+
exports.isAzureBlobStorageOptions = isAzureBlobStorageOptions;
|
|
1689
|
+
exports.isRetryableError = isRetryableError;
|
|
1690
|
+
exports.mapAwsError = mapAwsError;
|
|
1691
|
+
exports.mapAzureError = mapAzureError;
|
|
1692
|
+
exports.mapBatched = mapBatched;
|
|
1693
|
+
exports.parseRange = parseRange;
|
|
1694
|
+
exports.resolveObjectStorageOptions = resolveObjectStorageOptions;
|
|
1695
|
+
exports.resolveOptionsFromEnv = resolveOptionsFromEnv;
|
|
1696
|
+
exports.toRangeHeader = toRangeHeader;
|
|
1697
|
+
exports.withRetry = withRetry;
|
|
1698
|
+
//# sourceMappingURL=out.js.map
|
|
1699
|
+
//# sourceMappingURL=index-v2.js.map
|