@lossless.org/client 1.23.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +2 -2
- package/dist_ts/objectstorage/classes.bucket.d.ts +11 -21
- package/dist_ts/objectstorage/classes.bucket.js +58 -29
- package/dist_ts/objectstorage/classes.connection.d.ts +2 -1
- package/dist_ts/objectstorage/classes.connection.js +3 -2
- package/dist_ts/objectstorage/classes.directory.js +9 -1
- package/dist_ts/objectstorage/classes.requestdeadlineerror.d.ts +11 -0
- package/dist_ts/objectstorage/classes.requestdeadlineerror.js +16 -0
- package/dist_ts/objectstorage/classes.smartbucket.d.ts +11 -2
- package/dist_ts/objectstorage/classes.smartbucket.js +20 -1
- package/dist_ts/objectstorage/helpers.d.ts +4 -0
- package/dist_ts/objectstorage/helpers.js +13 -1
- package/dist_ts/objectstorage/index.d.ts +1 -0
- package/dist_ts/objectstorage/index.js +2 -1
- package/dist_ts/objectstorage/interfaces.d.ts +49 -0
- package/dist_ts/objectstorage/internal.exactupload.capability.js +4 -2
- package/dist_ts/objectstorage/internal.requestbounds.d.ts +37 -0
- package/dist_ts/objectstorage/internal.requestbounds.js +154 -0
- package/dist_ts/objectstorage/plugins.d.ts +2 -1
- package/dist_ts/objectstorage/plugins.js +3 -2
- package/package.json +2 -2
- package/readme.md +6 -0
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/objectstorage/classes.bucket.ts +80 -56
- package/ts/objectstorage/classes.connection.ts +4 -2
- package/ts/objectstorage/classes.directory.ts +8 -0
- package/ts/objectstorage/classes.requestdeadlineerror.ts +16 -0
- package/ts/objectstorage/classes.smartbucket.ts +30 -1
- package/ts/objectstorage/helpers.ts +16 -0
- package/ts/objectstorage/index.ts +1 -0
- package/ts/objectstorage/interfaces.ts +58 -0
- package/ts/objectstorage/internal.exactupload.capability.ts +3 -1
- package/ts/objectstorage/internal.requestbounds.ts +192 -0
- package/ts/objectstorage/plugins.ts +2 -1
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import * as crypto from 'node:crypto';
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import * as stream from 'node:stream';
|
|
4
|
-
|
|
4
|
+
import * as nodeUtil from 'node:util';
|
|
5
|
+
export { crypto, nodeUtil, path, stream };
|
|
5
6
|
import * as smartmime from '@push.rocks/smartmime';
|
|
6
7
|
import * as smartpath from '@push.rocks/smartpath';
|
|
7
8
|
import * as smartpromise from '@push.rocks/smartpromise';
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
import * as crypto from 'node:crypto';
|
|
4
4
|
import * as path from 'node:path';
|
|
5
5
|
import * as stream from 'node:stream';
|
|
6
|
-
|
|
6
|
+
import * as nodeUtil from 'node:util';
|
|
7
|
+
export { crypto, nodeUtil, path, stream };
|
|
7
8
|
// @push.rocks scope
|
|
8
9
|
import * as smartmime from '@push.rocks/smartmime';
|
|
9
10
|
import * as smartpath from '@push.rocks/smartpath';
|
|
@@ -20,4 +21,4 @@ export { tsclass, };
|
|
|
20
21
|
import * as s3 from '@aws-sdk/client-s3';
|
|
21
22
|
import { Minimatch } from 'minimatch';
|
|
22
23
|
export { s3, Minimatch, };
|
|
23
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
24
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGx1Z2lucy5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3RzL29iamVjdHN0b3JhZ2UvcGx1Z2lucy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxhQUFhO0FBRWIsY0FBYztBQUNkLE9BQU8sS0FBSyxNQUFNLE1BQU0sYUFBYSxDQUFDO0FBQ3RDLE9BQU8sS0FBSyxJQUFJLE1BQU0sV0FBVyxDQUFDO0FBQ2xDLE9BQU8sS0FBSyxNQUFNLE1BQU0sYUFBYSxDQUFDO0FBQ3RDLE9BQU8sS0FBSyxRQUFRLE1BQU0sV0FBVyxDQUFDO0FBRXRDLE9BQU8sRUFBRSxNQUFNLEVBQUUsUUFBUSxFQUFFLElBQUksRUFBRSxNQUFNLEVBQUUsQ0FBQztBQUUxQyxvQkFBb0I7QUFDcEIsT0FBTyxLQUFLLFNBQVMsTUFBTSx1QkFBdUIsQ0FBQztBQUNuRCxPQUFPLEtBQUssU0FBUyxNQUFNLHVCQUF1QixDQUFDO0FBQ25ELE9BQU8sS0FBSyxZQUFZLE1BQU0sMEJBQTBCLENBQUM7QUFDekQsT0FBTyxLQUFLLE9BQU8sTUFBTSxxQkFBcUIsQ0FBQztBQUMvQyxPQUFPLEtBQUssV0FBVyxNQUFNLHlCQUF5QixDQUFDO0FBQ3ZELE9BQU8sS0FBSyxXQUFXLE1BQU0seUJBQXlCLENBQUM7QUFDdkQsT0FBTyxLQUFLLFdBQVcsTUFBTSx5QkFBeUIsQ0FBQztBQUV2RCxPQUFPLEVBQUUsU0FBUyxFQUFFLFNBQVMsRUFBRSxZQUFZLEVBQUUsT0FBTyxFQUFFLFdBQVcsRUFBRSxXQUFXLEVBQUUsV0FBVyxFQUFFLENBQUM7QUFFOUYsV0FBVztBQUNYLE9BQU8sS0FBSyxPQUFPLE1BQU0sa0JBQWtCLENBQUM7QUFFNUMsT0FBTyxFQUNMLE9BQU8sR0FDUixDQUFBO0FBRUQsb0JBQW9CO0FBQ3BCLE9BQU8sS0FBSyxFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDekMsT0FBTyxFQUFFLFNBQVMsRUFBRSxNQUFNLFdBQVcsQ0FBQztBQUV0QyxPQUFPLEVBQ0wsRUFBRSxFQUNGLFNBQVMsR0FDVixDQUFBIn0=
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lossless.org/client",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"registry": "https://verdaccio.lossless.digital"
|
|
@@ -86,7 +86,7 @@
|
|
|
86
86
|
"@git.zone/tsbuild": "6.0.2",
|
|
87
87
|
"@git.zone/tsbundle": "3.2.1",
|
|
88
88
|
"@git.zone/tsrun": "3.0.3",
|
|
89
|
-
"@git.zone/tstest": "7.2.
|
|
89
|
+
"@git.zone/tstest": "7.2.1",
|
|
90
90
|
"@lossless.org/nosqldb": "8.0.2",
|
|
91
91
|
"@lossless.org/nosqldb-qualified": "npm:@lossless.org/nosqldb@10.9.0",
|
|
92
92
|
"@lossless.org/objectstorage": "10.3.1",
|
package/readme.md
CHANGED
|
@@ -137,6 +137,8 @@ Family imports preserve the established SmartData and SmartBucket names and cons
|
|
|
137
137
|
| Bucket lookup for bucket-scoped credentials | `smartBucket.getBucketByName(name, { signal? })` and `smartBucket.bucketExists(name, { signal? })` send one HeadBucket; `createBucket(name, { signal? })` and `removeBucket(name, { signal? })` take the same options |
|
|
138
138
|
| Byte-range downloads | `bucket.fastGetRangeStream({ path, range: { start, end? }, signal? }, typeArg?)` — one ranged GetObject resolving `{ stream, range, size }`; range failures reject with `ObjectRangeError` code `invalid_argument`, `range_not_satisfiable` or `invalid_response` |
|
|
139
139
|
| Reads of objects that may be missing | `bucket.fastGetIfPresent({ path, signal? })` — one GetObject resolving the object's bytes, or `null` when the key does not exist |
|
|
140
|
+
| Bounded requests | `new SmartBucket(config, { connectionTimeoutMs?, requestTimeoutMs?, maxAttempts? })` and the same options on `new ObjectStorageConnection({ ... })` — every request of the client a SmartBucket builds ends within `smartBucket.requestBounds.requestDeadlineMs`, retries included, or rejects with `ObjectStorageRequestDeadlineError` |
|
|
141
|
+
| Signals on writes, moves and listings | `bucket.fastPut({ ..., signal? })`, `bucket.fastMove({ ..., signal? })`, `bucket.listObjectKeysPage({ ..., signal? })`, `bucket.listAllObjects(prefix?, { signal? })` and `bucket.listAllObjectsArray(prefix?, { signal? })`; `fastGet`, `fastPutStream`, `fastCopy` and `fastRemove` already took one |
|
|
140
142
|
|
|
141
143
|
A bucket lookup sends one HeadBucket for the named bucket, never ListBuckets, so a credential scoped to its bucket can open it without the account-wide list permission, and a lookup does not depend on which buckets a listing returns. A missing bucket (404) keeps its answer: `getBucketByName()` rejects with `Bucket '<name>' not found.` and `bucketExists()` resolves `false`. A bucket the credentials may not access (403) rejects both with an error that names the denial and keeps the provider error as `cause`; S3 does not say whether such a bucket exists. Any other provider failure is rethrown unchanged. `signal` aborts the in-flight request and rejects with the signal's reason. Lookups, creations and removals run as lifecycle-managed operations: `SmartBucket.close()` aborts and awaits them, and the storage client cannot be replaced while one is active.
|
|
142
144
|
|
|
@@ -146,6 +148,10 @@ A bucket lookup sends one HeadBucket for the named bucket, never ListBuckets, so
|
|
|
146
148
|
|
|
147
149
|
`fastGetIfPresent()` answers "read it, or tell me it is gone" with one GetObject, so an object removed between an existence check and a read can no longer turn into a failed read: a provider answer that the key does not exist — `NoSuchKey`, `NotFound` or any other HTTP 404 — resolves `null`. Every other failure rejects as `fastGet()` does, the provider error unchanged: a missing bucket (`NoSuchBucket`), a denied request, a failure while the body streams, and an abort through `signal` or `SmartBucket.close()`, which rejects with the abort reason even when the provider answers not-found after it. `fastExists()` and the create-only check of `fastPutStream()` classify a missing object the same way.
|
|
148
150
|
|
|
151
|
+
Every request of the S3 client a `SmartBucket` builds is bounded. An attempt whose socket does not connect within `connectionTimeoutMs` (default 10 s, at most 60 s), or whose response headers do not arrive within `requestTimeoutMs` of its start (default 60 s, at most 1 h; the upload of the request body counts), fails with the SDK's `TimeoutError`; the SDK retries it up to `maxAttempts` attempts in all (default 3, at most 10, standard retry mode, `AWS_MAX_ATTEMPTS` and shared config files ignored). Each value must be a positive safe integer; anything else is refused with a `TypeError` before a client is built. `smartBucket.requestBounds` (and `connection.requestBounds`) reads the frozen bounds, including `requestDeadlineMs = maxAttempts × requestTimeoutMs + (maxAttempts − 1) × 20 000 ms + 1 000 ms` — the attempts, the longest wait the retry strategy takes before a retry (a provider's retry-after hint included), and a second for the client's own work. 221 000 ms with the defaults. A request still unsettled at that deadline, a stalled error body included, rejects with `ObjectStorageRequestDeadlineError`, its attempt in flight is destroyed and no further attempt of it starts, so a consumer can hold the client to an upper bound of its own (`requestDeadlineMs < bound`). The bound does not undo a request: a PutObject the provider received completely before the deadline or an abort may still take effect. It covers requests, not the time a caller takes to read a download it received: a response body streams at the caller's pace until it is consumed, the caller's signal aborts or the SmartBucket closes. While a client assigned through `storageClient` or `replaceStorageClient()` is in use, `requestBounds` is `undefined` and that client's own configuration applies. A single-request upload (`fastPut`, `fastPutStream`) must finish sending its body within `requestTimeoutMs`; for large objects raise it or use `fastPutStreamExact()`, whose parts are separate requests of 5 MiB each.
|
|
152
|
+
|
|
153
|
+
A `signal` aborts the request in flight, stops its retries and rejects with the signal's reason; an already aborted signal sends nothing. A signal of a bucket read, write, copy, move, removal, existence check or listing that is not an `AbortSignal`, a `Proxy` of one included, is refused with a `TypeError` before any request. A per-call time limit is a signal as well, `AbortSignal.timeout(ms)` or an own `AbortController`.
|
|
154
|
+
|
|
149
155
|
### Document APIs that need a MongoDB backend
|
|
150
156
|
|
|
151
157
|
The document model runs on both NoSQLDB and MongoDB, but several APIs do not mean the same thing on both. The NoSQLDB engine is qualified at 8.0.2 and 10.9.0 (see [Verification](#verification)); the two releases part on the grouped-totals and guarded-`$expr` rows below — 10.9.0 groups composite keys exactly and evaluates the guard, 8.0.2 does neither — on whether a migration can create a partial index or a collection validator at all, on when a transaction learns of a write conflict, and on whether `$elemMatch` applies query operators to scalar array elements.
|
package/ts/00_commitinfo_data.ts
CHANGED
|
@@ -30,6 +30,8 @@ export interface IListObjectKeysPageOptions {
|
|
|
30
30
|
prefix?: string;
|
|
31
31
|
limit: number;
|
|
32
32
|
startAfter?: string;
|
|
33
|
+
/** Optional caller-owned cancellation signal */
|
|
34
|
+
signal?: AbortSignal;
|
|
33
35
|
}
|
|
34
36
|
|
|
35
37
|
export interface IListObjectKeysPageResult {
|
|
@@ -37,10 +39,7 @@ export interface IListObjectKeysPageResult {
|
|
|
37
39
|
nextStartAfter?: string;
|
|
38
40
|
}
|
|
39
41
|
|
|
40
|
-
export interface IListObjectEntriesPageOptions extends IListObjectKeysPageOptions {
|
|
41
|
-
/** Optional caller-owned cancellation signal */
|
|
42
|
-
signal?: AbortSignal;
|
|
43
|
-
}
|
|
42
|
+
export interface IListObjectEntriesPageOptions extends IListObjectKeysPageOptions {}
|
|
44
43
|
|
|
45
44
|
/**
|
|
46
45
|
* One object as ListObjectsV2 reported it when the page was listed.
|
|
@@ -429,17 +428,15 @@ export class Bucket {
|
|
|
429
428
|
// ===============
|
|
430
429
|
|
|
431
430
|
/**
|
|
432
|
-
*
|
|
431
|
+
* Stores an object with one PutObject request; without `overwrite: true`
|
|
432
|
+
* a HeadObject request first refuses an existing object. `signal` aborts
|
|
433
|
+
* the request in flight and rejects with its reason.
|
|
433
434
|
*/
|
|
434
|
-
public async fastPut(
|
|
435
|
-
|
|
436
|
-
contents: string | Buffer;
|
|
437
|
-
overwrite?: boolean;
|
|
438
|
-
}
|
|
439
|
-
): Promise<File> {
|
|
435
|
+
public async fastPut(optionsArg: interfaces.IFastPutOptions): Promise<File> {
|
|
436
|
+
const signal = helpers.requireOptionalAbortSignal(optionsArg.signal, 'fastPut signal');
|
|
440
437
|
const reducedPath = await helpers.reducePathDescriptorToPath(optionsArg);
|
|
441
438
|
if (optionsArg.overwrite !== true) {
|
|
442
|
-
const exists = await this.fastExists({ path: reducedPath });
|
|
439
|
+
const exists = await this.fastExists({ path: reducedPath, signal });
|
|
443
440
|
|
|
444
441
|
if (exists) {
|
|
445
442
|
throw new Error(
|
|
@@ -449,12 +446,19 @@ export class Bucket {
|
|
|
449
446
|
}
|
|
450
447
|
}
|
|
451
448
|
|
|
452
|
-
const
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
449
|
+
const operation = beginBasicBucketOperation(this, signal);
|
|
450
|
+
try {
|
|
451
|
+
const command = new plugins.s3.PutObjectCommand({
|
|
452
|
+
Bucket: operation.bucketName,
|
|
453
|
+
Key: reducedPath,
|
|
454
|
+
Body: optionsArg.contents,
|
|
455
|
+
});
|
|
456
|
+
await operation.runProvider(
|
|
457
|
+
() => operation.client.send(command, { abortSignal: operation.signal }),
|
|
458
|
+
);
|
|
459
|
+
} finally {
|
|
460
|
+
operation.finish();
|
|
461
|
+
}
|
|
458
462
|
|
|
459
463
|
const parsedPath = plugins.path.parse(reducedPath);
|
|
460
464
|
return new File({
|
|
@@ -469,7 +473,7 @@ export class Bucket {
|
|
|
469
473
|
/**
|
|
470
474
|
* get file
|
|
471
475
|
*/
|
|
472
|
-
public async fastGet(optionsArg:
|
|
476
|
+
public async fastGet(optionsArg: interfaces.IFastGetStreamOptions): Promise<Buffer> {
|
|
473
477
|
return await readObjectBody(await this.fastGetStream(optionsArg));
|
|
474
478
|
}
|
|
475
479
|
|
|
@@ -502,9 +506,9 @@ export class Bucket {
|
|
|
502
506
|
* @param optionsArg
|
|
503
507
|
* @returns
|
|
504
508
|
*/
|
|
505
|
-
public async fastGetReplaySubject(
|
|
506
|
-
|
|
507
|
-
|
|
509
|
+
public async fastGetReplaySubject(
|
|
510
|
+
optionsArg: interfaces.IFastGetStreamOptions,
|
|
511
|
+
): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> {
|
|
508
512
|
const source = await this.fastGetStream(optionsArg);
|
|
509
513
|
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>();
|
|
510
514
|
void (async () => {
|
|
@@ -965,15 +969,12 @@ export class Bucket {
|
|
|
965
969
|
/**
|
|
966
970
|
* Move object from one path to another within the same bucket or to another bucket
|
|
967
971
|
*/
|
|
968
|
-
public async fastMove(optionsArg: {
|
|
969
|
-
|
|
970
|
-
destinationPath: string;
|
|
971
|
-
targetBucket?: Bucket;
|
|
972
|
-
overwrite?: boolean;
|
|
973
|
-
}): Promise<void> {
|
|
972
|
+
public async fastMove(optionsArg: interfaces.IFastMoveOptions): Promise<void> {
|
|
973
|
+
const signal = helpers.requireOptionalAbortSignal(optionsArg.signal, 'fastMove signal');
|
|
974
974
|
const destinationBucket = optionsArg.targetBucket || this;
|
|
975
975
|
const exists = await destinationBucket.fastExists({
|
|
976
976
|
path: optionsArg.destinationPath,
|
|
977
|
+
signal,
|
|
977
978
|
});
|
|
978
979
|
|
|
979
980
|
if (exists && !optionsArg.overwrite) {
|
|
@@ -984,7 +985,7 @@ export class Bucket {
|
|
|
984
985
|
}
|
|
985
986
|
|
|
986
987
|
await this.fastCopy(optionsArg);
|
|
987
|
-
await this.fastRemove({ path: optionsArg.sourcePath });
|
|
988
|
+
await this.fastRemove({ path: optionsArg.sourcePath, signal });
|
|
988
989
|
}
|
|
989
990
|
|
|
990
991
|
/**
|
|
@@ -1120,17 +1121,30 @@ export class Bucket {
|
|
|
1120
1121
|
* }
|
|
1121
1122
|
* ```
|
|
1122
1123
|
*/
|
|
1123
|
-
public async *listAllObjects(
|
|
1124
|
+
public async *listAllObjects(
|
|
1125
|
+
prefix: string = '',
|
|
1126
|
+
optionsArg: interfaces.IListAllObjectsOptions = {},
|
|
1127
|
+
): AsyncIterableIterator<string> {
|
|
1128
|
+
const signal = helpers.requireOptionalAbortSignal(optionsArg.signal, 'listAllObjects signal');
|
|
1124
1129
|
let continuationToken: string | undefined;
|
|
1125
1130
|
|
|
1126
1131
|
do {
|
|
1127
|
-
const
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1132
|
+
const operation = beginBasicBucketOperation(this, signal);
|
|
1133
|
+
let response: plugins.s3.ListObjectsV2CommandOutput;
|
|
1134
|
+
try {
|
|
1135
|
+
operation.assertCurrent();
|
|
1136
|
+
const command = new plugins.s3.ListObjectsV2Command({
|
|
1137
|
+
Bucket: operation.bucketName,
|
|
1138
|
+
Prefix: prefix,
|
|
1139
|
+
ContinuationToken: continuationToken,
|
|
1140
|
+
});
|
|
1141
|
+
response = await operation.runProvider(
|
|
1142
|
+
() => operation.client.send(command, { abortSignal: operation.signal }),
|
|
1143
|
+
);
|
|
1144
|
+
operation.assertCurrent();
|
|
1145
|
+
} finally {
|
|
1146
|
+
operation.finish();
|
|
1147
|
+
}
|
|
1134
1148
|
|
|
1135
1149
|
for (const obj of response.Contents || []) {
|
|
1136
1150
|
if (obj.Key) yield obj.Key;
|
|
@@ -1210,19 +1224,28 @@ export class Bucket {
|
|
|
1210
1224
|
optionsArg: IListObjectKeysPageOptions,
|
|
1211
1225
|
): Promise<IListObjectKeysPageResult> {
|
|
1212
1226
|
const prefix = assertListingPageOptions(optionsArg, 'Object key page');
|
|
1213
|
-
const
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1227
|
+
const signal = helpers.requireOptionalAbortSignal(optionsArg.signal, 'Object key page signal');
|
|
1228
|
+
const operation = beginBasicBucketOperation(this, signal);
|
|
1229
|
+
try {
|
|
1230
|
+
operation.assertCurrent();
|
|
1231
|
+
const command = createListingPageCommand(operation.bucketName, prefix, optionsArg);
|
|
1232
|
+
const response = await operation.runProvider(
|
|
1233
|
+
() => operation.client.send(command, { abortSignal: operation.signal }),
|
|
1234
|
+
);
|
|
1235
|
+
operation.assertCurrent();
|
|
1236
|
+
const { keys, nextStartAfter } = readListingPageContents(
|
|
1237
|
+
response,
|
|
1238
|
+
prefix,
|
|
1239
|
+
optionsArg,
|
|
1240
|
+
'key page',
|
|
1241
|
+
);
|
|
1242
|
+
return {
|
|
1243
|
+
keys,
|
|
1244
|
+
...(nextStartAfter !== undefined ? { nextStartAfter } : {}),
|
|
1245
|
+
};
|
|
1246
|
+
} finally {
|
|
1247
|
+
operation.finish();
|
|
1248
|
+
}
|
|
1226
1249
|
}
|
|
1227
1250
|
|
|
1228
1251
|
/**
|
|
@@ -1238,10 +1261,8 @@ export class Bucket {
|
|
|
1238
1261
|
optionsArg: IListObjectEntriesPageOptions,
|
|
1239
1262
|
): Promise<IListObjectEntriesPageResult> {
|
|
1240
1263
|
const prefix = assertListingPageOptions(optionsArg, 'Object entry page');
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
}
|
|
1244
|
-
const operation = beginBasicBucketOperation(this, optionsArg.signal);
|
|
1264
|
+
const signal = helpers.requireOptionalAbortSignal(optionsArg.signal, 'Object entry page signal');
|
|
1265
|
+
const operation = beginBasicBucketOperation(this, signal);
|
|
1245
1266
|
try {
|
|
1246
1267
|
operation.assertCurrent();
|
|
1247
1268
|
const command = createListingPageCommand(operation.bucketName, prefix, optionsArg);
|
|
@@ -1345,9 +1366,12 @@ export class Bucket {
|
|
|
1345
1366
|
* console.log(`Found ${allKeys.length} objects`);
|
|
1346
1367
|
* ```
|
|
1347
1368
|
*/
|
|
1348
|
-
public async listAllObjectsArray(
|
|
1369
|
+
public async listAllObjectsArray(
|
|
1370
|
+
prefix: string = '',
|
|
1371
|
+
optionsArg: interfaces.IListAllObjectsOptions = {},
|
|
1372
|
+
): Promise<string[]> {
|
|
1349
1373
|
const keys: string[] = [];
|
|
1350
|
-
for await (const key of this.listAllObjects(prefix)) {
|
|
1374
|
+
for await (const key of this.listAllObjects(prefix, optionsArg)) {
|
|
1351
1375
|
keys.push(key);
|
|
1352
1376
|
}
|
|
1353
1377
|
return keys;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { SmartBucket } from './classes.smartbucket.js';
|
|
2
2
|
import type { ICapabilities, IOperationOptions, IReadiness } from '../core/interfaces.js';
|
|
3
|
+
import type { IObjectStorageRequestOptions } from './interfaces.js';
|
|
3
4
|
|
|
4
|
-
export interface IObjectStorageConnectionOptions {
|
|
5
|
+
export interface IObjectStorageConnectionOptions extends IObjectStorageRequestOptions {
|
|
5
6
|
backend: 'objectstorage' | 's3';
|
|
6
7
|
endpoint: string;
|
|
7
8
|
region?: string;
|
|
@@ -23,7 +24,8 @@ export class ObjectStorageConnection extends SmartBucket {
|
|
|
23
24
|
super({ endpoint: endpoint.hostname, port: Number(endpoint.port || (endpoint.protocol === 'https:' ? 443 : 80)),
|
|
24
25
|
useSsl: endpoint.protocol === 'https:', accessKey: options.credentials.accessKeyId,
|
|
25
26
|
accessSecret: options.credentials.secretAccessKey, region: options.region ?? 'us-east-1' },
|
|
26
|
-
{ forcePathStyle: options.forcePathStyle
|
|
27
|
+
{ forcePathStyle: options.forcePathStyle, connectionTimeoutMs: options.connectionTimeoutMs,
|
|
28
|
+
requestTimeoutMs: options.requestTimeoutMs, maxAttempts: options.maxAttempts });
|
|
27
29
|
this.backend = options.backend;
|
|
28
30
|
this.capabilities = Object.freeze({ backend: this.backend, transactions: 'unsupported', changeStreams: 'unsupported',
|
|
29
31
|
compositeGrouping: 'unsupported', objectMetrics: 'unsupported', diskResidentScans: 'unknown' });
|
|
@@ -243,6 +243,14 @@ export class Directory {
|
|
|
243
243
|
|
|
244
244
|
|
|
245
245
|
const getDirectory = async (directoryArg: Directory, dirNameToSearch: string, isFinalDirectory: boolean) => {
|
|
246
|
+
// A listed directory and a new one of the same name are the same handle. Only whether the
|
|
247
|
+
// final directory exists decides anything, and only when it is created with an initializer
|
|
248
|
+
// file; otherwise a caller that accepts an empty directory needs no listing, which walks every
|
|
249
|
+
// key below the prefix and costs a large bucket seconds per level.
|
|
250
|
+
const createsInitializer = isFinalDirectory && optionsArg.createWithInitializerFile;
|
|
251
|
+
if ((optionsArg.getEmptyDirectory || optionsArg.createWithInitializerFile) && !createsInitializer) {
|
|
252
|
+
return new Directory(this.bucketRef, directoryArg, dirNameToSearch);
|
|
253
|
+
}
|
|
246
254
|
const directories = await directoryArg.listDirectories();
|
|
247
255
|
let returnDirectory = directories.find((directory) => {
|
|
248
256
|
return directory.name === dirNameToSearch;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The rejection of a request that reached the request deadline of the
|
|
3
|
+
* storage client a SmartBucket built (`SmartBucket.requestBounds`). The
|
|
4
|
+
* attempt in flight was destroyed and no further attempt starts. A request
|
|
5
|
+
* the provider had received completely before the deadline may still take
|
|
6
|
+
* effect.
|
|
7
|
+
*/
|
|
8
|
+
export class ObjectStorageRequestDeadlineError extends Error {
|
|
9
|
+
public readonly deadlineMs: number;
|
|
10
|
+
|
|
11
|
+
constructor(deadlineMsArg: number) {
|
|
12
|
+
super(`The object storage request exceeded its deadline of ${deadlineMsArg} ms.`);
|
|
13
|
+
this.name = 'ObjectStorageRequestDeadlineError';
|
|
14
|
+
this.deadlineMs = deadlineMsArg;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -4,6 +4,11 @@ import * as plugins from './plugins.js';
|
|
|
4
4
|
import { Bucket } from './classes.bucket.js';
|
|
5
5
|
import { normalizeStorageDescriptor } from './helpers.js';
|
|
6
6
|
import { headBucketExists } from './internal.bucketrequests.js';
|
|
7
|
+
import {
|
|
8
|
+
bindRequestDeadline,
|
|
9
|
+
createBoundedClientConfig,
|
|
10
|
+
resolveRequestBounds,
|
|
11
|
+
} from './internal.requestbounds.js';
|
|
7
12
|
import {
|
|
8
13
|
beginOperationLifecycleClose,
|
|
9
14
|
commitOperationLifecycleClient,
|
|
@@ -14,6 +19,8 @@ import {
|
|
|
14
19
|
} from './internal.exactupload.capability.js';
|
|
15
20
|
import type {
|
|
16
21
|
IBucketRequestOptions,
|
|
22
|
+
IObjectStorageRequestBounds,
|
|
23
|
+
IObjectStorageRequestOptions,
|
|
17
24
|
ISmartBucketReadinessOptions,
|
|
18
25
|
TSmartBucketReadinessFailureReason,
|
|
19
26
|
TSmartBucketReadinessResult,
|
|
@@ -31,6 +38,8 @@ export class SmartBucket {
|
|
|
31
38
|
|
|
32
39
|
private currentStorageClient: plugins.s3.S3Client;
|
|
33
40
|
private ownedStorageClient: plugins.s3.S3Client;
|
|
41
|
+
private readonly boundedStorageClient: plugins.s3.S3Client;
|
|
42
|
+
private readonly boundedStorageClientBounds: IObjectStorageRequestBounds;
|
|
34
43
|
private lifecycleState: TSmartBucketLifecycleState = 'open';
|
|
35
44
|
private lifecycleEpoch = 0;
|
|
36
45
|
private activeReadinessControllers = new Set<AbortController>();
|
|
@@ -85,6 +94,18 @@ export class SmartBucket {
|
|
|
85
94
|
}
|
|
86
95
|
}
|
|
87
96
|
|
|
97
|
+
/**
|
|
98
|
+
* The bounds every request of the storage client this SmartBucket built
|
|
99
|
+
* runs within, or `undefined` while a client assigned through
|
|
100
|
+
* `storageClient` or `replaceStorageClient()` is in use: its requests are
|
|
101
|
+
* bounded only by its own configuration.
|
|
102
|
+
*/
|
|
103
|
+
public get requestBounds(): IObjectStorageRequestBounds | undefined {
|
|
104
|
+
return this.currentStorageClient === this.boundedStorageClient
|
|
105
|
+
? this.boundedStorageClientBounds
|
|
106
|
+
: undefined;
|
|
107
|
+
}
|
|
108
|
+
|
|
88
109
|
/** @deprecated Use storageClient instead */
|
|
89
110
|
public get s3Client(): plugins.s3.S3Client {
|
|
90
111
|
return this.currentStorageClient;
|
|
@@ -93,18 +114,26 @@ export class SmartBucket {
|
|
|
93
114
|
/**
|
|
94
115
|
* the constructor of SmartBucket
|
|
95
116
|
*/
|
|
96
|
-
constructor(
|
|
117
|
+
constructor(
|
|
118
|
+
configArg: plugins.tsclass.storage.IStorageDescriptor,
|
|
119
|
+
clientOptions: { forcePathStyle?: boolean } & IObjectStorageRequestOptions = {},
|
|
120
|
+
) {
|
|
97
121
|
this.config = configArg;
|
|
98
122
|
|
|
99
123
|
// Use the normalizer to handle various endpoint formats
|
|
100
124
|
const { normalized } = normalizeStorageDescriptor(configArg);
|
|
125
|
+
const requestBounds = resolveRequestBounds(clientOptions);
|
|
101
126
|
|
|
102
127
|
this.currentStorageClient = new plugins.s3.S3Client({
|
|
103
128
|
endpoint: normalized.endpointUrl,
|
|
104
129
|
region: normalized.region,
|
|
105
130
|
credentials: normalized.credentials,
|
|
106
131
|
forcePathStyle: clientOptions.forcePathStyle ?? normalized.forcePathStyle,
|
|
132
|
+
...createBoundedClientConfig(requestBounds),
|
|
107
133
|
});
|
|
134
|
+
bindRequestDeadline(this.currentStorageClient, requestBounds);
|
|
135
|
+
this.boundedStorageClient = this.currentStorageClient;
|
|
136
|
+
this.boundedStorageClientBounds = requestBounds;
|
|
108
137
|
this.ownedStorageClient = this.currentStorageClient;
|
|
109
138
|
registerOperationLifecycle(this, this.currentStorageClient);
|
|
110
139
|
}
|
|
@@ -21,6 +21,22 @@ export const reducePathDescriptorToPath = async (pathDescriptorArg: interfaces.I
|
|
|
21
21
|
return returnPath;
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Validates a caller's cancellation signal: a genuine `AbortSignal`.
|
|
26
|
+
*/
|
|
27
|
+
export const requireOptionalAbortSignal = (
|
|
28
|
+
valueArg: unknown,
|
|
29
|
+
labelArg: string,
|
|
30
|
+
): AbortSignal | undefined => {
|
|
31
|
+
if (valueArg === undefined) {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
if (plugins.nodeUtil.types.isProxy(valueArg) || !(valueArg instanceof AbortSignal)) {
|
|
35
|
+
throw new TypeError(`${labelArg} must be an AbortSignal.`);
|
|
36
|
+
}
|
|
37
|
+
return valueArg;
|
|
38
|
+
};
|
|
39
|
+
|
|
24
40
|
// Storage Descriptor Normalization
|
|
25
41
|
export interface IStorageWarning {
|
|
26
42
|
code: string;
|
|
@@ -3,6 +3,7 @@ export * from './classes.bucket.js';
|
|
|
3
3
|
export * from './classes.exactuploaderror.js';
|
|
4
4
|
export * from './classes.exactpathpurgeerror.js';
|
|
5
5
|
export * from './classes.objectrangeerror.js';
|
|
6
|
+
export * from './classes.requestdeadlineerror.js';
|
|
6
7
|
export * from './classes.directory.js';
|
|
7
8
|
export * from './classes.file.js';
|
|
8
9
|
export * from './classes.listcursor.js';
|
|
@@ -87,6 +87,25 @@ export interface IFastStatOptions extends IPathDecriptor {
|
|
|
87
87
|
signal?: AbortSignal;
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
+
export interface IFastPutOptions extends IPathDecriptor {
|
|
91
|
+
contents: string | Buffer;
|
|
92
|
+
overwrite?: boolean;
|
|
93
|
+
signal?: AbortSignal;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export interface IFastMoveOptions {
|
|
97
|
+
sourcePath: string;
|
|
98
|
+
destinationPath: string;
|
|
99
|
+
targetBucket?: Bucket;
|
|
100
|
+
overwrite?: boolean;
|
|
101
|
+
signal?: AbortSignal;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export interface IListAllObjectsOptions {
|
|
105
|
+
/** Optional caller-owned cancellation signal */
|
|
106
|
+
signal?: AbortSignal;
|
|
107
|
+
}
|
|
108
|
+
|
|
90
109
|
export interface IDirectoryFastPutStreamOptions {
|
|
91
110
|
path: string;
|
|
92
111
|
stream: Readable;
|
|
@@ -103,6 +122,45 @@ export interface IBucketRequestOptions {
|
|
|
103
122
|
signal?: AbortSignal;
|
|
104
123
|
}
|
|
105
124
|
|
|
125
|
+
// ================================
|
|
126
|
+
// Request Bound Interfaces
|
|
127
|
+
// ================================
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Bounds of every request sent by the storage client a SmartBucket builds.
|
|
131
|
+
* Each is a positive safe integer; an omitted one takes its default.
|
|
132
|
+
*/
|
|
133
|
+
export interface IObjectStorageRequestOptions {
|
|
134
|
+
/** Time one attempt may take to connect its socket in milliseconds (default: 10000, maximum: 60000) */
|
|
135
|
+
connectionTimeoutMs?: number;
|
|
136
|
+
/**
|
|
137
|
+
* Time from the start of one attempt to its response headers in
|
|
138
|
+
* milliseconds, connecting and uploading the request body included
|
|
139
|
+
* (default: 60000, maximum: 3600000)
|
|
140
|
+
*/
|
|
141
|
+
requestTimeoutMs?: number;
|
|
142
|
+
/** Attempts of one request, the first one included (default: 3, maximum: 10) */
|
|
143
|
+
maxAttempts?: number;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* The effective bounds of the storage client a SmartBucket built.
|
|
148
|
+
*/
|
|
149
|
+
export interface IObjectStorageRequestBounds {
|
|
150
|
+
readonly connectionTimeoutMs: number;
|
|
151
|
+
readonly requestTimeoutMs: number;
|
|
152
|
+
readonly maxAttempts: number;
|
|
153
|
+
/** The longest wait of the SDK before a retry */
|
|
154
|
+
readonly maxRetryDelayMs: number;
|
|
155
|
+
/**
|
|
156
|
+
* `maxAttempts * requestTimeoutMs + (maxAttempts - 1) * maxRetryDelayMs`
|
|
157
|
+
* plus 1000 ms for the client's own work around the attempts: the client
|
|
158
|
+
* destroys a request still unsettled after this time, retries included,
|
|
159
|
+
* and starts no further attempt of it
|
|
160
|
+
*/
|
|
161
|
+
readonly requestDeadlineMs: number;
|
|
162
|
+
}
|
|
163
|
+
|
|
106
164
|
// ================================
|
|
107
165
|
// SmartBucket Readiness Interfaces
|
|
108
166
|
// ================================
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
|
+
import { requireOptionalAbortSignal } from './helpers.js';
|
|
2
3
|
import type {
|
|
3
4
|
IExactPathPurgeCapability,
|
|
4
5
|
IExactUploadCapability,
|
|
@@ -541,6 +542,7 @@ export const beginBasicBucketOperation = (
|
|
|
541
542
|
bucketRefArg: IBucketCapabilityRef,
|
|
542
543
|
callerSignalArg?: AbortSignal,
|
|
543
544
|
): IBasicBucketOperation => {
|
|
545
|
+
const callerSignal = requireOptionalAbortSignal(callerSignalArg, 'Object storage request signal');
|
|
544
546
|
const smartbucketRef = bucketRefArg.smartbucketRef;
|
|
545
547
|
const lifecycleRecord = getLifecycleRecord(smartbucketRef);
|
|
546
548
|
const binding: IExactCapabilityBinding = {
|
|
@@ -550,7 +552,7 @@ export const beginBasicBucketOperation = (
|
|
|
550
552
|
storageClient: lifecycleRecord.storageClient,
|
|
551
553
|
lifecycleGeneration: lifecycleRecord.generation,
|
|
552
554
|
};
|
|
553
|
-
return beginOperation(binding,
|
|
555
|
+
return beginOperation(binding, callerSignal);
|
|
554
556
|
};
|
|
555
557
|
|
|
556
558
|
export const beginExactUploadProbeOperation = (
|