@lossless.org/client 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/nosqldb/classes.atomicdelete.d.ts +1 -0
- package/dist_ts/nosqldb/classes.atomicdelete.js +10 -7
- package/dist_ts/nosqldb/classes.atomicfindoneandupdate.d.ts +2 -0
- package/dist_ts/nosqldb/classes.atomicfindoneandupdate.js +9 -5
- package/dist_ts/nosqldb/classes.atomicupdate.d.ts +2 -0
- package/dist_ts/nosqldb/classes.atomicupdate.js +13 -8
- package/dist_ts/nosqldb/classes.collection.d.ts +13 -4
- package/dist_ts/nosqldb/classes.collection.js +57 -64
- package/dist_ts/nosqldb/classes.db.d.ts +1 -1
- package/dist_ts/nosqldb/classes.doc.d.ts +60 -3
- package/dist_ts/nosqldb/classes.doc.js +68 -26
- package/dist_ts/nosqldb/classes.exactpersistence.d.ts +36 -7
- package/dist_ts/nosqldb/classes.exactpersistence.js +92 -81
- package/dist_ts/nosqldb/classes.operationdeadline.d.ts +61 -0
- package/dist_ts/nosqldb/classes.operationdeadline.js +149 -0
- package/dist_ts/nosqldb/classes.persistence.d.ts +16 -1
- package/dist_ts/nosqldb/classes.persistence.js +20 -1
- package/dist_ts/nosqldb/classes.session.d.ts +73 -2
- package/dist_ts/nosqldb/classes.session.js +164 -38
- package/dist_ts/objectstorage/classes.bucket.d.ts +61 -3
- package/dist_ts/objectstorage/classes.bucket.js +243 -184
- package/dist_ts/objectstorage/classes.directory.js +17 -28
- package/dist_ts/objectstorage/classes.smartbucket.d.ts +13 -6
- package/dist_ts/objectstorage/classes.smartbucket.js +18 -12
- package/dist_ts/objectstorage/classes.watcher.js +1 -2
- package/dist_ts/objectstorage/interfaces.d.ts +8 -0
- package/dist_ts/objectstorage/internal.bucketrequests.d.ts +14 -0
- package/dist_ts/objectstorage/internal.bucketrequests.js +53 -0
- package/dist_ts/objectstorage/internal.exactupload.capability.d.ts +12 -0
- package/dist_ts/objectstorage/internal.exactupload.capability.js +46 -17
- package/package.json +3 -3
- package/readme.md +13 -2
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/nosqldb/classes.atomicdelete.ts +10 -6
- package/ts/nosqldb/classes.atomicfindoneandupdate.ts +10 -4
- package/ts/nosqldb/classes.atomicupdate.ts +14 -7
- package/ts/nosqldb/classes.collection.ts +82 -73
- package/ts/nosqldb/classes.db.ts +1 -1
- package/ts/nosqldb/classes.doc.ts +162 -33
- package/ts/nosqldb/classes.exactpersistence.ts +131 -93
- package/ts/nosqldb/classes.operationdeadline.ts +179 -0
- package/ts/nosqldb/classes.persistence.ts +36 -1
- package/ts/nosqldb/classes.session.ts +221 -39
- package/ts/objectstorage/classes.bucket.ts +350 -218
- package/ts/objectstorage/classes.directory.ts +18 -27
- package/ts/objectstorage/classes.smartbucket.ts +21 -11
- package/ts/objectstorage/classes.watcher.ts +0 -1
- package/ts/objectstorage/interfaces.ts +9 -0
- package/ts/objectstorage/internal.bucketrequests.ts +70 -0
- package/ts/objectstorage/internal.exactupload.capability.ts +105 -24
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
purgeExactPath as purgeExactPathOperation,
|
|
19
19
|
} from './internal.exactpathpurge.operations.js';
|
|
20
20
|
import { beginBasicBucketOperation } from './internal.exactupload.capability.js';
|
|
21
|
+
import { headBucketExists, runAccountRequest } from './internal.bucketrequests.js';
|
|
21
22
|
|
|
22
23
|
export interface IListObjectKeysPageOptions {
|
|
23
24
|
prefix?: string;
|
|
@@ -30,9 +31,171 @@ export interface IListObjectKeysPageResult {
|
|
|
30
31
|
nextStartAfter?: string;
|
|
31
32
|
}
|
|
32
33
|
|
|
34
|
+
export interface IListObjectEntriesPageOptions extends IListObjectKeysPageOptions {
|
|
35
|
+
/** Optional caller-owned cancellation signal */
|
|
36
|
+
signal?: AbortSignal;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* One object as ListObjectsV2 reported it when the page was listed.
|
|
41
|
+
* - `size`: the `Size` field, the object's byte length; HEAD reports the same
|
|
42
|
+
* value as `ContentLength`.
|
|
43
|
+
* - `etag`: the `ETag` field without its surrounding double quotes; HEAD
|
|
44
|
+
* reports the same entity tag, quoted. It identifies an object version, it
|
|
45
|
+
* is not a content digest: AWS S3 derives it from the content MD5 only for
|
|
46
|
+
* single-part uploads without SSE-KMS or SSE-C, and other providers define
|
|
47
|
+
* it themselves.
|
|
48
|
+
* - `lastModified`: the `LastModified` field when the provider sends one.
|
|
49
|
+
*/
|
|
50
|
+
export interface IObjectListingEntry {
|
|
51
|
+
key: string;
|
|
52
|
+
size: number;
|
|
53
|
+
etag: string;
|
|
54
|
+
lastModified?: Date;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface IListObjectEntriesPageResult {
|
|
58
|
+
entries: IObjectListingEntry[];
|
|
59
|
+
nextStartAfter?: string;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface IListAllObjectEntriesOptions {
|
|
63
|
+
/** Entries fetched per ListObjectsV2 request, 1 to 1000 (default 1000) */
|
|
64
|
+
pageSize?: number;
|
|
65
|
+
/** Optional caller-owned cancellation signal */
|
|
66
|
+
signal?: AbortSignal;
|
|
67
|
+
}
|
|
68
|
+
|
|
33
69
|
const compareUtf8ObjectKeys = (leftArg: string, rightArg: string): number =>
|
|
34
70
|
Buffer.compare(Buffer.from(leftArg, 'utf8'), Buffer.from(rightArg, 'utf8'));
|
|
35
71
|
|
|
72
|
+
const assertListingPageOptions = (
|
|
73
|
+
optionsArg: IListObjectKeysPageOptions,
|
|
74
|
+
labelArg: string,
|
|
75
|
+
): string => {
|
|
76
|
+
if (!optionsArg || typeof optionsArg !== 'object' || Array.isArray(optionsArg)) {
|
|
77
|
+
throw new TypeError(`${labelArg} options must be a plain object`);
|
|
78
|
+
}
|
|
79
|
+
const prefix = optionsArg.prefix ?? '';
|
|
80
|
+
if (typeof prefix !== 'string') {
|
|
81
|
+
throw new TypeError(`${labelArg} prefix must be a string`);
|
|
82
|
+
}
|
|
83
|
+
if (!Number.isInteger(optionsArg.limit) || optionsArg.limit < 1 || optionsArg.limit > 1_000) {
|
|
84
|
+
throw new Error(`${labelArg} limit must be an integer from 1 to 1000`);
|
|
85
|
+
}
|
|
86
|
+
if (
|
|
87
|
+
optionsArg.startAfter !== undefined
|
|
88
|
+
&& (
|
|
89
|
+
typeof optionsArg.startAfter !== 'string'
|
|
90
|
+
|| optionsArg.startAfter.length === 0
|
|
91
|
+
|| !optionsArg.startAfter.startsWith(prefix)
|
|
92
|
+
)
|
|
93
|
+
) {
|
|
94
|
+
throw new Error(`${labelArg} startAfter must be a non-empty key under the prefix`);
|
|
95
|
+
}
|
|
96
|
+
return prefix;
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
const createListingPageCommand = (
|
|
100
|
+
bucketNameArg: string,
|
|
101
|
+
prefixArg: string,
|
|
102
|
+
optionsArg: IListObjectKeysPageOptions,
|
|
103
|
+
): plugins.s3.ListObjectsV2Command => new plugins.s3.ListObjectsV2Command({
|
|
104
|
+
Bucket: bucketNameArg,
|
|
105
|
+
Prefix: prefixArg,
|
|
106
|
+
MaxKeys: optionsArg.limit,
|
|
107
|
+
...(optionsArg.startAfter ? { StartAfter: optionsArg.startAfter } : {}),
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Validate one ListObjectsV2 keyset page: a truncation flag, at most `limit`
|
|
112
|
+
* string keys under the prefix in strict UTF-8 byte order after `startAfter`,
|
|
113
|
+
* and no empty truncated page.
|
|
114
|
+
*/
|
|
115
|
+
const readListingPageContents = (
|
|
116
|
+
responseArg: plugins.s3.ListObjectsV2Output,
|
|
117
|
+
prefixArg: string,
|
|
118
|
+
optionsArg: IListObjectKeysPageOptions,
|
|
119
|
+
labelArg: string,
|
|
120
|
+
): { contents: plugins.s3._Object[]; keys: string[]; nextStartAfter?: string } => {
|
|
121
|
+
if (typeof responseArg.IsTruncated !== 'boolean') {
|
|
122
|
+
throw new Error(`Object storage returned an invalid ${labelArg} truncation flag`);
|
|
123
|
+
}
|
|
124
|
+
if (responseArg.Contents !== undefined && !Array.isArray(responseArg.Contents)) {
|
|
125
|
+
throw new Error(`Object storage returned invalid ${labelArg} contents`);
|
|
126
|
+
}
|
|
127
|
+
const contents = responseArg.Contents ?? [];
|
|
128
|
+
const keys = contents.map((entry) => {
|
|
129
|
+
if (!entry || typeof entry.Key !== 'string') {
|
|
130
|
+
throw new Error(`Object storage returned a ${labelArg} entry without a string key`);
|
|
131
|
+
}
|
|
132
|
+
return entry.Key;
|
|
133
|
+
});
|
|
134
|
+
if (keys.length > optionsArg.limit) {
|
|
135
|
+
throw new Error(`Object storage returned more ${labelArg === 'key page' ? 'keys' : 'entries'} than the requested page limit`);
|
|
136
|
+
}
|
|
137
|
+
if (new Set(keys).size !== keys.length) {
|
|
138
|
+
throw new Error('Object storage returned duplicate keys in one page');
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
let previousKey = optionsArg.startAfter;
|
|
142
|
+
for (const key of keys) {
|
|
143
|
+
if (!key.startsWith(prefixArg)) {
|
|
144
|
+
throw new Error('Object storage returned a key outside the requested prefix');
|
|
145
|
+
}
|
|
146
|
+
if (previousKey !== undefined && compareUtf8ObjectKeys(key, previousKey) <= 0) {
|
|
147
|
+
throw new Error('Object storage returned keys outside strict UTF-8 byte order');
|
|
148
|
+
}
|
|
149
|
+
previousKey = key;
|
|
150
|
+
}
|
|
151
|
+
if (responseArg.IsTruncated && keys.length === 0) {
|
|
152
|
+
throw new Error(`Object storage returned an empty truncated ${labelArg}`);
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
contents,
|
|
156
|
+
keys,
|
|
157
|
+
...(responseArg.IsTruncated ? { nextStartAfter: keys[keys.length - 1] } : {}),
|
|
158
|
+
};
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Strip the double quotes S3 puts around an entity tag. A provider that
|
|
163
|
+
* returns the tag unquoted is accepted as is; anything else is refused.
|
|
164
|
+
*/
|
|
165
|
+
const normalizeListingEtag = (etagArg: unknown, keyArg: string): string => {
|
|
166
|
+
if (typeof etagArg !== 'string') {
|
|
167
|
+
throw new Error(`Object storage returned a listing entry without an ETag for key '${keyArg}'`);
|
|
168
|
+
}
|
|
169
|
+
const quoted = /^"([^"]+)"$/.exec(etagArg);
|
|
170
|
+
const etag = quoted ? quoted[1] : etagArg;
|
|
171
|
+
if (etag.length === 0 || etag.includes('"')) {
|
|
172
|
+
throw new Error(`Object storage returned an invalid ETag for key '${keyArg}'`);
|
|
173
|
+
}
|
|
174
|
+
return etag;
|
|
175
|
+
};
|
|
176
|
+
|
|
177
|
+
const toObjectListingEntry = (
|
|
178
|
+
entryArg: plugins.s3._Object,
|
|
179
|
+
keyArg: string,
|
|
180
|
+
): IObjectListingEntry => {
|
|
181
|
+
if (!Number.isSafeInteger(entryArg.Size) || (entryArg.Size as number) < 0) {
|
|
182
|
+
throw new Error(`Object storage returned a listing entry without a valid size for key '${keyArg}'`);
|
|
183
|
+
}
|
|
184
|
+
const lastModified = entryArg.LastModified;
|
|
185
|
+
if (
|
|
186
|
+
lastModified !== undefined
|
|
187
|
+
&& (!(lastModified instanceof Date) || Number.isNaN(lastModified.getTime()))
|
|
188
|
+
) {
|
|
189
|
+
throw new Error(`Object storage returned an invalid LastModified for key '${keyArg}'`);
|
|
190
|
+
}
|
|
191
|
+
return {
|
|
192
|
+
key: keyArg,
|
|
193
|
+
size: entryArg.Size as number,
|
|
194
|
+
etag: normalizeListingEtag(entryArg.ETag, keyArg),
|
|
195
|
+
...(lastModified ? { lastModified } : {}),
|
|
196
|
+
};
|
|
197
|
+
};
|
|
198
|
+
|
|
36
199
|
const destroyReadableAndWaitForSettlement = async (
|
|
37
200
|
readableArg: plugins.stream.Readable,
|
|
38
201
|
): Promise<void> => {
|
|
@@ -78,29 +241,45 @@ const destroyReadableAndWaitForSettlement = async (
|
|
|
78
241
|
* operate on blobs of data in an S3-compatible object store.
|
|
79
242
|
*/
|
|
80
243
|
export class Bucket {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
244
|
+
/**
|
|
245
|
+
* Returns a reference to an existing bucket, checked with one HeadBucket
|
|
246
|
+
* request. Rejects when the bucket does not exist or the credentials may
|
|
247
|
+
* not access it.
|
|
248
|
+
*/
|
|
249
|
+
public static async getBucketByName(
|
|
250
|
+
smartbucketRef: SmartBucket,
|
|
251
|
+
bucketNameArg: string,
|
|
252
|
+
optionsArg: interfaces.IBucketRequestOptions = {},
|
|
253
|
+
): Promise<Bucket> {
|
|
254
|
+
if (!(await headBucketExists(smartbucketRef, bucketNameArg, optionsArg.signal))) {
|
|
91
255
|
throw new Error(`Bucket '${bucketNameArg}' not found.`);
|
|
92
256
|
}
|
|
257
|
+
return new this(smartbucketRef, bucketNameArg);
|
|
93
258
|
}
|
|
94
259
|
|
|
95
|
-
public static async createBucketByName(
|
|
96
|
-
|
|
97
|
-
|
|
260
|
+
public static async createBucketByName(
|
|
261
|
+
smartbucketRef: SmartBucket,
|
|
262
|
+
bucketName: string,
|
|
263
|
+
optionsArg: interfaces.IBucketRequestOptions = {},
|
|
264
|
+
) {
|
|
265
|
+
await runAccountRequest(smartbucketRef, optionsArg.signal, (clientArg, abortSignalArg) =>
|
|
266
|
+
clientArg.send(new plugins.s3.CreateBucketCommand({ Bucket: bucketName }), {
|
|
267
|
+
abortSignal: abortSignalArg,
|
|
268
|
+
}),
|
|
269
|
+
);
|
|
98
270
|
return new Bucket(smartbucketRef, bucketName);
|
|
99
271
|
}
|
|
100
272
|
|
|
101
|
-
public static async removeBucketByName(
|
|
102
|
-
|
|
103
|
-
|
|
273
|
+
public static async removeBucketByName(
|
|
274
|
+
smartbucketRef: SmartBucket,
|
|
275
|
+
bucketName: string,
|
|
276
|
+
optionsArg: interfaces.IBucketRequestOptions = {},
|
|
277
|
+
) {
|
|
278
|
+
await runAccountRequest(smartbucketRef, optionsArg.signal, (clientArg, abortSignalArg) =>
|
|
279
|
+
clientArg.send(new plugins.s3.DeleteBucketCommand({ Bucket: bucketName }), {
|
|
280
|
+
abortSignal: abortSignalArg,
|
|
281
|
+
}),
|
|
282
|
+
);
|
|
104
283
|
}
|
|
105
284
|
|
|
106
285
|
public smartbucketRef: SmartBucket;
|
|
@@ -191,43 +370,32 @@ export class Bucket {
|
|
|
191
370
|
overwrite?: boolean;
|
|
192
371
|
}
|
|
193
372
|
): Promise<File> {
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
);
|
|
204
|
-
} else {
|
|
205
|
-
console.log(`Creating new object at path '${reducedPath}' in bucket '${this.name}'.`);
|
|
206
|
-
}
|
|
373
|
+
const reducedPath = await helpers.reducePathDescriptorToPath(optionsArg);
|
|
374
|
+
if (optionsArg.overwrite !== true) {
|
|
375
|
+
const exists = await this.fastExists({ path: reducedPath });
|
|
376
|
+
|
|
377
|
+
if (exists) {
|
|
378
|
+
throw new Error(
|
|
379
|
+
`Object already exists at path '${reducedPath}' in bucket '${this.name}'. ` +
|
|
380
|
+
`Set overwrite:true to replace it.`
|
|
381
|
+
);
|
|
207
382
|
}
|
|
208
|
-
|
|
209
|
-
const command = new plugins.s3.PutObjectCommand({
|
|
210
|
-
Bucket: this.name,
|
|
211
|
-
Key: reducedPath,
|
|
212
|
-
Body: optionsArg.contents,
|
|
213
|
-
});
|
|
214
|
-
await this.smartbucketRef.storageClient.send(command);
|
|
215
|
-
|
|
216
|
-
console.log(`Object '${reducedPath}' has been successfully stored in bucket '${this.name}'.`);
|
|
217
|
-
const parsedPath = plugins.path.parse(reducedPath);
|
|
218
|
-
return new File({
|
|
219
|
-
directoryRefArg: await this.getDirectoryFromPath({
|
|
220
|
-
path: parsedPath.dir,
|
|
221
|
-
}),
|
|
222
|
-
fileName: parsedPath.base,
|
|
223
|
-
});
|
|
224
|
-
} catch (error) {
|
|
225
|
-
console.error(
|
|
226
|
-
`Error storing object at path '${optionsArg.path}' in bucket '${this.name}':`,
|
|
227
|
-
error
|
|
228
|
-
);
|
|
229
|
-
throw error;
|
|
230
383
|
}
|
|
384
|
+
|
|
385
|
+
const command = new plugins.s3.PutObjectCommand({
|
|
386
|
+
Bucket: this.name,
|
|
387
|
+
Key: reducedPath,
|
|
388
|
+
Body: optionsArg.contents,
|
|
389
|
+
});
|
|
390
|
+
await this.smartbucketRef.storageClient.send(command);
|
|
391
|
+
|
|
392
|
+
const parsedPath = plugins.path.parse(reducedPath);
|
|
393
|
+
return new File({
|
|
394
|
+
directoryRefArg: await this.getDirectoryFromPath({
|
|
395
|
+
path: parsedPath.dir,
|
|
396
|
+
}),
|
|
397
|
+
fileName: parsedPath.base,
|
|
398
|
+
});
|
|
231
399
|
}
|
|
232
400
|
|
|
233
401
|
|
|
@@ -552,8 +720,6 @@ export class Bucket {
|
|
|
552
720
|
`Object already exists at path '${optionsArg.path}' in bucket '${this.name}'. ` +
|
|
553
721
|
`Set overwrite:true to replace it.`
|
|
554
722
|
);
|
|
555
|
-
} else {
|
|
556
|
-
console.log(`Creating new object at path '${optionsArg.path}' in bucket '${this.name}'.`);
|
|
557
723
|
}
|
|
558
724
|
}
|
|
559
725
|
|
|
@@ -572,18 +738,10 @@ export class Bucket {
|
|
|
572
738
|
await operation.runProvider(
|
|
573
739
|
() => operation.client.send(command, { abortSignal: operation.signal }),
|
|
574
740
|
);
|
|
575
|
-
|
|
576
|
-
console.log(
|
|
577
|
-
`Object '${optionsArg.path}' has been successfully stored in bucket '${this.name}'.`
|
|
578
|
-
);
|
|
579
741
|
} catch (error) {
|
|
580
742
|
if (sourceOwnershipStarted || putDispatched || operation.signal.aborted) {
|
|
581
743
|
await disposeSource();
|
|
582
744
|
}
|
|
583
|
-
console.error(
|
|
584
|
-
`Error storing object at path '${optionsArg.path}' in bucket '${this.name}':`,
|
|
585
|
-
error
|
|
586
|
-
);
|
|
587
745
|
throw error;
|
|
588
746
|
} finally {
|
|
589
747
|
operation.finish();
|
|
@@ -647,9 +805,6 @@ export class Bucket {
|
|
|
647
805
|
await operation.runProvider(
|
|
648
806
|
() => operation.client.send(command, { abortSignal: operation.signal }),
|
|
649
807
|
);
|
|
650
|
-
} catch (err) {
|
|
651
|
-
console.error('Error updating metadata:', err);
|
|
652
|
-
throw err; // rethrow to allow caller to handle
|
|
653
808
|
} finally {
|
|
654
809
|
operation.finish();
|
|
655
810
|
}
|
|
@@ -664,40 +819,20 @@ export class Bucket {
|
|
|
664
819
|
targetBucket?: Bucket;
|
|
665
820
|
overwrite?: boolean;
|
|
666
821
|
}): Promise<void> {
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
});
|
|
672
|
-
|
|
673
|
-
if (exists && !optionsArg.overwrite) {
|
|
674
|
-
console.error(
|
|
675
|
-
`Object already exists at destination path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`
|
|
676
|
-
);
|
|
677
|
-
return;
|
|
678
|
-
} else if (exists && optionsArg.overwrite) {
|
|
679
|
-
console.log(
|
|
680
|
-
`Overwriting existing object at destination path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`
|
|
681
|
-
);
|
|
682
|
-
} else {
|
|
683
|
-
console.log(
|
|
684
|
-
`Moving object to path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`
|
|
685
|
-
);
|
|
686
|
-
}
|
|
687
|
-
|
|
688
|
-
await this.fastCopy(optionsArg);
|
|
689
|
-
await this.fastRemove({ path: optionsArg.sourcePath });
|
|
822
|
+
const destinationBucket = optionsArg.targetBucket || this;
|
|
823
|
+
const exists = await destinationBucket.fastExists({
|
|
824
|
+
path: optionsArg.destinationPath,
|
|
825
|
+
});
|
|
690
826
|
|
|
691
|
-
|
|
692
|
-
`Object '${optionsArg.sourcePath}' has been successfully moved to '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`
|
|
693
|
-
);
|
|
694
|
-
} catch (error) {
|
|
827
|
+
if (exists && !optionsArg.overwrite) {
|
|
695
828
|
console.error(
|
|
696
|
-
`
|
|
697
|
-
error
|
|
829
|
+
`Object already exists at destination path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`
|
|
698
830
|
);
|
|
699
|
-
|
|
831
|
+
return;
|
|
700
832
|
}
|
|
833
|
+
|
|
834
|
+
await this.fastCopy(optionsArg);
|
|
835
|
+
await this.fastRemove({ path: optionsArg.sourcePath });
|
|
701
836
|
}
|
|
702
837
|
|
|
703
838
|
/**
|
|
@@ -733,7 +868,6 @@ export class Bucket {
|
|
|
733
868
|
await operation.runProvider(
|
|
734
869
|
() => operation.client.send(command, { abortSignal: operation.signal }),
|
|
735
870
|
);
|
|
736
|
-
console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`);
|
|
737
871
|
return true;
|
|
738
872
|
} catch (error: any) {
|
|
739
873
|
try {
|
|
@@ -742,10 +876,8 @@ export class Bucket {
|
|
|
742
876
|
throw abortError;
|
|
743
877
|
}
|
|
744
878
|
if (error?.name === 'NotFound') {
|
|
745
|
-
console.log(`Object '${optionsArg.path}' does not exist in bucket '${this.name}'.`);
|
|
746
879
|
return false;
|
|
747
880
|
} else {
|
|
748
|
-
console.error('Error checking object existence:', error);
|
|
749
881
|
throw error; // Rethrow if it's not a NotFound error to handle unexpected issues
|
|
750
882
|
}
|
|
751
883
|
} finally {
|
|
@@ -805,27 +937,19 @@ export class Bucket {
|
|
|
805
937
|
}
|
|
806
938
|
|
|
807
939
|
public async getMagicBytes(optionsArg: { path: string; length: number }): Promise<Buffer> {
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
const stream = response.Body as any; // SdkStreamMixin includes readable stream
|
|
940
|
+
const command = new plugins.s3.GetObjectCommand({
|
|
941
|
+
Bucket: this.name,
|
|
942
|
+
Key: optionsArg.path,
|
|
943
|
+
Range: `bytes=0-${optionsArg.length - 1}`,
|
|
944
|
+
});
|
|
945
|
+
const response = await this.smartbucketRef.storageClient.send(command);
|
|
946
|
+
const chunks: Buffer[] = [];
|
|
947
|
+
const stream = response.Body as any; // SdkStreamMixin includes readable stream
|
|
817
948
|
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
}
|
|
821
|
-
return Buffer.concat(chunks);
|
|
822
|
-
} catch (error) {
|
|
823
|
-
console.error(
|
|
824
|
-
`Error retrieving magic bytes from object at path '${optionsArg.path}' in bucket '${this.name}':`,
|
|
825
|
-
error
|
|
826
|
-
);
|
|
827
|
-
throw error;
|
|
949
|
+
for await (const chunk of stream) {
|
|
950
|
+
chunks.push(chunk);
|
|
828
951
|
}
|
|
952
|
+
return Buffer.concat(chunks);
|
|
829
953
|
}
|
|
830
954
|
|
|
831
955
|
// ==========================================
|
|
@@ -933,73 +1057,90 @@ export class Bucket {
|
|
|
933
1057
|
public async listObjectKeysPage(
|
|
934
1058
|
optionsArg: IListObjectKeysPageOptions,
|
|
935
1059
|
): Promise<IListObjectKeysPageResult> {
|
|
936
|
-
|
|
937
|
-
throw new TypeError('Object key page options must be a plain object');
|
|
938
|
-
}
|
|
939
|
-
const prefix = optionsArg.prefix ?? '';
|
|
940
|
-
if (typeof prefix !== 'string') {
|
|
941
|
-
throw new TypeError('Object key page prefix must be a string');
|
|
942
|
-
}
|
|
943
|
-
if (!Number.isInteger(optionsArg.limit) || optionsArg.limit < 1 || optionsArg.limit > 1_000) {
|
|
944
|
-
throw new Error('Object key page limit must be an integer from 1 to 1000');
|
|
945
|
-
}
|
|
946
|
-
if (
|
|
947
|
-
optionsArg.startAfter !== undefined
|
|
948
|
-
&& (
|
|
949
|
-
typeof optionsArg.startAfter !== 'string'
|
|
950
|
-
|| optionsArg.startAfter.length === 0
|
|
951
|
-
|| !optionsArg.startAfter.startsWith(prefix)
|
|
952
|
-
)
|
|
953
|
-
) {
|
|
954
|
-
throw new Error('Object key page startAfter must be a non-empty key under the prefix');
|
|
955
|
-
}
|
|
956
|
-
|
|
1060
|
+
const prefix = assertListingPageOptions(optionsArg, 'Object key page');
|
|
957
1061
|
const response = await this.smartbucketRef.storageClient.send(
|
|
958
|
-
|
|
959
|
-
Bucket: this.name,
|
|
960
|
-
Prefix: prefix,
|
|
961
|
-
MaxKeys: optionsArg.limit,
|
|
962
|
-
...(optionsArg.startAfter ? { StartAfter: optionsArg.startAfter } : {}),
|
|
963
|
-
}),
|
|
1062
|
+
createListingPageCommand(this.name, prefix, optionsArg),
|
|
964
1063
|
);
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
1064
|
+
const { keys, nextStartAfter } = readListingPageContents(
|
|
1065
|
+
response,
|
|
1066
|
+
prefix,
|
|
1067
|
+
optionsArg,
|
|
1068
|
+
'key page',
|
|
1069
|
+
);
|
|
1070
|
+
return {
|
|
1071
|
+
keys,
|
|
1072
|
+
...(nextStartAfter !== undefined ? { nextStartAfter } : {}),
|
|
1073
|
+
};
|
|
1074
|
+
}
|
|
971
1075
|
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
1076
|
+
/**
|
|
1077
|
+
* List one keyset page of objects with the size, ETag and last-modified time
|
|
1078
|
+
* ListObjectsV2 already returns, so a caller needs no HEAD per object.
|
|
1079
|
+
* Pagination, prefix, limit (1 to 1000) and `startAfter` behave exactly like
|
|
1080
|
+
* `listObjectKeysPage()`: `nextStartAfter` is present only while the provider
|
|
1081
|
+
* reports more entries. `signal` aborts the in-flight request and rejects with
|
|
1082
|
+
* its reason. This requires a general-purpose S3 provider with UTF-8
|
|
1083
|
+
* byte-ordered keys.
|
|
1084
|
+
*/
|
|
1085
|
+
public async listObjectEntriesPage(
|
|
1086
|
+
optionsArg: IListObjectEntriesPageOptions,
|
|
1087
|
+
): Promise<IListObjectEntriesPageResult> {
|
|
1088
|
+
const prefix = assertListingPageOptions(optionsArg, 'Object entry page');
|
|
1089
|
+
if (optionsArg.signal !== undefined && !(optionsArg.signal instanceof AbortSignal)) {
|
|
1090
|
+
throw new TypeError('Object entry page signal must be an AbortSignal');
|
|
980
1091
|
}
|
|
981
|
-
|
|
982
|
-
|
|
1092
|
+
const operation = beginBasicBucketOperation(this, optionsArg.signal);
|
|
1093
|
+
try {
|
|
1094
|
+
operation.assertCurrent();
|
|
1095
|
+
const command = createListingPageCommand(operation.bucketName, prefix, optionsArg);
|
|
1096
|
+
const response = await operation.runProvider(
|
|
1097
|
+
() => operation.client.send(command, { abortSignal: operation.signal }),
|
|
1098
|
+
);
|
|
1099
|
+
operation.assertCurrent();
|
|
1100
|
+
const { contents, keys, nextStartAfter } = readListingPageContents(
|
|
1101
|
+
response,
|
|
1102
|
+
prefix,
|
|
1103
|
+
optionsArg,
|
|
1104
|
+
'entry page',
|
|
1105
|
+
);
|
|
1106
|
+
return {
|
|
1107
|
+
entries: contents.map((entry, index) => toObjectListingEntry(entry, keys[index])),
|
|
1108
|
+
...(nextStartAfter !== undefined ? { nextStartAfter } : {}),
|
|
1109
|
+
};
|
|
1110
|
+
} finally {
|
|
1111
|
+
operation.finish();
|
|
983
1112
|
}
|
|
1113
|
+
}
|
|
984
1114
|
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
1115
|
+
/**
|
|
1116
|
+
* Iterate every object under a prefix with its size, ETag and last-modified
|
|
1117
|
+
* time, one `listObjectEntriesPage()` request per page. No request is in
|
|
1118
|
+
* flight while the caller holds an entry; breaking out stops the listing.
|
|
1119
|
+
* @example
|
|
1120
|
+
* ```ts
|
|
1121
|
+
* for await (const entry of bucket.listAllObjectEntries('npm/', { signal })) {
|
|
1122
|
+
* console.log(entry.key, entry.size, entry.etag);
|
|
1123
|
+
* }
|
|
1124
|
+
* ```
|
|
1125
|
+
*/
|
|
1126
|
+
public async *listAllObjectEntries(
|
|
1127
|
+
prefix: string = '',
|
|
1128
|
+
optionsArg: IListAllObjectEntriesOptions = {},
|
|
1129
|
+
): AsyncIterableIterator<IObjectListingEntry> {
|
|
1130
|
+
const limit = optionsArg.pageSize ?? 1_000;
|
|
1131
|
+
let startAfter: string | undefined;
|
|
1132
|
+
do {
|
|
1133
|
+
const page = await this.listObjectEntriesPage({
|
|
1134
|
+
prefix,
|
|
1135
|
+
limit,
|
|
1136
|
+
...(startAfter !== undefined ? { startAfter } : {}),
|
|
1137
|
+
...(optionsArg.signal ? { signal: optionsArg.signal } : {}),
|
|
1138
|
+
});
|
|
1139
|
+
for (const entry of page.entries) {
|
|
1140
|
+
yield entry;
|
|
992
1141
|
}
|
|
993
|
-
|
|
994
|
-
}
|
|
995
|
-
if (response.IsTruncated && keys.length === 0) {
|
|
996
|
-
throw new Error('Object storage returned an empty truncated key page');
|
|
997
|
-
}
|
|
998
|
-
|
|
999
|
-
return {
|
|
1000
|
-
keys,
|
|
1001
|
-
...(response.IsTruncated ? { nextStartAfter: keys[keys.length - 1] } : {}),
|
|
1002
|
-
};
|
|
1142
|
+
startAfter = page.nextStartAfter;
|
|
1143
|
+
} while (startAfter !== undefined);
|
|
1003
1144
|
}
|
|
1004
1145
|
|
|
1005
1146
|
/**
|
|
@@ -1061,50 +1202,41 @@ export class Bucket {
|
|
|
1061
1202
|
}
|
|
1062
1203
|
|
|
1063
1204
|
public async cleanAllContents(): Promise<void> {
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1205
|
+
// Define the command type explicitly
|
|
1206
|
+
const listCommandInput: plugins.s3.ListObjectsV2CommandInput = {
|
|
1207
|
+
Bucket: this.name,
|
|
1208
|
+
};
|
|
1209
|
+
|
|
1210
|
+
let isTruncated = true;
|
|
1211
|
+
let continuationToken: string | undefined = undefined;
|
|
1212
|
+
|
|
1213
|
+
while (isTruncated) {
|
|
1214
|
+
// Add the continuation token to the input if present
|
|
1215
|
+
const listCommand = new plugins.s3.ListObjectsV2Command({
|
|
1216
|
+
...listCommandInput,
|
|
1217
|
+
ContinuationToken: continuationToken,
|
|
1218
|
+
});
|
|
1219
|
+
|
|
1220
|
+
// Explicitly type the response
|
|
1221
|
+
const response: plugins.s3.ListObjectsV2Output =
|
|
1222
|
+
await this.smartbucketRef.storageClient.send(listCommand);
|
|
1223
|
+
|
|
1224
|
+
if (response.Contents && response.Contents.length > 0) {
|
|
1225
|
+
// Delete objects in batches, mapping each item to { Key: string }
|
|
1226
|
+
const deleteCommand = new plugins.s3.DeleteObjectsCommand({
|
|
1227
|
+
Bucket: this.name,
|
|
1228
|
+
Delete: {
|
|
1229
|
+
Objects: response.Contents.map((item) => ({ Key: item.Key! })),
|
|
1230
|
+
Quiet: true,
|
|
1231
|
+
},
|
|
1078
1232
|
});
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
const response: plugins.s3.ListObjectsV2Output =
|
|
1082
|
-
await this.smartbucketRef.storageClient.send(listCommand);
|
|
1083
|
-
|
|
1084
|
-
console.log(`Cleaning contents of bucket '${this.name}': Now deleting ${response.Contents?.length} items...`);
|
|
1085
|
-
|
|
1086
|
-
if (response.Contents && response.Contents.length > 0) {
|
|
1087
|
-
// Delete objects in batches, mapping each item to { Key: string }
|
|
1088
|
-
const deleteCommand = new plugins.s3.DeleteObjectsCommand({
|
|
1089
|
-
Bucket: this.name,
|
|
1090
|
-
Delete: {
|
|
1091
|
-
Objects: response.Contents.map((item) => ({ Key: item.Key! })),
|
|
1092
|
-
Quiet: true,
|
|
1093
|
-
},
|
|
1094
|
-
});
|
|
1095
|
-
|
|
1096
|
-
await this.smartbucketRef.storageClient.send(deleteCommand);
|
|
1097
|
-
}
|
|
1098
|
-
|
|
1099
|
-
// Update continuation token and truncation status
|
|
1100
|
-
isTruncated = response.IsTruncated || false;
|
|
1101
|
-
continuationToken = response.NextContinuationToken;
|
|
1233
|
+
|
|
1234
|
+
await this.smartbucketRef.storageClient.send(deleteCommand);
|
|
1102
1235
|
}
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
throw error;
|
|
1236
|
+
|
|
1237
|
+
// Update continuation token and truncation status
|
|
1238
|
+
isTruncated = response.IsTruncated || false;
|
|
1239
|
+
continuationToken = response.NextContinuationToken;
|
|
1108
1240
|
}
|
|
1109
1241
|
}
|
|
1110
1242
|
}
|