@cloudbitmaps/s3 0.10.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +11 -0
- package/README.md +76 -0
- package/dist/backend.d.ts +55 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.js +508 -0
- package/dist/index.js.map +7 -0
- package/dist/keys.d.ts +18 -0
- package/dist/registry.d.ts +38 -0
- package/dist/s3-errors.d.ts +32 -0
- package/dist/storage.d.ts +46 -0
- package/package.json +65 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
package/NOTICE
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
@cloudbitmaps/s3
|
|
2
|
+
Copyright 2026 Sharvil Kadam
|
|
3
|
+
|
|
4
|
+
This product includes software developed as an open-source project.
|
|
5
|
+
Licensed under the Apache License, Version 2.0 (the "License"); see the LICENSE
|
|
6
|
+
file. You may obtain a copy of the License at:
|
|
7
|
+
|
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
|
|
10
|
+
@cloudbitmaps/s3 is a storage driver for CloudBitmaps. It contains no bitmap code and is
|
|
11
|
+
codec-agnostic: it moves opaque payload bytes.
|
package/README.md
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# @cloudbitmaps/s3
|
|
2
|
+
|
|
3
|
+
**S3 and S3-compatible object storage for [CloudBitmaps](https://github.com/cloudbitmaps/cloudbitmaps).**
|
|
4
|
+
|
|
5
|
+
> **ESM-only, Node ≥ 22.12.** This package ships as ES modules; there is no CommonJS bundle.
|
|
6
|
+
> `require()` works on Node 22.12+ through `require(esm)`, but a runner with its own CommonJS loader
|
|
7
|
+
> (notably Jest in its default configuration) does not get that and needs `import` instead. On TypeScript,
|
|
8
|
+
> a CommonJS project needs `"module": "nodenext"` or `"node20"`. See the
|
|
9
|
+
> [repository README](https://github.com/cloudbitmaps/cloudbitmaps#install--entry-points) for the details.
|
|
10
|
+
|
|
11
|
+
AWS S3 — and every S3-compatible service: Cloudflare R2, MinIO, Ceph, Wasabi, Backblaze B2.
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pnpm add @cloudbitmaps/roaring @cloudbitmaps/s3
|
|
17
|
+
# npm i @cloudbitmaps/roaring @cloudbitmaps/s3 # the same, with npm
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
> **On pnpm 10+, allow the one build script.** pnpm 10 skips dependency build scripts by default, so
|
|
21
|
+
> the `roaring` native addon never downloads and the package throws at `import` — while the install
|
|
22
|
+
> itself prints a warning and **exits 0**. Add this to your `package.json`, then install:
|
|
23
|
+
>
|
|
24
|
+
> ```json
|
|
25
|
+
> { "pnpm": { "onlyBuiltDependencies": ["roaring"] } }
|
|
26
|
+
> ```
|
|
27
|
+
>
|
|
28
|
+
> pnpm 9 and npm run it already. [Full symptoms and fixes](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/getting-started.md#cannot-find-module-buildreleaseroaringnode-after-a-successful-install).
|
|
29
|
+
|
|
30
|
+
Two packages: the **codec** you want and the **storage** you have. `@aws-sdk/client-s3` (`>=3.645.0 <4`) is a real dependency of
|
|
31
|
+
this package, so installing it is the whole step — there is no optional peer to remember. `@cloudbitmaps/core` is one
|
|
32
|
+
too, so the engine lands in your tree without you installing it — you never name it yourself.
|
|
33
|
+
|
|
34
|
+
> **The `>=3.645.0` floor is a correctness floor, not a preference.** Below it the SDK does not model the
|
|
35
|
+
> conditional write this library's write-once guarantee is built on: measured against MinIO, **3.640.0
|
|
36
|
+
> silently overwrites** an existing object instead of refusing, which loses a published generation without
|
|
37
|
+
> an error; 3.641.0 rejects correctly. If you pin `@aws-sdk/client-s3` yourself, raise the pin to at least
|
|
38
|
+
> `3.645.0` — an older pin will fail to resolve against this package rather than quietly downgrading you.
|
|
39
|
+
|
|
40
|
+
## Use
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import { CloudRoaring } from '@cloudbitmaps/roaring';
|
|
44
|
+
import { S3Storage } from '@cloudbitmaps/s3';
|
|
45
|
+
|
|
46
|
+
const store = new CloudRoaring({
|
|
47
|
+
storage: new S3Storage({ bucket: 'bitmaps', prefix: 'cr', region: 'us-east-1' }),
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
await store.load({ segment: 'active-users' }, [1, 2, 3]);
|
|
51
|
+
const seg = store.segment('active-users');
|
|
52
|
+
await seg.has(2); // true
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`S3Storage` configures both halves — the immutable generation objects and the registry pointer row — from one set
|
|
56
|
+
of values. Need them apart? This package also exports the two drivers and their option types; see the
|
|
57
|
+
[API reference](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md).
|
|
58
|
+
|
|
59
|
+
## What this package is
|
|
60
|
+
|
|
61
|
+
These drivers move opaque payload bytes, so they are codec-agnostic: the same package serves every codec
|
|
62
|
+
flavor. That is why storage is a package rather than a subpath of one — as a subpath, each flavor needed a
|
|
63
|
+
re-export barrel per service, and the count multiplied with every codec added.
|
|
64
|
+
|
|
65
|
+
Built against `@cloudbitmaps/core/driver-kit`, the declared contract for a driver — the same surface a
|
|
66
|
+
third-party driver would use.
|
|
67
|
+
|
|
68
|
+
## Documentation
|
|
69
|
+
|
|
70
|
+
- [Getting started](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/getting-started.md)
|
|
71
|
+
- [API reference](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md)
|
|
72
|
+
- [Changelog](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/CHANGELOG.md)
|
|
73
|
+
|
|
74
|
+
## License
|
|
75
|
+
|
|
76
|
+
Apache-2.0
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `S3Storage` — the S3 backend as one object: the generations and the pointer, in one bucket, stated once.
|
|
3
|
+
*
|
|
4
|
+
* Replaces two constructor calls that each repeated `client`, `bucket` and `prefix`. Repeating them is how
|
|
5
|
+
* they come apart: point the registry at one prefix and the objects at another and the store answers *empty*
|
|
6
|
+
* rather than *misconfigured*, which is the hardest kind of wrong answer to debug. Here the location is written
|
|
7
|
+
* once and shared, so the mismatch cannot be expressed.
|
|
8
|
+
*
|
|
9
|
+
* **It will build a client for you**, which is the common case — `new S3Storage({ bucket })` picks up the
|
|
10
|
+
* ambient credential chain and region exactly as the SDK would. Pass `client` instead when you need a
|
|
11
|
+
* credential chain the SDK cannot infer (SSO, an assumed role, a custom retry strategy); pass `endpoint` +
|
|
12
|
+
* `pathStyle` + `credentials` for an S3-compatible store (MinIO, Ceph, R2). Both halves stay reachable as `.storage` and
|
|
13
|
+
* `.registry` for anyone wiring something the facade does not cover.
|
|
14
|
+
*/
|
|
15
|
+
import { STORAGE_BACKEND } from '@cloudbitmaps/core/driver-kit';
|
|
16
|
+
import type { IRegistryDriver, IStorageDriver, StorageBackend } from '@cloudbitmaps/core/driver-kit';
|
|
17
|
+
import { S3Client } from '@aws-sdk/client-s3';
|
|
18
|
+
export interface S3StorageOptions {
|
|
19
|
+
/** Target bucket (must already exist). */
|
|
20
|
+
readonly bucket: string;
|
|
21
|
+
/** Optional key prefix under which everything lives — generations and the registry alike. */
|
|
22
|
+
readonly prefix?: string;
|
|
23
|
+
/** A constructed client. Supply one for a credential chain the SDK cannot infer; otherwise one is built. */
|
|
24
|
+
readonly client?: S3Client;
|
|
25
|
+
/** Region for the client built when `client` is absent. Falls back to the SDK's own resolution. */
|
|
26
|
+
readonly region?: string;
|
|
27
|
+
/** Endpoint for an S3-compatible store (MinIO, Ceph, R2). Ignored when `client` is supplied. */
|
|
28
|
+
readonly endpoint?: string;
|
|
29
|
+
/** Path-style addressing, which most S3-compatible stores require. Ignored when `client` is supplied. */
|
|
30
|
+
readonly pathStyle?: boolean;
|
|
31
|
+
/**
|
|
32
|
+
* Static credentials, for the S3-compatible stores that issue them (MinIO, Ceph, R2).
|
|
33
|
+
*
|
|
34
|
+
* On AWS itself, leave this unset — the SDK's own chain (instance role, SSO, environment, profile) is what
|
|
35
|
+
* you want, and hard-coding keys to reach it would be a downgrade. It exists because the alternative for a
|
|
36
|
+
* MinIO user was to construct an `S3Client` purely to carry two strings, which is the ergonomics this class
|
|
37
|
+
* is here to remove. Ignored when `client` is supplied.
|
|
38
|
+
*/
|
|
39
|
+
readonly credentials?: {
|
|
40
|
+
readonly accessKeyId: string;
|
|
41
|
+
readonly secretAccessKey: string;
|
|
42
|
+
readonly sessionToken?: string;
|
|
43
|
+
};
|
|
44
|
+
/** Injected clock for the registry's `createdAt`/`updatedAt`; defaults to `Date.now`. */
|
|
45
|
+
readonly now?: () => number;
|
|
46
|
+
}
|
|
47
|
+
export declare class S3Storage implements StorageBackend {
|
|
48
|
+
/** Cross-bundle brand, stamped non-enumerably in the constructor so a spread cannot carry it. */
|
|
49
|
+
readonly [STORAGE_BACKEND]: true;
|
|
50
|
+
readonly storage: IStorageDriver;
|
|
51
|
+
readonly registry: IRegistryDriver;
|
|
52
|
+
/** The client both halves share — built here unless one was supplied. */
|
|
53
|
+
readonly client: S3Client;
|
|
54
|
+
constructor(options: S3StorageOptions);
|
|
55
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@cloudbitmaps/s3` — S3 and every S3-compatible service — Cloudflare R2, MinIO, Ceph, Wasabi, Backblaze B2.
|
|
3
|
+
*
|
|
4
|
+
* One package per storage SERVICE, with `@aws-sdk/client-s3` as a real dependency. Install it alongside a
|
|
5
|
+
* flavor package and wire the backend in one line:
|
|
6
|
+
*
|
|
7
|
+
* ```ts
|
|
8
|
+
* import { CloudRoaring } from '@cloudbitmaps/roaring';
|
|
9
|
+
* import { S3Storage } from '@cloudbitmaps/s3';
|
|
10
|
+
*
|
|
11
|
+
* const store = new CloudRoaring({ storage: new S3Storage({ bucket: 'bitmaps', prefix: 'cr' }) });
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* These drivers move opaque payload bytes, so they are codec-agnostic: the same package serves every
|
|
15
|
+
* flavor. That is why they are a package rather than a subpath of one — as a subpath, each flavor needed a
|
|
16
|
+
* re-export barrel per service, and the count multiplied with every new codec.
|
|
17
|
+
*
|
|
18
|
+
* The engine, the `.crbm` format and the ports live in `@cloudbitmaps/core`, which is a real dependency of
|
|
19
|
+
* THIS package: it lands in your tree without you installing it, and you never name it yourself. What this
|
|
20
|
+
* package builds against is `@cloudbitmaps/core/driver-kit`, the declared contract for a driver — the same
|
|
21
|
+
* surface a third-party driver would use.
|
|
22
|
+
*/
|
|
23
|
+
export { S3StorageDriver } from './storage.js';
|
|
24
|
+
export type { S3StorageDriverOptions } from './storage.js';
|
|
25
|
+
export { S3RegistryDriver } from './registry.js';
|
|
26
|
+
export type { S3RegistryDriverOptions } from './registry.js';
|
|
27
|
+
export { S3Storage } from './backend.js';
|
|
28
|
+
export type { S3StorageOptions } from './backend.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,508 @@
|
|
|
1
|
+
// src/storage.ts
|
|
2
|
+
import {
|
|
3
|
+
NotFoundError,
|
|
4
|
+
TransientError,
|
|
5
|
+
ValidationError as ValidationError2,
|
|
6
|
+
WriteConflictError,
|
|
7
|
+
isNotFoundError,
|
|
8
|
+
isValidationError,
|
|
9
|
+
isWriteConflictError
|
|
10
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
11
|
+
import { createHash } from "node:crypto";
|
|
12
|
+
import {
|
|
13
|
+
AbortMultipartUploadCommand,
|
|
14
|
+
CompleteMultipartUploadCommand,
|
|
15
|
+
CreateMultipartUploadCommand,
|
|
16
|
+
DeleteObjectCommand,
|
|
17
|
+
GetObjectCommand,
|
|
18
|
+
HeadObjectCommand,
|
|
19
|
+
ListObjectsV2Command,
|
|
20
|
+
PutObjectCommand,
|
|
21
|
+
UploadPartCommand
|
|
22
|
+
} from "@aws-sdk/client-s3";
|
|
23
|
+
|
|
24
|
+
// src/keys.ts
|
|
25
|
+
import {
|
|
26
|
+
ValidationError,
|
|
27
|
+
encodeNameForKey,
|
|
28
|
+
namespaceKeyPart,
|
|
29
|
+
prefixPart,
|
|
30
|
+
validateSegmentRef
|
|
31
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
32
|
+
import { normalizeObjectPrefix } from "@cloudbitmaps/core/driver-kit";
|
|
33
|
+
import {
|
|
34
|
+
registryPrefix,
|
|
35
|
+
registryObjectKey,
|
|
36
|
+
registryListPrefix,
|
|
37
|
+
parseRegistryKey
|
|
38
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
39
|
+
var SUFFIX = ".crbm";
|
|
40
|
+
function segmentObjectPrefix(prefix, ref) {
|
|
41
|
+
validateSegmentRef(ref);
|
|
42
|
+
return `${prefixPart(prefix)}${namespaceKeyPart(ref.namespace)}/segments/${encodeNameForKey(ref.segment)}.`;
|
|
43
|
+
}
|
|
44
|
+
function storageObjectKey(prefix, key) {
|
|
45
|
+
if (!Number.isInteger(key.generation) || key.generation < 0) {
|
|
46
|
+
throw new ValidationError(`generation must be a non-negative integer; got ${key.generation}`);
|
|
47
|
+
}
|
|
48
|
+
return `${segmentObjectPrefix(prefix, key)}${key.generation}${SUFFIX}`;
|
|
49
|
+
}
|
|
50
|
+
function parseGenerationFromKey(segmentPrefix, objectKey) {
|
|
51
|
+
if (!objectKey.startsWith(segmentPrefix) || !objectKey.endsWith(SUFFIX)) return null;
|
|
52
|
+
const middle = objectKey.slice(segmentPrefix.length, objectKey.length - SUFFIX.length);
|
|
53
|
+
if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
|
|
54
|
+
const generation = Number(middle);
|
|
55
|
+
return Number.isSafeInteger(generation) ? generation : null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// src/s3-errors.ts
|
|
59
|
+
import {
|
|
60
|
+
errorName,
|
|
61
|
+
httpStatus,
|
|
62
|
+
isNetworkOrTimeout,
|
|
63
|
+
isSdkRetryable,
|
|
64
|
+
isServerSide
|
|
65
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
66
|
+
function isPreconditionFailed(err) {
|
|
67
|
+
return errorName(err) === "PreconditionFailed" || httpStatus(err) === 412;
|
|
68
|
+
}
|
|
69
|
+
function isConditionalConflict(err) {
|
|
70
|
+
return isPreconditionFailed(err) || errorName(err) === "ConditionalRequestConflict" || httpStatus(err) === 409;
|
|
71
|
+
}
|
|
72
|
+
function isNotFound(err) {
|
|
73
|
+
const name = errorName(err);
|
|
74
|
+
return name === "NoSuchKey" || name === "NotFound" || httpStatus(err) === 404;
|
|
75
|
+
}
|
|
76
|
+
function isInvalidRange(err) {
|
|
77
|
+
return errorName(err) === "InvalidRange" || httpStatus(err) === 416;
|
|
78
|
+
}
|
|
79
|
+
function isTransient(err) {
|
|
80
|
+
if (isConditionalConflict(err) || isNotFound(err) || isInvalidRange(err)) return false;
|
|
81
|
+
return errorName(err) === "SlowDown" || isServerSide(err) || isNetworkOrTimeout(err) || isSdkRetryable(err);
|
|
82
|
+
}
|
|
83
|
+
function totalFromContentRange(contentRange) {
|
|
84
|
+
if (contentRange === void 0) return void 0;
|
|
85
|
+
const match = /\/(\d+)\s*$/.exec(contentRange);
|
|
86
|
+
if (match === null) return void 0;
|
|
87
|
+
const total = Number(match[1]);
|
|
88
|
+
return Number.isSafeInteger(total) ? total : void 0;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// src/storage.ts
|
|
92
|
+
var S3_PART_BYTES = 8 * 1024 * 1024;
|
|
93
|
+
var S3_MAX_PARTS = 1e4;
|
|
94
|
+
var S3StorageDriver = class {
|
|
95
|
+
client;
|
|
96
|
+
bucket;
|
|
97
|
+
prefix;
|
|
98
|
+
maxObjectBytes;
|
|
99
|
+
partBytes;
|
|
100
|
+
constructor(options) {
|
|
101
|
+
this.client = options.client;
|
|
102
|
+
this.bucket = options.bucket;
|
|
103
|
+
this.prefix = normalizeObjectPrefix(options.prefix);
|
|
104
|
+
const requestedPart = Math.max(options.partBytes ?? S3_PART_BYTES, 5 * 1024 * 1024);
|
|
105
|
+
this.maxObjectBytes = options.maxObjectBytes ?? requestedPart * S3_MAX_PARTS;
|
|
106
|
+
this.partBytes = Math.max(requestedPart, Math.ceil(this.maxObjectBytes / S3_MAX_PARTS));
|
|
107
|
+
}
|
|
108
|
+
capabilities() {
|
|
109
|
+
return { rangeRead: true, maxObjectBytes: this.maxObjectBytes, conditionalPut: true };
|
|
110
|
+
}
|
|
111
|
+
async putImmutable(key, write) {
|
|
112
|
+
const objectKey = storageObjectKey(this.prefix, key);
|
|
113
|
+
const sink = new S3MultipartSink(
|
|
114
|
+
this.client,
|
|
115
|
+
this.bucket,
|
|
116
|
+
objectKey,
|
|
117
|
+
this.partBytes,
|
|
118
|
+
this.maxObjectBytes
|
|
119
|
+
);
|
|
120
|
+
try {
|
|
121
|
+
await write(sink);
|
|
122
|
+
return await sink.finish();
|
|
123
|
+
} catch (err) {
|
|
124
|
+
await sink.abort();
|
|
125
|
+
if (isConditionalConflict(err)) {
|
|
126
|
+
throw new WriteConflictError(
|
|
127
|
+
`generation already exists (write-once): ${key.segment}.${key.generation}`
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
if (isValidationError(err) || isWriteConflictError(err) || isNotFoundError(err)) {
|
|
131
|
+
throw err;
|
|
132
|
+
}
|
|
133
|
+
throw this.mapError(err);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
async getRange(key, offset, length) {
|
|
137
|
+
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0) {
|
|
138
|
+
throw new ValidationError2(`invalid range offset=${offset} length=${length}`);
|
|
139
|
+
}
|
|
140
|
+
const objectKey = storageObjectKey(this.prefix, key);
|
|
141
|
+
if (length === 0) return new Uint8Array(0);
|
|
142
|
+
try {
|
|
143
|
+
const res = await this.client.send(
|
|
144
|
+
new GetObjectCommand({
|
|
145
|
+
Bucket: this.bucket,
|
|
146
|
+
Key: objectKey,
|
|
147
|
+
Range: `bytes=${offset}-${offset + length - 1}`
|
|
148
|
+
})
|
|
149
|
+
);
|
|
150
|
+
const bytes = await collect(res.Body);
|
|
151
|
+
if (bytes.length !== length) {
|
|
152
|
+
throw new ValidationError2(
|
|
153
|
+
`range [${offset}, ${offset + length}) out of bounds (got ${bytes.length}B)`
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
return bytes;
|
|
157
|
+
} catch (err) {
|
|
158
|
+
throw this.mapReadError(err, key);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
async getTail(key, maxBytes) {
|
|
162
|
+
const objectKey = storageObjectKey(this.prefix, key);
|
|
163
|
+
if (maxBytes <= 0) {
|
|
164
|
+
try {
|
|
165
|
+
const head = await this.client.send(
|
|
166
|
+
new HeadObjectCommand({ Bucket: this.bucket, Key: objectKey })
|
|
167
|
+
);
|
|
168
|
+
return { bytes: new Uint8Array(0), size: head.ContentLength ?? 0 };
|
|
169
|
+
} catch (err) {
|
|
170
|
+
throw this.mapReadError(err, key);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
try {
|
|
174
|
+
const res = await this.client.send(
|
|
175
|
+
new GetObjectCommand({ Bucket: this.bucket, Key: objectKey, Range: `bytes=-${maxBytes}` })
|
|
176
|
+
);
|
|
177
|
+
const bytes = await collect(res.Body);
|
|
178
|
+
let size = totalFromContentRange(res.ContentRange);
|
|
179
|
+
if (size === void 0) {
|
|
180
|
+
if (bytes.length === maxBytes) {
|
|
181
|
+
const head = await this.client.send(
|
|
182
|
+
new HeadObjectCommand({ Bucket: this.bucket, Key: objectKey })
|
|
183
|
+
);
|
|
184
|
+
size = head.ContentLength ?? bytes.length;
|
|
185
|
+
} else {
|
|
186
|
+
size = bytes.length;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return { bytes, size };
|
|
190
|
+
} catch (err) {
|
|
191
|
+
throw this.mapReadError(err, key);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
async delete(key) {
|
|
195
|
+
try {
|
|
196
|
+
await this.client.send(
|
|
197
|
+
new DeleteObjectCommand({ Bucket: this.bucket, Key: storageObjectKey(this.prefix, key) })
|
|
198
|
+
);
|
|
199
|
+
} catch (err) {
|
|
200
|
+
throw this.mapError(err);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
async *list(ref) {
|
|
204
|
+
const prefix = segmentObjectPrefix(this.prefix, ref);
|
|
205
|
+
let token;
|
|
206
|
+
do {
|
|
207
|
+
let res;
|
|
208
|
+
try {
|
|
209
|
+
res = await this.client.send(
|
|
210
|
+
new ListObjectsV2Command({
|
|
211
|
+
Bucket: this.bucket,
|
|
212
|
+
Prefix: prefix,
|
|
213
|
+
ContinuationToken: token
|
|
214
|
+
})
|
|
215
|
+
);
|
|
216
|
+
} catch (err) {
|
|
217
|
+
throw this.mapError(err);
|
|
218
|
+
}
|
|
219
|
+
for (const obj of res.Contents ?? []) {
|
|
220
|
+
if (obj.Key === void 0) continue;
|
|
221
|
+
const generation = parseGenerationFromKey(prefix, obj.Key);
|
|
222
|
+
if (generation !== null) {
|
|
223
|
+
yield { namespace: ref.namespace, segment: ref.segment, generation };
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
token = res.IsTruncated === true ? res.NextContinuationToken : void 0;
|
|
227
|
+
} while (token !== void 0);
|
|
228
|
+
}
|
|
229
|
+
/** Map S3 read errors to the driver vocabulary; pass everything else through {@link mapError}. */
|
|
230
|
+
mapReadError(err, key) {
|
|
231
|
+
if (isValidationError(err)) return err;
|
|
232
|
+
if (isNotFound(err)) {
|
|
233
|
+
return new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
234
|
+
}
|
|
235
|
+
if (isInvalidRange(err)) {
|
|
236
|
+
return new ValidationError2(`range out of bounds for ${key.segment}.${key.generation}`);
|
|
237
|
+
}
|
|
238
|
+
return this.mapError(err);
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Reclassify a transient S3 fault (throttle/5xx/dropped connection) as a retryable {@link TransientError}
|
|
242
|
+
* so the retry decorator can ride it out; everything else propagates unchanged. The final fallback at every
|
|
243
|
+
* `client.send` site, so callers and the decorator only ever see typed errors.
|
|
244
|
+
*/
|
|
245
|
+
mapError(err) {
|
|
246
|
+
if (isTransient(err)) {
|
|
247
|
+
return new TransientError(
|
|
248
|
+
`transient S3 fault: ${err?.name ?? "unknown"}`,
|
|
249
|
+
{ cause: err }
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
return err;
|
|
253
|
+
}
|
|
254
|
+
};
|
|
255
|
+
function concatBytes(parts, total) {
|
|
256
|
+
const out = new Uint8Array(total);
|
|
257
|
+
let offset = 0;
|
|
258
|
+
for (const p of parts) {
|
|
259
|
+
out.set(p, offset);
|
|
260
|
+
offset += p.length;
|
|
261
|
+
}
|
|
262
|
+
return out;
|
|
263
|
+
}
|
|
264
|
+
var S3MultipartSink = class {
|
|
265
|
+
constructor(client, bucket, objectKey, partBytes, maxObjectBytes) {
|
|
266
|
+
this.client = client;
|
|
267
|
+
this.bucket = bucket;
|
|
268
|
+
this.objectKey = objectKey;
|
|
269
|
+
this.partBytes = partBytes;
|
|
270
|
+
this.maxObjectBytes = maxObjectBytes;
|
|
271
|
+
}
|
|
272
|
+
client;
|
|
273
|
+
bucket;
|
|
274
|
+
objectKey;
|
|
275
|
+
partBytes;
|
|
276
|
+
maxObjectBytes;
|
|
277
|
+
hash = createHash("sha256");
|
|
278
|
+
pending = [];
|
|
279
|
+
pendingLen = 0;
|
|
280
|
+
total = 0;
|
|
281
|
+
uploadId;
|
|
282
|
+
partNumber = 0;
|
|
283
|
+
parts = [];
|
|
284
|
+
async write(bytes) {
|
|
285
|
+
if (bytes.length === 0) return;
|
|
286
|
+
this.total += bytes.length;
|
|
287
|
+
if (this.total > this.maxObjectBytes) {
|
|
288
|
+
throw new ValidationError2(`object exceeds maxObjectBytes ${this.maxObjectBytes}`);
|
|
289
|
+
}
|
|
290
|
+
this.hash.update(bytes);
|
|
291
|
+
this.pending.push(bytes);
|
|
292
|
+
this.pendingLen += bytes.length;
|
|
293
|
+
if (this.pendingLen >= this.partBytes) await this.flushPart();
|
|
294
|
+
}
|
|
295
|
+
/** Upload the buffered bytes (≥ one part) as a single part, freeing them. Starts the upload on first call. */
|
|
296
|
+
async flushPart() {
|
|
297
|
+
if (this.uploadId === void 0) {
|
|
298
|
+
const res2 = await this.client.send(
|
|
299
|
+
new CreateMultipartUploadCommand({ Bucket: this.bucket, Key: this.objectKey })
|
|
300
|
+
);
|
|
301
|
+
if (res2.UploadId === void 0) {
|
|
302
|
+
throw new TransientError("S3 CreateMultipartUpload returned no UploadId");
|
|
303
|
+
}
|
|
304
|
+
this.uploadId = res2.UploadId;
|
|
305
|
+
}
|
|
306
|
+
const body = concatBytes(this.pending, this.pendingLen);
|
|
307
|
+
this.pending.length = 0;
|
|
308
|
+
this.pendingLen = 0;
|
|
309
|
+
this.partNumber += 1;
|
|
310
|
+
if (this.partNumber > S3_MAX_PARTS) {
|
|
311
|
+
throw new ValidationError2(`multipart upload exceeded the S3 ${S3_MAX_PARTS}-part limit`);
|
|
312
|
+
}
|
|
313
|
+
const res = await this.client.send(
|
|
314
|
+
new UploadPartCommand({
|
|
315
|
+
Bucket: this.bucket,
|
|
316
|
+
Key: this.objectKey,
|
|
317
|
+
UploadId: this.uploadId,
|
|
318
|
+
PartNumber: this.partNumber,
|
|
319
|
+
Body: body
|
|
320
|
+
})
|
|
321
|
+
);
|
|
322
|
+
this.parts.push({ ETag: res.ETag, PartNumber: this.partNumber });
|
|
323
|
+
}
|
|
324
|
+
/** Commit the object: a single conditional PUT if it fit in one part, else complete the multipart upload. */
|
|
325
|
+
async finish() {
|
|
326
|
+
const sha256 = this.hash.digest("hex");
|
|
327
|
+
if (this.uploadId === void 0) {
|
|
328
|
+
await this.client.send(
|
|
329
|
+
new PutObjectCommand({
|
|
330
|
+
Bucket: this.bucket,
|
|
331
|
+
Key: this.objectKey,
|
|
332
|
+
Body: concatBytes(this.pending, this.pendingLen),
|
|
333
|
+
IfNoneMatch: "*"
|
|
334
|
+
// write-once
|
|
335
|
+
})
|
|
336
|
+
);
|
|
337
|
+
return { size: this.total, sha256 };
|
|
338
|
+
}
|
|
339
|
+
if (this.pendingLen > 0) await this.flushPart();
|
|
340
|
+
await this.client.send(
|
|
341
|
+
new CompleteMultipartUploadCommand({
|
|
342
|
+
Bucket: this.bucket,
|
|
343
|
+
Key: this.objectKey,
|
|
344
|
+
UploadId: this.uploadId,
|
|
345
|
+
MultipartUpload: { Parts: this.parts },
|
|
346
|
+
IfNoneMatch: "*"
|
|
347
|
+
// write-once: fail if the object already exists
|
|
348
|
+
})
|
|
349
|
+
);
|
|
350
|
+
this.uploadId = void 0;
|
|
351
|
+
return { size: this.total, sha256 };
|
|
352
|
+
}
|
|
353
|
+
/** Best-effort cleanup of an in-flight multipart upload after an error (a leaked MPU is reaped by a bucket
|
|
354
|
+
* lifecycle rule; never a correctness issue). No-op if nothing was started or it already completed. */
|
|
355
|
+
async abort() {
|
|
356
|
+
if (this.uploadId === void 0) return;
|
|
357
|
+
const id = this.uploadId;
|
|
358
|
+
this.uploadId = void 0;
|
|
359
|
+
try {
|
|
360
|
+
await this.client.send(
|
|
361
|
+
new AbortMultipartUploadCommand({ Bucket: this.bucket, Key: this.objectKey, UploadId: id })
|
|
362
|
+
);
|
|
363
|
+
} catch {
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
};
|
|
367
|
+
async function collect(body) {
|
|
368
|
+
if (body === void 0) {
|
|
369
|
+
throw new NotFoundError("S3 GetObject returned an empty body");
|
|
370
|
+
}
|
|
371
|
+
return body.transformToByteArray();
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
// src/registry.ts
|
|
375
|
+
import {
|
|
376
|
+
IntegrityError,
|
|
377
|
+
MAX_ROW_BYTES,
|
|
378
|
+
ObjectStoreRegistry,
|
|
379
|
+
TransientError as TransientError2,
|
|
380
|
+
WriteConflictError as WriteConflictError2,
|
|
381
|
+
normalizeObjectPrefix as normalizeObjectPrefix2
|
|
382
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
383
|
+
import {
|
|
384
|
+
GetObjectCommand as GetObjectCommand2,
|
|
385
|
+
ListObjectsV2Command as ListObjectsV2Command2,
|
|
386
|
+
PutObjectCommand as PutObjectCommand2
|
|
387
|
+
} from "@aws-sdk/client-s3";
|
|
388
|
+
var S3Store = class {
|
|
389
|
+
constructor(client, bucket) {
|
|
390
|
+
this.client = client;
|
|
391
|
+
this.bucket = bucket;
|
|
392
|
+
}
|
|
393
|
+
client;
|
|
394
|
+
bucket;
|
|
395
|
+
label = "S3";
|
|
396
|
+
async read(key) {
|
|
397
|
+
let res;
|
|
398
|
+
try {
|
|
399
|
+
res = await this.client.send(new GetObjectCommand2({ Bucket: this.bucket, Key: key }));
|
|
400
|
+
} catch (err) {
|
|
401
|
+
if (isNotFound(err)) return null;
|
|
402
|
+
throw mapError(err);
|
|
403
|
+
}
|
|
404
|
+
if ((res.ContentLength ?? 0) > MAX_ROW_BYTES) {
|
|
405
|
+
throw new IntegrityError(
|
|
406
|
+
`registry object ${res.ContentLength}B exceeds cap ${MAX_ROW_BYTES}B`
|
|
407
|
+
);
|
|
408
|
+
}
|
|
409
|
+
if (res.Body === void 0) {
|
|
410
|
+
throw new IntegrityError(`registry object has an empty body: ${key}`);
|
|
411
|
+
}
|
|
412
|
+
const bytes = await res.Body.transformToByteArray();
|
|
413
|
+
return { bytes, version: res.ETag ?? "" };
|
|
414
|
+
}
|
|
415
|
+
async write(key, body, expect) {
|
|
416
|
+
try {
|
|
417
|
+
await this.client.send(
|
|
418
|
+
new PutObjectCommand2({
|
|
419
|
+
Bucket: this.bucket,
|
|
420
|
+
Key: key,
|
|
421
|
+
Body: body,
|
|
422
|
+
ContentType: "application/json",
|
|
423
|
+
IfNoneMatch: expect === "absent" ? "*" : void 0,
|
|
424
|
+
IfMatch: expect === "absent" ? void 0 : expect.version
|
|
425
|
+
})
|
|
426
|
+
);
|
|
427
|
+
} catch (err) {
|
|
428
|
+
if (isConditionalConflict(err)) {
|
|
429
|
+
throw new WriteConflictError2(`registry OCC conflict for ${key}`);
|
|
430
|
+
}
|
|
431
|
+
throw mapError(err);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
async *listKeys(prefix) {
|
|
435
|
+
let token;
|
|
436
|
+
do {
|
|
437
|
+
let res;
|
|
438
|
+
try {
|
|
439
|
+
res = await this.client.send(
|
|
440
|
+
new ListObjectsV2Command2({
|
|
441
|
+
Bucket: this.bucket,
|
|
442
|
+
Prefix: prefix,
|
|
443
|
+
ContinuationToken: token
|
|
444
|
+
})
|
|
445
|
+
);
|
|
446
|
+
} catch (err) {
|
|
447
|
+
throw mapError(err);
|
|
448
|
+
}
|
|
449
|
+
for (const obj of res.Contents ?? []) {
|
|
450
|
+
if (obj.Key !== void 0) yield obj.Key;
|
|
451
|
+
}
|
|
452
|
+
token = res.IsTruncated === true ? res.NextContinuationToken : void 0;
|
|
453
|
+
} while (token !== void 0);
|
|
454
|
+
}
|
|
455
|
+
};
|
|
456
|
+
function mapError(err) {
|
|
457
|
+
if (isTransient(err)) {
|
|
458
|
+
return new TransientError2(
|
|
459
|
+
`transient S3 fault: ${err?.name ?? "unknown"}`,
|
|
460
|
+
{ cause: err }
|
|
461
|
+
);
|
|
462
|
+
}
|
|
463
|
+
return err;
|
|
464
|
+
}
|
|
465
|
+
var S3RegistryDriver = class extends ObjectStoreRegistry {
|
|
466
|
+
constructor(options) {
|
|
467
|
+
super(
|
|
468
|
+
new S3Store(options.client, options.bucket),
|
|
469
|
+
normalizeObjectPrefix2(options.prefix),
|
|
470
|
+
options.now ?? (() => Date.now())
|
|
471
|
+
);
|
|
472
|
+
}
|
|
473
|
+
};
|
|
474
|
+
|
|
475
|
+
// src/backend.ts
|
|
476
|
+
import { brandAsBackend } from "@cloudbitmaps/core/driver-kit";
|
|
477
|
+
import { S3Client } from "@aws-sdk/client-s3";
|
|
478
|
+
var S3Storage = class {
|
|
479
|
+
storage;
|
|
480
|
+
registry;
|
|
481
|
+
/** The client both halves share — built here unless one was supplied. */
|
|
482
|
+
client;
|
|
483
|
+
constructor(options) {
|
|
484
|
+
this.client = options.client ?? new S3Client({
|
|
485
|
+
...options.region === void 0 ? {} : { region: options.region },
|
|
486
|
+
...options.endpoint === void 0 ? {} : { endpoint: options.endpoint },
|
|
487
|
+
...options.pathStyle === void 0 ? {} : { forcePathStyle: options.pathStyle },
|
|
488
|
+
...options.credentials === void 0 ? {} : { credentials: options.credentials }
|
|
489
|
+
});
|
|
490
|
+
const shared = {
|
|
491
|
+
client: this.client,
|
|
492
|
+
bucket: options.bucket,
|
|
493
|
+
...options.prefix === void 0 ? {} : { prefix: options.prefix }
|
|
494
|
+
};
|
|
495
|
+
this.storage = new S3StorageDriver(shared);
|
|
496
|
+
this.registry = new S3RegistryDriver({
|
|
497
|
+
...shared,
|
|
498
|
+
...options.now === void 0 ? {} : { now: options.now }
|
|
499
|
+
});
|
|
500
|
+
brandAsBackend(this);
|
|
501
|
+
}
|
|
502
|
+
};
|
|
503
|
+
export {
|
|
504
|
+
S3RegistryDriver,
|
|
505
|
+
S3Storage,
|
|
506
|
+
S3StorageDriver
|
|
507
|
+
};
|
|
508
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/storage.ts", "../src/keys.ts", "../src/s3-errors.ts", "../src/registry.ts", "../src/backend.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * `S3StorageDriver` \u2014 an {@link IStorageDriver} over S3-compatible object storage.\n *\n * Works with AWS S3 and any compatible backend (MinIO, etc.) via the official `@aws-sdk/client-s3`, a real\n * dependency of this package \u2014 installing `@cloudbitmaps/s3` is what installs it. The client is\n * **injected** (dependency injection): the driver owns no credential/region/endpoint logic, so it's thin,\n * testable against MinIO (point a client at its endpoint), and reuses the caller's existing client.\n *\n * Generations are write-once immutable objects: a conditional `PutObject` with `If-None-Match: *` makes the\n * publish atomic \u2014 a second write to the same key fails with `WriteConflictError`, never a silent overwrite\n * (hard invariant 2: storage objects are immutable and never overwritten in place), the cloud analogue of\n * the LocalFs atomic `link`. **This requires a backend that honors\n * `If-None-Match: *`** (AWS S3 \u2014 GA Aug 2024; recent MinIO): a backend that silently ignored the\n * precondition would break write-once immutability. **Writes stream:** the object is uploaded in\n * constant memory \u2014 a small object is a single conditional `PutObject`; a large one is a **multipart upload**\n * (parts flushed as the codec writes, freed as they go) finished with a conditional `CompleteMultipartUpload`,\n * so a load's footprint stays ~one part regardless of segment size, up to the advertised `maxObjectBytes`\n * (default `partBytes \u00D7 10,000` \u2014 S3's per-upload part limit). Drivers may use `node:crypto`; only `core/`\n * is bound by the determinism lint.\n */\nimport {\n NotFoundError,\n TransientError,\n ValidationError,\n WriteConflictError,\n isNotFoundError,\n isValidationError,\n isWriteConflictError,\n} from '@cloudbitmaps/core/driver-kit';\nimport type {\n BlobSink,\n GenKey,\n IStorageDriver,\n SegmentRef,\n StorageCaps,\n} from '@cloudbitmaps/core/driver-kit';\nimport { createHash, type Hash } from 'node:crypto';\nimport {\n AbortMultipartUploadCommand,\n CompleteMultipartUploadCommand,\n CreateMultipartUploadCommand,\n DeleteObjectCommand,\n GetObjectCommand,\n HeadObjectCommand,\n ListObjectsV2Command,\n PutObjectCommand,\n UploadPartCommand,\n type S3Client,\n} from '@aws-sdk/client-s3';\nimport {\n storageObjectKey,\n normalizeS3Prefix,\n parseGenerationFromKey,\n segmentObjectPrefix,\n} from './keys';\nimport {\n isConditionalConflict,\n isInvalidRange,\n isNotFound,\n isTransient,\n totalFromContentRange,\n} from './s3-errors';\n\n/** Part size for multipart uploads. \u2265 the S3 5 MiB minimum; an object that fits in one part uses a single\n * conditional PUT instead (no multipart overhead, strongest write-once). Peak write memory \u2248 one part. */\nconst S3_PART_BYTES = 8 * 1024 * 1024;\n/** S3 hard limit: a multipart upload has at most 10,000 parts. This \u00D7 the part size is the real object ceiling. */\nconst S3_MAX_PARTS = 10_000;\n\nexport interface S3StorageDriverOptions {\n /** A constructed S3 client (point its `endpoint` at MinIO for local/integration use). */\n readonly client: S3Client;\n /** Target bucket (must already exist). */\n readonly bucket: string;\n /** Optional key prefix under which all objects live (e.g. `cloudroaring/`). */\n readonly prefix?: string;\n /**\n * Largest object this driver will write/advertise. Default = `partBytes \u00D7 10,000` (\u2248 80 GiB at the default\n * 8 MiB part) \u2014 the honest ceiling reachable within S3's 10,000-part limit. Set it higher and `partBytes`\n * auto-grows so 10,000 parts still cover it (raising peak write memory to ~one part); up to the 5 TiB S3 max.\n */\n readonly maxObjectBytes?: number;\n /** Multipart part size in bytes (default 8 MiB; clamped to the S3 5 MiB minimum). Tunes peak write memory. */\n readonly partBytes?: number;\n}\n\nexport class S3StorageDriver implements IStorageDriver {\n private readonly client: S3Client;\n private readonly bucket: string;\n private readonly prefix: string | undefined;\n private readonly maxObjectBytes: number;\n private readonly partBytes: number;\n\n constructor(options: S3StorageDriverOptions) {\n this.client = options.client;\n this.bucket = options.bucket;\n this.prefix = normalizeS3Prefix(options.prefix);\n const requestedPart = Math.max(options.partBytes ?? S3_PART_BYTES, 5 * 1024 * 1024);\n // Default the object cap to what the requested part size can actually cover within S3's 10,000-part limit;\n // if a larger cap is requested, grow the part size to keep it reachable (so the advertised cap is honest).\n this.maxObjectBytes = options.maxObjectBytes ?? requestedPart * S3_MAX_PARTS;\n this.partBytes = Math.max(requestedPart, Math.ceil(this.maxObjectBytes / S3_MAX_PARTS));\n }\n\n capabilities(): StorageCaps {\n return { rangeRead: true, maxObjectBytes: this.maxObjectBytes, conditionalPut: true };\n }\n\n async putImmutable(\n key: GenKey,\n write: (sink: BlobSink) => Promise<void>,\n ): Promise<{ size: number; sha256: string }> {\n const objectKey = storageObjectKey(this.prefix, key); // validates ref + generation\n const sink = new S3MultipartSink(\n this.client,\n this.bucket,\n objectKey,\n this.partBytes,\n this.maxObjectBytes,\n );\n try {\n await write(sink);\n return await sink.finish();\n } catch (err) {\n await sink.abort(); // best-effort cleanup of any in-flight multipart upload\n // A lost conditional-write race \u2014 the precondition failed (412) or S3 rejected concurrent conditional\n // writes to the key (409) \u2014 is the write-once conflict, never a silent overwrite.\n if (isConditionalConflict(err)) {\n throw new WriteConflictError(\n `generation already exists (write-once): ${key.segment}.${key.generation}`,\n );\n }\n if (isValidationError(err) || isWriteConflictError(err) || isNotFoundError(err)) {\n throw err;\n }\n throw this.mapError(err);\n }\n }\n\n async getRange(key: GenKey, offset: number, length: number): Promise<Uint8Array> {\n if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0) {\n throw new ValidationError(`invalid range offset=${offset} length=${length}`);\n }\n const objectKey = storageObjectKey(this.prefix, key);\n if (length === 0) return new Uint8Array(0);\n try {\n const res = await this.client.send(\n new GetObjectCommand({\n Bucket: this.bucket,\n Key: objectKey,\n Range: `bytes=${offset}-${offset + length - 1}`,\n }),\n );\n const bytes = await collect(res.Body);\n // A short read means the range ran past EOF \u2014 treat as out-of-bounds, never a partial result.\n if (bytes.length !== length) {\n throw new ValidationError(\n `range [${offset}, ${offset + length}) out of bounds (got ${bytes.length}B)`,\n );\n }\n return bytes;\n } catch (err) {\n throw this.mapReadError(err, key);\n }\n }\n\n async getTail(key: GenKey, maxBytes: number): Promise<{ bytes: Uint8Array; size: number }> {\n const objectKey = storageObjectKey(this.prefix, key);\n if (maxBytes <= 0) {\n // No tail bytes wanted \u2014 just resolve the size via a HEAD.\n try {\n const head = await this.client.send(\n new HeadObjectCommand({ Bucket: this.bucket, Key: objectKey }),\n );\n return { bytes: new Uint8Array(0), size: head.ContentLength ?? 0 };\n } catch (err) {\n throw this.mapReadError(err, key);\n }\n }\n try {\n const res = await this.client.send(\n new GetObjectCommand({ Bucket: this.bucket, Key: objectKey, Range: `bytes=-${maxBytes}` }),\n );\n const bytes = await collect(res.Body);\n let size = totalFromContentRange(res.ContentRange);\n if (size === undefined) {\n // A spec-compliant backend omits Content-Range only on a 200 (whole object), where bytes.length\n // IS the size. If the body is exactly maxBytes we can't rule out a clamped partial from a\n // non-compliant backend \u2014 confirm the true size with a HEAD rather than trust a possibly-short read.\n if (bytes.length === maxBytes) {\n const head = await this.client.send(\n new HeadObjectCommand({ Bucket: this.bucket, Key: objectKey }),\n );\n size = head.ContentLength ?? bytes.length;\n } else {\n size = bytes.length;\n }\n }\n return { bytes, size };\n } catch (err) {\n throw this.mapReadError(err, key);\n }\n }\n\n async delete(key: GenKey): Promise<void> {\n // Idempotent: S3 DeleteObject succeeds even if the key is absent (GC may race / retry).\n try {\n await this.client.send(\n new DeleteObjectCommand({ Bucket: this.bucket, Key: storageObjectKey(this.prefix, key) }),\n );\n } catch (err) {\n throw this.mapError(err);\n }\n }\n\n async *list(ref: SegmentRef): AsyncIterable<GenKey> {\n const prefix = segmentObjectPrefix(this.prefix, ref); // validates ref\n let token: string | undefined;\n do {\n let res;\n try {\n res = await this.client.send(\n new ListObjectsV2Command({\n Bucket: this.bucket,\n Prefix: prefix,\n ContinuationToken: token,\n }),\n );\n } catch (err) {\n throw this.mapError(err);\n }\n for (const obj of res.Contents ?? []) {\n if (obj.Key === undefined) continue;\n const generation = parseGenerationFromKey(prefix, obj.Key);\n if (generation !== null) {\n yield { namespace: ref.namespace, segment: ref.segment, generation };\n }\n }\n token = res.IsTruncated === true ? res.NextContinuationToken : undefined;\n } while (token !== undefined);\n }\n\n /** Map S3 read errors to the driver vocabulary; pass everything else through {@link mapError}. */\n private mapReadError(err: unknown, key: GenKey): unknown {\n if (isValidationError(err)) return err;\n if (isNotFound(err)) {\n return new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);\n }\n // A fully out-of-range request (start past EOF) \u2014 the BlobReader contract treats range errors as\n // ValidationError, never a short/empty read.\n if (isInvalidRange(err)) {\n return new ValidationError(`range out of bounds for ${key.segment}.${key.generation}`);\n }\n return this.mapError(err);\n }\n\n /**\n * Reclassify a transient S3 fault (throttle/5xx/dropped connection) as a retryable {@link TransientError}\n * so the retry decorator can ride it out; everything else propagates unchanged. The final fallback at every\n * `client.send` site, so callers and the decorator only ever see typed errors.\n */\n private mapError(err: unknown): unknown {\n if (isTransient(err)) {\n return new TransientError(\n `transient S3 fault: ${(err as { name?: string } | null)?.name ?? 'unknown'}`,\n { cause: err },\n );\n }\n return err;\n }\n}\n\n/** Concatenate a list of byte chunks of known total length into one buffer. */\nfunction concatBytes(parts: readonly Uint8Array[], total: number): Uint8Array {\n const out = new Uint8Array(total);\n let offset = 0;\n for (const p of parts) {\n out.set(p, offset);\n offset += p.length;\n }\n return out;\n}\n\n/**\n * Streaming {@link BlobSink} that uploads one S3 object in **constant memory**. It buffers at most\n * one part: as the codec writes, full parts are flushed via `UploadPart` and freed. A small object that never\n * reaches one part is committed as a single conditional `PutObject`; a larger one is finished with a\n * conditional `CompleteMultipartUpload` \u2014 **both enforce write-once** via `If-None-Match: *`. SHA-256 is hashed\n * incrementally. On any error the caller invokes {@link abort} to clean up the in-flight multipart upload.\n */\nclass S3MultipartSink implements BlobSink {\n private readonly hash: Hash = createHash('sha256');\n private readonly pending: Uint8Array[] = [];\n private pendingLen = 0;\n private total = 0;\n private uploadId: string | undefined;\n private partNumber = 0;\n private readonly parts: { ETag: string | undefined; PartNumber: number }[] = [];\n\n constructor(\n private readonly client: S3Client,\n private readonly bucket: string,\n private readonly objectKey: string,\n private readonly partBytes: number,\n private readonly maxObjectBytes: number,\n ) {}\n\n async write(bytes: Uint8Array): Promise<void> {\n if (bytes.length === 0) return;\n this.total += bytes.length;\n if (this.total > this.maxObjectBytes) {\n // Fail fast + typed, rather than a late opaque S3 error (and abort the in-flight upload via the caller).\n throw new ValidationError(`object exceeds maxObjectBytes ${this.maxObjectBytes}`);\n }\n this.hash.update(bytes);\n this.pending.push(bytes);\n this.pendingLen += bytes.length;\n if (this.pendingLen >= this.partBytes) await this.flushPart();\n }\n\n /** Upload the buffered bytes (\u2265 one part) as a single part, freeing them. Starts the upload on first call. */\n private async flushPart(): Promise<void> {\n if (this.uploadId === undefined) {\n const res = await this.client.send(\n new CreateMultipartUploadCommand({ Bucket: this.bucket, Key: this.objectKey }),\n );\n if (res.UploadId === undefined) {\n throw new TransientError('S3 CreateMultipartUpload returned no UploadId');\n }\n this.uploadId = res.UploadId;\n }\n const body = concatBytes(this.pending, this.pendingLen);\n this.pending.length = 0;\n this.pendingLen = 0;\n this.partNumber += 1;\n if (this.partNumber > S3_MAX_PARTS) {\n // Unreachable for valid input (the maxObjectBytes byte-cap, sized to \u2264 S3_MAX_PARTS parts, fires first) \u2014\n // a typed guard so the S3 hard limit is never a raw 400.\n throw new ValidationError(`multipart upload exceeded the S3 ${S3_MAX_PARTS}-part limit`);\n }\n const res = await this.client.send(\n new UploadPartCommand({\n Bucket: this.bucket,\n Key: this.objectKey,\n UploadId: this.uploadId,\n PartNumber: this.partNumber,\n Body: body,\n }),\n );\n this.parts.push({ ETag: res.ETag, PartNumber: this.partNumber });\n }\n\n /** Commit the object: a single conditional PUT if it fit in one part, else complete the multipart upload. */\n async finish(): Promise<{ size: number; sha256: string }> {\n const sha256 = this.hash.digest('hex');\n if (this.uploadId === undefined) {\n await this.client.send(\n new PutObjectCommand({\n Bucket: this.bucket,\n Key: this.objectKey,\n Body: concatBytes(this.pending, this.pendingLen),\n IfNoneMatch: '*', // write-once\n }),\n );\n return { size: this.total, sha256 };\n }\n if (this.pendingLen > 0) await this.flushPart(); // the final part may be < partBytes (allowed)\n await this.client.send(\n new CompleteMultipartUploadCommand({\n Bucket: this.bucket,\n Key: this.objectKey,\n UploadId: this.uploadId,\n MultipartUpload: { Parts: this.parts },\n IfNoneMatch: '*', // write-once: fail if the object already exists\n }),\n );\n this.uploadId = undefined; // completed \u2014 nothing left to abort\n return { size: this.total, sha256 };\n }\n\n /** Best-effort cleanup of an in-flight multipart upload after an error (a leaked MPU is reaped by a bucket\n * lifecycle rule; never a correctness issue). No-op if nothing was started or it already completed. */\n async abort(): Promise<void> {\n if (this.uploadId === undefined) return;\n const id = this.uploadId;\n this.uploadId = undefined;\n try {\n await this.client.send(\n new AbortMultipartUploadCommand({ Bucket: this.bucket, Key: this.objectKey, UploadId: id }),\n );\n } catch {\n // swallow \u2014 best-effort\n }\n }\n}\n\n/**\n * Collect an S3 response body into a `Uint8Array`. `transformToByteArray` is added at runtime to the SDK's\n * Node stream by `@aws-sdk`'s sdk-stream-mixin, so the structural cast is sound on Node.\n */\nasync function collect(body: GetObjectCommandBody): Promise<Uint8Array> {\n if (body === undefined) {\n throw new NotFoundError('S3 GetObject returned an empty body');\n }\n return body.transformToByteArray();\n}\n\n/** The S3 `GetObject` Body type, narrowed to the part we use (`transformToByteArray`). */\ntype GetObjectCommandBody = { transformToByteArray(): Promise<Uint8Array> } | undefined;\n", "/**\n * Logical-ref \u2192 S3 object-key mapping for {@link S3StorageDriver}.\n *\n * Pure string logic with no SDK dependency, so it's unit-testable without S3/MinIO. Mirrors the LocalFs\n * layout (`<namespace>/segments/<segment>.<gen>.crbm`) under an optional caller prefix, and re-validates\n * names at the boundary \u2014 defense in depth, because a driver can be constructed and driven directly rather\n * than through the engine that would otherwise have validated for it. The default\n * (absent) namespace maps to `_default`, which cannot collide with a real namespace because a caller's\n * `_default` encodes to `%5Fdefault` while the sentinel is emitted literally.\n */\n// `prefixPart` is imported, never redefined: the storage and registry layouts sit under the SAME caller\n// prefix, so they must normalize it identically \u2014 a second copy of that three-line function is how the two\n// halves of one bucket drift apart.\nimport {\n ValidationError,\n encodeNameForKey,\n namespaceKeyPart,\n prefixPart,\n validateSegmentRef,\n} from '@cloudbitmaps/core/driver-kit';\nimport type { GenKey, SegmentRef } from '@cloudbitmaps/core/driver-kit';\n\nconst SUFFIX = '.crbm';\n\n/** Validate a caller-supplied key prefix. The rule is shared with every other object store. */\nexport { normalizeObjectPrefix as normalizeS3Prefix } from '@cloudbitmaps/core/driver-kit';\n\n/**\n * The S3 key prefix shared by all of a segment's generations: `<prefix><ns>/segments/<segment>.`. Used\n * both as the `ListObjectsV2` prefix and as the string stripped by {@link parseGenerationFromKey}.\n */\nexport function segmentObjectPrefix(prefix: string | undefined, ref: SegmentRef): string {\n validateSegmentRef(ref);\n return `${prefixPart(prefix)}${namespaceKeyPart(ref.namespace)}/segments/${encodeNameForKey(ref.segment)}.`;\n}\n\n/** The full S3 key of one `.crbm` generation: `<segmentPrefix><gen>.crbm`. */\nexport function storageObjectKey(prefix: string | undefined, key: GenKey): string {\n if (!Number.isInteger(key.generation) || key.generation < 0) {\n throw new ValidationError(`generation must be a non-negative integer; got ${key.generation}`);\n }\n return `${segmentObjectPrefix(prefix, key)}${key.generation}${SUFFIX}`;\n}\n\n// \u2500\u2500\u2500 Registry keys \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n// The registry layout is identical across S3, GCS and Azure Blob \u2014 all three encode names the same way and\n// build byte-identical keys \u2014 so it lives in `_shared/object-registry-keys` and is re-exported here for the\n// callers (and tests) that already name it through this module.\nexport {\n registryPrefix,\n registryObjectKey,\n registryListPrefix,\n parseRegistryKey,\n} from '@cloudbitmaps/core/driver-kit';\n\n/**\n * Parse a generation number out of a full object key, given its segment prefix, or `null` if it doesn't\n * match. Canonical decimal only \u2014 no leading zeros (so `\u2026s.07.crbm` can't alias `\u2026s.7.crbm`) and within\n * safe-integer range. This also rejects a *different* segment whose name merely shares the prefix (e.g. a\n * key for segment `s.x` won't parse under segment `s`'s prefix, since its middle isn't all digits).\n */\nexport function parseGenerationFromKey(segmentPrefix: string, objectKey: string): number | null {\n if (!objectKey.startsWith(segmentPrefix) || !objectKey.endsWith(SUFFIX)) return null;\n const middle = objectKey.slice(segmentPrefix.length, objectKey.length - SUFFIX.length);\n if (!/^(0|[1-9]\\d*)$/.test(middle)) return null;\n const generation = Number(middle);\n return Number.isSafeInteger(generation) ? generation : null;\n}\n", "/**\n * Pure helpers for classifying S3 SDK errors + parsing response headers (conflict and transient classification).\n *\n * Kept SDK-free and side-effect-free (they only read structural shapes \u2014 `err.name`,\n * `$metadata.httpStatusCode`, a `Content-Range` string) so the subtle S3-specific translation logic is\n * unit-testable without a live MinIO/S3 or even the AWS SDK. Shared AWS shapes come from `_shared/aws-errors`.\n */\n\nimport {\n errorName,\n httpStatus,\n isNetworkOrTimeout,\n isSdkRetryable,\n isServerSide,\n} from '@cloudbitmaps/core/driver-kit';\n\n/** A conditional `If-None-Match: *` PUT lost the write-once race (the object already existed). */\nexport function isPreconditionFailed(err: unknown): boolean {\n return errorName(err) === 'PreconditionFailed' || httpStatus(err) === 412;\n}\n\n/**\n * A conditional write (`If-None-Match: *` / `If-Match: <etag>`) lost the race \u2014 **either** outcome S3 uses:\n * the precondition evaluated false (`412 PreconditionFailed`), **or** S3 rejected concurrent conditional\n * writes to the same key to prevent a lost update (`409 ConditionalRequestConflict`, which AWS documents and\n * asks you to retry). Both mean \"you lost; re-read and retry\" \u2014 so both must map to `WriteConflictError` and\n * route through the caller's OCC path, never a blind transient retry (which would just replay a doomed PUT).\n */\nexport function isConditionalConflict(err: unknown): boolean {\n return (\n isPreconditionFailed(err) ||\n errorName(err) === 'ConditionalRequestConflict' ||\n httpStatus(err) === 409\n );\n}\n\n/** The object / generation does not exist (GetObject \u2192 `NoSuchKey`, HeadObject \u2192 `NotFound`; both 404). */\nexport function isNotFound(err: unknown): boolean {\n const name = errorName(err);\n return name === 'NoSuchKey' || name === 'NotFound' || httpStatus(err) === 404;\n}\n\n/** A range request started past EOF (HTTP 416). */\nexport function isInvalidRange(err: unknown): boolean {\n return errorName(err) === 'InvalidRange' || httpStatus(err) === 416;\n}\n\n/**\n * A transient S3 fault that is safe to retry: throttling (`SlowDown` / 503), any 5xx, a dropped/timed-out\n * connection, or anything the SDK itself marks retryable. Excludes the deterministic outcomes above\n * (412/404/416) \u2014 those are caller-meaningful and must never be retried/reclassified.\n */\nexport function isTransient(err: unknown): boolean {\n // A conditional-write conflict (412/409) is caller-meaningful OCC, not a blind-retryable transient.\n if (isConditionalConflict(err) || isNotFound(err) || isInvalidRange(err)) return false;\n return (\n errorName(err) === 'SlowDown' ||\n isServerSide(err) ||\n isNetworkOrTimeout(err) ||\n isSdkRetryable(err)\n );\n}\n\n/**\n * Parse the total object size out of a `Content-Range: bytes <start>-<end>/<total>` header, or `undefined`\n * if absent/unparseable/unsafe. The total is the part after the final `/`.\n */\nexport function totalFromContentRange(contentRange: string | undefined): number | undefined {\n if (contentRange === undefined) return undefined;\n const match = /\\/(\\d+)\\s*$/.exec(contentRange);\n if (match === null) return undefined;\n const total = Number(match[1]);\n return Number.isSafeInteger(total) ? total : undefined;\n}\n", "/**\n * `S3RegistryDriver` \u2014 an {@link IRegistryDriver} over S3-compatible object storage.\n *\n * Lets a **read-mostly deployment run on S3 alone** \u2014 storage `.crbm` generations + the registry in one bucket,\n * no separate database. The protocol (an ABA-safe OCC counter, tombstoning delete, the bounded retry, the key layout)\n * lives once in {@link ObjectStoreRegistry}; this file is only the three I/O calls S3 makes, so the S3, GCS\n * and Azure registries cannot drift from one another.\n *\n * **The atomic swap is offloaded to S3's conditional writes** (GA Nov 2024): `If-None-Match: *` for\n * create-only and `If-Match: <etag>` for compare-and-swap, so a concurrent writer between our read and our\n * PUT loses with a `412` \u2192 {@link WriteConflictError}. Reads are strongly consistent (S3, since 2020),\n * satisfying the registry's `strongRead` contract. The client is **injected**, exactly like\n * {@link S3StorageDriver}.\n *\n * **Deployment requirements** (a backend/policy that violates these silently corrupts the registry):\n * - The backend **must honor `If-Match`** (AWS S3; recent MinIO). One that returns ETags but ignores the\n * precondition degrades compare-and-swap to last-write-wins \u2192 lost `currentGen` swaps. Verified against\n * real S3 semantics by the MinIO integration lane.\n * - The IAM principal needs **`s3:ListBucket`** on the bucket. Without it, `GetObject` on a missing key\n * returns `403` (not `404`), so the \"absent segment \u2192 `null`\" contract (and `create`'s bootstrap read)\n * breaks \u2014 and `list()` needs it regardless.\n * - **Do not apply an S3 lifecycle-expiration rule to the `registry/` prefix.** See {@link ObjectStoreRegistry}.\n */\nimport {\n IntegrityError,\n MAX_ROW_BYTES,\n ObjectStoreRegistry,\n TransientError,\n WriteConflictError,\n normalizeObjectPrefix,\n} from '@cloudbitmaps/core/driver-kit';\nimport type { ObjectRegistryStore, ObjectRow } from '@cloudbitmaps/core/driver-kit';\nimport {\n GetObjectCommand,\n ListObjectsV2Command,\n PutObjectCommand,\n type S3Client,\n} from '@aws-sdk/client-s3';\nimport { isConditionalConflict, isNotFound, isTransient } from './s3-errors';\n\nexport interface S3RegistryDriverOptions {\n /** A constructed S3 client (point its `endpoint` at MinIO for local/integration use). */\n readonly client: S3Client;\n /** Target bucket (must already exist). */\n readonly bucket: string;\n /** Optional key prefix under which all registry objects live (e.g. `cloudroaring/`). */\n readonly prefix?: string;\n /** Injected clock for `createdAt`/`updatedAt`; defaults to `Date.now`. */\n readonly now?: () => number;\n}\n\n/** The three calls {@link ObjectStoreRegistry} needs, in S3's dialect. */\nclass S3Store implements ObjectRegistryStore {\n readonly label = 'S3';\n\n constructor(\n private readonly client: S3Client,\n private readonly bucket: string,\n ) {}\n\n async read(key: string): Promise<ObjectRow | null> {\n let res;\n try {\n res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key }));\n } catch (err) {\n if (isNotFound(err)) return null;\n throw mapError(err);\n }\n // Check the advertised length BEFORE allocating, so a hostile object cannot make us buffer it first.\n if ((res.ContentLength ?? 0) > MAX_ROW_BYTES) {\n throw new IntegrityError(\n `registry object ${res.ContentLength}B exceeds cap ${MAX_ROW_BYTES}B`,\n );\n }\n if (res.Body === undefined) {\n throw new IntegrityError(`registry object has an empty body: ${key}`);\n }\n const bytes = await (\n res.Body as { transformToByteArray(): Promise<Uint8Array> }\n ).transformToByteArray();\n return { bytes, version: res.ETag ?? '' };\n }\n\n async write(\n key: string,\n body: Uint8Array,\n expect: 'absent' | { version: string },\n ): Promise<void> {\n try {\n await this.client.send(\n new PutObjectCommand({\n Bucket: this.bucket,\n Key: key,\n Body: body,\n ContentType: 'application/json',\n IfNoneMatch: expect === 'absent' ? '*' : undefined,\n IfMatch: expect === 'absent' ? undefined : expect.version,\n }),\n );\n } catch (err) {\n // A lost conditional-write race (412 precondition, or 409 concurrent-conflict) is an OCC conflict.\n if (isConditionalConflict(err)) {\n throw new WriteConflictError(`registry OCC conflict for ${key}`);\n }\n throw mapError(err);\n }\n }\n\n async *listKeys(prefix: string): AsyncIterable<string> {\n let token: string | undefined;\n do {\n let res;\n try {\n res = await this.client.send(\n new ListObjectsV2Command({\n Bucket: this.bucket,\n Prefix: prefix,\n ContinuationToken: token,\n }),\n );\n } catch (err) {\n throw mapError(err);\n }\n for (const obj of res.Contents ?? []) {\n if (obj.Key !== undefined) yield obj.Key;\n }\n token = res.IsTruncated === true ? res.NextContinuationToken : undefined;\n } while (token !== undefined);\n }\n}\n\n/** Reclassify a transient S3 fault as a retryable {@link TransientError}; pass everything else through. */\nfunction mapError(err: unknown): unknown {\n if (isTransient(err)) {\n return new TransientError(\n `transient S3 fault: ${(err as { name?: string } | null)?.name ?? 'unknown'}`,\n { cause: err },\n );\n }\n return err;\n}\n\nexport class S3RegistryDriver extends ObjectStoreRegistry {\n constructor(options: S3RegistryDriverOptions) {\n super(\n new S3Store(options.client, options.bucket),\n normalizeObjectPrefix(options.prefix),\n options.now ?? ((): number => Date.now()),\n );\n }\n}\n", "/**\n * `S3Storage` \u2014 the S3 backend as one object: the generations and the pointer, in one bucket, stated once.\n *\n * Replaces two constructor calls that each repeated `client`, `bucket` and `prefix`. Repeating them is how\n * they come apart: point the registry at one prefix and the objects at another and the store answers *empty*\n * rather than *misconfigured*, which is the hardest kind of wrong answer to debug. Here the location is written\n * once and shared, so the mismatch cannot be expressed.\n *\n * **It will build a client for you**, which is the common case \u2014 `new S3Storage({ bucket })` picks up the\n * ambient credential chain and region exactly as the SDK would. Pass `client` instead when you need a\n * credential chain the SDK cannot infer (SSO, an assumed role, a custom retry strategy); pass `endpoint` +\n * `pathStyle` + `credentials` for an S3-compatible store (MinIO, Ceph, R2). Both halves stay reachable as `.storage` and\n * `.registry` for anyone wiring something the facade does not cover.\n */\nimport { STORAGE_BACKEND, brandAsBackend } from '@cloudbitmaps/core/driver-kit';\nimport type {\n IRegistryDriver,\n IStorageDriver,\n StorageBackend,\n} from '@cloudbitmaps/core/driver-kit';\nimport { S3Client } from '@aws-sdk/client-s3';\nimport { S3StorageDriver } from './storage';\nimport { S3RegistryDriver } from './registry';\n\nexport interface S3StorageOptions {\n /** Target bucket (must already exist). */\n readonly bucket: string;\n /** Optional key prefix under which everything lives \u2014 generations and the registry alike. */\n readonly prefix?: string;\n /** A constructed client. Supply one for a credential chain the SDK cannot infer; otherwise one is built. */\n readonly client?: S3Client;\n /** Region for the client built when `client` is absent. Falls back to the SDK's own resolution. */\n readonly region?: string;\n /** Endpoint for an S3-compatible store (MinIO, Ceph, R2). Ignored when `client` is supplied. */\n readonly endpoint?: string;\n /** Path-style addressing, which most S3-compatible stores require. Ignored when `client` is supplied. */\n readonly pathStyle?: boolean;\n /**\n * Static credentials, for the S3-compatible stores that issue them (MinIO, Ceph, R2).\n *\n * On AWS itself, leave this unset \u2014 the SDK's own chain (instance role, SSO, environment, profile) is what\n * you want, and hard-coding keys to reach it would be a downgrade. It exists because the alternative for a\n * MinIO user was to construct an `S3Client` purely to carry two strings, which is the ergonomics this class\n * is here to remove. Ignored when `client` is supplied.\n */\n readonly credentials?: {\n readonly accessKeyId: string;\n readonly secretAccessKey: string;\n readonly sessionToken?: string;\n };\n /** Injected clock for the registry's `createdAt`/`updatedAt`; defaults to `Date.now`. */\n readonly now?: () => number;\n}\n\nexport class S3Storage implements StorageBackend {\n /** Cross-bundle brand, stamped non-enumerably in the constructor so a spread cannot carry it. */\n declare readonly [STORAGE_BACKEND]: true;\n readonly storage: IStorageDriver;\n readonly registry: IRegistryDriver;\n /** The client both halves share \u2014 built here unless one was supplied. */\n readonly client: S3Client;\n\n constructor(options: S3StorageOptions) {\n this.client =\n options.client ??\n new S3Client({\n ...(options.region === undefined ? {} : { region: options.region }),\n ...(options.endpoint === undefined ? {} : { endpoint: options.endpoint }),\n ...(options.pathStyle === undefined ? {} : { forcePathStyle: options.pathStyle }),\n ...(options.credentials === undefined ? {} : { credentials: options.credentials }),\n });\n const shared = {\n client: this.client,\n bucket: options.bucket,\n ...(options.prefix === undefined ? {} : { prefix: options.prefix }),\n };\n this.storage = new S3StorageDriver(shared);\n this.registry = new S3RegistryDriver({\n ...shared,\n ...(options.now === undefined ? {} : { now: options.now }),\n });\n brandAsBackend(this);\n }\n}\n"],
|
|
5
|
+
"mappings": ";AAoBA;AAAA,EACE;AAAA,EACA;AAAA,EACA,mBAAAA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAQP,SAAS,kBAA6B;AACtC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;;;ACnCP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAMP,SAAkC,6BAAyB;AAuB3D;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AA/BP,IAAM,SAAS;AASR,SAAS,oBAAoB,QAA4B,KAAyB;AACvF,qBAAmB,GAAG;AACtB,SAAO,GAAG,WAAW,MAAM,CAAC,GAAG,iBAAiB,IAAI,SAAS,CAAC,aAAa,iBAAiB,IAAI,OAAO,CAAC;AAC1G;AAGO,SAAS,iBAAiB,QAA4B,KAAqB;AAChF,MAAI,CAAC,OAAO,UAAU,IAAI,UAAU,KAAK,IAAI,aAAa,GAAG;AAC3D,UAAM,IAAI,gBAAgB,kDAAkD,IAAI,UAAU,EAAE;AAAA,EAC9F;AACA,SAAO,GAAG,oBAAoB,QAAQ,GAAG,CAAC,GAAG,IAAI,UAAU,GAAG,MAAM;AACtE;AAmBO,SAAS,uBAAuB,eAAuB,WAAkC;AAC9F,MAAI,CAAC,UAAU,WAAW,aAAa,KAAK,CAAC,UAAU,SAAS,MAAM,EAAG,QAAO;AAChF,QAAM,SAAS,UAAU,MAAM,cAAc,QAAQ,UAAU,SAAS,OAAO,MAAM;AACrF,MAAI,CAAC,iBAAiB,KAAK,MAAM,EAAG,QAAO;AAC3C,QAAM,aAAa,OAAO,MAAM;AAChC,SAAO,OAAO,cAAc,UAAU,IAAI,aAAa;AACzD;;;AC3DA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAGA,SAAS,qBAAqB,KAAuB;AAC1D,SAAO,UAAU,GAAG,MAAM,wBAAwB,WAAW,GAAG,MAAM;AACxE;AASO,SAAS,sBAAsB,KAAuB;AAC3D,SACE,qBAAqB,GAAG,KACxB,UAAU,GAAG,MAAM,gCACnB,WAAW,GAAG,MAAM;AAExB;AAGO,SAAS,WAAW,KAAuB;AAChD,QAAM,OAAO,UAAU,GAAG;AAC1B,SAAO,SAAS,eAAe,SAAS,cAAc,WAAW,GAAG,MAAM;AAC5E;AAGO,SAAS,eAAe,KAAuB;AACpD,SAAO,UAAU,GAAG,MAAM,kBAAkB,WAAW,GAAG,MAAM;AAClE;AAOO,SAAS,YAAY,KAAuB;AAEjD,MAAI,sBAAsB,GAAG,KAAK,WAAW,GAAG,KAAK,eAAe,GAAG,EAAG,QAAO;AACjF,SACE,UAAU,GAAG,MAAM,cACnB,aAAa,GAAG,KAChB,mBAAmB,GAAG,KACtB,eAAe,GAAG;AAEtB;AAMO,SAAS,sBAAsB,cAAsD;AAC1F,MAAI,iBAAiB,OAAW,QAAO;AACvC,QAAM,QAAQ,cAAc,KAAK,YAAY;AAC7C,MAAI,UAAU,KAAM,QAAO;AAC3B,QAAM,QAAQ,OAAO,MAAM,CAAC,CAAC;AAC7B,SAAO,OAAO,cAAc,KAAK,IAAI,QAAQ;AAC/C;;;AFRA,IAAM,gBAAgB,IAAI,OAAO;AAEjC,IAAM,eAAe;AAmBd,IAAM,kBAAN,MAAgD;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,SAAiC;AAC3C,SAAK,SAAS,QAAQ;AACtB,SAAK,SAAS,QAAQ;AACtB,SAAK,SAAS,sBAAkB,QAAQ,MAAM;AAC9C,UAAM,gBAAgB,KAAK,IAAI,QAAQ,aAAa,eAAe,IAAI,OAAO,IAAI;AAGlF,SAAK,iBAAiB,QAAQ,kBAAkB,gBAAgB;AAChE,SAAK,YAAY,KAAK,IAAI,eAAe,KAAK,KAAK,KAAK,iBAAiB,YAAY,CAAC;AAAA,EACxF;AAAA,EAEA,eAA4B;AAC1B,WAAO,EAAE,WAAW,MAAM,gBAAgB,KAAK,gBAAgB,gBAAgB,KAAK;AAAA,EACtF;AAAA,EAEA,MAAM,aACJ,KACA,OAC2C;AAC3C,UAAM,YAAY,iBAAiB,KAAK,QAAQ,GAAG;AACnD,UAAM,OAAO,IAAI;AAAA,MACf,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AACA,QAAI;AACF,YAAM,MAAM,IAAI;AAChB,aAAO,MAAM,KAAK,OAAO;AAAA,IAC3B,SAAS,KAAK;AACZ,YAAM,KAAK,MAAM;AAGjB,UAAI,sBAAsB,GAAG,GAAG;AAC9B,cAAM,IAAI;AAAA,UACR,2CAA2C,IAAI,OAAO,IAAI,IAAI,UAAU;AAAA,QAC1E;AAAA,MACF;AACA,UAAI,kBAAkB,GAAG,KAAK,qBAAqB,GAAG,KAAK,gBAAgB,GAAG,GAAG;AAC/E,cAAM;AAAA,MACR;AACA,YAAM,KAAK,SAAS,GAAG;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,KAAa,QAAgB,QAAqC;AAC/E,QAAI,CAAC,OAAO,UAAU,MAAM,KAAK,CAAC,OAAO,UAAU,MAAM,KAAK,SAAS,KAAK,SAAS,GAAG;AACtF,YAAM,IAAIC,iBAAgB,wBAAwB,MAAM,WAAW,MAAM,EAAE;AAAA,IAC7E;AACA,UAAM,YAAY,iBAAiB,KAAK,QAAQ,GAAG;AACnD,QAAI,WAAW,EAAG,QAAO,IAAI,WAAW,CAAC;AACzC,QAAI;AACF,YAAM,MAAM,MAAM,KAAK,OAAO;AAAA,QAC5B,IAAI,iBAAiB;AAAA,UACnB,QAAQ,KAAK;AAAA,UACb,KAAK;AAAA,UACL,OAAO,SAAS,MAAM,IAAI,SAAS,SAAS,CAAC;AAAA,QAC/C,CAAC;AAAA,MACH;AACA,YAAM,QAAQ,MAAM,QAAQ,IAAI,IAAI;AAEpC,UAAI,MAAM,WAAW,QAAQ;AAC3B,cAAM,IAAIA;AAAA,UACR,UAAU,MAAM,KAAK,SAAS,MAAM,wBAAwB,MAAM,MAAM;AAAA,QAC1E;AAAA,MACF;AACA,aAAO;AAAA,IACT,SAAS,KAAK;AACZ,YAAM,KAAK,aAAa,KAAK,GAAG;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ,KAAa,UAAgE;AACzF,UAAM,YAAY,iBAAiB,KAAK,QAAQ,GAAG;AACnD,QAAI,YAAY,GAAG;AAEjB,UAAI;AACF,cAAM,OAAO,MAAM,KAAK,OAAO;AAAA,UAC7B,IAAI,kBAAkB,EAAE,QAAQ,KAAK,QAAQ,KAAK,UAAU,CAAC;AAAA,QAC/D;AACA,eAAO,EAAE,OAAO,IAAI,WAAW,CAAC,GAAG,MAAM,KAAK,iBAAiB,EAAE;AAAA,MACnE,SAAS,KAAK;AACZ,cAAM,KAAK,aAAa,KAAK,GAAG;AAAA,MAClC;AAAA,IACF;AACA,QAAI;AACF,YAAM,MAAM,MAAM,KAAK,OAAO;AAAA,QAC5B,IAAI,iBAAiB,EAAE,QAAQ,KAAK,QAAQ,KAAK,WAAW,OAAO,UAAU,QAAQ,GAAG,CAAC;AAAA,MAC3F;AACA,YAAM,QAAQ,MAAM,QAAQ,IAAI,IAAI;AACpC,UAAI,OAAO,sBAAsB,IAAI,YAAY;AACjD,UAAI,SAAS,QAAW;AAItB,YAAI,MAAM,WAAW,UAAU;AAC7B,gBAAM,OAAO,MAAM,KAAK,OAAO;AAAA,YAC7B,IAAI,kBAAkB,EAAE,QAAQ,KAAK,QAAQ,KAAK,UAAU,CAAC;AAAA,UAC/D;AACA,iBAAO,KAAK,iBAAiB,MAAM;AAAA,QACrC,OAAO;AACL,iBAAO,MAAM;AAAA,QACf;AAAA,MACF;AACA,aAAO,EAAE,OAAO,KAAK;AAAA,IACvB,SAAS,KAAK;AACZ,YAAM,KAAK,aAAa,KAAK,GAAG;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,KAA4B;AAEvC,QAAI;AACF,YAAM,KAAK,OAAO;AAAA,QAChB,IAAI,oBAAoB,EAAE,QAAQ,KAAK,QAAQ,KAAK,iBAAiB,KAAK,QAAQ,GAAG,EAAE,CAAC;AAAA,MAC1F;AAAA,IACF,SAAS,KAAK;AACZ,YAAM,KAAK,SAAS,GAAG;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,OAAO,KAAK,KAAwC;AAClD,UAAM,SAAS,oBAAoB,KAAK,QAAQ,GAAG;AACnD,QAAI;AACJ,OAAG;AACD,UAAI;AACJ,UAAI;AACF,cAAM,MAAM,KAAK,OAAO;AAAA,UACtB,IAAI,qBAAqB;AAAA,YACvB,QAAQ,KAAK;AAAA,YACb,QAAQ;AAAA,YACR,mBAAmB;AAAA,UACrB,CAAC;AAAA,QACH;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,KAAK,SAAS,GAAG;AAAA,MACzB;AACA,iBAAW,OAAO,IAAI,YAAY,CAAC,GAAG;AACpC,YAAI,IAAI,QAAQ,OAAW;AAC3B,cAAM,aAAa,uBAAuB,QAAQ,IAAI,GAAG;AACzD,YAAI,eAAe,MAAM;AACvB,gBAAM,EAAE,WAAW,IAAI,WAAW,SAAS,IAAI,SAAS,WAAW;AAAA,QACrE;AAAA,MACF;AACA,cAAQ,IAAI,gBAAgB,OAAO,IAAI,wBAAwB;AAAA,IACjE,SAAS,UAAU;AAAA,EACrB;AAAA;AAAA,EAGQ,aAAa,KAAc,KAAsB;AACvD,QAAI,kBAAkB,GAAG,EAAG,QAAO;AACnC,QAAI,WAAW,GAAG,GAAG;AACnB,aAAO,IAAI,cAAc,uBAAuB,IAAI,OAAO,IAAI,IAAI,UAAU,EAAE;AAAA,IACjF;AAGA,QAAI,eAAe,GAAG,GAAG;AACvB,aAAO,IAAIA,iBAAgB,2BAA2B,IAAI,OAAO,IAAI,IAAI,UAAU,EAAE;AAAA,IACvF;AACA,WAAO,KAAK,SAAS,GAAG;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOQ,SAAS,KAAuB;AACtC,QAAI,YAAY,GAAG,GAAG;AACpB,aAAO,IAAI;AAAA,QACT,uBAAwB,KAAkC,QAAQ,SAAS;AAAA,QAC3E,EAAE,OAAO,IAAI;AAAA,MACf;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;AAGA,SAAS,YAAY,OAA8B,OAA2B;AAC5E,QAAM,MAAM,IAAI,WAAW,KAAK;AAChC,MAAI,SAAS;AACb,aAAW,KAAK,OAAO;AACrB,QAAI,IAAI,GAAG,MAAM;AACjB,cAAU,EAAE;AAAA,EACd;AACA,SAAO;AACT;AASA,IAAM,kBAAN,MAA0C;AAAA,EASxC,YACmB,QACA,QACA,WACA,WACA,gBACjB;AALiB;AACA;AACA;AACA;AACA;AAAA,EAChB;AAAA,EALgB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAbF,OAAa,WAAW,QAAQ;AAAA,EAChC,UAAwB,CAAC;AAAA,EAClC,aAAa;AAAA,EACb,QAAQ;AAAA,EACR;AAAA,EACA,aAAa;AAAA,EACJ,QAA4D,CAAC;AAAA,EAU9E,MAAM,MAAM,OAAkC;AAC5C,QAAI,MAAM,WAAW,EAAG;AACxB,SAAK,SAAS,MAAM;AACpB,QAAI,KAAK,QAAQ,KAAK,gBAAgB;AAEpC,YAAM,IAAIA,iBAAgB,iCAAiC,KAAK,cAAc,EAAE;AAAA,IAClF;AACA,SAAK,KAAK,OAAO,KAAK;AACtB,SAAK,QAAQ,KAAK,KAAK;AACvB,SAAK,cAAc,MAAM;AACzB,QAAI,KAAK,cAAc,KAAK,UAAW,OAAM,KAAK,UAAU;AAAA,EAC9D;AAAA;AAAA,EAGA,MAAc,YAA2B;AACvC,QAAI,KAAK,aAAa,QAAW;AAC/B,YAAMC,OAAM,MAAM,KAAK,OAAO;AAAA,QAC5B,IAAI,6BAA6B,EAAE,QAAQ,KAAK,QAAQ,KAAK,KAAK,UAAU,CAAC;AAAA,MAC/E;AACA,UAAIA,KAAI,aAAa,QAAW;AAC9B,cAAM,IAAI,eAAe,+CAA+C;AAAA,MAC1E;AACA,WAAK,WAAWA,KAAI;AAAA,IACtB;AACA,UAAM,OAAO,YAAY,KAAK,SAAS,KAAK,UAAU;AACtD,SAAK,QAAQ,SAAS;AACtB,SAAK,aAAa;AAClB,SAAK,cAAc;AACnB,QAAI,KAAK,aAAa,cAAc;AAGlC,YAAM,IAAID,iBAAgB,oCAAoC,YAAY,aAAa;AAAA,IACzF;AACA,UAAM,MAAM,MAAM,KAAK,OAAO;AAAA,MAC5B,IAAI,kBAAkB;AAAA,QACpB,QAAQ,KAAK;AAAA,QACb,KAAK,KAAK;AAAA,QACV,UAAU,KAAK;AAAA,QACf,YAAY,KAAK;AAAA,QACjB,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,SAAK,MAAM,KAAK,EAAE,MAAM,IAAI,MAAM,YAAY,KAAK,WAAW,CAAC;AAAA,EACjE;AAAA;AAAA,EAGA,MAAM,SAAoD;AACxD,UAAM,SAAS,KAAK,KAAK,OAAO,KAAK;AACrC,QAAI,KAAK,aAAa,QAAW;AAC/B,YAAM,KAAK,OAAO;AAAA,QAChB,IAAI,iBAAiB;AAAA,UACnB,QAAQ,KAAK;AAAA,UACb,KAAK,KAAK;AAAA,UACV,MAAM,YAAY,KAAK,SAAS,KAAK,UAAU;AAAA,UAC/C,aAAa;AAAA;AAAA,QACf,CAAC;AAAA,MACH;AACA,aAAO,EAAE,MAAM,KAAK,OAAO,OAAO;AAAA,IACpC;AACA,QAAI,KAAK,aAAa,EAAG,OAAM,KAAK,UAAU;AAC9C,UAAM,KAAK,OAAO;AAAA,MAChB,IAAI,+BAA+B;AAAA,QACjC,QAAQ,KAAK;AAAA,QACb,KAAK,KAAK;AAAA,QACV,UAAU,KAAK;AAAA,QACf,iBAAiB,EAAE,OAAO,KAAK,MAAM;AAAA,QACrC,aAAa;AAAA;AAAA,MACf,CAAC;AAAA,IACH;AACA,SAAK,WAAW;AAChB,WAAO,EAAE,MAAM,KAAK,OAAO,OAAO;AAAA,EACpC;AAAA;AAAA;AAAA,EAIA,MAAM,QAAuB;AAC3B,QAAI,KAAK,aAAa,OAAW;AACjC,UAAM,KAAK,KAAK;AAChB,SAAK,WAAW;AAChB,QAAI;AACF,YAAM,KAAK,OAAO;AAAA,QAChB,IAAI,4BAA4B,EAAE,QAAQ,KAAK,QAAQ,KAAK,KAAK,WAAW,UAAU,GAAG,CAAC;AAAA,MAC5F;AAAA,IACF,QAAQ;AAAA,IAER;AAAA,EACF;AACF;AAMA,eAAe,QAAQ,MAAiD;AACtE,MAAI,SAAS,QAAW;AACtB,UAAM,IAAI,cAAc,qCAAqC;AAAA,EAC/D;AACA,SAAO,KAAK,qBAAqB;AACnC;;;AG9XA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA,kBAAAE;AAAA,EACA,sBAAAC;AAAA,EACA,yBAAAC;AAAA,OACK;AAEP;AAAA,EACE,oBAAAC;AAAA,EACA,wBAAAC;AAAA,EACA,oBAAAC;AAAA,OAEK;AAeP,IAAM,UAAN,MAA6C;AAAA,EAG3C,YACmB,QACA,QACjB;AAFiB;AACA;AAAA,EAChB;AAAA,EAFgB;AAAA,EACA;AAAA,EAJV,QAAQ;AAAA,EAOjB,MAAM,KAAK,KAAwC;AACjD,QAAI;AACJ,QAAI;AACF,YAAM,MAAM,KAAK,OAAO,KAAK,IAAIC,kBAAiB,EAAE,QAAQ,KAAK,QAAQ,KAAK,IAAI,CAAC,CAAC;AAAA,IACtF,SAAS,KAAK;AACZ,UAAI,WAAW,GAAG,EAAG,QAAO;AAC5B,YAAM,SAAS,GAAG;AAAA,IACpB;AAEA,SAAK,IAAI,iBAAiB,KAAK,eAAe;AAC5C,YAAM,IAAI;AAAA,QACR,mBAAmB,IAAI,aAAa,iBAAiB,aAAa;AAAA,MACpE;AAAA,IACF;AACA,QAAI,IAAI,SAAS,QAAW;AAC1B,YAAM,IAAI,eAAe,sCAAsC,GAAG,EAAE;AAAA,IACtE;AACA,UAAM,QAAQ,MACZ,IAAI,KACJ,qBAAqB;AACvB,WAAO,EAAE,OAAO,SAAS,IAAI,QAAQ,GAAG;AAAA,EAC1C;AAAA,EAEA,MAAM,MACJ,KACA,MACA,QACe;AACf,QAAI;AACF,YAAM,KAAK,OAAO;AAAA,QAChB,IAAIC,kBAAiB;AAAA,UACnB,QAAQ,KAAK;AAAA,UACb,KAAK;AAAA,UACL,MAAM;AAAA,UACN,aAAa;AAAA,UACb,aAAa,WAAW,WAAW,MAAM;AAAA,UACzC,SAAS,WAAW,WAAW,SAAY,OAAO;AAAA,QACpD,CAAC;AAAA,MACH;AAAA,IACF,SAAS,KAAK;AAEZ,UAAI,sBAAsB,GAAG,GAAG;AAC9B,cAAM,IAAIC,oBAAmB,6BAA6B,GAAG,EAAE;AAAA,MACjE;AACA,YAAM,SAAS,GAAG;AAAA,IACpB;AAAA,EACF;AAAA,EAEA,OAAO,SAAS,QAAuC;AACrD,QAAI;AACJ,OAAG;AACD,UAAI;AACJ,UAAI;AACF,cAAM,MAAM,KAAK,OAAO;AAAA,UACtB,IAAIC,sBAAqB;AAAA,YACvB,QAAQ,KAAK;AAAA,YACb,QAAQ;AAAA,YACR,mBAAmB;AAAA,UACrB,CAAC;AAAA,QACH;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,SAAS,GAAG;AAAA,MACpB;AACA,iBAAW,OAAO,IAAI,YAAY,CAAC,GAAG;AACpC,YAAI,IAAI,QAAQ,OAAW,OAAM,IAAI;AAAA,MACvC;AACA,cAAQ,IAAI,gBAAgB,OAAO,IAAI,wBAAwB;AAAA,IACjE,SAAS,UAAU;AAAA,EACrB;AACF;AAGA,SAAS,SAAS,KAAuB;AACvC,MAAI,YAAY,GAAG,GAAG;AACpB,WAAO,IAAIC;AAAA,MACT,uBAAwB,KAAkC,QAAQ,SAAS;AAAA,MAC3E,EAAE,OAAO,IAAI;AAAA,IACf;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,mBAAN,cAA+B,oBAAoB;AAAA,EACxD,YAAY,SAAkC;AAC5C;AAAA,MACE,IAAI,QAAQ,QAAQ,QAAQ,QAAQ,MAAM;AAAA,MAC1CC,uBAAsB,QAAQ,MAAM;AAAA,MACpC,QAAQ,QAAQ,MAAc,KAAK,IAAI;AAAA,IACzC;AAAA,EACF;AACF;;;ACxIA,SAA0B,sBAAsB;AAMhD,SAAS,gBAAgB;AAkClB,IAAM,YAAN,MAA0C;AAAA,EAGtC;AAAA,EACA;AAAA;AAAA,EAEA;AAAA,EAET,YAAY,SAA2B;AACrC,SAAK,SACH,QAAQ,UACR,IAAI,SAAS;AAAA,MACX,GAAI,QAAQ,WAAW,SAAY,CAAC,IAAI,EAAE,QAAQ,QAAQ,OAAO;AAAA,MACjE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,MACvE,GAAI,QAAQ,cAAc,SAAY,CAAC,IAAI,EAAE,gBAAgB,QAAQ,UAAU;AAAA,MAC/E,GAAI,QAAQ,gBAAgB,SAAY,CAAC,IAAI,EAAE,aAAa,QAAQ,YAAY;AAAA,IAClF,CAAC;AACH,UAAM,SAAS;AAAA,MACb,QAAQ,KAAK;AAAA,MACb,QAAQ,QAAQ;AAAA,MAChB,GAAI,QAAQ,WAAW,SAAY,CAAC,IAAI,EAAE,QAAQ,QAAQ,OAAO;AAAA,IACnE;AACA,SAAK,UAAU,IAAI,gBAAgB,MAAM;AACzC,SAAK,WAAW,IAAI,iBAAiB;AAAA,MACnC,GAAG;AAAA,MACH,GAAI,QAAQ,QAAQ,SAAY,CAAC,IAAI,EAAE,KAAK,QAAQ,IAAI;AAAA,IAC1D,CAAC;AACD,mBAAe,IAAI;AAAA,EACrB;AACF;",
|
|
6
|
+
"names": ["ValidationError", "ValidationError", "res", "TransientError", "WriteConflictError", "normalizeObjectPrefix", "GetObjectCommand", "ListObjectsV2Command", "PutObjectCommand", "GetObjectCommand", "PutObjectCommand", "WriteConflictError", "ListObjectsV2Command", "TransientError", "normalizeObjectPrefix"]
|
|
7
|
+
}
|
package/dist/keys.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { GenKey, SegmentRef } from '@cloudbitmaps/core/driver-kit';
|
|
2
|
+
/** Validate a caller-supplied key prefix. The rule is shared with every other object store. */
|
|
3
|
+
export { normalizeObjectPrefix as normalizeS3Prefix } from '@cloudbitmaps/core/driver-kit';
|
|
4
|
+
/**
|
|
5
|
+
* The S3 key prefix shared by all of a segment's generations: `<prefix><ns>/segments/<segment>.`. Used
|
|
6
|
+
* both as the `ListObjectsV2` prefix and as the string stripped by {@link parseGenerationFromKey}.
|
|
7
|
+
*/
|
|
8
|
+
export declare function segmentObjectPrefix(prefix: string | undefined, ref: SegmentRef): string;
|
|
9
|
+
/** The full S3 key of one `.crbm` generation: `<segmentPrefix><gen>.crbm`. */
|
|
10
|
+
export declare function storageObjectKey(prefix: string | undefined, key: GenKey): string;
|
|
11
|
+
export { registryPrefix, registryObjectKey, registryListPrefix, parseRegistryKey, } from '@cloudbitmaps/core/driver-kit';
|
|
12
|
+
/**
|
|
13
|
+
* Parse a generation number out of a full object key, given its segment prefix, or `null` if it doesn't
|
|
14
|
+
* match. Canonical decimal only — no leading zeros (so `…s.07.crbm` can't alias `…s.7.crbm`) and within
|
|
15
|
+
* safe-integer range. This also rejects a *different* segment whose name merely shares the prefix (e.g. a
|
|
16
|
+
* key for segment `s.x` won't parse under segment `s`'s prefix, since its middle isn't all digits).
|
|
17
|
+
*/
|
|
18
|
+
export declare function parseGenerationFromKey(segmentPrefix: string, objectKey: string): number | null;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `S3RegistryDriver` — an {@link IRegistryDriver} over S3-compatible object storage.
|
|
3
|
+
*
|
|
4
|
+
* Lets a **read-mostly deployment run on S3 alone** — storage `.crbm` generations + the registry in one bucket,
|
|
5
|
+
* no separate database. The protocol (an ABA-safe OCC counter, tombstoning delete, the bounded retry, the key layout)
|
|
6
|
+
* lives once in {@link ObjectStoreRegistry}; this file is only the three I/O calls S3 makes, so the S3, GCS
|
|
7
|
+
* and Azure registries cannot drift from one another.
|
|
8
|
+
*
|
|
9
|
+
* **The atomic swap is offloaded to S3's conditional writes** (GA Nov 2024): `If-None-Match: *` for
|
|
10
|
+
* create-only and `If-Match: <etag>` for compare-and-swap, so a concurrent writer between our read and our
|
|
11
|
+
* PUT loses with a `412` → {@link WriteConflictError}. Reads are strongly consistent (S3, since 2020),
|
|
12
|
+
* satisfying the registry's `strongRead` contract. The client is **injected**, exactly like
|
|
13
|
+
* {@link S3StorageDriver}.
|
|
14
|
+
*
|
|
15
|
+
* **Deployment requirements** (a backend/policy that violates these silently corrupts the registry):
|
|
16
|
+
* - The backend **must honor `If-Match`** (AWS S3; recent MinIO). One that returns ETags but ignores the
|
|
17
|
+
* precondition degrades compare-and-swap to last-write-wins → lost `currentGen` swaps. Verified against
|
|
18
|
+
* real S3 semantics by the MinIO integration lane.
|
|
19
|
+
* - The IAM principal needs **`s3:ListBucket`** on the bucket. Without it, `GetObject` on a missing key
|
|
20
|
+
* returns `403` (not `404`), so the "absent segment → `null`" contract (and `create`'s bootstrap read)
|
|
21
|
+
* breaks — and `list()` needs it regardless.
|
|
22
|
+
* - **Do not apply an S3 lifecycle-expiration rule to the `registry/` prefix.** See {@link ObjectStoreRegistry}.
|
|
23
|
+
*/
|
|
24
|
+
import { ObjectStoreRegistry } from '@cloudbitmaps/core/driver-kit';
|
|
25
|
+
import { type S3Client } from '@aws-sdk/client-s3';
|
|
26
|
+
export interface S3RegistryDriverOptions {
|
|
27
|
+
/** A constructed S3 client (point its `endpoint` at MinIO for local/integration use). */
|
|
28
|
+
readonly client: S3Client;
|
|
29
|
+
/** Target bucket (must already exist). */
|
|
30
|
+
readonly bucket: string;
|
|
31
|
+
/** Optional key prefix under which all registry objects live (e.g. `cloudroaring/`). */
|
|
32
|
+
readonly prefix?: string;
|
|
33
|
+
/** Injected clock for `createdAt`/`updatedAt`; defaults to `Date.now`. */
|
|
34
|
+
readonly now?: () => number;
|
|
35
|
+
}
|
|
36
|
+
export declare class S3RegistryDriver extends ObjectStoreRegistry {
|
|
37
|
+
constructor(options: S3RegistryDriverOptions);
|
|
38
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure helpers for classifying S3 SDK errors + parsing response headers (conflict and transient classification).
|
|
3
|
+
*
|
|
4
|
+
* Kept SDK-free and side-effect-free (they only read structural shapes — `err.name`,
|
|
5
|
+
* `$metadata.httpStatusCode`, a `Content-Range` string) so the subtle S3-specific translation logic is
|
|
6
|
+
* unit-testable without a live MinIO/S3 or even the AWS SDK. Shared AWS shapes come from `_shared/aws-errors`.
|
|
7
|
+
*/
|
|
8
|
+
/** A conditional `If-None-Match: *` PUT lost the write-once race (the object already existed). */
|
|
9
|
+
export declare function isPreconditionFailed(err: unknown): boolean;
|
|
10
|
+
/**
|
|
11
|
+
* A conditional write (`If-None-Match: *` / `If-Match: <etag>`) lost the race — **either** outcome S3 uses:
|
|
12
|
+
* the precondition evaluated false (`412 PreconditionFailed`), **or** S3 rejected concurrent conditional
|
|
13
|
+
* writes to the same key to prevent a lost update (`409 ConditionalRequestConflict`, which AWS documents and
|
|
14
|
+
* asks you to retry). Both mean "you lost; re-read and retry" — so both must map to `WriteConflictError` and
|
|
15
|
+
* route through the caller's OCC path, never a blind transient retry (which would just replay a doomed PUT).
|
|
16
|
+
*/
|
|
17
|
+
export declare function isConditionalConflict(err: unknown): boolean;
|
|
18
|
+
/** The object / generation does not exist (GetObject → `NoSuchKey`, HeadObject → `NotFound`; both 404). */
|
|
19
|
+
export declare function isNotFound(err: unknown): boolean;
|
|
20
|
+
/** A range request started past EOF (HTTP 416). */
|
|
21
|
+
export declare function isInvalidRange(err: unknown): boolean;
|
|
22
|
+
/**
|
|
23
|
+
* A transient S3 fault that is safe to retry: throttling (`SlowDown` / 503), any 5xx, a dropped/timed-out
|
|
24
|
+
* connection, or anything the SDK itself marks retryable. Excludes the deterministic outcomes above
|
|
25
|
+
* (412/404/416) — those are caller-meaningful and must never be retried/reclassified.
|
|
26
|
+
*/
|
|
27
|
+
export declare function isTransient(err: unknown): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Parse the total object size out of a `Content-Range: bytes <start>-<end>/<total>` header, or `undefined`
|
|
30
|
+
* if absent/unparseable/unsafe. The total is the part after the final `/`.
|
|
31
|
+
*/
|
|
32
|
+
export declare function totalFromContentRange(contentRange: string | undefined): number | undefined;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { BlobSink, GenKey, IStorageDriver, SegmentRef, StorageCaps } from '@cloudbitmaps/core/driver-kit';
|
|
2
|
+
import { type S3Client } from '@aws-sdk/client-s3';
|
|
3
|
+
export interface S3StorageDriverOptions {
|
|
4
|
+
/** A constructed S3 client (point its `endpoint` at MinIO for local/integration use). */
|
|
5
|
+
readonly client: S3Client;
|
|
6
|
+
/** Target bucket (must already exist). */
|
|
7
|
+
readonly bucket: string;
|
|
8
|
+
/** Optional key prefix under which all objects live (e.g. `cloudroaring/`). */
|
|
9
|
+
readonly prefix?: string;
|
|
10
|
+
/**
|
|
11
|
+
* Largest object this driver will write/advertise. Default = `partBytes × 10,000` (≈ 80 GiB at the default
|
|
12
|
+
* 8 MiB part) — the honest ceiling reachable within S3's 10,000-part limit. Set it higher and `partBytes`
|
|
13
|
+
* auto-grows so 10,000 parts still cover it (raising peak write memory to ~one part); up to the 5 TiB S3 max.
|
|
14
|
+
*/
|
|
15
|
+
readonly maxObjectBytes?: number;
|
|
16
|
+
/** Multipart part size in bytes (default 8 MiB; clamped to the S3 5 MiB minimum). Tunes peak write memory. */
|
|
17
|
+
readonly partBytes?: number;
|
|
18
|
+
}
|
|
19
|
+
export declare class S3StorageDriver implements IStorageDriver {
|
|
20
|
+
private readonly client;
|
|
21
|
+
private readonly bucket;
|
|
22
|
+
private readonly prefix;
|
|
23
|
+
private readonly maxObjectBytes;
|
|
24
|
+
private readonly partBytes;
|
|
25
|
+
constructor(options: S3StorageDriverOptions);
|
|
26
|
+
capabilities(): StorageCaps;
|
|
27
|
+
putImmutable(key: GenKey, write: (sink: BlobSink) => Promise<void>): Promise<{
|
|
28
|
+
size: number;
|
|
29
|
+
sha256: string;
|
|
30
|
+
}>;
|
|
31
|
+
getRange(key: GenKey, offset: number, length: number): Promise<Uint8Array>;
|
|
32
|
+
getTail(key: GenKey, maxBytes: number): Promise<{
|
|
33
|
+
bytes: Uint8Array;
|
|
34
|
+
size: number;
|
|
35
|
+
}>;
|
|
36
|
+
delete(key: GenKey): Promise<void>;
|
|
37
|
+
list(ref: SegmentRef): AsyncIterable<GenKey>;
|
|
38
|
+
/** Map S3 read errors to the driver vocabulary; pass everything else through {@link mapError}. */
|
|
39
|
+
private mapReadError;
|
|
40
|
+
/**
|
|
41
|
+
* Reclassify a transient S3 fault (throttle/5xx/dropped connection) as a retryable {@link TransientError}
|
|
42
|
+
* so the retry decorator can ride it out; everything else propagates unchanged. The final fallback at every
|
|
43
|
+
* `client.send` site, so callers and the decorator only ever see typed errors.
|
|
44
|
+
*/
|
|
45
|
+
private mapError;
|
|
46
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@cloudbitmaps/s3",
|
|
3
|
+
"version": "0.10.0-rc.0",
|
|
4
|
+
"description": "S3 and S3-compatible storage (R2, MinIO, Ceph, Wasabi, B2) for CloudBitmaps",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"cloudbitmaps",
|
|
7
|
+
"bitmap",
|
|
8
|
+
"roaring-bitmap",
|
|
9
|
+
"serverless",
|
|
10
|
+
"cloud",
|
|
11
|
+
"typescript",
|
|
12
|
+
"s3",
|
|
13
|
+
"aws",
|
|
14
|
+
"r2",
|
|
15
|
+
"minio",
|
|
16
|
+
"ceph",
|
|
17
|
+
"wasabi",
|
|
18
|
+
"backblaze",
|
|
19
|
+
"object-storage"
|
|
20
|
+
],
|
|
21
|
+
"license": "Apache-2.0",
|
|
22
|
+
"author": "Sharvil Kadam",
|
|
23
|
+
"repository": {
|
|
24
|
+
"type": "git",
|
|
25
|
+
"url": "git+https://github.com/cloudbitmaps/cloudbitmaps.git",
|
|
26
|
+
"directory": "packages/s3"
|
|
27
|
+
},
|
|
28
|
+
"homepage": "https://github.com/cloudbitmaps/cloudbitmaps#readme",
|
|
29
|
+
"bugs": {
|
|
30
|
+
"url": "https://github.com/cloudbitmaps/cloudbitmaps/issues"
|
|
31
|
+
},
|
|
32
|
+
"type": "module",
|
|
33
|
+
"engines": {
|
|
34
|
+
"node": ">=22.12"
|
|
35
|
+
},
|
|
36
|
+
"sideEffects": false,
|
|
37
|
+
"main": "./dist/index.js",
|
|
38
|
+
"types": "./dist/index.d.ts",
|
|
39
|
+
"exports": {
|
|
40
|
+
".": {
|
|
41
|
+
"types": "./dist/index.d.ts",
|
|
42
|
+
"default": "./dist/index.js"
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
"files": [
|
|
46
|
+
"dist",
|
|
47
|
+
"README.md",
|
|
48
|
+
"LICENSE",
|
|
49
|
+
"NOTICE"
|
|
50
|
+
],
|
|
51
|
+
"publishConfig": {
|
|
52
|
+
"access": "public"
|
|
53
|
+
},
|
|
54
|
+
"dependencies": {
|
|
55
|
+
"@aws-sdk/client-s3": ">=3.645.0 <4",
|
|
56
|
+
"@cloudbitmaps/core": "^0.10.0"
|
|
57
|
+
},
|
|
58
|
+
"devDependencies": {
|
|
59
|
+
"@types/node": "^22.0.0",
|
|
60
|
+
"typescript": "^5.5.0"
|
|
61
|
+
},
|
|
62
|
+
"scripts": {
|
|
63
|
+
"build": "node ../../scripts/build.mjs"
|
|
64
|
+
}
|
|
65
|
+
}
|