@cloudbitmaps/s3 0.21.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cloudbitmaps/s3",
3
- "version": "0.21.0",
3
+ "version": "0.23.0",
4
4
  "description": "S3 and S3-compatible storage (R2, MinIO, Ceph, Wasabi, B2) for CloudBitmaps",
5
5
  "keywords": [
6
6
  "cloudbitmaps",
@@ -40,7 +40,8 @@
40
40
  ".": {
41
41
  "types": "./dist/index.d.ts",
42
42
  "default": "./dist/index.js"
43
- }
43
+ },
44
+ "./package.json": "./package.json"
44
45
  },
45
46
  "files": [
46
47
  "dist",
@@ -53,7 +54,7 @@
53
54
  },
54
55
  "dependencies": {
55
56
  "@aws-sdk/client-s3": ">=3.700.0 <4",
56
- "@cloudbitmaps/core": "^0.21.0"
57
+ "@cloudbitmaps/core": "0.23.0"
57
58
  },
58
59
  "devDependencies": {
59
60
  "@types/node": "^22.0.0",
@@ -1,13 +0,0 @@
1
- export declare function httpStatus(err: unknown): number | undefined;
2
- export declare function errorName(err: unknown): string | undefined;
3
- /** A lower-level transport code (e.g. `ECONNRESET`) — the Node networking layer sets `code`. */
4
- export declare function errorCode(err: unknown): string | undefined;
5
- /** The AWS SDK v3 tags retryable errors with a `$retryable` object (throttling faults carry `.throttling`). */
6
- export declare function isSdkRetryable(err: unknown): boolean;
7
- /** Any 5xx is a server-side fault that's safe to retry (the request didn't deterministically fail). */
8
- export declare function isServerSide(err: unknown): boolean;
9
- /**
10
- * A dropped/timed-out connection — transient by nature; a retry on a fresh connection usually succeeds. The SDK's
11
- * transport error names, the codes Node gives a failed connection ({@link isTransportFault}), and the messages above.
12
- */
13
- export declare function isNetworkOrTimeout(err: unknown): boolean;
@@ -1,15 +0,0 @@
1
- /** The `GetObject` response body, narrowed to the parts used. */
2
- type ResponseBody = {
3
- transformToByteArray?: () => Promise<Uint8Array>;
4
- [Symbol.asyncIterator]?: () => AsyncIterator<unknown>;
5
- destroy?: () => void;
6
- } | undefined;
7
- /** Destroy a response body left unread, which releases its connection; a body with no `destroy` is left alone. */
8
- export declare function destroyBody(body: unknown): void;
9
- /**
10
- * The body's bytes, at most `maxBytes` of them. `oversize` builds the typed error thrown when the count passes
11
- * `maxBytes`, or when the response advertises more than that before a byte is read; any other failure of the stream
12
- * propagates as it is.
13
- */
14
- export declare function readBounded(body: ResponseBody, maxBytes: number, oversize: () => Error, advertised?: number): Promise<Uint8Array>;
15
- export {};
@@ -1,36 +0,0 @@
1
- import type { S3Client } from '@aws-sdk/client-s3';
2
- /** What a client does with the requests the registry sends. */
3
- export interface ClientFacts {
4
- /** The host a request to the bucket is addressed to, as the client resolves it now. */
5
- readonly host: string;
6
- /** Whether a `DeleteObject` given `IfMatch` goes out with an `If-Match` header. */
7
- readonly sendsDeleteIfMatch: boolean;
8
- /** Whether a `PutObject` given `IfMatch` goes out with an `If-Match` header: the registry's compare-and-swap. */
9
- readonly sendsPutIfMatch: boolean;
10
- /** Whether a `PutObject` given `IfNoneMatch` goes out with an `If-None-Match` header: the registry's create. */
11
- readonly sendsPutIfNoneMatch: boolean;
12
- }
13
- /** A key no registry row or generation can hold; the probe's requests are never sent, so it is never used. */
14
- export declare const PROBE_KEY = "cloudbitmaps-probe";
15
- /**
16
- * What `client` does with a conditional `DeleteObject` and `PutObject` to `bucket`, or `undefined` when it cannot be
17
- * found out. Never throws, never sends a request, and never runs anything the caller added to `client`.
18
- */
19
- export declare function probeClient(client: S3Client, bucket: string): Promise<ClientFacts | undefined>;
20
- /**
21
- * Whether a `DeleteObject` may rely on `If-Match`, for the registry's row delete and the storage half's conditional
22
- * delete alike: `explicit` when the caller set it, but never for an SDK seen not to send the header; otherwise only
23
- * when the client resolves to an AWS S3 host, which applies the precondition, and `false` until the client has been
24
- * asked, since an S3-compatible store (MinIO, for one) can accept the header and ignore it.
25
- */
26
- export declare function appliesDeleteIfMatch(explicit: boolean | undefined, facts: ClientFacts | undefined): boolean;
27
- /**
28
- * Whether `hostname` is an AWS S3 host, by its structure: under an AWS domain, with a label that names S3 (`s3`,
29
- * `s3-fips`, an access point's, a multi-region access point's `s3-global`, Object Lambda's, an Outpost's, a dash-joined
30
- * `s3-<region>`, a directory bucket's `s3express-…`), followed only by what S3's hosts put there: `dualstack`, a
31
- * region and `vpce`, in that order. Anything may come before it (a bucket, an access point, a VPC endpoint id). So
32
- * the FIPS, dual-stack, access-point and VPC interface forms all count, and another AWS service's host whose name
33
- * starts with `s3` does not: a load balancer, an API Gateway, a website endpoint. A host of any other domain is not,
34
- * whatever it speaks: a store behind one has to be vouched for by the caller.
35
- */
36
- export declare function isAwsS3Host(hostname: string): boolean;
package/dist/defined.d.ts DELETED
@@ -1,8 +0,0 @@
1
- /**
2
- * `values` without the entries that are `undefined`, typed as optional properties, so a constructor can pass on the
3
- * options it was given: `exactOptionalPropertyTypes` refuses `{ x: undefined }` for an optional `x`, and a sub-driver
4
- * must see an option the caller left out as absent.
5
- */
6
- export declare function defined<T extends Record<string, unknown>>(values: T): {
7
- [K in keyof T]?: Exclude<T[K], undefined>;
8
- };
@@ -1,11 +0,0 @@
1
- /** Validate a caller's `readTimeoutMs`, defaulting it: an integer from 0 (no timeout) to {@link MAX_TIMER_MS}. */
2
- export declare function resolveReadTimeoutMs(value: number | undefined): number;
3
- /** The options a timed read passes to its `client.send`: the abort signal, or nothing when the timeout is off. */
4
- export type ReadSendOptions = {
5
- readonly abortSignal: AbortSignal;
6
- } | undefined;
7
- /**
8
- * Run `read` under a `timeoutMs` timer (`0`: none). `read` passes `options` to its `client.send` and reads the body
9
- * before it returns, so the timer covers both. The timer is cleared when the read settles.
10
- */
11
- export declare function timedRead<T>(operation: 'GetObject' | 'HeadObject', timeoutMs: number, read: (options: ReadSendOptions) => Promise<T>): Promise<T>;
@@ -1,118 +0,0 @@
1
- /**
2
- * `S3RegistryDriver` — an {@link IRegistryDriver} over S3-compatible object storage.
3
- *
4
- * Lets a **read-mostly deployment run on S3 alone** — storage `.crbm` generations + the registry in one bucket,
5
- * no separate database. The protocol (an ABA-safe OCC token, the delete and its tombstone, the bounded retry, the key
6
- * layout) lives once in {@link ObjectStoreRegistry}; this file is only the I/O calls S3 makes, so the S3, GCS
7
- * and Azure registries cannot drift from one another.
8
- *
9
- * **A delete removes the row for good** with a `DeleteObject` under `If-Match: <etag>`, sent once like the writes,
10
- * when `conditionalDelete` is on. By default it is on when the host the client resolves is an AWS S3 host, whichever way
11
- * the endpoint was set (a constructor `endpoint`, `AWS_ENDPOINT_URL_S3`, `AWS_ENDPOINT_URL`, the shared config file), and
12
- * off for any other, since an S3-compatible store may accept the header and ignore it (MinIO does). The client is asked
13
- * once, before the registry's first request, and the SDK is checked to send the header at all ({@link probeClient}).
14
- *
15
- * **The atomic swap is offloaded to S3's conditional writes** (GA Nov 2024): `If-None-Match: *` for
16
- * create-only and `If-Match: <etag>` for compare-and-swap, so a concurrent writer between our read and our
17
- * PUT loses with a `412` → {@link WriteConflictError}. Each conditional PUT is sent once, with the SDK's retry off
18
- * for it ({@link sendOnce}), so a `412` means another write got there first, never this one meeting itself after a
19
- * lost response. A transient failure reaches the caller as {@link TransientError}: the write may or may not have
20
- * landed, and the caller re-reads the row to learn where it stands. Reads are strongly consistent (S3, since 2020),
21
- * satisfying the registry's `strongRead` contract, and each can be timed as the storage driver's are ({@link timedRead}):
22
- * with `readTimeoutMs` set, a row's `GetObject` that has not finished, body included, after it throws
23
- * {@link TransientError}. It is off by default, and the writes and listings are never timed. The client is **injected**, exactly like {@link S3StorageDriver}.
24
- *
25
- * **Deployment requirements** (a backend/policy that violates these silently corrupts the registry):
26
- * - The backend **must honor `If-Match`** (AWS S3; recent MinIO). One that returns ETags but ignores the
27
- * precondition degrades compare-and-swap to last-write-wins → lost `currentGen` swaps. Verified against
28
- * real S3 semantics by the MinIO integration lane.
29
- * - The IAM principal needs **`s3:ListBucket`** on the bucket. Without it, `GetObject` on a missing key
30
- * returns `403` (not `404`), so the "absent segment → `null`" contract (and `create`'s bootstrap read)
31
- * breaks — and `list()` needs it regardless.
32
- * - **Do not apply an S3 lifecycle-expiration rule to the `registry/` prefix that expires a current version**
33
- * (`NoncurrentVersionExpiration` is safe). See {@link ObjectStoreRegistry}.
34
- */
35
- import { ObjectStoreRegistry } from '@cloudbitmaps/core/driver-kit';
36
- import type { ObjectRegistryStore, ObjectRow } from '@cloudbitmaps/core/driver-kit';
37
- import { type S3Client } from '@aws-sdk/client-s3';
38
- interface S3RegistryDriverOptions {
39
- /** A constructed S3 client (point its `endpoint` at MinIO for local/integration use). */
40
- readonly client: S3Client;
41
- /** Target bucket (must already exist). */
42
- readonly bucket: string;
43
- /** Optional key prefix under which all registry objects live (e.g. `cloudbitmaps/`). */
44
- readonly prefix?: string;
45
- /** Injected clock for `createdAt`/`updatedAt`; defaults to `Date.now`. */
46
- readonly now?: () => number;
47
- /**
48
- * How long one read of a row — its `GetObject`, the body included — may take before it is abandoned and throws
49
- * `TransientError`, in ms. `0`, the default, sets no timeout. Must be a non-negative safe integer no larger than
50
- * 2,147,483,647. Writes and listings are not timed.
51
- *
52
- * The clock starts when the read is handed to the SDK, so it also counts the time the read waits for one of the
53
- * client's sockets (128 by default on the client the store builds, 50 on the SDK's own) and the time spent fetching
54
- * credentials, and under `retryMode: 'adaptive'` the SDK's rate-limiter wait. A burst of concurrent reads larger than
55
- * the socket pool can therefore time out with nothing slow on the wire: size the timeout above the worst queueing
56
- * your concurrency implies, or raise `maxSockets` (the store's option, or your own client's). On a client built with
57
- * `cacheMiddleware: true`, a timed read resolves its middleware each time.
58
- */
59
- readonly readTimeoutMs?: number;
60
- /**
61
- * Whether a delete removes a row for good, by a `DeleteObject` sent with `If-Match: <the ETag it read>`, rather than
62
- * leaving a tombstone.
63
- *
64
- * Defaults to `true` when the host the client resolves is an AWS S3 host (`*.amazonaws.com`, `*.amazonaws.com.cn`: the
65
- * FIPS, dual-stack, access-point and VPC interface forms included), where `If-Match` on `DeleteObject` is documented
66
- * for general purpose and directory buckets, and to `false` for any other host, or when the host cannot be resolved.
67
- * The host is the one the SDK resolves for a request, so an endpoint set through `AWS_ENDPOINT_URL_S3`,
68
- * `AWS_ENDPOINT_URL` or an `endpoint_url` in the shared config file counts exactly as a constructor `endpoint` does.
69
- * It is resolved once, lazily: until the registry's first read or listing the answer reads `false`, and a client that
70
- * cannot be resolved then (no region) keeps tombstoning for the life of the registry, so set the
71
- * option to override it.
72
- *
73
- * Set it explicitly to override the host. An S3-compatible store must apply the precondition before you set it there:
74
- * MinIO, for one, ignores it and deletes anyway, and on such a store two sweepers and a re-create can delete a live
75
- * row. A `true` never overrides the SDK: one that does not send `If-Match` on a `DeleteObject` leaves the row
76
- * tombstoned whatever is set.
77
- */
78
- readonly conditionalDelete?: boolean;
79
- }
80
- /**
81
- * The calls {@link ObjectStoreRegistry} needs, in S3's dialect. Exported for the tests that drive one directly.
82
- *
83
- * `conditionalDelete` is the caller's explicit setting, or `undefined` to follow the client. What the client does is
84
- * found out once, before the first read, listing or write ({@link probeClient}), so by the time the registry decides
85
- * how to delete it has been.
86
- */
87
- export declare class S3RegistryStore implements ObjectRegistryStore {
88
- private readonly client;
89
- private readonly bucket;
90
- private readonly readTimeoutMs;
91
- private readonly explicitConditionalDelete?;
92
- readonly label = "S3";
93
- private probed;
94
- private facts;
95
- constructor(client: S3Client, bucket: string, readTimeoutMs: number, explicitConditionalDelete?: boolean | undefined);
96
- /**
97
- * Whether a delete may rely on `If-Match`: the caller's setting, else whether the client's host is AWS S3, and in
98
- * neither case when the SDK is seen not to send the header. `false` until the client has been asked, unless the caller
99
- * vouched for the store.
100
- */
101
- get conditionalDelete(): boolean;
102
- /** Ask the client once what it does with the registry's requests; every caller waits for the same answer. */
103
- private settle;
104
- /** Refuse a write the SDK would send without its precondition: it would land as a plain overwrite. */
105
- private refuseUnfencedWrites;
106
- read(key: string): Promise<ObjectRow | null>;
107
- write(key: string, body: Uint8Array, expect: 'absent' | {
108
- version: string;
109
- }): Promise<void>;
110
- delete(key: string, expect: {
111
- version: string;
112
- }): Promise<void>;
113
- listKeys(prefix: string): AsyncIterable<string>;
114
- }
115
- export declare class S3RegistryDriver extends ObjectStoreRegistry {
116
- constructor(options: S3RegistryDriverOptions);
117
- }
118
- export {};
@@ -1,66 +0,0 @@
1
- /**
2
- * Pure helpers for classifying S3 SDK errors + parsing response headers (conflict and transient classification).
3
- *
4
- * Kept SDK-free and side-effect-free (they only read structural shapes — `err.name`,
5
- * `$metadata.httpStatusCode`, a `Content-Range` string) so the subtle S3-specific translation logic is
6
- * unit-testable without a live MinIO/S3 or even the AWS SDK. The AWS error shapes come from `./aws-errors`.
7
- */
8
- /** A conditional `If-None-Match: *` PUT lost the write-once race (the object already existed). */
9
- export declare function isPreconditionFailed(err: unknown): boolean;
10
- /**
11
- * A conditional write (`If-None-Match: *` / `If-Match: <etag>`) lost the race — **either** outcome S3 uses:
12
- * the precondition evaluated false (`412 PreconditionFailed`), **or** S3 rejected concurrent conditional
13
- * writes to the same key to prevent a lost update (`409 ConditionalRequestConflict`, which AWS documents and
14
- * asks you to retry). Both mean "you lost; re-read and retry" — so both must map to `WriteConflictError` and
15
- * route through the caller's OCC path, never a blind transient retry (which would just replay a doomed PUT).
16
- */
17
- export declare function isConditionalConflict(err: unknown): boolean;
18
- /**
19
- * S3 asked the client to slow down: `503 SlowDown`, or any other `503` (`ServiceUnavailable`). The SDK raises it with
20
- * `name` set to the code and `$metadata.httpStatusCode` to 503, and no `$retryable` marker. S3 does not document that a
21
- * throttled request was not applied, so a write sent again after one must still tell a landed first send apart.
22
- */
23
- export declare function isThrottle(err: unknown): boolean;
24
- /** A multipart upload id S3 no longer knows: completed, aborted or expired (`404 NoSuchUpload`). */
25
- export declare function isNoSuchUpload(err: unknown): boolean;
26
- /**
27
- * The bucket itself does not exist (`404 NoSuchBucket`): a wiring fault, never an absent object. Read as one, a
28
- * misnamed or deleted bucket would answer every read as an empty segment. A HEAD carries no code, so it cannot be told
29
- * apart there; every request that reaches a missing bucket with a body says so.
30
- */
31
- export declare function isMissingBucket(err: unknown): boolean;
32
- /**
33
- * `err`, or the {@link ValidationError} a missing bucket is: it names the bucket and what to check, and carries the
34
- * service's error as its `cause`. A wiring fault is not an absent object, so it is not a `NotFoundError`: a read
35
- * re-resolves on that, and would treat a misnamed bucket as a miss it can heal.
36
- */
37
- export declare function asMissingBucket(err: unknown, bucket: string): unknown;
38
- /**
39
- * The object / generation does not exist (GetObject → `NoSuchKey`, HeadObject → `NotFound`; both 404), in a bucket
40
- * that does: a 404 that says the bucket is missing is not one.
41
- */
42
- export declare function isNotFound(err: unknown): boolean;
43
- /** A range request started past EOF (HTTP 416). */
44
- export declare function isInvalidRange(err: unknown): boolean;
45
- /**
46
- * A transient S3 fault that is safe to retry: throttling (`SlowDown` / 503), any 5xx, a dropped/timed-out
47
- * connection, a clock-skew refusal the SDK has corrected for, or anything the SDK itself marks retryable. Excludes
48
- * the deterministic outcomes above (412/404/416) — those are caller-meaningful and must never be
49
- * retried/reclassified.
50
- */
51
- export declare function isTransient(err: unknown): boolean;
52
- /**
53
- * The message a transient S3 fault is handed on with: `transient S3 fault: <name>`, the SDK error's name, which names
54
- * S3's error code (`SlowDown`, `InternalError`) or the transport failure (`TimeoutError`).
55
- */
56
- export declare function transientFaultMessage(err: unknown): string;
57
- /** The first and last byte a `Content-Range: bytes <start>-<end>/<total>` header names, or `undefined` if unparseable. */
58
- export declare function spanFromContentRange(contentRange: string | undefined): {
59
- start: number;
60
- end: number;
61
- } | undefined;
62
- /**
63
- * Parse the total object size out of a `Content-Range: bytes <start>-<end>/<total>` header, or `undefined`
64
- * if absent/unparseable/unsafe. The total is the part after the final `/`.
65
- */
66
- export declare function totalFromContentRange(contentRange: string | undefined): number | undefined;
@@ -1,12 +0,0 @@
1
- /**
2
- * Remove credentials from an error the AWS SDK raised, before the driver throws it or keeps it as a `cause`.
3
- *
4
- * The SDK keeps the HTTP response it got on the error (`$response`), and through its socket the request it sent, with
5
- * the signed `Authorization` header and `x-amz-security-token`, on properties `util.inspect` shows only with
6
- * `showHidden`. Those objects are live (a socket still closing reads them), so they are never changed: the error is
7
- * replaced by a copy that has none of them (an error with none is returned as it is). The copy keeps its prototype, name, message, stack, status, code, request
8
- * id and every other plain field, so classification by name, code or status works on it as on the original. A
9
- * credential header or header line left anywhere in the copy's plain data is redacted.
10
- */
11
- /** `err` as it is safe to throw: a credential-free copy of an error object, any other value as it is. */
12
- export declare function scrubCredentials<T>(err: T): T;
@@ -1,23 +0,0 @@
1
- /**
2
- * `sendOnce` — send a conditional write exactly once, with the SDK's retry off for that one command.
3
- *
4
- * The SDK re-sends a request whose response it did not get: a timeout, a reset connection, a 5xx. For a conditional
5
- * write that is the wrong thing to do. When the write landed and only its response was lost, the second send meets
6
- * the first — `If-None-Match: *` finds the object it created, `If-Match` finds the ETag it replaced — and fails with
7
- * `412`, which the driver can only report as a lost race. The caller is told its write lost when it won. Only the
8
- * caller can find out which happened, by reading the pointer or listing the generations, so the write is sent once
9
- * and a transient failure reaches it as `TransientError`.
10
- *
11
- * **How.** The client runs its retry as one middleware, `retryMiddleware`, at high priority in the `finalizeRequest`
12
- * step, whatever retry strategy or `maxAttempts` it was built with. When a command is sent, its own middleware stack
13
- * is merged over the client's, and an entry with the same name, step and priority that sets `override` replaces the
14
- * client's. So this command runs with a pass-through where the retry was, and nothing about the client changes: a
15
- * caller's own client keeps its configuration, and every other command sent through it keeps its retry.
16
- *
17
- * The options object passed to `send` matters too. A client built with `cacheMiddleware: true` reuses the handler it
18
- * resolved for the first command of a class, and that handler holds the retry; `send` resolves afresh whenever it is
19
- * given options, so the replacement always takes effect.
20
- */
21
- import type { $Command, S3Client, S3ClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from '@aws-sdk/client-s3';
22
- /** Send `command` once: its retry step is a pass-through, and the client's stays as it is for every other command. */
23
- export declare function sendOnce<Input extends ServiceInputTypes, Output extends ServiceOutputTypes>(client: S3Client, command: $Command<Input, Output, S3ClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes>): Promise<Output>;
@@ -1,57 +0,0 @@
1
- /**
2
- * The socket-pool advisory: once the first read has run, the SDK's own request handler has made its agents and its
3
- * public `httpHandlerConfigs()` reports their `maxSockets`. A pool smaller than twice the default window of a combine
4
- * (32 ranges held ahead per operand, two operands) makes reads queue for a socket, so one `advisory` event goes to the
5
- * store's metrics sink, once. Nothing is logged or printed.
6
- *
7
- * A handler this cannot read (a custom one, HTTP/2, Fetch, an agent class that hides `maxSockets`) gives no event and
8
- * no error. The window it compares against is the default, because a combine's `concurrency` is chosen per call.
9
- */
10
- import type { S3Client } from '@aws-sdk/client-s3';
11
- import type { IMetricsSink } from '@cloudbitmaps/core/driver-kit';
12
- /** The default window of a combine, which the engine owns; a test holds the two equal. */
13
- export declare const SOCKET_ADVISORY_CONCURRENCY = 32;
14
- /** What of the SDK's request handler the limit and the advisory need: its `handle` and the agents it exposes once it has run. */
15
- export interface PooledHandler {
16
- handle(request: unknown, options?: unknown): Promise<unknown>;
17
- httpHandlerConfigs?: () => {
18
- httpAgent?: {
19
- maxSockets: number;
20
- };
21
- httpsAgent?: {
22
- maxSockets: number;
23
- };
24
- };
25
- }
26
- export declare class SocketAdvisory {
27
- private readonly client;
28
- private readonly bucket;
29
- /** True from the first `attach` until the one check has started: the only thing a request pays for after the first. */
30
- private pending;
31
- /** Every sink ever attached, so a sink is never given the event twice. */
32
- private readonly sinks;
33
- /** The check's result, once it has started; a sink attached after that is given the result as it settles. */
34
- private outcome;
35
- constructor(client: S3Client, bucket: string);
36
- /**
37
- * Hand a sink to the advisory. Each sink gets the event at most once: a sink attached before the check runs gets it
38
- * when the first read finishes, one attached later gets it as soon as the (single) check's result is known, and
39
- * one attached again is ignored.
40
- */
41
- attach(sink: IMetricsSink): void;
42
- /**
43
- * Called as a request finishes. The first call after a sink is attached reads the pool, never throws and never
44
- * rejects; every later one costs one boolean. Returns the pending check (for a test to await), or `undefined` once
45
- * it has run.
46
- */
47
- afterRequest(): Promise<void> | undefined;
48
- private read;
49
- /**
50
- * The pool the reads use, or `undefined` when it cannot be read. With an endpoint on the client, that is the
51
- * `maxSockets` of the agent for its scheme. Without one (the SDK's own endpoints, or an endpoint set by the
52
- * environment, which the client does not expose) the scheme is not known, so every agent the handler has made is
53
- * read and the pool counts as small only if all of them are: the larger one is returned. That never warns on a pool
54
- * the reads do not use, at the cost of a missed warning when only the unused agent is small.
55
- */
56
- private pool;
57
- }
package/dist/storage.d.ts DELETED
@@ -1,100 +0,0 @@
1
- import type { BlobSink, GenKey, IStorageDriver, SegmentRef, StorageCaps, StorageDeleteOptions, TailRead } from '@cloudbitmaps/core/driver-kit';
2
- import type { Clock, Rng } from '@cloudbitmaps/core';
3
- import { type S3Client } from '@aws-sdk/client-s3';
4
- import type { SocketAdvisory } from './socket-advisory.js';
5
- /** What the throttle backoff waits on. */
6
- type Sleeper = Pick<Clock, 'sleep'>;
7
- interface S3StorageDriverOptions {
8
- /** A constructed S3 client (point its `endpoint` at MinIO for local/integration use). */
9
- readonly client: S3Client;
10
- /** Target bucket (must already exist). */
11
- readonly bucket: string;
12
- /** Optional key prefix under which all objects live (e.g. `cloudbitmaps/`). */
13
- readonly prefix?: string;
14
- /**
15
- * Largest object this driver will write/advertise. Default = `partBytes × 10,000` (≈ 80 GiB at the default
16
- * 8 MiB part) — the honest ceiling reachable within S3's 10,000-part limit. Set it higher and `partBytes`
17
- * auto-grows so 10,000 parts still cover it (raising peak write memory to ~one part); up to the 5 TiB S3 max.
18
- */
19
- readonly maxObjectBytes?: number;
20
- /** Multipart part size in bytes (default 8 MiB; a smaller value is raised to the S3 5 MiB minimum). Must be a
21
- * positive safe integer. Tunes peak write memory. */
22
- readonly partBytes?: number;
23
- /**
24
- * How long one read — a `GetObject` or `HeadObject`, its body included — may take before it is abandoned and throws
25
- * `TransientError`, in ms. `0`, the default, sets no timeout. Must be a non-negative safe integer no larger than
26
- * 2,147,483,647. Writes and listings are not timed.
27
- *
28
- * The clock starts when the read is handed to the SDK, so it also counts the time the read waits for one of the
29
- * client's sockets (128 by default on the client the store builds, 50 on the SDK's own) and the time spent fetching
30
- * credentials, and under `retryMode: 'adaptive'` the SDK's rate-limiter wait. A burst of concurrent reads larger than
31
- * the socket pool can therefore time out with nothing slow on the wire: size the timeout above the worst queueing
32
- * your concurrency implies, or raise `maxSockets` (the store's option, or your own client's). On a client built with
33
- * `cacheMiddleware: true`, a timed read resolves its middleware each time.
34
- */
35
- readonly readTimeoutMs?: number;
36
- /** The socket-pool check, run as each read finishes; absent for a driver built on its own. */
37
- readonly sockets?: SocketAdvisory;
38
- /** What the backoff before re-sending a throttled commit waits on; real time when absent. */
39
- readonly clock?: Sleeper;
40
- /** Where the jitter of that backoff is drawn from; `Math.random` when absent. */
41
- readonly rng?: Rng;
42
- /**
43
- * Whether a delete given `ifVersion` is sent with `If-Match`, so it removes the object only while it is the one that
44
- * ETag names. Defaults as the registry's option does: on when the host the client resolves is an AWS S3 host and the
45
- * SDK sends the header, resolved on the first such delete (until then `capabilities()` reads `false`), and off for any
46
- * other host, and for a client that cannot be resolved then (no region), whose deletes stay unconditional. Set it for
47
- * an S3-compatible store only once you know the store applies `If-Match` on a delete. A `true` never overrides an SDK
48
- * that does not send the header.
49
- */
50
- readonly conditionalDelete?: boolean;
51
- }
52
- export declare class S3StorageDriver implements IStorageDriver {
53
- private readonly client;
54
- private readonly bucket;
55
- private readonly prefix;
56
- private readonly maxObjectBytes;
57
- private readonly partBytes;
58
- private readonly readTimeoutMs;
59
- private readonly clock;
60
- private readonly rng;
61
- private readonly sockets;
62
- private readonly explicitConditionalDelete;
63
- private probed;
64
- private facts;
65
- constructor(options: S3StorageDriverOptions);
66
- capabilities(): StorageCaps;
67
- /** Ask the client once where its requests go and whether it sends `If-Match` on a delete; it sends nothing. */
68
- private settle;
69
- putImmutable(key: GenKey, write: (sink: BlobSink) => Promise<void>): Promise<{
70
- size: number;
71
- sha256: string;
72
- }>;
73
- getRange(key: GenKey, offset: number, length: number): Promise<Uint8Array>;
74
- getTail(key: GenKey, maxBytes: number): Promise<TailRead>;
75
- private badRead;
76
- /**
77
- * The object's size and ETag from a `HeadObject`, each `undefined` when the response does not carry it. A missing
78
- * object throws {@link NotFoundError}.
79
- */
80
- private head;
81
- /**
82
- * One read request under the read timeout, its failures mapped to the driver vocabulary. The timeout's own
83
- * `TransientError` is raised outside the mapping, so it reaches the caller as it is.
84
- */
85
- private read;
86
- delete(key: GenKey, options?: StorageDeleteOptions): Promise<void>;
87
- /** Whether an object is under the key, from one `HeadObject`. A read that fails in any other way throws. */
88
- private exists;
89
- list(ref: SegmentRef): AsyncIterable<GenKey>;
90
- /** Map S3 read errors to the driver vocabulary; pass everything else through {@link mapError}. */
91
- private mapReadError;
92
- /**
93
- * Reclassify a transient S3 fault (throttle/5xx/dropped connection) as a retryable {@link TransientError}, so the
94
- * store's read retry can ride it out and a write's caller can tell it from a deterministic failure, and a missing
95
- * bucket as a `ValidationError` that names it; everything else propagates unchanged. The final fallback at every
96
- * `client.send` site, so callers and the read retry only ever see typed errors.
97
- */
98
- private mapError;
99
- }
100
- export {};