@hold-baby/bucket-db 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +331 -62
- package/dist/bucket-db.d.ts +25 -0
- package/dist/bucket-db.js +36 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +133 -0
- package/dist/collection.d.ts +78 -0
- package/dist/collection.js +321 -0
- package/dist/doctor.d.ts +7 -0
- package/dist/doctor.js +100 -0
- package/dist/errors.d.ts +59 -0
- package/dist/errors.js +57 -0
- package/dist/index.d.ts +13 -9
- package/dist/index.js +6 -10
- package/dist/keys.d.ts +12 -0
- package/dist/keys.js +31 -0
- package/dist/query.d.ts +16 -0
- package/dist/query.js +133 -0
- package/dist/snapshot.d.ts +16 -0
- package/dist/snapshot.js +40 -0
- package/dist/storage/adapter.d.ts +75 -0
- package/dist/storage/adapter.js +27 -0
- package/dist/storage/fs.d.ts +28 -0
- package/dist/storage/fs.js +175 -0
- package/dist/storage/memory.d.ts +18 -0
- package/dist/storage/memory.js +49 -0
- package/dist/storage/s3/adapters.d.ts +38 -0
- package/dist/storage/s3/adapters.js +65 -0
- package/dist/storage/s3/client.d.ts +52 -0
- package/dist/storage/s3/client.js +231 -0
- package/dist/storage/s3/credentials.d.ts +17 -0
- package/dist/storage/s3/credentials.js +42 -0
- package/dist/storage/s3/sigv4.d.ts +32 -0
- package/dist/storage/s3/sigv4.js +75 -0
- package/dist/types.d.ts +55 -0
- package/dist/types.js +1 -0
- package/dist/url.d.ts +16 -0
- package/dist/url.js +66 -0
- package/llms.txt +116 -20
- package/package.json +7 -5
- package/dist/cache/lru-cache.d.ts +0 -47
- package/dist/cache/lru-cache.d.ts.map +0 -1
- package/dist/cache/lru-cache.js +0 -110
- package/dist/cache/lru-cache.js.map +0 -1
- package/dist/core/bucket-db.d.ts +0 -20
- package/dist/core/bucket-db.d.ts.map +0 -1
- package/dist/core/bucket-db.js +0 -32
- package/dist/core/bucket-db.js.map +0 -1
- package/dist/core/collection.d.ts +0 -26
- package/dist/core/collection.d.ts.map +0 -1
- package/dist/core/collection.js +0 -213
- package/dist/core/collection.js.map +0 -1
- package/dist/core/wal.d.ts +0 -45
- package/dist/core/wal.d.ts.map +0 -1
- package/dist/core/wal.js +0 -80
- package/dist/core/wal.js.map +0 -1
- package/dist/index/shard-manager.d.ts +0 -36
- package/dist/index/shard-manager.d.ts.map +0 -1
- package/dist/index/shard-manager.js +0 -177
- package/dist/index/shard-manager.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/query/evaluator.d.ts +0 -10
- package/dist/query/evaluator.d.ts.map +0 -1
- package/dist/query/evaluator.js +0 -127
- package/dist/query/evaluator.js.map +0 -1
- package/dist/storage/fs-adapter.d.ts +0 -24
- package/dist/storage/fs-adapter.d.ts.map +0 -1
- package/dist/storage/fs-adapter.js +0 -208
- package/dist/storage/fs-adapter.js.map +0 -1
- package/dist/storage/memory-adapter.d.ts +0 -22
- package/dist/storage/memory-adapter.d.ts.map +0 -1
- package/dist/storage/memory-adapter.js +0 -78
- package/dist/storage/memory-adapter.js.map +0 -1
- package/dist/storage/oss-adapter.d.ts +0 -26
- package/dist/storage/oss-adapter.d.ts.map +0 -1
- package/dist/storage/oss-adapter.js +0 -145
- package/dist/storage/oss-adapter.js.map +0 -1
- package/dist/storage/s3-adapter.d.ts +0 -26
- package/dist/storage/s3-adapter.d.ts.map +0 -1
- package/dist/storage/s3-adapter.js +0 -170
- package/dist/storage/s3-adapter.js.map +0 -1
- package/dist/types/collection.d.ts +0 -40
- package/dist/types/collection.d.ts.map +0 -1
- package/dist/types/collection.js +0 -2
- package/dist/types/collection.js.map +0 -1
- package/dist/types/document.d.ts +0 -18
- package/dist/types/document.d.ts.map +0 -1
- package/dist/types/document.js +0 -2
- package/dist/types/document.js.map +0 -1
- package/dist/types/errors.d.ts +0 -32
- package/dist/types/errors.d.ts.map +0 -1
- package/dist/types/errors.js +0 -48
- package/dist/types/errors.js.map +0 -1
- package/dist/types/index.d.ts +0 -6
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -3
- package/dist/types/index.js.map +0 -1
- package/dist/types/query.d.ts +0 -37
- package/dist/types/query.d.ts.map +0 -1
- package/dist/types/query.js +0 -2
- package/dist/types/query.js.map +0 -1
- package/dist/types/storage.d.ts +0 -45
- package/dist/types/storage.d.ts.map +0 -1
- package/dist/types/storage.js +0 -2
- package/dist/types/storage.js.map +0 -1
- package/dist/utils/hash.d.ts +0 -13
- package/dist/utils/hash.d.ts.map +0 -1
- package/dist/utils/hash.js +0 -23
- package/dist/utils/hash.js.map +0 -1
- package/dist/utils/path-validator.d.ts +0 -46
- package/dist/utils/path-validator.d.ts.map +0 -1
- package/dist/utils/path-validator.js +0 -99
- package/dist/utils/path-validator.js.map +0 -1
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { mkdir, open, link, rename, unlink, readdir, writeFile, stat } from 'node:fs/promises';
|
|
2
|
+
import { join, dirname, resolve } from 'node:path';
|
|
3
|
+
import { randomUUID } from 'node:crypto';
|
|
4
|
+
import { MAX_LIST_LIMIT } from './adapter.js';
|
|
5
|
+
import { StorageError } from '../errors.js';
|
|
6
|
+
/** Temporary write files and 0.3.x `.meta` sidecars are not objects. */
|
|
7
|
+
const isInternalFile = (name) => name.endsWith('.tmp') || name.endsWith('.meta');
|
|
8
|
+
function errorCode(error) {
|
|
9
|
+
return error instanceof Error && 'code' in error ? String(error.code) : undefined;
|
|
10
|
+
}
|
|
11
|
+
/** Every write replaces the file with a new inode, so the etag changes on every write. */
|
|
12
|
+
function etagOf(stats) {
|
|
13
|
+
return `${stats.ino.toString(36)}-${stats.mtimeNs.toString(36)}-${stats.size.toString(36)}`;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Stores each object as a file under `basePath`. Create-if-absent uses
|
|
17
|
+
* `link()`, which is atomic across processes; compare-and-swap is serialized
|
|
18
|
+
* only within this process.
|
|
19
|
+
*/
|
|
20
|
+
export class FileSystemAdapter {
|
|
21
|
+
capabilities = { createIfAbsent: 'atomic', compareAndSwap: 'single-process' };
|
|
22
|
+
basePath;
|
|
23
|
+
locks = new Map();
|
|
24
|
+
constructor(config) {
|
|
25
|
+
this.basePath = resolve(config.basePath);
|
|
26
|
+
}
|
|
27
|
+
pathOf(key) {
|
|
28
|
+
const segments = key.split('/');
|
|
29
|
+
if (segments.some(segment => segment === '' || segment === '.' || segment === '..' || segment.includes('\\') || isInternalFile(segment))) {
|
|
30
|
+
throw new StorageError(`Invalid key: ${key}`, { key });
|
|
31
|
+
}
|
|
32
|
+
return join(this.basePath, ...segments);
|
|
33
|
+
}
|
|
34
|
+
/** Runs `fn` after every earlier writer of `path` in this process has finished. */
|
|
35
|
+
async withLock(path, fn) {
|
|
36
|
+
const run = (this.locks.get(path) ?? Promise.resolve()).then(fn);
|
|
37
|
+
const tail = run.then(() => undefined, () => undefined);
|
|
38
|
+
this.locks.set(path, tail);
|
|
39
|
+
try {
|
|
40
|
+
return await run;
|
|
41
|
+
}
|
|
42
|
+
finally {
|
|
43
|
+
if (this.locks.get(path) === tail)
|
|
44
|
+
this.locks.delete(path);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
async get(key) {
|
|
48
|
+
const path = this.pathOf(key);
|
|
49
|
+
let handle;
|
|
50
|
+
try {
|
|
51
|
+
handle = await open(path, 'r');
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
const code = errorCode(error);
|
|
55
|
+
if (code === 'ENOENT' || code === 'ENOTDIR') {
|
|
56
|
+
throw new StorageError(`Object not found: ${key}`, { kind: 'not-found', key });
|
|
57
|
+
}
|
|
58
|
+
throw new StorageError(`Failed to read ${key}`, { key, cause: error });
|
|
59
|
+
}
|
|
60
|
+
try {
|
|
61
|
+
// stat and read through one descriptor so the etag matches the bytes
|
|
62
|
+
const stats = await handle.stat({ bigint: true });
|
|
63
|
+
if (!stats.isFile()) {
|
|
64
|
+
throw new StorageError(`Object not found: ${key}`, { kind: 'not-found', key });
|
|
65
|
+
}
|
|
66
|
+
return { body: new Uint8Array(await handle.readFile()), etag: etagOf(stats) };
|
|
67
|
+
}
|
|
68
|
+
finally {
|
|
69
|
+
await handle.close();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
async put(key, body, condition) {
|
|
73
|
+
const path = this.pathOf(key);
|
|
74
|
+
return this.withLock(path, async () => {
|
|
75
|
+
await mkdir(dirname(path), { recursive: true });
|
|
76
|
+
const temp = `${path}.${randomUUID()}.tmp`;
|
|
77
|
+
await writeFile(temp, body, { flag: 'wx' });
|
|
78
|
+
try {
|
|
79
|
+
// rename and link keep the inode and mtime, so this is the etag of the stored object
|
|
80
|
+
const etag = etagOf(await stat(temp, { bigint: true }));
|
|
81
|
+
if (condition && 'ifNoneMatch' in condition) {
|
|
82
|
+
try {
|
|
83
|
+
await link(temp, path);
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
if (errorCode(error) === 'EEXIST') {
|
|
87
|
+
throw new StorageError(`Object already exists: ${key}`, { kind: 'conflict', key });
|
|
88
|
+
}
|
|
89
|
+
throw error;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
if (condition && 'ifMatch' in condition) {
|
|
94
|
+
const current = await stat(path, { bigint: true }).catch((error) => {
|
|
95
|
+
if (errorCode(error) === 'ENOENT') {
|
|
96
|
+
throw new StorageError(`Object not found: ${key}`, { kind: 'not-found', key });
|
|
97
|
+
}
|
|
98
|
+
throw error;
|
|
99
|
+
});
|
|
100
|
+
if (etagOf(current) !== condition.ifMatch) {
|
|
101
|
+
throw new StorageError(`ETag mismatch: ${key}`, { kind: 'conflict', key });
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
await rename(temp, path);
|
|
105
|
+
}
|
|
106
|
+
return { etag };
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
if (error instanceof StorageError)
|
|
110
|
+
throw error;
|
|
111
|
+
throw new StorageError(`Failed to write ${key}`, { key, cause: error });
|
|
112
|
+
}
|
|
113
|
+
finally {
|
|
114
|
+
await unlink(temp).catch(() => undefined);
|
|
115
|
+
}
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
async delete(key) {
|
|
119
|
+
const path = this.pathOf(key);
|
|
120
|
+
await this.withLock(path, async () => {
|
|
121
|
+
try {
|
|
122
|
+
await unlink(path);
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
if (errorCode(error) !== 'ENOENT') {
|
|
126
|
+
throw new StorageError(`Failed to delete ${key}`, { key, cause: error });
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
async list(prefix, options = {}) {
|
|
132
|
+
const limit = Math.min(options.limit ?? MAX_LIST_LIMIT, MAX_LIST_LIMIT);
|
|
133
|
+
const slash = prefix.lastIndexOf('/');
|
|
134
|
+
const directory = prefix.slice(0, slash + 1);
|
|
135
|
+
const keys = [];
|
|
136
|
+
const walk = async (keyPrefix) => {
|
|
137
|
+
let entries;
|
|
138
|
+
try {
|
|
139
|
+
entries = await readdir(join(this.basePath, ...keyPrefix.split('/').filter(Boolean)), { withFileTypes: true });
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
const code = errorCode(error);
|
|
143
|
+
if (code === 'ENOENT' || code === 'ENOTDIR')
|
|
144
|
+
return;
|
|
145
|
+
throw new StorageError(`Failed to list ${prefix}`, { cause: error });
|
|
146
|
+
}
|
|
147
|
+
for (const entry of entries) {
|
|
148
|
+
const key = keyPrefix + entry.name;
|
|
149
|
+
if (entry.isDirectory()) {
|
|
150
|
+
if (prefix.startsWith(`${key}/`) || `${key}/`.startsWith(prefix))
|
|
151
|
+
await walk(`${key}/`);
|
|
152
|
+
}
|
|
153
|
+
else if (entry.isFile() && !isInternalFile(entry.name) && key.startsWith(prefix)) {
|
|
154
|
+
if (options.cursor === undefined || key > options.cursor)
|
|
155
|
+
keys.push(key);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
await walk(directory);
|
|
160
|
+
keys.sort();
|
|
161
|
+
const page = keys.slice(0, limit);
|
|
162
|
+
const entries = [];
|
|
163
|
+
for (const key of page) {
|
|
164
|
+
try {
|
|
165
|
+
entries.push({ key, etag: etagOf(await stat(this.pathOf(key), { bigint: true })) });
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
// deleted after the directory walk
|
|
169
|
+
if (errorCode(error) !== 'ENOENT')
|
|
170
|
+
throw new StorageError(`Failed to list ${prefix}`, { key, cause: error });
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return { entries, cursor: keys.length > limit ? page[page.length - 1] : undefined };
|
|
174
|
+
}
|
|
175
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { StorageAdapter, Capabilities, WriteCondition, ListOptions, ListPage } from './adapter.js';
|
|
2
|
+
/**
|
|
3
|
+
* In-process storage for tests and local development. Nothing is persisted.
|
|
4
|
+
*/
|
|
5
|
+
export declare class MemoryStorageAdapter implements StorageAdapter {
|
|
6
|
+
readonly capabilities: Capabilities;
|
|
7
|
+
private readonly objects;
|
|
8
|
+
private version;
|
|
9
|
+
get(key: string): Promise<{
|
|
10
|
+
body: Uint8Array;
|
|
11
|
+
etag: string;
|
|
12
|
+
}>;
|
|
13
|
+
put(key: string, body: Uint8Array, condition?: WriteCondition): Promise<{
|
|
14
|
+
etag: string;
|
|
15
|
+
}>;
|
|
16
|
+
delete(key: string): Promise<void>;
|
|
17
|
+
list(prefix: string, options?: ListOptions): Promise<ListPage>;
|
|
18
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { MAX_LIST_LIMIT } from './adapter.js';
|
|
2
|
+
import { StorageError } from '../errors.js';
|
|
3
|
+
/**
|
|
4
|
+
* In-process storage for tests and local development. Nothing is persisted.
|
|
5
|
+
*/
|
|
6
|
+
export class MemoryStorageAdapter {
|
|
7
|
+
capabilities = { createIfAbsent: 'atomic', compareAndSwap: 'atomic' };
|
|
8
|
+
objects = new Map();
|
|
9
|
+
version = 0;
|
|
10
|
+
async get(key) {
|
|
11
|
+
const object = this.objects.get(key);
|
|
12
|
+
if (!object) {
|
|
13
|
+
throw new StorageError(`Object not found: ${key}`, { kind: 'not-found', key });
|
|
14
|
+
}
|
|
15
|
+
return { body: object.body.slice(), etag: object.etag };
|
|
16
|
+
}
|
|
17
|
+
async put(key, body, condition) {
|
|
18
|
+
const existing = this.objects.get(key);
|
|
19
|
+
if (condition && 'ifNoneMatch' in condition && existing) {
|
|
20
|
+
throw new StorageError(`Object already exists: ${key}`, { kind: 'conflict', key });
|
|
21
|
+
}
|
|
22
|
+
if (condition && 'ifMatch' in condition) {
|
|
23
|
+
if (!existing) {
|
|
24
|
+
throw new StorageError(`Object not found: ${key}`, { kind: 'not-found', key });
|
|
25
|
+
}
|
|
26
|
+
if (existing.etag !== condition.ifMatch) {
|
|
27
|
+
throw new StorageError(`ETag mismatch: ${key}`, { kind: 'conflict', key });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
const etag = `m${++this.version}`;
|
|
31
|
+
this.objects.set(key, { body: body.slice(), etag });
|
|
32
|
+
return { etag };
|
|
33
|
+
}
|
|
34
|
+
async delete(key) {
|
|
35
|
+
this.objects.delete(key);
|
|
36
|
+
}
|
|
37
|
+
async list(prefix, options = {}) {
|
|
38
|
+
const limit = Math.min(options.limit ?? MAX_LIST_LIMIT, MAX_LIST_LIMIT);
|
|
39
|
+
const entries = [...this.objects]
|
|
40
|
+
.filter(([key]) => key.startsWith(prefix) && (options.cursor === undefined || key > options.cursor))
|
|
41
|
+
.sort(([a], [b]) => (a < b ? -1 : 1))
|
|
42
|
+
.map(([key, { etag }]) => ({ key, etag }));
|
|
43
|
+
const page = entries.slice(0, limit);
|
|
44
|
+
return {
|
|
45
|
+
entries: page,
|
|
46
|
+
cursor: entries.length > limit ? page[page.length - 1].key : undefined,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { S3Client } from './client.js';
|
|
2
|
+
import { type CredentialProvider, type Credentials } from './credentials.js';
|
|
3
|
+
export type { CredentialProvider, Credentials };
|
|
4
|
+
export interface S3AdapterOptions {
|
|
5
|
+
bucket: string;
|
|
6
|
+
/** Default: `AWS_REGION`, then `AWS_DEFAULT_REGION`. */
|
|
7
|
+
region?: string;
|
|
8
|
+
/** Default: `AWS_ENDPOINT_URL_S3`, then `AWS_ENDPOINT_URL`, then `https://s3.<region>.amazonaws.com`. */
|
|
9
|
+
endpoint?: string;
|
|
10
|
+
/** Default: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`. */
|
|
11
|
+
credentials?: Credentials | CredentialProvider;
|
|
12
|
+
/** Default: true when the endpoint host is an IP address or `localhost`, else virtual-hosted. */
|
|
13
|
+
forcePathStyle?: boolean;
|
|
14
|
+
/** Per-attempt timeout; default 10000. */
|
|
15
|
+
timeoutMs?: number;
|
|
16
|
+
}
|
|
17
|
+
export interface OSSAdapterOptions {
|
|
18
|
+
bucket: string;
|
|
19
|
+
/** `cn-hangzhou` (or 0.3's `oss-cn-hangzhou`). Default: `OSS_REGION`. */
|
|
20
|
+
region?: string;
|
|
21
|
+
/** S3-compatible endpoint. Default: `https://s3.oss-<region>.aliyuncs.com`. */
|
|
22
|
+
endpoint?: string;
|
|
23
|
+
/** Default: `OSS_ACCESS_KEY_ID`, `OSS_ACCESS_KEY_SECRET`, `OSS_SESSION_TOKEN`, else the `ALIBABA_CLOUD_*` equivalents. */
|
|
24
|
+
credentials?: Credentials | CredentialProvider;
|
|
25
|
+
/** Per-attempt timeout; default 10000. */
|
|
26
|
+
timeoutMs?: number;
|
|
27
|
+
}
|
|
28
|
+
/** Amazon S3, or any service speaking its protocol (MinIO, R2, …). */
|
|
29
|
+
export declare class S3Adapter extends S3Client {
|
|
30
|
+
constructor(options: S3AdapterOptions);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Alibaba Cloud OSS through its S3-compatible endpoint. Compare-and-swap
|
|
34
|
+
* (`put` with `ifMatch`) is not supported.
|
|
35
|
+
*/
|
|
36
|
+
export declare class OSSAdapter extends S3Client {
|
|
37
|
+
constructor(options: OSSAdapterOptions);
|
|
38
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { ValidationError } from '../../errors.js';
|
|
2
|
+
import { AWS_DIALECT, OSS_DIALECT, S3Client } from './client.js';
|
|
3
|
+
import { resolveCredentials } from './credentials.js';
|
|
4
|
+
function resolveOptions(name, options) {
|
|
5
|
+
const { bucket, region, timeoutMs = 10_000 } = options;
|
|
6
|
+
if (!bucket)
|
|
7
|
+
throw new ValidationError(`${name}: bucket is required`);
|
|
8
|
+
if (!region)
|
|
9
|
+
throw new ValidationError(`${name}: region is required (option or environment variable)`);
|
|
10
|
+
if (!(timeoutMs > 0))
|
|
11
|
+
throw new ValidationError(`${name}: timeoutMs must be positive`);
|
|
12
|
+
let endpoint;
|
|
13
|
+
try {
|
|
14
|
+
endpoint = new URL(options.endpoint);
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
throw new ValidationError(`${name}: invalid endpoint ${options.endpoint}`, { cause: error });
|
|
18
|
+
}
|
|
19
|
+
const { hostname } = endpoint;
|
|
20
|
+
const local = hostname === 'localhost' || hostname.startsWith('[') || /^\d+\.\d+\.\d+\.\d+$/.test(hostname);
|
|
21
|
+
return {
|
|
22
|
+
bucket,
|
|
23
|
+
region,
|
|
24
|
+
endpoint,
|
|
25
|
+
forcePathStyle: options.forcePathStyle ?? local,
|
|
26
|
+
credentials: options.credentials,
|
|
27
|
+
timeoutMs,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
/** Amazon S3, or any service speaking its protocol (MinIO, R2, …). */
|
|
31
|
+
export class S3Adapter extends S3Client {
|
|
32
|
+
constructor(options) {
|
|
33
|
+
const { env } = process;
|
|
34
|
+
const region = options.region ?? (env.AWS_REGION || env.AWS_DEFAULT_REGION);
|
|
35
|
+
const resolved = resolveOptions('S3Adapter', {
|
|
36
|
+
...options,
|
|
37
|
+
region,
|
|
38
|
+
endpoint: options.endpoint ?? (env.AWS_ENDPOINT_URL_S3 || env.AWS_ENDPOINT_URL || `https://s3.${region}.amazonaws.com`),
|
|
39
|
+
credentials: resolveCredentials(options.credentials, [
|
|
40
|
+
['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'],
|
|
41
|
+
]),
|
|
42
|
+
});
|
|
43
|
+
super(resolved, AWS_DIALECT);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Alibaba Cloud OSS through its S3-compatible endpoint. Compare-and-swap
|
|
48
|
+
* (`put` with `ifMatch`) is not supported.
|
|
49
|
+
*/
|
|
50
|
+
export class OSSAdapter extends S3Client {
|
|
51
|
+
constructor(options) {
|
|
52
|
+
const region = (options.region ?? process.env.OSS_REGION)?.replace(/^oss-/, '');
|
|
53
|
+
const resolved = resolveOptions('OSSAdapter', {
|
|
54
|
+
...options,
|
|
55
|
+
region,
|
|
56
|
+
endpoint: options.endpoint ?? `https://s3.oss-${region}.aliyuncs.com`,
|
|
57
|
+
// ossutil's names first, then the Alibaba Cloud CLI / OIDC action's
|
|
58
|
+
credentials: resolveCredentials(options.credentials, [
|
|
59
|
+
['OSS_ACCESS_KEY_ID', 'OSS_ACCESS_KEY_SECRET', 'OSS_SESSION_TOKEN'],
|
|
60
|
+
['ALIBABA_CLOUD_ACCESS_KEY_ID', 'ALIBABA_CLOUD_ACCESS_KEY_SECRET', 'ALIBABA_CLOUD_SECURITY_TOKEN'],
|
|
61
|
+
]),
|
|
62
|
+
});
|
|
63
|
+
super(resolved, OSS_DIALECT);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { type Capabilities, type ListOptions, type ListPage, type StorageAdapter, type WriteCondition } from '../adapter.js';
|
|
2
|
+
import type { CredentialProvider } from './credentials.js';
|
|
3
|
+
/** Where an S3-compatible service departs from AWS S3. */
|
|
4
|
+
export interface Dialect {
|
|
5
|
+
/** Header that makes a PUT fail when the key exists. */
|
|
6
|
+
createIfAbsent: [header: string, value: string];
|
|
7
|
+
supportsIfMatch: boolean;
|
|
8
|
+
/** Whether a failed conditional PUT means "condition not met". */
|
|
9
|
+
isConflict(status: number, code: string): boolean;
|
|
10
|
+
}
|
|
11
|
+
export declare const AWS_DIALECT: Dialect;
|
|
12
|
+
export declare const OSS_DIALECT: Dialect;
|
|
13
|
+
export interface ResolvedOptions {
|
|
14
|
+
bucket: string;
|
|
15
|
+
/** Signing region */
|
|
16
|
+
region: string;
|
|
17
|
+
endpoint: URL;
|
|
18
|
+
forcePathStyle: boolean;
|
|
19
|
+
credentials: CredentialProvider;
|
|
20
|
+
/** Per attempt */
|
|
21
|
+
timeoutMs: number;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* StorageAdapter over the S3 REST protocol: SigV4 signing, per-attempt
|
|
25
|
+
* timeout, retries with full-jitter backoff, and error classification.
|
|
26
|
+
*/
|
|
27
|
+
export declare class S3Client implements StorageAdapter {
|
|
28
|
+
private readonly options;
|
|
29
|
+
private readonly dialect;
|
|
30
|
+
readonly capabilities: Capabilities;
|
|
31
|
+
readonly remote = true;
|
|
32
|
+
constructor(options: ResolvedOptions, dialect: Dialect);
|
|
33
|
+
get(key: string): Promise<{
|
|
34
|
+
body: Uint8Array;
|
|
35
|
+
etag: string;
|
|
36
|
+
}>;
|
|
37
|
+
put(key: string, body: Uint8Array, condition?: WriteCondition): Promise<{
|
|
38
|
+
etag: string;
|
|
39
|
+
}>;
|
|
40
|
+
delete(key: string): Promise<void>;
|
|
41
|
+
list(prefix: string, options?: ListOptions): Promise<ListPage>;
|
|
42
|
+
private etag;
|
|
43
|
+
private failure;
|
|
44
|
+
private url;
|
|
45
|
+
private credentials;
|
|
46
|
+
/**
|
|
47
|
+
* Sends with retries. Returns the first response that is not retryable;
|
|
48
|
+
* `retried` tells whether an earlier attempt failed transiently.
|
|
49
|
+
*/
|
|
50
|
+
private send;
|
|
51
|
+
private attempt;
|
|
52
|
+
}
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { BucketDBError, StorageError, UnsupportedError, ValidationError } from '../../errors.js';
|
|
2
|
+
import { MAX_LIST_LIMIT, } from '../adapter.js';
|
|
3
|
+
import { EMPTY_SHA256, encodeRfc3986, sha256Hex, signRequest } from './sigv4.js';
|
|
4
|
+
export const AWS_DIALECT = {
|
|
5
|
+
createIfAbsent: ['if-none-match', '*'],
|
|
6
|
+
supportsIfMatch: true,
|
|
7
|
+
isConflict: (status, code) => status === 412 || (status === 409 && code === 'ConditionalRequestConflict'),
|
|
8
|
+
};
|
|
9
|
+
export const OSS_DIALECT = {
|
|
10
|
+
createIfAbsent: ['x-oss-forbid-overwrite', 'true'],
|
|
11
|
+
supportsIfMatch: false,
|
|
12
|
+
isConflict: (status, code) => status === 412 || (status === 409 && code === 'FileAlreadyExists'),
|
|
13
|
+
};
|
|
14
|
+
const MAX_ATTEMPTS = 3;
|
|
15
|
+
const BACKOFF_BASE_MS = 100;
|
|
16
|
+
const BACKOFF_CAP_MS = 2000;
|
|
17
|
+
const RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 504]);
|
|
18
|
+
const RETRYABLE_CODES = new Set(['SlowDown', 'RequestTimeout']);
|
|
19
|
+
const XML_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
|
|
20
|
+
const decoder = new TextDecoder();
|
|
21
|
+
function decodeXml(text) {
|
|
22
|
+
return text.replace(/&(#x[0-9a-f]+|#[0-9]+|amp|lt|gt|quot|apos);/gi, (_, entity) => {
|
|
23
|
+
if (entity[0] !== '#')
|
|
24
|
+
return XML_ENTITIES[entity.toLowerCase()];
|
|
25
|
+
const hex = entity[1] === 'x' || entity[1] === 'X';
|
|
26
|
+
return String.fromCodePoint(Number.parseInt(entity.slice(hex ? 2 : 1), hex ? 16 : 10));
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
/** Entity-decoded text of the first `<tag>`, or ''. */
|
|
30
|
+
function xmlValue(xml, tag) {
|
|
31
|
+
const match = new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`).exec(xml);
|
|
32
|
+
return match ? decodeXml(match[1]) : '';
|
|
33
|
+
}
|
|
34
|
+
function bytesEqual(a, b) {
|
|
35
|
+
if (a.length !== b.length)
|
|
36
|
+
return false;
|
|
37
|
+
for (let i = 0; i < a.length; i++)
|
|
38
|
+
if (a[i] !== b[i])
|
|
39
|
+
return false;
|
|
40
|
+
return true;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* StorageAdapter over the S3 REST protocol: SigV4 signing, per-attempt
|
|
44
|
+
* timeout, retries with full-jitter backoff, and error classification.
|
|
45
|
+
*/
|
|
46
|
+
export class S3Client {
|
|
47
|
+
options;
|
|
48
|
+
dialect;
|
|
49
|
+
capabilities;
|
|
50
|
+
remote = true;
|
|
51
|
+
constructor(options, dialect) {
|
|
52
|
+
this.options = options;
|
|
53
|
+
this.dialect = dialect;
|
|
54
|
+
this.capabilities = { createIfAbsent: 'atomic', compareAndSwap: dialect.supportsIfMatch ? 'atomic' : 'none' };
|
|
55
|
+
}
|
|
56
|
+
async get(key) {
|
|
57
|
+
const { response } = await this.send('GET', key);
|
|
58
|
+
if (response.status < 300)
|
|
59
|
+
return { body: response.body, etag: this.etag(response) };
|
|
60
|
+
throw this.failure('GET', key, response);
|
|
61
|
+
}
|
|
62
|
+
async put(key, body, condition) {
|
|
63
|
+
const headers = {};
|
|
64
|
+
if (condition && 'ifMatch' in condition) {
|
|
65
|
+
if (!this.dialect.supportsIfMatch) {
|
|
66
|
+
throw new UnsupportedError(`This backend cannot compare-and-swap (put with ifMatch): ${key}`);
|
|
67
|
+
}
|
|
68
|
+
headers['if-match'] = `"${condition.ifMatch}"`;
|
|
69
|
+
}
|
|
70
|
+
else if (condition) {
|
|
71
|
+
const [name, value] = this.dialect.createIfAbsent;
|
|
72
|
+
headers[name] = value;
|
|
73
|
+
}
|
|
74
|
+
const { response, retried } = await this.send('PUT', key, { headers, body });
|
|
75
|
+
if (response.status < 300)
|
|
76
|
+
return { etag: this.etag(response) };
|
|
77
|
+
if (!condition || !this.dialect.isConflict(response.status, response.code)) {
|
|
78
|
+
throw this.failure('PUT', key, response);
|
|
79
|
+
}
|
|
80
|
+
if (retried) {
|
|
81
|
+
// An earlier attempt may have written the object before its response was
|
|
82
|
+
// lost; then the conflict is with ourselves.
|
|
83
|
+
const current = await this.get(key).catch((error) => {
|
|
84
|
+
if (error instanceof StorageError && error.kind === 'not-found')
|
|
85
|
+
return undefined;
|
|
86
|
+
throw error;
|
|
87
|
+
});
|
|
88
|
+
if (current && bytesEqual(current.body, body))
|
|
89
|
+
return { etag: current.etag };
|
|
90
|
+
}
|
|
91
|
+
throw new StorageError(`PUT ${key}: write condition not met (${response.status} ${response.code})`, {
|
|
92
|
+
kind: 'conflict',
|
|
93
|
+
key,
|
|
94
|
+
status: response.status,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
async delete(key) {
|
|
98
|
+
const { response } = await this.send('DELETE', key);
|
|
99
|
+
if (response.status >= 300 && response.status !== 404)
|
|
100
|
+
throw this.failure('DELETE', key, response);
|
|
101
|
+
}
|
|
102
|
+
async list(prefix, options = {}) {
|
|
103
|
+
const limit = options.limit ?? MAX_LIST_LIMIT;
|
|
104
|
+
if (!Number.isInteger(limit) || limit < 1)
|
|
105
|
+
throw new ValidationError(`Invalid list limit: ${limit}`);
|
|
106
|
+
const query = {
|
|
107
|
+
'list-type': '2',
|
|
108
|
+
prefix,
|
|
109
|
+
'max-keys': String(Math.min(limit, MAX_LIST_LIMIT)),
|
|
110
|
+
'encoding-type': 'url',
|
|
111
|
+
};
|
|
112
|
+
if (options.cursor !== undefined)
|
|
113
|
+
query['continuation-token'] = options.cursor;
|
|
114
|
+
const { response } = await this.send('GET', prefix, { query });
|
|
115
|
+
if (response.status >= 300)
|
|
116
|
+
throw this.failure('LIST', prefix, response, false);
|
|
117
|
+
const xml = decoder.decode(response.body);
|
|
118
|
+
const entries = Array.from(xml.matchAll(/<Contents>([\s\S]*?)<\/Contents>/g), ([, contents]) => ({
|
|
119
|
+
// encoding-type=url encodes keys like a form: `+` is a space.
|
|
120
|
+
key: decodeURIComponent(xmlValue(contents, 'Key').replace(/\+/g, ' ')),
|
|
121
|
+
etag: xmlValue(contents, 'ETag').replace(/"/g, ''),
|
|
122
|
+
}));
|
|
123
|
+
const cursor = xmlValue(xml, 'IsTruncated') === 'true' ? xmlValue(xml, 'NextContinuationToken') : '';
|
|
124
|
+
return cursor ? { entries, cursor } : { entries };
|
|
125
|
+
}
|
|
126
|
+
etag(response) {
|
|
127
|
+
return (response.headers.get('etag') ?? '').replace(/"/g, '');
|
|
128
|
+
}
|
|
129
|
+
failure(operation, key, response, notFound = true) {
|
|
130
|
+
const { status, code, message } = response;
|
|
131
|
+
return new StorageError(`${operation} ${key}: ${status} ${code}${message ? `: ${message}` : ''}`, {
|
|
132
|
+
kind: notFound && status === 404 && code !== 'NoSuchBucket' ? 'not-found' : 'permanent',
|
|
133
|
+
key,
|
|
134
|
+
status,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
url(key, query) {
|
|
138
|
+
const { endpoint, bucket, forcePathStyle } = this.options;
|
|
139
|
+
const segments = query ? [''] : key.split('/');
|
|
140
|
+
if (segments.some(segment => segment === '.' || segment === '..')) {
|
|
141
|
+
// URL parsing would resolve them and address a different key.
|
|
142
|
+
throw new ValidationError(`Key has a '.' or '..' segment: ${key}`);
|
|
143
|
+
}
|
|
144
|
+
const url = new URL(endpoint);
|
|
145
|
+
if (!forcePathStyle)
|
|
146
|
+
url.hostname = `${bucket}.${url.hostname}`;
|
|
147
|
+
url.pathname =
|
|
148
|
+
url.pathname.replace(/\/$/, '') +
|
|
149
|
+
(forcePathStyle ? `/${encodeRfc3986(bucket)}` : '') +
|
|
150
|
+
`/${segments.map(encodeRfc3986).join('/')}`;
|
|
151
|
+
url.search = query
|
|
152
|
+
? Object.entries(query)
|
|
153
|
+
.map(([name, value]) => `${encodeRfc3986(name)}=${encodeRfc3986(value)}`)
|
|
154
|
+
.join('&')
|
|
155
|
+
: '';
|
|
156
|
+
return url;
|
|
157
|
+
}
|
|
158
|
+
async credentials(key) {
|
|
159
|
+
try {
|
|
160
|
+
return await this.options.credentials();
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
if (error instanceof BucketDBError)
|
|
164
|
+
throw error;
|
|
165
|
+
throw new StorageError(`Failed to obtain credentials for ${key}`, { kind: 'permanent', key, cause: error });
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Sends with retries. Returns the first response that is not retryable;
|
|
170
|
+
* `retried` tells whether an earlier attempt failed transiently.
|
|
171
|
+
*/
|
|
172
|
+
async send(method, key, init = {}) {
|
|
173
|
+
const url = this.url(key, init.query);
|
|
174
|
+
const payloadHash = init.body ? await sha256Hex(init.body) : EMPTY_SHA256;
|
|
175
|
+
let lastStatus;
|
|
176
|
+
let lastError;
|
|
177
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
178
|
+
if (attempt > 0) {
|
|
179
|
+
const ceiling = Math.min(BACKOFF_CAP_MS, BACKOFF_BASE_MS * 2 ** (attempt - 1));
|
|
180
|
+
await new Promise(resolve => setTimeout(resolve, Math.random() * ceiling));
|
|
181
|
+
}
|
|
182
|
+
const credentials = await this.credentials(key);
|
|
183
|
+
try {
|
|
184
|
+
const response = await this.attempt(method, url, init, payloadHash, credentials);
|
|
185
|
+
if (!RETRYABLE_STATUSES.has(response.status) && !RETRYABLE_CODES.has(response.code)) {
|
|
186
|
+
return { response, retried: attempt > 0 };
|
|
187
|
+
}
|
|
188
|
+
lastStatus = response.status;
|
|
189
|
+
lastError = new Error(`${response.status} ${response.code}: ${response.message}`);
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
// Network failure or per-attempt timeout.
|
|
193
|
+
lastError = error;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
throw new StorageError(`${method} ${key}: failed after ${MAX_ATTEMPTS} attempts`, {
|
|
197
|
+
kind: 'transient',
|
|
198
|
+
key,
|
|
199
|
+
status: lastStatus,
|
|
200
|
+
cause: lastError,
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
async attempt(method, url, init, payloadHash, credentials) {
|
|
204
|
+
const headers = init.headers ?? {};
|
|
205
|
+
const signed = await signRequest({
|
|
206
|
+
method,
|
|
207
|
+
url,
|
|
208
|
+
headers,
|
|
209
|
+
payloadHash,
|
|
210
|
+
credentials,
|
|
211
|
+
region: this.options.region,
|
|
212
|
+
date: new Date(),
|
|
213
|
+
});
|
|
214
|
+
const response = await fetch(url, {
|
|
215
|
+
method,
|
|
216
|
+
headers: { ...headers, ...signed },
|
|
217
|
+
body: init.body,
|
|
218
|
+
signal: AbortSignal.timeout(this.options.timeoutMs),
|
|
219
|
+
});
|
|
220
|
+
// Read inside the attempt so the timeout and connection errors cover the body.
|
|
221
|
+
const body = new Uint8Array(await response.arrayBuffer());
|
|
222
|
+
const xml = response.status >= 300 ? decoder.decode(body) : '';
|
|
223
|
+
return {
|
|
224
|
+
status: response.status,
|
|
225
|
+
code: xmlValue(xml, 'Code'),
|
|
226
|
+
message: xmlValue(xml, 'Message'),
|
|
227
|
+
headers: response.headers,
|
|
228
|
+
body,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface Credentials {
|
|
2
|
+
accessKeyId: string;
|
|
3
|
+
secretAccessKey: string;
|
|
4
|
+
sessionToken?: string;
|
|
5
|
+
/** When a provider returns this, the provider is called again 5 minutes before it. */
|
|
6
|
+
expiresAt?: Date;
|
|
7
|
+
}
|
|
8
|
+
export type CredentialProvider = () => Promise<Credentials>;
|
|
9
|
+
/** Environment variable names for the access key id, secret and session token. */
|
|
10
|
+
export type CredentialEnv = [accessKeyId: string, secretAccessKey: string, sessionToken: string];
|
|
11
|
+
/**
|
|
12
|
+
* Turns the `credentials` option into a function returning current
|
|
13
|
+
* credentials. Without the option, credentials come from the first variable
|
|
14
|
+
* set in `envSets` that has both an id and a secret, read on first use;
|
|
15
|
+
* none set throws ValidationError then.
|
|
16
|
+
*/
|
|
17
|
+
export declare function resolveCredentials(option: Credentials | CredentialProvider | undefined, envSets: CredentialEnv[]): CredentialProvider;
|