@dxos/blob 0.11.1 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -4
- package/dist/lib/chunk-ni-uri.mjs +103 -0
- package/dist/lib/chunk-ni-uri.mjs.map +1 -0
- package/dist/lib/hosted.mjs +55 -0
- package/dist/lib/hosted.mjs.map +1 -0
- package/dist/lib/index.mjs +2 -0
- package/dist/lib/s3.mjs +515 -0
- package/dist/lib/s3.mjs.map +1 -0
- package/dist/types/src/backend.d.ts +31 -0
- package/dist/types/src/backend.d.ts.map +1 -1
- package/dist/types/src/hosted/blob-backend.d.ts +21 -0
- package/dist/types/src/hosted/blob-backend.d.ts.map +1 -0
- package/dist/types/src/hosted/blob-backend.test.d.ts +2 -0
- package/dist/types/src/hosted/blob-backend.test.d.ts.map +1 -0
- package/dist/types/src/hosted/index.d.ts +2 -0
- package/dist/types/src/hosted/index.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +2 -1
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/ni-uri.d.ts +51 -0
- package/dist/types/src/ni-uri.d.ts.map +1 -0
- package/dist/types/src/ni-uri.test.d.ts +2 -0
- package/dist/types/src/ni-uri.test.d.ts.map +1 -0
- package/dist/types/src/s3/blob-backend.d.ts +38 -0
- package/dist/types/src/s3/blob-backend.d.ts.map +1 -0
- package/dist/types/src/s3/constants.d.ts +17 -0
- package/dist/types/src/s3/constants.d.ts.map +1 -0
- package/dist/types/src/s3/index.d.ts +6 -0
- package/dist/types/src/s3/index.d.ts.map +1 -0
- package/dist/types/src/s3/s3-client.d.ts +93 -0
- package/dist/types/src/s3/s3-client.d.ts.map +1 -0
- package/dist/types/src/s3/s3-client.test.d.ts +2 -0
- package/dist/types/src/s3/s3-client.test.d.ts.map +1 -0
- package/dist/types/src/s3/s3-uri.d.ts +44 -0
- package/dist/types/src/s3/s3-uri.d.ts.map +1 -0
- package/dist/types/src/s3/s3-uri.test.d.ts +2 -0
- package/dist/types/src/s3/s3-uri.test.d.ts.map +1 -0
- package/dist/types/src/s3/sigv4.d.ts +55 -0
- package/dist/types/src/s3/sigv4.d.ts.map +1 -0
- package/dist/types/src/s3/sigv4.test.d.ts +2 -0
- package/dist/types/src/s3/sigv4.test.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +14 -2
- package/src/backend.ts +23 -0
- package/src/hosted/blob-backend.test.ts +113 -0
- package/src/hosted/blob-backend.ts +67 -0
- package/src/hosted/index.ts +5 -0
- package/src/index.ts +2 -1
- package/src/ni-uri.test.ts +31 -0
- package/src/ni-uri.ts +136 -0
- package/src/s3/blob-backend.ts +75 -0
- package/src/s3/constants.ts +24 -0
- package/src/s3/index.ts +13 -0
- package/src/s3/s3-client.test.ts +145 -0
- package/src/s3/s3-client.ts +331 -0
- package/src/s3/s3-uri.test.ts +81 -0
- package/src/s3/s3-uri.ts +93 -0
- package/src/s3/sigv4.test.ts +99 -0
- package/src/s3/sigv4.ts +203 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/blob",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Blob storage contract and registry: pluggable content-addressed backends for ECHO.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -17,6 +17,16 @@
|
|
|
17
17
|
"source": "./src/index.ts",
|
|
18
18
|
"types": "./dist/types/src/index.d.ts",
|
|
19
19
|
"import": "./dist/lib/index.mjs"
|
|
20
|
+
},
|
|
21
|
+
"./hosted": {
|
|
22
|
+
"source": "./src/hosted/index.ts",
|
|
23
|
+
"types": "./dist/types/src/hosted/index.d.ts",
|
|
24
|
+
"import": "./dist/lib/hosted.mjs"
|
|
25
|
+
},
|
|
26
|
+
"./s3": {
|
|
27
|
+
"source": "./src/s3/index.ts",
|
|
28
|
+
"types": "./dist/types/src/s3/index.d.ts",
|
|
29
|
+
"import": "./dist/lib/s3.mjs"
|
|
20
30
|
}
|
|
21
31
|
},
|
|
22
32
|
"types": "dist/types/src/index.d.ts",
|
|
@@ -25,7 +35,9 @@
|
|
|
25
35
|
"src"
|
|
26
36
|
],
|
|
27
37
|
"dependencies": {
|
|
28
|
-
"@dxos/
|
|
38
|
+
"@dxos/invariant": "0.14.0",
|
|
39
|
+
"@dxos/keys": "0.14.0",
|
|
40
|
+
"@dxos/util": "0.14.0"
|
|
29
41
|
},
|
|
30
42
|
"devDependencies": {
|
|
31
43
|
"vitest": "4.1.10"
|
package/src/backend.ts
CHANGED
|
@@ -38,6 +38,14 @@ export interface BlobTransport {
|
|
|
38
38
|
/** `undefined` means the key was not found. */
|
|
39
39
|
get(key: string): Promise<Uint8Array | undefined>;
|
|
40
40
|
has(key: string): Promise<boolean>;
|
|
41
|
+
/**
|
|
42
|
+
* Admits bytes already staged by a direct upload, returning the key they landed under.
|
|
43
|
+
*
|
|
44
|
+
* Optional because it is meaningful only for a hosted store: the bytes are moved server-side and
|
|
45
|
+
* never reach this process, which is the entire reason the path exists — the uploader was an
|
|
46
|
+
* agent's shell, and routing the file back through the client to hash it would undo the saving.
|
|
47
|
+
*/
|
|
48
|
+
finalizeUpload?(uploadId: string): Promise<{ key: string; size: number; contentType?: string }>;
|
|
41
49
|
}
|
|
42
50
|
|
|
43
51
|
/**
|
|
@@ -58,4 +66,19 @@ export interface BlobBackend {
|
|
|
58
66
|
get(request: { spaceId: SpaceId; uri: string }): Promise<Uint8Array | undefined>;
|
|
59
67
|
has(request: { spaceId: SpaceId; uri: string }): Promise<boolean>;
|
|
60
68
|
getUrl?(request: { spaceId: SpaceId; uri: string; contentType?: string }): Promise<string | undefined>;
|
|
69
|
+
/**
|
|
70
|
+
* Turns a completed direct upload into a stored blob, returning its URI and what the store
|
|
71
|
+
* actually received.
|
|
72
|
+
*
|
|
73
|
+
* The size and content type come back from the store rather than from the caller because the
|
|
74
|
+
* caller never saw the bytes: a direct upload is written by a third party (an agent's `curl`), so
|
|
75
|
+
* the only honest source for what landed is whatever received it.
|
|
76
|
+
*
|
|
77
|
+
* Implemented only by backends with a direct-upload path; `undefined` on the others, which is how
|
|
78
|
+
* the manager tells "this backend cannot adopt uploads" from "the upload failed".
|
|
79
|
+
*/
|
|
80
|
+
adoptUpload?(request: {
|
|
81
|
+
spaceId: SpaceId;
|
|
82
|
+
uploadId: string;
|
|
83
|
+
}): Promise<{ uri: string; size: number; contentType?: string }>;
|
|
61
84
|
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { describe, test, vi } from 'vitest';
|
|
6
|
+
|
|
7
|
+
import { SpaceId } from '@dxos/keys';
|
|
8
|
+
|
|
9
|
+
import { type BlobTransport } from '../backend.ts';
|
|
10
|
+
import { fromDigestHex } from '../ni-uri.ts';
|
|
11
|
+
import { createEdgeBlobBackend } from './blob-backend.ts';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* A transport whose unused operations reject. Before the backend took a `BlobTransport` these tests
|
|
15
|
+
* built partial `EdgeHttpClient`s behind `as unknown as`, which asserted a 33-method class from a
|
|
16
|
+
* one-method object; the narrow interface makes each stub honest.
|
|
17
|
+
*/
|
|
18
|
+
const transportWith = (overrides: Partial<BlobTransport>): BlobTransport => ({
|
|
19
|
+
url: () => {
|
|
20
|
+
throw new Error('url not stubbed');
|
|
21
|
+
},
|
|
22
|
+
put: async () => {
|
|
23
|
+
throw new Error('put not stubbed');
|
|
24
|
+
},
|
|
25
|
+
get: async () => {
|
|
26
|
+
throw new Error('get not stubbed');
|
|
27
|
+
},
|
|
28
|
+
has: async () => {
|
|
29
|
+
throw new Error('has not stubbed');
|
|
30
|
+
},
|
|
31
|
+
...overrides,
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
describe('createEdgeBlobBackend', () => {
|
|
35
|
+
const niUri = fromDigestHex('deadbeef');
|
|
36
|
+
|
|
37
|
+
test('put uploads via the transport and returns an ni: URI', async ({ expect }) => {
|
|
38
|
+
const put = vi.fn(async () => undefined);
|
|
39
|
+
const backend = createEdgeBlobBackend({ transport: transportWith({ put }) });
|
|
40
|
+
|
|
41
|
+
const spaceId = SpaceId.random();
|
|
42
|
+
const data = new Uint8Array([1, 2, 3]);
|
|
43
|
+
const response = await backend.put({ spaceId, data, contentType: 'image/png', contentHash: 'deadbeef' });
|
|
44
|
+
|
|
45
|
+
expect(response.uri).toBe(niUri);
|
|
46
|
+
expect(put).toHaveBeenCalledWith('deadbeef', data, { contentType: 'image/png' });
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test('get downloads using the digest encoded in the URI', async ({ expect }) => {
|
|
50
|
+
const bytes = new Uint8Array([9, 8, 7]);
|
|
51
|
+
const get = vi.fn(async () => bytes);
|
|
52
|
+
const backend = createEdgeBlobBackend({ transport: transportWith({ get }) });
|
|
53
|
+
|
|
54
|
+
const spaceId = SpaceId.random();
|
|
55
|
+
const result = await backend.get({ spaceId, uri: niUri });
|
|
56
|
+
|
|
57
|
+
expect(result).toBe(bytes);
|
|
58
|
+
expect(get).toHaveBeenCalledWith('deadbeef');
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test('get returns undefined when the transport reports a miss', async ({ expect }) => {
|
|
62
|
+
const get = vi.fn(async () => undefined);
|
|
63
|
+
const backend = createEdgeBlobBackend({ transport: transportWith({ get }) });
|
|
64
|
+
|
|
65
|
+
const spaceId = SpaceId.random();
|
|
66
|
+
const result = await backend.get({ spaceId, uri: fromDigestHex('c0ffee') });
|
|
67
|
+
|
|
68
|
+
expect(result).toBeUndefined();
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test('has checks using the digest encoded in the URI', async ({ expect }) => {
|
|
72
|
+
const has = vi.fn(async () => true);
|
|
73
|
+
const backend = createEdgeBlobBackend({ transport: transportWith({ has }) });
|
|
74
|
+
|
|
75
|
+
const spaceId = SpaceId.random();
|
|
76
|
+
const result = await backend.has({ spaceId, uri: niUri });
|
|
77
|
+
|
|
78
|
+
expect(result).toBe(true);
|
|
79
|
+
expect(has).toHaveBeenCalledWith('deadbeef');
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test('adoptUpload turns a finalized upload into an ni: URI', async ({ expect }) => {
|
|
83
|
+
const finalizeUpload = vi.fn(async () => ({ key: 'deadbeef', size: 1234, contentType: 'video/webm' }));
|
|
84
|
+
const backend = createEdgeBlobBackend({ transport: transportWith({ finalizeUpload }) });
|
|
85
|
+
|
|
86
|
+
const spaceId = SpaceId.random();
|
|
87
|
+
const result = await backend.adoptUpload?.({ spaceId, uploadId: 'upload-1' });
|
|
88
|
+
|
|
89
|
+
// The service's key becomes the blob's identity; its size and type are reported verbatim,
|
|
90
|
+
// because nothing on this side of the wire ever saw the bytes.
|
|
91
|
+
expect(result).toEqual({ uri: niUri, size: 1234, contentType: 'video/webm' });
|
|
92
|
+
expect(finalizeUpload).toHaveBeenCalledWith('upload-1');
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test('adoptUpload is absent when the transport cannot finalize uploads', async ({ expect }) => {
|
|
96
|
+
const backend = createEdgeBlobBackend({ transport: transportWith({}) });
|
|
97
|
+
|
|
98
|
+
// Absent rather than throwing: the manager distinguishes "this backend cannot adopt uploads"
|
|
99
|
+
// from "the adoption failed", and only the missing method expresses the first.
|
|
100
|
+
expect(backend.adoptUpload).toBeUndefined();
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test('getUrl builds a direct URL from the digest encoded in the URI', async ({ expect }) => {
|
|
104
|
+
const url = vi.fn((key: string) => new URL(`/blob/file/${key}`, 'https://edge.example.com'));
|
|
105
|
+
const backend = createEdgeBlobBackend({ transport: transportWith({ url }) });
|
|
106
|
+
|
|
107
|
+
const spaceId = SpaceId.random();
|
|
108
|
+
const result = await backend.getUrl?.({ spaceId, uri: niUri });
|
|
109
|
+
|
|
110
|
+
expect(result).toBe('https://edge.example.com/blob/file/deadbeef');
|
|
111
|
+
expect(url).toHaveBeenCalledWith('deadbeef');
|
|
112
|
+
});
|
|
113
|
+
});
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { invariant } from '@dxos/invariant';
|
|
6
|
+
|
|
7
|
+
import { type BlobBackend, type BlobTransport } from '../backend.ts';
|
|
8
|
+
import { SCHEME, digestHex, fromDigestHex } from '../ni-uri.ts';
|
|
9
|
+
|
|
10
|
+
export interface CreateEdgeBlobBackendOptions {
|
|
11
|
+
transport: BlobTransport;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// `SCHEME` from `ni-uri` rather than `Blob.Scheme.ni` from `@dxos/echo`: both are the string `ni`,
|
|
15
|
+
// and taking it from here is what keeps this package below `@dxos/echo` in the graph. `@dxos/echo`
|
|
16
|
+
// exposes `registerBlobBackend(…: BlobBackend)`, so anything it depends on cannot depend back on it.
|
|
17
|
+
const parseNiUri = (uri: string): string => {
|
|
18
|
+
invariant(uri.startsWith(`${SCHEME}:///`), `Invalid ni: URI: ${uri}`);
|
|
19
|
+
return digestHex(uri);
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
/** Largest blob the edge blob service accepts, in bytes. */
|
|
23
|
+
export const MAX_EDGE_BLOB_SIZE = 50 * 1024 * 1024;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Blob backend that stores bytes in the edge blob service, addressed by RFC 6920 `ni:` URIs over a
|
|
27
|
+
* SHA-256 digest of the complete blob. The edge blob service itself is a flat key-value store keyed
|
|
28
|
+
* by hex digest — online-only in v1; reads and writes reject if the edge is unreachable and there is
|
|
29
|
+
* no local cache.
|
|
30
|
+
*
|
|
31
|
+
* @see {@link https://www.rfc-editor.org/rfc/rfc6920 RFC 6920}
|
|
32
|
+
*
|
|
33
|
+
* TODO(wittjosiah): Add a local cache (e.g. via `@dxos/random-access-storage`, keyed by content
|
|
34
|
+
* hash) so reads/writes don't always round-trip to the edge. Blocked on `BlobBackend` gaining a
|
|
35
|
+
* `remove` and GC/refcounting for content-addressed blobs — without that, a local cache has no way
|
|
36
|
+
* to know when a cached entry is safe to evict once its owning Blob object is deleted.
|
|
37
|
+
*/
|
|
38
|
+
export const createEdgeBlobBackend = ({ transport }: CreateEdgeBlobBackendOptions): BlobBackend => {
|
|
39
|
+
// Bound once here rather than re-read inside the closure: the capability check and the call then
|
|
40
|
+
// share one proof that the method exists, and `bind` keeps a method-style transport working.
|
|
41
|
+
const finalizeUpload = transport.finalizeUpload?.bind(transport);
|
|
42
|
+
|
|
43
|
+
return {
|
|
44
|
+
schemes: [SCHEME],
|
|
45
|
+
maxSize: MAX_EDGE_BLOB_SIZE,
|
|
46
|
+
|
|
47
|
+
put: async ({ data, contentType, contentHash }) => {
|
|
48
|
+
await transport.put(contentHash, data, { contentType });
|
|
49
|
+
return { uri: fromDigestHex(contentHash) };
|
|
50
|
+
},
|
|
51
|
+
|
|
52
|
+
get: async ({ uri }) => transport.get(parseNiUri(uri)),
|
|
53
|
+
|
|
54
|
+
has: async ({ uri }) => transport.has(parseNiUri(uri)),
|
|
55
|
+
|
|
56
|
+
getUrl: async ({ uri }) => transport.url(parseNiUri(uri)).toString(),
|
|
57
|
+
|
|
58
|
+
// Spread rather than always present: `adoptUpload` being absent is how the manager tells a
|
|
59
|
+
// backend that cannot adopt uploads from an adoption that failed.
|
|
60
|
+
...(finalizeUpload && {
|
|
61
|
+
adoptUpload: async ({ uploadId }: { uploadId: string }) => {
|
|
62
|
+
const { key, size, contentType } = await finalizeUpload(uploadId);
|
|
63
|
+
return { uri: fromDigestHex(key), size, contentType };
|
|
64
|
+
},
|
|
65
|
+
}),
|
|
66
|
+
};
|
|
67
|
+
};
|
package/src/index.ts
CHANGED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { describe, test } from 'vitest';
|
|
6
|
+
|
|
7
|
+
import * as NiUri from './ni-uri.ts';
|
|
8
|
+
|
|
9
|
+
describe('ni-uri', () => {
|
|
10
|
+
test('encode produces an RFC 6920 ni: URI', async ({ expect }) => {
|
|
11
|
+
const bytes = new Uint8Array([1, 2, 3]);
|
|
12
|
+
const uri = await NiUri.encode(bytes);
|
|
13
|
+
expect(uri).toMatch(/^ni:\/\/\/sha-256;/);
|
|
14
|
+
expect(uri).toBe(NiUri.fromDigest(await NiUri.digestBytes(bytes)));
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test('fromDigestHex roundtrips through decode and digestHex', ({ expect }) => {
|
|
18
|
+
const uri = NiUri.fromDigestHex('deadbeef');
|
|
19
|
+
expect(uri).toBe('ni:///sha-256;3q2-7w');
|
|
20
|
+
expect(NiUri.decode(uri)).toEqual(new Uint8Array([0xde, 0xad, 0xbe, 0xef]));
|
|
21
|
+
expect(NiUri.digestHex(uri)).toBe('deadbeef');
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
test('decode rejects an unknown hash algorithm', ({ expect }) => {
|
|
25
|
+
expect(() => NiUri.decode('ni:///sha-512;abc')).toThrow(/Invalid ni: URI/);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test('decode rejects a malformed URI', ({ expect }) => {
|
|
29
|
+
expect(() => NiUri.decode('not-a-uri')).toThrow(/Invalid ni: URI/);
|
|
30
|
+
});
|
|
31
|
+
});
|
package/src/ni-uri.ts
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* IANA hash-algorithm name for SHA-256 in RFC 6920 Named Information URIs.
|
|
7
|
+
*
|
|
8
|
+
* @see https://www.rfc-editor.org/rfc/rfc6920
|
|
9
|
+
*/
|
|
10
|
+
export const ALG = 'sha-256';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* URI scheme for Named Information URIs (`ni:`).
|
|
14
|
+
*
|
|
15
|
+
* @see https://www.rfc-editor.org/rfc/rfc6920#section-4
|
|
16
|
+
*/
|
|
17
|
+
export const SCHEME = 'ni';
|
|
18
|
+
|
|
19
|
+
const BASE64URL_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_';
|
|
20
|
+
|
|
21
|
+
/** RFC 4648 §5 base64url encoding without padding. */
|
|
22
|
+
export const toBase64url = (bytes: Uint8Array): string => {
|
|
23
|
+
let out = '';
|
|
24
|
+
for (let offset = 0; offset < bytes.length; offset += 3) {
|
|
25
|
+
const byte0 = bytes[offset];
|
|
26
|
+
const byte1 = bytes[offset + 1];
|
|
27
|
+
const byte2 = bytes[offset + 2];
|
|
28
|
+
out += BASE64URL_ALPHABET[byte0 >> 2];
|
|
29
|
+
out += BASE64URL_ALPHABET[((byte0 & 0x03) << 4) | (byte1 === undefined ? 0 : byte1 >> 4)];
|
|
30
|
+
if (byte1 === undefined) {
|
|
31
|
+
break;
|
|
32
|
+
}
|
|
33
|
+
out += BASE64URL_ALPHABET[((byte1 & 0x0f) << 2) | (byte2 === undefined ? 0 : byte2 >> 6)];
|
|
34
|
+
if (byte2 === undefined) {
|
|
35
|
+
break;
|
|
36
|
+
}
|
|
37
|
+
out += BASE64URL_ALPHABET[byte2 & 0x3f];
|
|
38
|
+
}
|
|
39
|
+
return out;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/** RFC 4648 §5 base64url decoding; ignores characters outside the alphabet. */
|
|
43
|
+
export const fromBase64url = (value: string): Uint8Array => {
|
|
44
|
+
const lookup = new Int8Array(128).fill(-1);
|
|
45
|
+
for (let index = 0; index < BASE64URL_ALPHABET.length; index++) {
|
|
46
|
+
lookup[BASE64URL_ALPHABET.charCodeAt(index)] = index;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const out = new Uint8Array((value.length * 3) >> 2);
|
|
50
|
+
let bits = 0;
|
|
51
|
+
let accumulator = 0;
|
|
52
|
+
let position = 0;
|
|
53
|
+
for (let index = 0; index < value.length; index++) {
|
|
54
|
+
const code = value.charCodeAt(index);
|
|
55
|
+
if (code >= lookup.length) {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const digit = lookup[code];
|
|
59
|
+
if (digit < 0) {
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
accumulator = (accumulator << 6) | digit;
|
|
63
|
+
bits += 6;
|
|
64
|
+
if (bits >= 8) {
|
|
65
|
+
bits -= 8;
|
|
66
|
+
out[position++] = (accumulator >> bits) & 0xff;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return out.subarray(0, position);
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
const hexFromDigest = (digest: Uint8Array): string =>
|
|
73
|
+
Array.from(digest)
|
|
74
|
+
.map((byte) => byte.toString(16).padStart(2, '0'))
|
|
75
|
+
.join('');
|
|
76
|
+
|
|
77
|
+
const digestFromHex = (hex: string): Uint8Array => {
|
|
78
|
+
const bytes = new Uint8Array(hex.length / 2);
|
|
79
|
+
for (let index = 0; index < bytes.length; index++) {
|
|
80
|
+
bytes[index] = Number.parseInt(hex.slice(index * 2, index * 2 + 2), 16);
|
|
81
|
+
}
|
|
82
|
+
return bytes;
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Computes the raw SHA-256 digest of `bytes`.
|
|
87
|
+
*/
|
|
88
|
+
export const digestBytes = async (bytes: Uint8Array): Promise<Uint8Array> => {
|
|
89
|
+
// WebCrypto's `BufferSource` type requires an `ArrayBuffer`-backed view, while `Uint8Array` is
|
|
90
|
+
// generic over `ArrayBufferLike` (which also covers `SharedArrayBuffer`) — a real gap between
|
|
91
|
+
// the DOM lib types and the TS standard lib, not fixable by typing `bytes` differently.
|
|
92
|
+
const digest = await crypto.subtle.digest('SHA-256', bytes as BufferSource);
|
|
93
|
+
return new Uint8Array(digest);
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Builds an RFC 6920 Named Information URI from a raw SHA-256 digest.
|
|
98
|
+
*
|
|
99
|
+
* @see https://www.rfc-editor.org/rfc/rfc6920#section-3
|
|
100
|
+
*/
|
|
101
|
+
export const fromDigest = (digest: Uint8Array): string => `ni:///${ALG};${toBase64url(digest)}`;
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Builds an RFC 6920 Named Information URI from a lowercase hex-encoded SHA-256 digest.
|
|
105
|
+
*/
|
|
106
|
+
export const fromDigestHex = (hex: string): string => fromDigest(digestFromHex(hex));
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Computes the RFC 6920 Named Information URI for `bytes` (`ni:///sha-256;<base64url-digest>`).
|
|
110
|
+
*
|
|
111
|
+
* @see https://www.rfc-editor.org/rfc/rfc6920
|
|
112
|
+
*/
|
|
113
|
+
export const encode = async (bytes: Uint8Array): Promise<string> => fromDigest(await digestBytes(bytes));
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Extracts the raw SHA-256 digest from a `ni:` URI.
|
|
117
|
+
*
|
|
118
|
+
* @throws If `uri` is not a well-formed `ni:///sha-256;…` URI.
|
|
119
|
+
*/
|
|
120
|
+
export const decode = (uri: string): Uint8Array => {
|
|
121
|
+
const match = /^ni:\/\/\/([a-z0-9-]+);([A-Za-z0-9_-]+)$/.exec(uri);
|
|
122
|
+
if (!match || match[1] !== ALG) {
|
|
123
|
+
throw new Error(`Invalid ni: URI: ${uri}`);
|
|
124
|
+
}
|
|
125
|
+
return fromBase64url(match[2]);
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Returns the lowercase hex SHA-256 digest encoded in a `ni:` URI.
|
|
130
|
+
*/
|
|
131
|
+
export const digestHex = (uri: string): string => hexFromDigest(decode(uri));
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Computes the lowercase hex SHA-256 digest of `bytes`.
|
|
135
|
+
*/
|
|
136
|
+
export const digestHexFromBytes = async (bytes: Uint8Array): Promise<string> => hexFromDigest(await digestBytes(bytes));
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { invariant } from '@dxos/invariant';
|
|
6
|
+
import { type SpaceId } from '@dxos/keys';
|
|
7
|
+
|
|
8
|
+
import { type BlobBackend } from '../backend.ts';
|
|
9
|
+
import { S3_SCHEME } from './constants.ts';
|
|
10
|
+
import { type S3Credentials, getObject, getObjectUrl, headObject, putObject } from './s3-client.ts';
|
|
11
|
+
import { formatUri, objectKey, parseUri } from './s3-uri.ts';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* How the host answers the two questions this backend cannot answer itself.
|
|
15
|
+
*
|
|
16
|
+
* Both are supplied rather than derived because they need a space's database, and the hosts that
|
|
17
|
+
* should be able to reach a bucket disagree about how to get one: the browser has a `Client`, while
|
|
18
|
+
* EDGE's `operation-service` has only `Database.Service`. Keeping both out of this package is what
|
|
19
|
+
* makes the backend registrable anywhere.
|
|
20
|
+
*/
|
|
21
|
+
export type S3Host = {
|
|
22
|
+
/** The key pair for a bucket endpoint, or `undefined` to attempt an unsigned (public) read. */
|
|
23
|
+
resolveCredentials: (request: { spaceId: SpaceId; host: string }) => Promise<S3Credentials | undefined>;
|
|
24
|
+
/**
|
|
25
|
+
* The bucket a space writes to. Reads take their endpoint from the URI, so this exists only for
|
|
26
|
+
* `put` — `BlobPutRequest` carries no endpoint of its own.
|
|
27
|
+
*/
|
|
28
|
+
resolveWriteEndpoint: (spaceId: SpaceId) => Promise<{ host: string; credentials: S3Credentials } | undefined>;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Blob backend storing bytes in an S3-compatible bucket, addressed by `s3://<host>/<key>` URIs.
|
|
33
|
+
*
|
|
34
|
+
* Reads are attempted unsigned when the space holds no credential for the endpoint, so a public
|
|
35
|
+
* bucket renders for a viewer who was never given the writer's keys. Writes always require one.
|
|
36
|
+
*
|
|
37
|
+
* The bucket must allow the app's origin in its CORS policy for `PUT` and for any programmatic
|
|
38
|
+
* read; rendering does not need it, since a presigned URL in an `<img>` is not CORS-gated. In a
|
|
39
|
+
* headless host CORS does not apply at all.
|
|
40
|
+
*/
|
|
41
|
+
export const createS3BlobBackend = ({ resolveCredentials, resolveWriteEndpoint }: S3Host): BlobBackend => {
|
|
42
|
+
const resolve = (uri: string) => {
|
|
43
|
+
const parsed = parseUri(uri);
|
|
44
|
+
invariant(parsed, `not an s3 uri: ${uri}`);
|
|
45
|
+
return parsed;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
return {
|
|
49
|
+
schemes: [S3_SCHEME],
|
|
50
|
+
|
|
51
|
+
put: async ({ spaceId, data, contentType, contentHash }) => {
|
|
52
|
+
const endpoint = await resolveWriteEndpoint(spaceId);
|
|
53
|
+
invariant(endpoint, 'no S3 connection in this space; connect a bucket before uploading');
|
|
54
|
+
|
|
55
|
+
const uri = { host: endpoint.host, key: objectKey({ spaceId, contentHash }) };
|
|
56
|
+
await putObject({ uri, data, contentType, contentHash, credentials: endpoint.credentials });
|
|
57
|
+
return { uri: formatUri(uri) };
|
|
58
|
+
},
|
|
59
|
+
|
|
60
|
+
get: async ({ spaceId, uri }) => {
|
|
61
|
+
const parsed = resolve(uri);
|
|
62
|
+
return getObject({ uri: parsed, credentials: await resolveCredentials({ spaceId, host: parsed.host }) });
|
|
63
|
+
},
|
|
64
|
+
|
|
65
|
+
has: async ({ spaceId, uri }) => {
|
|
66
|
+
const parsed = resolve(uri);
|
|
67
|
+
return headObject({ uri: parsed, credentials: await resolveCredentials({ spaceId, host: parsed.host }) });
|
|
68
|
+
},
|
|
69
|
+
|
|
70
|
+
getUrl: async ({ spaceId, uri }) => {
|
|
71
|
+
const parsed = resolve(uri);
|
|
72
|
+
return getObjectUrl({ uri: parsed, credentials: await resolveCredentials({ spaceId, host: parsed.host }) });
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Name this backend registers under on the ECHO Hypergraph, and the value `Blob.fromBytes`'s
|
|
7
|
+
* `storage` option takes when it is selected.
|
|
8
|
+
*/
|
|
9
|
+
export const S3_BACKEND = 's3';
|
|
10
|
+
|
|
11
|
+
/** URI scheme this backend resolves at read time. */
|
|
12
|
+
export const S3_SCHEME = 's3';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Signing region for endpoints that do not encode one. R2 ignores the region but SigV4 still
|
|
16
|
+
* requires a value in the credential scope, and `auto` is what Cloudflare documents.
|
|
17
|
+
*/
|
|
18
|
+
export const DEFAULT_REGION = 'auto';
|
|
19
|
+
|
|
20
|
+
/** Lifetime of a presigned GET URL. Long enough to render a large asset, short enough to not be a handout. */
|
|
21
|
+
export const PRESIGN_EXPIRY_SECONDS = 15 * 60;
|
|
22
|
+
|
|
23
|
+
/** Abort a bucket request after this many ms so a hung call cannot wedge an upload or a render. */
|
|
24
|
+
export const S3_TIMEOUT_MS = 60_000;
|
package/src/s3/index.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
// S3-compatible blob storage. Lives beside the blob manager rather than in a plugin so any host —
|
|
6
|
+
// browser, CLI, or an edge function runtime — can register it; the two host-specific answers it
|
|
7
|
+
// needs (a bucket's credentials, and which bucket a space writes to) are supplied by the caller.
|
|
8
|
+
|
|
9
|
+
export * from './blob-backend.ts';
|
|
10
|
+
export * from './constants.ts';
|
|
11
|
+
export * from './s3-client.ts';
|
|
12
|
+
export * from './s3-uri.ts';
|
|
13
|
+
export * from './sigv4.ts';
|