@beam-network/sdk 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -2
- package/dist/client.js +9 -3
- package/dist/models.d.ts +25 -1
- package/dist/models.js +19 -0
- package/dist/provider-signing.d.ts +8 -1
- package/dist/provider-signing.js +115 -94
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -34,13 +34,61 @@ const transfer = await beam.createTransfer({
|
|
|
34
34
|
secret_access_key: "aws-secret-key"
|
|
35
35
|
})
|
|
36
36
|
],
|
|
37
|
-
name: "r2-to-s3-report"
|
|
37
|
+
name: "r2-to-s3-report",
|
|
38
|
+
testMode: true
|
|
38
39
|
});
|
|
39
40
|
|
|
40
41
|
const status = await beam.waitForTransfer(transfer.transfer_id);
|
|
41
42
|
console.log(status.status);
|
|
42
43
|
```
|
|
43
44
|
|
|
44
|
-
The main `createTransfer` API is provider-aware and strictly typed for S3, R2, and Hippius configs.
|
|
45
|
+
The main `createTransfer` API is provider-aware and strictly typed for S3, R2, S3-compatible, and Hippius configs.
|
|
46
|
+
Use `testMode: true` to create a BeamCore test-mode transfer.
|
|
47
|
+
|
|
48
|
+
## S3-Compatible Providers
|
|
49
|
+
|
|
50
|
+
Use `S3CompatibleProviderConfig` for S3-compatible providers with custom
|
|
51
|
+
endpoints, such as Wasabi, MinIO, Backblaze B2 S3 API, DigitalOcean Spaces,
|
|
52
|
+
Scaleway Object Storage, and self-hosted S3-compatible systems.
|
|
53
|
+
|
|
54
|
+
```ts
|
|
55
|
+
import {
|
|
56
|
+
BeamClient,
|
|
57
|
+
S3CompatibleProviderConfig
|
|
58
|
+
} from "@beam-network/sdk";
|
|
59
|
+
|
|
60
|
+
const beam = new BeamClient({ apiKey: process.env.BEAM_API_KEY! });
|
|
61
|
+
|
|
62
|
+
await beam.prepareProviderTransfer({
|
|
63
|
+
sources: [
|
|
64
|
+
S3CompatibleProviderConfig.create({
|
|
65
|
+
provider: "wasabi",
|
|
66
|
+
bucket: "my-bucket",
|
|
67
|
+
key: "input/file.bin",
|
|
68
|
+
region: "us-east-1",
|
|
69
|
+
endpoint_url: "https://s3.us-east-1.wasabisys.com",
|
|
70
|
+
access_key_id: process.env.WASABI_ACCESS_KEY_ID!,
|
|
71
|
+
secret_access_key: process.env.WASABI_SECRET_ACCESS_KEY!
|
|
72
|
+
})
|
|
73
|
+
],
|
|
74
|
+
destinations: [
|
|
75
|
+
S3CompatibleProviderConfig.create({
|
|
76
|
+
provider: "minio",
|
|
77
|
+
bucket: "archive",
|
|
78
|
+
key: "file.bin",
|
|
79
|
+
endpoint_url: "https://minio.example.com",
|
|
80
|
+
force_path_style: true,
|
|
81
|
+
access_key_id: process.env.MINIO_ACCESS_KEY_ID!,
|
|
82
|
+
secret_access_key: process.env.MINIO_SECRET_ACCESS_KEY!
|
|
83
|
+
})
|
|
84
|
+
],
|
|
85
|
+
name: "S3-compatible transfer"
|
|
86
|
+
});
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Use `S3ProviderConfig` for ordinary AWS S3 if preferred, and keep using
|
|
90
|
+
`R2ProviderConfig` for existing Cloudflare R2 code. Use
|
|
91
|
+
`S3CompatibleProviderConfig` when a provider needs a custom S3-compatible
|
|
92
|
+
endpoint.
|
|
45
93
|
|
|
46
94
|
For low-level BeamCore `/transfers/create` calls with raw HTTP-style configs, use `createRawTransfer`.
|
package/dist/client.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { abortMultipartUpload, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, signAbortMultipartUpload, signCompleteMultipartUpload, signDestinationRoute } from "./provider-signing.js";
|
|
1
|
+
import { abortMultipartUpload, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, signAbortMultipartUpload, signCompleteMultipartUpload, signDestinationRoute, signListMultipartUpload } from "./provider-signing.js";
|
|
2
2
|
export const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
|
|
3
3
|
export class BeamApiError extends Error {
|
|
4
4
|
status;
|
|
@@ -112,13 +112,14 @@ export class BeamClient {
|
|
|
112
112
|
let upload = multipartUploads.get(multipartKey);
|
|
113
113
|
if (!upload) {
|
|
114
114
|
const multipartCreatedAt = nowIso();
|
|
115
|
-
if (destination
|
|
115
|
+
if (isHippiusDestination(destination)) {
|
|
116
116
|
upload = {
|
|
117
117
|
destination,
|
|
118
118
|
objectKey: finalObjectKey,
|
|
119
119
|
uploadId: "",
|
|
120
120
|
completeUrl: "",
|
|
121
121
|
abortUrl: "",
|
|
122
|
+
listUrl: "",
|
|
122
123
|
multipartGroupId: multipartKey,
|
|
123
124
|
multipartCreatedAt
|
|
124
125
|
};
|
|
@@ -131,6 +132,7 @@ export class BeamClient {
|
|
|
131
132
|
uploadId,
|
|
132
133
|
completeUrl: await signCompleteMultipartUpload(destination, finalObjectKey, uploadId, expiresIn),
|
|
133
134
|
abortUrl: await signAbortMultipartUpload(destination, finalObjectKey, uploadId, expiresIn),
|
|
135
|
+
listUrl: await signListMultipartUpload(destination, finalObjectKey, uploadId, expiresIn),
|
|
134
136
|
multipartGroupId: multipartKey,
|
|
135
137
|
multipartCreatedAt
|
|
136
138
|
};
|
|
@@ -149,6 +151,7 @@ export class BeamClient {
|
|
|
149
151
|
multipartObjectKey: finalObjectKey,
|
|
150
152
|
completeUrl: upload.completeUrl,
|
|
151
153
|
abortUrl: upload.abortUrl,
|
|
154
|
+
listUrl: upload.listUrl,
|
|
152
155
|
transferId: prepared.transfer_id,
|
|
153
156
|
multipartGroupId: upload.multipartGroupId,
|
|
154
157
|
multipartCreatedAt: upload.multipartCreatedAt,
|
|
@@ -262,12 +265,15 @@ function validateId(value, name) {
|
|
|
262
265
|
}
|
|
263
266
|
async function abortCreatedUploads(uploads) {
|
|
264
267
|
await Promise.all([...uploads.values()].map(async (upload) => {
|
|
265
|
-
if (!upload.uploadId || upload.destination
|
|
268
|
+
if (!upload.uploadId || isHippiusDestination(upload.destination)) {
|
|
266
269
|
return;
|
|
267
270
|
}
|
|
268
271
|
await abortMultipartUpload(upload.destination, upload.objectKey, upload.uploadId);
|
|
269
272
|
}));
|
|
270
273
|
}
|
|
274
|
+
function isHippiusDestination(destination) {
|
|
275
|
+
return destination.provider === "hippius" && "api_token" in destination;
|
|
276
|
+
}
|
|
271
277
|
function sleep(ms) {
|
|
272
278
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
273
279
|
}
|
package/dist/models.d.ts
CHANGED
|
@@ -54,6 +54,9 @@ export interface RawTransferCreateInput {
|
|
|
54
54
|
merkleRoot?: string;
|
|
55
55
|
chunkHashes?: string[];
|
|
56
56
|
callbacks?: CallbackConfig[];
|
|
57
|
+
/**
|
|
58
|
+
* Maps to BeamCore test_mode.
|
|
59
|
+
*/
|
|
57
60
|
testMode?: boolean;
|
|
58
61
|
progressiveMode?: boolean;
|
|
59
62
|
}
|
|
@@ -137,6 +140,20 @@ export interface R2ProviderConfig {
|
|
|
137
140
|
account_id?: string;
|
|
138
141
|
endpoint_url?: string;
|
|
139
142
|
}
|
|
143
|
+
export interface S3CompatibleProviderConfig {
|
|
144
|
+
provider: string;
|
|
145
|
+
driver?: "s3-compatible";
|
|
146
|
+
id?: string;
|
|
147
|
+
bucket: string;
|
|
148
|
+
key: string;
|
|
149
|
+
region?: string;
|
|
150
|
+
endpoint_url?: string;
|
|
151
|
+
access_key_id: string;
|
|
152
|
+
secret_access_key: string;
|
|
153
|
+
session_token?: string;
|
|
154
|
+
force_path_style?: boolean;
|
|
155
|
+
account_id?: string;
|
|
156
|
+
}
|
|
140
157
|
export interface HippiusProviderConfig {
|
|
141
158
|
provider: "hippius";
|
|
142
159
|
id?: string;
|
|
@@ -145,11 +162,12 @@ export interface HippiusProviderConfig {
|
|
|
145
162
|
api_token: string;
|
|
146
163
|
base_url?: string;
|
|
147
164
|
}
|
|
148
|
-
export type ProviderConfig = S3ProviderConfig | R2ProviderConfig | HippiusProviderConfig;
|
|
165
|
+
export type ProviderConfig = S3ProviderConfig | R2ProviderConfig | S3CompatibleProviderConfig | HippiusProviderConfig;
|
|
149
166
|
export type ProviderSourceConfig = ProviderConfig;
|
|
150
167
|
export type ProviderDestinationConfig = ProviderConfig;
|
|
151
168
|
export type S3ProviderConfigInput = Omit<S3ProviderConfig, "provider">;
|
|
152
169
|
export type R2ProviderConfigInput = Omit<R2ProviderConfig, "provider">;
|
|
170
|
+
export type S3CompatibleProviderConfigInput = Omit<S3CompatibleProviderConfig, "driver">;
|
|
153
171
|
export type HippiusProviderConfigInput = Omit<HippiusProviderConfig, "provider">;
|
|
154
172
|
export declare const S3ProviderConfig: Readonly<{
|
|
155
173
|
create(input: S3ProviderConfigInput): S3ProviderConfig;
|
|
@@ -157,6 +175,9 @@ export declare const S3ProviderConfig: Readonly<{
|
|
|
157
175
|
export declare const R2ProviderConfig: Readonly<{
|
|
158
176
|
create(input: R2ProviderConfigInput): R2ProviderConfig;
|
|
159
177
|
}>;
|
|
178
|
+
export declare const S3CompatibleProviderConfig: Readonly<{
|
|
179
|
+
create(input: S3CompatibleProviderConfigInput): S3CompatibleProviderConfig;
|
|
180
|
+
}>;
|
|
160
181
|
export declare const HippiusProviderConfig: Readonly<{
|
|
161
182
|
create(input: HippiusProviderConfigInput): HippiusProviderConfig;
|
|
162
183
|
}>;
|
|
@@ -164,6 +185,9 @@ export interface ProviderTransferCreateInput {
|
|
|
164
185
|
sources: ProviderSourceConfig[];
|
|
165
186
|
destinations: ProviderDestinationConfig[];
|
|
166
187
|
name?: string;
|
|
188
|
+
/**
|
|
189
|
+
* Maps to BeamCore test_mode.
|
|
190
|
+
*/
|
|
167
191
|
testMode?: boolean;
|
|
168
192
|
expiresIn?: number;
|
|
169
193
|
/**
|
package/dist/models.js
CHANGED
|
@@ -13,6 +13,25 @@ export const R2ProviderConfig = Object.freeze({
|
|
|
13
13
|
return { provider: "r2", ...input };
|
|
14
14
|
}
|
|
15
15
|
});
|
|
16
|
+
export const S3CompatibleProviderConfig = Object.freeze({
|
|
17
|
+
create(input) {
|
|
18
|
+
requireFields(input.provider || "s3-compatible", input, [
|
|
19
|
+
"provider",
|
|
20
|
+
"bucket",
|
|
21
|
+
"key",
|
|
22
|
+
"access_key_id",
|
|
23
|
+
"secret_access_key"
|
|
24
|
+
]);
|
|
25
|
+
const provider = input.provider.trim().toLowerCase();
|
|
26
|
+
if (provider === "r2" && !hasText(input.endpoint_url) && !hasText(input.account_id)) {
|
|
27
|
+
throw new Error("r2 config requires account_id or endpoint_url.");
|
|
28
|
+
}
|
|
29
|
+
if (provider !== "s3" && provider !== "r2" && !hasText(input.endpoint_url)) {
|
|
30
|
+
throw new Error(`${provider} config requires endpoint_url.`);
|
|
31
|
+
}
|
|
32
|
+
return { ...input, provider, driver: "s3-compatible" };
|
|
33
|
+
}
|
|
34
|
+
});
|
|
16
35
|
export const HippiusProviderConfig = Object.freeze({
|
|
17
36
|
create(input) {
|
|
18
37
|
requireFields("hippius", input, ["bucket", "key", "api_token"]);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, SignedChunkRoute } from "./models.js";
|
|
1
|
+
import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, R2ProviderConfig, S3CompatibleProviderConfig, S3ProviderConfig, SignedChunkRoute } from "./models.js";
|
|
2
|
+
export type AnyS3CompatibleProviderConfig = S3ProviderConfig | R2ProviderConfig | S3CompatibleProviderConfig;
|
|
2
3
|
export declare function nowIso(): string;
|
|
3
4
|
export declare function expiresAtIso(expiresIn: number): string;
|
|
4
5
|
export declare function prepareProviderSource(source: ProviderSourceConfig, options?: {
|
|
@@ -12,6 +13,7 @@ export declare function prepareProviderDestination(destination: ProviderDestinat
|
|
|
12
13
|
export declare function createMultipartUpload(destination: ProviderDestinationConfig, objectKey: string): Promise<string>;
|
|
13
14
|
export declare function signCompleteMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
|
|
14
15
|
export declare function signAbortMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
|
|
16
|
+
export declare function signListMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
|
|
15
17
|
export declare function abortMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string): Promise<void>;
|
|
16
18
|
export declare function signDestinationRoute(input: {
|
|
17
19
|
chunk: ChunkSigningPlanItem;
|
|
@@ -23,8 +25,13 @@ export declare function signDestinationRoute(input: {
|
|
|
23
25
|
multipartObjectKey?: string;
|
|
24
26
|
completeUrl?: string;
|
|
25
27
|
abortUrl?: string;
|
|
28
|
+
listUrl?: string;
|
|
26
29
|
transferId?: string;
|
|
27
30
|
multipartGroupId?: string;
|
|
28
31
|
multipartCreatedAt?: string;
|
|
29
32
|
fetchImpl?: typeof fetch;
|
|
30
33
|
}): Promise<SignedChunkRoute>;
|
|
34
|
+
export declare function isS3CompatibleProvider(config: ProviderSourceConfig | ProviderDestinationConfig): config is AnyS3CompatibleProviderConfig;
|
|
35
|
+
export declare function s3CompatibleEndpoint(source: AnyS3CompatibleProviderConfig): string | undefined;
|
|
36
|
+
export declare function s3CompatibleRegion(source: AnyS3CompatibleProviderConfig): string;
|
|
37
|
+
export declare function s3CompatibleForcePathStyle(source: AnyS3CompatibleProviderConfig, endpoint?: string | undefined): boolean | undefined;
|
package/dist/provider-signing.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, GetObjectCommand, HeadObjectCommand, PutObjectCommand, S3Client, UploadPartCommand } from "@aws-sdk/client-s3";
|
|
1
|
+
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, GetObjectCommand, HeadObjectCommand, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCommand } from "@aws-sdk/client-s3";
|
|
2
2
|
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
3
3
|
export function nowIso() {
|
|
4
4
|
return new Date().toISOString();
|
|
@@ -9,37 +9,23 @@ export function expiresAtIso(expiresIn) {
|
|
|
9
9
|
export async function prepareProviderSource(source, options = {}) {
|
|
10
10
|
const index = options.index ?? 0;
|
|
11
11
|
const expiresIn = options.expiresIn ?? 3600;
|
|
12
|
-
if (source
|
|
13
|
-
const
|
|
14
|
-
const
|
|
15
|
-
const getUrl = await getSignedUrl(client, new GetObjectCommand({ Bucket: source.bucket, Key: source.key }), { expiresIn });
|
|
16
|
-
return {
|
|
17
|
-
source_id: source.id ?? `src_${index}`,
|
|
18
|
-
type: "http",
|
|
19
|
-
provider: "s3",
|
|
20
|
-
url: getUrl,
|
|
21
|
-
size: Number(head.ContentLength ?? 0),
|
|
22
|
-
filename: filename(source.key),
|
|
23
|
-
expires_at: expiresAtIso(expiresIn),
|
|
24
|
-
metadata: { bucket: source.bucket, key: source.key, region: source.region ?? "us-east-1" }
|
|
25
|
-
};
|
|
26
|
-
}
|
|
27
|
-
if (source.provider === "r2") {
|
|
28
|
-
const client = createR2Client(source);
|
|
12
|
+
if (isS3CompatibleProvider(source)) {
|
|
13
|
+
const endpoint = s3CompatibleEndpoint(source);
|
|
14
|
+
const client = createS3CompatibleClient(source, endpoint);
|
|
29
15
|
const head = await client.send(new HeadObjectCommand({ Bucket: source.bucket, Key: source.key }));
|
|
30
16
|
const getUrl = await getSignedUrl(client, new GetObjectCommand({ Bucket: source.bucket, Key: source.key }), { expiresIn });
|
|
31
17
|
return {
|
|
32
18
|
source_id: source.id ?? `src_${index}`,
|
|
33
19
|
type: "http",
|
|
34
|
-
provider:
|
|
20
|
+
provider: source.provider,
|
|
35
21
|
url: getUrl,
|
|
36
22
|
size: Number(head.ContentLength ?? 0),
|
|
37
23
|
filename: filename(source.key),
|
|
38
24
|
expires_at: expiresAtIso(expiresIn),
|
|
39
|
-
metadata:
|
|
25
|
+
metadata: s3CompatibleMetadata(source, endpoint)
|
|
40
26
|
};
|
|
41
27
|
}
|
|
42
|
-
if (source
|
|
28
|
+
if (isHippiusProvider(source)) {
|
|
43
29
|
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
44
30
|
const baseUrl = source.base_url ?? "https://api.hippius.com";
|
|
45
31
|
const size = await hippiusObjectSize(fetchImpl, baseUrl, source.api_token, source.bucket, source.key);
|
|
@@ -55,45 +41,39 @@ export async function prepareProviderSource(source, options = {}) {
|
|
|
55
41
|
metadata: { bucket: source.bucket, key: source.key, base_url: baseUrl }
|
|
56
42
|
};
|
|
57
43
|
}
|
|
58
|
-
return
|
|
44
|
+
return unsupportedProviderConfig(source);
|
|
59
45
|
}
|
|
60
46
|
export function prepareProviderDestination(destination, options = {}) {
|
|
61
47
|
const index = options.index ?? 0;
|
|
62
48
|
const destinationId = destination.id ?? `dst_${index}`;
|
|
63
|
-
if (destination
|
|
49
|
+
if (isS3CompatibleProvider(destination)) {
|
|
50
|
+
const endpoint = s3CompatibleEndpoint(destination);
|
|
64
51
|
return {
|
|
65
52
|
destination_id: destinationId,
|
|
66
|
-
provider:
|
|
53
|
+
provider: destination.provider,
|
|
67
54
|
logical_prefix: destination.key,
|
|
68
|
-
metadata: {
|
|
55
|
+
metadata: {
|
|
56
|
+
...s3CompatibleMetadata(destination, endpoint)
|
|
57
|
+
}
|
|
69
58
|
};
|
|
70
59
|
}
|
|
71
|
-
if (destination
|
|
60
|
+
if (isHippiusProvider(destination)) {
|
|
72
61
|
return {
|
|
73
62
|
destination_id: destinationId,
|
|
74
|
-
provider: "
|
|
75
|
-
logical_prefix: destination.key,
|
|
76
|
-
metadata: {
|
|
63
|
+
provider: "hippius",
|
|
64
|
+
logical_prefix: destination.key.replace(/\/+$/, ""),
|
|
65
|
+
metadata: {
|
|
66
|
+
bucket: destination.bucket,
|
|
67
|
+
key: destination.key,
|
|
68
|
+
base_url: destination.base_url ?? "https://api.hippius.com"
|
|
69
|
+
}
|
|
77
70
|
};
|
|
78
71
|
}
|
|
79
|
-
return
|
|
80
|
-
destination_id: destinationId,
|
|
81
|
-
provider: "hippius",
|
|
82
|
-
logical_prefix: destination.key.replace(/\/+$/, ""),
|
|
83
|
-
metadata: { bucket: destination.bucket, key: destination.key, base_url: destination.base_url ?? "https://api.hippius.com" }
|
|
84
|
-
};
|
|
72
|
+
return unsupportedProviderConfig(destination);
|
|
85
73
|
}
|
|
86
74
|
export async function createMultipartUpload(destination, objectKey) {
|
|
87
|
-
if (destination
|
|
88
|
-
const client =
|
|
89
|
-
const response = await client.send(new CreateMultipartUploadCommand({ Bucket: destination.bucket, Key: objectKey }));
|
|
90
|
-
if (!response.UploadId) {
|
|
91
|
-
throw new Error(`provider did not return UploadId for ${objectKey}`);
|
|
92
|
-
}
|
|
93
|
-
return response.UploadId;
|
|
94
|
-
}
|
|
95
|
-
if (destination.provider === "r2") {
|
|
96
|
-
const client = createR2Client(destination);
|
|
75
|
+
if (isS3CompatibleProvider(destination)) {
|
|
76
|
+
const client = createS3CompatibleClient(destination);
|
|
97
77
|
const response = await client.send(new CreateMultipartUploadCommand({ Bucket: destination.bucket, Key: objectKey }));
|
|
98
78
|
if (!response.UploadId) {
|
|
99
79
|
throw new Error(`provider did not return UploadId for ${objectKey}`);
|
|
@@ -103,26 +83,29 @@ export async function createMultipartUpload(destination, objectKey) {
|
|
|
103
83
|
throw new Error(`multipart upload is not supported for ${destination.provider}`);
|
|
104
84
|
}
|
|
105
85
|
export async function signCompleteMultipartUpload(destination, objectKey, uploadId, expiresIn) {
|
|
106
|
-
if (destination
|
|
107
|
-
const client =
|
|
86
|
+
if (isS3CompatibleProvider(destination)) {
|
|
87
|
+
const client = createS3CompatibleClient(destination);
|
|
108
88
|
return getSignedUrl(client, new CompleteMultipartUploadCommand({ Bucket: destination.bucket, Key: objectKey, UploadId: uploadId }), { expiresIn });
|
|
109
89
|
}
|
|
110
90
|
throw new Error(`multipart completion is not supported for ${destination.provider}`);
|
|
111
91
|
}
|
|
112
92
|
export async function signAbortMultipartUpload(destination, objectKey, uploadId, expiresIn) {
|
|
113
|
-
if (destination
|
|
114
|
-
const client =
|
|
93
|
+
if (isS3CompatibleProvider(destination)) {
|
|
94
|
+
const client = createS3CompatibleClient(destination);
|
|
115
95
|
return getSignedUrl(client, new AbortMultipartUploadCommand({ Bucket: destination.bucket, Key: objectKey, UploadId: uploadId }), { expiresIn });
|
|
116
96
|
}
|
|
117
97
|
throw new Error(`multipart abort is not supported for ${destination.provider}`);
|
|
118
98
|
}
|
|
119
|
-
export async function
|
|
120
|
-
if (destination
|
|
121
|
-
|
|
122
|
-
return;
|
|
99
|
+
export async function signListMultipartUpload(destination, objectKey, uploadId, expiresIn) {
|
|
100
|
+
if (isS3CompatibleProvider(destination)) {
|
|
101
|
+
const client = createS3CompatibleClient(destination);
|
|
102
|
+
return getSignedUrl(client, new ListPartsCommand({ Bucket: destination.bucket, Key: objectKey, UploadId: uploadId }), { expiresIn });
|
|
123
103
|
}
|
|
124
|
-
|
|
125
|
-
|
|
104
|
+
throw new Error(`multipart list-parts is not supported for ${destination.provider}`);
|
|
105
|
+
}
|
|
106
|
+
export async function abortMultipartUpload(destination, objectKey, uploadId) {
|
|
107
|
+
if (isS3CompatibleProvider(destination)) {
|
|
108
|
+
await createS3CompatibleClient(destination).send(new AbortMultipartUploadCommand({ Bucket: destination.bucket, Key: objectKey, UploadId: uploadId }));
|
|
126
109
|
return;
|
|
127
110
|
}
|
|
128
111
|
throw new Error(`multipart abort is not supported for ${destination.provider}`);
|
|
@@ -159,62 +142,94 @@ export async function signDestinationRoute(input) {
|
|
|
159
142
|
...(input.uploadId ? { upload_id: input.uploadId } : {}),
|
|
160
143
|
...(input.partNumber ? { part_number: input.partNumber } : {}),
|
|
161
144
|
...(input.completeUrl ? { complete_url: input.completeUrl } : {}),
|
|
162
|
-
...(input.abortUrl ? { abort_url: input.abortUrl } : {})
|
|
145
|
+
...(input.abortUrl ? { abort_url: input.abortUrl } : {}),
|
|
146
|
+
...(input.listUrl ? { list_url: input.listUrl } : {})
|
|
163
147
|
}
|
|
164
148
|
};
|
|
165
149
|
}
|
|
166
150
|
async function signDestinationUrl(input) {
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
:
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
Key: input.uploadPartObjectKey,
|
|
177
|
-
UploadId: input.uploadId,
|
|
178
|
-
PartNumber: input.partNumber
|
|
179
|
-
}), { expiresIn: input.expiresIn });
|
|
180
|
-
}
|
|
181
|
-
return getSignedUrl(client, new PutObjectCommand({ Bucket: input.destination.bucket, Key: input.objectKey }), { expiresIn: input.expiresIn });
|
|
151
|
+
if (isS3CompatibleProvider(input.destination)) {
|
|
152
|
+
const client = createS3CompatibleClient(input.destination);
|
|
153
|
+
if (input.uploadId && input.partNumber) {
|
|
154
|
+
return getSignedUrl(client, new UploadPartCommand({
|
|
155
|
+
Bucket: input.destination.bucket,
|
|
156
|
+
Key: input.uploadPartObjectKey,
|
|
157
|
+
UploadId: input.uploadId,
|
|
158
|
+
PartNumber: input.partNumber
|
|
159
|
+
}), { expiresIn: input.expiresIn });
|
|
182
160
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
161
|
+
return getSignedUrl(client, new PutObjectCommand({ Bucket: input.destination.bucket, Key: input.objectKey }), { expiresIn: input.expiresIn });
|
|
162
|
+
}
|
|
163
|
+
if (isHippiusProvider(input.destination)) {
|
|
164
|
+
return hippiusPresign(input.fetchImpl ?? globalThis.fetch, input.destination.base_url ?? "https://api.hippius.com", input.destination.api_token, input.destination.bucket, input.objectKey, "put", input.expiresIn);
|
|
187
165
|
}
|
|
166
|
+
return unsupportedProviderConfig(input.destination);
|
|
188
167
|
}
|
|
189
|
-
function
|
|
168
|
+
function createS3CompatibleClient(source, endpoint = s3CompatibleEndpoint(source)) {
|
|
190
169
|
return new S3Client({
|
|
191
|
-
region: source
|
|
192
|
-
endpoint
|
|
170
|
+
region: s3CompatibleRegion(source),
|
|
171
|
+
endpoint,
|
|
172
|
+
forcePathStyle: s3CompatibleForcePathStyle(source, endpoint),
|
|
193
173
|
credentials: {
|
|
194
174
|
accessKeyId: source.access_key_id,
|
|
195
175
|
secretAccessKey: source.secret_access_key,
|
|
196
|
-
sessionToken: source.session_token
|
|
176
|
+
sessionToken: "session_token" in source ? source.session_token : undefined
|
|
197
177
|
}
|
|
198
178
|
});
|
|
199
179
|
}
|
|
200
|
-
function
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
});
|
|
180
|
+
export function isS3CompatibleProvider(config) {
|
|
181
|
+
if (config.provider === "s3" || config.provider === "r2") {
|
|
182
|
+
return true;
|
|
183
|
+
}
|
|
184
|
+
if ("driver" in config && config.driver === "s3-compatible") {
|
|
185
|
+
return true;
|
|
186
|
+
}
|
|
187
|
+
return "access_key_id" in config && "secret_access_key" in config;
|
|
209
188
|
}
|
|
210
|
-
function
|
|
211
|
-
if (source.endpoint_url) {
|
|
189
|
+
export function s3CompatibleEndpoint(source) {
|
|
190
|
+
if (hasText(source.endpoint_url)) {
|
|
212
191
|
return source.endpoint_url;
|
|
213
192
|
}
|
|
214
|
-
if (source.
|
|
215
|
-
|
|
193
|
+
if (source.provider === "r2") {
|
|
194
|
+
if ("account_id" in source && hasText(source.account_id)) {
|
|
195
|
+
return `https://${source.account_id}.r2.cloudflarestorage.com`;
|
|
196
|
+
}
|
|
197
|
+
throw new Error("r2 config requires account_id or endpoint_url.");
|
|
198
|
+
}
|
|
199
|
+
if (source.provider !== "s3") {
|
|
200
|
+
throw new Error(`${source.provider} config requires endpoint_url.`);
|
|
216
201
|
}
|
|
217
|
-
|
|
202
|
+
return undefined;
|
|
203
|
+
}
|
|
204
|
+
export function s3CompatibleRegion(source) {
|
|
205
|
+
if ("region" in source && hasText(source.region)) {
|
|
206
|
+
return source.region;
|
|
207
|
+
}
|
|
208
|
+
if (source.provider === "r2") {
|
|
209
|
+
return "auto";
|
|
210
|
+
}
|
|
211
|
+
return "us-east-1";
|
|
212
|
+
}
|
|
213
|
+
export function s3CompatibleForcePathStyle(source, endpoint = s3CompatibleEndpoint(source)) {
|
|
214
|
+
if ("force_path_style" in source && typeof source.force_path_style === "boolean") {
|
|
215
|
+
return source.force_path_style;
|
|
216
|
+
}
|
|
217
|
+
if (source.provider === "s3") {
|
|
218
|
+
return undefined;
|
|
219
|
+
}
|
|
220
|
+
return Boolean(endpoint);
|
|
221
|
+
}
|
|
222
|
+
function s3CompatibleMetadata(source, endpoint = s3CompatibleEndpoint(source)) {
|
|
223
|
+
return compactMetadata({
|
|
224
|
+
driver: "s3-compatible",
|
|
225
|
+
bucket: source.bucket,
|
|
226
|
+
key: source.key,
|
|
227
|
+
region: s3CompatibleRegion(source),
|
|
228
|
+
endpoint_url: endpoint
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
function isHippiusProvider(config) {
|
|
232
|
+
return config.provider === "hippius" && "api_token" in config;
|
|
218
233
|
}
|
|
219
234
|
async function hippiusPresign(fetchImpl, baseUrl, token, bucket, key, action, expiresIn) {
|
|
220
235
|
const url = new URL(`/api/objectstore/buckets/${bucket}/presigned-url/`, baseUrl);
|
|
@@ -245,6 +260,12 @@ async function hippiusObjectSize(fetchImpl, baseUrl, token, bucket, key) {
|
|
|
245
260
|
function filename(key) {
|
|
246
261
|
return key.split("/").filter(Boolean).at(-1) ?? key;
|
|
247
262
|
}
|
|
248
|
-
function
|
|
263
|
+
function compactMetadata(input) {
|
|
264
|
+
return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined));
|
|
265
|
+
}
|
|
266
|
+
function hasText(value) {
|
|
267
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
268
|
+
}
|
|
269
|
+
function unsupportedProviderConfig(value) {
|
|
249
270
|
throw new Error(`Unsupported provider config: ${JSON.stringify(value)}`);
|
|
250
271
|
}
|