@zero-transfer/sdk 0.4.0 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +146 -21
- package/dist/index.cjs +724 -46
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.mts +738 -25
- package/dist/index.d.ts +738 -25
- package/dist/index.mjs +714 -38
- package/dist/index.mjs.map +1 -1
- package/package.json +4 -1
package/dist/index.mjs
CHANGED
|
@@ -6104,7 +6104,7 @@ var SshTransportPacketUnprotector = class {
|
|
|
6104
6104
|
}
|
|
6105
6105
|
/**
|
|
6106
6106
|
* Feeds raw encrypted bytes from the socket and returns any fully decoded payloads.
|
|
6107
|
-
* Maintains internal framing state across calls
|
|
6107
|
+
* Maintains internal framing state across calls - pass each `data` event chunk directly.
|
|
6108
6108
|
*/
|
|
6109
6109
|
pushBytes(chunk) {
|
|
6110
6110
|
this.framePendingRaw = Buffer16.concat([this.framePendingRaw, chunk]);
|
|
@@ -6612,7 +6612,7 @@ var SshTransportConnection = class {
|
|
|
6612
6612
|
assertConnected() {
|
|
6613
6613
|
if (!this.connected) {
|
|
6614
6614
|
throw new ProtocolError({
|
|
6615
|
-
message: "SshTransportConnection is not yet connected
|
|
6615
|
+
message: "SshTransportConnection is not yet connected - call connect() first",
|
|
6616
6616
|
protocol: "sftp",
|
|
6617
6617
|
retryable: false
|
|
6618
6618
|
});
|
|
@@ -6952,14 +6952,14 @@ function sftpStatusToError(status, path2) {
|
|
|
6952
6952
|
case SFTP_STATUS.NO_SUCH_FILE:
|
|
6953
6953
|
return new PathNotFoundError({
|
|
6954
6954
|
details: { path: path2, sftpMessage: status.errorMessage },
|
|
6955
|
-
message: `SFTP: no such file or directory${path2 !== void 0 ? `
|
|
6955
|
+
message: `SFTP: no such file or directory${path2 !== void 0 ? ` - ${path2}` : ""}`,
|
|
6956
6956
|
protocol: "sftp",
|
|
6957
6957
|
retryable: false
|
|
6958
6958
|
});
|
|
6959
6959
|
case SFTP_STATUS.PERMISSION_DENIED:
|
|
6960
6960
|
return new PermissionDeniedError({
|
|
6961
6961
|
details: { path: path2, sftpMessage: status.errorMessage },
|
|
6962
|
-
message: `SFTP: permission denied${path2 !== void 0 ? `
|
|
6962
|
+
message: `SFTP: permission denied${path2 !== void 0 ? ` - ${path2}` : ""}`,
|
|
6963
6963
|
protocol: "sftp",
|
|
6964
6964
|
retryable: false
|
|
6965
6965
|
});
|
|
@@ -6967,21 +6967,21 @@ function sftpStatusToError(status, path2) {
|
|
|
6967
6967
|
case SFTP_STATUS.CONNECTION_LOST:
|
|
6968
6968
|
return new ConnectionError({
|
|
6969
6969
|
details: { sftpMessage: status.errorMessage, statusCode: status.statusCode },
|
|
6970
|
-
message: `SFTP: connection error
|
|
6970
|
+
message: `SFTP: connection error - ${status.errorMessage}`,
|
|
6971
6971
|
protocol: "sftp",
|
|
6972
6972
|
retryable: true
|
|
6973
6973
|
});
|
|
6974
6974
|
case SFTP_STATUS.OP_UNSUPPORTED:
|
|
6975
6975
|
return new UnsupportedFeatureError({
|
|
6976
6976
|
details: { sftpMessage: status.errorMessage },
|
|
6977
|
-
message: `SFTP: operation unsupported
|
|
6977
|
+
message: `SFTP: operation unsupported - ${status.errorMessage}`,
|
|
6978
6978
|
protocol: "sftp",
|
|
6979
6979
|
retryable: false
|
|
6980
6980
|
});
|
|
6981
6981
|
case SFTP_STATUS.BAD_MESSAGE:
|
|
6982
6982
|
return new ProtocolError({
|
|
6983
6983
|
details: { sftpMessage: status.errorMessage },
|
|
6984
|
-
message: `SFTP: bad message
|
|
6984
|
+
message: `SFTP: bad message - ${status.errorMessage}`,
|
|
6985
6985
|
protocol: "sftp",
|
|
6986
6986
|
retryable: false
|
|
6987
6987
|
});
|
|
@@ -6989,7 +6989,7 @@ function sftpStatusToError(status, path2) {
|
|
|
6989
6989
|
return new ZeroTransferError({
|
|
6990
6990
|
code: "SFTP_FAILURE",
|
|
6991
6991
|
details: { sftpMessage: status.errorMessage, statusCode: status.statusCode },
|
|
6992
|
-
message: `SFTP: operation failed (status ${status.statusCode})
|
|
6992
|
+
message: `SFTP: operation failed (status ${status.statusCode}) - ${status.errorMessage}`,
|
|
6993
6993
|
protocol: "sftp",
|
|
6994
6994
|
retryable: false
|
|
6995
6995
|
});
|
|
@@ -9092,6 +9092,9 @@ function concatChunks(chunks, totalSize) {
|
|
|
9092
9092
|
// src/providers/cloud/OneDriveProvider.ts
|
|
9093
9093
|
var ONEDRIVE_DRIVE_BASE = "https://graph.microsoft.com/v1.0/me/drive";
|
|
9094
9094
|
var ONEDRIVE_CHECKSUM_CAPABILITIES = ["sha1", "sha256", "quickxorhash"];
|
|
9095
|
+
var ONEDRIVE_CHUNK_ALIGNMENT = 320 * 1024;
|
|
9096
|
+
var DEFAULT_ONEDRIVE_PART_SIZE = 10 * 1024 * 1024;
|
|
9097
|
+
var DEFAULT_ONEDRIVE_THRESHOLD = 4 * 1024 * 1024;
|
|
9095
9098
|
function createOneDriveProviderFactory(options = {}) {
|
|
9096
9099
|
const id = options.id ?? "one-drive";
|
|
9097
9100
|
const fetchImpl = options.fetch ?? globalThis.fetch;
|
|
@@ -9102,6 +9105,37 @@ function createOneDriveProviderFactory(options = {}) {
|
|
|
9102
9105
|
retryable: false
|
|
9103
9106
|
});
|
|
9104
9107
|
}
|
|
9108
|
+
const multipartEnabled = options.multipart?.enabled ?? true;
|
|
9109
|
+
const partSizeBytes = options.multipart?.partSizeBytes ?? DEFAULT_ONEDRIVE_PART_SIZE;
|
|
9110
|
+
const thresholdBytes = options.multipart?.thresholdBytes ?? DEFAULT_ONEDRIVE_THRESHOLD;
|
|
9111
|
+
if (multipartEnabled) {
|
|
9112
|
+
if (!Number.isInteger(partSizeBytes) || partSizeBytes <= 0) {
|
|
9113
|
+
throw new ConfigurationError({
|
|
9114
|
+
details: { partSizeBytes },
|
|
9115
|
+
message: "OneDriveMultipartOptions.partSizeBytes must be a positive integer",
|
|
9116
|
+
retryable: false
|
|
9117
|
+
});
|
|
9118
|
+
}
|
|
9119
|
+
if (partSizeBytes % ONEDRIVE_CHUNK_ALIGNMENT !== 0) {
|
|
9120
|
+
throw new ConfigurationError({
|
|
9121
|
+
details: { partSizeBytes },
|
|
9122
|
+
message: `OneDrive multipart partSizeBytes must be a multiple of ${String(ONEDRIVE_CHUNK_ALIGNMENT)} bytes (320 KiB)`,
|
|
9123
|
+
retryable: false
|
|
9124
|
+
});
|
|
9125
|
+
}
|
|
9126
|
+
if (!Number.isInteger(thresholdBytes) || thresholdBytes < 0) {
|
|
9127
|
+
throw new ConfigurationError({
|
|
9128
|
+
details: { thresholdBytes },
|
|
9129
|
+
message: "OneDriveMultipartOptions.thresholdBytes must be a non-negative integer",
|
|
9130
|
+
retryable: false
|
|
9131
|
+
});
|
|
9132
|
+
}
|
|
9133
|
+
}
|
|
9134
|
+
const multipart = {
|
|
9135
|
+
enabled: multipartEnabled,
|
|
9136
|
+
partSizeBytes,
|
|
9137
|
+
thresholdBytes
|
|
9138
|
+
};
|
|
9105
9139
|
const capabilities = {
|
|
9106
9140
|
atomicRename: false,
|
|
9107
9141
|
authentication: ["token", "oauth"],
|
|
@@ -9111,13 +9145,17 @@ function createOneDriveProviderFactory(options = {}) {
|
|
|
9111
9145
|
list: true,
|
|
9112
9146
|
maxConcurrency: 4,
|
|
9113
9147
|
metadata: ["modifiedAt", "createdAt", "uniqueId"],
|
|
9114
|
-
notes: [
|
|
9115
|
-
|
|
9148
|
+
notes: multipartEnabled ? [
|
|
9149
|
+
`OneDrive upload session enabled by default (partSize=${String(multipart.partSizeBytes)}B, threshold=${String(multipart.thresholdBytes)}B).`,
|
|
9150
|
+
"Payloads at or below the threshold automatically fall back to single-shot PUT /content.",
|
|
9151
|
+
"Pass `multipart: { enabled: false }` to force the legacy single-shot behaviour."
|
|
9152
|
+
] : [
|
|
9153
|
+
"OneDrive provider performs single-shot uploads via PUT /content; resumable upload sessions are disabled."
|
|
9116
9154
|
],
|
|
9117
9155
|
provider: id,
|
|
9118
9156
|
readStream: true,
|
|
9119
9157
|
resumeDownload: true,
|
|
9120
|
-
resumeUpload:
|
|
9158
|
+
resumeUpload: multipartEnabled,
|
|
9121
9159
|
serverSideCopy: false,
|
|
9122
9160
|
serverSideMove: false,
|
|
9123
9161
|
stat: true,
|
|
@@ -9131,7 +9169,8 @@ function createOneDriveProviderFactory(options = {}) {
|
|
|
9131
9169
|
defaultHeaders: { ...options.defaultHeaders ?? {} },
|
|
9132
9170
|
driveBaseUrl,
|
|
9133
9171
|
fetch: fetchImpl,
|
|
9134
|
-
id
|
|
9172
|
+
id,
|
|
9173
|
+
multipart
|
|
9135
9174
|
}),
|
|
9136
9175
|
id
|
|
9137
9176
|
};
|
|
@@ -9165,6 +9204,7 @@ var OneDriveProvider = class {
|
|
|
9165
9204
|
driveBaseUrl: this.internals.driveBaseUrl,
|
|
9166
9205
|
fetch: this.internals.fetch,
|
|
9167
9206
|
id: this.internals.id,
|
|
9207
|
+
multipart: this.internals.multipart,
|
|
9168
9208
|
token
|
|
9169
9209
|
};
|
|
9170
9210
|
if (profile.timeoutMs !== void 0) sessionOptions.timeoutMs = profile.timeoutMs;
|
|
@@ -9268,12 +9308,19 @@ var OneDriveTransferOperations = class {
|
|
|
9268
9308
|
if (request.offset !== void 0 && request.offset > 0) {
|
|
9269
9309
|
throw new UnsupportedFeatureError({
|
|
9270
9310
|
details: { offset: request.offset },
|
|
9271
|
-
message: "OneDrive provider does not yet support
|
|
9311
|
+
message: "OneDrive provider does not yet support cross-attempt resume of upload sessions",
|
|
9272
9312
|
retryable: false
|
|
9273
9313
|
});
|
|
9274
9314
|
}
|
|
9275
9315
|
const normalized = normalizeRemotePath(request.endpoint.path);
|
|
9316
|
+
const multipart = this.options.multipart;
|
|
9276
9317
|
const buffered = await collectChunks3(request.content);
|
|
9318
|
+
if (!multipart.enabled || buffered.byteLength <= multipart.thresholdBytes) {
|
|
9319
|
+
return this.singleShotPut(request, normalized, buffered);
|
|
9320
|
+
}
|
|
9321
|
+
return this.writeUploadSession(request, normalized, buffered);
|
|
9322
|
+
}
|
|
9323
|
+
async singleShotPut(request, normalized, buffered) {
|
|
9277
9324
|
const url = `${this.options.driveBaseUrl}${itemSegment(normalized)}/content`;
|
|
9278
9325
|
const response = await graphFetch(this.options, "PUT", url, {
|
|
9279
9326
|
...request.signal !== void 0 ? { signal: request.signal } : {},
|
|
@@ -9293,6 +9340,77 @@ var OneDriveTransferOperations = class {
|
|
|
9293
9340
|
if (checksum !== void 0) result.checksum = checksum;
|
|
9294
9341
|
return result;
|
|
9295
9342
|
}
|
|
9343
|
+
async writeUploadSession(request, normalized, buffered) {
|
|
9344
|
+
const partSize = this.options.multipart.partSizeBytes;
|
|
9345
|
+
const total = buffered.byteLength;
|
|
9346
|
+
const sessionUrl = `${this.options.driveBaseUrl}${itemSegment(normalized)}/createUploadSession`;
|
|
9347
|
+
const initiate = await graphFetch(this.options, "POST", sessionUrl, {
|
|
9348
|
+
...request.signal !== void 0 ? { signal: request.signal } : {},
|
|
9349
|
+
body: new TextEncoder().encode(
|
|
9350
|
+
JSON.stringify({ item: { "@microsoft.graph.conflictBehavior": "replace" } })
|
|
9351
|
+
),
|
|
9352
|
+
extraHeaders: { "content-type": "application/json" }
|
|
9353
|
+
});
|
|
9354
|
+
if (!initiate.ok) {
|
|
9355
|
+
throw mapOneDriveResponseError(initiate, normalized, await safeReadText3(initiate));
|
|
9356
|
+
}
|
|
9357
|
+
const initiateBody = await initiate.json();
|
|
9358
|
+
const uploadUrl = initiateBody.uploadUrl;
|
|
9359
|
+
if (typeof uploadUrl !== "string" || uploadUrl === "") {
|
|
9360
|
+
throw new ConnectionError({
|
|
9361
|
+
details: { path: normalized },
|
|
9362
|
+
message: "OneDrive createUploadSession returned no uploadUrl",
|
|
9363
|
+
retryable: true
|
|
9364
|
+
});
|
|
9365
|
+
}
|
|
9366
|
+
let bytesTransferred = 0;
|
|
9367
|
+
let finalItem;
|
|
9368
|
+
try {
|
|
9369
|
+
while (bytesTransferred < total) {
|
|
9370
|
+
request.throwIfAborted();
|
|
9371
|
+
const chunkEnd = Math.min(bytesTransferred + partSize, total);
|
|
9372
|
+
const chunk = buffered.subarray(bytesTransferred, chunkEnd);
|
|
9373
|
+
const response = await graphSessionFetch(this.options, uploadUrl, {
|
|
9374
|
+
...request.signal !== void 0 ? { signal: request.signal } : {},
|
|
9375
|
+
body: chunk,
|
|
9376
|
+
extraHeaders: {
|
|
9377
|
+
"content-length": String(chunk.byteLength),
|
|
9378
|
+
"content-range": `bytes ${String(bytesTransferred)}-${String(chunkEnd - 1)}/${String(total)}`,
|
|
9379
|
+
"content-type": "application/octet-stream"
|
|
9380
|
+
}
|
|
9381
|
+
});
|
|
9382
|
+
if (response.status === 202) {
|
|
9383
|
+
bytesTransferred = chunkEnd;
|
|
9384
|
+
request.reportProgress(bytesTransferred, total);
|
|
9385
|
+
continue;
|
|
9386
|
+
}
|
|
9387
|
+
if (response.status === 200 || response.status === 201) {
|
|
9388
|
+
bytesTransferred = chunkEnd;
|
|
9389
|
+
request.reportProgress(bytesTransferred, total);
|
|
9390
|
+
finalItem = await response.json();
|
|
9391
|
+
break;
|
|
9392
|
+
}
|
|
9393
|
+
throw mapOneDriveResponseError(response, normalized, await safeReadText3(response));
|
|
9394
|
+
}
|
|
9395
|
+
} catch (error) {
|
|
9396
|
+
void graphSessionFetch(this.options, uploadUrl, { method: "DELETE" }).catch(() => void 0);
|
|
9397
|
+
throw error;
|
|
9398
|
+
}
|
|
9399
|
+
if (finalItem === void 0) {
|
|
9400
|
+
throw new ConnectionError({
|
|
9401
|
+
details: { path: normalized },
|
|
9402
|
+
message: "OneDrive upload session did not return a final DriveItem",
|
|
9403
|
+
retryable: true
|
|
9404
|
+
});
|
|
9405
|
+
}
|
|
9406
|
+
const result = {
|
|
9407
|
+
bytesTransferred,
|
|
9408
|
+
totalBytes: bytesTransferred
|
|
9409
|
+
};
|
|
9410
|
+
const checksum = preferHash(finalItem.file?.hashes);
|
|
9411
|
+
if (checksum !== void 0) result.checksum = checksum;
|
|
9412
|
+
return result;
|
|
9413
|
+
}
|
|
9296
9414
|
async fetchItem(normalized) {
|
|
9297
9415
|
const url = `${this.options.driveBaseUrl}${itemSegment(normalized)}`;
|
|
9298
9416
|
const response = await graphFetch(this.options, "GET", url);
|
|
@@ -9339,6 +9457,45 @@ async function graphFetch(options, method, url, fetchOptions = {}) {
|
|
|
9339
9457
|
if (timer !== void 0) clearTimeout(timer);
|
|
9340
9458
|
}
|
|
9341
9459
|
}
|
|
9460
|
+
async function graphSessionFetch(options, uploadUrl, fetchOptions = {}) {
|
|
9461
|
+
const headers = {
|
|
9462
|
+
...fetchOptions.extraHeaders ?? {}
|
|
9463
|
+
};
|
|
9464
|
+
const init = { headers, method: fetchOptions.method ?? "PUT" };
|
|
9465
|
+
if (fetchOptions.body !== void 0) {
|
|
9466
|
+
init.body = fetchOptions.body;
|
|
9467
|
+
}
|
|
9468
|
+
const controller = new AbortController();
|
|
9469
|
+
const upstream = fetchOptions.signal ?? null;
|
|
9470
|
+
if (upstream !== null) {
|
|
9471
|
+
if (upstream.aborted) controller.abort(upstream.reason);
|
|
9472
|
+
else upstream.addEventListener("abort", () => controller.abort(upstream.reason));
|
|
9473
|
+
}
|
|
9474
|
+
let timer;
|
|
9475
|
+
if (options.timeoutMs !== void 0 && options.timeoutMs > 0) {
|
|
9476
|
+
timer = setTimeout(
|
|
9477
|
+
() => controller.abort(new Error("OneDrive upload session request timed out")),
|
|
9478
|
+
options.timeoutMs
|
|
9479
|
+
);
|
|
9480
|
+
}
|
|
9481
|
+
try {
|
|
9482
|
+
return await options.fetch(uploadUrl, { ...init, signal: controller.signal });
|
|
9483
|
+
} catch (error) {
|
|
9484
|
+
const safeUrl = redactSessionUrl(uploadUrl);
|
|
9485
|
+
throw new ConnectionError({
|
|
9486
|
+
cause: error,
|
|
9487
|
+
details: { url: safeUrl },
|
|
9488
|
+
message: `OneDrive upload session request to ${safeUrl} failed`,
|
|
9489
|
+
retryable: true
|
|
9490
|
+
});
|
|
9491
|
+
} finally {
|
|
9492
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
9493
|
+
}
|
|
9494
|
+
}
|
|
9495
|
+
function redactSessionUrl(url) {
|
|
9496
|
+
const queryStart = url.indexOf("?");
|
|
9497
|
+
return queryStart === -1 ? url : `${url.slice(0, queryStart)}?<redacted>`;
|
|
9498
|
+
}
|
|
9342
9499
|
function mapOneDriveResponseError(response, contextPath, bodyText) {
|
|
9343
9500
|
const details = {
|
|
9344
9501
|
bodyText: bodyText.slice(0, 500),
|
|
@@ -9460,8 +9617,12 @@ async function collectChunks3(source) {
|
|
|
9460
9617
|
}
|
|
9461
9618
|
|
|
9462
9619
|
// src/providers/cloud/AzureBlobProvider.ts
|
|
9620
|
+
import { randomBytes as cryptoRandomBytes } from "crypto";
|
|
9463
9621
|
var AZURE_BLOB_API_VERSION = "2023-11-03";
|
|
9464
9622
|
var AZURE_CHECKSUM_CAPABILITIES = ["md5"];
|
|
9623
|
+
var DEFAULT_AZURE_PART_SIZE = 8 * 1024 * 1024;
|
|
9624
|
+
var DEFAULT_AZURE_THRESHOLD = 8 * 1024 * 1024;
|
|
9625
|
+
var AZURE_MAX_PART_SIZE = 4e3 * 1024 * 1024;
|
|
9465
9626
|
function createAzureBlobProviderFactory(options) {
|
|
9466
9627
|
if (typeof options.container !== "string" || options.container === "") {
|
|
9467
9628
|
throw new ConfigurationError({
|
|
@@ -9479,6 +9640,37 @@ function createAzureBlobProviderFactory(options) {
|
|
|
9479
9640
|
}
|
|
9480
9641
|
const endpoint = resolveAzureEndpoint(options);
|
|
9481
9642
|
const apiVersion = options.apiVersion ?? AZURE_BLOB_API_VERSION;
|
|
9643
|
+
const multipartEnabled = options.multipart?.enabled ?? true;
|
|
9644
|
+
const partSizeBytes = options.multipart?.partSizeBytes ?? DEFAULT_AZURE_PART_SIZE;
|
|
9645
|
+
const thresholdBytes = options.multipart?.thresholdBytes ?? DEFAULT_AZURE_THRESHOLD;
|
|
9646
|
+
if (multipartEnabled) {
|
|
9647
|
+
if (!Number.isInteger(partSizeBytes) || partSizeBytes <= 0) {
|
|
9648
|
+
throw new ConfigurationError({
|
|
9649
|
+
details: { partSizeBytes },
|
|
9650
|
+
message: "AzureBlobMultipartOptions.partSizeBytes must be a positive integer",
|
|
9651
|
+
retryable: false
|
|
9652
|
+
});
|
|
9653
|
+
}
|
|
9654
|
+
if (partSizeBytes > AZURE_MAX_PART_SIZE) {
|
|
9655
|
+
throw new ConfigurationError({
|
|
9656
|
+
details: { maxBytes: AZURE_MAX_PART_SIZE, partSizeBytes },
|
|
9657
|
+
message: `AzureBlobMultipartOptions.partSizeBytes must not exceed ${String(AZURE_MAX_PART_SIZE)} bytes (4000 MiB)`,
|
|
9658
|
+
retryable: false
|
|
9659
|
+
});
|
|
9660
|
+
}
|
|
9661
|
+
if (!Number.isInteger(thresholdBytes) || thresholdBytes < 0) {
|
|
9662
|
+
throw new ConfigurationError({
|
|
9663
|
+
details: { thresholdBytes },
|
|
9664
|
+
message: "AzureBlobMultipartOptions.thresholdBytes must be a non-negative integer",
|
|
9665
|
+
retryable: false
|
|
9666
|
+
});
|
|
9667
|
+
}
|
|
9668
|
+
}
|
|
9669
|
+
const multipart = {
|
|
9670
|
+
enabled: multipartEnabled,
|
|
9671
|
+
partSizeBytes,
|
|
9672
|
+
thresholdBytes
|
|
9673
|
+
};
|
|
9482
9674
|
const capabilities = {
|
|
9483
9675
|
atomicRename: false,
|
|
9484
9676
|
authentication: ["token", "oauth"],
|
|
@@ -9488,13 +9680,17 @@ function createAzureBlobProviderFactory(options) {
|
|
|
9488
9680
|
list: true,
|
|
9489
9681
|
maxConcurrency: 4,
|
|
9490
9682
|
metadata: ["modifiedAt", "uniqueId"],
|
|
9491
|
-
notes: [
|
|
9492
|
-
|
|
9683
|
+
notes: multipartEnabled ? [
|
|
9684
|
+
`Azure Blob staged-block upload enabled by default (partSize=${String(multipart.partSizeBytes)}B, threshold=${String(multipart.thresholdBytes)}B).`,
|
|
9685
|
+
"Payloads at or below the threshold automatically fall back to single-shot block-blob PUT.",
|
|
9686
|
+
"Pass `multipart: { enabled: false }` to force the legacy single-shot behaviour."
|
|
9687
|
+
] : [
|
|
9688
|
+
"Azure Blob provider performs single-shot block-blob uploads via PUT; entire object is buffered in memory before transmission."
|
|
9493
9689
|
],
|
|
9494
9690
|
provider: id,
|
|
9495
9691
|
readStream: true,
|
|
9496
9692
|
resumeDownload: true,
|
|
9497
|
-
resumeUpload:
|
|
9693
|
+
resumeUpload: multipartEnabled,
|
|
9498
9694
|
serverSideCopy: false,
|
|
9499
9695
|
serverSideMove: false,
|
|
9500
9696
|
stat: true,
|
|
@@ -9511,6 +9707,7 @@ function createAzureBlobProviderFactory(options) {
|
|
|
9511
9707
|
endpoint,
|
|
9512
9708
|
fetch: fetchImpl,
|
|
9513
9709
|
id,
|
|
9710
|
+
multipart,
|
|
9514
9711
|
...options.sasToken !== void 0 ? { sasToken: options.sasToken } : {}
|
|
9515
9712
|
}),
|
|
9516
9713
|
id
|
|
@@ -9556,7 +9753,8 @@ var AzureBlobProvider = class {
|
|
|
9556
9753
|
defaultHeaders: this.internals.defaultHeaders,
|
|
9557
9754
|
endpoint: this.internals.endpoint,
|
|
9558
9755
|
fetch: this.internals.fetch,
|
|
9559
|
-
id: this.internals.id
|
|
9756
|
+
id: this.internals.id,
|
|
9757
|
+
multipart: this.internals.multipart
|
|
9560
9758
|
};
|
|
9561
9759
|
if (bearerToken !== void 0) sessionOptions.bearerToken = bearerToken;
|
|
9562
9760
|
if (this.internals.sasToken !== void 0) sessionOptions.sasToken = this.internals.sasToken;
|
|
@@ -9691,12 +9889,19 @@ var AzureBlobTransferOperations = class {
|
|
|
9691
9889
|
if (request.offset !== void 0 && request.offset > 0) {
|
|
9692
9890
|
throw new UnsupportedFeatureError({
|
|
9693
9891
|
details: { offset: request.offset },
|
|
9694
|
-
message: "Azure Blob provider does not yet support staged-block
|
|
9892
|
+
message: "Azure Blob provider does not yet support cross-attempt resume of staged-block uploads",
|
|
9695
9893
|
retryable: false
|
|
9696
9894
|
});
|
|
9697
9895
|
}
|
|
9698
9896
|
const normalized = normalizeRemotePath(request.endpoint.path);
|
|
9699
|
-
const
|
|
9897
|
+
const multipart = this.options.multipart;
|
|
9898
|
+
if (!multipart.enabled) {
|
|
9899
|
+
const buffered = await collectChunks4(request.content);
|
|
9900
|
+
return this.singleShotPut(request, normalized, buffered);
|
|
9901
|
+
}
|
|
9902
|
+
return this.writeStagedBlocks(request, normalized);
|
|
9903
|
+
}
|
|
9904
|
+
async singleShotPut(request, normalized, buffered) {
|
|
9700
9905
|
const url = buildBlobUrl(this.options, normalized);
|
|
9701
9906
|
const response = await azureFetch(this.options, "PUT", url, {
|
|
9702
9907
|
...request.signal !== void 0 ? { signal: request.signal } : {},
|
|
@@ -9718,6 +9923,92 @@ var AzureBlobTransferOperations = class {
|
|
|
9718
9923
|
if (md5 !== null && md5 !== "") result.checksum = md5;
|
|
9719
9924
|
return result;
|
|
9720
9925
|
}
|
|
9926
|
+
async writeStagedBlocks(request, normalized) {
|
|
9927
|
+
const multipart = this.options.multipart;
|
|
9928
|
+
const partSize = multipart.partSizeBytes;
|
|
9929
|
+
const iterator = request.content[Symbol.asyncIterator]();
|
|
9930
|
+
const uploadNonce = generateUploadNonce();
|
|
9931
|
+
const initialBuffer = [];
|
|
9932
|
+
let initialSize = 0;
|
|
9933
|
+
while (initialSize <= multipart.thresholdBytes) {
|
|
9934
|
+
const next = await iterator.next();
|
|
9935
|
+
if (next.done === true) break;
|
|
9936
|
+
const chunk = next.value;
|
|
9937
|
+
if (chunk.byteLength === 0) continue;
|
|
9938
|
+
initialBuffer.push(chunk);
|
|
9939
|
+
initialSize += chunk.byteLength;
|
|
9940
|
+
}
|
|
9941
|
+
if (initialSize <= multipart.thresholdBytes) {
|
|
9942
|
+
return this.singleShotPut(request, normalized, concatChunks2(initialBuffer, initialSize));
|
|
9943
|
+
}
|
|
9944
|
+
const blockIds = [];
|
|
9945
|
+
let bytesTransferred = 0;
|
|
9946
|
+
let partNumber = 1;
|
|
9947
|
+
let buffer = [...initialBuffer];
|
|
9948
|
+
let bufferSize = initialSize;
|
|
9949
|
+
const flushBlocks = async (final) => {
|
|
9950
|
+
while (bufferSize >= partSize || final && bufferSize > 0) {
|
|
9951
|
+
const take = Math.min(bufferSize, partSize);
|
|
9952
|
+
const sliced = sliceFromBuffers(buffer, take);
|
|
9953
|
+
buffer = sliced.remaining;
|
|
9954
|
+
bufferSize -= sliced.bytes.byteLength;
|
|
9955
|
+
const blockId = encodeBlockId(uploadNonce, partNumber);
|
|
9956
|
+
const blockUrl = buildBlobUrl(this.options, normalized, {
|
|
9957
|
+
blockid: blockId,
|
|
9958
|
+
comp: "block"
|
|
9959
|
+
});
|
|
9960
|
+
const response = await azureFetch(this.options, "PUT", blockUrl, {
|
|
9961
|
+
...request.signal !== void 0 ? { signal: request.signal } : {},
|
|
9962
|
+
body: sliced.bytes,
|
|
9963
|
+
extraHeaders: { "content-type": "application/octet-stream" }
|
|
9964
|
+
});
|
|
9965
|
+
if (!response.ok) {
|
|
9966
|
+
throw mapAzureResponseError(response, normalized, await safeReadText4(response));
|
|
9967
|
+
}
|
|
9968
|
+
blockIds.push(blockId);
|
|
9969
|
+
bytesTransferred += sliced.bytes.byteLength;
|
|
9970
|
+
request.reportProgress(bytesTransferred, void 0);
|
|
9971
|
+
partNumber += 1;
|
|
9972
|
+
}
|
|
9973
|
+
};
|
|
9974
|
+
await flushBlocks(false);
|
|
9975
|
+
while (true) {
|
|
9976
|
+
request.throwIfAborted();
|
|
9977
|
+
const next = await iterator.next();
|
|
9978
|
+
if (next.done === true) break;
|
|
9979
|
+
if (next.value.byteLength === 0) continue;
|
|
9980
|
+
buffer.push(next.value);
|
|
9981
|
+
bufferSize += next.value.byteLength;
|
|
9982
|
+
await flushBlocks(false);
|
|
9983
|
+
}
|
|
9984
|
+
await flushBlocks(true);
|
|
9985
|
+
if (blockIds.length === 0) {
|
|
9986
|
+
throw new ConnectionError({
|
|
9987
|
+
message: "Azure Blob staged-block upload completed with zero blocks",
|
|
9988
|
+
retryable: false
|
|
9989
|
+
});
|
|
9990
|
+
}
|
|
9991
|
+
const commitUrl = buildBlobUrl(this.options, normalized, { comp: "blocklist" });
|
|
9992
|
+
const xmlBody = buildBlockListXml(blockIds);
|
|
9993
|
+
const commitResponse = await azureFetch(this.options, "PUT", commitUrl, {
|
|
9994
|
+
...request.signal !== void 0 ? { signal: request.signal } : {},
|
|
9995
|
+
body: new TextEncoder().encode(xmlBody),
|
|
9996
|
+
extraHeaders: {
|
|
9997
|
+
"content-type": "application/xml",
|
|
9998
|
+
"x-ms-blob-content-type": "application/octet-stream"
|
|
9999
|
+
}
|
|
10000
|
+
});
|
|
10001
|
+
if (!commitResponse.ok) {
|
|
10002
|
+
throw mapAzureResponseError(commitResponse, normalized, await safeReadText4(commitResponse));
|
|
10003
|
+
}
|
|
10004
|
+
const result = {
|
|
10005
|
+
bytesTransferred,
|
|
10006
|
+
totalBytes: bytesTransferred
|
|
10007
|
+
};
|
|
10008
|
+
const md5 = commitResponse.headers.get("content-md5");
|
|
10009
|
+
if (md5 !== null && md5 !== "") result.checksum = md5;
|
|
10010
|
+
return result;
|
|
10011
|
+
}
|
|
9721
10012
|
};
|
|
9722
10013
|
async function azureFetch(options, method, url, fetchOptions = {}) {
|
|
9723
10014
|
const headers = {
|
|
@@ -9763,14 +10054,17 @@ function buildContainerUrl(options, params) {
|
|
|
9763
10054
|
appendSas(search, options.sasToken);
|
|
9764
10055
|
return `${options.endpoint}/${encodeURIComponent(options.container)}?${search.toString()}`;
|
|
9765
10056
|
}
|
|
9766
|
-
function buildBlobUrl(options, normalized) {
|
|
10057
|
+
function buildBlobUrl(options, normalized, extraParams) {
|
|
9767
10058
|
const blobPath = normalized.replace(/^\/+/u, "");
|
|
9768
10059
|
const encoded = blobPath.split("/").map((segment) => encodeURIComponent(segment)).join("/");
|
|
9769
10060
|
const base = `${options.endpoint}/${encodeURIComponent(options.container)}/${encoded}`;
|
|
9770
|
-
|
|
9771
|
-
|
|
10061
|
+
const search = new URLSearchParams();
|
|
10062
|
+
if (extraParams !== void 0) {
|
|
10063
|
+
for (const [k, v] of Object.entries(extraParams)) search.set(k, v);
|
|
9772
10064
|
}
|
|
9773
|
-
|
|
10065
|
+
appendSas(search, options.sasToken);
|
|
10066
|
+
const query = search.toString();
|
|
10067
|
+
return query === "" ? base : `${base}?${query}`;
|
|
9774
10068
|
}
|
|
9775
10069
|
function appendSas(search, sasToken) {
|
|
9776
10070
|
if (sasToken === void 0 || sasToken === "") return;
|
|
@@ -9922,11 +10216,63 @@ async function collectChunks4(source) {
|
|
|
9922
10216
|
}
|
|
9923
10217
|
return out;
|
|
9924
10218
|
}
|
|
10219
|
+
function concatChunks2(chunks, totalSize) {
|
|
10220
|
+
const out = new Uint8Array(totalSize);
|
|
10221
|
+
let offset = 0;
|
|
10222
|
+
for (const chunk of chunks) {
|
|
10223
|
+
out.set(chunk, offset);
|
|
10224
|
+
offset += chunk.byteLength;
|
|
10225
|
+
}
|
|
10226
|
+
return out;
|
|
10227
|
+
}
|
|
10228
|
+
function sliceFromBuffers(buffers, size) {
|
|
10229
|
+
const out = new Uint8Array(size);
|
|
10230
|
+
let offset = 0;
|
|
10231
|
+
let i = 0;
|
|
10232
|
+
while (offset < size && i < buffers.length) {
|
|
10233
|
+
const chunk = buffers[i];
|
|
10234
|
+
if (chunk === void 0) {
|
|
10235
|
+
i += 1;
|
|
10236
|
+
continue;
|
|
10237
|
+
}
|
|
10238
|
+
const remaining = size - offset;
|
|
10239
|
+
if (chunk.byteLength <= remaining) {
|
|
10240
|
+
out.set(chunk, offset);
|
|
10241
|
+
offset += chunk.byteLength;
|
|
10242
|
+
i += 1;
|
|
10243
|
+
} else {
|
|
10244
|
+
out.set(chunk.subarray(0, remaining), offset);
|
|
10245
|
+
const leftover = chunk.subarray(remaining);
|
|
10246
|
+
const next = buffers.slice(i + 1);
|
|
10247
|
+
next.unshift(leftover);
|
|
10248
|
+
return { bytes: out, remaining: next };
|
|
10249
|
+
}
|
|
10250
|
+
}
|
|
10251
|
+
return { bytes: out.subarray(0, offset), remaining: buffers.slice(i) };
|
|
10252
|
+
}
|
|
10253
|
+
function encodeBlockId(nonce, partNumber) {
|
|
10254
|
+
const padded = String(partNumber).padStart(9, "0");
|
|
10255
|
+
const raw = `${nonce}-${padded}`;
|
|
10256
|
+
return Buffer.from(raw, "utf8").toString("base64");
|
|
10257
|
+
}
|
|
10258
|
+
function generateUploadNonce() {
|
|
10259
|
+
return cryptoRandomBytes(4).toString("hex");
|
|
10260
|
+
}
|
|
10261
|
+
function buildBlockListXml(blockIds) {
|
|
10262
|
+
const items = blockIds.map((id) => `<Latest>${escapeXml(id)}</Latest>`).join("");
|
|
10263
|
+
return `<?xml version="1.0" encoding="utf-8"?><BlockList>${items}</BlockList>`;
|
|
10264
|
+
}
|
|
10265
|
+
function escapeXml(value) {
|
|
10266
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
10267
|
+
}
|
|
9925
10268
|
|
|
9926
10269
|
// src/providers/cloud/GcsProvider.ts
|
|
9927
10270
|
var GCS_JSON_API_BASE = "https://storage.googleapis.com/storage/v1";
|
|
9928
10271
|
var GCS_UPLOAD_API_BASE = "https://storage.googleapis.com/upload/storage/v1";
|
|
9929
10272
|
var GCS_CHECKSUM_CAPABILITIES = ["md5", "crc32c"];
|
|
10273
|
+
var GCS_CHUNK_ALIGNMENT = 256 * 1024;
|
|
10274
|
+
var DEFAULT_GCS_PART_SIZE = 8 * 1024 * 1024;
|
|
10275
|
+
var DEFAULT_GCS_THRESHOLD = 8 * 1024 * 1024;
|
|
9930
10276
|
function createGcsProviderFactory(options) {
|
|
9931
10277
|
if (typeof options.bucket !== "string" || options.bucket === "") {
|
|
9932
10278
|
throw new ConfigurationError({
|
|
@@ -9944,6 +10290,37 @@ function createGcsProviderFactory(options) {
|
|
|
9944
10290
|
}
|
|
9945
10291
|
const apiBaseUrl = (options.apiBaseUrl ?? GCS_JSON_API_BASE).replace(/\/+$/u, "");
|
|
9946
10292
|
const uploadBaseUrl = (options.uploadBaseUrl ?? GCS_UPLOAD_API_BASE).replace(/\/+$/u, "");
|
|
10293
|
+
const multipartEnabled = options.multipart?.enabled ?? true;
|
|
10294
|
+
const partSizeBytes = options.multipart?.partSizeBytes ?? DEFAULT_GCS_PART_SIZE;
|
|
10295
|
+
const thresholdBytes = options.multipart?.thresholdBytes ?? DEFAULT_GCS_THRESHOLD;
|
|
10296
|
+
if (multipartEnabled) {
|
|
10297
|
+
if (!Number.isInteger(partSizeBytes) || partSizeBytes <= 0) {
|
|
10298
|
+
throw new ConfigurationError({
|
|
10299
|
+
details: { partSizeBytes },
|
|
10300
|
+
message: "GcsMultipartOptions.partSizeBytes must be a positive integer",
|
|
10301
|
+
retryable: false
|
|
10302
|
+
});
|
|
10303
|
+
}
|
|
10304
|
+
if (partSizeBytes % GCS_CHUNK_ALIGNMENT !== 0) {
|
|
10305
|
+
throw new ConfigurationError({
|
|
10306
|
+
details: { partSizeBytes },
|
|
10307
|
+
message: `GCS multipart partSizeBytes must be a multiple of ${String(GCS_CHUNK_ALIGNMENT)} bytes (256 KiB)`,
|
|
10308
|
+
retryable: false
|
|
10309
|
+
});
|
|
10310
|
+
}
|
|
10311
|
+
if (!Number.isInteger(thresholdBytes) || thresholdBytes < 0) {
|
|
10312
|
+
throw new ConfigurationError({
|
|
10313
|
+
details: { thresholdBytes },
|
|
10314
|
+
message: "GcsMultipartOptions.thresholdBytes must be a non-negative integer",
|
|
10315
|
+
retryable: false
|
|
10316
|
+
});
|
|
10317
|
+
}
|
|
10318
|
+
}
|
|
10319
|
+
const multipart = {
|
|
10320
|
+
enabled: multipartEnabled,
|
|
10321
|
+
partSizeBytes,
|
|
10322
|
+
thresholdBytes
|
|
10323
|
+
};
|
|
9947
10324
|
const capabilities = {
|
|
9948
10325
|
atomicRename: false,
|
|
9949
10326
|
authentication: ["token", "oauth"],
|
|
@@ -9953,13 +10330,17 @@ function createGcsProviderFactory(options) {
|
|
|
9953
10330
|
list: true,
|
|
9954
10331
|
maxConcurrency: 4,
|
|
9955
10332
|
metadata: ["modifiedAt", "createdAt", "uniqueId"],
|
|
9956
|
-
notes: [
|
|
9957
|
-
|
|
10333
|
+
notes: multipartEnabled ? [
|
|
10334
|
+
`GCS resumable-upload session enabled by default (partSize=${String(multipart.partSizeBytes)}B, threshold=${String(multipart.thresholdBytes)}B).`,
|
|
10335
|
+
"Payloads at or below the threshold automatically fall back to single-shot uploadType=media POST.",
|
|
10336
|
+
"Pass `multipart: { enabled: false }` to force the legacy single-shot behaviour."
|
|
10337
|
+
] : [
|
|
10338
|
+
"GCS provider performs single-shot media uploads via /upload?uploadType=media; resumable upload sessions are disabled."
|
|
9958
10339
|
],
|
|
9959
10340
|
provider: id,
|
|
9960
10341
|
readStream: true,
|
|
9961
10342
|
resumeDownload: true,
|
|
9962
|
-
resumeUpload:
|
|
10343
|
+
resumeUpload: multipartEnabled,
|
|
9963
10344
|
serverSideCopy: false,
|
|
9964
10345
|
serverSideMove: false,
|
|
9965
10346
|
stat: true,
|
|
@@ -9975,6 +10356,7 @@ function createGcsProviderFactory(options) {
|
|
|
9975
10356
|
defaultHeaders: { ...options.defaultHeaders ?? {} },
|
|
9976
10357
|
fetch: fetchImpl,
|
|
9977
10358
|
id,
|
|
10359
|
+
multipart,
|
|
9978
10360
|
uploadBaseUrl
|
|
9979
10361
|
}),
|
|
9980
10362
|
id
|
|
@@ -10010,6 +10392,7 @@ var GcsProvider = class {
|
|
|
10010
10392
|
defaultHeaders: this.internals.defaultHeaders,
|
|
10011
10393
|
fetch: this.internals.fetch,
|
|
10012
10394
|
id: this.internals.id,
|
|
10395
|
+
multipart: this.internals.multipart,
|
|
10013
10396
|
token,
|
|
10014
10397
|
uploadBaseUrl: this.internals.uploadBaseUrl
|
|
10015
10398
|
};
|
|
@@ -10131,13 +10514,20 @@ var GcsTransferOperations = class {
|
|
|
10131
10514
|
if (request.offset !== void 0 && request.offset > 0) {
|
|
10132
10515
|
throw new UnsupportedFeatureError({
|
|
10133
10516
|
details: { offset: request.offset },
|
|
10134
|
-
message: "GCS provider does not yet support
|
|
10517
|
+
message: "GCS provider does not yet support cross-attempt resume of upload sessions",
|
|
10135
10518
|
retryable: false
|
|
10136
10519
|
});
|
|
10137
10520
|
}
|
|
10138
10521
|
const normalized = normalizeRemotePath(request.endpoint.path);
|
|
10139
10522
|
const objectName = toGcsObjectName(normalized);
|
|
10140
|
-
const
|
|
10523
|
+
const multipart = this.options.multipart;
|
|
10524
|
+
if (!multipart.enabled) {
|
|
10525
|
+
const buffered = await collectChunks5(request.content);
|
|
10526
|
+
return this.singleShotMedia(request, normalized, objectName, buffered);
|
|
10527
|
+
}
|
|
10528
|
+
return this.writeResumableSession(request, normalized, objectName);
|
|
10529
|
+
}
|
|
10530
|
+
async singleShotMedia(request, normalized, objectName, buffered) {
|
|
10141
10531
|
const url = `${this.options.uploadBaseUrl}/b/${encodeURIComponent(this.options.bucket)}/o?uploadType=media&name=${encodeURIComponent(objectName)}`;
|
|
10142
10532
|
const response = await gcsFetch(this.options, "POST", url, {
|
|
10143
10533
|
...request.signal !== void 0 ? { signal: request.signal } : {},
|
|
@@ -10156,6 +10546,124 @@ var GcsTransferOperations = class {
|
|
|
10156
10546
|
if (typeof item.md5Hash === "string" && item.md5Hash !== "") result.checksum = item.md5Hash;
|
|
10157
10547
|
return result;
|
|
10158
10548
|
}
|
|
10549
|
+
async writeResumableSession(request, normalized, objectName) {
|
|
10550
|
+
const multipart = this.options.multipart;
|
|
10551
|
+
const partSize = multipart.partSizeBytes;
|
|
10552
|
+
const iterator = request.content[Symbol.asyncIterator]();
|
|
10553
|
+
const initialBuffer = [];
|
|
10554
|
+
let initialSize = 0;
|
|
10555
|
+
while (initialSize <= multipart.thresholdBytes) {
|
|
10556
|
+
const next = await iterator.next();
|
|
10557
|
+
if (next.done === true) break;
|
|
10558
|
+
const chunk = next.value;
|
|
10559
|
+
if (chunk.byteLength === 0) continue;
|
|
10560
|
+
initialBuffer.push(chunk);
|
|
10561
|
+
initialSize += chunk.byteLength;
|
|
10562
|
+
}
|
|
10563
|
+
if (initialSize <= multipart.thresholdBytes) {
|
|
10564
|
+
return this.singleShotMedia(
|
|
10565
|
+
request,
|
|
10566
|
+
normalized,
|
|
10567
|
+
objectName,
|
|
10568
|
+
concatChunks3(initialBuffer, initialSize)
|
|
10569
|
+
);
|
|
10570
|
+
}
|
|
10571
|
+
const initiateUrl = `${this.options.uploadBaseUrl}/b/${encodeURIComponent(this.options.bucket)}/o?uploadType=resumable&name=${encodeURIComponent(objectName)}`;
|
|
10572
|
+
const initiateResponse = await gcsFetch(this.options, "POST", initiateUrl, {
|
|
10573
|
+
...request.signal !== void 0 ? { signal: request.signal } : {},
|
|
10574
|
+
body: new TextEncoder().encode("{}"),
|
|
10575
|
+
extraHeaders: {
|
|
10576
|
+
"content-type": "application/json; charset=UTF-8",
|
|
10577
|
+
"x-upload-content-type": "application/octet-stream"
|
|
10578
|
+
}
|
|
10579
|
+
});
|
|
10580
|
+
if (!initiateResponse.ok) {
|
|
10581
|
+
throw mapGcsResponseError(initiateResponse, normalized, await safeReadText5(initiateResponse));
|
|
10582
|
+
}
|
|
10583
|
+
const sessionUri = initiateResponse.headers.get("location");
|
|
10584
|
+
if (sessionUri === null || sessionUri === "") {
|
|
10585
|
+
throw new ConnectionError({
|
|
10586
|
+
details: { path: normalized },
|
|
10587
|
+
message: "GCS resumable session initiation returned no Location header",
|
|
10588
|
+
retryable: true
|
|
10589
|
+
});
|
|
10590
|
+
}
|
|
10591
|
+
let bytesTransferred = 0;
|
|
10592
|
+
let buffer = [...initialBuffer];
|
|
10593
|
+
let bufferSize = initialSize;
|
|
10594
|
+
let sourceExhausted = false;
|
|
10595
|
+
let finalItem;
|
|
10596
|
+
const flushChunks = async (final) => {
|
|
10597
|
+
while (bufferSize >= partSize || final && bufferSize > 0) {
|
|
10598
|
+
const take = final ? bufferSize : partSize;
|
|
10599
|
+
const sliced = sliceFromBuffers2(buffer, take);
|
|
10600
|
+
buffer = sliced.remaining;
|
|
10601
|
+
bufferSize -= sliced.bytes.byteLength;
|
|
10602
|
+
const chunkStart = bytesTransferred;
|
|
10603
|
+
const chunkEnd = chunkStart + sliced.bytes.byteLength - 1;
|
|
10604
|
+
const totalRange = final ? String(chunkEnd + 1) : "*";
|
|
10605
|
+
const headers = {
|
|
10606
|
+
"content-length": String(sliced.bytes.byteLength),
|
|
10607
|
+
"content-range": `bytes ${String(chunkStart)}-${String(chunkEnd)}/${totalRange}`,
|
|
10608
|
+
"content-type": "application/octet-stream"
|
|
10609
|
+
};
|
|
10610
|
+
const response = await gcsSessionFetch(this.options, sessionUri, {
|
|
10611
|
+
...request.signal !== void 0 ? { signal: request.signal } : {},
|
|
10612
|
+
body: sliced.bytes,
|
|
10613
|
+
extraHeaders: headers
|
|
10614
|
+
});
|
|
10615
|
+
if (response.status === 308) {
|
|
10616
|
+
bytesTransferred += sliced.bytes.byteLength;
|
|
10617
|
+
request.reportProgress(bytesTransferred, void 0);
|
|
10618
|
+
continue;
|
|
10619
|
+
}
|
|
10620
|
+
if (response.status === 200 || response.status === 201) {
|
|
10621
|
+
bytesTransferred += sliced.bytes.byteLength;
|
|
10622
|
+
request.reportProgress(bytesTransferred, bytesTransferred);
|
|
10623
|
+
finalItem = await response.json();
|
|
10624
|
+
return;
|
|
10625
|
+
}
|
|
10626
|
+
throw mapGcsResponseError(response, normalized, await safeReadText5(response));
|
|
10627
|
+
}
|
|
10628
|
+
};
|
|
10629
|
+
try {
|
|
10630
|
+
await flushChunks(false);
|
|
10631
|
+
while (!sourceExhausted) {
|
|
10632
|
+
request.throwIfAborted();
|
|
10633
|
+
const next = await iterator.next();
|
|
10634
|
+
if (next.done === true) {
|
|
10635
|
+
sourceExhausted = true;
|
|
10636
|
+
break;
|
|
10637
|
+
}
|
|
10638
|
+
if (next.value.byteLength === 0) continue;
|
|
10639
|
+
buffer.push(next.value);
|
|
10640
|
+
bufferSize += next.value.byteLength;
|
|
10641
|
+
await flushChunks(false);
|
|
10642
|
+
if (finalItem !== void 0) break;
|
|
10643
|
+
}
|
|
10644
|
+
if (finalItem === void 0) {
|
|
10645
|
+
await flushChunks(true);
|
|
10646
|
+
}
|
|
10647
|
+
} catch (error) {
|
|
10648
|
+
void gcsSessionFetch(this.options, sessionUri, { method: "DELETE" }).catch(() => void 0);
|
|
10649
|
+
throw error;
|
|
10650
|
+
}
|
|
10651
|
+
if (finalItem === void 0) {
|
|
10652
|
+
throw new ConnectionError({
|
|
10653
|
+
details: { path: normalized },
|
|
10654
|
+
message: "GCS resumable upload did not return a final object",
|
|
10655
|
+
retryable: true
|
|
10656
|
+
});
|
|
10657
|
+
}
|
|
10658
|
+
const result = {
|
|
10659
|
+
bytesTransferred,
|
|
10660
|
+
totalBytes: bytesTransferred
|
|
10661
|
+
};
|
|
10662
|
+
if (typeof finalItem.md5Hash === "string" && finalItem.md5Hash !== "") {
|
|
10663
|
+
result.checksum = finalItem.md5Hash;
|
|
10664
|
+
}
|
|
10665
|
+
return result;
|
|
10666
|
+
}
|
|
10159
10667
|
};
|
|
10160
10668
|
async function gcsFetch(options, method, url, fetchOptions = {}) {
|
|
10161
10669
|
const headers = {
|
|
@@ -10314,6 +10822,81 @@ async function collectChunks5(source) {
|
|
|
10314
10822
|
}
|
|
10315
10823
|
return out;
|
|
10316
10824
|
}
|
|
10825
|
+
function concatChunks3(chunks, totalSize) {
|
|
10826
|
+
const out = new Uint8Array(totalSize);
|
|
10827
|
+
let offset = 0;
|
|
10828
|
+
for (const chunk of chunks) {
|
|
10829
|
+
out.set(chunk, offset);
|
|
10830
|
+
offset += chunk.byteLength;
|
|
10831
|
+
}
|
|
10832
|
+
return out;
|
|
10833
|
+
}
|
|
10834
|
+
function sliceFromBuffers2(buffers, size) {
|
|
10835
|
+
const out = new Uint8Array(size);
|
|
10836
|
+
let offset = 0;
|
|
10837
|
+
let i = 0;
|
|
10838
|
+
while (offset < size && i < buffers.length) {
|
|
10839
|
+
const chunk = buffers[i];
|
|
10840
|
+
if (chunk === void 0) {
|
|
10841
|
+
i += 1;
|
|
10842
|
+
continue;
|
|
10843
|
+
}
|
|
10844
|
+
const remaining = size - offset;
|
|
10845
|
+
if (chunk.byteLength <= remaining) {
|
|
10846
|
+
out.set(chunk, offset);
|
|
10847
|
+
offset += chunk.byteLength;
|
|
10848
|
+
i += 1;
|
|
10849
|
+
} else {
|
|
10850
|
+
out.set(chunk.subarray(0, remaining), offset);
|
|
10851
|
+
const leftover = chunk.subarray(remaining);
|
|
10852
|
+
const next = buffers.slice(i + 1);
|
|
10853
|
+
next.unshift(leftover);
|
|
10854
|
+
return { bytes: out, remaining: next };
|
|
10855
|
+
}
|
|
10856
|
+
}
|
|
10857
|
+
return { bytes: out.subarray(0, offset), remaining: buffers.slice(i) };
|
|
10858
|
+
}
|
|
10859
|
+
async function gcsSessionFetch(options, sessionUri, fetchOptions = {}) {
|
|
10860
|
+
const headers = {
|
|
10861
|
+
...options.defaultHeaders,
|
|
10862
|
+
...fetchOptions.extraHeaders ?? {},
|
|
10863
|
+
authorization: `Bearer ${options.token}`
|
|
10864
|
+
};
|
|
10865
|
+
const init = { headers, method: fetchOptions.method ?? "PUT" };
|
|
10866
|
+
if (fetchOptions.body !== void 0) {
|
|
10867
|
+
init.body = fetchOptions.body;
|
|
10868
|
+
}
|
|
10869
|
+
const controller = new AbortController();
|
|
10870
|
+
const upstream = fetchOptions.signal ?? null;
|
|
10871
|
+
if (upstream !== null) {
|
|
10872
|
+
if (upstream.aborted) controller.abort(upstream.reason);
|
|
10873
|
+
else upstream.addEventListener("abort", () => controller.abort(upstream.reason));
|
|
10874
|
+
}
|
|
10875
|
+
let timer;
|
|
10876
|
+
if (options.timeoutMs !== void 0 && options.timeoutMs > 0) {
|
|
10877
|
+
timer = setTimeout(
|
|
10878
|
+
() => controller.abort(new Error("GCS resumable session request timed out")),
|
|
10879
|
+
options.timeoutMs
|
|
10880
|
+
);
|
|
10881
|
+
}
|
|
10882
|
+
try {
|
|
10883
|
+
return await options.fetch(sessionUri, { ...init, signal: controller.signal });
|
|
10884
|
+
} catch (error) {
|
|
10885
|
+
const safeUrl = redactSessionUrl2(sessionUri);
|
|
10886
|
+
throw new ConnectionError({
|
|
10887
|
+
cause: error,
|
|
10888
|
+
details: { url: safeUrl },
|
|
10889
|
+
message: `GCS resumable session request to ${safeUrl} failed`,
|
|
10890
|
+
retryable: true
|
|
10891
|
+
});
|
|
10892
|
+
} finally {
|
|
10893
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
10894
|
+
}
|
|
10895
|
+
}
|
|
10896
|
+
function redactSessionUrl2(url) {
|
|
10897
|
+
const queryStart = url.indexOf("?");
|
|
10898
|
+
return queryStart === -1 ? url : `${url.slice(0, queryStart)}?<redacted>`;
|
|
10899
|
+
}
|
|
10317
10900
|
|
|
10318
10901
|
// src/providers/local/LocalProvider.ts
|
|
10319
10902
|
import { createReadStream } from "fs";
|
|
@@ -10534,9 +11117,9 @@ async function collectTransferContent(request) {
|
|
|
10534
11117
|
byteLength += clonedChunk.byteLength;
|
|
10535
11118
|
request.reportProgress(byteLength, request.totalBytes);
|
|
10536
11119
|
}
|
|
10537
|
-
return
|
|
11120
|
+
return concatChunks4(chunks, byteLength);
|
|
10538
11121
|
}
|
|
10539
|
-
function
|
|
11122
|
+
function concatChunks4(chunks, byteLength) {
|
|
10540
11123
|
const content = new Uint8Array(byteLength);
|
|
10541
11124
|
let offset = 0;
|
|
10542
11125
|
for (const chunk of chunks) {
|
|
@@ -11110,9 +11693,9 @@ async function collectTransferContent2(request) {
|
|
|
11110
11693
|
byteLength += clonedChunk.byteLength;
|
|
11111
11694
|
request.reportProgress(byteLength, request.totalBytes);
|
|
11112
11695
|
}
|
|
11113
|
-
return
|
|
11696
|
+
return concatChunks5(chunks, byteLength);
|
|
11114
11697
|
}
|
|
11115
|
-
function
|
|
11698
|
+
function concatChunks5(chunks, byteLength) {
|
|
11116
11699
|
const content = new Uint8Array(byteLength);
|
|
11117
11700
|
let offset = 0;
|
|
11118
11701
|
for (const chunk of chunks) {
|
|
@@ -12285,7 +12868,7 @@ var S3TransferOperations = class {
|
|
|
12285
12868
|
const flushPart = async (final) => {
|
|
12286
12869
|
while (bufferSize >= partSize || final && bufferSize > 0) {
|
|
12287
12870
|
const take = final ? bufferSize : partSize;
|
|
12288
|
-
const partBytes =
|
|
12871
|
+
const partBytes = sliceFromBuffers3(buffer, take);
|
|
12289
12872
|
buffer = partBytes.remaining;
|
|
12290
12873
|
bufferSize -= partBytes.bytes.byteLength;
|
|
12291
12874
|
const partUrl = new URL(objectUrl.toString());
|
|
@@ -12474,7 +13057,7 @@ function concat(chunks, totalSize) {
|
|
|
12474
13057
|
}
|
|
12475
13058
|
return out;
|
|
12476
13059
|
}
|
|
12477
|
-
function
|
|
13060
|
+
function sliceFromBuffers3(buffers, size) {
|
|
12478
13061
|
const out = new Uint8Array(size);
|
|
12479
13062
|
let offset = 0;
|
|
12480
13063
|
let i = 0;
|
|
@@ -12506,11 +13089,11 @@ async function abortMultipart(options, objectUrl, uploadId) {
|
|
|
12506
13089
|
}
|
|
12507
13090
|
function buildCompleteMultipartBody(parts) {
|
|
12508
13091
|
const partsXml = parts.map(
|
|
12509
|
-
(part) => `<Part><PartNumber>${String(part.partNumber)}</PartNumber><ETag>${
|
|
13092
|
+
(part) => `<Part><PartNumber>${String(part.partNumber)}</PartNumber><ETag>${escapeXml2(part.etag)}</ETag></Part>`
|
|
12510
13093
|
).join("");
|
|
12511
13094
|
return `<?xml version="1.0" encoding="UTF-8"?><CompleteMultipartUpload>${partsXml}</CompleteMultipartUpload>`;
|
|
12512
13095
|
}
|
|
12513
|
-
function
|
|
13096
|
+
function escapeXml2(value) {
|
|
12514
13097
|
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
12515
13098
|
}
|
|
12516
13099
|
function parseListObjectsV2(xml, prefix) {
|
|
@@ -12659,8 +13242,8 @@ function formatCapabilityMatrixMarkdown(matrix = getBuiltinCapabilityMatrix()) {
|
|
|
12659
13242
|
const c = entry.capabilities;
|
|
12660
13243
|
const yesNo = (value) => value ? "\u2705" : "\u274C";
|
|
12661
13244
|
const sideways = `${yesNo(c.serverSideCopy)} / ${yesNo(c.serverSideMove)}`;
|
|
12662
|
-
const checksums = c.checksum.length === 0 ? "
|
|
12663
|
-
const auth = c.authentication.length === 0 ? "
|
|
13245
|
+
const checksums = c.checksum.length === 0 ? "-" : c.checksum.join(", ");
|
|
13246
|
+
const auth = c.authentication.length === 0 ? "-" : c.authentication.join(", ");
|
|
12664
13247
|
return `| ${entry.label} | ${yesNo(c.list)} | ${yesNo(c.stat)} | ${yesNo(c.readStream)} | ${yesNo(c.writeStream)} | ${yesNo(c.resumeDownload)} | ${yesNo(c.resumeUpload)} | ${sideways} | ${checksums} | ${auth} |`;
|
|
12665
13248
|
});
|
|
12666
13249
|
return [header, divider, ...rows].join("\n");
|
|
@@ -13246,6 +13829,84 @@ function mapFtp550(details) {
|
|
|
13246
13829
|
return new PermissionDeniedError(details);
|
|
13247
13830
|
}
|
|
13248
13831
|
|
|
13832
|
+
// src/protocols/ssh/runSshCommand.ts
|
|
13833
|
+
import { connect } from "net";
|
|
13834
|
+
var DEFAULT_PORT = 22;
|
|
13835
|
+
var DEFAULT_CONNECT_TIMEOUT_MS = 1e4;
|
|
13836
|
+
var DEFAULT_HANDSHAKE_TIMEOUT_MS = 1e4;
|
|
13837
|
+
var DEFAULT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
|
|
13838
|
+
async function runSshCommand(options) {
|
|
13839
|
+
const {
|
|
13840
|
+
host,
|
|
13841
|
+
port = DEFAULT_PORT,
|
|
13842
|
+
command,
|
|
13843
|
+
auth,
|
|
13844
|
+
transport: transportOptions,
|
|
13845
|
+
connectTimeoutMs = DEFAULT_CONNECT_TIMEOUT_MS,
|
|
13846
|
+
maxOutputBytes = DEFAULT_MAX_OUTPUT_BYTES
|
|
13847
|
+
} = options;
|
|
13848
|
+
const socket = await openTcpSocket(host, port, connectTimeoutMs);
|
|
13849
|
+
const transport = new SshTransportConnection({
|
|
13850
|
+
handshakeTimeoutMs: DEFAULT_HANDSHAKE_TIMEOUT_MS,
|
|
13851
|
+
...transportOptions
|
|
13852
|
+
});
|
|
13853
|
+
try {
|
|
13854
|
+
const handshake = await transport.connect(socket);
|
|
13855
|
+
const authSession = new SshAuthSession(transport);
|
|
13856
|
+
await authSession.authenticate({
|
|
13857
|
+
credential: auth,
|
|
13858
|
+
sessionId: handshake.keyExchange.sessionId
|
|
13859
|
+
});
|
|
13860
|
+
const conn = new SshConnectionManager(transport);
|
|
13861
|
+
const channel = await conn.openExecChannel(command);
|
|
13862
|
+
const pump = conn.start();
|
|
13863
|
+
pump.catch(() => {
|
|
13864
|
+
});
|
|
13865
|
+
const chunks = [];
|
|
13866
|
+
let bytesReceived = 0;
|
|
13867
|
+
try {
|
|
13868
|
+
for await (const chunk of channel.receiveData()) {
|
|
13869
|
+
bytesReceived += chunk.length;
|
|
13870
|
+
if (bytesReceived > maxOutputBytes) {
|
|
13871
|
+
throw new Error(
|
|
13872
|
+
`runSshCommand: stdout exceeded ${maxOutputBytes} bytes (set maxOutputBytes to allow more)`
|
|
13873
|
+
);
|
|
13874
|
+
}
|
|
13875
|
+
chunks.push(chunk);
|
|
13876
|
+
}
|
|
13877
|
+
} finally {
|
|
13878
|
+
channel.close();
|
|
13879
|
+
}
|
|
13880
|
+
const stdout = Buffer.concat(chunks);
|
|
13881
|
+
return {
|
|
13882
|
+
stdout,
|
|
13883
|
+
stdoutText: stdout.toString("utf8"),
|
|
13884
|
+
bytesReceived
|
|
13885
|
+
};
|
|
13886
|
+
} finally {
|
|
13887
|
+
transport.disconnect();
|
|
13888
|
+
}
|
|
13889
|
+
}
|
|
13890
|
+
function openTcpSocket(host, port, timeoutMs) {
|
|
13891
|
+
return new Promise((resolve, reject) => {
|
|
13892
|
+
const socket = connect({ host, port });
|
|
13893
|
+
const timer = setTimeout(() => {
|
|
13894
|
+
socket.destroy();
|
|
13895
|
+
reject(
|
|
13896
|
+
new Error(`runSshCommand: TCP connect to ${host}:${port} timed out after ${timeoutMs}ms`)
|
|
13897
|
+
);
|
|
13898
|
+
}, timeoutMs);
|
|
13899
|
+
socket.once("connect", () => {
|
|
13900
|
+
clearTimeout(timer);
|
|
13901
|
+
resolve(socket);
|
|
13902
|
+
});
|
|
13903
|
+
socket.once("error", (error) => {
|
|
13904
|
+
clearTimeout(timer);
|
|
13905
|
+
reject(error);
|
|
13906
|
+
});
|
|
13907
|
+
});
|
|
13908
|
+
}
|
|
13909
|
+
|
|
13249
13910
|
// src/transfers/TransferPlan.ts
|
|
13250
13911
|
function createTransferPlan(input) {
|
|
13251
13912
|
const plan = {
|
|
@@ -15284,6 +15945,19 @@ var defaultRunner = ({ client, route, signal }) => {
|
|
|
15284
15945
|
const options = { client, route, signal };
|
|
15285
15946
|
return runRoute(options);
|
|
15286
15947
|
};
|
|
15948
|
+
|
|
15949
|
+
// src/utils/mainModule.ts
|
|
15950
|
+
import { fileURLToPath } from "url";
|
|
15951
|
+
function isMainModule(importMetaUrl) {
|
|
15952
|
+
if (typeof process === "undefined" || !process.argv || process.argv.length < 2) {
|
|
15953
|
+
return false;
|
|
15954
|
+
}
|
|
15955
|
+
try {
|
|
15956
|
+
return process.argv[1] === fileURLToPath(importMetaUrl);
|
|
15957
|
+
} catch {
|
|
15958
|
+
return false;
|
|
15959
|
+
}
|
|
15960
|
+
}
|
|
15287
15961
|
export {
|
|
15288
15962
|
AbortError,
|
|
15289
15963
|
ApprovalRegistry,
|
|
@@ -15383,6 +16057,7 @@ export {
|
|
|
15383
16057
|
inboxFailedPath,
|
|
15384
16058
|
inboxProcessedPath,
|
|
15385
16059
|
isClassicProviderId,
|
|
16060
|
+
isMainModule,
|
|
15386
16061
|
isSensitiveKey,
|
|
15387
16062
|
joinRemotePath,
|
|
15388
16063
|
matchKnownHosts,
|
|
@@ -15415,6 +16090,7 @@ export {
|
|
|
15415
16090
|
resolveSecret,
|
|
15416
16091
|
runConnectionDiagnostics,
|
|
15417
16092
|
runRoute,
|
|
16093
|
+
runSshCommand,
|
|
15418
16094
|
serializeRemoteManifest,
|
|
15419
16095
|
signWebhookPayload,
|
|
15420
16096
|
sortRemoteEntries,
|