@flowscripter/pluggable-io-framework 2.0.1 → 2.1.0

package/README.md

@@ -10,9 +10,12 @@
 
 ## Key Features
 
-- Protocol-aware locations: `file:///data/a.txt`, `s3://bucket/key`,
-  `https://host/a` and composite schemes such as `tams+https:` select the
-  installed provider plugin for that protocol. Bare paths default to `file`.
+- Protocol-aware locations, given as a string (`file:///data/a.txt`,
+  `s3://bucket/key`, `https://host/a`, composite schemes such as
+  `tams+https:`; bare paths default to `file`) or as a structured
+  `{ protocol, location }` object. Either form selects the installed provider
+  plugin for that protocol. See
+  [String and Structured Locations](README/key-concepts.md#string-and-structured-locations).
 - A `ProviderRegistry` keyed by protocol and payload kind, with more than one
   implementation of a protocol installed side by side (e.g. JS and native
   `file` providers).

package/dist/index.d.ts

@@ -4,6 +4,7 @@ export * from "./src/decorators/locallyCached.ts";
 export * from "./src/decorators/seekable.ts";
 export * from "./src/registry/detectProtocol.ts";
 export * from "./src/registry/ProviderRegistry.ts";
+export * from "./src/registry/StructuredLocation.ts";
 export * from "./src/retry/RetryOptions.ts";
 export * from "./src/retry/withRetry.ts";
 export * from "./src/transfer/copyMove.ts";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,cAAc,yCAAyC,CAAC;AACxD,cAAc,mDAAmD,CAAC;AAClE,cAAc,mCAAmC,CAAC;AAClD,cAAc,8BAA8B,CAAC;AAC7C,cAAc,kCAAkC,CAAC;AACjD,cAAc,oCAAoC,CAAC;AACnD,cAAc,6BAA6B,CAAC;AAC5C,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,gDAAgD,CAAC;AAC/D,cAAc,mCAAmC,CAAC;AAClD,cAAc,kCAAkC,CAAC;AACjD,cAAc,2BAA2B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,cAAc,yCAAyC,CAAC;AACxD,cAAc,mDAAmD,CAAC;AAClE,cAAc,mCAAmC,CAAC;AAClD,cAAc,8BAA8B,CAAC;AAC7C,cAAc,kCAAkC,CAAC;AACjD,cAAc,oCAAoC,CAAC;AACnD,cAAc,sCAAsC,CAAC;AACrD,cAAc,6BAA6B,CAAC;AAC5C,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,gDAAgD,CAAC;AAC/D,cAAc,mCAAmC,CAAC;AAClD,cAAc,kCAAkC,CAAC;AACjD,cAAc,2BAA2B,CAAC"}

package/dist/index.js

@@ -4,6 +4,7 @@ export * from "./src/decorators/locallyCached.js";
 export * from "./src/decorators/seekable.js";
 export * from "./src/registry/detectProtocol.js";
 export * from "./src/registry/ProviderRegistry.js";
+export * from "./src/registry/StructuredLocation.js";
 export * from "./src/retry/RetryOptions.js";
 export * from "./src/retry/withRetry.js";
 export * from "./src/transfer/copyMove.js";

package/dist/src/registry/ProviderRegistry.d.ts

@@ -1,6 +1,7 @@
 import type { PluginManager } from "@flowscripter/dynamic-plugin-framework";
 import { type IOProvider, type IOProviderFactory, type LocationTarget, PayloadKind, type PayloadConverter, type ProviderResolver } from "@flowscripter/pluggable-io-framework-api";
 import type { TransferOptions } from "../transfer/TransferOptions.ts";
+import type { StructuredLocation } from "./StructuredLocation.ts";
 /** Nested provider resolutions allowed before resolution fails. */
 export declare const MAX_RESOLUTION_DEPTH = 4;
 export interface ResolvedProvider {
@@ -19,7 +20,8 @@ export interface TransferProviders {
 /**
  * Discovers provider factories and payload converters through a
  * `dynamic-plugin-framework` `PluginManager`, keyed by protocol and
- * payload kind, and creates providers for location strings. It is also the
+ * payload kind, and creates providers for location strings or
+ * {@link StructuredLocation}s. It is also the
  * `ProviderResolver` handed to every provider it creates.
  */
 export declare class ProviderRegistry implements ProviderResolver {
@@ -37,7 +39,7 @@ export declare class ProviderRegistry implements ProviderResolver {
     /** The factory for `protocol` and `kind`; without `kind`, the `js` one, else the only one. */
     getFactory(protocol: string, kind?: PayloadKind): IOProviderFactory | undefined;
     getConverters(): readonly PayloadConverter[];
-    createProviderForLocation(location: string, options?: {
+    createProviderForLocation(location: string | StructuredLocation, options?: {
         kind?: PayloadKind;
         domain?: string;
     }): Promise<ResolvedProvider>;
@@ -45,7 +47,7 @@ export declare class ProviderRegistry implements ProviderResolver {
      * Creates providers for both sides of a transfer, negotiating a common
      * (kind, domain) and payload type, or a converter.
      */
-    createProvidersForTransfer(source: string, dest: string, options?: {
+    createProvidersForTransfer(source: string | StructuredLocation, dest: string | StructuredLocation, options?: {
         kind?: PayloadKind;
     }): Promise<TransferProviders>;
 }

package/dist/src/registry/ProviderRegistry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ProviderRegistry.d.ts","sourceRoot":"","sources":["../../../src/registry/ProviderRegistry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wCAAwC,CAAC;AAC5E,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,iBAAiB,EACtB,KAAK,cAAc,EACnB,WAAW,EACX,KAAK,gBAAgB,EAIrB,KAAK,gBAAgB,EACtB,MAAM,0CAA0C,CAAC;AAClD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAItE,mEAAmE;AACnE,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAEtC,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;CACnC;AAOD;;;;;GAKG;AACH,qBAAa,gBAAiB,YAAW,gBAAgB;;IAIpC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAAjD,YAAoC,aAAa,EAAE,aAAa,EAAI;IAEpE;;;;OAIG;IACU,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAqCrC;IAEM,YAAY,IAAI,MAAM,EAAE,CAE9B;IAEM,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,WAAW,EAAE,CAE/C;IAED,8FAA8F;IACvF,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,iBAAiB,GAAG,SAAS,CAKrF;IAEM,aAAa,IAAI,SAAS,gBAAgB,EAAE,CAElD;IAEM,yBAAyB,CAC9B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,WAAW,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAChD,OAAO,CAAC,gBAAgB,CAAC,CAE3B;IAED;;;OAGG;IACU,0BAA0B,CACrC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,WAAW,CAAA;KAAE,GAC/B,OAAO,CAAC,iBAAiB,CAAC,CAwC5B;CAqEF"}
+{"version":3,"file":"ProviderRegistry.d.ts","sourceRoot":"","sources":["../../../src/registry/ProviderRegistry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wCAAwC,CAAC;AAC5E,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,iBAAiB,EACtB,KAAK,cAAc,EACnB,WAAW,EACX,KAAK,gBAAgB,EAIrB,KAAK,gBAAgB,EACtB,MAAM,0CAA0C,CAAC;AAClD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAEtE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAGlE,mEAAmE;AACnE,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAEtC,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;CACnC;AAWD;;;;;;GAMG;AACH,qBAAa,gBAAiB,YAAW,gBAAgB;;IAIpC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAAjD,YAAoC,aAAa,EAAE,aAAa,EAAI;IAEpE;;;;OAIG;IACU,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAqCrC;IAEM,YAAY,IAAI,MAAM,EAAE,CAE9B;IAEM,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,WAAW,EAAE,CAE/C;IAED,8FAA8F;IACvF,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,iBAAiB,GAAG,SAAS,CAKrF;IAEM,aAAa,IAAI,SAAS,gBAAgB,EAAE,CAElD;IAEM,yBAAyB,CAC9B,QAAQ,EAAE,MAAM,GAAG,kBAAkB,EACrC,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,WAAW,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAChD,OAAO,CAAC,gBAAgB,CAAC,CAE3B;IAED;;;OAGG;IACU,0BAA0B,CACrC,MAAM,EAAE,MAAM,GAAG,kBAAkB,EACnC,IAAI,EAAE,MAAM,GAAG,kBAAkB,EACjC,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,WAAW,CAAA;KAAE,GAC/B,OAAO,CAAC,iBAAiB,CAAC,CAwC5B;CAuEF"}

package/dist/src/registry/ProviderRegistry.js

@@ -3,6 +3,9 @@ import { detectProtocol } from "./detectProtocol.js";
 import { DEFAULT_NATIVE_DOMAINS, negotiateTransfer } from "./negotiateTransfer.js";
 /** Nested provider resolutions allowed before resolution fails. */
 export const MAX_RESOLUTION_DEPTH = 4;
+function protocolOf(location) {
+    return typeof location === "string" ? detectProtocol(location) : location.protocol;
+}
 function locationFieldNames(factory) {
     const shape = factory.locationSchema.shape;
     return Object.keys(shape ?? {}).sort();
@@ -10,7 +13,8 @@ function locationFieldNames(factory) {
 /**
  * Discovers provider factories and payload converters through a
  * `dynamic-plugin-framework` `PluginManager`, keyed by protocol and
- * payload kind, and creates providers for location strings. It is also the
+ * payload kind, and creates providers for location strings or
+ * {@link StructuredLocation}s. It is also the
  * `ProviderResolver` handed to every provider it creates.
  */
 export class ProviderRegistry {
@@ -77,8 +81,8 @@ export class ProviderRegistry {
      * (kind, domain) and payload type, or a converter.
      */
     async createProvidersForTransfer(source, dest, options) {
-        const sourceProtocol = detectProtocol(source);
-        const destProtocol = detectProtocol(dest);
+        const sourceProtocol = protocolOf(source);
+        const destProtocol = protocolOf(dest);
         const sourceFactories = this.#factoriesFor(sourceProtocol, options?.kind);
         const destFactories = this.#factoriesFor(destProtocol, options?.kind);
         const negotiation = negotiateTransfer(sourceProtocol, sourceFactories, destProtocol, destFactories, this.#converters, options?.kind);
@@ -120,7 +124,7 @@ export class ProviderRegistry {
         if (depth > MAX_RESOLUTION_DEPTH) {
             throw new PermanentIOError("provider resolution too deep");
         }
-        const protocol = detectProtocol(location);
+        const protocol = protocolOf(location);
         this.#factoriesFor(protocol, options?.kind);
         const factory = this.getFactory(protocol, options?.kind);
         let domain;
@@ -134,7 +138,8 @@ export class ProviderRegistry {
         return this.#instantiate(factory, location, domain, depth);
     }
     async #instantiate(factory, location, domain, depth) {
-        const parsed = factory.locationSchema.parse(factory.parseLocationString(location));
+        const raw = typeof location === "string" ? factory.parseLocationString(location) : location.location;
+        const parsed = factory.locationSchema.parse(raw);
         const { config, target } = factory.toProviderInputs(parsed);
         const resolver = {
             createProviderForLocation: (inner, opts) => this.#resolve(inner, { kind: opts?.kind ?? factory.kind, domain: opts?.domain ?? domain }, depth + 1),

package/dist/src/registry/StructuredLocation.d.ts

@@ -0,0 +1,11 @@
+/**
+ * A location given as its protocol plus the raw location object that the
+ * protocol's factory `locationSchema` validates, instead of a location
+ * string. It can carry fields a string cannot, such as `filename`,
+ * `pattern` or credentials.
+ */
+export interface StructuredLocation {
+    readonly protocol: string;
+    readonly location: unknown;
+}
+//# sourceMappingURL=StructuredLocation.d.ts.map

package/dist/src/registry/StructuredLocation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"StructuredLocation.d.ts","sourceRoot":"","sources":["../../../src/registry/StructuredLocation.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC5B"}

package/dist/src/registry/StructuredLocation.js

@@ -0,0 +1 @@
+export {};

package/dist/src/transfer/multipartTransfer.d.ts

@@ -1,14 +1,14 @@
-import { type IOProvider, type Part, type RangeReadable, type StreamHandle } from "@flowscripter/pluggable-io-framework-api";
+import { type IOProvider, type MultipartWriter, type RangeReadable, type StreamHandle } from "@flowscripter/pluggable-io-framework-api";
 import type { ConcurrencyLimiter } from "../concurrency/ConcurrencyLimiter.ts";
 import type { EntryContext, EntryOutcome } from "./EntryContext.ts";
 export interface MultipartTransferInput {
     readonly readable: StreamHandle & RangeReadable;
     readonly sink: IOProvider;
-    readonly writer: {
-        write(parts: AsyncIterable<Part>): Promise<void>;
-    };
+    readonly writer: MultipartWriter;
     readonly totalBytes: number;
     readonly partSize: number;
+    /** Where a resumed upload continues: parts start at the part containing this offset. */
+    readonly startOffset?: number;
     readonly limiter: ConcurrencyLimiter;
     readonly context: EntryContext;
 }
@@ -17,7 +17,8 @@ export interface MultipartTransferInput {
  * writer, bounded by `limiter`. A part whose read fails with a
  * `TransientIOError` is re-read on its own, up to `retry.maxRetries` times,
  * within the same upload. `stop` ends the transfer after the parts already
- * started; `signal` aborts it.
+ * started; `signal` aborts it. Reported bytes include the parts before
+ * `startOffset`.
  */
 export declare function multipartTransfer(input: MultipartTransferInput): Promise<EntryOutcome>;
 //# sourceMappingURL=multipartTransfer.d.ts.map

package/dist/src/transfer/multipartTransfer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"multipartTransfer.d.ts","sourceRoot":"","sources":["../../../src/transfer/multipartTransfer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,UAAU,EAEf,KAAK,IAAI,EACT,KAAK,aAAa,EAClB,KAAK,YAAY,EAElB,MAAM,0CAA0C,CAAC;AAClD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AAM/E,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGpE,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,QAAQ,EAAE,YAAY,GAAG,aAAa,CAAC;IAChD,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE;QAAE,KAAK,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;KAAE,CAAC;IACtE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IACrC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;CAChC;AAED;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,YAAY,CAAC,CAoF5F"}
+{"version":3,"file":"multipartTransfer.d.ts","sourceRoot":"","sources":["../../../src/transfer/multipartTransfer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,UAAU,EAEf,KAAK,eAAe,EAEpB,KAAK,aAAa,EAClB,KAAK,YAAY,EAElB,MAAM,0CAA0C,CAAC;AAClD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AAM/E,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGpE,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,QAAQ,EAAE,YAAY,GAAG,aAAa,CAAC;IAChD,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,wFAAwF;IACxF,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IACrC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,YAAY,CAAC,CA0F5F"}

package/dist/src/transfer/multipartTransfer.js

@@ -10,13 +10,15 @@ import { rangeReadableMultipartReader } from "./rangeReadableMultipartReader.js"
  * writer, bounded by `limiter`. A part whose read fails with a
  * `TransientIOError` is re-read on its own, up to `retry.maxRetries` times,
  * within the same upload. `stop` ends the transfer after the parts already
- * started; `signal` aborts it.
+ * started; `signal` aborts it. Reported bytes include the parts before
+ * `startOffset`.
  */
 export async function multipartTransfer(input) {
     const { readable, sink, writer, totalBytes, partSize, limiter, context } = input;
     const { options, hooks, operationId, type } = context;
     const retry = options.retry ?? DEFAULT_RETRY;
-    let bytes = 0;
+    const firstOffset = Math.floor((input.startOffset ?? 0) / partSize) * partSize;
+    let bytes = firstOffset;
     let items = 0;
     let stopped = false;
     async function readPart(sourcePart) {
@@ -78,7 +80,7 @@ export async function multipartTransfer(input) {
         };
     }
     async function* sourceParts() {
-        for await (const part of rangeReadableMultipartReader(readable, totalBytes, partSize)) {
+        for await (const part of rangeReadableMultipartReader(readable, totalBytes, partSize, firstOffset)) {
             if (options.signal?.aborted)
                 throw createAbortError();
             if (options.stop?.aborted) {

package/dist/src/transfer/rangeReadableMultipartReader.d.ts

@@ -2,7 +2,8 @@ import type { Part, PayloadKind, RangeReadable, StreamHandle } from "@flowscript
 /**
  * Splits a `RangeReadable` handle into parts of `partSize` bytes covering
  * `totalSize`. Each part's stream is opened with `readRange(start, end)`
- * (`end` exclusive) when the part is pulled.
+ * (`end` exclusive) when the part is pulled. With `startOffset`, parts begin
+ * at the part containing that offset.
  */
-export declare function rangeReadableMultipartReader<K extends PayloadKind>(handle: StreamHandle<K> & RangeReadable<K>, totalSize: number, partSize: number): AsyncGenerator<Part<K>>;
+export declare function rangeReadableMultipartReader<K extends PayloadKind>(handle: StreamHandle<K> & RangeReadable<K>, totalSize: number, partSize: number, startOffset?: number): AsyncGenerator<Part<K>>;
 //# sourceMappingURL=rangeReadableMultipartReader.d.ts.map

package/dist/src/transfer/rangeReadableMultipartReader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rangeReadableMultipartReader.d.ts","sourceRoot":"","sources":["../../../src/transfer/rangeReadableMultipartReader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,IAAI,EACJ,WAAW,EACX,aAAa,EACb,YAAY,EACb,MAAM,0CAA0C,CAAC;AAElD;;;;GAIG;AACH,wBAAuB,4BAA4B,CAAC,CAAC,SAAS,WAAW,EACvE,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,EAC1C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAazB"}
+{"version":3,"file":"rangeReadableMultipartReader.d.ts","sourceRoot":"","sources":["../../../src/transfer/rangeReadableMultipartReader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,IAAI,EACJ,WAAW,EACX,aAAa,EACb,YAAY,EACb,MAAM,0CAA0C,CAAC;AAElD;;;;;GAKG;AACH,wBAAuB,4BAA4B,CAAC,CAAC,SAAS,WAAW,EACvE,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,EAC1C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,WAAW,SAAI,GACd,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAazB"}

package/dist/src/transfer/rangeReadableMultipartReader.js

@@ -1,11 +1,12 @@
 /**
  * Splits a `RangeReadable` handle into parts of `partSize` bytes covering
  * `totalSize`. Each part's stream is opened with `readRange(start, end)`
- * (`end` exclusive) when the part is pulled.
+ * (`end` exclusive) when the part is pulled. With `startOffset`, parts begin
+ * at the part containing that offset.
  */
-export async function* rangeReadableMultipartReader(handle, totalSize, partSize) {
+export async function* rangeReadableMultipartReader(handle, totalSize, partSize, startOffset = 0) {
     const partCount = Math.max(1, Math.ceil(totalSize / partSize));
-    for (let index = 0; index < partCount; index += 1) {
+    for (let index = Math.floor(startOffset / partSize); index < partCount; index += 1) {
         const start = index * partSize;
         const end = Math.min(start + partSize, totalSize);
         yield {

package/dist/src/transfer/transferEntry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transferEntry.d.ts","sourceRoot":"","sources":["../../../src/transfer/transferEntry.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,UAAU,EAOhB,MAAM,0CAA0C,CAAC;AAYlD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAI1D,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC,mEAAmE;IACnE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAC/B,qEAAqE;IACrE,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;CAChC;AAED,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,GAAG,MAAM,CAM/F;AAgBD;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,cAAc,CAAC,CAmHtF"}
+{"version":3,"file":"transferEntry.d.ts","sourceRoot":"","sources":["../../../src/transfer/transferEntry.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,UAAU,EAShB,MAAM,0CAA0C,CAAC;AAYlD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAI1D,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC,mEAAmE;IACnE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAC/B,qEAAqE;IACrE,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;CAChC;AAED,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,GAAG,MAAM,CAM/F;AAgBD;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,cAAc,CAAC,CAwHtF"}

package/dist/src/transfer/transferEntry.js

@@ -1,4 +1,4 @@
-import { BYTES_PAYLOAD_TYPE, isBufferProvider, isFillReadable, isRangeReadable, PermanentIOError, } from "@flowscripter/pluggable-io-framework-api";
+import { BYTES_PAYLOAD_TYPE, isBufferProvider, isFillReadable, isRangeReadable, isResumableWritable, PermanentIOError, } from "@flowscripter/pluggable-io-framework-api";
 import { defaultConcurrencyLimiter } from "../concurrency/ConcurrencyLimiter.js";
 import { DEFAULT_RETRY } from "../retry/RetryOptions.js";
 import { withRetry } from "../retry/withRetry.js";
@@ -83,15 +83,24 @@ export async function transferEntry(input) {
         isRangeReadable(readable)) {
         await cancelStream(readable);
         const rangeReadable = readable;
-        outcome = await withRetry(() => multipartTransfer({
-            readable: rangeReadable,
-            sink,
-            writer: multipartWriter(destKey, partSize),
-            totalBytes: size,
-            partSize,
-            limiter: options.concurrencyLimiter ?? defaultConcurrencyLimiter,
-            context,
-        }), retry);
+        let writer;
+        outcome = await withRetry(() => {
+            // A retry continues the failed writer's upload when it has a resume token.
+            const token = writer && isResumableWritable(writer) ? writer.resumeToken() : undefined;
+            writer = token
+                ? multipartWriter(destKey, partSize, { resume: token })
+                : multipartWriter(destKey, partSize);
+            return multipartTransfer({
+                readable: rangeReadable,
+                sink,
+                writer,
+                totalBytes: size,
+                partSize,
+                startOffset: token?.offset,
+                limiter: options.concurrencyLimiter ?? defaultConcurrencyLimiter,
+                context,
+            });
+        }, retry);
         strategy = "multipart";
     }
     else {

package/index.ts

@@ -4,6 +4,7 @@ export * from "./src/decorators/locallyCached.ts";
 export * from "./src/decorators/seekable.ts";
 export * from "./src/registry/detectProtocol.ts";
 export * from "./src/registry/ProviderRegistry.ts";
+export * from "./src/registry/StructuredLocation.ts";
 export * from "./src/retry/RetryOptions.ts";
 export * from "./src/retry/withRetry.ts";
 export * from "./src/transfer/copyMove.ts";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowscripter/pluggable-io-framework",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "description": "A pluggable source/sink IO framework using https://github.com/flowscripter/dynamic-plugin-framework",
   "keywords": [
     "bun",

package/src/registry/ProviderRegistry.ts

@@ -13,6 +13,7 @@ import {
 } from "@flowscripter/pluggable-io-framework-api";
 import type { TransferOptions } from "../transfer/TransferOptions.ts";
 import { detectProtocol } from "./detectProtocol.ts";
+import type { StructuredLocation } from "./StructuredLocation.ts";
 import { DEFAULT_NATIVE_DOMAINS, negotiateTransfer } from "./negotiateTransfer.ts";
 
 /** Nested provider resolutions allowed before resolution fails. */
@@ -33,6 +34,10 @@ export interface TransferProviders {
   readonly options: TransferOptions;
 }
 
+function protocolOf(location: string | StructuredLocation): string {
+  return typeof location === "string" ? detectProtocol(location) : location.protocol;
+}
+
 function locationFieldNames(factory: IOProviderFactory): string[] {
   const shape = (factory.locationSchema as { shape?: Record<string, unknown> }).shape;
   return Object.keys(shape ?? {}).sort();
@@ -41,7 +46,8 @@ function locationFieldNames(factory: IOProviderFactory): string[] {
 /**
  * Discovers provider factories and payload converters through a
  * `dynamic-plugin-framework` `PluginManager`, keyed by protocol and
- * payload kind, and creates providers for location strings. It is also the
+ * payload kind, and creates providers for location strings or
+ * {@link StructuredLocation}s. It is also the
  * `ProviderResolver` handed to every provider it creates.
  */
 export class ProviderRegistry implements ProviderResolver {
@@ -115,7 +121,7 @@ export class ProviderRegistry implements ProviderResolver {
   }
 
   public createProviderForLocation(
-    location: string,
+    location: string | StructuredLocation,
     options?: { kind?: PayloadKind; domain?: string },
   ): Promise<ResolvedProvider> {
     return this.#resolve(location, options, 0);
@@ -126,12 +132,12 @@ export class ProviderRegistry implements ProviderResolver {
    * (kind, domain) and payload type, or a converter.
    */
   public async createProvidersForTransfer(
-    source: string,
-    dest: string,
+    source: string | StructuredLocation,
+    dest: string | StructuredLocation,
     options?: { kind?: PayloadKind },
   ): Promise<TransferProviders> {
-    const sourceProtocol = detectProtocol(source);
-    const destProtocol = detectProtocol(dest);
+    const sourceProtocol = protocolOf(source);
+    const destProtocol = protocolOf(dest);
     const sourceFactories = this.#factoriesFor(sourceProtocol, options?.kind);
     const destFactories = this.#factoriesFor(destProtocol, options?.kind);
     const negotiation = negotiateTransfer(
@@ -193,14 +199,14 @@ export class ProviderRegistry implements ProviderResolver {
   }
 
   async #resolve(
-    location: string,
+    location: string | StructuredLocation,
     options: { kind?: PayloadKind; domain?: string } | undefined,
     depth: number,
   ): Promise<ResolvedProvider> {
     if (depth > MAX_RESOLUTION_DEPTH) {
       throw new PermanentIOError("provider resolution too deep");
     }
-    const protocol = detectProtocol(location);
+    const protocol = protocolOf(location);
     this.#factoriesFor(protocol, options?.kind);
     const factory = this.getFactory(protocol, options?.kind) as IOProviderFactory;
     let domain: string | undefined;
@@ -218,11 +224,13 @@ export class ProviderRegistry implements ProviderResolver {
 
   async #instantiate(
     factory: IOProviderFactory,
-    location: string,
+    location: string | StructuredLocation,
     domain: string | undefined,
     depth: number,
   ): Promise<ResolvedProvider> {
-    const parsed = factory.locationSchema.parse(factory.parseLocationString(location));
+    const raw =
+      typeof location === "string" ? factory.parseLocationString(location) : location.location;
+    const parsed = factory.locationSchema.parse(raw);
     const { config, target } = factory.toProviderInputs(parsed);
     const resolver: ProviderResolver = {
       createProviderForLocation: (inner, opts) =>

package/src/registry/StructuredLocation.ts

@@ -0,0 +1,10 @@
+/**
+ * A location given as its protocol plus the raw location object that the
+ * protocol's factory `locationSchema` validates, instead of a location
+ * string. It can carry fields a string cannot, such as `filename`,
+ * `pattern` or credentials.
+ */
+export interface StructuredLocation {
+  readonly protocol: string;
+  readonly location: unknown;
+}

package/src/transfer/multipartTransfer.ts

@@ -1,6 +1,7 @@
 import {
   type IOProvider,
   type Item,
+  type MultipartWriter,
   type Part,
   type RangeReadable,
   type StreamHandle,
@@ -18,9 +19,11 @@ import { rangeReadableMultipartReader } from "./rangeReadableMultipartReader.ts"
 export interface MultipartTransferInput {
   readonly readable: StreamHandle & RangeReadable;
   readonly sink: IOProvider;
-  readonly writer: { write(parts: AsyncIterable<Part>): Promise<void> };
+  readonly writer: MultipartWriter;
   readonly totalBytes: number;
   readonly partSize: number;
+  /** Where a resumed upload continues: parts start at the part containing this offset. */
+  readonly startOffset?: number;
   readonly limiter: ConcurrencyLimiter;
   readonly context: EntryContext;
 }
@@ -30,13 +33,15 @@ export interface MultipartTransferInput {
  * writer, bounded by `limiter`. A part whose read fails with a
  * `TransientIOError` is re-read on its own, up to `retry.maxRetries` times,
  * within the same upload. `stop` ends the transfer after the parts already
- * started; `signal` aborts it.
+ * started; `signal` aborts it. Reported bytes include the parts before
+ * `startOffset`.
  */
 export async function multipartTransfer(input: MultipartTransferInput): Promise<EntryOutcome> {
   const { readable, sink, writer, totalBytes, partSize, limiter, context } = input;
   const { options, hooks, operationId, type } = context;
   const retry = options.retry ?? DEFAULT_RETRY;
-  let bytes = 0;
+  const firstOffset = Math.floor((input.startOffset ?? 0) / partSize) * partSize;
+  let bytes = firstOffset;
   let items = 0;
   let stopped = false;
 
@@ -103,7 +108,12 @@ export async function multipartTransfer(input: MultipartTransferInput): Promise<
   }
 
   async function* sourceParts(): AsyncGenerator<Part> {
-    for await (const part of rangeReadableMultipartReader(readable, totalBytes, partSize)) {
+    for await (const part of rangeReadableMultipartReader(
+      readable,
+      totalBytes,
+      partSize,
+      firstOffset,
+    )) {
       if (options.signal?.aborted) throw createAbortError();
       if (options.stop?.aborted) {
         stopped = true;

package/src/transfer/rangeReadableMultipartReader.ts

@@ -8,15 +8,17 @@ import type {
 /**
  * Splits a `RangeReadable` handle into parts of `partSize` bytes covering
  * `totalSize`. Each part's stream is opened with `readRange(start, end)`
- * (`end` exclusive) when the part is pulled.
+ * (`end` exclusive) when the part is pulled. With `startOffset`, parts begin
+ * at the part containing that offset.
  */
 export async function* rangeReadableMultipartReader<K extends PayloadKind>(
   handle: StreamHandle<K> & RangeReadable<K>,
   totalSize: number,
   partSize: number,
+  startOffset = 0,
 ): AsyncGenerator<Part<K>> {
   const partCount = Math.max(1, Math.ceil(totalSize / partSize));
-  for (let index = 0; index < partCount; index += 1) {
+  for (let index = Math.floor(startOffset / partSize); index < partCount; index += 1) {
     const start = index * partSize;
     const end = Math.min(start + partSize, totalSize);
     yield {

package/src/transfer/transferEntry.ts

@@ -5,7 +5,9 @@ import {
   isBufferProvider,
   isFillReadable,
   isRangeReadable,
+  isResumableWritable,
   type Item,
+  type MultipartWriter,
   PermanentIOError,
   type StreamHandle,
 } from "@flowscripter/pluggable-io-framework-api";
@@ -124,19 +126,24 @@ export async function transferEntry(input: TransferEntryInput): Promise<Transfer
   ) {
     await cancelStream(readable);
     const rangeReadable = readable;
-    outcome = await withRetry(
-      () =>
-        multipartTransfer({
-          readable: rangeReadable,
-          sink,
-          writer: multipartWriter(destKey, partSize),
-          totalBytes: size,
-          partSize,
-          limiter: options.concurrencyLimiter ?? defaultConcurrencyLimiter,
-          context,
-        }),
-      retry,
-    );
+    let writer: MultipartWriter | undefined;
+    outcome = await withRetry(() => {
+      // A retry continues the failed writer's upload when it has a resume token.
+      const token = writer && isResumableWritable(writer) ? writer.resumeToken() : undefined;
+      writer = token
+        ? multipartWriter(destKey, partSize, { resume: token })
+        : multipartWriter(destKey, partSize);
+      return multipartTransfer({
+        readable: rangeReadable,
+        sink,
+        writer,
+        totalBytes: size,
+        partSize,
+        startOffset: token?.offset,
+        limiter: options.concurrencyLimiter ?? defaultConcurrencyLimiter,
+        context,
+      });
+    }, retry);
     strategy = "multipart";
   } else {
     let writable = await withRetry(() => sink.getWritableStream(destKey), retry);